diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000..6f98693add --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,13 @@ +# Use the latest 2.1 version of CircleCI pipeline process engine. See: https://circleci.com/docs/2.0/configuration-reference +version: 2.1 +# Use a package of configuration called an orb. +orbs: + # Declare a dependency on the welcome-orb + welcome: circleci/welcome-orb@0.4.1 +# Orchestrate or schedule a set of jobs +workflows: + # Name the workflow "welcome" + welcome: + # Run the welcome/run job in its own container + jobs: + - welcome/run diff --git a/.gitmodules b/.gitmodules index 74afbbf997..0e65b02221 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,12 +1,12 @@ [submodule "src/connector/go"] path = src/connector/go - url = https://github.com/taosdata/driver-go + url = git@github.com:taosdata/driver-go.git [submodule "src/connector/grafanaplugin"] path = src/connector/grafanaplugin - url = https://github.com/taosdata/grafanaplugin + url = git@github.com:taosdata/grafanaplugin.git [submodule "src/connector/hivemq-tdengine-extension"] path = src/connector/hivemq-tdengine-extension - url = https://github.com/huskar-t/hivemq-tdengine-extension.git + url = git@github.com:taosdata/hivemq-tdengine-extension.git [submodule "tests/examples/rust"] path = tests/examples/rust url = https://github.com/songtianyi/tdengine-rust-bindings.git diff --git a/README-CN.md b/README-CN.md index d4c10e71d6..afb242d621 100644 --- a/README-CN.md +++ b/README-CN.md @@ -116,7 +116,7 @@ mkdir debug && cd debug cmake .. && cmake --build . ``` -在X86-64、X86、arm64 和 arm32 平台上，TDengine 生成脚本可以自动检测机器架构。也可以手动配置 CPUTYPE 参数来指定 CPU 类型，如 aarch64 或 aarch32 等。 +在X86-64、X86、arm64、arm32 和 mips64 平台上，TDengine 生成脚本可以自动检测机器架构。也可以手动配置 CPUTYPE 参数来指定 CPU 类型，如 aarch64 或 aarch32 等。 aarch64： @@ -130,6 +130,12 @@ aarch32： cmake .. -DCPUTYPE=aarch32 && cmake --build . ``` +mips64： + +```bash +cmake .. -DCPUTYPE=mips64 && cmake --build . +``` + ### Windows 系统 如果你使用的是 Visual Studio 2013 版本： diff --git a/README.md b/README.md index 78f902babe..2c33119e34 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ mkdir debug && cd debug cmake .. && cmake --build . ``` -TDengine build script can detect the host machine's architecture on X86-64, X86, arm64 and arm32 platform. +TDengine build script can detect the host machine's architecture on X86-64, X86, arm64, arm32 and mips64 platform. You can also specify CPUTYPE option like aarch64 or aarch32 too if the detection result is not correct: aarch64: @@ -123,6 +123,11 @@ aarch32: cmake .. -DCPUTYPE=aarch32 && cmake --build . ``` +mips64: +```bash +cmake .. -DCPUTYPE=mips64 && cmake --build . +``` + ### On Windows platform If you use the Visual Studio 2013, please open a command window by executing "cmd.exe". diff --git a/cmake/platform.inc b/cmake/platform.inc index dcd0183e27..5f7391c996 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -102,6 +102,12 @@ IF ("${CPUTYPE}" STREQUAL "") SET(TD_LINUX TRUE) SET(TD_LINUX_64 FALSE) SET(TD_ARM_64 TRUE) + ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "mips64") + SET(CPUTYPE "mips64") + MESSAGE(STATUS "Set CPUTYPE to mips64") + SET(TD_LINUX TRUE) + SET(TD_LINUX_64 FALSE) + SET(TD_MIPS_64 TRUE) ENDIF () ELSE () diff --git a/cmake/version.inc b/cmake/version.inc index 0ee23f319a..ed8e7a156c 100755 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.1.0.0") + SET(TD_VER_NUMBER "2.1.1.0") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/documentation20/cn/02.getting-started/docs.md b/documentation20/cn/02.getting-started/docs.md index a98159d8c4..6eb58a1433 100644 --- a/documentation20/cn/02.getting-started/docs.md +++ b/documentation20/cn/02.getting-started/docs.md @@ -24,7 +24,7 @@ TDengine的安装非常简单，从下载到安装成功仅仅只要几秒钟。 ## 轻松启动 -安装成功后，用户可使用`systemctl`命令来启动TDengine的服务进程。 +安装成功后，用户可使用 `systemctl` 命令来启动 TDengine 的服务进程。 ```bash $ systemctl start taosd @@ -35,21 +35,22 @@ $ systemctl start taosd $ systemctl status taosd ``` -如果TDengine服务正常工作，那么您可以通过TDengine的命令行程序`taos`来访问并体验TDengine。 +如果 TDengine 服务正常工作，那么您可以通过 TDengine 的命令行程序 `taos` 来访问并体验 TDengine。 **注意：** -- systemctl命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 sudo -- 为更好的获得产品反馈，改善产品，TDengine会采集基本的使用信息，但您可以修改系统配置文件taos.cfg里的配置参数telemetryReporting, 将其设为0，就可将其关闭。 -- TDengine采用FQDN(一般就是hostname)作为节点的ID，为保证正常运行，需要给运行taosd的服务器配置好hostname，在客户端应用运行的机器配置好DNS服务或hosts文件，保证FQDN能够解析。 +- systemctl 命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 sudo 。 +- 为更好的获得产品反馈，改善产品，TDengine 会采集基本的使用信息，但您可以修改系统配置文件 taos.cfg 里的配置参数 telemetryReporting, 将其设为 0，就可将其关闭。 +- TDengine 采用 FQDN (一般就是 hostname )作为节点的 ID，为保证正常运行，需要给运行 taosd 的服务器配置好 hostname，在客户端应用运行的机器配置好 DNS 服务或 hosts 文件，保证 FQDN 能够解析。 +- `systemctl stop taosd` 指令在执行后并不会马上停止 TDengine 服务，而是会等待系统中必要的落盘工作正常完成。在数据量很大的情况下，这可能会消耗较长时间。 -* TDengine 支持在使用[`systemd`](https://en.wikipedia.org/wiki/Systemd)做进程服务管理的linux系统上安装，用`which systemctl`命令来检测系统中是否存在`systemd`包: +* TDengine 支持在使用 [`systemd`](https://en.wikipedia.org/wiki/Systemd) 做进程服务管理的 linux 系统上安装，用 `which systemctl` 命令来检测系统中是否存在 `systemd` 包: ```bash $ which systemctl ``` - 如果系统中不支持systemd，也可以用手动运行 /usr/local/taos/bin/taosd 方式启动 TDengine 服务。 + 如果系统中不支持 systemd，也可以用手动运行 /usr/local/taos/bin/taosd 方式启动 TDengine 服务。 ## TDengine命令行程序 diff --git a/documentation20/cn/11.administrator/docs.md b/documentation20/cn/11.administrator/docs.md index bfa0456c7d..ee2f34ff11 100644 --- a/documentation20/cn/11.administrator/docs.md +++ b/documentation20/cn/11.administrator/docs.md @@ -129,7 +129,7 @@ taosd -C - blocks：每个VNODE（TSDB）中有多少cache大小的内存块。因此一个VNODE的用的内存大小粗略为（cache * blocks）。单位为块，默认值：4。（可通过 alter database 修改） - replica：副本个数，取值范围：1-3。单位为个，默认值：1。（可通过 alter database 修改） - precision：时间戳精度标识，ms表示毫秒，us表示微秒。默认值：ms。 -- cacheLast：是否在内存中缓存子表 last_row，0：关闭；1：开启。默认值：0。（可通过 alter database 修改）（从 2.0.11 版本开始支持此参数） +- cacheLast：是否在内存中缓存子表的最近数据，0：关闭；1：缓存子表最近一行数据；2：缓存子表每一列的最近的非NULL值，3：同时打开缓存最近行和列功能，默认值：0。（可通过 alter database 修改）（从 2.0.11 版本开始支持此参数） 对于一个应用场景，可能有多种数据特征的数据并存，最佳的设计是将具有相同数据特征的表放在一个库里，这样一个应用有多个库，而每个库可以配置不同的存储参数，从而保证系统有最优的性能。TDengine允许应用在创建库时指定上述存储参数，如果指定，该参数就将覆盖对应的系统配置参数。举例，有下述SQL： diff --git a/documentation20/cn/12.taos-sql/01.error-code/docs.md b/documentation20/cn/12.taos-sql/01.error-code/docs.md index 95975dba5a..867aa18715 100644 --- a/documentation20/cn/12.taos-sql/01.error-code/docs.md +++ b/documentation20/cn/12.taos-sql/01.error-code/docs.md @@ -1,172 +1,172 @@ # TDengine 2.0 错误码以及对应的十进制码 -| 状态码 | 模 | 错误码（十六进制） | 错误描述 | 错误码（十进制） | -|-----------------------| :---: | :---------: | :------------------------ | ---------------- | -|TSDB_CODE_RPC_ACTION_IN_PROGRESS| 0 | 0x0001| "Action in progress"| -2147483647| -|TSDB_CODE_RPC_AUTH_REQUIRED| 0 | 0x0002 | "Authentication required"| -2147483646| -|TSDB_CODE_RPC_AUTH_FAILURE| 0| 0x0003 | "Authentication failure"| -2147483645| -|TSDB_CODE_RPC_REDIRECT |0 | 0x0004| "Redirect"| -2147483644| -|TSDB_CODE_RPC_NOT_READY| 0 | 0x0005 | "System not ready"| -2147483643| -|TSDB_CODE_RPC_ALREADY_PROCESSED| 0 | 0x0006 |"Message already processed"| -2147483642| -|TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED| 0 |0x0007| "Last session not finished"| -2147483641| -|TSDB_CODE_RPC_MISMATCHED_LINK_ID| 0| 0x0008 | "Mismatched meter id"| -2147483640| -|TSDB_CODE_RPC_TOO_SLOW| 0 | 0x0009 | "Processing of request timed out"| -2147483639| -|TSDB_CODE_RPC_MAX_SESSIONS| 0 | 0x000A | "Number of sessions reached limit"| -2147483638| -|TSDB_CODE_RPC_NETWORK_UNAVAIL| 0 |0x000B | "Unable to establish connection" |-2147483637| -|TSDB_CODE_RPC_APP_ERROR| 0| 0x000C | "Unexpected generic error in RPC"| -2147483636| -|TSDB_CODE_RPC_UNEXPECTED_RESPONSE| 0 |0x000D | "Unexpected response"| -2147483635| -|TSDB_CODE_RPC_INVALID_VALUE| 0 | 0x000E | "Invalid value"| -2147483634| -|TSDB_CODE_RPC_INVALID_TRAN_ID| 0 | 0x000F | "Invalid transaction id"| -2147483633| -|TSDB_CODE_RPC_INVALID_SESSION_ID| 0| 0x0010 | "Invalid session id"| -2147483632| -|TSDB_CODE_RPC_INVALID_MSG_TYPE| 0| 0x0011| "Invalid message type"| -2147483631| -|TSDB_CODE_RPC_INVALID_RESPONSE_TYPE| 0 | 0x0012| "Invalid response type"| -2147483630| -|TSDB_CODE_RPC_INVALID_TIME_STAMP| 0| 0x0013| "Invalid timestamp"| -2147483629| -|TSDB_CODE_COM_OPS_NOT_SUPPORT| 0 | 0x0100| "Operation not supported"| -2147483392| -|TSDB_CODE_COM_MEMORY_CORRUPTED |0| 0x0101 | "Memory corrupted"| -2147483391| -|TSDB_CODE_COM_OUT_OF_MEMORY| 0| 0x0102| "Out of memory"| -2147483390| -|TSDB_CODE_COM_INVALID_CFG_MSG| 0 | 0x0103| "Invalid config message"| -2147483389| -|TSDB_CODE_COM_FILE_CORRUPTED| 0| 0x0104| "Data file corrupted" |-2147483388| -|TSDB_CODE_TSC_INVALID_SQL| 0| 0x0200 | "Invalid SQL statement"| -2147483136| -|TSDB_CODE_TSC_INVALID_QHANDLE| 0 | 0x0201 | "Invalid qhandle"| -2147483135| -|TSDB_CODE_TSC_INVALID_TIME_STAMP| 0 | 0x0202 | "Invalid combination of client/service time"| -2147483134| -|TSDB_CODE_TSC_INVALID_VALUE| 0 | 0x0203| "Invalid value in client"| -2147483133| -|TSDB_CODE_TSC_INVALID_VERSION| 0 | 0x0204 | "Invalid client version" |-2147483132| -|TSDB_CODE_TSC_INVALID_IE| 0 | 0x0205 | "Invalid client ie" |-2147483131| -|TSDB_CODE_TSC_INVALID_FQDN| 0 | 0x0206| "Invalid host name"| -2147483130| -|TSDB_CODE_TSC_INVALID_USER_LENGTH| 0 | 0x0207| "Invalid user name"| -2147483129| -|TSDB_CODE_TSC_INVALID_PASS_LENGTH| 0 | 0x0208 | "Invalid password"| -2147483128| -|TSDB_CODE_TSC_INVALID_DB_LENGTH| 0 | 0x0209| "Database name too long"| -2147483127| -|TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH| 0 | 0x020A | "Table name too long"| -2147483126| -|TSDB_CODE_TSC_INVALID_CONNECTION| 0 | 0x020B| "Invalid connection"| -2147483125| -|TSDB_CODE_TSC_OUT_OF_MEMORY| 0 | 0x020C | "System out of memory" |-2147483124| -|TSDB_CODE_TSC_NO_DISKSPACE| 0 | 0x020D | "System out of disk space"| -2147483123| -|TSDB_CODE_TSC_QUERY_CACHE_ERASED| 0 | 0x020E| "Query cache erased"| -2147483122| -|TSDB_CODE_TSC_QUERY_CANCELLED| 0 | 0x020F |"Query terminated"| -2147483121| -|TSDB_CODE_TSC_SORTED_RES_TOO_MANY| 0 |0x0210 | "Result set too large to be sorted"| -2147483120| -|TSDB_CODE_TSC_APP_ERROR| 0 | 0x0211 | "Application error"| -2147483119| -|TSDB_CODE_TSC_ACTION_IN_PROGRESS| 0 |0x0212 | "Action in progress"| -2147483118| -|TSDB_CODE_TSC_DISCONNECTED| 0 | 0x0213 |"Disconnected from service" |-2147483117| -|TSDB_CODE_TSC_NO_WRITE_AUTH| 0 | 0x0214 | "No write permission" |-2147483116| -|TSDB_CODE_MND_MSG_NOT_PROCESSED| 0| 0x0300| "Message not processed"| -2147482880| -|TSDB_CODE_MND_ACTION_IN_PROGRESS| 0 | 0x0301 |"Message is progressing"| -2147482879| -|TSDB_CODE_MND_ACTION_NEED_REPROCESSED| 0 | 0x0302 |"Messag need to be reprocessed"| -2147482878| -|TSDB_CODE_MND_NO_RIGHTS| 0 | 0x0303| "Insufficient privilege for operation"| -2147482877| -|TSDB_CODE_MND_APP_ERROR| 0 | 0x0304 | "Unexpected generic error in mnode"| -2147482876| -|TSDB_CODE_MND_INVALID_CONNECTION| 0 | 0x0305 | "Invalid message connection"| -2147482875| -|TSDB_CODE_MND_INVALID_MSG_VERSION| 0 | 0x0306 | "Incompatible protocol version"| -2147482874| -|TSDB_CODE_MND_INVALID_MSG_LEN| 0| 0x0307 | "Invalid message length"| -2147482873| -|TSDB_CODE_MND_INVALID_MSG_TYPE| 0 | 0x0308 | "Invalid message type" |-2147482872| -|TSDB_CODE_MND_TOO_MANY_SHELL_CONNS| 0 |0x0309 | "Too many connections"| -2147482871| -|TSDB_CODE_MND_OUT_OF_MEMORY| 0 |0x030A | "Out of memory in mnode"| -2147482870| -|TSDB_CODE_MND_INVALID_SHOWOBJ| 0 | 0x030B |"Data expired"| -2147482869| -|TSDB_CODE_MND_INVALID_QUERY_ID |0 | 0x030C |"Invalid query id" |-2147482868| -|TSDB_CODE_MND_INVALID_STREAM_ID| 0 |0x030D | "Invalid stream id"| -2147482867| -|TSDB_CODE_MND_INVALID_CONN_ID| 0| 0x030E | "Invalid connection id" |-2147482866| -|TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE| 0 | 0x0320| "Object already there"| -2147482848| -|TSDB_CODE_MND_SDB_ERROR| 0 |0x0321 | "Unexpected generic error in sdb" |-2147482847| -|TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE| 0 | 0x0322| "Invalid table type" |-2147482846| -|TSDB_CODE_MND_SDB_OBJ_NOT_THERE| 0 | 0x0323 |"Object not there" |-2147482845| -|TSDB_CODE_MND_SDB_INVAID_META_ROW| 0 | 0x0324| "Invalid meta row" |-2147482844| -|TSDB_CODE_MND_SDB_INVAID_KEY_TYPE| 0 | 0x0325 |"Invalid key type" |-2147482843| -|TSDB_CODE_MND_DNODE_ALREADY_EXIST| 0 | 0x0330 | "DNode already exists"| -2147482832| -|TSDB_CODE_MND_DNODE_NOT_EXIST| 0 | 0x0331| "DNode does not exist" |-2147482831| -|TSDB_CODE_MND_VGROUP_NOT_EXIST| 0 | 0x0332 |"VGroup does not exist"| -2147482830| -|TSDB_CODE_MND_NO_REMOVE_MASTER |0 | 0x0333 | "Master DNode cannot be removed"| -2147482829| -|TSDB_CODE_MND_NO_ENOUGH_DNODES |0 | 0x0334| "Out of DNodes"| -2147482828| -|TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT |0 | 0x0335 | "Cluster cfg inconsistent"| -2147482827| -|TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION| 0 | 0x0336 | "Invalid dnode cfg option"| -2147482826| -|TSDB_CODE_MND_BALANCE_ENABLED| 0 | 0x0337 | "Balance already enabled" |-2147482825| -|TSDB_CODE_MND_VGROUP_NOT_IN_DNODE| 0 |0x0338 | "Vgroup not in dnode"| -2147482824| -|TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE| 0 | 0x0339 | "Vgroup already in dnode"| -2147482823| -|TSDB_CODE_MND_DNODE_NOT_FREE |0 | 0x033A |"Dnode not avaliable"| -2147482822| -|TSDB_CODE_MND_INVALID_CLUSTER_ID |0 |0x033B | "Cluster id not match"| -2147482821| -|TSDB_CODE_MND_NOT_READY| 0 | 0x033C |"Cluster not ready"| -2147482820| -|TSDB_CODE_MND_ACCT_ALREADY_EXIST| 0 | 0x0340 | "Account already exists" |-2147482816| -|TSDB_CODE_MND_INVALID_ACCT| 0 | 0x0341| "Invalid account"| -2147482815| -|TSDB_CODE_MND_INVALID_ACCT_OPTION| 0 | 0x0342 | "Invalid account options"| -2147482814| -|TSDB_CODE_MND_USER_ALREADY_EXIST| 0 | 0x0350 | "User already exists"| -2147482800| -|TSDB_CODE_MND_INVALID_USER |0 | 0x0351 | "Invalid user" |-2147482799| -|TSDB_CODE_MND_INVALID_USER_FORMAT| 0 |0x0352 |"Invalid user format" |-2147482798| -|TSDB_CODE_MND_INVALID_PASS_FORMAT| 0| 0x0353 | "Invalid password format"| -2147482797| -|TSDB_CODE_MND_NO_USER_FROM_CONN| 0 | 0x0354 | "Can not get user from conn"| -2147482796| -|TSDB_CODE_MND_TOO_MANY_USERS| 0 | 0x0355| "Too many users"| -2147482795| -|TSDB_CODE_MND_TABLE_ALREADY_EXIST| 0| 0x0360| "Table already exists"| -2147482784| -|TSDB_CODE_MND_INVALID_TABLE_ID| 0| 0x0361| "Table name too long"| -2147482783| -|TSDB_CODE_MND_INVALID_TABLE_NAME| 0| 0x0362 | "Table does not exist"| -2147482782| -|TSDB_CODE_MND_INVALID_TABLE_TYPE| 0| 0x0363 | "Invalid table type in tsdb"| -2147482781| -|TSDB_CODE_MND_TOO_MANY_TAGS| 0 | 0x0364| "Too many tags"| -2147482780| -|TSDB_CODE_MND_TOO_MANY_TIMESERIES| 0| 0x0366| "Too many time series"| -2147482778| -|TSDB_CODE_MND_NOT_SUPER_TABLE| 0 |0x0367| "Not super table"| -2147482777| -|TSDB_CODE_MND_COL_NAME_TOO_LONG| 0| 0x0368| "Tag name too long"| -2147482776| -|TSDB_CODE_MND_TAG_ALREAY_EXIST| 0| 0x0369| "Tag already exists"| -2147482775| -|TSDB_CODE_MND_TAG_NOT_EXIST| 0 |0x036A | "Tag does not exist" |-2147482774| -|TSDB_CODE_MND_FIELD_ALREAY_EXIST| 0 | 0x036B| "Field already exists"| -2147482773| -|TSDB_CODE_MND_FIELD_NOT_EXIST| 0 | 0x036C | "Field does not exist"| -2147482772| -|TSDB_CODE_MND_INVALID_STABLE_NAME |0 | 0x036D |"Super table does not exist" |-2147482771| -|TSDB_CODE_MND_DB_NOT_SELECTED| 0 | 0x0380 | "Database not specified or available"| -2147482752| -|TSDB_CODE_MND_DB_ALREADY_EXIST| 0 | 0x0381 | "Database already exists"| -2147482751| -|TSDB_CODE_MND_INVALID_DB_OPTION| 0 | 0x0382 | "Invalid database options"| -2147482750| -|TSDB_CODE_MND_INVALID_DB| 0 | 0x0383 | "Invalid database name"| -2147482749| -|TSDB_CODE_MND_MONITOR_DB_FORBIDDEN| 0 | 0x0384 | "Cannot delete monitor database"| -2147482748| -|TSDB_CODE_MND_TOO_MANY_DATABASES| 0| 0x0385 | "Too many databases for account"| -2147482747| -|TSDB_CODE_MND_DB_IN_DROPPING| 0 | 0x0386| "Database not available" |-2147482746| -|TSDB_CODE_DND_MSG_NOT_PROCESSED| 0| 0x0400 | "Message not processed"| -2147482624| -|TSDB_CODE_DND_OUT_OF_MEMORY |0 | 0x0401 | "Dnode out of memory"| -2147482623| -|TSDB_CODE_DND_NO_WRITE_ACCESS| 0 | 0x0402 | "No permission for disk files in dnode"| -2147482622| -|TSDB_CODE_DND_INVALID_MSG_LEN| 0 | 0x0403 | "Invalid message length"| -2147482621| -|TSDB_CODE_VND_ACTION_IN_PROGRESS |0 |0x0500| "Action in progress" |-2147482368| -|TSDB_CODE_VND_MSG_NOT_PROCESSED| 0 |0x0501 | "Message not processed" |-2147482367| -|TSDB_CODE_VND_ACTION_NEED_REPROCESSED |0 |0x0502| "Action need to be reprocessed"| -2147482366| -|TSDB_CODE_VND_INVALID_VGROUP_ID |0 | 0x0503| "Invalid Vgroup ID"| -2147482365| -|TSDB_CODE_VND_INIT_FAILED| 0 | 0x0504 | "Vnode initialization failed"| -2147482364| -|TSDB_CODE_VND_NO_DISKSPACE| 0 |0x0505| "System out of disk space" |-2147482363| -|TSDB_CODE_VND_NO_DISK_PERMISSIONS| 0 | 0x0506| "No write permission for disk files" |-2147482362| -|TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR| 0 | 0x0507 | "Missing data file"| -2147482361| -|TSDB_CODE_VND_OUT_OF_MEMORY |0| 0x0508 | "Out of memory"| -2147482360| -|TSDB_CODE_VND_APP_ERROR| 0| 0x0509 | "Unexpected generic error in vnode"| -2147482359| -|TSDB_CODE_VND_INVALID_STATUS |0| 0x0510 | "Database not ready"| -2147482352| -|TSDB_CODE_VND_NOT_SYNCED| 0 | 0x0511 | "Database suspended"| -2147482351| -|TSDB_CODE_VND_NO_WRITE_AUTH| 0 | 0x0512| "Write operation denied" |-2147482350| -|TSDB_CODE_TDB_INVALID_TABLE_ID |0 | 0x0600 | "Invalid table ID"| -2147482112| -|TSDB_CODE_TDB_INVALID_TABLE_TYPE| 0| 0x0601 |"Invalid table type"| -2147482111| -|TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION| 0| 0x0602| "Invalid table schema version"| -2147482110| -|TSDB_CODE_TDB_TABLE_ALREADY_EXIST| 0 | 0x0603| "Table already exists"| -2147482109| -|TSDB_CODE_TDB_INVALID_CONFIG| 0 | 0x0604| "Invalid configuration"| -2147482108| -|TSDB_CODE_TDB_INIT_FAILED| 0 | 0x0605| "Tsdb init failed"| -2147482107| -|TSDB_CODE_TDB_NO_DISKSPACE| 0 | 0x0606| "No diskspace for tsdb"| -2147482106| -|TSDB_CODE_TDB_NO_DISK_PERMISSIONS| 0 | 0x0607| "No permission for disk files"| -2147482105| -|TSDB_CODE_TDB_FILE_CORRUPTED| 0 | 0x0608| "Data file(s) corrupted"| -2147482104| -|TSDB_CODE_TDB_OUT_OF_MEMORY| 0 | 0x0609| "Out of memory"| -2147482103| -|TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE| 0 | 0x060A| "Tag too old"| -2147482102| -|TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE |0| 0x060B | "Timestamp data out of range"| -2147482101| -|TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP| 0| 0x060C| "Submit message is messed up"| -2147482100| -|TSDB_CODE_TDB_INVALID_ACTION| 0 | 0x060D | "Invalid operation"| -2147482099| -|TSDB_CODE_TDB_INVALID_CREATE_TB_MSG| 0 | 0x060E| "Invalid creation of table"| -2147482098| -|TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM| 0 | 0x060F| "No table data in memory skiplist" |-2147482097| -|TSDB_CODE_TDB_FILE_ALREADY_EXISTS| 0 | 0x0610| "File already exists"| -2147482096| -|TSDB_CODE_TDB_TABLE_RECONFIGURE| 0 | 0x0611| "Need to reconfigure table"| -2147482095| -|TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO| 0 | 0x0612| "Invalid information to create table"| -2147482094| -|TSDB_CODE_QRY_INVALID_QHANDLE| 0 | 0x0700| "Invalid handle"| -2147481856| -|TSDB_CODE_QRY_INVALID_MSG| 0 | 0x0701| "Invalid message"| -2147481855| -|TSDB_CODE_QRY_NO_DISKSPACE| 0 | 0x0702 | "No diskspace for query"| -2147481854| -|TSDB_CODE_QRY_OUT_OF_MEMORY| 0 | 0x0703 | "System out of memory"| -2147481853| -|TSDB_CODE_QRY_APP_ERROR| 0 | 0x0704 | "Unexpected generic error in query"| -2147481852| -|TSDB_CODE_QRY_DUP_JOIN_KEY| 0 | 0x0705| "Duplicated join key"| -2147481851| -|TSDB_CODE_QRY_EXCEED_TAGS_LIMIT| 0 | 0x0706 | "Tag conditon too many"| -2147481850| -|TSDB_CODE_QRY_NOT_READY |0| 0x0707 | "Query not ready" |-2147481849| -|TSDB_CODE_QRY_HAS_RSP| 0 | 0x0708| "Query should response"| -2147481848| -|TSDB_CODE_GRANT_EXPIRED| 0 | 0x0800| "License expired"| -2147481600| -|TSDB_CODE_GRANT_DNODE_LIMITED| 0 | 0x0801 | "DNode creation limited by licence"| -2147481599| -|TSDB_CODE_GRANT_ACCT_LIMITED |0| 0x0802 |"Account creation limited by license"| -2147481598| -|TSDB_CODE_GRANT_TIMESERIES_LIMITED| 0 | 0x0803 | "Table creation limited by license"| -2147481597| -|TSDB_CODE_GRANT_DB_LIMITED| 0 | 0x0804 | "DB creation limited by license"| -2147481596| -|TSDB_CODE_GRANT_USER_LIMITED| 0 | 0x0805 | "User creation limited by license"| -2147481595| -|TSDB_CODE_GRANT_CONN_LIMITED| 0| 0x0806 | "Conn creation limited by license" |-2147481594| -|TSDB_CODE_GRANT_STREAM_LIMITED| 0 | 0x0807 | "Stream creation limited by license"| -2147481593| -|TSDB_CODE_GRANT_SPEED_LIMITED| 0 | 0x0808 | "Write speed limited by license" |-2147481592| -|TSDB_CODE_GRANT_STORAGE_LIMITED| 0 |0x0809 | "Storage capacity limited by license"| -2147481591| -|TSDB_CODE_GRANT_QUERYTIME_LIMITED| 0 | 0x080A | "Query time limited by license" |-2147481590| -|TSDB_CODE_GRANT_CPU_LIMITED| 0 |0x080B |"CPU cores limited by license"| -2147481589| -|TSDB_CODE_SYN_INVALID_CONFIG| 0 | 0x0900| "Invalid Sync Configuration"| -2147481344| -|TSDB_CODE_SYN_NOT_ENABLED| 0 | 0x0901 | "Sync module not enabled" |-2147481343| -|TSDB_CODE_WAL_APP_ERROR| 0| 0x1000 | "Unexpected generic error in wal" |-2147479552| \ No newline at end of file +| 状态码 | 模 | 错误码（十六进制） | 错误描述 | 错误码（十进制） | +| :-------------------------------------- | :--: | :----------------: | :------------------------------------------- | :--------------- | +| TSDB_CODE_RPC_ACTION_IN_PROGRESS | 0 | 0x0001 | "Action in progress" | -2147483647 | +| TSDB_CODE_RPC_AUTH_REQUIRED | 0 | 0x0002 | "Authentication required" | -2147483646 | +| TSDB_CODE_RPC_AUTH_FAILURE | 0 | 0x0003 | "Authentication failure" | -2147483645 | +| TSDB_CODE_RPC_REDIRECT | 0 | 0x0004 | "Redirect" | -2147483644 | +| TSDB_CODE_RPC_NOT_READY | 0 | 0x0005 | "System not ready" | -2147483643 | +| TSDB_CODE_RPC_ALREADY_PROCESSED | 0 | 0x0006 | "Message already processed" | -2147483642 | +| TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED | 0 | 0x0007 | "Last session not finished" | -2147483641 | +| TSDB_CODE_RPC_MISMATCHED_LINK_ID | 0 | 0x0008 | "Mismatched meter id" | -2147483640 | +| TSDB_CODE_RPC_TOO_SLOW | 0 | 0x0009 | "Processing of request timed out" | -2147483639 | +| TSDB_CODE_RPC_MAX_SESSIONS | 0 | 0x000A | "Number of sessions reached limit" | -2147483638 | +| TSDB_CODE_RPC_NETWORK_UNAVAIL | 0 | 0x000B | "Unable to establish connection" | -2147483637 | +| TSDB_CODE_RPC_APP_ERROR | 0 | 0x000C | "Unexpected generic error in RPC" | -2147483636 | +| TSDB_CODE_RPC_UNEXPECTED_RESPONSE | 0 | 0x000D | "Unexpected response" | -2147483635 | +| TSDB_CODE_RPC_INVALID_VALUE | 0 | 0x000E | "Invalid value" | -2147483634 | +| TSDB_CODE_RPC_INVALID_TRAN_ID | 0 | 0x000F | "Invalid transaction id" | -2147483633 | +| TSDB_CODE_RPC_INVALID_SESSION_ID | 0 | 0x0010 | "Invalid session id" | -2147483632 | +| TSDB_CODE_RPC_INVALID_MSG_TYPE | 0 | 0x0011 | "Invalid message type" | -2147483631 | +| TSDB_CODE_RPC_INVALID_RESPONSE_TYPE | 0 | 0x0012 | "Invalid response type" | -2147483630 | +| TSDB_CODE_RPC_INVALID_TIME_STAMP | 0 | 0x0013 | "Invalid timestamp" | -2147483629 | +| TSDB_CODE_COM_OPS_NOT_SUPPORT | 0 | 0x0100 | "Operation not supported" | -2147483392 | +| TSDB_CODE_COM_MEMORY_CORRUPTED | 0 | 0x0101 | "Memory corrupted" | -2147483391 | +| TSDB_CODE_COM_OUT_OF_MEMORY | 0 | 0x0102 | "Out of memory" | -2147483390 | +| TSDB_CODE_COM_INVALID_CFG_MSG | 0 | 0x0103 | "Invalid config message" | -2147483389 | +| TSDB_CODE_COM_FILE_CORRUPTED | 0 | 0x0104 | "Data file corrupted" | -2147483388 | +| TSDB_CODE_TSC_INVALID_OPERATION | 0 | 0x0200 | "Invalid SQL statement" | -2147483136 | +| TSDB_CODE_TSC_INVALID_QHANDLE | 0 | 0x0201 | "Invalid qhandle" | -2147483135 | +| TSDB_CODE_TSC_INVALID_TIME_STAMP | 0 | 0x0202 | "Invalid combination of client/service time" | -2147483134 | +| TSDB_CODE_TSC_INVALID_VALUE | 0 | 0x0203 | "Invalid value in client" | -2147483133 | +| TSDB_CODE_TSC_INVALID_VERSION | 0 | 0x0204 | "Invalid client version" | -2147483132 | +| TSDB_CODE_TSC_INVALID_IE | 0 | 0x0205 | "Invalid client ie" | -2147483131 | +| TSDB_CODE_TSC_INVALID_FQDN | 0 | 0x0206 | "Invalid host name" | -2147483130 | +| TSDB_CODE_TSC_INVALID_USER_LENGTH | 0 | 0x0207 | "Invalid user name" | -2147483129 | +| TSDB_CODE_TSC_INVALID_PASS_LENGTH | 0 | 0x0208 | "Invalid password" | -2147483128 | +| TSDB_CODE_TSC_INVALID_DB_LENGTH | 0 | 0x0209 | "Database name too long" | -2147483127 | +| TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH | 0 | 0x020A | "Table name too long" | -2147483126 | +| TSDB_CODE_TSC_INVALID_CONNECTION | 0 | 0x020B | "Invalid connection" | -2147483125 | +| TSDB_CODE_TSC_OUT_OF_MEMORY | 0 | 0x020C | "System out of memory" | -2147483124 | +| TSDB_CODE_TSC_NO_DISKSPACE | 0 | 0x020D | "System out of disk space" | -2147483123 | +| TSDB_CODE_TSC_QUERY_CACHE_ERASED | 0 | 0x020E | "Query cache erased" | -2147483122 | +| TSDB_CODE_TSC_QUERY_CANCELLED | 0 | 0x020F | "Query terminated" | -2147483121 | +| TSDB_CODE_TSC_SORTED_RES_TOO_MANY | 0 | 0x0210 | "Result set too large to be sorted" | -2147483120 | +| TSDB_CODE_TSC_APP_ERROR | 0 | 0x0211 | "Application error" | -2147483119 | +| TSDB_CODE_TSC_ACTION_IN_PROGRESS | 0 | 0x0212 | "Action in progress" | -2147483118 | +| TSDB_CODE_TSC_DISCONNECTED | 0 | 0x0213 | "Disconnected from service" | -2147483117 | +| TSDB_CODE_TSC_NO_WRITE_AUTH | 0 | 0x0214 | "No write permission" | -2147483116 | +| TSDB_CODE_MND_MSG_NOT_PROCESSED | 0 | 0x0300 | "Message not processed" | -2147482880 | +| TSDB_CODE_MND_ACTION_IN_PROGRESS | 0 | 0x0301 | "Message is progressing" | -2147482879 | +| TSDB_CODE_MND_ACTION_NEED_REPROCESSED | 0 | 0x0302 | "Messag need to be reprocessed" | -2147482878 | +| TSDB_CODE_MND_NO_RIGHTS | 0 | 0x0303 | "Insufficient privilege for operation" | -2147482877 | +| TSDB_CODE_MND_APP_ERROR | 0 | 0x0304 | "Unexpected generic error in mnode" | -2147482876 | +| TSDB_CODE_MND_INVALID_CONNECTION | 0 | 0x0305 | "Invalid message connection" | -2147482875 | +| TSDB_CODE_MND_INVALID_MSG_VERSION | 0 | 0x0306 | "Incompatible protocol version" | -2147482874 | +| TSDB_CODE_MND_INVALID_MSG_LEN | 0 | 0x0307 | "Invalid message length" | -2147482873 | +| TSDB_CODE_MND_INVALID_MSG_TYPE | 0 | 0x0308 | "Invalid message type" | -2147482872 | +| TSDB_CODE_MND_TOO_MANY_SHELL_CONNS | 0 | 0x0309 | "Too many connections" | -2147482871 | +| TSDB_CODE_MND_OUT_OF_MEMORY | 0 | 0x030A | "Out of memory in mnode" | -2147482870 | +| TSDB_CODE_MND_INVALID_SHOWOBJ | 0 | 0x030B | "Data expired" | -2147482869 | +| TSDB_CODE_MND_INVALID_QUERY_ID | 0 | 0x030C | "Invalid query id" | -2147482868 | +| TSDB_CODE_MND_INVALID_STREAM_ID | 0 | 0x030D | "Invalid stream id" | -2147482867 | +| TSDB_CODE_MND_INVALID_CONN_ID | 0 | 0x030E | "Invalid connection id" | -2147482866 | +| TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE | 0 | 0x0320 | "Object already there" | -2147482848 | +| TSDB_CODE_MND_SDB_ERROR | 0 | 0x0321 | "Unexpected generic error in sdb" | -2147482847 | +| TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE | 0 | 0x0322 | "Invalid table type" | -2147482846 | +| TSDB_CODE_MND_SDB_OBJ_NOT_THERE | 0 | 0x0323 | "Object not there" | -2147482845 | +| TSDB_CODE_MND_SDB_INVAID_META_ROW | 0 | 0x0324 | "Invalid meta row" | -2147482844 | +| TSDB_CODE_MND_SDB_INVAID_KEY_TYPE | 0 | 0x0325 | "Invalid key type" | -2147482843 | +| TSDB_CODE_MND_DNODE_ALREADY_EXIST | 0 | 0x0330 | "DNode already exists" | -2147482832 | +| TSDB_CODE_MND_DNODE_NOT_EXIST | 0 | 0x0331 | "DNode does not exist" | -2147482831 | +| TSDB_CODE_MND_VGROUP_NOT_EXIST | 0 | 0x0332 | "VGroup does not exist" | -2147482830 | +| TSDB_CODE_MND_NO_REMOVE_MASTER | 0 | 0x0333 | "Master DNode cannot be removed" | -2147482829 | +| TSDB_CODE_MND_NO_ENOUGH_DNODES | 0 | 0x0334 | "Out of DNodes" | -2147482828 | +| TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT | 0 | 0x0335 | "Cluster cfg inconsistent" | -2147482827 | +| TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION | 0 | 0x0336 | "Invalid dnode cfg option" | -2147482826 | +| TSDB_CODE_MND_BALANCE_ENABLED | 0 | 0x0337 | "Balance already enabled" | -2147482825 | +| TSDB_CODE_MND_VGROUP_NOT_IN_DNODE | 0 | 0x0338 | "Vgroup not in dnode" | -2147482824 | +| TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE | 0 | 0x0339 | "Vgroup already in dnode" | -2147482823 | +| TSDB_CODE_MND_DNODE_NOT_FREE | 0 | 0x033A | "Dnode not avaliable" | -2147482822 | +| TSDB_CODE_MND_INVALID_CLUSTER_ID | 0 | 0x033B | "Cluster id not match" | -2147482821 | +| TSDB_CODE_MND_NOT_READY | 0 | 0x033C | "Cluster not ready" | -2147482820 | +| TSDB_CODE_MND_ACCT_ALREADY_EXIST | 0 | 0x0340 | "Account already exists" | -2147482816 | +| TSDB_CODE_MND_INVALID_ACCT | 0 | 0x0341 | "Invalid account" | -2147482815 | +| TSDB_CODE_MND_INVALID_ACCT_OPTION | 0 | 0x0342 | "Invalid account options" | -2147482814 | +| TSDB_CODE_MND_USER_ALREADY_EXIST | 0 | 0x0350 | "User already exists" | -2147482800 | +| TSDB_CODE_MND_INVALID_USER | 0 | 0x0351 | "Invalid user" | -2147482799 | +| TSDB_CODE_MND_INVALID_USER_FORMAT | 0 | 0x0352 | "Invalid user format" | -2147482798 | +| TSDB_CODE_MND_INVALID_PASS_FORMAT | 0 | 0x0353 | "Invalid password format" | -2147482797 | +| TSDB_CODE_MND_NO_USER_FROM_CONN | 0 | 0x0354 | "Can not get user from conn" | -2147482796 | +| TSDB_CODE_MND_TOO_MANY_USERS | 0 | 0x0355 | "Too many users" | -2147482795 | +| TSDB_CODE_MND_TABLE_ALREADY_EXIST | 0 | 0x0360 | "Table already exists" | -2147482784 | +| TSDB_CODE_MND_INVALID_TABLE_ID | 0 | 0x0361 | "Table name too long" | -2147482783 | +| TSDB_CODE_MND_INVALID_TABLE_NAME | 0 | 0x0362 | "Table does not exist" | -2147482782 | +| TSDB_CODE_MND_INVALID_TABLE_TYPE | 0 | 0x0363 | "Invalid table type in tsdb" | -2147482781 | +| TSDB_CODE_MND_TOO_MANY_TAGS | 0 | 0x0364 | "Too many tags" | -2147482780 | +| TSDB_CODE_MND_TOO_MANY_TIMESERIES | 0 | 0x0366 | "Too many time series" | -2147482778 | +| TSDB_CODE_MND_NOT_SUPER_TABLE | 0 | 0x0367 | "Not super table" | -2147482777 | +| TSDB_CODE_MND_COL_NAME_TOO_LONG | 0 | 0x0368 | "Tag name too long" | -2147482776 | +| TSDB_CODE_MND_TAG_ALREAY_EXIST | 0 | 0x0369 | "Tag already exists" | -2147482775 | +| TSDB_CODE_MND_TAG_NOT_EXIST | 0 | 0x036A | "Tag does not exist" | -2147482774 | +| TSDB_CODE_MND_FIELD_ALREAY_EXIST | 0 | 0x036B | "Field already exists" | -2147482773 | +| TSDB_CODE_MND_FIELD_NOT_EXIST | 0 | 0x036C | "Field does not exist" | -2147482772 | +| TSDB_CODE_MND_INVALID_STABLE_NAME | 0 | 0x036D | "Super table does not exist" | -2147482771 | +| TSDB_CODE_MND_DB_NOT_SELECTED | 0 | 0x0380 | "Database not specified or available" | -2147482752 | +| TSDB_CODE_MND_DB_ALREADY_EXIST | 0 | 0x0381 | "Database already exists" | -2147482751 | +| TSDB_CODE_MND_INVALID_DB_OPTION | 0 | 0x0382 | "Invalid database options" | -2147482750 | +| TSDB_CODE_MND_INVALID_DB | 0 | 0x0383 | "Invalid database name" | -2147482749 | +| TSDB_CODE_MND_MONITOR_DB_FORBIDDEN | 0 | 0x0384 | "Cannot delete monitor database" | -2147482748 | +| TSDB_CODE_MND_TOO_MANY_DATABASES | 0 | 0x0385 | "Too many databases for account" | -2147482747 | +| TSDB_CODE_MND_DB_IN_DROPPING | 0 | 0x0386 | "Database not available" | -2147482746 | +| TSDB_CODE_DND_MSG_NOT_PROCESSED | 0 | 0x0400 | "Message not processed" | -2147482624 | +| TSDB_CODE_DND_OUT_OF_MEMORY | 0 | 0x0401 | "Dnode out of memory" | -2147482623 | +| TSDB_CODE_DND_NO_WRITE_ACCESS | 0 | 0x0402 | "No permission for disk files in dnode" | -2147482622 | +| TSDB_CODE_DND_INVALID_MSG_LEN | 0 | 0x0403 | "Invalid message length" | -2147482621 | +| TSDB_CODE_VND_ACTION_IN_PROGRESS | 0 | 0x0500 | "Action in progress" | -2147482368 | +| TSDB_CODE_VND_MSG_NOT_PROCESSED | 0 | 0x0501 | "Message not processed" | -2147482367 | +| TSDB_CODE_VND_ACTION_NEED_REPROCESSED | 0 | 0x0502 | "Action need to be reprocessed" | -2147482366 | +| TSDB_CODE_VND_INVALID_VGROUP_ID | 0 | 0x0503 | "Invalid Vgroup ID" | -2147482365 | +| TSDB_CODE_VND_INIT_FAILED | 0 | 0x0504 | "Vnode initialization failed" | -2147482364 | +| TSDB_CODE_VND_NO_DISKSPACE | 0 | 0x0505 | "System out of disk space" | -2147482363 | +| TSDB_CODE_VND_NO_DISK_PERMISSIONS | 0 | 0x0506 | "No write permission for disk files" | -2147482362 | +| TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR | 0 | 0x0507 | "Missing data file" | -2147482361 | +| TSDB_CODE_VND_OUT_OF_MEMORY | 0 | 0x0508 | "Out of memory" | -2147482360 | +| TSDB_CODE_VND_APP_ERROR | 0 | 0x0509 | "Unexpected generic error in vnode" | -2147482359 | +| TSDB_CODE_VND_INVALID_STATUS | 0 | 0x0510 | "Database not ready" | -2147482352 | +| TSDB_CODE_VND_NOT_SYNCED | 0 | 0x0511 | "Database suspended" | -2147482351 | +| TSDB_CODE_VND_NO_WRITE_AUTH | 0 | 0x0512 | "Write operation denied" | -2147482350 | +| TSDB_CODE_TDB_INVALID_TABLE_ID | 0 | 0x0600 | "Invalid table ID" | -2147482112 | +| TSDB_CODE_TDB_INVALID_TABLE_TYPE | 0 | 0x0601 | "Invalid table type" | -2147482111 | +| TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION | 0 | 0x0602 | "Invalid table schema version" | -2147482110 | +| TSDB_CODE_TDB_TABLE_ALREADY_EXIST | 0 | 0x0603 | "Table already exists" | -2147482109 | +| TSDB_CODE_TDB_INVALID_CONFIG | 0 | 0x0604 | "Invalid configuration" | -2147482108 | +| TSDB_CODE_TDB_INIT_FAILED | 0 | 0x0605 | "Tsdb init failed" | -2147482107 | +| TSDB_CODE_TDB_NO_DISKSPACE | 0 | 0x0606 | "No diskspace for tsdb" | -2147482106 | +| TSDB_CODE_TDB_NO_DISK_PERMISSIONS | 0 | 0x0607 | "No permission for disk files" | -2147482105 | +| TSDB_CODE_TDB_FILE_CORRUPTED | 0 | 0x0608 | "Data file(s) corrupted" | -2147482104 | +| TSDB_CODE_TDB_OUT_OF_MEMORY | 0 | 0x0609 | "Out of memory" | -2147482103 | +| TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE | 0 | 0x060A | "Tag too old" | -2147482102 | +| TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE | 0 | 0x060B | "Timestamp data out of range" | -2147482101 | +| TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP | 0 | 0x060C | "Submit message is messed up" | -2147482100 | +| TSDB_CODE_TDB_INVALID_ACTION | 0 | 0x060D | "Invalid operation" | -2147482099 | +| TSDB_CODE_TDB_INVALID_CREATE_TB_MSG | 0 | 0x060E | "Invalid creation of table" | -2147482098 | +| TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM | 0 | 0x060F | "No table data in memory skiplist" | -2147482097 | +| TSDB_CODE_TDB_FILE_ALREADY_EXISTS | 0 | 0x0610 | "File already exists" | -2147482096 | +| TSDB_CODE_TDB_TABLE_RECONFIGURE | 0 | 0x0611 | "Need to reconfigure table" | -2147482095 | +| TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO | 0 | 0x0612 | "Invalid information to create table" | -2147482094 | +| TSDB_CODE_QRY_INVALID_QHANDLE | 0 | 0x0700 | "Invalid handle" | -2147481856 | +| TSDB_CODE_QRY_INVALID_MSG | 0 | 0x0701 | "Invalid message" | -2147481855 | +| TSDB_CODE_QRY_NO_DISKSPACE | 0 | 0x0702 | "No diskspace for query" | -2147481854 | +| TSDB_CODE_QRY_OUT_OF_MEMORY | 0 | 0x0703 | "System out of memory" | -2147481853 | +| TSDB_CODE_QRY_APP_ERROR | 0 | 0x0704 | "Unexpected generic error in query" | -2147481852 | +| TSDB_CODE_QRY_DUP_JOIN_KEY | 0 | 0x0705 | "Duplicated join key" | -2147481851 | +| TSDB_CODE_QRY_EXCEED_TAGS_LIMIT | 0 | 0x0706 | "Tag conditon too many" | -2147481850 | +| TSDB_CODE_QRY_NOT_READY | 0 | 0x0707 | "Query not ready" | -2147481849 | +| TSDB_CODE_QRY_HAS_RSP | 0 | 0x0708 | "Query should response" | -2147481848 | +| TSDB_CODE_GRANT_EXPIRED | 0 | 0x0800 | "License expired" | -2147481600 | +| TSDB_CODE_GRANT_DNODE_LIMITED | 0 | 0x0801 | "DNode creation limited by licence" | -2147481599 | +| TSDB_CODE_GRANT_ACCT_LIMITED | 0 | 0x0802 | "Account creation limited by license" | -2147481598 | +| TSDB_CODE_GRANT_TIMESERIES_LIMITED | 0 | 0x0803 | "Table creation limited by license" | -2147481597 | +| TSDB_CODE_GRANT_DB_LIMITED | 0 | 0x0804 | "DB creation limited by license" | -2147481596 | +| TSDB_CODE_GRANT_USER_LIMITED | 0 | 0x0805 | "User creation limited by license" | -2147481595 | +| TSDB_CODE_GRANT_CONN_LIMITED | 0 | 0x0806 | "Conn creation limited by license" | -2147481594 | +| TSDB_CODE_GRANT_STREAM_LIMITED | 0 | 0x0807 | "Stream creation limited by license" | -2147481593 | +| TSDB_CODE_GRANT_SPEED_LIMITED | 0 | 0x0808 | "Write speed limited by license" | -2147481592 | +| TSDB_CODE_GRANT_STORAGE_LIMITED | 0 | 0x0809 | "Storage capacity limited by license" | -2147481591 | +| TSDB_CODE_GRANT_QUERYTIME_LIMITED | 0 | 0x080A | "Query time limited by license" | -2147481590 | +| TSDB_CODE_GRANT_CPU_LIMITED | 0 | 0x080B | "CPU cores limited by license" | -2147481589 | +| TSDB_CODE_SYN_INVALID_CONFIG | 0 | 0x0900 | "Invalid Sync Configuration" | -2147481344 | +| TSDB_CODE_SYN_NOT_ENABLED | 0 | 0x0901 | "Sync module not enabled" | -2147481343 | +| TSDB_CODE_WAL_APP_ERROR | 0 | 0x1000 | "Unexpected generic error in wal" | -2147479552 | diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index fbb82ee140..af78074eb2 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -41,9 +41,9 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM 在TDengine中，普通表的数据模型中可使用以下 10 种数据类型。 -| | 类型 | Bytes | 说明 | +| # | **类型** | **Bytes** | **说明** | | ---- | :-------: | ------ | ------------------------------------------------------------ | -| 1 | TIMESTAMP | 8 | 时间戳。缺省精度毫秒，可支持微秒。从格林威治时间 1970-01-01 00:00:00.000 (UTC/GMT) 开始，计时不能早于该时间。（从 2.0.18 版本开始，已经去除了这一时间范围限制） | +| 1 | TIMESTAMP | 8 | 时间戳。缺省精度毫秒，可支持微秒。从格林威治时间 1970-01-01 00:00:00.000 (UTC/GMT) 开始，计时不能早于该时间。（从 2.0.18.0 版本开始，已经去除了这一时间范围限制） | | 2 | INT | 4 | 整型，范围 [-2^31+1, 2^31-1], -2^31 用作 NULL | | 3 | BIGINT | 8 | 长整型，范围 [-2^63+1, 2^63-1], -2^63 用于 NULL | | 4 | FLOAT | 4 | 浮点型，有效位数 6-7，范围 [-3.4E38, 3.4E38] | @@ -53,6 +53,7 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM | 8 | TINYINT | 1 | 单字节整型，范围 [-127, 127], -128 用于 NULL | | 9 | BOOL | 1 | 布尔型，{true, false} | | 10 | NCHAR | 自定义 | 记录包含多字节字符在内的字符串，如中文字符。每个 nchar 字符占用 4 bytes 的存储空间。字符串两端使用单引号引用，字符串内的单引号需用转义字符 `\’`。nchar 使用时须指定字符串大小，类型为 nchar(10) 的列表示此列的字符串最多存储 10 个 nchar 字符，会固定占用 40 bytes 的空间。如果用户字符串长度超出声明长度，将会报错。 | + **Tips**: 1. TDengine 对 SQL 语句中的英文字符不区分大小写，自动转化为小写执行。因此用户大小写敏感的字符串及密码，需要使用单引号将字符串引起来。 @@ -63,11 +64,11 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM - **创建数据库** ```mysql - CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep] [UPDATE 1]; + CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep] [DAYS days] [UPDATE 1]; ``` - 说明： + 说明： - 1) KEEP是该数据库的数据保留多长天数，缺省是3650天(10年)，数据库会自动删除超过时限的数据； + 1) KEEP是该数据库的数据保留多长天数，缺省是3650天(10年)，数据库会自动删除超过时限的数据； 2) UPDATE 标志数据库支持更新相同时间戳数据； @@ -75,7 +76,7 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM 4) 一条SQL 语句的最大长度为65480个字符； - 5) 数据库还有更多与存储相关的配置参数，请参见系统管理。 + 5) 数据库还有更多与存储相关的配置参数，请参见 [服务端配置](https://www.taosdata.com/cn/documentation/administrator#config) 章节。 - **显示系统当前参数** @@ -125,7 +126,7 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM ```mysql ALTER DATABASE db_name CACHELAST 0; ``` - CACHELAST 参数控制是否在内存中缓存数据子表的 last_row。缺省值为 0，取值范围 [0, 1]。其中 0 表示不启用、1 表示启用。（从 2.0.11 版本开始支持，修改后需要重启服务器生效。） + CACHELAST 参数控制是否在内存中缓存数据子表的 last_row。缺省值为 0，取值范围 [0, 1]。其中 0 表示不启用、1 表示启用。（从 2.0.11.0 版本开始支持。从 2.1.1.0 版本开始，修改此参数后无需重启服务器即可生效。） **Tips**: 以上所有参数修改后都可以用show databases来确认是否修改成功。 @@ -167,22 +168,22 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM ```mysql CREATE TABLE [IF NOT EXISTS] tb_name USING stb_name TAGS (tag_value1, ...); ``` - 以指定的超级表为模板，指定 tags 的值来创建数据表。 + 以指定的超级表为模板，指定 TAGS 的值来创建数据表。 -- **以超级表为模板创建数据表，并指定具体的 tags 列** +- **以超级表为模板创建数据表，并指定具体的 TAGS 列** ```mysql CREATE TABLE [IF NOT EXISTS] tb_name USING stb_name (tag_name1, ...) TAGS (tag_value1, ...); ``` - 以指定的超级表为模板，指定一部分 tags 列的值来创建数据表。（没被指定的 tags 列会设为空值。） - 说明：从 2.0.17 版本开始支持这种方式。在之前的版本中，不允许指定 tags 列，而必须显式给出所有 tags 列的取值。 + 以指定的超级表为模板，指定一部分 TAGS 列的值来创建数据表（没被指定的 TAGS 列会设为空值）。 + 说明：从 2.0.17.0 版本开始支持这种方式。在之前的版本中，不允许指定 TAGS 列，而必须显式给出所有 TAGS 列的取值。 - **批量创建数据表** ```mysql CREATE TABLE [IF NOT EXISTS] tb_name1 USING stb_name TAGS (tag_value1, ...) tb_name2 USING stb_name TAGS (tag_value2, ...) ...; ``` - 以更快的速度批量创建大量数据表。（服务器端 2.0.14 及以上版本） + 以更快的速度批量创建大量数据表（服务器端 2.0.14 及以上版本）。 说明： @@ -220,6 +221,7 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM ```mysql SET MAX_BINARY_DISPLAY_WIDTH ; ``` + 如显示的内容后面以...结尾时，表示该内容已被截断，可通过本命令修改显示字符宽度以显示完整的内容。 - **获取表的结构信息** @@ -236,14 +238,14 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM 1) 列的最大个数为1024，最小个数为2； - 2) 列名最大长度为64； + 2) 列名最大长度为64。 - **表删除列** ```mysql ALTER TABLE tb_name DROP COLUMN field_name; ``` - 如果表是通过[超级表](../super-table/)创建，更改表结构的操作只能对超级表进行。同时针对超级表的结构更改对所有通过该结构创建的表生效。对于不是通过超级表创建的表，可以直接修改表结构 + 如果表是通过超级表创建，更改表结构的操作只能对超级表进行。同时针对超级表的结构更改对所有通过该结构创建的表生效。对于不是通过超级表创建的表，可以直接修改表结构。 ## 超级表STable管理 @@ -254,7 +256,7 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM ```mysql CREATE STABLE [IF NOT EXISTS] stb_name (timestamp_field_name TIMESTAMP, field1_name data_type1 [, field2_name data_type2 ...]) TAGS (tag1_name tag_type1, tag2_name tag_type2 [, tag3_name tag_type3]); ``` - 创建 STable，与创建表的 SQL 语法相似，但需指定 TAGS 字段的名称和类型 + 创建 STable，与创建表的 SQL 语法相似，但需要指定 TAGS 字段的名称和类型 说明： @@ -276,7 +278,7 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM - **显示当前数据库下的所有超级表信息** ```mysql - SHOW STABLES [LIKE tb_name_wildcar]; + SHOW STABLES [LIKE tb_name_wildcard]; ``` 查看数据库内全部 STable，及其相关信息，包括 STable 的名称、创建时间、列数量、标签（TAG）数量、通过该 STable 建表的数量。 @@ -341,7 +343,7 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM ```mysql INSERT INTO tb_name VALUES (field_value, ...); ``` - 向表tb_name中插入一条记录 + 向表tb_name中插入一条记录。 - **插入一条记录，数据对应到指定的列** ```mysql @@ -353,42 +355,56 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM ```mysql INSERT INTO tb_name VALUES (field1_value1, ...) (field1_value2, ...) ...; ``` - 向表tb_name中插入多条记录 + 向表tb_name中插入多条记录。 **注意**：在使用“插入多条记录”方式写入数据时，不能把第一列的时间戳取值都设为now，否则会导致语句中的多条记录使用相同的时间戳，于是就可能出现相互覆盖以致这些数据行无法全部被正确保存。 - **按指定的列插入多条记录** ```mysql INSERT INTO tb_name (field1_name, ...) VALUES (field1_value1, ...) (field1_value2, ...) ...; ``` - 向表tb_name中按指定的列插入多条记录 + 向表tb_name中按指定的列插入多条记录。 - **向多个表插入多条记录** ```mysql INSERT INTO tb1_name VALUES (field1_value1, ...) (field1_value2, ...) ... tb2_name VALUES (field1_value1, ...) (field1_value2, ...) ...; ``` - 同时向表tb1_name和tb2_name中分别插入多条记录 + 同时向表tb1_name和tb2_name中分别插入多条记录。 - **同时向多个表按列插入多条记录** ```mysql INSERT INTO tb1_name (tb1_field1_name, ...) VALUES (field1_value1, ...) (field1_value2, ...) ... tb2_name (tb2_field1_name, ...) VALUES (field1_value1, ...) (field1_value2, ...) ...; ``` - 同时向表tb1_name和tb2_name中按列分别插入多条记录 + 同时向表tb1_name和tb2_name中按列分别插入多条记录。 - 注意：允许插入的最老记录的时间戳，是相对于当前服务器时间，减去配置的keep值（数据保留的天数），允许插入的最新记录的时间戳，是相对于当前服务器时间，加上配置的days值（数据文件存储数据的时间跨度，单位为天）。keep和days都是可以在创建数据库时指定的，缺省值分别是3650天和10天。 + 注意： + 1) 如果时间戳为now，系统将自动使用客户端当前时间作为该记录的时间戳； + 2) 允许插入的最老记录的时间戳，是相对于当前服务器时间，减去配置的keep值（数据保留的天数），允许插入的最新记录的时间戳，是相对于当前服务器时间，加上配置的days值（数据文件存储数据的时间跨度，单位为天）。keep和days都是可以在创建数据库时指定的，缺省值分别是3650天和10天。 - **插入记录时自动建表** ```mysql INSERT INTO tb_name USING stb_name TAGS (tag_value1, ...) VALUES (field_value1, ...); ``` - 如果用户在写数据时并不确定某个表是否存在，此时可以在写入数据时使用自动建表语法来创建不存在的表，若该表已存在则不会建立新表。自动建表时，要求必须以超级表为模板，并写明数据表的 tags 取值。 + 如果用户在写数据时并不确定某个表是否存在，此时可以在写入数据时使用自动建表语法来创建不存在的表，若该表已存在则不会建立新表。自动建表时，要求必须以超级表为模板，并写明数据表的 TAGS 取值。 -- **插入记录时自动建表，并指定具体的 tags 列** +- **插入记录时自动建表，并指定具体的 TAGS 列** ```mysql INSERT INTO tb_name USING stb_name (tag_name1, ...) TAGS (tag_value1, ...) VALUES (field_value1, ...); ``` - 在自动建表时，可以只是指定部分 tags 列的取值，未被指定的 tags 列将取为空值。 + 在自动建表时，可以只是指定部分 TAGS 列的取值，未被指定的 TAGS 列将取为空值。 + +- **同时向多个表按列插入多条记录，自动建表** + ```mysql + INSERT INTO tb1_name (tb1_field1_name, ...) [USING stb1_name TAGS (tag_value1, ...)] VALUES (field1_value1, ...) (field1_value2, ...) ... + tb2_name (tb2_field1_name, ...) [USING stb2_name TAGS (tag_value2, ...)] VALUES (field1_value1, ...) (field1_value2, ...) ...; + ``` + 以自动建表的方式，同时向表tb1_name和tb2_name中按列分别插入多条记录。 + 从 2.0.20.5 版本开始，子表的列名可以不跟在子表名称后面，而是可以放在 TAGS 和 VALUES 之间，例如像下面这样写： + ```mysql + INSERT INTO tb1_name [USING stb1_name TAGS (tag_value1, ...)] (tb1_field1_name, ...) VALUES (field1_value1, ...) (field1_value2, ...) ...; + ``` + 注意：虽然两种写法都可以，但并不能在一条 SQL 语句中混用，否则会报语法错误。 **历史记录写入**：可使用IMPORT或者INSERT命令，IMPORT的语法，功能与INSERT完全一样。 @@ -471,7 +487,7 @@ Query OK, 9 row(s) in set (0.002022s) SELECT * FROM d1001; SELECT d1001.* FROM d1001; ``` -在Join查询中，带前缀的\*和不带前缀\*返回的结果有差别， \*返回全部表的所有列数据（不包含标签），带前缀的通配符，则只返回该表的列数据。 +在JOIN查询中，带前缀的\*和不带前缀\*返回的结果有差别， \*返回全部表的所有列数据（不包含标签），带前缀的通配符，则只返回该表的列数据。 ```mysql taos> SELECT * FROM d1001, d1003 WHERE d1001.ts=d1003.ts; ts | current | voltage | phase | ts | current | voltage | phase | @@ -487,7 +503,7 @@ taos> SELECT d1001.* FROM d1001,d1003 WHERE d1001.ts = d1003.ts; Query OK, 1 row(s) in set (0.020443s) ``` -在使用SQL函数来进行查询过程中，部分SQL函数支持通配符操作。其中的区别在于： +在使用SQL函数来进行查询的过程中，部分SQL函数支持通配符操作。其中的区别在于： ```count(*)```函数只返回一列。```first```、```last```、```last_row```函数则是返回全部列。 ```mysql @@ -522,12 +538,12 @@ Query OK, 2 row(s) in set (0.003112s) ##### 获取标签列的去重取值 -从 2.0.15 版本开始，支持在超级表查询标签列时，指定 distinct 关键字，这样将返回指定标签列的所有不重复取值。 +从 2.0.15 版本开始，支持在超级表查询标签列时，指定 DISTINCT 关键字，这样将返回指定标签列的所有不重复取值。 ```mysql SELECT DISTINCT tag_name FROM stb_name; ``` -注意：目前 distinct 关键字只支持对超级表的标签列进行去重，而不能用于普通列。 +注意：目前 DISTINCT 关键字只支持对超级表的标签列进行去重，而不能用于普通列。 @@ -562,7 +578,7 @@ SELECT * FROM d1001; #### 特殊功能 -部分特殊的查询功能可以不使用FROM子句执行。获取当前所在的数据库 database() +部分特殊的查询功能可以不使用FROM子句执行。获取当前所在的数据库 database()： ```mysql taos> SELECT DATABASE(); database() | @@ -570,7 +586,7 @@ taos> SELECT DATABASE(); power | Query OK, 1 row(s) in set (0.000079s) ``` -如果登录的时候没有指定默认数据库，且没有使用```use```命令切换数据，则返回NULL。 +如果登录的时候没有指定默认数据库，且没有使用```USE```命令切换数据，则返回NULL。 ```mysql taos> SELECT DATABASE(); database() | @@ -578,7 +594,7 @@ taos> SELECT DATABASE(); NULL | Query OK, 1 row(s) in set (0.000184s) ``` -获取服务器和客户端版本号: +获取服务器和客户端版本号： ```mysql taos> SELECT CLIENT_VERSION(); client_version() | @@ -622,7 +638,7 @@ SELECT TBNAME, location FROM meters; ```mysql SELECT COUNT(TBNAME) FROM meters; ``` -以上两个查询均只支持在Where条件子句中添加针对标签（TAGS）的过滤条件。例如： +以上两个查询均只支持在WHERE条件子句中添加针对标签（TAGS）的过滤条件。例如： ```mysql taos> SELECT TBNAME, location FROM meters; tbname | location | @@ -648,30 +664,31 @@ Query OK, 1 row(s) in set (0.001091s) - 参数 LIMIT 控制输出条数，OFFSET 指定从第几条开始输出。LIMIT/OFFSET 对结果集的执行顺序在 ORDER BY 之后。 * 在有 GROUP BY 子句的情况下，LIMIT 参数控制的是每个分组中至多允许输出的条数。 - 参数 SLIMIT 控制由 GROUP BY 指令划分的分组中，至多允许输出几个分组的数据。 -- 通过 ">>" 输出结果可以导出到指定文件。 +- 通过 “>>” 输出结果可以导出到指定文件。 ### 支持的条件过滤操作 -| Operation | Note | Applicable Data Types | -| ----------- | ----------------------------- | ------------------------------------- | -| > | larger than | **`timestamp`** and all numeric types | -| < | smaller than | **`timestamp`** and all numeric types | -| >= | larger than or equal to | **`timestamp`** and all numeric types | -| <= | smaller than or equal to | **`timestamp`** and all numeric types | -| = | equal to | all types | -| <> | not equal to | all types | -| between and | within a certain range | **`timestamp`** and all numeric types | -| % | match with any char sequences | **`binary`** **`nchar`** | -| _ | match with a single char | **`binary`** **`nchar`** | +| **Operation** | **Note** | **Applicable Data Types** | +| --------------- | ----------------------------- | ------------------------------------- | +| > | larger than | **`timestamp`** and all numeric types | +| < | smaller than | **`timestamp`** and all numeric types | +| >= | larger than or equal to | **`timestamp`** and all numeric types | +| <= | smaller than or equal to | **`timestamp`** and all numeric types | +| = | equal to | all types | +| <> | not equal to | all types | +| between and | within a certain range | **`timestamp`** and all numeric types | +| % | match with any char sequences | **`binary`** **`nchar`** | +| _ | match with a single char | **`binary`** **`nchar`** | 1. 同时进行多个字段的范围过滤，需要使用关键词 AND 来连接不同的查询条件，暂不支持 OR 连接的不同列之间的查询过滤条件。 2. 针对单一字段的过滤，如果是时间过滤条件，则一条语句中只支持设定一个；但针对其他的（普通）列或标签列，则可以使用 `OR` 关键字进行组合条件的查询过滤。例如：((value > 20 AND value < 30) OR (value < 12)) 。 3. 从 2.0.17 版本开始，条件过滤开始支持 BETWEEN AND 语法，例如 `WHERE col2 BETWEEN 1.5 AND 3.25` 表示查询条件为“1.5 ≤ col2 ≤ 3.25”。 -### UNION ALL 操作符 + +### UNION ALL 操作符 ```mysql SELECT ... @@ -691,37 +709,38 @@ TDengine 支持 UNION ALL 操作符。也就是说，如果多个 SELECT 子句 ### SQL 示例 -- 对于下面的例子，表tb1用以下语句创建 +- 对于下面的例子，表tb1用以下语句创建： ```mysql CREATE TABLE tb1 (ts TIMESTAMP, col1 INT, col2 FLOAT, col3 BINARY(50)); ``` -- 查询tb1刚过去的一个小时的所有记录 +- 查询tb1刚过去的一个小时的所有记录： ```mysql SELECT * FROM tb1 WHERE ts >= NOW - 1h; ``` -- 查询表tb1从2018-06-01 08:00:00.000 到2018-06-02 08:00:00.000时间范围，并且col3的字符串是'nny'结尾的记录，结果按照时间戳降序 +- 查询表tb1从2018-06-01 08:00:00.000 到2018-06-02 08:00:00.000时间范围，并且col3的字符串是'nny'结尾的记录，结果按照时间戳降序： ```mysql SELECT * FROM tb1 WHERE ts > '2018-06-01 08:00:00.000' AND ts <= '2018-06-02 08:00:00.000' AND col3 LIKE '%nny' ORDER BY ts DESC; ``` -- 查询col1与col2的和，并取名complex, 时间大于2018-06-01 08:00:00.000, col2大于1.2，结果输出仅仅10条记录，从第5条开始 +- 查询col1与col2的和，并取名complex, 时间大于2018-06-01 08:00:00.000, col2大于1.2，结果输出仅仅10条记录，从第5条开始： ```mysql SELECT (col1 + col2) AS 'complex' FROM tb1 WHERE ts > '2018-06-01 08:00:00.000' AND col2 > 1.2 LIMIT 10 OFFSET 5; ``` -- 查询过去10分钟的记录，col2的值大于3.14，并且将结果输出到文件 `/home/testoutpu.csv`. +- 查询过去10分钟的记录，col2的值大于3.14，并且将结果输出到文件 `/home/testoutpu.csv`： ```mysql SELECT COUNT(*) FROM tb1 WHERE ts >= NOW - 10m AND col2 > 3.14 >> /home/testoutpu.csv; ``` -## SQL 函数 + +## SQL 函数 ### 聚合函数 @@ -741,7 +760,7 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 说明： - 1）可以使用星号\*来替代具体的字段，使用星号(\*)返回全部记录数量。 + 1）可以使用星号(\*)来替代具体的字段，使用星号(\*)返回全部记录数量。 2）针对同一表的（不包含NULL值）字段查询结果均相同。 @@ -1012,7 +1031,9 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 1）*k*值取值范围1≤*k*≤100； - 2）系统同时返回该记录关联的时间戳列。 + 2）系统同时返回该记录关联的时间戳列； + + 3）限制：TOP函数不支持FILL子句。 示例： ```mysql @@ -1048,7 +1069,9 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 1）*k*值取值范围1≤*k*≤100； - 2）系统同时返回该记录关联的时间戳列。 + 2）系统同时返回该记录关联的时间戳列； + + 3）限制：BOTTOM函数不支持FILL子句。 示例： ```mysql @@ -1124,7 +1147,9 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 适用于：**表、超级表**。 - 说明：与last函数不同，last_row不支持时间范围限制，强制返回最后一条记录。 + 说明：与LAST函数不同，LAST_ROW不支持时间范围限制，强制返回最后一条记录。 + + 限制：LAST_ROW()不能与INTERVAL一起使用。 示例： ```mysql @@ -1233,40 +1258,40 @@ SELECT function_list FROM tb_name [WHERE where_condition] INTERVAL (interval [, offset]) [SLIDING sliding] - [FILL ({NONE | VALUE | PREV | NULL | LINEAR})] + [FILL ({NONE | VALUE | PREV | NULL | LINEAR | NEXT})] SELECT function_list FROM stb_name [WHERE where_condition] INTERVAL (interval [, offset]) [SLIDING sliding] - [FILL ({ VALUE | PREV | NULL | LINEAR})] + [FILL ({ VALUE | PREV | NULL | LINEAR | NEXT})] [GROUP BY tags] ``` - 聚合时间段的长度由关键词INTERVAL指定，最短时间间隔10毫秒（10a），并且支持偏移（偏移必须小于间隔）。聚合查询中，能够同时执行的聚合和选择函数仅限于单个输出的函数：count、avg、sum 、stddev、leastsquares、percentile、min、max、first、last，不能使用具有多行输出结果的函数（例如：top、bottom、diff以及四则运算）。 -- WHERE语句可以指定查询的起止时间和其他过滤条件 -- SLIDING语句用于指定聚合时间段的前向增量 +- WHERE语句可以指定查询的起止时间和其他过滤条件。 +- SLIDING语句用于指定聚合时间段的前向增量。 - FILL语句指定某一时间区间数据缺失的情况下的填充模式。填充模式包括以下几种： - * 不进行填充：NONE(默认填充模式)。 - * VALUE填充：固定值填充，此时需要指定填充的数值。例如：fill(value, 1.23)。 - * NULL填充：使用NULL填充数据。例如：fill(null)。 - * PREV填充：使用前一个非NULL值填充数据。例如：fill(prev)。 + 1. 不进行填充：NONE(默认填充模式)。 + 2. VALUE填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。 + 3. NULL填充：使用NULL填充数据。例如：FILL(NULL)。 + 4. PREV填充：使用前一个非NULL值填充数据。例如：FILL(PREV)。 + 5. NEXT填充：使用下一个非NULL值填充数据。例如：FILL(NEXT)。 说明： 1. 使用FILL语句的时候可能生成大量的填充输出，务必指定查询的时间区间。针对每次查询，系统可返回不超过1千万条具有插值的结果。 2. 在时间维度聚合中，返回的结果中时间序列严格单调递增。 - 3. 如果查询对象是超级表，则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用group by语句，则返回的结果按照时间序列严格单调递增；如果查询中使用了group by语句分组，则返回结果中每个group内不按照时间序列严格单调递增。 + 3. 如果查询对象是超级表，则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用GROUP BY语句，则返回的结果按照时间序列严格单调递增；如果查询中使用了GROUP BY语句分组，则返回结果中每个GROUP内不按照时间序列严格单调递增。 时间聚合也常被用于连续查询场景，可以参考文档 [连续查询(Continuous Query)](https://www.taosdata.com/cn/documentation/advanced-features#continuous-query)。 -**示例:** 智能电表的建表语句如下： +**示例**： 智能电表的建表语句如下： ```mysql CREATE TABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT); ``` -针对智能电表采集的数据，以10分钟为一个阶段，计算过去24小时的电流数据的平均值、最大值、电流的中位数、以及随着时间变化的电流走势拟合直线。如果没有计算值，用前一个非NULL值填充。 -使用的查询语句如下： +针对智能电表采集的数据，以10分钟为一个阶段，计算过去24小时的电流数据的平均值、最大值、电流的中位数、以及随着时间变化的电流走势拟合直线。如果没有计算值，用前一个非NULL值填充。使用的查询语句如下： ```mysql SELECT AVG(current), MAX(current), LEASTSQUARES(current, start_val, step_val), PERCENTILE(current, 50) FROM meters @@ -1287,15 +1312,15 @@ SELECT AVG(current), MAX(current), LEASTSQUARES(current, start_val, step_val), P ## TAOS SQL其他约定 -**group by的限制** +**GROUP BY的限制** -TAOS SQL支持对标签、tbname进行group by操作，也支持普通列进行group by，前提是：仅限一列且该列的唯一值小于10万个。 +TAOS SQL支持对标签、TBNAME进行GROUP BY操作，也支持普通列进行GROUP BY，前提是：仅限一列且该列的唯一值小于10万个。 -**join操作的限制** +**JOIN操作的限制** TAOS SQL支持表之间按主键时间戳来join两张表的列，暂不支持两个表之间聚合后的四则运算。 -**is not null与不为空的表达式适用范围** +**IS NOT NULL与不为空的表达式适用范围** -is not null支持所有类型的列。不为空的表达式为 <>""，仅对非数值类型的列适用。 +IS NOT NULL支持所有类型的列。不为空的表达式为 <>""，仅对非数值类型的列适用。 diff --git a/documentation20/cn/13.faq/docs.md b/documentation20/cn/13.faq/docs.md index e561f91c94..8d18ae269d 100644 --- a/documentation20/cn/13.faq/docs.md +++ b/documentation20/cn/13.faq/docs.md @@ -26,17 +26,17 @@ ## 2. Windows平台下JDBCDriver找不到动态链接库，怎么办？ -请看为此问题撰写的[技术博客](https://www.taosdata.com/blog/2019/12/03/jdbcdriver找不到动态链接库/) +请看为此问题撰写的[技术博客](https://www.taosdata.com/blog/2019/12/03/950.html) ## 3. 创建数据表时提示more dnodes are needed -请看为此问题撰写的[技术博客](https://www.taosdata.com/blog/2019/12/03/创建数据表时提示more-dnodes-are-needed/) +请看为此问题撰写的[技术博客](https://www.taosdata.com/blog/2019/12/03/965.html) ## 4. 如何让TDengine crash时生成core文件？ -请看为此问题撰写的[技术博客](https://www.taosdata.com/blog/2019/12/06/tdengine-crash时生成core文件的方法/) +请看为此问题撰写的[技术博客](https://www.taosdata.com/blog/2019/12/06/974.html) -## 5. 遇到错误"Unable to establish connection", 我怎么办？ +## 5. 遇到错误“Unable to establish connection”, 我怎么办？ 客户端遇到连接故障，请按照下面的步骤进行检查： @@ -51,13 +51,13 @@ 4. 确认客户端连接时指定了正确的服务器FQDN (Fully Qualified Domain Name(可在服务器上执行Linux命令hostname -f获得)）,FQDN配置参考：[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)。 -5. ping服务器FQDN，如果没有反应，请检查你的网络，DNS设置，或客户端所在计算机的系统hosts文件 +5. ping服务器FQDN，如果没有反应，请检查你的网络，DNS设置，或客户端所在计算机的系统hosts文件。如果部署的是TDengine集群，客户端需要能ping通所有集群节点的FQDN。 6. 检查防火墙设置（Ubuntu 使用 ufw status，CentOS 使用 firewall-cmd --list-port），确认TCP/UDP 端口6030-6042 是打开的 7. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/taos/driver*里, 并且*/usr/local/taos/driver*在系统库函数搜索路径*LD_LIBRARY_PATH*里 -8. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*C:\TDengine\driver\taos.dll*在你的系统库函数搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*) +8. 对于Windows上的JDBC, ODBC, Python, Go等连接，确保*C:\TDengine\driver\taos.dll*在你的系统库函数搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*) 9. 如果仍不能排除连接故障 @@ -70,7 +70,8 @@ 10. 也可以使用taos程序内嵌的网络连通检测功能，来验证服务器和客户端之间指定的端口连接是否通畅（包括TCP和UDP）：[TDengine 内嵌网络检测工具使用指南](https://www.taosdata.com/blog/2020/09/08/1816.html)。 -## 6. 遇到错误“Unexpected generic error in RPC”或者"TDengine Error: Unable to resolve FQDN"， 我怎么办？ +## 6. 遇到错误“Unexpected generic error in RPC”或者“Unable to resolve FQDN”，我怎么办？ + 产生这个错误，是由于客户端或数据节点无法解析FQDN(Fully Qualified Domain Name)导致。对于TAOS Shell或客户端应用，请做如下检查： 1. 请检查连接的服务器的FQDN是否正确,FQDN配置参考：[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)。 @@ -102,7 +103,7 @@ TDengine 目前尚不支持删除功能，未来根据用户需求可能会支 批量插入。每条写入语句可以一张表同时插入多条记录，也可以同时插入多张表的多条记录。 -## 12. windows系统下插入的nchar类数据中的汉字被解析成了乱码如何解决？ +## 12. Windows系统下插入的nchar类数据中的汉字被解析成了乱码如何解决？ Windows下插入nchar类的数据中如果有中文，请先确认系统的地区设置成了中国（在Control Panel里可以设置），这时cmd中的`taos`客户端应该已经可以正常工作了；如果是在IDE里开发Java应用，比如Eclipse， Intellij，请确认IDE里的文件编码为GBK（这是Java默认的编码类型），然后在生成Connection时，初始化客户端的配置，具体语句如下： ```JAVA @@ -115,15 +116,15 @@ Connection = DriverManager.getConnection(url, properties); ## 13.JDBC报错： the excuted SQL is not a DML or a DDL？ 请更新至最新的JDBC驱动 -```JAVA +```xml com.taosdata.jdbc taos-jdbcdriver - 2.0.4 + 2.0.27 ``` -## 14. taos connect failed, reason: invalid timestamp +## 14. taos connect failed, reason: invalid timestamp 常见原因是服务器和客户端时间没有校准，可以通过和时间服务器同步的方式（Linux 下使用 ntpdate 命令，Windows 在系统时间设置中选择自动同步）校准。 @@ -157,7 +158,8 @@ ALTER LOCAL RESETLOG; 其含义是，清空本机所有由客户端生成的日志文件。 -## 18. 时间戳的时区信息是怎样处理的？ + +## 18. 时间戳的时区信息是怎样处理的？ TDengine 中时间戳的时区总是由客户端进行处理，而与服务端无关。具体来说，客户端会对 SQL 语句中的时间戳进行时区转换，转为 UTC 时区（即 Unix 时间戳——Unix Timestamp）再交由服务端进行写入和查询；在读取数据时，服务端也是采用 UTC 时区提供原始数据，客户端收到后再根据本地设置，把时间戳转换为本地系统所要求的时区进行显示。 @@ -167,12 +169,13 @@ TDengine 中时间戳的时区总是由客户端进行处理，而与服务端 3. 如果在 C/C++/Java/Python 等各种编程语言的 Connector Driver 中，在建立数据库连接时显式指定了 timezone，那么会以这个指定的时区设置为准。例如 Java Connector 的 JDBC URL 中就有 timezone 参数。 4. 在书写 SQL 语句时，也可以直接使用 Unix 时间戳（例如 `1554984068000`）或带有时区的时间戳字符串，也即以 RFC 3339 格式（例如 `2013-04-12T15:52:01.123+08:00`）或 ISO-8601 格式（例如 `2013-04-12T15:52:01.123+0800`）来书写时间戳，此时这些时间戳的取值将不再受其他时区设置的影响。 -## 19. TDengine 都会用到哪些网络端口？ + +## 19. TDengine 都会用到哪些网络端口？ 在 TDengine 2.0 版本中，会用到以下这些网络端口（以默认端口 6030 为前提进行说明，如果修改了配置文件中的设置，那么这里列举的端口都会出现变化），管理员可以参考这里的信息调整防火墙设置： | 协议 | 默认端口 | 用途说明 | 修改方法 | -| --- | --------- | ------------------------------- | ------------------------------ | +| :--- | :-------- | :---------------------------------- | :------------------------------- | | TCP | 6030 | 客户端与服务端之间通讯。 | 由配置文件设置 serverPort 决定。 | | TCP | 6035 | 多节点集群的节点间通讯。 | 随 serverPort 端口变化。 | | TCP | 6040 | 多节点集群的节点间数据同步。 | 随 serverPort 端口变化。 | diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index a593c7ca16..28be037e6c 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -58,7 +58,12 @@ cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_pat cp ${compile_dir}/../src/inc/taos.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../src/inc/taoserror.h ${pkg_dir}${install_home_path}/include cp -r ${top_dir}/tests/examples/* ${pkg_dir}${install_home_path}/examples -cp -r ${top_dir}/src/connector/grafanaplugin/dist ${pkg_dir}${install_home_path}/connector/grafanaplugin +if [ -d "${top_dir}/src/connector/grafanaplugin/dist" ]; then + cp -r ${top_dir}/src/connector/grafanaplugin/dist ${pkg_dir}${install_home_path}/connector/grafanaplugin +else + echo "grafanaplugin bundled directory not found!" + exit 1 +fi cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector cp -r ${top_dir}/src/connector/go ${pkg_dir}${install_home_path}/connector cp -r ${top_dir}/src/connector/nodejs ${pkg_dir}${install_home_path}/connector diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 73efe2e1ab..9910e20bfe 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -66,7 +66,12 @@ cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include cp %{_compiledir}/../src/inc/taoserror.h %{buildroot}%{homepath}/include -cp -r %{_compiledir}/../src/connector/grafanaplugin/dist %{buildroot}%{homepath}/connector/grafanaplugin +if [ -d %{_compiledir}/../src/connector/grafanaplugin/dist ]; then + cp -r %{_compiledir}/../src/connector/grafanaplugin/dist %{buildroot}%{homepath}/connector/grafanaplugin +else + echo grafanaplugin bundled directory not found! + exit 1 +fi cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector cp -r %{_compiledir}/../src/connector/nodejs %{buildroot}%{homepath}/connector diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 5f1739d6e0..d6ace0a063 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -243,9 +243,17 @@ function install_data() { } function install_connector() { - ${csudo} cp -rf ${source_dir}/src/connector/grafanaplugin/dist ${install_main_dir}/connector/grafanaplugin + if [ -d "${source_dir}/src/connector/grafanaplugin/dist" ]; then + ${csudo} cp -rf ${source_dir}/src/connector/grafanaplugin/dist ${install_main_dir}/connector/grafanaplugin + else + echo "WARNING: grafanaplugin bundled dir not found, please check if want to use it!" + fi + if find ${source_dir}/src/connector/go -mindepth 1 -maxdepth 1 | read; then + ${csudo} cp -r ${source_dir}/src/connector/go ${install_main_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector - ${csudo} cp -rf ${source_dir}/src/connector/go ${install_main_dir}/connector ${csudo} cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &> /dev/null && ${csudo} chmod 777 ${install_main_dir}/connector/*.jar || echo &> /dev/null } diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 7418bcc312..d0eeffc86a 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -117,9 +117,17 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: fi - cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin - cp -r ${connector_dir}/python ${install_dir}/connector/ - cp -r ${connector_dir}/go ${install_dir}/connector + if [ -d "${connector_dir}/grafanaplugin/dist" ]; then + cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin + else + echo "WARNING: grafanaplugin bundled dir not found, please check if want to use it!" + fi + if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then + cp -r ${connector_dir}/go ${install_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi + cp -r ${connector_dir}/python ${install_dir}/connector cp -r ${connector_dir}/nodejs ${install_dir}/connector fi # Copy release note diff --git a/packaging/tools/makeclient_power.sh b/packaging/tools/makeclient_power.sh index 911cc574a5..8241319e4f 100755 --- a/packaging/tools/makeclient_power.sh +++ b/packaging/tools/makeclient_power.sh @@ -144,9 +144,17 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: fi - cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin - cp -r ${connector_dir}/python ${install_dir}/connector/ - cp -r ${connector_dir}/go ${install_dir}/connector + if [ -d "${connector_dir}/grafanaplugin/dist" ]; then + cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin + else + echo "WARNING: grafanaplugin bunlded dir not found, please check if want to use it!" + fi + if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then + cp -r ${connector_dir}/go ${install_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi + cp -r ${connector_dir}/python ${install_dir}/connector sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/taos/cinterface.py diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 343e2a4224..e4d2d71b01 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -114,6 +114,25 @@ mkdir -p ${install_dir}/examples examples_dir="${top_dir}/tests/examples" cp -r ${examples_dir}/c ${install_dir}/examples if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then + if [ -d ${examples_dir}/JDBC/connectionPools/target ]; then + rm -rf ${examples_dir}/JDBC/connectionPools/target + fi + if [ -d ${examples_dir}/JDBC/JDBCDemo/target ]; then + rm -rf ${examples_dir}/JDBC/JDBCDemo/target + fi + if [ -d ${examples_dir}/JDBC/mybatisplus-demo/target ]; then + rm -rf ${examples_dir}/JDBC/mybatisplus-demo/target + fi + if [ -d ${examples_dir}/JDBC/springbootdemo/target ]; then + rm -rf ${examples_dir}/JDBC/springbootdemo/target + fi + if [ -d ${examples_dir}/JDBC/SpringJdbcTemplate/target ]; then + rm -rf ${examples_dir}/JDBC/SpringJdbcTemplate/target + fi + if [ -d ${examples_dir}/JDBC/taosdemo/target ]; then + rm -rf ${examples_dir}/JDBC/taosdemo/target + fi + cp -r ${examples_dir}/JDBC ${install_dir}/examples cp -r ${examples_dir}/matlab ${install_dir}/examples cp -r ${examples_dir}/python ${install_dir}/examples @@ -131,9 +150,17 @@ connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: - cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin - cp -r ${connector_dir}/python ${install_dir}/connector/ - cp -r ${connector_dir}/go ${install_dir}/connector + if [ -d "${connector_dir}/grafanaplugin/dist" ]; then + cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin + else + echo "WARNING: grafanaplugin bundled dir not found, please check if you want to use it!" + fi + if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then + cp -r ${connector_dir}/go ${install_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi + cp -r ${connector_dir}/python ${install_dir}/connector cp -r ${connector_dir}/nodejs ${install_dir}/connector fi # Copy release note diff --git a/packaging/tools/makepkg_power.sh b/packaging/tools/makepkg_power.sh index 8ea7f363c8..633a135c14 100755 --- a/packaging/tools/makepkg_power.sh +++ b/packaging/tools/makepkg_power.sh @@ -166,9 +166,18 @@ connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: - cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin + + if [ -d "${connector_dir}/grafanaplugin/dist" ]; then + cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin + else + echo "WARNING: grafanaplugin bundled dir not found, please check if want to use it!" + fi + if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then + cp -r ${connector_dir}/go ${install_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi cp -r ${connector_dir}/python ${install_dir}/connector/ - cp -r ${connector_dir}/go ${install_dir}/connector sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/taos/cinterface.py diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 43006928a6..a63225ab32 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,7 +1,6 @@ name: tdengine base: core18 - -version: '2.1.0.0' +version: '2.1.1.0' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -73,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.1.0.0 + - usr/lib/libtaos.so.2.1.1.0 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so diff --git a/src/client/inc/tscLocalMerge.h b/src/client/inc/tscLocalMerge.h index dddc767534..3c0bde0000 100644 --- a/src/client/inc/tscLocalMerge.h +++ b/src/client/inc/tscLocalMerge.h @@ -39,38 +39,29 @@ typedef struct SLocalDataSource { } SLocalDataSource; typedef struct SLocalMerger { - SLocalDataSource ** pLocalDataSrc; + SLocalDataSource **pLocalDataSrc; int32_t numOfBuffer; int32_t numOfCompleted; int32_t numOfVnode; - SLoserTreeInfo * pLoserTree; - tFilePage * pResultBuf; - int32_t nResultBufSize; - tFilePage * pTempBuffer; - struct SQLFunctionCtx *pCtx; - int32_t rowSize; // size of each intermediate result. - tOrderDescriptor * pDesc; - SColumnModel * resColModel; - SColumnModel* finalModel; - tExtMemBuffer ** pExtMemBuffer; // disk-based buffer + SLoserTreeInfo *pLoserTree; + int32_t rowSize; // size of each intermediate result. + tOrderDescriptor *pDesc; + tExtMemBuffer **pExtMemBuffer; // disk-based buffer + char *buf; // temp buffer } SLocalMerger; typedef struct SRetrieveSupport { tExtMemBuffer ** pExtMemBuffer; // for build loser tree tOrderDescriptor *pOrderDescriptor; - SColumnModel* pFinalColModel; // colModel for final result - SColumnModel* pFFColModel; int32_t subqueryIndex; // index of current vnode in vnode list SSqlObj * pParentSql; tFilePage * localBuffer; // temp buffer, there is a buffer for each vnode to uint32_t numOfRetry; // record the number of retry times } SRetrieveSupport; -int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOrderDescriptor **pDesc, - SColumnModel **pFinalModel, SColumnModel** pFFModel, uint32_t nBufferSize); +int32_t tscLocalReducerEnvCreate(SQueryInfo* pQueryInfo, tExtMemBuffer ***pMemBuffer, int32_t numOfSub, tOrderDescriptor **pDesc, uint32_t nBufferSize, int64_t id); -void tscLocalReducerEnvDestroy(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDesc, SColumnModel *pFinalModel, SColumnModel* pFFModel, - int32_t numOfVnodes); +void tscLocalReducerEnvDestroy(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDesc, int32_t numOfVnodes); int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePage *pPage, void *data, int32_t numOfRows, int32_t orderType); @@ -80,12 +71,10 @@ int32_t tscFlushTmpBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tF /* * create local reducer to launch the second-stage reduce process at client site */ -void tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc, - SColumnModel *finalModel, SColumnModel *pFFModel, SSqlObj* pSql); +int32_t tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc, + SQueryInfo *pQueryInfo, SLocalMerger **pMerger, int64_t id); -void tscDestroyLocalMerger(SSqlObj *pSql); - -//int32_t tscDoLocalMerge(SSqlObj *pSql); +void tscDestroyLocalMerger(SLocalMerger* pLocalMerger); #ifdef __cplusplus } diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 851adf94a4..5bc87a14e9 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -20,6 +20,7 @@ extern "C" { #endif +#include "tsched.h" #include "exception.h" #include "os.h" #include "qExtbuffer.h" @@ -36,6 +37,9 @@ extern "C" { #define UTIL_TABLE_IS_NORMAL_TABLE(metaInfo)\ (!(UTIL_TABLE_IS_SUPER_TABLE(metaInfo) || UTIL_TABLE_IS_CHILD_TABLE(metaInfo))) +#define UTIL_TABLE_IS_TMP_TABLE(metaInfo) \ + (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_TEMP_TABLE)) + #pragma pack(push,1) // this struct is transfered as binary, padding two bytes to avoid // an 'uid' whose low bytes is 0xff being recoginized as NULL, @@ -59,7 +63,7 @@ typedef struct SJoinSupporter { SArray* exprList; SFieldInfo fieldsInfo; STagCond tagCond; - SSqlGroupbyExpr groupInfo; // group by info + SGroupbyExpr groupInfo; // group by info struct STSBuf* pTSBuf; // the TSBuf struct that holds the compressed timestamp array FILE* f; // temporary file in order to create TSBuf char path[PATH_MAX]; // temporary file path, todo dynamic allocate memory @@ -90,22 +94,14 @@ typedef struct SVgroupTableInfo { SArray *itemList; // SArray } SVgroupTableInfo; -static FORCE_INLINE SQueryInfo* tscGetQueryInfo(SSqlCmd* pCmd, int32_t subClauseIndex) { - assert(pCmd != NULL && subClauseIndex >= 0); - if (pCmd->pQueryInfo == NULL || subClauseIndex >= pCmd->numOfClause) { - return NULL; - } - - return pCmd->pQueryInfo[subClauseIndex]; -} - -SQueryInfo* tscGetActiveQueryInfo(SSqlCmd* pCmd); +int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len); int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks); void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta); void tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf); void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo); +void doRetrieveSubqueryData(SSchedMsg *pMsg); SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes, uint32_t offset); @@ -128,7 +124,7 @@ int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, i */ bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo); bool tscIsTWAQuery(SQueryInfo* pQueryInfo); -bool tscIsSecondStageQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); +bool tsIsArithmeticQueryOnAggResult(SQueryInfo* pQueryInfo); bool tscGroupbyColumn(SQueryInfo* pQueryInfo); bool tscIsTopBotQuery(SQueryInfo* pQueryInfo); bool hasTagValOutput(SQueryInfo* pQueryInfo); @@ -137,13 +133,14 @@ bool isStabledev(SQueryInfo* pQueryInfo); bool isTsCompQuery(SQueryInfo* pQueryInfo); bool isSimpleAggregate(SQueryInfo* pQueryInfo); bool isBlockDistQuery(SQueryInfo* pQueryInfo); -int32_t tscGetTopbotQueryParam(SQueryInfo* pQueryInfo); +bool isSimpleAggregateRv(SQueryInfo* pQueryInfo); bool tscNonOrderedProjectionQueryOnSTable(SQueryInfo *pQueryInfo, int32_t tableIndex); bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex); bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex); bool tscIsProjectionQuery(SQueryInfo* pQueryInfo); +bool tscHasColumnFilter(SQueryInfo* pQueryInfo); bool tscIsTwoStageSTableQuery(SSqlCmd *pCmd, SQueryInfo* pQueryInfo, int32_t tableIndex); bool tscQueryTags(SQueryInfo* pQueryInfo); @@ -151,9 +148,9 @@ bool tscMultiRoundQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); bool tscQueryBlockInfo(SQueryInfo* pQueryInfo); SExprInfo* tscAddFuncInSelectClause(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, - SColumnIndex* pIndex, SSchema* pColSchema, int16_t colType); + SColumnIndex* pIndex, SSchema* pColSchema, int16_t colType, int16_t colId); -int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableName, SSqlObj* pSql); +int32_t tscSetTableFullName(SName* pName, SStrToken* pzTableName, SSqlObj* pSql); void tscClearInterpInfo(SQueryInfo* pQueryInfo); bool tscIsInsertData(char* sqlstr); @@ -172,36 +169,49 @@ void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo); int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index); void tscFieldInfoClear(SFieldInfo* pFieldInfo); +void tscFieldInfoCopy(SFieldInfo* pFieldInfo, const SFieldInfo* pSrc, const SArray* pExprList); static FORCE_INLINE int32_t tscNumOfFields(SQueryInfo* pQueryInfo) { return pQueryInfo->fieldsInfo.numOfOutput; } int32_t tscFieldInfoCompare(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2, int32_t *diffSize); -int32_t tscFieldInfoSetSize(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2); +void tscInsertPrimaryTsSourceColumn(SQueryInfo* pQueryInfo, uint64_t uid); +int32_t tscFieldInfoSetSize(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2); void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes); int32_t tscGetResRowLength(SArray* pExprList); -SExprInfo* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, +SExprInfo* tscExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, int16_t size, int16_t resColId, int16_t interSize, bool isTagCol); -SExprInfo* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, +SExprInfo* tscExprCreate(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, + int16_t size, int16_t resColId, int16_t interSize, int32_t colType); + +void tscExprAddParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes); + +SExprInfo* tscExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, int16_t size, int16_t resColId, int16_t interSize, bool isTagCol); -SExprInfo* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type, +SExprInfo* tscExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type, int16_t size); -size_t tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo); -void tscInsertPrimaryTsSourceColumn(SQueryInfo* pQueryInfo, uint64_t uid); -SExprInfo* tscSqlExprGet(SQueryInfo* pQueryInfo, int32_t index); -int32_t tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy); -void tscSqlExprAssign(SExprInfo* dst, const SExprInfo* src); -void tscSqlExprInfoDestroy(SArray* pExprInfo); +size_t tscNumOfExprs(SQueryInfo* pQueryInfo); +SExprInfo *tscExprGet(SQueryInfo* pQueryInfo, int32_t index); +int32_t tscExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy); +int32_t tscExprCopyAll(SArray* dst, const SArray* src, bool deepcopy); +void tscExprAssign(SExprInfo* dst, const SExprInfo* src); +void tscExprDestroy(SArray* pExprInfo); + +int32_t createProjectionExpr(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SExprInfo*** pExpr, int32_t* num); + +void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, bool removeMeta); SColumn* tscColumnClone(const SColumn* src); bool tscColumnExists(SArray* pColumnList, int32_t columnIndex, uint64_t uid); SColumn* tscColumnListInsert(SArray* pColumnList, int32_t columnIndex, uint64_t uid, SSchema* pSchema); void tscColumnListDestroy(SArray* pColList); +void tscColumnListCopy(SArray* dst, const SArray* src, uint64_t tableUid); +void tscColumnListCopyAll(SArray* dst, const SArray* src); void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo); @@ -223,14 +233,14 @@ void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo); bool tscShouldBeFreed(SSqlObj* pSql); -STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd *pCmd, int32_t subClauseIndex, int32_t tableIndex); +STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd *pCmd, int32_t tableIndex); STableMetaInfo* tscGetMetaInfo(SQueryInfo *pQueryInfo, int32_t tableIndex); void tscInitQueryInfo(SQueryInfo* pQueryInfo); void tscClearSubqueryInfo(SSqlCmd* pCmd); int32_t tscAddQueryInfo(SSqlCmd *pCmd); -SQueryInfo *tscGetQueryInfo(SSqlCmd* pCmd, int32_t subClauseIndex); -SQueryInfo *tscGetQueryInfoS(SSqlCmd *pCmd, int32_t subClauseIndex); +SQueryInfo *tscGetQueryInfo(SSqlCmd* pCmd); +SQueryInfo *tscGetQueryInfoS(SSqlCmd *pCmd); void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo); @@ -244,13 +254,12 @@ SArray* tscVgroupTableInfoDup(SArray* pVgroupTables); void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index); void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo); -int tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex); +int tscGetSTableVgroupInfo(SSqlObj* pSql, SQueryInfo* pQueryInfo); int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo); int tscGetTableMetaEx(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, bool createIfNotExists); int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo); void tscResetForNextRetrieve(SSqlRes* pRes); -void tscDoQuery(SSqlObj* pSql); void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo); void doExecuteQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo); @@ -281,7 +290,7 @@ void registerSqlObj(SSqlObj* pSql); SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t fp, void* param, int32_t cmd, SSqlObj* pPrevSql); void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClauseIndex, int32_t tableIndex); -void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex); +void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex, SSqlCmd* pCmd); int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid); int16_t tscGetTagColIndexById(STableMeta* pTableMeta, int16_t colId); @@ -297,6 +306,11 @@ void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp); void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows); void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp); int tscSetMgmtEpSetFromCfg(const char *first, const char *second, SRpcCorEpSet *corEpSet); +int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, __async_cb_func_t fp); + +int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t length, SArray* pNameArray); + +bool subAndCheckDone(SSqlObj *pSql, SSqlObj *pParentSql, int idx); bool tscSetSqlOwner(SSqlObj* pSql); void tscClearSqlOwner(SSqlObj* pSql); @@ -309,12 +323,14 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild); uint32_t tscGetTableMetaSize(STableMeta* pTableMeta); CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta); uint32_t tscGetTableMetaMaxSize(); -int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name, void* buf); +int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, void* buf); STableMeta* tscTableMetaDup(STableMeta* pTableMeta); +SVgroupsInfo* tscVgroupsInfoDup(SVgroupsInfo* pVgroupsInfo); + int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAttr, void* addr); void tsCreateSQLFunctionCtx(SQueryInfo* pQueryInfo, SQLFunctionCtx* pCtx, SSchema* pSchema); -void* createQueryInfoFromQueryNode(SQueryInfo* pQueryInfo, SExprInfo* pExprs, STableGroupInfo* pTableGroupInfo, SOperatorInfo* pOperator, char* sql, void* addr, int32_t stage); +void* createQInfoFromQueryNode(SQueryInfo* pQueryInfo, SExprInfo* pExprs, STableGroupInfo* pTableGroupInfo, SOperatorInfo* pOperator, char* sql, void* addr, int32_t stage); void* malloc_throw(size_t size); void* calloc_throw(size_t nmemb, size_t size); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index bc301b2b42..948ed11343 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -42,12 +42,6 @@ extern "C" { struct SSqlInfo; struct SLocalMerger; -// data source from sql string or from file -enum { - DATA_FROM_SQL_STRING = 1, - DATA_FROM_DATA_FILE = 2, -}; - typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int32_t numOfRows); typedef struct STableComInfo { @@ -68,14 +62,16 @@ typedef struct CChildTableMeta { int32_t vgId; STableId id; uint8_t tableType; - char sTableName[TSDB_TABLE_FNAME_LEN]; //super table name, not full name + char sTableName[TSDB_TABLE_FNAME_LEN]; // TODO: refactor super table name, not full name + uint64_t suid; // super table id } CChildTableMeta; typedef struct STableMeta { int32_t vgId; STableId id; uint8_t tableType; - char sTableName[TSDB_TABLE_FNAME_LEN]; + char sTableName[TSDB_TABLE_FNAME_LEN]; // super table name + uint64_t suid; // super table id int16_t sversion; int16_t tversion; STableComInfo tableInfo; @@ -83,10 +79,10 @@ typedef struct STableMeta { } STableMeta; typedef struct STableMetaInfo { - STableMeta *pTableMeta; // table meta, cached in client side and acquired by name + STableMeta *pTableMeta; // table meta, cached in client side and acquired by name uint32_t tableMetaSize; - SVgroupsInfo *vgroupList; - SArray *pVgroupTables; // SArray + SVgroupsInfo *vgroupList; + SArray *pVgroupTables; // SArray /* * 1. keep the vgroup index during the multi-vnode super table projection query @@ -135,8 +131,8 @@ typedef struct SJoinNode { } SJoinNode; typedef struct SJoinInfo { - bool hasJoin; - SJoinNode* joinTables[TSDB_MAX_JOIN_TABLE_NUM]; + bool hasJoin; + SJoinNode *joinTables[TSDB_MAX_JOIN_TABLE_NUM]; } SJoinInfo; typedef struct STagCond { @@ -203,10 +199,11 @@ typedef struct SQueryInfo { SInterval interval; // tumble time window SSessionWindow sessionWindow; // session time window - SSqlGroupbyExpr groupbyExpr; // groupby tags info + SGroupbyExpr groupbyExpr; // groupby tags info SArray * colList; // SArray SFieldInfo fieldsInfo; SArray * exprList; // SArray + SArray * exprList1; // final exprlist in case of arithmetic expression exists SLimitVal limit; SLimitVal slimit; STagCond tagCond; @@ -229,35 +226,55 @@ typedef struct SQueryInfo { int32_t round; // 0/1/.... int32_t bufLen; char* buf; - SQInfo* pQInfo; // global merge operator - SArray* pDSOperator; // data source operator - SArray* pPhyOperator; // physical query execution plan + SQInfo* pQInfo; // global merge operator SQueryAttr* pQueryAttr; // query object struct SQueryInfo *sibling; // sibling SArray *pUpstream; // SArray struct SQueryInfo *pDownstream; int32_t havingFieldNum; + bool stableQuery; + bool groupbyColumn; + bool simpleAgg; + bool arithmeticOnAgg; + bool projectionQuery; + bool hasFilter; + bool onlyTagQuery; bool globalMerge; // need global merge - bool arithmCalOnAgg; // arithmetic calculation on aggregate result. - SArray *pUdfInfo; // user defined function information SArray + SArray *pUdfInfo; // user defined function information SArray } SQueryInfo; +typedef struct { + STableMeta *pTableMeta; + SVgroupsInfo *pVgroupInfo; +} STableMetaVgroupInfo; + +typedef struct SInsertStatementParam { + SName **pTableNameList; // all involved tableMeta list of current insert sql statement. + int32_t numOfTables; // number of tables in table name list + SHashObj *pTableBlockHashList; // data block for each table + SArray *pDataBlocks; // SArray. Merged submit block for each vgroup + int8_t schemaAttached; // denote if submit block is built with table schema or not + STagData tagData; // NOTE: pTagData->data is used as a variant length array + + char msg[512]; // error message + char *sql; // current sql statement position + uint32_t insertType; // insert data from [file|sql statement| bound statement] +} SInsertStatementParam; + +// TODO extract sql parser supporter typedef struct { int command; uint8_t msgType; + SInsertStatementParam insertParam; char reserve1[3]; // fix bus error on arm32 - bool autoCreated; // create table if it is not existed during retrieve table meta in mnode bool subCmd; union { int32_t count; - int32_t numOfTablesInSubmit; }; - uint32_t insertType; // TODO remove it char * curSql; // current sql, resume position of sql after parsing paused - int8_t parseFinished; char reserve2[3]; // fix bus error on arm32 int16_t numOfCols; @@ -266,25 +283,13 @@ typedef struct { char * payload; int32_t payloadLen; - SQueryInfo **pQueryInfo; - int32_t numOfClause; - int32_t clauseIndex; // index of multiple subclause query - SQueryInfo *active; // current active query info - - int32_t batchSize; // for parameter ('?') binding and batch processing + SHashObj *pTableMetaMap; // local buffer to keep the queried table meta, before validating the AST + SQueryInfo *pQueryInfo; + SQueryInfo *active; // current active query info + int32_t batchSize; // for parameter ('?') binding and batch processing int32_t numOfParams; - - int8_t dataSourceType; // load data from file or not - char reserve4[3]; // fix bus error on arm32 - int8_t submitSchema; // submit block is built with table schema - char reserve5[3]; // fix bus error on arm32 STagData tagData; // NOTE: pTagData->data is used as a variant length array - - SName **pTableNameList; // all involved tableMeta list of current insert sql statement. - int32_t numOfTables; - - SHashObj *pTableBlockHashList; // data block for each table - SArray *pDataBlocks; // SArray. Merged submit block for each vgroup + int32_t resColumnId; } SSqlCmd; typedef struct SResRec { @@ -445,7 +450,7 @@ int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo); void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo); void tscSetResRawPtrRv(SSqlRes* pRes, SQueryInfo* pQueryInfo, SSDataBlock* pBlock); -void handleDownstreamOperator(SSqlRes* pRes, SQueryInfo* pQueryInfo); +void handleDownstreamOperator(SSqlObj** pSqlList, int32_t numOfUpstream, SQueryInfo* px, SSqlRes* pOutput); void destroyTableNameList(SSqlCmd* pCmd); void tscResetSqlCmd(SSqlCmd *pCmd, bool removeMeta); @@ -491,7 +496,7 @@ char *tscGetErrorMsgPayload(SSqlCmd *pCmd); int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *sql); int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* sql); -int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo); +int32_t tscValidateSqlInfo(SSqlObj *pSql, struct SSqlInfo *pInfo); extern int32_t sentinel; extern SHashObj *tscVgroupMap; @@ -507,7 +512,7 @@ extern int tscNumOfObj; // number of existed sqlObj in current process. extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables); -int16_t getNewResColId(SQueryInfo* pQueryInfo); +int16_t getNewResColId(SSqlCmd* pCmd); #ifdef __cplusplus } diff --git a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h index 04bccc1a4a..d16b672f38 100644 --- a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h +++ b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h @@ -218,11 +218,19 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeBatchImp(J /* * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: executeBatchImp + * Method: closeStmt * Signature: (JJ)I */ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeStmt(JNIEnv *env, jobject jobj, jlong stmt, jlong con); +/** + * Class: com_taosdata_jdbc_TSDBJNIConnector + * Method: setTableNameTagsImp + * Signature: (JLjava/lang/String;I[B[B[B[BJ)I + */ +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setTableNameTagsImp + (JNIEnv *, jobject, jlong, jstring, jint, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jlong); + #ifdef __cplusplus } #endif diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index da7da17aa3..324c436dce 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -749,7 +749,6 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setBindTableNameI } jniDebug("jobj:%p, conn:%p, set stmt bind table name:%s", jobj, tsconn, name); - (*env)->ReleaseStringUTFChars(env, jname, name); return JNI_SUCCESS; } @@ -762,7 +761,7 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp(J return JNI_CONNECTION_NULL; } - TAOS_STMT* pStmt = (TAOS_STMT*) stmt; + TAOS_STMT *pStmt = (TAOS_STMT *)stmt; if (pStmt == NULL) { jniError("jobj:%p, conn:%p, invalid stmt", jobj, tscon); return JNI_SQL_NULL; @@ -777,14 +776,14 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp(J } len = (*env)->GetArrayLength(env, lengthList); - char *lengthArray = (char*) calloc(1, len); - (*env)->GetByteArrayRegion(env, lengthList, 0, len, (jbyte*) lengthArray); + char *lengthArray = (char *)calloc(1, len); + (*env)->GetByteArrayRegion(env, lengthList, 0, len, (jbyte *)lengthArray); if ((*env)->ExceptionCheck(env)) { } len = (*env)->GetArrayLength(env, nullList); - char *nullArray = (char*) calloc(1, len); - (*env)->GetByteArrayRegion(env, nullList, 0, len, (jbyte*) nullArray); + char *nullArray = (char *)calloc(1, len); + (*env)->GetByteArrayRegion(env, nullList, 0, len, (jbyte *)nullArray); if ((*env)->ExceptionCheck(env)) { } @@ -799,22 +798,10 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp(J b->length = (int32_t*)lengthArray; // set the length and is_null array - switch(dataType) { - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: { - int32_t bytes = tDataTypes[dataType].bytes; - for(int32_t i = 0; i < numOfRows; ++i) { - b->length[i] = bytes; - } - break; - } - - case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_BINARY: { - // do nothing + if (!IS_VAR_DATA_TYPE(dataType)) { + int32_t bytes = tDataTypes[dataType].bytes; + for (int32_t i = 0; i < numOfRows; ++i) { + b->length[i] = bytes; } } @@ -878,3 +865,74 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeStmt(JNIEnv jniDebug("jobj:%p, conn:%p, stmt closed", jobj, tscon); return JNI_SUCCESS; } + +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setTableNameTagsImp(JNIEnv *env, jobject jobj, + jlong stmt, jstring tableName, jint numOfTags, jbyteArray tags, jbyteArray typeList, jbyteArray lengthList, jbyteArray nullList, jlong conn) { + TAOS *tsconn = (TAOS *)conn; + if (tsconn == NULL) { + jniError("jobj:%p, connection already closed", jobj); + return JNI_CONNECTION_NULL; + } + + TAOS_STMT* pStmt = (TAOS_STMT*) stmt; + if (pStmt == NULL) { + jniError("jobj:%p, conn:%p, invalid stmt handle", jobj, tsconn); + return JNI_SQL_NULL; + } + + jsize len = (*env)->GetArrayLength(env, tags); + char *tagsData = (char *)calloc(1, len); + (*env)->GetByteArrayRegion(env, tags, 0, len, (jbyte *)tagsData); + if ((*env)->ExceptionCheck(env)) { + // todo handle error + } + + len = (*env)->GetArrayLength(env, lengthList); + int64_t *lengthArray = (int64_t*) calloc(1, len); + (*env)->GetByteArrayRegion(env, lengthList, 0, len, (jbyte*) lengthArray); + if ((*env)->ExceptionCheck(env)) { + } + + len = (*env)->GetArrayLength(env, typeList); + char *typeArray = (char*) calloc(1, len); + (*env)->GetByteArrayRegion(env, typeList, 0, len, (jbyte*) typeArray); + if ((*env)->ExceptionCheck(env)) { + } + + len = (*env)->GetArrayLength(env, nullList); + int32_t *nullArray = (int32_t*) calloc(1, len); + (*env)->GetByteArrayRegion(env, nullList, 0, len, (jbyte*) nullArray); + if ((*env)->ExceptionCheck(env)) { + } + + const char *name = (*env)->GetStringUTFChars(env, tableName, NULL); + char* curTags = tagsData; + + TAOS_BIND *tagsBind = calloc(numOfTags, sizeof(TAOS_BIND)); + for(int32_t i = 0; i < numOfTags; ++i) { + tagsBind[i].buffer_type = typeArray[i]; + tagsBind[i].buffer = curTags; + tagsBind[i].is_null = &nullArray[i]; + tagsBind[i].length = (uintptr_t*) &lengthArray[i]; + + curTags += lengthArray[i]; + } + + int32_t code = taos_stmt_set_tbname_tags((void*)stmt, name, tagsBind); + + int32_t nTags = (int32_t) numOfTags; + jniDebug("jobj:%p, conn:%p, set table name:%s, numOfTags:%d", jobj, tsconn, name, nTags); + + tfree(tagsData); + tfree(lengthArray); + tfree(typeArray); + tfree(nullArray); + (*env)->ReleaseStringUTFChars(env, tableName, name); + + if (code != TSDB_CODE_SUCCESS) { + jniError("jobj:%p, conn:%p, code:%s", jobj, tsconn, tstrerror(code)); + return JNI_TDENGINE_ERROR; + } + + return JNI_SUCCESS; +} diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 09b31e4b19..47798f4e33 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -59,6 +59,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* para tscDebugL("0x%"PRIx64" SQL: %s", pSql->self, pSql->sqlstr); pCmd->curSql = pSql->sqlstr; + pCmd->resColumnId = TSDB_RES_COL_ID; int32_t code = tsParseSql(pSql, true); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return; @@ -69,7 +70,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* para return; } - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); executeQuery(pSql, pQueryInfo); } @@ -127,7 +128,8 @@ static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows) { * all available virtual node has been checked already, now we need to check * for the next subclause queries */ - if (pCmd->clauseIndex < pCmd->numOfClause - 1) { + if (pCmd->active->sibling != NULL) { + pCmd->active = pCmd->active->sibling; tscTryQueryNextClause(pSql, tscAsyncQueryRowsForNextVnode); return; } @@ -220,6 +222,17 @@ void taos_fetch_rows_a(TAOS_RES *tres, __async_cb_func_t fp, void *param) { tscResetForNextRetrieve(pRes); // handle the sub queries of join query + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) { + SSchedMsg schedMsg = {0}; + schedMsg.fp = doRetrieveSubqueryData; + schedMsg.ahandle = (void *)pSql; + schedMsg.thandle = (void *)1; + schedMsg.msg = 0; + taosScheduleTask(tscQhandle, &schedMsg); + return; + } + if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) { tscFetchDatablockForSubquery(pSql); } else if (pRes->completed) { @@ -231,7 +244,8 @@ void taos_fetch_rows_a(TAOS_RES *tres, __async_cb_func_t fp, void *param) { * all available virtual nodes in current clause has been checked already, now try the * next one in the following union subclause */ - if (pCmd->clauseIndex < pCmd->numOfClause - 1) { + if (pCmd->active->sibling != NULL) { + pCmd->active = pCmd->active->sibling; // todo refactor tscTryQueryNextClause(pSql, tscAsyncQueryRowsForNextVnode); return; } @@ -255,7 +269,7 @@ void taos_fetch_rows_a(TAOS_RES *tres, __async_cb_func_t fp, void *param) { pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; } - SQueryInfo* pQueryInfo1 = tscGetActiveQueryInfo(&pSql->cmd); + SQueryInfo* pQueryInfo1 = tscGetQueryInfo(&pSql->cmd); tscBuildAndSendRequest(pSql, pQueryInfo1); } } @@ -317,26 +331,38 @@ static int32_t updateMetaBeforeRetryQuery(SSqlObj* pSql, STableMetaInfo* pTableM // update the pExpr info, colList info, number of table columns // TODO Re-parse this sql and issue the corresponding subquery as an alternative for this case. if (pSql->retryReason == TSDB_CODE_TDB_INVALID_TABLE_ID) { - int32_t numOfExprs = (int32_t) tscSqlExprNumOfExprs(pQueryInfo); + int32_t numOfExprs = (int32_t) tscNumOfExprs(pQueryInfo); int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); int32_t numOfTags = tscGetNumOfTags(pTableMetaInfo->pTableMeta); SSchema *pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); + SSchema *pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); + for (int32_t i = 0; i < numOfExprs; ++i) { - SSqlExpr *pExpr = &(tscSqlExprGet(pQueryInfo, i)->base); + SSqlExpr *pExpr = &(tscExprGet(pQueryInfo, i)->base); + + // update the table uid pExpr->uid = pTableMetaInfo->pTableMeta->id.uid; if (pExpr->colInfo.colIndex >= 0) { int32_t index = pExpr->colInfo.colIndex; if ((TSDB_COL_IS_NORMAL_COL(pExpr->colInfo.flag) && index >= numOfCols) || - (TSDB_COL_IS_TAG(pExpr->colInfo.flag) && (index < numOfCols || index >= (numOfCols + numOfTags)))) { + (TSDB_COL_IS_TAG(pExpr->colInfo.flag) && (index < 0 || index >= numOfTags))) { return pSql->retryReason; } - if ((pSchema[pExpr->colInfo.colIndex].colId != pExpr->colInfo.colId) && - strcasecmp(pExpr->colInfo.name, pSchema[pExpr->colInfo.colIndex].name) != 0) { - return pSql->retryReason; + if (TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { + if ((pTagSchema[pExpr->colInfo.colIndex].colId != pExpr->colInfo.colId) && + strcasecmp(pExpr->colInfo.name, pTagSchema[pExpr->colInfo.colIndex].name) != 0) { + return pSql->retryReason; + } + } else if (TSDB_COL_IS_NORMAL_COL(pExpr->colInfo.flag)) { + if ((pSchema[pExpr->colInfo.colIndex].colId != pExpr->colInfo.colId) && + strcasecmp(pExpr->colInfo.name, pSchema[pExpr->colInfo.colIndex].name) != 0) { + return pSql->retryReason; + } + } else { // do nothing for udc } } } @@ -374,12 +400,12 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { tscDebug("0x%"PRIx64" get %s successfully", pSql->self, msg); if (pSql->pStream == NULL) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); // check if it is a sub-query of super table query first, if true, enter another routine - if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY|TSDB_QUERY_TYPE_SUBQUERY|TSDB_QUERY_TYPE_TAG_FILTER_QUERY))) { - tscDebug("0x%"PRIx64" update local table meta, continue to process sql and send the corresponding query", pSql->self); - + if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY | TSDB_QUERY_TYPE_SUBQUERY | + TSDB_QUERY_TYPE_TAG_FILTER_QUERY))) { + tscDebug("0x%" PRIx64 " update cached table-meta, continue to process sql and send the corresponding query", pSql->self); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); code = tscGetTableMeta(pSql, pTableMetaInfo); @@ -401,42 +427,8 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { taosReleaseRef(tscObjRef, pSql->self); return; } else { // continue to process normal async query - if (pCmd->parseFinished) { - tscDebug("0x%"PRIx64" update local table meta, continue to process sql and send corresponding query", pSql->self); - - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - code = tscGetTableMeta(pSql, pTableMetaInfo); - - assert(code == TSDB_CODE_TSC_ACTION_IN_PROGRESS || code == TSDB_CODE_SUCCESS); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - taosReleaseRef(tscObjRef, pSql->self); - return; - } - - assert(pCmd->command != TSDB_SQL_INSERT); - - if (pCmd->command == TSDB_SQL_SELECT) { - tscDebug("0x%"PRIx64" redo parse sql string and proceed", pSql->self); - pCmd->parseFinished = false; - tscResetSqlCmd(pCmd, true); - - code = tsParseSql(pSql, true); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - taosReleaseRef(tscObjRef, pSql->self); - return; - } else if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - tscBuildAndSendRequest(pSql, NULL); - } else { // in all other cases, simple retry - tscBuildAndSendRequest(pSql, NULL); - } - - taosReleaseRef(tscObjRef, pSql->self); - return; - } else { - tscDebug("0x%"PRIx64" continue parse sql after get table meta", pSql->self); + if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT)) { + tscDebug("0x%" PRIx64 " continue parse sql after get table-meta", pSql->self); code = tsParseSql(pSql, false); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { @@ -446,8 +438,8 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { goto _error; } - if (pCmd->insertType == TSDB_QUERY_TYPE_STMT_INSERT) { - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + if (TSDB_QUERY_HAS_TYPE(pCmd->insertParam.insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); code = tscGetTableMeta(pSql, pTableMetaInfo); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { taosReleaseRef(tscObjRef, pSql->self); @@ -457,59 +449,52 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { } (*pSql->fp)(pSql->param, pSql, code); - } else if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT)) { - if (pCmd->dataSourceType == DATA_FROM_DATA_FILE) { + } else { + if (TSDB_QUERY_HAS_TYPE(pCmd->insertParam.insertType, TSDB_QUERY_TYPE_FILE_INSERT)) { tscImportDataFromFile(pSql); } else { tscHandleMultivnodeInsert(pSql); } + } + } else { + if (pSql->retryReason != TSDB_CODE_SUCCESS) { + tscDebug("0x%" PRIx64 " update cached table-meta, re-validate sql statement and send query again", + pSql->self); + tscResetSqlCmd(pCmd, false); + pSql->retryReason = TSDB_CODE_SUCCESS; } else { - SQueryInfo* pQueryInfo1 = tscGetQueryInfo(pCmd, pCmd->clauseIndex); - executeQuery(pSql, pQueryInfo1); + tscDebug("0x%" PRIx64 " cached table-meta, continue validate sql statement and send query", pSql->self); } - taosReleaseRef(tscObjRef, pSql->self); - return; + code = tsParseSql(pSql, true); + if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + taosReleaseRef(tscObjRef, pSql->self); + return; + } else if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + SQueryInfo *pQueryInfo1 = tscGetQueryInfo(pCmd); + executeQuery(pSql, pQueryInfo1); } - } - } else { // stream computing - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - - code = tscGetTableMeta(pSql, pTableMetaInfo); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { taosReleaseRef(tscObjRef, pSql->self); return; - } else if (code != TSDB_CODE_SUCCESS) { - goto _error; } + } else { // stream computing + tscDebug("0x%"PRIx64" stream:%p meta is updated, start new query, command:%d", pSql->self, pSql->pStream, pCmd->command); - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - code = tscGetSTableVgroupInfo(pSql, pCmd->clauseIndex); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - taosReleaseRef(tscObjRef, pSql->self); - return; - } else if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - } - - tscDebug("0x%"PRIx64" stream:%p meta is updated, start new query, command:%d", pSql->self, pSql->pStream, pSql->cmd.command); - if (!pSql->cmd.parseFinished) { + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + if (tscNumOfExprs(pQueryInfo) == 0) { tsParseSql(pSql, false); } (*pSql->fp)(pSql->param, pSql, code); - taosReleaseRef(tscObjRef, pSql->self); - return; } -// tscDoQuery(pSql); - taosReleaseRef(tscObjRef, pSql->self); - return; _error: diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 1be5e29230..abac2407fb 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -53,7 +53,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { SSqlRes *pRes = &pSql->res; // one column for each row - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMeta * pMeta = pTableMetaInfo->pTableMeta; @@ -154,14 +154,14 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, pSql->cmd.numOfCols = numOfCols; - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); pQueryInfo->order.order = TSDB_ORDER_ASC; TAOS_FIELD f = {.type = TSDB_DATA_TYPE_BINARY, .bytes = (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE}; tstrncpy(f.name, "Field", sizeof(f.name)); SInternalField* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, + pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE, -1000, (TSDB_COL_NAME_LEN - 1), false); rowLen += ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE); @@ -171,7 +171,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, tstrncpy(f.name, "Type", sizeof(f.name)); pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, (int16_t)(typeColLength + VARSTR_HEADER_SIZE), + pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, (int16_t)(typeColLength + VARSTR_HEADER_SIZE), -1000, typeColLength, false); rowLen += typeColLength + VARSTR_HEADER_SIZE; @@ -181,7 +181,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, tstrncpy(f.name, "Length", sizeof(f.name)); pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t), + pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t), -1000, sizeof(int32_t), false); rowLen += sizeof(int32_t); @@ -191,7 +191,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, tstrncpy(f.name, "Note", sizeof(f.name)); pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, (int16_t)(noteColLength + VARSTR_HEADER_SIZE), + pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, (int16_t)(noteColLength + VARSTR_HEADER_SIZE), -1000, noteColLength, false); rowLen += noteColLength + VARSTR_HEADER_SIZE; @@ -199,7 +199,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, } static int32_t tscProcessDescribeTable(SSqlObj *pSql) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); assert(tscGetMetaInfo(pQueryInfo, 0)->pTableMeta != NULL); @@ -390,7 +390,7 @@ static int32_t tscSCreateBuildResultFields(SSqlObj *pSql, BuildType type, const SColumnIndex index = {0}; pSql->cmd.numOfCols = 2; - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); pQueryInfo->order.order = TSDB_ORDER_ASC; TAOS_FIELD f; @@ -405,7 +405,7 @@ static int32_t tscSCreateBuildResultFields(SSqlObj *pSql, BuildType type, const } SInternalField* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, f.bytes, -1000, f.bytes - VARSTR_HEADER_SIZE, false); + pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, f.bytes, -1000, f.bytes - VARSTR_HEADER_SIZE, false); rowLen += f.bytes; @@ -418,7 +418,7 @@ static int32_t tscSCreateBuildResultFields(SSqlObj *pSql, BuildType type, const } pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, + pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, (int16_t)(ddlLen + VARSTR_HEADER_SIZE), -1000, ddlLen, false); rowLen += ddlLen + VARSTR_HEADER_SIZE; @@ -428,7 +428,7 @@ static int32_t tscSCreateBuildResultFields(SSqlObj *pSql, BuildType type, const static int32_t tscSCreateSetValueToResObj(SSqlObj *pSql, int32_t rowLen, const char *tableName, const char *ddl) { SSqlRes *pRes = &pSql->res; - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); int32_t numOfRows = 1; if (strlen(ddl) == 0) { @@ -445,7 +445,7 @@ static int32_t tscSCreateSetValueToResObj(SSqlObj *pSql, int32_t rowLen, const c return 0; } static int32_t tscSCreateBuildResult(SSqlObj *pSql, BuildType type, const char *str, const char *result) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); int32_t rowLen = tscSCreateBuildResultFields(pSql, type, result); tscFieldInfoUpdateOffset(pQueryInfo); @@ -532,7 +532,7 @@ static int32_t tscGetTableTagColumnName(SSqlObj *pSql, char **result) { } buf[0] = 0; - STableMeta *pMeta = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0)->pTableMeta; + STableMeta *pMeta = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0)->pTableMeta; if (pMeta->tableType == TSDB_SUPER_TABLE || pMeta->tableType == TSDB_NORMAL_TABLE || pMeta->tableType == TSDB_STREAM_TABLE) { free(buf); @@ -553,7 +553,7 @@ static int32_t tscGetTableTagColumnName(SSqlObj *pSql, char **result) { return TSDB_CODE_SUCCESS; } static int32_t tscRebuildDDLForSubTable(SSqlObj *pSql, const char *tableName, char *ddl) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMeta * pMeta = pTableMetaInfo->pTableMeta; @@ -607,7 +607,7 @@ static int32_t tscRebuildDDLForSubTable(SSqlObj *pSql, const char *tableName, ch } static int32_t tscRebuildDDLForNormalTable(SSqlObj *pSql, const char *tableName, char *ddl) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMeta * pMeta = pTableMetaInfo->pTableMeta; @@ -634,7 +634,7 @@ static int32_t tscRebuildDDLForNormalTable(SSqlObj *pSql, const char *tableName, } static int32_t tscRebuildDDLForSuperTable(SSqlObj *pSql, const char *tableName, char *ddl) { char *result = ddl; - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMeta * pMeta = pTableMetaInfo->pTableMeta; @@ -675,7 +675,7 @@ static int32_t tscRebuildDDLForSuperTable(SSqlObj *pSql, const char *tableName, } static int32_t tscProcessShowCreateTable(SSqlObj *pSql) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); assert(pTableMetaInfo->pTableMeta != NULL); @@ -704,7 +704,7 @@ static int32_t tscProcessShowCreateTable(SSqlObj *pSql) { } static int32_t tscProcessShowCreateDatabase(SSqlObj *pSql) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -730,7 +730,7 @@ static int32_t tscProcessShowCreateDatabase(SSqlObj *pSql) { return TSDB_CODE_TSC_ACTION_IN_PROGRESS; } static int32_t tscProcessCurrentUser(SSqlObj *pSql) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); pExpr->resBytes = TSDB_USER_LEN + TSDB_DATA_TYPE_BINARY; @@ -757,7 +757,7 @@ static int32_t tscProcessCurrentDB(SSqlObj *pSql) { extractDBName(pSql->pTscObj->db, db); pthread_mutex_unlock(&pSql->pTscObj->mutex); - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, pSql->cmd.clauseIndex); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); pExpr->resType = TSDB_DATA_TYPE_BINARY; @@ -784,7 +784,7 @@ static int32_t tscProcessCurrentDB(SSqlObj *pSql) { static int32_t tscProcessServerVer(SSqlObj *pSql) { const char* v = pSql->pTscObj->sversion; - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, pSql->cmd.clauseIndex); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); pExpr->resType = TSDB_DATA_TYPE_BINARY; @@ -807,7 +807,7 @@ static int32_t tscProcessServerVer(SSqlObj *pSql) { } static int32_t tscProcessClientVer(SSqlObj *pSql) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); pExpr->resType = TSDB_DATA_TYPE_BINARY; @@ -859,7 +859,7 @@ static int32_t tscProcessServStatus(SSqlObj *pSql) { return pSql->res.code; } - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); int32_t val = 1; @@ -873,7 +873,7 @@ void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnNa pCmd->numOfCols = 1; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); pQueryInfo->order.order = TSDB_ORDER_ASC; tscFieldInfoClear(&pQueryInfo->fieldsInfo); @@ -928,7 +928,7 @@ int tscProcessLocalCmd(SSqlObj *pSql) { } else if (pCmd->command == TSDB_SQL_SERV_STATUS) { pRes->code = tscProcessServStatus(pSql); } else { - pRes->code = TSDB_CODE_TSC_INVALID_SQL; + pRes->code = TSDB_CODE_TSC_INVALID_OPERATION; tscError("0x%"PRIx64" not support command:%d", pSql->self, pCmd->command); } diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index f5b691324a..4fffe5f128 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -19,8 +19,6 @@ #include "texpr.h" #include "tlosertree.h" #include "tscLog.h" -#include "tscUtil.h" -#include "tschemautil.h" #include "tsclient.h" #include "qUtil.h" @@ -59,77 +57,25 @@ int32_t treeComparator(const void *pLeft, const void *pRight, void *param) { } } -// todo merge with vnode side function -void tsCreateSQLFunctionCtx(SQueryInfo* pQueryInfo, SQLFunctionCtx* pCtx, SSchema* pSchema) { - size_t size = tscSqlExprNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < size; ++i) { - SExprInfo *pExpr = tscSqlExprGet(pQueryInfo, i); - - pCtx[i].order = pQueryInfo->order.order; - pCtx[i].functionId = pExpr->base.functionId; - - pCtx[i].order = pQueryInfo->order.order; - pCtx[i].functionId = pExpr->base.functionId; - - // input data format comes from pModel - pCtx[i].inputType = pSchema[i].type; - pCtx[i].inputBytes = pSchema[i].bytes; - - pCtx[i].outputBytes = pExpr->base.resBytes; - pCtx[i].outputType = pExpr->base.resType; - - // input buffer hold only one point data - pCtx[i].size = 1; - pCtx[i].hasNull = true; - pCtx[i].currentStage = MERGE_STAGE; - - // for top/bottom function, the output of timestamp is the first column - int32_t functionId = pExpr->base.functionId; - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - pCtx[i].ptsOutputBuf = pCtx[0].pOutput; - pCtx[i].param[2].i64 = pQueryInfo->order.order; - pCtx[i].param[2].nType = TSDB_DATA_TYPE_BIGINT; - pCtx[i].param[1].i64 = pQueryInfo->order.orderColId; - pCtx[i].param[0].i64 = pExpr->base.param[0].i64; // top/bot parameter - } else if (functionId == TSDB_FUNC_APERCT) { - pCtx[i].param[0].i64 = pExpr->base.param[0].i64; - pCtx[i].param[0].nType = pExpr->base.param[0].nType; - } else if (functionId == TSDB_FUNC_BLKINFO) { - pCtx[i].param[0].i64 = pExpr->base.param[0].i64; - pCtx[i].param[0].nType = pExpr->base.param[0].nType; - pCtx[i].numOfParams = 1; - } - - pCtx[i].interBufBytes = pExpr->base.interBytes; - pCtx[i].stableQuery = true; - } -} - -void tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc, - SColumnModel *finalmodel, SColumnModel *pFFModel, SSqlObj *pSql) { - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - +int32_t tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc, + SQueryInfo* pQueryInfo, SLocalMerger **pMerger, int64_t id) { if (pMemBuffer == NULL) { - tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, pFFModel, numOfBuffer); - tscError("pMemBuffer:%p is NULL", pMemBuffer); - pRes->code = TSDB_CODE_TSC_APP_ERROR; - return; + tscLocalReducerEnvDestroy(pMemBuffer, pDesc, numOfBuffer); + tscError("0x%"PRIx64" %p pMemBuffer is NULL", id, pMemBuffer); + return TSDB_CODE_TSC_APP_ERROR; } if (pDesc->pColumnModel == NULL) { - tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, pFFModel, numOfBuffer); - tscError("0x%"PRIx64" no local buffer or intermediate result format model", pSql->self); - pRes->code = TSDB_CODE_TSC_APP_ERROR; - return; + tscLocalReducerEnvDestroy(pMemBuffer, pDesc, numOfBuffer); + tscError("0x%"PRIx64" no local buffer or intermediate result format model", id); + return TSDB_CODE_TSC_APP_ERROR; } int32_t numOfFlush = 0; for (int32_t i = 0; i < numOfBuffer; ++i) { int32_t len = pMemBuffer[i]->fileMeta.flushoutData.nLength; if (len == 0) { - tscDebug("0x%"PRIx64" no data retrieved from orderOfVnode:%d", pSql->self, i + 1); + tscDebug("0x%"PRIx64" no data retrieved from orderOfVnode:%d", id, i + 1); continue; } @@ -137,41 +83,36 @@ void tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrde } if (numOfFlush == 0 || numOfBuffer == 0) { - tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, pFFModel, numOfBuffer); - pCmd->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; // no result, set the result empty - tscDebug("0x%"PRIx64" retrieved no data", pSql->self); - return; + tscLocalReducerEnvDestroy(pMemBuffer, pDesc, numOfBuffer); + tscDebug("0x%"PRIx64" no data to retrieve", id); + return TSDB_CODE_SUCCESS; } if (pDesc->pColumnModel->capacity >= pMemBuffer[0]->pageSize) { - tscError("0x%"PRIx64" Invalid value of buffer capacity %d and page size %d ", pSql->self, pDesc->pColumnModel->capacity, + tscError("0x%"PRIx64" Invalid value of buffer capacity %d and page size %d ", id, pDesc->pColumnModel->capacity, pMemBuffer[0]->pageSize); - tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, pFFModel, numOfBuffer); - pRes->code = TSDB_CODE_TSC_APP_ERROR; - return; + tscLocalReducerEnvDestroy(pMemBuffer, pDesc, numOfBuffer); + return TSDB_CODE_TSC_APP_ERROR; } - size_t size = sizeof(SLocalMerger) + POINTER_BYTES * numOfFlush; - - SLocalMerger *pMerger = (SLocalMerger *) calloc(1, size); - if (pMerger == NULL) { - tscError("0x%"PRIx64" failed to create local merge structure, out of memory", pSql->self); + *pMerger = (SLocalMerger *) calloc(1, sizeof(SLocalMerger)); + if ((*pMerger) == NULL) { + tscError("0x%"PRIx64" failed to create local merge structure, out of memory", id); - tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, pFFModel, numOfBuffer); - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return; + tscLocalReducerEnvDestroy(pMemBuffer, pDesc, numOfBuffer); + return TSDB_CODE_TSC_OUT_OF_MEMORY; } - pMerger->pExtMemBuffer = pMemBuffer; - pMerger->pLocalDataSrc = (SLocalDataSource **)&pMerger[1]; - assert(pMerger->pLocalDataSrc != NULL); + (*pMerger)->pExtMemBuffer = pMemBuffer; + (*pMerger)->pLocalDataSrc = calloc(numOfFlush, POINTER_BYTES); + assert((*pMerger)->pLocalDataSrc != NULL); - pMerger->numOfBuffer = numOfFlush; - pMerger->numOfVnode = numOfBuffer; + (*pMerger)->numOfBuffer = numOfFlush; + (*pMerger)->numOfVnode = numOfBuffer; - pMerger->pDesc = pDesc; - tscDebug("0x%"PRIx64" the number of merged leaves is: %d", pSql->self, pMerger->numOfBuffer); + (*pMerger)->pDesc = pDesc; + tscDebug("0x%"PRIx64" the number of merged leaves is: %d", id, (*pMerger)->numOfBuffer); int32_t idx = 0; for (int32_t i = 0; i < numOfBuffer; ++i) { @@ -180,13 +121,12 @@ void tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrde for (int32_t j = 0; j < numOfFlushoutInFile; ++j) { SLocalDataSource *ds = (SLocalDataSource *)malloc(sizeof(SLocalDataSource) + pMemBuffer[0]->pageSize); if (ds == NULL) { - tscError("0x%"PRIx64" failed to create merge structure", pSql->self); - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + tscError("0x%"PRIx64" failed to create merge structure", id); tfree(pMerger); - return; + return TSDB_CODE_TSC_OUT_OF_MEMORY; } - pMerger->pLocalDataSrc[idx] = ds; + (*pMerger)->pLocalDataSrc[idx] = ds; ds->pMemBuffer = pMemBuffer[i]; ds->flushoutIdx = j; @@ -194,12 +134,12 @@ void tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrde ds->pageId = 0; ds->rowIdx = 0; - tscDebug("0x%"PRIx64" load data from disk into memory, orderOfVnode:%d, total:%d", pSql->self, i + 1, idx + 1); + tscDebug("0x%"PRIx64" load data from disk into memory, orderOfVnode:%d, total:%d", id, i + 1, idx + 1); tExtMemBufferLoadData(pMemBuffer[i], &(ds->filePage), j, 0); #ifdef _DEBUG_VIEW printf("load data page into mem for build loser tree: %" PRIu64 " rows\n", ds->filePage.num); SSrcColumnInfo colInfo[256] = {0}; - SQueryInfo * pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); + SQueryInfo * pQueryInfo = tscGetQueryInfo(pCmd); tscGetSrcColumnInfo(colInfo, pQueryInfo); @@ -208,7 +148,7 @@ void tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrde #endif if (ds->filePage.num == 0) { // no data in this flush, the index does not increase - tscDebug("0x%"PRIx64" flush data is empty, ignore %d flush record", pSql->self, idx); + tscDebug("0x%"PRIx64" flush data is empty, ignore %d flush record", id, idx); tfree(ds); continue; } @@ -219,91 +159,43 @@ void tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrde // no data actually, no need to merge result. if (idx == 0) { - tfree(pMerger); - return; + tscDebug("0x%"PRIx64" retrieved no data", id); + tscLocalReducerEnvDestroy(pMemBuffer, pDesc, numOfBuffer); + return TSDB_CODE_SUCCESS; } - pMerger->numOfBuffer = idx; + (*pMerger)->numOfBuffer = idx; SCompareParam *param = malloc(sizeof(SCompareParam)); if (param == NULL) { - tfree(pMerger); - return; + tfree((*pMerger)); + return TSDB_CODE_TSC_OUT_OF_MEMORY; } - param->pLocalData = pMerger->pLocalDataSrc; - param->pDesc = pMerger->pDesc; - param->num = pMerger->pLocalDataSrc[0]->pMemBuffer->numOfElemsPerPage; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); + param->pLocalData = (*pMerger)->pLocalDataSrc; + param->pDesc = (*pMerger)->pDesc; + param->num = (*pMerger)->pLocalDataSrc[0]->pMemBuffer->numOfElemsPerPage; param->groupOrderType = pQueryInfo->groupbyExpr.orderType; - pRes->code = tLoserTreeCreate(&pMerger->pLoserTree, pMerger->numOfBuffer, param, treeComparator); - if (pMerger->pLoserTree == NULL || pRes->code != 0) { + int32_t code = tLoserTreeCreate(&(*pMerger)->pLoserTree, (*pMerger)->numOfBuffer, param, treeComparator); + if ((*pMerger)->pLoserTree == NULL || code != TSDB_CODE_SUCCESS) { tfree(param); - tfree(pMerger); - return; + tfree((*pMerger)); + return code; } - // the input data format follows the old format, but output in a new format. - // so, all the input must be parsed as old format - pMerger->pCtx = (SQLFunctionCtx *)calloc(tscSqlExprNumOfExprs(pQueryInfo), sizeof(SQLFunctionCtx)); - pMerger->rowSize = pMemBuffer[0]->nElemSize; + (*pMerger)->rowSize = pMemBuffer[0]->nElemSize; - tscFieldInfoUpdateOffset(pQueryInfo); + // todo fixed row size is larger than the minimum page size; + assert((*pMerger)->rowSize <= pMemBuffer[0]->pageSize); - if (pMerger->rowSize > pMemBuffer[0]->pageSize) { - assert(false); // todo fixed row size is larger than the minimum page size; - } - - // used to keep the latest input row - pMerger->pTempBuffer = (tFilePage *)calloc(1, pMerger->rowSize + sizeof(tFilePage)); - - pMerger->nResultBufSize = pMemBuffer[0]->pageSize * 16; - pMerger->pResultBuf = (tFilePage *)calloc(1, pMerger->nResultBufSize + sizeof(tFilePage)); - - pMerger->resColModel = finalmodel; - pMerger->resColModel->capacity = pMerger->nResultBufSize; - pMerger->finalModel = pFFModel; - - if (finalmodel->rowSize > 0) { - pMerger->resColModel->capacity /= finalmodel->rowSize; - } - - assert(finalmodel->rowSize > 0 && finalmodel->rowSize <= pMerger->rowSize); - - if (pMerger->pTempBuffer == NULL || pMerger->pLoserTree == NULL) { - tfree(pMerger->pTempBuffer); - tfree(pMerger->pLoserTree); + if ((*pMerger)->pLoserTree == NULL) { + tfree((*pMerger)->pLoserTree); tfree(param); - tfree(pMerger); - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return; + tfree((*pMerger)); + return TSDB_CODE_TSC_OUT_OF_MEMORY; } - - pMerger->pTempBuffer->num = 0; - tscCreateResPointerInfo(pRes, pQueryInfo); - - SSchema* pschema = calloc(pDesc->pColumnModel->numOfCols, sizeof(SSchema)); - for(int32_t i = 0; i < pDesc->pColumnModel->numOfCols; ++i) { - pschema[i] = pDesc->pColumnModel->pFields[i].field; - } - - tsCreateSQLFunctionCtx(pQueryInfo, pMerger->pCtx, pschema); -// setCtxInputOutputBuffer(pQueryInfo, pMerger->pCtx, pMerger, pDesc); - - tfree(pschema); - - int32_t maxBufSize = 0; - for (int32_t k = 0; k < tscSqlExprNumOfExprs(pQueryInfo); ++k) { - SExprInfo *pExpr = tscSqlExprGet(pQueryInfo, k); - if (maxBufSize < pExpr->base.resBytes && pExpr->base.functionId == TSDB_FUNC_TAG) { - maxBufSize = pExpr->base.resBytes; - } - } - - // we change the capacity of schema to denote that there is only one row in temp buffer - pMerger->pDesc->pColumnModel->capacity = 1; // restore the limitation value at the last stage if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { @@ -311,22 +203,10 @@ void tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrde pQueryInfo->limit.offset = pQueryInfo->prjOffset; } - pRes->pLocalMerger = pMerger; - pRes->numOfGroups = 0; + // we change the capacity of schema to denote that there is only one row in temp buffer + (*pMerger)->pDesc->pColumnModel->capacity = 1; -// STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); -// STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - -// TSKEY stime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.skey : pQueryInfo->window.ekey; -// int64_t revisedSTime = taosTimeTruncate(stime, &pQueryInfo->interval, tinfo.precision); - -// if (pQueryInfo->fillType != TSDB_FILL_NONE) { -// SFillColInfo* pFillCol = createFillColInfo(pQueryInfo); -// pMerger->pFillInfo = -// taosCreateFillInfo(pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols, 4096, -// (int32_t)pQueryInfo->fieldsInfo.numOfOutput, pQueryInfo->interval.sliding, -// pQueryInfo->interval.slidingUnit, tinfo.precision, pQueryInfo->fillType, pFillCol, pSql); -// } + return TSDB_CODE_SUCCESS; } static int32_t tscFlushTmpBufferImpl(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePage *pPage, @@ -417,44 +297,32 @@ int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePa return 0; } -void tscDestroyLocalMerger(SSqlObj *pSql) { - if (pSql == NULL) { +void tscDestroyLocalMerger(SLocalMerger* pLocalMerger) { + if (pLocalMerger == NULL) { return; } - SSqlRes *pRes = &(pSql->res); - if (pRes->pLocalMerger == NULL) { - return; + for (int32_t i = 0; i < pLocalMerger->numOfBuffer; ++i) { + tfree(pLocalMerger->pLocalDataSrc[i]); } - // there is no more result, so we release all allocated resource - SLocalMerger *pLocalMerge = (SLocalMerger *)atomic_exchange_ptr(&pRes->pLocalMerger, NULL); - tfree(pLocalMerge->pResultBuf); - tfree(pLocalMerge->pCtx); + pLocalMerger->numOfBuffer = 0; + tscLocalReducerEnvDestroy(pLocalMerger->pExtMemBuffer, pLocalMerger->pDesc, pLocalMerger->numOfVnode); - if (pLocalMerge->pLoserTree) { - tfree(pLocalMerge->pLoserTree->param); - tfree(pLocalMerge->pLoserTree); + pLocalMerger->numOfCompleted = 0; + + if (pLocalMerger->pLoserTree) { + tfree(pLocalMerger->pLoserTree->param); + tfree(pLocalMerger->pLoserTree); } - tscLocalReducerEnvDestroy(pLocalMerge->pExtMemBuffer, pLocalMerge->pDesc, pLocalMerge->resColModel, - pLocalMerge->finalModel, pLocalMerge->numOfVnode); - for (int32_t i = 0; i < pLocalMerge->numOfBuffer; ++i) { - tfree(pLocalMerge->pLocalDataSrc[i]); - } - - pLocalMerge->numOfBuffer = 0; - pLocalMerge->numOfCompleted = 0; - tfree(pLocalMerge->pTempBuffer); - - free(pLocalMerge); - - tscDebug("0x%"PRIx64" free local reducer finished", pSql->self); + tfree(pLocalMerger->buf); + tfree(pLocalMerger->pLocalDataSrc); + free(pLocalMerger); } -static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCmd, SColumnModel *pModel) { - int32_t numOfGroupByCols = 0; - SQueryInfo *pQueryInfo = tscGetActiveQueryInfo(pCmd); +static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SQueryInfo* pQueryInfo, SColumnModel *pModel) { + int32_t numOfGroupByCols = 0; if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) { numOfGroupByCols = pQueryInfo->groupbyExpr.numOfGroupCols; @@ -473,13 +341,13 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm if (numOfGroupByCols > 0) { if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) { - int32_t numOfInternalOutput = (int32_t) tscSqlExprNumOfExprs(pQueryInfo); + int32_t numOfInternalOutput = (int32_t) tscNumOfExprs(pQueryInfo); // the last "pQueryInfo->groupbyExpr.numOfGroupCols" columns are order-by columns for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, i); for(int32_t j = 0; j < numOfInternalOutput; ++j) { - SExprInfo* pExprInfo = tscSqlExprGet(pQueryInfo, j); + SExprInfo* pExprInfo = tscExprGet(pQueryInfo, j); int32_t functionId = pExprInfo->base.functionId; if (pColIndex->colId == pExprInfo->base.colInfo.colId && (functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TAG)) { @@ -501,9 +369,9 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm if (pQueryInfo->interval.interval != 0) { orderColIndexList[0] = PRIMARYKEY_TIMESTAMP_COL_INDEX; } else { - size_t size = tscSqlExprNumOfExprs(pQueryInfo); + size_t size = tscNumOfExprs(pQueryInfo); for (int32_t i = 0; i < size; ++i) { - SExprInfo *pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo *pExpr = tscExprGet(pQueryInfo, i); if (pExpr->base.functionId == TSDB_FUNC_PRJ && pExpr->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { orderColIndexList[0] = i; } @@ -524,37 +392,30 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm } } -int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOrderDescriptor **pOrderDesc, - SColumnModel **pFinalModel, SColumnModel** pFFModel, uint32_t nBufferSizes) { - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - SSchema * pSchema = NULL; +int32_t tscLocalReducerEnvCreate(SQueryInfo *pQueryInfo, tExtMemBuffer ***pMemBuffer, int32_t numOfSub, + tOrderDescriptor **pOrderDesc, uint32_t nBufferSizes, int64_t id) { + SSchema *pSchema = NULL; SColumnModel *pModel = NULL; - *pFinalModel = NULL; - SQueryInfo * pQueryInfo = tscGetActiveQueryInfo(pCmd); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pSql->subState.numOfSub); + (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * numOfSub); if (*pMemBuffer == NULL) { - tscError("0x%"PRIx64" failed to allocate memory", pSql->self); - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return pRes->code; + tscError("0x%"PRIx64" failed to allocate memory", id); + return TSDB_CODE_TSC_OUT_OF_MEMORY; } - size_t size = tscSqlExprNumOfExprs(pQueryInfo); + size_t size = tscNumOfExprs(pQueryInfo); pSchema = (SSchema *)calloc(1, sizeof(SSchema) * size); if (pSchema == NULL) { - tscError("0x%"PRIx64" failed to allocate memory", pSql->self); - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return pRes->code; + tscError("0x%"PRIx64" failed to allocate memory", id); + return TSDB_CODE_TSC_OUT_OF_MEMORY; } int32_t rlen = 0; for (int32_t i = 0; i < size; ++i) { - SExprInfo *pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo *pExpr = tscExprGet(pQueryInfo, i); pSchema[i].bytes = pExpr->base.resBytes; pSchema[i].type = (int8_t)pExpr->base.resType; @@ -569,6 +430,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr } pModel = createColumnModel(pSchema, (int32_t)size, capacity); + tfree(pSchema); int32_t pg = DEFAULT_PAGE_SIZE; int32_t overhead = sizeof(tFilePage); @@ -576,100 +438,26 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr pg *= 2; } - size_t numOfSubs = pSql->subState.numOfSub; - assert(numOfSubs <= pTableMetaInfo->vgroupList->numOfVgroups); - for (int32_t i = 0; i < numOfSubs; ++i) { + assert(numOfSub <= pTableMetaInfo->vgroupList->numOfVgroups); + for (int32_t i = 0; i < numOfSub; ++i) { (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pg, pModel); (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL; } - if (createOrderDescriptor(pOrderDesc, pCmd, pModel) != TSDB_CODE_SUCCESS) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tfree(pSchema); - return pRes->code; + if (createOrderDescriptor(pOrderDesc, pQueryInfo, pModel) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; } - // final result depends on the fields number - memset(pSchema, 0, sizeof(SSchema) * size); - - for (int32_t i = 0; i < size; ++i) { - SExprInfo *pExpr = tscSqlExprGet(pQueryInfo, i); - - SSchema p1 = {0}; - if (pExpr->base.colInfo.colIndex == TSDB_TBNAME_COLUMN_INDEX) { - p1 = *tGetTbnameColumnSchema(); - } else if (TSDB_COL_IS_UD_COL(pExpr->base.colInfo.flag)) { - p1.bytes = pExpr->base.resBytes; - p1.type = (uint8_t) pExpr->base.resType; - tstrncpy(p1.name, pExpr->base.aliasName, tListLen(p1.name)); - } else { - p1 = *tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->base.colInfo.colIndex); - } - - int32_t inter = 0; - int16_t type = -1; - int16_t bytes = 0; - - // the final result size and type in the same as query on single table. - // so here, set the flag to be false; - int32_t functionId = pExpr->base.functionId; - if (functionId >= TSDB_FUNC_TS && functionId <= TSDB_FUNC_DIFF) { - type = pModel->pFields[i].field.type; - bytes = pModel->pFields[i].field.bytes; - } else if (functionId < 0) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); - int32_t ret = getResultDataInfo(p1.type, p1.bytes, functionId, 0, &type, &bytes, &inter, 0, false, pUdfInfo); - assert(ret == TSDB_CODE_SUCCESS); - - } else { - if (functionId == TSDB_FUNC_FIRST_DST) { - functionId = TSDB_FUNC_FIRST; - } else if (functionId == TSDB_FUNC_LAST_DST) { - functionId = TSDB_FUNC_LAST; - } else if (functionId == TSDB_FUNC_STDDEV_DST) { - functionId = TSDB_FUNC_STDDEV; - } - - int32_t ret = getResultDataInfo(p1.type, p1.bytes, functionId, 0, &type, &bytes, &inter, 0, false, NULL); - assert(ret == TSDB_CODE_SUCCESS); - } - - pSchema[i].type = (uint8_t)type; - pSchema[i].bytes = bytes; - strcpy(pSchema[i].name, pModel->pFields[i].field.name); - } - - *pFinalModel = createColumnModel(pSchema, (int32_t)size, capacity); - - memset(pSchema, 0, sizeof(SSchema) * size); - size = tscNumOfFields(pQueryInfo); - - for(int32_t i = 0; i < size; ++i) { - SInternalField* pField = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, i); - pSchema[i].bytes = pField->field.bytes; - pSchema[i].type = pField->field.type; - tstrncpy(pSchema[i].name, pField->field.name, tListLen(pSchema[i].name)); - } - - *pFFModel = createColumnModel(pSchema, (int32_t) size, capacity); - - tfree(pSchema); return TSDB_CODE_SUCCESS; } /** * @param pMemBuffer * @param pDesc - * @param pFinalModel * @param numOfVnodes */ -void tscLocalReducerEnvDestroy(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDesc, SColumnModel *pFinalModel, SColumnModel *pFFModel, - int32_t numOfVnodes) { - destroyColumnModel(pFinalModel); - destroyColumnModel(pFFModel); - +void tscLocalReducerEnvDestroy(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDesc, int32_t numOfVnodes) { tOrderDescDestroy(pDesc); - for (int32_t i = 0; i < numOfVnodes; ++i) { pMemBuffer[i] = destoryExtMemBuffer(pMemBuffer[i]); } @@ -804,12 +592,12 @@ static void doExecuteFinalMergeRv(SOperatorInfo* pOperator, int32_t numOfExpr, S if (functionId < 0) { SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1); - + doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_MERGE); continue; } - + aAggs[functionId].mergeFunc(&pCtx[j]); } } else { @@ -821,12 +609,12 @@ static void doExecuteFinalMergeRv(SOperatorInfo* pOperator, int32_t numOfExpr, S if (functionId < 0) { SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1); - + doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); continue; } - + aAggs[functionId].xFinalize(&pCtx[j]); } @@ -846,7 +634,7 @@ static void doExecuteFinalMergeRv(SOperatorInfo* pOperator, int32_t numOfExpr, S if (pCtx[j].functionId < 0) { continue; } - + aAggs[pCtx[j].functionId].init(&pCtx[j], pCtx[j].resultInfo); } @@ -862,12 +650,12 @@ static void doExecuteFinalMergeRv(SOperatorInfo* pOperator, int32_t numOfExpr, S if (functionId < 0) { SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1); - + doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_MERGE); continue; } - + aAggs[functionId].mergeFunc(&pCtx[j]); } } @@ -884,12 +672,12 @@ static void doExecuteFinalMergeRv(SOperatorInfo* pOperator, int32_t numOfExpr, S if (functionId < 0) { SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1); - + doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_MERGE); continue; } - + aAggs[functionId].mergeFunc(&pCtx[j]); } } @@ -921,10 +709,12 @@ static bool isAllSourcesCompleted(SLocalMerger *pLocalMerge) { return (pLocalMerge->numOfBuffer == pLocalMerge->numOfCompleted); } -void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen) { - SSqlRes *pRes = &pObj->res; +void tscInitResObjForLocalQuery(SSqlObj *pSql, int32_t numOfRes, int32_t rowLen) { + SSqlRes *pRes = &pSql->res; if (pRes->pLocalMerger != NULL) { - tscDestroyLocalMerger(pObj); + tscDestroyLocalMerger(pRes->pLocalMerger); + pRes->pLocalMerger = NULL; + tscDebug("0x%"PRIx64" free local reducer finished", pSql->self); } pRes->qId = 1; // hack to pass the safety check in fetch_row function @@ -935,14 +725,12 @@ void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen) pRes->pLocalMerger = (SLocalMerger *)calloc(1, sizeof(SLocalMerger)); /* - * we need one additional byte space - * the sprintf function needs one additional space to put '\0' at the end of string + * One more byte space is required, since the sprintf function needs one additional space to put '\0' at + * the end of string */ - size_t allocSize = numOfRes * rowLen + sizeof(tFilePage) + 1; - pRes->pLocalMerger->pResultBuf = (tFilePage *)calloc(1, allocSize); - - pRes->pLocalMerger->pResultBuf->num = numOfRes; - pRes->data = pRes->pLocalMerger->pResultBuf->data; + size_t size = numOfRes * rowLen + 1; + pRes->pLocalMerger->buf = calloc(1, size); + pRes->data = pRes->pLocalMerger->buf; } int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize) { @@ -954,12 +742,12 @@ int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_ // todo refactor arithSup.offset = 0; - arithSup.numOfCols = (int32_t) tscSqlExprNumOfExprs(pQueryInfo); + arithSup.numOfCols = (int32_t) tscNumOfExprs(pQueryInfo); arithSup.exprList = pQueryInfo->exprList; arithSup.data = calloc(arithSup.numOfCols, POINTER_BYTES); for(int32_t k = 0; k < arithSup.numOfCols; ++k) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, k); + SExprInfo* pExpr = tscExprGet(pQueryInfo, k); arithSup.data[k] = (pOutput->data + pOutput->num* pExpr->base.offset); } @@ -988,8 +776,8 @@ int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_ return offset; } -#define COLMODEL_GET_VAL(data, schema, allrow, rowId, colId) \ - (data + (schema)->pFields[colId].offset * (allrow) + (rowId) * (schema)->pFields[colId].field.bytes) +#define COLMODEL_GET_VAL(data, schema, rowId, colId) \ + (data + (schema)->pFields[colId].offset * ((schema)->capacity) + (rowId) * (schema)->pFields[colId].field.bytes) static void appendOneRowToDataBlock(SSDataBlock *pBlock, char *buf, SColumnModel *pModel, int32_t rowIndex, int32_t maxRows) { @@ -997,7 +785,7 @@ static void appendOneRowToDataBlock(SSDataBlock *pBlock, char *buf, SColumnModel SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); char* p = pColInfo->pData + pBlock->info.rows * pColInfo->info.bytes; - char *src = COLMODEL_GET_VAL(buf, pModel, maxRows, rowIndex, i); + char *src = COLMODEL_GET_VAL(buf, pModel, rowIndex, i); memmove(p, src, pColInfo->info.bytes); } @@ -1014,8 +802,6 @@ SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup) { SLocalMerger *pMerger = pInfo->pMerge; SLoserTreeInfo *pTree = pMerger->pLoserTree; - SColumnModel *pModel = pMerger->pDesc->pColumnModel; - tFilePage *tmpBuffer = pMerger->pTempBuffer; pInfo->binfo.pRes->info.rows = 0; @@ -1028,7 +814,7 @@ SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup) { printf("chosen data in pTree[0] = %d\n", pTree->pNode[0].index); #endif - assert((pTree->pNode[0].index < pMerger->numOfBuffer) && (pTree->pNode[0].index >= 0) && tmpBuffer->num == 0); + assert((pTree->pNode[0].index < pMerger->numOfBuffer) && (pTree->pNode[0].index >= 0)); // chosen from loser tree SLocalDataSource *pOneDataSrc = pMerger->pLocalDataSrc[pTree->pNode[0].index]; @@ -1041,11 +827,10 @@ SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup) { SColIndex * pIndex = taosArrayGet(pInfo->orderColumnList, i); SColumnInfoData *pColInfo = taosArrayGet(pInfo->binfo.pRes->pDataBlock, pIndex->colIndex); - char *newRow = - COLMODEL_GET_VAL(pOneDataSrc->filePage.data, pModel, pOneDataSrc->pMemBuffer->pColumnModel->capacity, - pOneDataSrc->rowIdx, pIndex->colIndex); + char *newRow = COLMODEL_GET_VAL(pOneDataSrc->filePage.data, pOneDataSrc->pMemBuffer->pColumnModel, + pOneDataSrc->rowIdx, pIndex->colIndex); - char * data = pInfo->prevRow[i]; + char *data = pInfo->prevRow[i]; int32_t ret = columnValueAscendingComparator(data, newRow, pColInfo->info.type, pColInfo->info.bytes); if (ret == 0) { continue; @@ -1064,9 +849,8 @@ SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup) { SColIndex * pIndex = taosArrayGet(pInfo->orderColumnList, i); SColumnInfoData *pColInfo = taosArrayGet(pInfo->binfo.pRes->pDataBlock, pIndex->colIndex); - char *curCol = - COLMODEL_GET_VAL(pOneDataSrc->filePage.data, pModel, pOneDataSrc->pMemBuffer->pColumnModel->capacity, - pOneDataSrc->rowIdx, pIndex->colIndex); + char *curCol = COLMODEL_GET_VAL(pOneDataSrc->filePage.data, pOneDataSrc->pMemBuffer->pColumnModel, + pOneDataSrc->rowIdx, pIndex->colIndex); memcpy(pInfo->prevRow[i], curCol, pColInfo->info.bytes); } @@ -1077,7 +861,8 @@ SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup) { return pInfo->binfo.pRes; } - appendOneRowToDataBlock(pInfo->binfo.pRes, pOneDataSrc->filePage.data, pModel, pOneDataSrc->rowIdx, pOneDataSrc->pMemBuffer->pColumnModel->capacity); + appendOneRowToDataBlock(pInfo->binfo.pRes, pOneDataSrc->filePage.data, pOneDataSrc->pMemBuffer->pColumnModel, + pOneDataSrc->rowIdx, pOneDataSrc->pMemBuffer->pColumnModel->capacity); #if defined(_DEBUG_VIEW) printf("chosen row:\t"); @@ -1126,7 +911,7 @@ SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) { } SMultiwayMergeInfo *pAggInfo = pOperator->info; - SOperatorInfo *upstream = pOperator->upstream; + SOperatorInfo *upstream = pOperator->upstream[0]; *newgroup = false; bool handleData = false; @@ -1147,7 +932,7 @@ SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) { clearOutputBuf(&pAggInfo->binfo, &pAggInfo->bufCapacity); continue; } - + aAggs[pCtx->functionId].init(pCtx, pCtx->resultInfo); } } @@ -1201,9 +986,9 @@ SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) { if (functionId < 0) { SUdfInfo* pUdfInfo = taosArrayGet(pAggInfo->udfInfo, -1 * functionId - 1); - + doInvokeUdf(pUdfInfo, &pAggInfo->binfo.pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); - + continue; } @@ -1223,7 +1008,6 @@ SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) { if (pInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP && pRes->info.rows > 0) { STimeWindow* w = &pRes->info.window; - // TODO in case of desc order, swap it w->skey = *(int64_t*)pInfoData->pData; w->ekey = *(int64_t*)(((char*)pInfoData->pData) + TSDB_KEYSIZE * (pRes->info.rows - 1)); @@ -1243,7 +1027,7 @@ static SSDataBlock* skipGroupBlock(SOperatorInfo* pOperator, bool* newgroup) { SSDataBlock* pBlock = NULL; if (pInfo->currentGroupOffset == 0) { - pBlock = pOperator->upstream->exec(pOperator->upstream, newgroup); + pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); if (pBlock == NULL) { setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); pOperator->status = OP_EXEC_DONE; @@ -1251,7 +1035,7 @@ static SSDataBlock* skipGroupBlock(SOperatorInfo* pOperator, bool* newgroup) { if (*newgroup == false && pInfo->limit.limit > 0 && pInfo->rowsTotal >= pInfo->limit.limit) { while ((*newgroup) == false) { // ignore the remain blocks - pBlock = pOperator->upstream->exec(pOperator->upstream, newgroup); + pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); if (pBlock == NULL) { setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); pOperator->status = OP_EXEC_DONE; @@ -1263,7 +1047,7 @@ static SSDataBlock* skipGroupBlock(SOperatorInfo* pOperator, bool* newgroup) { return pBlock; } - pBlock = pOperator->upstream->exec(pOperator->upstream, newgroup); + pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); if (pBlock == NULL) { setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); pOperator->status = OP_EXEC_DONE; @@ -1277,7 +1061,7 @@ static SSDataBlock* skipGroupBlock(SOperatorInfo* pOperator, bool* newgroup) { } while ((*newgroup) == false) { - pBlock = pOperator->upstream->exec(pOperator->upstream, newgroup); + pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); if (pBlock == NULL) { setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); pOperator->status = OP_EXEC_DONE; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index d96e25dd37..cc89fd6220 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -107,7 +107,7 @@ int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int1 } if (parseAbsoluteDuration(valueToken.z, valueToken.n, &interval) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } if (timePrec == TSDB_TIME_PRECISION_MILLI) { @@ -441,7 +441,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, SSqlCmd *pCmd, int1 *str += index; if (sToken.type == TK_QUESTION) { - if (pCmd->insertType != TSDB_QUERY_TYPE_STMT_INSERT) { + if (pCmd->insertParam.insertType != TSDB_QUERY_TYPE_STMT_INSERT) { return tscSQLSyntaxErrMsg(pCmd->payload, "? only allowed in binding insertion", *str); } @@ -647,7 +647,7 @@ static int32_t tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, pBlocks->sversion = pTableMeta->sversion; if (pBlocks->numOfRows + numOfRows >= INT16_MAX) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } else { pBlocks->numOfRows += numOfRows; return TSDB_CODE_SUCCESS; @@ -708,7 +708,7 @@ static int32_t doParseInsertStatement(SSqlCmd* pCmd, char **str, STableDataBlock return TSDB_CODE_TSC_OUT_OF_MEMORY; } - code = TSDB_CODE_TSC_INVALID_SQL; + code = TSDB_CODE_TSC_INVALID_OPERATION; char tmpTokenBuf[16*1024] = {0}; // used for deleting Escape character: \\, \', \" int32_t numOfRows = 0; @@ -747,12 +747,10 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC const int32_t STABLE_INDEX = 1; SSqlCmd * pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); char *sql = *sqlstr; - pSql->cmd.autoCreated = false; - // get the token of specified table index = 0; tableToken = tStrGetToken(sql, &index, false); @@ -786,7 +784,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC } if (numOfColList == 0 && (*boundColumn) != NULL) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, TABLE_INDEX); @@ -802,7 +800,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC } STableMetaInfo *pSTableMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); - code = tscSetTableFullName(pSTableMetaInfo, &sToken, pSql); + code = tscSetTableFullName(&pSTableMetaInfo->name, &sToken, pSql); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -879,7 +877,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC if (TK_ILLEGAL == sToken.type) { tdDestroyKVRowBuilder(&kvRowBuilder); tscDestroyBoundColumnInfo(&spd); - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } if (sToken.n == 0 || sToken.type == TK_RP) { @@ -961,7 +959,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC } if (numOfColsAfterTags == 0 && (*boundColumn) != NULL) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } sToken = tStrGetToken(sql, &index, false); @@ -973,13 +971,13 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC return tscInvalidSQLErrMsg(pCmd->payload, "invalid table name", *sqlstr); } - int32_t ret = tscSetTableFullName(pTableMetaInfo, &tableToken, pSql); + int32_t ret = tscSetTableFullName(&pTableMetaInfo->name, &tableToken, pSql); if (ret != TSDB_CODE_SUCCESS) { return ret; } if (sql == NULL) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } code = tscGetTableMetaEx(pSql, pTableMetaInfo, true); @@ -991,7 +989,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC sql = sToken.z; if (sql == NULL) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } code = tscGetTableMetaEx(pSql, pTableMetaInfo, false); @@ -1015,12 +1013,17 @@ int validateTableName(char *tblName, int len, SStrToken* psTblToken) { return tscValidateName(psTblToken); } -static int32_t validateDataSource(SSqlCmd *pCmd, int8_t type, const char *sql) { - if (pCmd->dataSourceType != 0 && pCmd->dataSourceType != type) { - return tscInvalidSQLErrMsg(pCmd->payload, "keyword VALUES and FILE are not allowed to mix up", sql); +static int32_t validateDataSource(SSqlCmd *pCmd, int32_t type, const char *sql) { + uint32_t *insertType = &pCmd->insertParam.insertType; + if (*insertType == TSDB_QUERY_TYPE_STMT_INSERT && type == TSDB_QUERY_TYPE_INSERT) { + return TSDB_CODE_SUCCESS; } - pCmd->dataSourceType = type; + if ((*insertType) != 0 && (*insertType) != type) { + return tscInvalidSQLErrMsg(pCmd->payload, "keyword VALUES and FILE are not allowed to mixed up", sql); + } + + *insertType = type; return TSDB_CODE_SUCCESS; } @@ -1090,7 +1093,6 @@ static int32_t parseBoundColumns(SSqlCmd* pCmd, SParsedDataColInfo* pColInfo, SS _clean: pCmd->curSql = NULL; - pCmd->parseFinished = 1; return code; } @@ -1106,7 +1108,7 @@ int tsParseInsertSql(SSqlObj *pSql) { int32_t totalNum = 0; int32_t code = TSDB_CODE_SUCCESS; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); assert(pQueryInfo != NULL); STableMetaInfo *pTableMetaInfo = (pQueryInfo->numOfTables == 0)? tscAddEmptyMetaInfo(pQueryInfo):tscGetMetaInfo(pQueryInfo, 0); @@ -1120,9 +1122,9 @@ int tsParseInsertSql(SSqlObj *pSql) { return code; } - if (NULL == pCmd->pTableBlockHashList) { - pCmd->pTableBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - if (NULL == pCmd->pTableBlockHashList) { + if (NULL == pCmd->insertParam.pTableBlockHashList) { + pCmd->insertParam.pTableBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + if (NULL == pCmd->insertParam.pTableBlockHashList) { code = TSDB_CODE_TSC_OUT_OF_MEMORY; goto _clean; } @@ -1130,7 +1132,7 @@ int tsParseInsertSql(SSqlObj *pSql) { str = pCmd->curSql; } - tscDebug("0x%"PRIx64" create data block list hashList:%p", pSql->self, pCmd->pTableBlockHashList); + tscDebug("0x%"PRIx64" create data block list hashList:%p", pSql->self, pCmd->insertParam.pTableBlockHashList); while (1) { int32_t index = 0; @@ -1142,7 +1144,7 @@ int tsParseInsertSql(SSqlObj *pSql) { * if the data is from the data file, no data has been generated yet. So, there no data to * merge or submit, save the file path and parse the file in other routines. */ - if (pCmd->dataSourceType == DATA_FROM_DATA_FILE) { + if (TSDB_QUERY_HAS_TYPE(pCmd->insertParam.insertType, TSDB_QUERY_TYPE_FILE_INSERT)) { goto _clean; } @@ -1151,7 +1153,7 @@ int tsParseInsertSql(SSqlObj *pSql) { * Otherwise, create the first submit block and submit to virtual node. */ if (totalNum == 0) { - code = TSDB_CODE_TSC_INVALID_SQL; + code = TSDB_CODE_TSC_INVALID_OPERATION; goto _clean; } else { break; @@ -1168,7 +1170,7 @@ int tsParseInsertSql(SSqlObj *pSql) { goto _clean; } - if ((code = tscSetTableFullName(pTableMetaInfo, &sTblToken, pSql)) != TSDB_CODE_SUCCESS) { + if ((code = tscSetTableFullName(&pTableMetaInfo->name, &sTblToken, pSql)) != TSDB_CODE_SUCCESS) { goto _clean; } @@ -1203,7 +1205,7 @@ int tsParseInsertSql(SSqlObj *pSql) { STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); if (sToken.type == TK_FILE) { - if (validateDataSource(pCmd, DATA_FROM_DATA_FILE, sToken.z) != TSDB_CODE_SUCCESS) { + if (validateDataSource(pCmd, TSDB_QUERY_TYPE_FILE_INSERT, sToken.z) != TSDB_CODE_SUCCESS) { goto _clean; } @@ -1236,12 +1238,12 @@ int tsParseInsertSql(SSqlObj *pSql) { if (bindedColumns == NULL) { STableMeta *pTableMeta = pTableMetaInfo->pTableMeta; - if (validateDataSource(pCmd, DATA_FROM_SQL_STRING, sToken.z) != TSDB_CODE_SUCCESS) { + if (validateDataSource(pCmd, TSDB_QUERY_TYPE_INSERT, sToken.z) != TSDB_CODE_SUCCESS) { goto _clean; } STableDataBlocks *dataBuf = NULL; - int32_t ret = tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_DEFAULT_PAYLOAD_SIZE, + int32_t ret = tscGetDataBlockFromList(pCmd->insertParam.pTableBlockHashList, pTableMeta->id.uid, TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), tinfo.rowSize, &pTableMetaInfo->name, pTableMeta, &dataBuf, NULL); if (ret != TSDB_CODE_SUCCESS) { @@ -1254,14 +1256,14 @@ int tsParseInsertSql(SSqlObj *pSql) { } } else { // bindedColumns != NULL // insert into tablename(col1, col2,..., coln) values(v1, v2,... vn); - STableMeta *pTableMeta = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0)->pTableMeta; + STableMeta *pTableMeta = tscGetTableMetaInfoFromCmd(pCmd, 0)->pTableMeta; - if (validateDataSource(pCmd, DATA_FROM_SQL_STRING, sToken.z) != TSDB_CODE_SUCCESS) { + if (validateDataSource(pCmd, TSDB_QUERY_TYPE_INSERT, sToken.z) != TSDB_CODE_SUCCESS) { goto _clean; } STableDataBlocks *dataBuf = NULL; - int32_t ret = tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_DEFAULT_PAYLOAD_SIZE, + int32_t ret = tscGetDataBlockFromList(pCmd->insertParam.pTableBlockHashList, pTableMeta->id.uid, TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), tinfo.rowSize, &pTableMetaInfo->name, pTableMeta, &dataBuf, NULL); if (ret != TSDB_CODE_SUCCESS) { @@ -1297,7 +1299,8 @@ int tsParseInsertSql(SSqlObj *pSql) { goto _clean; } - if ((pCmd->insertType != TSDB_QUERY_TYPE_STMT_INSERT) && taosHashGetSize(pCmd->pTableBlockHashList) > 0) { // merge according to vgId + // merge according to vgId + if (!TSDB_QUERY_HAS_TYPE(pCmd->insertParam.insertType, TSDB_QUERY_TYPE_STMT_INSERT) && taosHashGetSize(pCmd->insertParam.pTableBlockHashList) > 0) { if ((code = tscMergeTableDataBlocks(pSql, true)) != TSDB_CODE_SUCCESS) { goto _clean; } @@ -1308,7 +1311,6 @@ int tsParseInsertSql(SSqlObj *pSql) { _clean: pCmd->curSql = NULL; - pCmd->parseFinished = 1; return code; } @@ -1326,9 +1328,8 @@ int tsInsertInitialCheck(SSqlObj *pSql) { pCmd->count = 0; pCmd->command = TSDB_SQL_INSERT; - SQueryInfo *pQueryInfo = tscGetQueryInfoS(pCmd, pCmd->clauseIndex); - - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT | pCmd->insertType); + SQueryInfo *pQueryInfo = tscGetQueryInfoS(pCmd); + TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT); sToken = tStrGetToken(pSql->sqlstr, &index, false); if (sToken.type != TK_INTO) { @@ -1343,11 +1344,11 @@ int tsParseSql(SSqlObj *pSql, bool initial) { int32_t ret = TSDB_CODE_SUCCESS; SSqlCmd* pCmd = &pSql->cmd; - if ((!pCmd->parseFinished) && (!initial)) { + if (!initial) { tscDebug("0x%"PRIx64" resume to parse sql: %s", pSql->self, pCmd->curSql); } - ret = tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE); + ret = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); if (TSDB_CODE_SUCCESS != ret) { return ret; } @@ -1357,31 +1358,32 @@ int tsParseSql(SSqlObj *pSql, bool initial) { return ret; } - // make a backup as tsParseInsertSql may modify the string - char* sqlstr = strdup(pSql->sqlstr); ret = tsParseInsertSql(pSql); - if ((sqlstr == NULL) || (pSql->parseRetry >= 1) || - (ret != TSDB_CODE_TSC_SQL_SYNTAX_ERROR && ret != TSDB_CODE_TSC_INVALID_SQL)) { - free(sqlstr); - } else { + assert(ret == TSDB_CODE_SUCCESS || ret == TSDB_CODE_TSC_ACTION_IN_PROGRESS || ret == TSDB_CODE_TSC_SQL_SYNTAX_ERROR || ret == TSDB_CODE_TSC_INVALID_OPERATION); + + if (pSql->parseRetry < 1 && (ret == TSDB_CODE_TSC_SQL_SYNTAX_ERROR || ret == TSDB_CODE_TSC_INVALID_OPERATION)) { + tscDebug("0x%"PRIx64 " parse insert sql statement failed, code:%s, clear meta cache and retry ", pSql->self, tstrerror(ret)); + tscResetSqlCmd(pCmd, true); - free(pSql->sqlstr); - pSql->sqlstr = sqlstr; pSql->parseRetry++; + if ((ret = tsInsertInitialCheck(pSql)) == TSDB_CODE_SUCCESS) { ret = tsParseInsertSql(pSql); } } } else { - SSqlInfo SQLInfo = qSqlParse(pSql->sqlstr); - ret = tscToSQLCmd(pSql, &SQLInfo); - if (ret == TSDB_CODE_TSC_INVALID_SQL && pSql->parseRetry == 0 && SQLInfo.type == TSDB_SQL_NULL) { + SSqlInfo sqlInfo = qSqlParse(pSql->sqlstr); + ret = tscValidateSqlInfo(pSql, &sqlInfo); + if (ret == TSDB_CODE_TSC_INVALID_OPERATION && pSql->parseRetry < 1 && sqlInfo.type == TSDB_SQL_SELECT) { + tscDebug("0x%"PRIx64 " parse query sql statement failed, code:%s, clear meta cache and retry ", pSql->self, tstrerror(ret)); + tscResetSqlCmd(pCmd, true); pSql->parseRetry++; - ret = tscToSQLCmd(pSql, &SQLInfo); + + ret = tscValidateSqlInfo(pSql, &sqlInfo); } - SqlInfoDestroy(&SQLInfo); + SqlInfoDestroy(&sqlInfo); } /* @@ -1398,8 +1400,7 @@ static int doPackSendDataBlock(SSqlObj *pSql, int32_t numOfRows, STableDataBlock SSqlCmd *pCmd = &pSql->cmd; pSql->res.numOfRows = 0; - assert(pCmd->numOfClause == 1); - STableMeta *pTableMeta = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0)->pTableMeta; + STableMeta *pTableMeta = tscGetTableMetaInfoFromCmd(pCmd, 0)->pTableMeta; SSubmitBlk *pBlocks = (SSubmitBlk *)(pTableDataBlocks->pData); code = tsSetBlockInfo(pBlocks, pTableMeta, numOfRows); @@ -1411,7 +1412,7 @@ static int doPackSendDataBlock(SSqlObj *pSql, int32_t numOfRows, STableDataBlock return code; } - STableDataBlocks *pDataBlock = taosArrayGetP(pCmd->pDataBlocks, 0); + STableDataBlocks *pDataBlock = taosArrayGetP(pCmd->insertParam.pDataBlocks, 0); if ((code = tscCopyDataBlockToPayload(pSql, pDataBlock)) != TSDB_CODE_SUCCESS) { return code; } @@ -1461,17 +1462,17 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow // accumulate the total submit records pParentSql->res.numOfRows += pSql->res.numOfRows; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; STableComInfo tinfo = tscGetTableInfo(pTableMeta); destroyTableNameList(pCmd); - pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); + pCmd->insertParam.pDataBlocks = tscDestroyBlockArrayList(pCmd->insertParam.pDataBlocks); - if (pCmd->pTableBlockHashList == NULL) { - pCmd->pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - if (pCmd->pTableBlockHashList == NULL) { + if (pCmd->insertParam.pTableBlockHashList == NULL) { + pCmd->insertParam.pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + if (pCmd->insertParam.pTableBlockHashList == NULL) { code = TSDB_CODE_TSC_OUT_OF_MEMORY; goto _error; } @@ -1479,7 +1480,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow STableDataBlocks *pTableDataBlock = NULL; int32_t ret = - tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), + tscGetDataBlockFromList(pCmd->insertParam.pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), tinfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pTableDataBlock, NULL); if (ret != TSDB_CODE_SUCCESS) { pParentSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -1561,8 +1562,8 @@ void tscImportDataFromFile(SSqlObj *pSql) { return; } - assert(pCmd->dataSourceType == DATA_FROM_DATA_FILE && strlen(pCmd->payload) != 0); - pCmd->active = pCmd->pQueryInfo[0]; + assert(TSDB_QUERY_HAS_TYPE(pCmd->insertParam.insertType, TSDB_QUERY_TYPE_FILE_INSERT) && strlen(pCmd->payload) != 0); + pCmd->active = pCmd->pQueryInfo; SImportFileSupport *pSupporter = calloc(1, sizeof(SImportFileSupport)); SSqlObj *pNew = createSubqueryObj(pSql, 0, parseFileSendDataBlock, pSupporter, TSDB_SQL_INSERT, NULL); diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 7794e3190c..b80bdabbfc 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -46,10 +46,14 @@ typedef struct SNormalStmt { typedef struct SMultiTbStmt { bool nameSet; + bool tagSet; uint64_t currentUid; uint32_t tbNum; SStrToken tbname; - SHashObj *pTableHash; + SStrToken stbname; + SStrToken values; + SArray *tags; + SHashObj *pTableHash; SHashObj *pTableBlockHashList; // data block for each table } SMultiTbStmt; @@ -283,9 +287,9 @@ static int fillColumnsNull(STableDataBlocks* pBlock, int32_t rowNum) { for (int32_t i = 0; i < spd->numOfCols; ++i) { if (!spd->cols[i].hasVal) { // current column do not have any value to insert, set it to null - for (int32_t n = 0; n < rowNum; ++n) { + for (int32_t n = 0; n < rowNum; ++n) { char *ptr = pBlock->pData + sizeof(SSubmitBlk) + pBlock->rowSize * n + offset; - + if (schema[i].type == TSDB_DATA_TYPE_BINARY) { varDataSetLen(ptr, sizeof(int8_t)); *(uint8_t*) varDataVal(ptr) = TSDB_DATA_BINARY_NULL; @@ -297,7 +301,7 @@ static int fillColumnsNull(STableDataBlocks* pBlock, int32_t rowNum) { } } } - + offset += schema[i].bytes; } @@ -308,7 +312,7 @@ static int fillColumnsNull(STableDataBlocks* pBlock, int32_t rowNum) { int32_t fillTablesColumnsNull(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; - STableDataBlocks** p = taosHashIterate(pCmd->pTableBlockHashList, NULL); + STableDataBlocks** p = taosHashIterate(pCmd->insertParam.pTableBlockHashList, NULL); STableDataBlocks* pOneTableBlock = *p; while(pOneTableBlock) { @@ -316,8 +320,8 @@ int32_t fillTablesColumnsNull(SSqlObj* pSql) { if (pBlocks->numOfRows > 0 && pOneTableBlock->boundColumnInfo.numOfBound < pOneTableBlock->boundColumnInfo.numOfCols) { fillColumnsNull(pOneTableBlock, pBlocks->numOfRows); } - - p = taosHashIterate(pCmd->pTableBlockHashList, p); + + p = taosHashIterate(pCmd->insertParam.pTableBlockHashList, p); if (p == NULL) { break; } @@ -775,7 +779,7 @@ static int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, return TSDB_CODE_TSC_INVALID_VALUE; } } - + return TSDB_CODE_SUCCESS; } @@ -800,7 +804,7 @@ static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MU if (!IS_VAR_DATA_TYPE(param->type)) { memcpy(data + param->offset, (char *)bind->buffer + bind->buffer_length * i, tDataTypes[param->type].bytes); - + if (param->offset == 0) { if (tsCheckTimestamp(pBlock, data + param->offset) != TSDB_CODE_SUCCESS) { tscError("invalid timestamp"); @@ -829,23 +833,23 @@ static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MU varDataSetLen(data + param->offset, output); } } - + return TSDB_CODE_SUCCESS; } static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { SSqlCmd* pCmd = &stmt->pSql->cmd; STscStmt* pStmt = (STscStmt*)stmt; - + STableDataBlocks* pBlock = NULL; - + if (pStmt->multiTbInsert) { - if (pCmd->pTableBlockHashList == NULL) { + if (pCmd->insertParam.pTableBlockHashList == NULL) { tscError("0x%"PRIx64" Table block hash list is empty", pStmt->pSql->self); return TSDB_CODE_TSC_APP_ERROR; } - - STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pCmd->pTableBlockHashList, (const char*)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid)); + + STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pCmd->insertParam.pTableBlockHashList, (const char*)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid)); if (t1 == NULL) { tscError("0x%"PRIx64" no table data block in hash list, uid:%" PRId64 , pStmt->pSql->self, pStmt->mtb.currentUid); return TSDB_CODE_TSC_APP_ERROR; @@ -853,15 +857,15 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { pBlock = *t1; } else { - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0, 0); + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (pCmd->pTableBlockHashList == NULL) { - pCmd->pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + if (pCmd->insertParam.pTableBlockHashList == NULL) { + pCmd->insertParam.pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); } int32_t ret = - tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), + tscGetDataBlockFromList(pCmd->insertParam.pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); if (ret != 0) { return ret; @@ -900,16 +904,16 @@ static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_MULTI_BIND* bind, int c SSqlCmd* pCmd = &stmt->pSql->cmd; STscStmt* pStmt = (STscStmt*)stmt; int rowNum = bind->num; - + STableDataBlocks* pBlock = NULL; - + if (pStmt->multiTbInsert) { - if (pCmd->pTableBlockHashList == NULL) { + if (pCmd->insertParam.pTableBlockHashList == NULL) { tscError("0x%"PRIx64" Table block hash list is empty", pStmt->pSql->self); return TSDB_CODE_TSC_APP_ERROR; } - STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pCmd->pTableBlockHashList, (const char*)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid)); + STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pCmd->insertParam.pTableBlockHashList, (const char*)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid)); if (t1 == NULL) { tscError("0x%"PRIx64" no table data block in hash list, uid:%" PRId64 , pStmt->pSql->self, pStmt->mtb.currentUid); return TSDB_CODE_TSC_APP_ERROR; @@ -917,15 +921,15 @@ static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_MULTI_BIND* bind, int c pBlock = *t1; } else { - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0, 0); + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (pCmd->pTableBlockHashList == NULL) { - pCmd->pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + if (pCmd->insertParam.pTableBlockHashList == NULL) { + pCmd->insertParam.pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); } int32_t ret = - tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), + tscGetDataBlockFromList(pCmd->insertParam.pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); if (ret != 0) { return ret; @@ -954,7 +958,7 @@ static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_MULTI_BIND* bind, int c tscError("0x%"PRIx64" param %d: num[%d:%d] not match", pStmt->pSql->self, param->idx, rowNum, bind[param->idx].num); return TSDB_CODE_TSC_INVALID_VALUE; } - + int code = doBindBatchParam(pBlock, param, &bind[param->idx], pCmd->batchSize); if (code != TSDB_CODE_SUCCESS) { tscError("0x%"PRIx64" bind column %d: type mismatch or invalid", pStmt->pSql->self, param->idx); @@ -965,7 +969,7 @@ static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_MULTI_BIND* bind, int c pCmd->batchSize += rowNum - 1; } else { SParamInfo* param = &pBlock->params[colIdx]; - + int code = doBindBatchParam(pBlock, param, bind, pCmd->batchSize); if (code != TSDB_CODE_SUCCESS) { tscError("0x%"PRIx64" bind column %d: type mismatch or invalid", pStmt->pSql->self, param->idx); @@ -990,13 +994,12 @@ static int insertStmtUpdateBatch(STscStmt* stmt) { tscError("too many record:%d", pCmd->batchSize); return TSDB_CODE_TSC_APP_ERROR; } - - assert(pCmd->numOfClause == 1); - if (taosHashGetSize(pCmd->pTableBlockHashList) == 0) { + + if (taosHashGetSize(pCmd->insertParam.pTableBlockHashList) == 0) { return TSDB_CODE_SUCCESS; } - STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pCmd->pTableBlockHashList, (const char*)&stmt->mtb.currentUid, sizeof(stmt->mtb.currentUid)); + STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pCmd->insertParam.pTableBlockHashList, (const char*)&stmt->mtb.currentUid, sizeof(stmt->mtb.currentUid)); if (t1 == NULL) { tscError("0x%"PRIx64" no table data block in hash list, uid:%" PRId64 , pSql->self, stmt->mtb.currentUid); return TSDB_CODE_TSC_APP_ERROR; @@ -1019,11 +1022,11 @@ static int insertStmtUpdateBatch(STscStmt* stmt) { static int insertStmtAddBatch(STscStmt* stmt) { SSqlCmd* pCmd = &stmt->pSql->cmd; ++pCmd->batchSize; - + if (stmt->multiTbInsert) { return insertStmtUpdateBatch(stmt); } - + return TSDB_CODE_SUCCESS; } @@ -1032,9 +1035,9 @@ static int insertStmtReset(STscStmt* pStmt) { if (pCmd->batchSize > 2) { int32_t alloced = (pCmd->batchSize + 1) / 2; - size_t size = taosArrayGetSize(pCmd->pDataBlocks); + size_t size = taosArrayGetSize(pCmd->insertParam.pDataBlocks); for (int32_t i = 0; i < size; ++i) { - STableDataBlocks* pBlock = taosArrayGetP(pCmd->pDataBlocks, i); + STableDataBlocks* pBlock = taosArrayGetP(pCmd->insertParam.pDataBlocks, i); uint32_t totalDataSize = pBlock->size - sizeof(SSubmitBlk); pBlock->size = sizeof(SSubmitBlk) + totalDataSize / alloced; @@ -1045,7 +1048,7 @@ static int insertStmtReset(STscStmt* pStmt) { } pCmd->batchSize = 0; - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); pTableMetaInfo->vgroupIndex = 0; return TSDB_CODE_SUCCESS; } @@ -1056,22 +1059,21 @@ static int insertStmtExecute(STscStmt* stmt) { return TSDB_CODE_TSC_INVALID_VALUE; } - assert(pCmd->numOfClause == 1); - if (taosHashGetSize(pCmd->pTableBlockHashList) == 0) { + if (taosHashGetSize(pCmd->insertParam.pTableBlockHashList) == 0) { return TSDB_CODE_SUCCESS; } - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0, 0); + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (pCmd->pTableBlockHashList == NULL) { - pCmd->pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + if (pCmd->insertParam.pTableBlockHashList == NULL) { + pCmd->insertParam.pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); } STableDataBlocks* pBlock = NULL; int32_t ret = - tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), + tscGetDataBlockFromList(pCmd->insertParam.pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); assert(ret == 0); pBlock->size = sizeof(SSubmitBlk) + pCmd->batchSize * pBlock->rowSize; @@ -1088,7 +1090,7 @@ static int insertStmtExecute(STscStmt* stmt) { return code; } - STableDataBlocks* pDataBlock = taosArrayGetP(pCmd->pDataBlocks, 0); + STableDataBlocks* pDataBlock = taosArrayGetP(pCmd->insertParam.pDataBlocks, 0); code = tscCopyDataBlockToPayload(stmt->pSql, pDataBlock); if (code != TSDB_CODE_SUCCESS) { return code; @@ -1106,15 +1108,15 @@ static int insertStmtExecute(STscStmt* stmt) { // data block reset pCmd->batchSize = 0; - for(int32_t i = 0; i < pCmd->numOfTables; ++i) { - if (pCmd->pTableNameList && pCmd->pTableNameList[i]) { - tfree(pCmd->pTableNameList[i]); + for(int32_t i = 0; i < pCmd->insertParam.numOfTables; ++i) { + if (pCmd->insertParam.pTableNameList && pCmd->insertParam.pTableNameList[i]) { + tfree(pCmd->insertParam.pTableNameList[i]); } } - pCmd->numOfTables = 0; - tfree(pCmd->pTableNameList); - pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); + pCmd->insertParam.numOfTables = 0; + tfree(pCmd->insertParam.pTableNameList); + pCmd->insertParam.pDataBlocks = tscDestroyBlockArrayList(pCmd->insertParam.pDataBlocks); return pSql->res.code; } @@ -1122,32 +1124,32 @@ static int insertStmtExecute(STscStmt* stmt) { static void insertBatchClean(STscStmt* pStmt) { SSqlCmd *pCmd = &pStmt->pSql->cmd; SSqlObj *pSql = pStmt->pSql; - int32_t size = taosHashGetSize(pCmd->pTableBlockHashList); - + int32_t size = taosHashGetSize(pCmd->insertParam.pTableBlockHashList); + // data block reset pCmd->batchSize = 0; - + for(int32_t i = 0; i < size; ++i) { - if (pCmd->pTableNameList && pCmd->pTableNameList[i]) { - tfree(pCmd->pTableNameList[i]); + if (pCmd->insertParam.pTableNameList && pCmd->insertParam.pTableNameList[i]) { + tfree(pCmd->insertParam.pTableNameList[i]); } } - tfree(pCmd->pTableNameList); + tfree(pCmd->insertParam.pTableNameList); /* - STableDataBlocks** p = taosHashIterate(pCmd->pTableBlockHashList, NULL); + STableDataBlocks** p = taosHashIterate(pCmd->insertParam.pTableBlockHashList, NULL); STableDataBlocks* pOneTableBlock = *p; - while (1) { + while (1) { SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; - + pOneTableBlock->size = sizeof(SSubmitBlk); pBlocks->numOfRows = 0; - - p = taosHashIterate(pCmd->pTableBlockHashList, p); + + p = taosHashIterate(pCmd->insertParam.pTableBlockHashList, p); if (p == NULL) { break; } @@ -1156,27 +1158,27 @@ static void insertBatchClean(STscStmt* pStmt) { } */ - pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); - pCmd->numOfTables = 0; + pCmd->insertParam.pDataBlocks = tscDestroyBlockArrayList(pCmd->insertParam.pDataBlocks); + pCmd->insertParam.numOfTables = 0; - taosHashEmpty(pCmd->pTableBlockHashList); + taosHashEmpty(pCmd->insertParam.pTableBlockHashList); tscFreeSqlResult(pSql); tscFreeSubobj(pSql); tfree(pSql->pSubs); - pSql->subState.numOfSub = 0; + pSql->subState.numOfSub = 0; } static int insertBatchStmtExecute(STscStmt* pStmt) { int32_t code = 0; - + if(pStmt->mtb.nameSet == false) { tscError("0x%"PRIx64" no table name set", pStmt->pSql->self); return TSDB_CODE_TSC_APP_ERROR; } - + pStmt->pSql->retry = pStmt->pSql->maxRetry + 1; //no retry - if (taosHashGetSize(pStmt->pSql->cmd.pTableBlockHashList) <= 0) { // merge according to vgId + if (taosHashGetSize(pStmt->pSql->cmd.insertParam.pTableBlockHashList) <= 0) { // merge according to vgId tscError("0x%"PRIx64" no data block to insert", pStmt->pSql->self); return TSDB_CODE_TSC_APP_ERROR; } @@ -1192,15 +1194,193 @@ static int insertBatchStmtExecute(STscStmt* pStmt) { if (code != TSDB_CODE_SUCCESS) { return code; } - + // wait for the callback function to post the semaphore tsem_wait(&pStmt->pSql->rspSem); insertBatchClean(pStmt); - + return pStmt->pSql->res.code; } + +int stmtParseInsertTbTags(SSqlObj* pSql, STscStmt* pStmt) { + SSqlCmd *pCmd = &pSql->cmd; + int32_t ret = TSDB_CODE_SUCCESS; + + if ((ret = tsInsertInitialCheck(pSql)) != TSDB_CODE_SUCCESS) { + return ret; + } + + int32_t index = 0; + SStrToken sToken = tStrGetToken(pCmd->curSql, &index, false); + if (sToken.n == 0) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + if (sToken.n == 1 && sToken.type == TK_QUESTION) { + pStmt->multiTbInsert = true; + pStmt->mtb.tbname = sToken; + pStmt->mtb.nameSet = false; + if (pStmt->mtb.pTableHash == NULL) { + pStmt->mtb.pTableHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + } + + if (pStmt->mtb.pTableBlockHashList == NULL) { + pStmt->mtb.pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + } + + pStmt->mtb.tagSet = true; + + sToken = tStrGetToken(pCmd->curSql, &index, false); + if (sToken.n > 0 && sToken.type == TK_VALUES) { + return TSDB_CODE_SUCCESS; + } + + if (sToken.n <= 0 || sToken.type != TK_USING) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + sToken = tStrGetToken(pCmd->curSql, &index, false); + if (sToken.n <= 0 || ((sToken.type != TK_ID) && (sToken.type != TK_STRING))) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + pStmt->mtb.stbname = sToken; + + sToken = tStrGetToken(pCmd->curSql, &index, false); + if (sToken.n <= 0 || sToken.type != TK_TAGS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + sToken = tStrGetToken(pCmd->curSql, &index, false); + if (sToken.n <= 0 || sToken.type != TK_LP) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + pStmt->mtb.tags = taosArrayInit(4, sizeof(SStrToken)); + + int32_t loopCont = 1; + + while (loopCont) { + sToken = tStrGetToken(pCmd->curSql, &index, false); + if (sToken.n <= 0) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + switch (sToken.type) { + case TK_RP: + loopCont = 0; + break; + case TK_VALUES: + return TSDB_CODE_TSC_INVALID_OPERATION; + case TK_QUESTION: + pStmt->mtb.tagSet = false; //continue + default: + taosArrayPush(pStmt->mtb.tags, &sToken); + break; + } + } + + if (taosArrayGetSize(pStmt->mtb.tags) <= 0) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + sToken = tStrGetToken(pCmd->curSql, &index, false); + if (sToken.n <= 0 || sToken.type != TK_VALUES) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + pStmt->mtb.values = sToken; + } + + return TSDB_CODE_SUCCESS; +} + + + + +int stmtGenInsertStatement(SSqlObj* pSql, STscStmt* pStmt, const char* name, TAOS_BIND* tags) { + size_t tagNum = taosArrayGetSize(pStmt->mtb.tags); + size_t size = 1048576; + char *str = calloc(1, size); + size_t len = 0; + int32_t ret = 0; + int32_t j = 0; + + while (1) { + len = (size_t)snprintf(str, size - 1, "insert into %s using %.*s tags(", name, pStmt->mtb.stbname.n, pStmt->mtb.stbname.z); + if (len >= (size -1)) { + size *= 2; + free(str); + str = calloc(1, size); + continue; + } + + j = 0; + + for (size_t i = 0; i < tagNum && len < (size - 1); ++i) { + SStrToken *t = taosArrayGet(pStmt->mtb.tags, i); + if (t->type == TK_QUESTION) { + int32_t l = 0; + if (i > 0) { + str[len++] = ','; + } + + if (tags[j].is_null && (*tags[j].is_null)) { + ret = converToStr(str + len, TSDB_DATA_TYPE_NULL, NULL, -1, &l); + } else { + if (tags[j].buffer == NULL) { + free(str); + tscError("empty"); + return TSDB_CODE_TSC_APP_ERROR; + } + + ret = converToStr(str + len, tags[j].buffer_type, tags[j].buffer, tags[j].length ? (int32_t)*tags[j].length : -1, &l); + } + + ++j; + + if (ret) { + free(str); + return ret; + } + + len += l; + } else { + len += (size_t)snprintf(str + len, size - len - 1, i > 0 ? ",%.*s" : "%.*s", t->n, t->z); + } + } + + if (len >= (size - 1)) { + size *= 2; + free(str); + str = calloc(1, size); + continue; + } + + strcat(str, ") "); + len += 2; + + if ((len + strlen(pStmt->mtb.values.z)) >= (size - 1)) { + size *= 2; + free(str); + str = calloc(1, size); + continue; + } + + strcat(str, pStmt->mtb.values.z); + + break; + } + + free(pSql->sqlstr); + pSql->sqlstr = str; + + return TSDB_CODE_SUCCESS; +} + + + //////////////////////////////////////////////////////////////////////////////// // interface functions @@ -1221,6 +1401,7 @@ TAOS_STMT* taos_stmt_init(TAOS* taos) { pStmt->taos = pObj; SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + if (pSql == NULL) { free(pStmt); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -1253,7 +1434,7 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { } pStmt->last = STMT_PREPARE; - + SSqlObj* pSql = pStmt->pSql; size_t sqlLen = strlen(sql); @@ -1263,7 +1444,7 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { pSql->fp = waitForQueryRsp; pSql->fetchFp = waitForQueryRsp; - pCmd->insertType = TSDB_QUERY_TYPE_STMT_INSERT; + pCmd->insertParam.insertType = TSDB_QUERY_TYPE_STMT_INSERT; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { tscError("%p failed to malloc payload buffer", pSql); @@ -1292,34 +1473,15 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { registerSqlObj(pSql); - int32_t ret = TSDB_CODE_SUCCESS; - - if ((ret = tsInsertInitialCheck(pSql)) != TSDB_CODE_SUCCESS) { + int32_t ret = stmtParseInsertTbTags(pSql, pStmt); + if (ret != TSDB_CODE_SUCCESS) { return ret; } - - int32_t index = 0; - SStrToken sToken = tStrGetToken(pCmd->curSql, &index, false); - if (sToken.n == 0) { - return TSDB_CODE_TSC_INVALID_SQL; + if (pStmt->multiTbInsert) { + return TSDB_CODE_SUCCESS; } - if (sToken.n == 1 && sToken.type == TK_QUESTION) { - pStmt->multiTbInsert = true; - pStmt->mtb.tbname = sToken; - pStmt->mtb.nameSet = false; - if (pStmt->mtb.pTableHash == NULL) { - pStmt->mtb.pTableHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - } - if (pStmt->mtb.pTableBlockHashList == NULL) { - pStmt->mtb.pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - } - - return TSDB_CODE_SUCCESS; - } - - pStmt->multiTbInsert = false; memset(&pStmt->mtb, 0, sizeof(pStmt->mtb)); int32_t code = tsParseSql(pSql, true); @@ -1336,8 +1498,7 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { return normalStmtPrepare(pStmt); } - -int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { +int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags) { STscStmt* pStmt = (STscStmt*)stmt; SSqlObj* pSql = pStmt->pSql; SSqlCmd* pCmd = &pSql->cmd; @@ -1376,44 +1537,58 @@ int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { return TSDB_CODE_TSC_APP_ERROR; } - SSubmitBlk* pBlk = (SSubmitBlk*) (*t1)->pData; + SSubmitBlk* pBlk = (SSubmitBlk*) (*t1)->pData; pCmd->batchSize = pBlk->numOfRows; - taosHashPut(pCmd->pTableBlockHashList, (void *)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid), (void*)t1, POINTER_BYTES); - + taosHashPut(pCmd->insertParam.pTableBlockHashList, (void *)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid), (void*)t1, POINTER_BYTES); + tscDebug("0x%"PRIx64" table:%s is already prepared, uid:%" PRIu64, pSql->self, name, pStmt->mtb.currentUid); return TSDB_CODE_SUCCESS; } - - pStmt->mtb.tbname = tscReplaceStrToken(&pSql->sqlstr, &pStmt->mtb.tbname, name); + + if (pStmt->mtb.tagSet) { + pStmt->mtb.tbname = tscReplaceStrToken(&pSql->sqlstr, &pStmt->mtb.tbname, name); + } else { + if (tags == NULL) { + tscError("No tags set"); + return TSDB_CODE_TSC_APP_ERROR; + } + + int32_t ret = stmtGenInsertStatement(pSql, pStmt, name, tags); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + } + pStmt->mtb.nameSet = true; + pStmt->mtb.tagSet = true; tscDebug("0x%"PRIx64" SQL: %s", pSql->self, pSql->sqlstr); - pSql->cmd.parseFinished = 0; pSql->cmd.numOfParams = 0; pSql->cmd.batchSize = 0; - if (taosHashGetSize(pCmd->pTableBlockHashList) > 0) { - SHashObj* hashList = pCmd->pTableBlockHashList; - pCmd->pTableBlockHashList = NULL; + if (taosHashGetSize(pCmd->insertParam.pTableBlockHashList) > 0) { + SHashObj* hashList = pCmd->insertParam.pTableBlockHashList; + pCmd->insertParam.pTableBlockHashList = NULL; tscResetSqlCmd(pCmd, true); - pCmd->pTableBlockHashList = hashList; + pCmd->insertParam.pTableBlockHashList = hashList; } - + int32_t code = tsParseSql(pStmt->pSql, true); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { // wait for the callback function to post the semaphore tsem_wait(&pStmt->pSql->rspSem); - + code = pStmt->pSql->res.code; } if (code == TSDB_CODE_SUCCESS) { - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0, 0); + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; STableDataBlocks* pBlock = NULL; - code = tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), + code = tscGetDataBlockFromList(pCmd->insertParam.pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); if (code != TSDB_CODE_SUCCESS) { return code; @@ -1426,15 +1601,20 @@ int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { pStmt->mtb.tbNum++; taosHashPut(pStmt->mtb.pTableBlockHashList, (void *)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid), (void*)&pBlock, POINTER_BYTES); - taosHashPut(pStmt->mtb.pTableHash, name, strlen(name), (char*) &pTableMeta->id.uid, sizeof(pTableMeta->id.uid)); tscDebug("0x%"PRIx64" table:%s is prepared, uid:%" PRIx64, pSql->self, name, pStmt->mtb.currentUid); } - + return code; } + +int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { + return taos_stmt_set_tbname_tags(stmt, name, NULL); +} + + int taos_stmt_close(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; if (!pStmt->isInsert) { @@ -1451,8 +1631,9 @@ int taos_stmt_close(TAOS_STMT* stmt) { if (pStmt->multiTbInsert) { taosHashCleanup(pStmt->mtb.pTableHash); pStmt->mtb.pTableBlockHashList = tscDestroyBlockHashTable(pStmt->mtb.pTableBlockHashList, true); - taosHashCleanup(pStmt->pSql->cmd.pTableBlockHashList); - pStmt->pSql->cmd.pTableBlockHashList = NULL; + taosHashCleanup(pStmt->pSql->cmd.insertParam.pTableBlockHashList); + pStmt->pSql->cmd.insertParam.pTableBlockHashList = NULL; + taosArrayDestroy(pStmt->mtb.tags); } } @@ -1467,7 +1648,7 @@ int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* bind) { terrno = TSDB_CODE_TSC_DISCONNECTED; return TSDB_CODE_TSC_DISCONNECTED; } - + if (pStmt->isInsert) { if (pStmt->multiTbInsert) { if (pStmt->last != STMT_SETTBNAME && pStmt->last != STMT_ADD_BATCH) { @@ -1482,7 +1663,7 @@ int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* bind) { } pStmt->last = STMT_BIND; - + return insertStmtBindParam(pStmt, bind); } else { return normalStmtBindParam(pStmt, bind); @@ -1502,7 +1683,7 @@ int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) { tscError("0x%"PRIx64" invalid parameter", pStmt->pSql->self); return TSDB_CODE_TSC_APP_ERROR; } - + if (!pStmt->isInsert) { tscError("0x%"PRIx64" not or invalid batch insert", pStmt->pSql->self); return TSDB_CODE_TSC_APP_ERROR; @@ -1512,7 +1693,7 @@ int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) { if (pStmt->last != STMT_SETTBNAME && pStmt->last != STMT_ADD_BATCH) { tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); return TSDB_CODE_TSC_APP_ERROR; - } + } } else { if (pStmt->last != STMT_PREPARE && pStmt->last != STMT_ADD_BATCH && pStmt->last != STMT_EXECUTE) { tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); @@ -1521,7 +1702,7 @@ int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) { } pStmt->last = STMT_BIND; - + return insertStmtBindParamBatch(pStmt, bind, -1); } @@ -1546,7 +1727,7 @@ int taos_stmt_bind_single_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, in if (pStmt->last != STMT_SETTBNAME && pStmt->last != STMT_ADD_BATCH && pStmt->last != STMT_BIND_COL) { tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); return TSDB_CODE_TSC_APP_ERROR; - } + } } else { if (pStmt->last != STMT_PREPARE && pStmt->last != STMT_ADD_BATCH && pStmt->last != STMT_BIND_COL && pStmt->last != STMT_EXECUTE) { tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); @@ -1567,7 +1748,7 @@ int taos_stmt_add_batch(TAOS_STMT* stmt) { terrno = TSDB_CODE_TSC_DISCONNECTED; return TSDB_CODE_TSC_DISCONNECTED; } - + if (pStmt->isInsert) { if (pStmt->last != STMT_BIND && pStmt->last != STMT_BIND_COL) { tscError("0x%"PRIx64" add batch status error, last:%d", pStmt->pSql->self, pStmt->last); @@ -1575,10 +1756,10 @@ int taos_stmt_add_batch(TAOS_STMT* stmt) { } pStmt->last = STMT_ADD_BATCH; - + return insertStmtAddBatch(pStmt); } - + return TSDB_CODE_COM_OPS_NOT_SUPPORT; } @@ -1597,7 +1778,7 @@ int taos_stmt_execute(TAOS_STMT* stmt) { terrno = TSDB_CODE_TSC_DISCONNECTED; return TSDB_CODE_TSC_DISCONNECTED; } - + if (pStmt->isInsert) { if (pStmt->last != STMT_ADD_BATCH) { tscError("0x%"PRIx64" exec status error, last:%d", pStmt->pSql->self, pStmt->last); @@ -1605,7 +1786,7 @@ int taos_stmt_execute(TAOS_STMT* stmt) { } pStmt->last = STMT_EXECUTE; - + if (pStmt->multiTbInsert) { ret = insertBatchStmtExecute(pStmt); } else { @@ -1617,9 +1798,10 @@ int taos_stmt_execute(TAOS_STMT* stmt) { ret = TSDB_CODE_TSC_OUT_OF_MEMORY; } else { if (pStmt->pSql != NULL) { - taos_free_result(pStmt->pSql); + tscFreeSqlObj(pStmt->pSql); pStmt->pSql = NULL; } + pStmt->pSql = taos_query((TAOS*)pStmt->taos, sql); ret = taos_errno(pStmt->pSql); free(sql); @@ -1689,16 +1871,16 @@ int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes) { if (pStmt->isInsert) { SSqlCmd* pCmd = &pStmt->pSql->cmd; - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0, 0); + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (pCmd->pTableBlockHashList == NULL) { - pCmd->pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + if (pCmd->insertParam.pTableBlockHashList == NULL) { + pCmd->insertParam.pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); } STableDataBlocks* pBlock = NULL; int32_t ret = - tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), + tscGetDataBlockFromList(pCmd->insertParam.pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); if (ret != 0) { // todo handle error diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 66bbd8db3b..f8ea81497f 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -36,16 +36,17 @@ #include "qScript.h" #include "ttype.h" #include "qUtil.h" +#include "qPlan.h" #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" #define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey)) -// -1 is tbname column index, so here use the -3 as the initial value -#define COLUMN_INDEX_INITIAL_VAL (-3) +// -1 is tbname column index, so here use the -2 as the initial value +#define COLUMN_INDEX_INITIAL_VAL (-2) #define COLUMN_INDEX_INITIALIZER \ { COLUMN_INDEX_INITIAL_VAL, COLUMN_INDEX_INITIAL_VAL } -#define COLUMN_INDEX_VALIDE(index) (((index).tableIndex >= 0) && ((index).columnIndex >= TSDB_BLOCK_DIST_COLUMN_INDEX)) +#define COLUMN_INDEX_VALIDE(index) (((index).tableIndex >= 0) && ((index).columnIndex >= TSDB_TBNAME_COLUMN_INDEX)) #define TBNAME_LIST_SEP "," typedef struct SColumnList { // todo refactor @@ -58,7 +59,7 @@ typedef struct SConvertFunc { int32_t execFuncId; } SConvertFunc; -static SExprInfo* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t colIndex, int32_t tableIndex); +static SExprInfo* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t colIndex, int32_t tableIndex, int32_t colId); static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo); static char* getAccountId(SSqlObj* pSql); @@ -71,7 +72,7 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC static int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* len); -static void getColumnName(tSqlExprItem* pItem, char* resultFieldName, int32_t nameLength); +static void getColumnName(tSqlExprItem* pItem, char* resultFieldName, char* rawName, int32_t nameLength); static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSqlExprItem* pItem, bool finalResult); static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes, @@ -112,7 +113,7 @@ static bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField); static bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo); static bool hasNormalColumnFilter(SQueryInfo* pQueryInfo); -static int32_t validateLimitNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t index, SSqlNode* pSqlNode, SSqlObj* pSql); +static int32_t validateLimitNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, SSqlObj* pSql); static int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDbInfo* pCreateDbSql); static int32_t getColumnIndexByName(SSqlCmd* pCmd, const SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t getTableIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); @@ -127,17 +128,18 @@ static SColumnList createColumnList(int32_t num, int16_t tableIndex, int32_t col static int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* pInfo); static int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo); -static int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, int32_t index); +static int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInfo); static int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, uint64_t *uid); static bool validateDebugFlag(int32_t v); static int32_t checkQueryRangeForFill(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); +static int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo); static bool isTimeWindowQuery(SQueryInfo* pQueryInfo) { return pQueryInfo->interval.interval > 0 || pQueryInfo->sessionWindow.gap > 0; } -int16_t getNewResColId(SQueryInfo* pQueryInfo) { - return pQueryInfo->resColumnId--; +int16_t getNewResColId(SSqlCmd* pCmd) { + return pCmd->resColumnId--; } static uint8_t convertOptr(SStrToken *pToken) { @@ -193,7 +195,7 @@ static bool validateDebugFlag(int32_t v) { * Used during parsing query sql. Since the query sql usually small in length, error position * is not needed in the final error message. */ -static int32_t invalidSqlErrMsg(char* dstBuffer, const char* errMsg) { +static int32_t invalidOperationMsg(char* dstBuffer, const char* errMsg) { return tscInvalidSQLErrMsg(dstBuffer, errMsg, NULL); } @@ -209,11 +211,11 @@ static int setColumnFilterInfoForTimestamp(SSqlCmd* pCmd, SQueryInfo* pQueryInfo if (seg != NULL) { if (taosParseTime(pVar->pz, &time, pVar->nLen, tinfo.precision, tsDaylight) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } } else { if (tVariantDump(pVar, (char*)&time, TSDB_DATA_TYPE_BIGINT, true)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } } @@ -229,18 +231,50 @@ static int32_t handlePassword(SSqlCmd* pCmd, SStrToken* pPwd) { const char* msg3 = "password needs single quote marks enclosed"; if (pPwd->type != TK_STRING) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } strdequote(pPwd->z); pPwd->n = (uint32_t)strtrim(pPwd->z); // trim space before and after passwords if (pPwd->n <= 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (pPwd->n >= TSDB_KEY_LEN) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + + return TSDB_CODE_SUCCESS; +} + +// validate the out put field type for "UNION ALL" subclause +static int32_t normalizeVarDataTypeLength(SSqlCmd* pCmd) { + const char* msg1 = "columns in select clause not identical"; + + int32_t diffSize = 0; + + // if there is only one element, the limit of clause is the limit of global result. + SQueryInfo* pQueryInfo1 = pCmd->pQueryInfo; + SQueryInfo* pSibling = pQueryInfo1->sibling; + + while(pSibling != NULL) { + int32_t ret = tscFieldInfoCompare(&pQueryInfo1->fieldsInfo, &pSibling->fieldsInfo, &diffSize); + if (ret != 0) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + + pSibling = pSibling->sibling; + } + + if (diffSize) { + pQueryInfo1 = pCmd->pQueryInfo; + pSibling = pQueryInfo1->sibling; + + while(pSibling->sibling != NULL) { + tscFieldInfoSetSize(&pQueryInfo1->fieldsInfo, &pSibling->fieldsInfo); + pSibling = pSibling->sibling; + } } return TSDB_CODE_SUCCESS; @@ -292,93 +326,92 @@ int32_t handleUserDefinedFunc(SSqlObj* pSql, struct SSqlInfo* pInfo) { SSqlCmd *pCmd = &pSql->cmd; switch (pInfo->type) { - case TSDB_SQL_CREATE_FUNCTION: { - SCreateFuncInfo *createInfo = &pInfo->pMiscInfo->funcOpt; - uint32_t len = 0; - void *buf = NULL; + case TSDB_SQL_CREATE_FUNCTION: { + SCreateFuncInfo *createInfo = &pInfo->pMiscInfo->funcOpt; + uint32_t len = 0; + void *buf = NULL; - if (createInfo->output.type == (uint8_t)-1 || createInfo->output.bytes < 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); - } + if (createInfo->output.type == (uint8_t)-1 || createInfo->output.bytes < 0) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); + } - createInfo->name.z[createInfo->name.n] = 0; + createInfo->name.z[createInfo->name.n] = 0; - strdequote(createInfo->name.z); + strdequote(createInfo->name.z); - if (strlen(createInfo->name.z) >= TSDB_FUNC_NAME_LEN) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); - } + if (strlen(createInfo->name.z) >= TSDB_FUNC_NAME_LEN) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); + } - createInfo->path.z[createInfo->path.n] = 0; + createInfo->path.z[createInfo->path.n] = 0; - strdequote(createInfo->path.z); + strdequote(createInfo->path.z); - if (strlen(createInfo->path.z) >= PATH_MAX) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); - } + if (strlen(createInfo->path.z) >= PATH_MAX) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + } - int32_t ret = readFromFile(createInfo->path.z, &len, &buf); - if (ret) { - return ret; - } - //distinguish *.lua and *.so - int32_t pathLen = (int32_t)strlen(createInfo->path.z); - if ((pathLen > 3) && (0 == strncmp(createInfo->path.z + pathLen - 3, "lua", 3)) && !isValidScript(buf, len)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - //TODO CHECK CODE - if (len + sizeof(SCreateFuncMsg) > pSql->cmd.allocSize) { - ret = tscAllocPayload(&pSql->cmd, len + sizeof(SCreateFuncMsg)); + int32_t ret = readFromFile(createInfo->path.z, &len, &buf); if (ret) { - tfree(buf); return ret; } + //distinguish *.lua and *.so + int32_t pathLen = (int32_t)strlen(createInfo->path.z); + if ((pathLen > 3) && (0 == strncmp(createInfo->path.z + pathLen - 3, "lua", 3)) && !isValidScript(buf, len)) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); + } + + //TODO CHECK CODE + if (len + sizeof(SCreateFuncMsg) > pSql->cmd.allocSize) { + ret = tscAllocPayload(&pSql->cmd, len + sizeof(SCreateFuncMsg)); + if (ret) { + tfree(buf); + return ret; + } + } + + SCreateFuncMsg *pMsg = (SCreateFuncMsg *)pSql->cmd.payload; + + strcpy(pMsg->name, createInfo->name.z); + strcpy(pMsg->path, createInfo->path.z); + + pMsg->funcType = htonl(createInfo->type); + pMsg->bufSize = htonl(createInfo->bufSize); + + pMsg->outputType = createInfo->output.type; + pMsg->outputLen = htons(createInfo->output.bytes); + + pMsg->codeLen = htonl(len); + memcpy(pMsg->code, buf, len); + tfree(buf); + + break; } + case TSDB_SQL_DROP_FUNCTION: { + SStrToken* t0 = taosArrayGet(pInfo->pMiscInfo->a, 0); - SCreateFuncMsg *pMsg = (SCreateFuncMsg *)pSql->cmd.payload; + SDropFuncMsg *pMsg = (SDropFuncMsg *)pSql->cmd.payload; - strcpy(pMsg->name, createInfo->name.z); - strcpy(pMsg->path, createInfo->path.z); + t0->z[t0->n] = 0; - pMsg->funcType = htonl(createInfo->type); - pMsg->bufSize = htonl(createInfo->bufSize); + strdequote(t0->z); - pMsg->outputType = createInfo->output.type; - pMsg->outputLen = htons(createInfo->output.bytes); + if (strlen(t0->z) >= TSDB_FUNC_NAME_LEN) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); + } - pMsg->codeLen = htonl(len); - memcpy(pMsg->code, buf, len); - tfree(buf); + strcpy(pMsg->name, t0->z); - break; + break; } - case TSDB_SQL_DROP_FUNCTION: { - SStrToken* t0 = taosArrayGet(pInfo->pMiscInfo->a, 0); - - SDropFuncMsg *pMsg = (SDropFuncMsg *)pSql->cmd.payload; - - t0->z[t0->n] = 0; - - strdequote(t0->z); - - if (strlen(t0->z) >= TSDB_FUNC_NAME_LEN) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - strcpy(pMsg->name, t0->z); - - break; - } - default: - return TSDB_CODE_TSC_APP_ERROR; + default: + return TSDB_CODE_TSC_APP_ERROR; } return TSDB_CODE_SUCCESS; } - -int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { +int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (pInfo == NULL || pSql == NULL) { return TSDB_CODE_TSC_APP_ERROR; } @@ -392,7 +425,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return tscSQLSyntaxErrMsg(tscGetErrorMsgPayload(pCmd), NULL, pInfo->msg); } - SQueryInfo* pQueryInfo = tscGetQueryInfoS(pCmd, pCmd->clauseIndex); + SQueryInfo* pQueryInfo = tscGetQueryInfoS(pCmd); if (pQueryInfo == NULL) { pRes->code = terrno; return pRes->code; @@ -414,37 +447,36 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { case TSDB_SQL_DROP_DB: { const char* msg2 = "invalid name"; const char* msg3 = "param name too long"; - const char* msg4 = "table is not super table"; SStrToken* pzName = taosArrayGet(pInfo->pMiscInfo->a, 0); if ((pInfo->type != TSDB_SQL_DROP_DNODE) && (tscValidateName(pzName) != TSDB_CODE_SUCCESS)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } if (pInfo->type == TSDB_SQL_DROP_DB) { assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1); code = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pzName); if (code != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } } else if (pInfo->type == TSDB_SQL_DROP_TABLE) { assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1); - code = tscSetTableFullName(pTableMetaInfo, pzName, pSql); + code = tscSetTableFullName(&pTableMetaInfo->name, pzName, pSql); if(code != TSDB_CODE_SUCCESS) { return code; } if (pInfo->pMiscInfo->tableType == TSDB_SUPER_TABLE) { - code = tscGetTableMeta(pSql, pTableMetaInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); - } +//// code = tscGetTableMeta(pSql, pTableMetaInfo); +//// if (code != TSDB_CODE_SUCCESS) { +//// return code; +//// } +// +// if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { +// return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); +// } } } else if (pInfo->type == TSDB_SQL_DROP_DNODE) { @@ -452,7 +484,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { strncpy(pCmd->payload, pzName->z, pzName->n); } else { // drop user/account if (pzName->n >= TSDB_USER_LEN) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } strncpy(pCmd->payload, pzName->z, pzName->n); @@ -466,12 +498,12 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pToken); if (ret != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } break; @@ -483,7 +515,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { case TSDB_SQL_SHOW: { if (setShowInfo(pSql, pInfo) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } break; @@ -506,16 +538,16 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { SCreateDbInfo* pCreateDB = &(pInfo->pMiscInfo->dbOpt); if (tscValidateName(&pCreateDB->dbname) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), &(pCreateDB->dbname)); if (ret != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } if (parseCreateDBOptions(pCmd, pCreateDB) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } break; @@ -525,7 +557,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg = "invalid host name (ip address)"; if (taosArrayGetSize(pInfo->pMiscInfo->a) > 1) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } SStrToken* id = taosArrayGet(pInfo->pMiscInfo->a, 0); @@ -543,15 +575,15 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { SStrToken* pPwd = &pInfo->pMiscInfo->user.passwd; if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } if (pName->n >= TSDB_USER_LEN) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } if (tscValidateName(pName) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } SCreateAcctInfo* pAcctOpt = &pInfo->pMiscInfo->acctOpt; @@ -561,7 +593,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) { } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) { } else { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } } @@ -573,10 +605,10 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } // additional msg has been attached already - code = tscSetTableFullName(pTableMetaInfo, pToken, pSql); + code = tscSetTableFullName(&pTableMetaInfo->name, pToken, pSql); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -589,10 +621,10 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } - code = tscSetTableFullName(pTableMetaInfo, pToken, pSql); + code = tscSetTableFullName(&pTableMetaInfo->name, pToken, pSql); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -604,11 +636,11 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (pToken->n > TSDB_DB_NAME_LEN) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } return tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pToken); } @@ -621,7 +653,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { /* validate the parameter names and options */ if (validateDNodeConfig(pMiscInfo) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } char* pMsg = pCmd->payload; @@ -635,7 +667,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { strncpy(pCfg->ep, t0->z, t0->n); if (validateEp(pCfg->ep) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } strncpy(pCfg->config, t1->z, t1->n); @@ -664,21 +696,21 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { SStrToken* pPwd = &pUser->passwd; if (pName->n >= TSDB_USER_LEN) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } if (tscValidateName(pName) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } if (pCmd->command == TSDB_SQL_CREATE_USER) { if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } else { if (pUser->type == TSDB_ALTER_USER_PASSWD) { if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } else if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { assert(pPwd->type == TSDB_DATA_TYPE_NULL); @@ -692,10 +724,10 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } else if (strncasecmp(pPrivilege->z, "write", 5) == 0 && pPrivilege->n == 5) { pCmd->count = 3; } else { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } } else { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); } } @@ -708,7 +740,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { // validate the parameter names and options if (validateLocalConfig(pMiscInfo) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } int32_t numOfToken = (int32_t) taosArrayGetSize(pMiscInfo->a); @@ -748,56 +780,41 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } case TSDB_SQL_SELECT: { - const char* msg1 = "columns in select clause not identical"; - - size_t size = taosArrayGetSize(pInfo->list); - for (int32_t i = pCmd->numOfClause; i < size; ++i) { - SQueryInfo* p = tscGetQueryInfoS(pCmd, i); - if (p == NULL) { - pRes->code = terrno; - return pRes->code; - } + code = loadAllTableMeta(pSql, pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; } - assert(pCmd->numOfClause == size); - for (int32_t i = pCmd->clauseIndex; i < size; ++i) { + pQueryInfo = tscGetQueryInfo(pCmd); + + size_t size = taosArrayGetSize(pInfo->list); + for (int32_t i = 0; i < size; ++i) { SSqlNode* pSqlNode = taosArrayGetP(pInfo->list, i); - tscTrace("%p start to parse %dth subclause, total:%d", pSql, i, (int32_t) size); - if ((code = validateSqlNode(pSql, pSqlNode, i)) != TSDB_CODE_SUCCESS) { + + tscTrace("%p start to parse %dth subclause, total:%"PRIzu, pSql, i, size); + if ((code = validateSqlNode(pSql, pSqlNode, pQueryInfo)) != TSDB_CODE_SUCCESS) { return code; } tscPrintSelNodeList(pSql, i); - pCmd->clauseIndex += 1; + + if ((i + 1) < size && pQueryInfo->sibling == NULL) { + if ((code = tscAddQueryInfo(pCmd)) != TSDB_CODE_SUCCESS) { + return code; + } + + pQueryInfo = pCmd->active; + } } - // restore the clause index - pCmd->clauseIndex = 0; + if ((code = normalizeVarDataTypeLength(pCmd)) != TSDB_CODE_SUCCESS) { + return code; + } // set the command/global limit parameters from the first subclause to the sqlcmd object - SQueryInfo* pQueryInfo1 = tscGetQueryInfo(pCmd, 0); - pCmd->command = pQueryInfo1->command; - int32_t diffSize = 0; + pCmd->active = pCmd->pQueryInfo; + pCmd->command = pCmd->pQueryInfo->command; - // if there is only one element, the limit of clause is the limit of global result. - // validate the select node for "UNION ALL" subclause - for (int32_t i = 1; i < pCmd->numOfClause; ++i) { - SQueryInfo* pQueryInfo2 = tscGetQueryInfo(pCmd, i); - - int32_t ret = tscFieldInfoCompare(&pQueryInfo1->fieldsInfo, &pQueryInfo2->fieldsInfo, &diffSize); - if (ret != 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } - - if (diffSize) { - for (int32_t i = 1; i < pCmd->numOfClause; ++i) { - SQueryInfo* pQueryInfo2 = tscGetQueryInfo(pCmd, i); - tscFieldInfoSetSize(&pQueryInfo1->fieldsInfo, &pQueryInfo2->fieldsInfo); - } - } - - pCmd->parseFinished = 1; return TSDB_CODE_SUCCESS; // do not build query message here } @@ -825,20 +842,19 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1); code = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pzName); if (code != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } break; } default: - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "not support sql expression"); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "not support sql expression"); } - pSql->cmd.parseFinished = 1; if (tscBuildMsg[pCmd->command] != NULL) { return tscBuildMsg[pCmd->command](pSql, pInfo); } else { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "not support sql expression"); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "not support sql expression"); } } @@ -847,10 +863,10 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { * are available. */ static bool isTopBottomQuery(SQueryInfo* pQueryInfo) { - size_t size = tscSqlExprNumOfExprs(pQueryInfo); + size_t size = tscNumOfExprs(pQueryInfo); for (int32_t i = 0; i < size; ++i) { - int32_t functionId = tscSqlExprGet(pQueryInfo, i)->base.functionId; + int32_t functionId = tscExprGet(pQueryInfo, i)->base.functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { return true; @@ -861,8 +877,8 @@ static bool isTopBottomQuery(SQueryInfo* pQueryInfo) { } // need to add timestamp column in result set, if it is a time window query -static int32_t addPrimaryTsColumnForTimeWindowQuery(SQueryInfo* pQueryInfo) { - uint64_t uid = tscSqlExprGet(pQueryInfo, 0)->base.uid; +static int32_t addPrimaryTsColumnForTimeWindowQuery(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) { + uint64_t uid = tscExprGet(pQueryInfo, 0)->base.uid; int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { @@ -874,14 +890,14 @@ static int32_t addPrimaryTsColumnForTimeWindowQuery(SQueryInfo* pQueryInfo) { } if (tableIndex == COLUMN_INDEX_INITIAL_VAL) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } SSchema s = {.bytes = TSDB_KEYSIZE, .type = TSDB_DATA_TYPE_TIMESTAMP, .colId = PRIMARYKEY_TIMESTAMP_COL_INDEX}; tstrncpy(s.name, aAggs[TSDB_FUNC_TS].name, sizeof(s.name)); SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TS, &index, &s, TSDB_COL_NORMAL); + tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TS, &index, &s, TSDB_COL_NORMAL, getNewResColId(pCmd)); return TSDB_CODE_SUCCESS; } @@ -897,7 +913,7 @@ static int32_t checkInvalidExprForTimeWindow(SSqlCmd* pCmd, SQueryInfo* pQueryIn // order by normal column is not supported int32_t colId = pQueryInfo->order.orderColId; if (isTimeWindowQuery(pQueryInfo) && colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } return TSDB_CODE_SUCCESS; @@ -907,11 +923,11 @@ static int32_t checkInvalidExprForTimeWindow(SSqlCmd* pCmd, SQueryInfo* pQueryIn * invalid sql: * select count(tbname)/count(tag1)/count(tag2) from super_table_name [interval(1d)|session(ts, 1d)]; */ - size_t size = tscSqlExprNumOfExprs(pQueryInfo); + size_t size = tscNumOfExprs(pQueryInfo); for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (pExpr->base.functionId == TSDB_FUNC_COUNT && TSDB_COL_IS_TAG(pExpr->base.colInfo.flag)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } } @@ -920,10 +936,10 @@ static int32_t checkInvalidExprForTimeWindow(SSqlCmd* pCmd, SQueryInfo* pQueryIn * select tbname, tags_fields from super_table_name [interval(1s)|session(ts,1s)] */ if (tscQueryTags(pQueryInfo) && isTimeWindowQuery(pQueryInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } - return addPrimaryTsColumnForTimeWindowQuery(pQueryInfo); + return addPrimaryTsColumnForTimeWindowQuery(pQueryInfo, pCmd); } int32_t validateIntervalNode(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode) { @@ -937,7 +953,7 @@ int32_t validateIntervalNode(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNode* pS if (!TPARSER_HAS_TOKEN(pSqlNode->interval.interval)) { if (TPARSER_HAS_TOKEN(pSqlNode->sliding)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } return TSDB_CODE_SUCCESS; @@ -951,7 +967,7 @@ int32_t validateIntervalNode(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNode* pS // interval is not null SStrToken *t = &pSqlNode->interval.interval; if (parseNatualDuration(t->z, t->n, &pQueryInfo->interval.interval, &pQueryInfo->interval.intervalUnit) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit != 'y') { @@ -962,16 +978,16 @@ int32_t validateIntervalNode(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNode* pS // interval cannot be less than 10 milliseconds if (pQueryInfo->interval.interval < tsMinIntervalTime) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } } if (parseIntervalOffset(pCmd, pQueryInfo, &pSqlNode->interval.offset) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } if (parseSlidingClause(pCmd, pQueryInfo, &pSqlNode->sliding) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } // The following part is used to check for the invalid query expression. @@ -994,11 +1010,11 @@ int32_t validateSessionNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode * pS char timeUnit = 0; if (parseNatualDuration(gap->z, gap->n, &pQueryInfo->sessionWindow.gap, &timeUnit) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } if (timeUnit == 'y' || timeUnit == 'n') { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } // if the unit of time window value is millisecond, change the value from microsecond @@ -1009,12 +1025,12 @@ int32_t validateSessionNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode * pS } if (pQueryInfo->sessionWindow.gap != 0 && pQueryInfo->interval.interval != 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, col, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } pQueryInfo->sessionWindow.primaryColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; @@ -1039,11 +1055,11 @@ int32_t parseIntervalOffset(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* of } if (parseNatualDuration(t->z, t->n, &pQueryInfo->interval.offset, &pQueryInfo->interval.offsetUnit) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } if (pQueryInfo->interval.offset < 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (pQueryInfo->interval.offsetUnit != 'n' && pQueryInfo->interval.offsetUnit != 'y') { @@ -1053,18 +1069,18 @@ int32_t parseIntervalOffset(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* of } if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit != 'y') { if (pQueryInfo->interval.offset >= pQueryInfo->interval.interval) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } } } else if (pQueryInfo->interval.offsetUnit == pQueryInfo->interval.intervalUnit) { if (pQueryInfo->interval.offset >= pQueryInfo->interval.interval) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } } else if (pQueryInfo->interval.intervalUnit == 'n' && pQueryInfo->interval.offsetUnit == 'y') { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } else if (pQueryInfo->interval.intervalUnit == 'y' && pQueryInfo->interval.offsetUnit == 'n') { if (pQueryInfo->interval.interval * 12 <= pQueryInfo->interval.offset) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } } else { // TODO: offset should be shorter than interval, but how to check @@ -1092,7 +1108,7 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* pSl } if (pQueryInfo->interval.intervalUnit == 'n' || pQueryInfo->interval.intervalUnit == 'y') { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } parseAbsoluteDuration(pSliding->z, pSliding->n, &pQueryInfo->interval.sliding); @@ -1101,25 +1117,25 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* pSl } if (pQueryInfo->interval.sliding < tsMinSlidingTime) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); } if (pQueryInfo->interval.sliding > pQueryInfo->interval.interval) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if ((pQueryInfo->interval.interval != 0) && (pQueryInfo->interval.interval/pQueryInfo->interval.sliding > INTERVAL_SLIDING_FACTOR)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } // if (pQueryInfo->interval.sliding != pQueryInfo->interval.interval && pSql->pStream == NULL) { -// return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); +// return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); // } return TSDB_CODE_SUCCESS; } -int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pTableName, SSqlObj* pSql) { +int32_t tscSetTableFullName(SName* pName, SStrToken* pTableName, SSqlObj* pSql) { const char* msg1 = "name too long"; const char* msg2 = "acctId too long"; const char* msg3 = "no acctId"; @@ -1132,27 +1148,27 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pTableNam if (idx != -1) { // db has been specified in sql string so we ignore current db path char* acctId = getAccountId(pSql); if (acctId == NULL || strlen(acctId) <= 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } - code = tNameSetAcctId(&pTableMetaInfo->name, acctId); + code = tNameSetAcctId(pName, acctId); if (code != 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } if (idx >= TSDB_DB_NAME_LEN) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } if (pTableName->n - 1 - idx >= TSDB_TABLE_NAME_LEN) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } char name[TSDB_TABLE_FNAME_LEN] = {0}; strncpy(name, pTableName->z, pTableName->n); - code = tNameFromString(&pTableMetaInfo->name, name, T_NAME_DB|T_NAME_TABLE); + code = tNameFromString(pName, name, T_NAME_DB|T_NAME_TABLE); if (code != 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } } else { // get current DB name first, and then set it into path char* t = cloneCurrentDBName(pSql); @@ -1160,7 +1176,7 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pTableNam return TSDB_CODE_TSC_DB_NOT_SELECTED; } - code = tNameFromString(&pTableMetaInfo->name, t, T_NAME_ACCT | T_NAME_DB); + code = tNameFromString(pName, t, T_NAME_ACCT | T_NAME_DB); if (code != 0) { free(t); return TSDB_CODE_TSC_DB_NOT_SELECTED; @@ -1169,15 +1185,15 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pTableNam free(t); if (pTableName->n >= TSDB_TABLE_NAME_LEN) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } char name[TSDB_TABLE_FNAME_LEN] = {0}; strncpy(name, pTableName->z, pTableName->n); - code = tNameFromString(&pTableMetaInfo->name, name, T_NAME_TABLE); + code = tNameFromString(pName, name, T_NAME_TABLE); if (code != 0) { - code = invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + code = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } } @@ -1198,14 +1214,14 @@ static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { // number of fields no less than 2 size_t numOfCols = taosArrayGetSize(pFieldList); if (numOfCols <= 1 || numOfCols > TSDB_MAX_COLUMNS) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); return false; } // first column must be timestamp TAOS_FIELD* pField = taosArrayGet(pFieldList, 0); if (pField->type != TSDB_DATA_TYPE_TIMESTAMP) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); return false; } @@ -1213,29 +1229,29 @@ static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { for (int32_t i = 0; i < numOfCols; ++i) { pField = taosArrayGet(pFieldList, i); if (!isValidDataType(pField->type)) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); return false; } if (pField->bytes == 0) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); return false; } if ((pField->type == TSDB_DATA_TYPE_BINARY && (pField->bytes <= 0 || pField->bytes > TSDB_MAX_BINARY_LEN)) || (pField->type == TSDB_DATA_TYPE_NCHAR && (pField->bytes <= 0 || pField->bytes > TSDB_MAX_NCHAR_LEN))) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); return false; } if (validateColumnName(pField->name) != TSDB_CODE_SUCCESS) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); return false; } // field name must be unique if (has(pFieldList, i + 1, pField->name) == true) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); return false; } @@ -1244,7 +1260,7 @@ static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { // max row length must be less than TSDB_MAX_BYTES_PER_ROW if (nLen > TSDB_MAX_BYTES_PER_ROW) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); return false; } @@ -1266,7 +1282,7 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC // number of fields at least 1 size_t numOfTags = taosArrayGetSize(pTagsList); if (numOfTags < 1 || numOfTags > TSDB_MAX_TAGS) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); return false; } @@ -1275,28 +1291,28 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC TAOS_FIELD* p = taosArrayGet(pTagsList, i); if (p->type == TSDB_DATA_TYPE_TIMESTAMP) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); return false; } if (!isValidDataType(p->type)) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); return false; } if ((p->type == TSDB_DATA_TYPE_BINARY && p->bytes <= 0) || (p->type == TSDB_DATA_TYPE_NCHAR && p->bytes <= 0)) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); return false; } if (validateColumnName(p->name) != TSDB_CODE_SUCCESS) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); return false; } if (has(pTagsList, i + 1, p->name) == true) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); return false; } } @@ -1305,7 +1321,7 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC for (int32_t i = 0; i < numOfTags; ++i) { TAOS_FIELD* p = taosArrayGet(pTagsList, i); if (p->bytes == 0) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); return false; } @@ -1314,7 +1330,7 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC // max tag row length must be less than TSDB_MAX_TAGS_LEN if (nLen > TSDB_MAX_TAGS_LEN) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); return false; } @@ -1323,7 +1339,7 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC TAOS_FIELD* p = taosArrayGet(pTagsList, i); if (has(pFieldList, 0, p->name) == true) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); return false; } } @@ -1342,9 +1358,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { const char* msg5 = "invalid binary/nchar tag length"; const char* msg6 = "invalid data type in tags"; - assert(pCmd->numOfClause == 1); - - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; int32_t numOfTags = tscGetNumOfTags(pTableMeta); @@ -1355,18 +1369,18 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { char msg[128] = {0}; sprintf(msg, "tags no more than %d", TSDB_MAX_TAGS); - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); return false; } // no timestamp allowable if (pTagField->type == TSDB_DATA_TYPE_TIMESTAMP) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); return false; } if ((pTagField->type < TSDB_DATA_TYPE_BOOL) || (pTagField->type > TSDB_DATA_TYPE_UBIGINT)) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); return false; } @@ -1379,19 +1393,19 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { // length less than TSDB_MAX_TASG_LEN if (nLen + pTagField->bytes > TSDB_MAX_TAGS_LEN) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); return false; } // tags name can not be a keyword if (validateColumnName(pTagField->name) != TSDB_CODE_SUCCESS) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); return false; } // binary(val), val can not be equalled to or less than 0 if ((pTagField->type == TSDB_DATA_TYPE_BINARY || pTagField->type == TSDB_DATA_TYPE_NCHAR) && pTagField->bytes <= 0) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); return false; } @@ -1400,7 +1414,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { for (int32_t i = 0; i < numOfTags + numOfCols; ++i) { if (strncasecmp(pTagField->name, pSchema[i].name, sizeof(pTagField->name) - 1) == 0) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); return false; } } @@ -1416,8 +1430,8 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { const char* msg5 = "invalid column name"; const char* msg6 = "invalid column length"; - assert(pCmd->numOfClause == 1); - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); +// assert(pCmd->numOfClause == 1); + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; int32_t numOfTags = tscGetNumOfTags(pTableMeta); @@ -1425,17 +1439,17 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { // no more max columns if (numOfCols >= TSDB_MAX_COLUMNS || numOfTags + numOfCols >= TSDB_MAX_COLUMNS) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); return false; } if (pColField->type < TSDB_DATA_TYPE_BOOL || pColField->type > TSDB_DATA_TYPE_UBIGINT) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); return false; } if (validateColumnName(pColField->name) != TSDB_CODE_SUCCESS) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); return false; } @@ -1447,20 +1461,20 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { } if (pColField->bytes <= 0) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); return false; } // length less than TSDB_MAX_BYTES_PER_ROW if (nLen + pColField->bytes > TSDB_MAX_BYTES_PER_ROW) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); return false; } // field name must be unique for (int32_t i = 0; i < numOfTags + numOfCols; ++i) { if (strncasecmp(pColField->name, pSchema[i].name, sizeof(pColField->name) - 1) == 0) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); return false; } } @@ -1512,7 +1526,7 @@ int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStr /* db name is not specified, the tableName dose not include db name */ if (pDB != NULL) { if (pDB->n >= TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN || pDB->n == 0) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } memcpy(&fullName[totalLen], pDB->z, pDB->n); @@ -1526,12 +1540,12 @@ int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStr /* here we only check the table name length limitation */ if (!tscValidateTableNameLength(tableName->n)) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } else { // pDB == NULL, the db prefix name is specified in tableName /* the length limitation includes tablename + dbname + sep */ if (tableName->n >= TSDB_TABLE_NAME_LEN + TSDB_DB_NAME_LEN) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } @@ -1547,7 +1561,7 @@ int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStr fullName[totalLen] = 0; } - return (totalLen < TSDB_TABLE_FNAME_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_TSC_INVALID_SQL; + return (totalLen < TSDB_TABLE_FNAME_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_TSC_INVALID_OPERATION; } void tscInsertPrimaryTsSourceColumn(SQueryInfo* pQueryInfo, uint64_t tableUid) { @@ -1565,7 +1579,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32 int32_t arithmeticType = NON_ARITHMEIC_EXPR; if (validateArithmeticSQLExpr(pCmd, pItem->pNode, pQueryInfo, &columnList, &arithmeticType) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } int32_t tableIndex = columnList.ids[0].tableIndex; @@ -1575,15 +1589,15 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32 // all columns in arithmetic expression must belong to the same table for (int32_t f = 1; f < columnList.num; ++f) { if (columnList.ids[f].tableIndex != tableIndex) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } } // expr string is set as the parameter of function SColumnIndex index = {.tableIndex = tableIndex}; - SExprInfo* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, sizeof(double), - getNewResColId(pQueryInfo), sizeof(double), false); + SExprInfo* pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, sizeof(double), + getNewResColId(pCmd), sizeof(double), false); char* name = (pItem->aliasName != NULL)? pItem->aliasName:pItem->pNode->token.z; size_t len = MIN(sizeof(pExpr->base.aliasName), pItem->pNode->token.n + 1); @@ -1596,7 +1610,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32 if (ret != TSDB_CODE_SUCCESS) { taosArrayDestroy(colList); tExprTreeDestroy(pNode, NULL); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } // check for if there is a tag in the arithmetic express @@ -1607,7 +1621,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32 tExprTreeDestroy(pNode, NULL); taosArrayDestroy(colList); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } } @@ -1625,7 +1639,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32 char* c = tbufGetData(&bw, false); // set the serialized binary string as the parameter of arithmetic expression - addExprParams(&pExpr->base, c, TSDB_DATA_TYPE_BINARY, (int32_t)len); + tscExprAddParams(&pExpr->base, c, TSDB_DATA_TYPE_BINARY, (int32_t)len); insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->base.aliasName, pExpr); // add ts column @@ -1638,40 +1652,37 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32 columnList.num = 0; columnList.ids[0] = (SColumnIndex) {0, 0}; + char rawName[TSDB_COL_NAME_LEN] = {0}; char aliasName[TSDB_COL_NAME_LEN] = {0}; - if (pItem->aliasName != NULL) { - tstrncpy(aliasName, pItem->aliasName, TSDB_COL_NAME_LEN); - } else { - int32_t nameLen = MIN(TSDB_COL_NAME_LEN, pItem->pNode->token.n + 1); - tstrncpy(aliasName, pItem->pNode->token.z, nameLen); - } + getColumnName(pItem, aliasName, rawName, TSDB_COL_NAME_LEN); insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, aliasName, NULL); int32_t slot = tscNumOfFields(pQueryInfo) - 1; SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, slot); + assert(pInfo->pExpr == NULL); - if (pInfo->pExpr == NULL) { - SExprInfo* pExprInfo = calloc(1, sizeof(SExprInfo)); + SExprInfo* pExprInfo = calloc(1, sizeof(SExprInfo)); - // arithmetic expression always return result in the format of double float - pExprInfo->base.resBytes = sizeof(double); - pExprInfo->base.interBytes = sizeof(double); - pExprInfo->base.resType = TSDB_DATA_TYPE_DOUBLE; + // arithmetic expression always return result in the format of double float + pExprInfo->base.resBytes = sizeof(double); + pExprInfo->base.interBytes = sizeof(double); + pExprInfo->base.resType = TSDB_DATA_TYPE_DOUBLE; - pExprInfo->base.functionId = TSDB_FUNC_ARITHM; - pExprInfo->base.numOfParams = 1; - pExprInfo->base.resColId = getNewResColId(pQueryInfo); + pExprInfo->base.functionId = TSDB_FUNC_ARITHM; + pExprInfo->base.numOfParams = 1; + pExprInfo->base.resColId = getNewResColId(pCmd); + strncpy(pExprInfo->base.aliasName, aliasName, tListLen(pExprInfo->base.aliasName)); + strncpy(pExprInfo->base.token, rawName, tListLen(pExprInfo->base.token)); - int32_t ret = exprTreeFromSqlExpr(pCmd, &pExprInfo->pExpr, pItem->pNode, pQueryInfo, NULL, &(pExprInfo->base.uid)); - if (ret != TSDB_CODE_SUCCESS) { - tExprTreeDestroy(pExprInfo->pExpr, NULL); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "invalid expression in select clause"); - } - - pInfo->pExpr = pExprInfo; + int32_t ret = exprTreeFromSqlExpr(pCmd, &pExprInfo->pExpr, pItem->pNode, pQueryInfo, NULL, &(pExprInfo->base.uid)); + if (ret != TSDB_CODE_SUCCESS) { + tExprTreeDestroy(pExprInfo->pExpr, NULL); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "invalid expression in select clause"); } + pInfo->pExpr = pExprInfo; + SBufferWriter bw = tbufInitWriter(NULL, false); TRY(0) { @@ -1693,8 +1704,8 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32 return TSDB_CODE_SUCCESS; } -static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSqlExprItem* pItem) { - SExprInfo* pExpr = doAddProjectCol(pQueryInfo, pIndex->columnIndex, pIndex->tableIndex); +static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSqlExprItem* pItem, int32_t colId) { + SExprInfo* pExpr = doAddProjectCol(pQueryInfo, pIndex->columnIndex, pIndex->tableIndex, colId); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; @@ -1716,11 +1727,11 @@ static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumn insertResultField(pQueryInfo, startPos, &ids, pExpr->base.resBytes, (int8_t)pExpr->base.resType, pExpr->base.aliasName, pExpr); } -static void addPrimaryTsColIntoResult(SQueryInfo* pQueryInfo) { +static void addPrimaryTsColIntoResult(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) { // primary timestamp column has been added already - size_t size = tscSqlExprNumOfExprs(pQueryInfo); + size_t size = tscNumOfExprs(pQueryInfo); for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (pExpr->base.functionId == TSDB_FUNC_PRJ && pExpr->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { return; } @@ -1733,8 +1744,8 @@ static void addPrimaryTsColIntoResult(SQueryInfo* pQueryInfo) { // add the timestamp column into the output columns SColumnIndex index = {0}; // primary timestamp column info - int32_t numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); - tscAddFuncInSelectClause(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL); + int32_t numOfCols = (int32_t)tscNumOfExprs(pQueryInfo); + tscAddFuncInSelectClause(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL, getNewResColId(pCmd)); SInternalField* pSupInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, numOfCols); pSupInfo->visible = false; @@ -1749,7 +1760,7 @@ bool isValidDistinctSql(SQueryInfo* pQueryInfo) { if ((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_QUERY) != TSDB_QUERY_TYPE_STABLE_QUERY) { return false; } - if (tscQueryTags(pQueryInfo) && tscSqlExprNumOfExprs(pQueryInfo) == 1){ + if (tscQueryTags(pQueryInfo) && tscNumOfExprs(pQueryInfo) == 1){ return true; } return false; @@ -1844,7 +1855,7 @@ int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pS // too many result columns not support order by in query if (taosArrayGetSize(pSelNodeList) > TSDB_MAX_COLUMNS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (pQueryInfo->colList == NULL) { @@ -1854,7 +1865,7 @@ int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pS bool hasDistinct = false; size_t numOfExpr = taosArrayGetSize(pSelNodeList); for (int32_t i = 0; i < numOfExpr; ++i) { - int32_t outputIndex = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); + int32_t outputIndex = (int32_t)tscNumOfExprs(pQueryInfo); tSqlExprItem* pItem = taosArrayGet(pSelNodeList, i); if (hasDistinct == false) { @@ -1867,7 +1878,7 @@ int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pS if (pItem->pNode->functionId < 0) { SUdfInfo* pUdfInfo = isValidUdf(pQueryInfo->pUdfInfo, pItem->pNode->operand.z, pItem->pNode->operand.n); if (pUdfInfo == NULL) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } pItem->pNode->functionId = pUdfInfo->functionId; @@ -1875,13 +1886,13 @@ int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pS // sql function in selection clause, append sql function info in pSqlCmd structure sequentially if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, pItem, true) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } else if (type == SQL_NODE_TABLE_COLUMN || type == SQL_NODE_VALUE) { // use the dynamic array list to decide if the function is valid or not // select table_name1.field_name1, table_name2.field_name2 from table_name1, table_name2 if (addProjectionExprAndResultField(pCmd, pQueryInfo, pItem) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } else if (type == SQL_NODE_EXPR) { int32_t code = handleArithmeticExpr(pCmd, pQueryInfo, i, pItem); @@ -1889,17 +1900,17 @@ int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pS return code; } } else { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } if (pQueryInfo->fieldsInfo.numOfOutput > TSDB_MAX_COLUMNS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } } if (hasDistinct == true) { if (!isValidDistinctSql(pQueryInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } pQueryInfo->distinctTag = true; } @@ -1907,11 +1918,11 @@ int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pS // there is only one user-defined column in the final result field, add the timestamp column. size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); if ((numOfSrcCols <= 0 || !hasNoneUserDefineExpr(pQueryInfo)) && !tscQueryTags(pQueryInfo) && !tscQueryBlockInfo(pQueryInfo)) { - addPrimaryTsColIntoResult(pQueryInfo); + addPrimaryTsColIntoResult(pQueryInfo, pCmd); } if (!functionCompatibleCheck(pQueryInfo, joinQuery, timeWindowQuery)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } return TSDB_CODE_SUCCESS; @@ -1940,7 +1951,7 @@ int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnLi return TSDB_CODE_SUCCESS; } -SExprInfo* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t colIndex, int32_t tableIndex) { +SExprInfo* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t colIndex, int32_t tableIndex, int32_t colId) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; int32_t numOfCols = tscGetNumOfColumns(pTableMeta); @@ -1957,18 +1968,16 @@ SExprInfo* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t colIndex, int32_t tab index.columnIndex = colIndex; } - int16_t colId = getNewResColId(pQueryInfo); - return tscSqlExprAppend(pQueryInfo, functionId, &index, pSchema->type, pSchema->bytes, colId, pSchema->bytes, + return tscExprAppend(pQueryInfo, functionId, &index, pSchema->type, pSchema->bytes, colId, pSchema->bytes, (functionId == TSDB_FUNC_TAGPRJ)); } SExprInfo* tscAddFuncInSelectClause(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, - SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag) { - int16_t colId = getNewResColId(pQueryInfo); - - SExprInfo* pExpr = tscSqlExprInsert(pQueryInfo, outputColIndex, functionId, pIndex, pColSchema->type, + SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag, int16_t colId) { + SExprInfo* pExpr = tscExprInsert(pQueryInfo, outputColIndex, functionId, pIndex, pColSchema->type, pColSchema->bytes, colId, pColSchema->bytes, TSDB_COL_IS_TAG(flag)); tstrncpy(pExpr->base.aliasName, pColSchema->name, sizeof(pExpr->base.aliasName)); + tstrncpy(pExpr->base.token, pColSchema->name, sizeof(pExpr->base.token)); SColumnList ids = createColumnList(1, pIndex->tableIndex, pIndex->columnIndex); if (TSDB_COL_IS_TAG(flag)) { @@ -1987,7 +1996,7 @@ SExprInfo* tscAddFuncInSelectClause(SQueryInfo* pQueryInfo, int32_t outputColInd return pExpr; } -static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColumnIndex* pIndex, int32_t startPos) { +static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColumnIndex* pIndex, int32_t startPos, SSqlCmd* pCmd) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); int32_t numOfTotalColumns = 0; @@ -2003,7 +2012,7 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum } for (int32_t j = 0; j < numOfTotalColumns; ++j) { - SExprInfo* pExpr = doAddProjectCol(pQueryInfo, j, pIndex->tableIndex); + SExprInfo* pExpr = doAddProjectCol(pQueryInfo, j, pIndex->tableIndex, getNewResColId(pCmd)); tstrncpy(pExpr->base.aliasName, pSchema[j].name, sizeof(pExpr->base.aliasName)); pIndex->columnIndex = j; @@ -2021,7 +2030,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t const char* msg0 = "invalid column name"; const char* msg1 = "tag for normal table query is not allowed"; - int32_t startPos = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); + int32_t startPos = (int32_t)tscNumOfExprs(pQueryInfo); int32_t optr = pItem->pNode->tokenId; if (optr == TK_ALL) { // project on all fields @@ -2029,18 +2038,18 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getTableIndexByName(&pItem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); } // all meters columns are required if (index.tableIndex == COLUMN_INDEX_INITIAL_VAL) { // all table columns are required. for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { index.tableIndex = i; - int32_t inc = doAddProjectionExprAndResultFields(pQueryInfo, &index, startPos); + int32_t inc = doAddProjectionExprAndResultFields(pQueryInfo, &index, startPos, pCmd); startPos += inc; } } else { - doAddProjectionExprAndResultFields(pQueryInfo, &index, startPos); + doAddProjectionExprAndResultFields(pQueryInfo, &index, startPos, pCmd); } // add the primary timestamp column even though it is not required by user @@ -2057,7 +2066,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t SSchema colSchema = tGetUserSpecifiedColumnSchema(&pItem->pNode->value, &pItem->pNode->token, pItem->aliasName); SExprInfo* pExpr = - tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC); + tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC, getNewResColId(pCmd)); // NOTE: the first parameter is reserved for the tag column id during join query process. pExpr->base.numOfParams = 2; @@ -2066,32 +2075,31 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &pItem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); } if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { SSchema* colSchema = tGetTbnameColumnSchema(); - tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, colSchema, TSDB_COL_TAG); - } else if (index.columnIndex == TSDB_BLOCK_DIST_COLUMN_INDEX) { - SSchema colSchema = tGetBlockDistColumnSchema(); - tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_TAG); + tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, colSchema, TSDB_COL_TAG, getNewResColId(pCmd)); } else { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) && UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } - addProjectQueryCol(pQueryInfo, startPos, &index, pItem); + addProjectQueryCol(pQueryInfo, startPos, &index, pItem, getNewResColId(pCmd)); pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; } // add the primary timestamp column even though it is not required by user STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMetaInfo->pTableMeta->id.uid); + if (!UTIL_TABLE_IS_TMP_TABLE(pTableMetaInfo)) { + tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMetaInfo->pTableMeta->id.uid); + } } else { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } return TSDB_CODE_SUCCESS; @@ -2108,7 +2116,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS if (functionID == TSDB_FUNC_SPREAD) { int32_t t1 = pSchema->type; if (t1 == TSDB_DATA_TYPE_BINARY || t1 == TSDB_DATA_TYPE_NCHAR || t1 == TSDB_DATA_TYPE_BOOL) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); return -1; } else { type = TSDB_DATA_TYPE_DOUBLE; @@ -2119,7 +2127,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS bytes = pSchema->bytes; } - SExprInfo* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, getNewResColId(pQueryInfo), bytes, false); + SExprInfo* pExpr = tscExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, getNewResColId(pCmd), bytes, false); tstrncpy(pExpr->base.aliasName, name, tListLen(pExpr->base.aliasName)); if (cvtFunc.originFuncId == TSDB_FUNC_LAST_ROW && cvtFunc.originFuncId != functionID) { @@ -2173,9 +2181,9 @@ void setResultColName(char* name, tSqlExprItem* pItem, int32_t functionId, SStrT static void updateLastScanOrderIfNeeded(SQueryInfo* pQueryInfo) { if (pQueryInfo->sessionWindow.gap > 0 || tscGroupbyColumn(pQueryInfo)) { - size_t numOfExpr = tscSqlExprNumOfExprs(pQueryInfo); + size_t numOfExpr = tscNumOfExprs(pQueryInfo); for (int32_t i = 0; i < numOfExpr; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (pExpr->base.functionId != TSDB_FUNC_LAST && pExpr->base.functionId != TSDB_FUNC_LAST_DST) { continue; } @@ -2206,7 +2214,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col case TSDB_FUNC_COUNT: { /* more than one parameter for count() function */ if (pItem->pNode->pParam != NULL && taosArrayGetSize(pItem->pNode->pParam) != 1) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } SExprInfo* pExpr = NULL; @@ -2215,43 +2223,28 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (pItem->pNode->pParam != NULL) { tSqlExprItem* pParamElem = taosArrayGet(pItem->pNode->pParam, 0); SStrToken* pToken = &pParamElem->pNode->colInfo; - int16_t sqlOptr = pParamElem->pNode->tokenId; - if ((pToken->z == NULL || pToken->n == 0) - && (TK_INTEGER != sqlOptr)) /*select count(1) from table*/ { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + int16_t tokenId = pParamElem->pNode->tokenId; + if ((pToken->z == NULL || pToken->n == 0) && (TK_INTEGER != tokenId)) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } - if (sqlOptr == TK_ALL) { - // select table.* + // select count(table.*) + // select count(1)|count(2) + if (tokenId == TK_ALL || tokenId == TK_INTEGER) { // check if the table name is valid or not SStrToken tmpToken = pParamElem->pNode->colInfo; if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, false); - } else if (sqlOptr == TK_INTEGER) { // select count(1) from table1 - char buf[8] = {0}; - int64_t val = -1; - tVariant* pVariant = &pParamElem->pNode->value; - if (pVariant->nType == TSDB_DATA_TYPE_BIGINT) { - tVariantDump(pVariant, buf, TSDB_DATA_TYPE_BIGINT, true); - val = GET_INT64_VAL(buf); - } - if (val == 1) { - index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, false); - } else { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); - } + pExpr = tscExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pCmd), size, false); } else { - // count the number of meters created according to the super table + // count the number of table created according to the super table if (getColumnIndexByName(pCmd, pToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); @@ -2264,18 +2257,18 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, isTag); + pExpr = tscExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pCmd), size, isTag); } } else { // count(*) is equalled to count(primary_timestamp_key) index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, false); + pExpr = tscExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pCmd), size, false); } pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); memset(pExpr->base.aliasName, 0, tListLen(pExpr->base.aliasName)); - getColumnName(pItem, pExpr->base.aliasName, sizeof(pExpr->base.aliasName) - 1); + getColumnName(pItem, pExpr->base.aliasName, pExpr->base.token,sizeof(pExpr->base.aliasName) - 1); SColumnList list = createColumnList(1, index.tableIndex, index.columnIndex); if (finalResult) { @@ -2314,21 +2307,21 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (pItem->pNode->pParam == NULL || (functionId != TSDB_FUNC_LEASTSQR && taosArrayGetSize(pItem->pNode->pParam) != 1) || (functionId == TSDB_FUNC_LEASTSQR && taosArrayGetSize(pItem->pNode->pParam) != 3)) { /* no parameters or more than one parameter for function */ - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } tSqlExprItem* pParamElem = taosArrayGet(pItem->pNode->pParam, 0); if (pParamElem->pNode->tokenId != TK_ALL && pParamElem->pNode->tokenId != TK_ID) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } SColumnIndex index = COLUMN_INDEX_INITIALIZER; if ((getColumnIndexByName(pCmd, &pParamElem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } // 2. check if sql function can be applied on this column data type @@ -2336,9 +2329,9 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); if (!IS_NUMERIC_TYPE(pSchema->type)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } else if (IS_UNSIGNED_NUMERIC_TYPE(pSchema->type) && functionId == TSDB_FUNC_DIFF) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); } int16_t resultType = 0; @@ -2347,15 +2340,15 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (getResultDataInfo(pSchema->type, pSchema->bytes, functionId, 0, &resultType, &resultSize, &intermediateResSize, 0, false, NULL) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } // set the first column ts for diff query if (functionId == TSDB_FUNC_DIFF) { colIndex += 1; SColumnIndex indexTS = {.tableIndex = index.tableIndex, .columnIndex = 0}; - SExprInfo* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, - getNewResColId(pQueryInfo), TSDB_KEYSIZE, false); + SExprInfo* pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, + getNewResColId(pCmd), TSDB_KEYSIZE, false); SColumnList ids = createColumnList(1, 0, 0); insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].name, pExpr); @@ -2363,32 +2356,32 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // functions can not be applied to tags if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } - SExprInfo* pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pQueryInfo), resultSize, false); + SExprInfo* pExpr = tscExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pCmd), resultSize, false); if (functionId == TSDB_FUNC_LEASTSQR) { /* set the leastsquares parameters */ char val[8] = {0}; if (tVariantDump(&pParamElem[1].pNode->value, val, TSDB_DATA_TYPE_DOUBLE, true) < 0) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } - addExprParams(&pExpr->base, val, TSDB_DATA_TYPE_DOUBLE, DOUBLE_BYTES); + tscExprAddParams(&pExpr->base, val, TSDB_DATA_TYPE_DOUBLE, DOUBLE_BYTES); memset(val, 0, tListLen(val)); if (tVariantDump(&pParamElem[2].pNode->value, val, TSDB_DATA_TYPE_DOUBLE, true) < 0) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } - addExprParams(&pExpr->base, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); + tscExprAddParams(&pExpr->base, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); } SColumnList ids = createColumnList(1, index.tableIndex, index.columnIndex); memset(pExpr->base.aliasName, 0, tListLen(pExpr->base.aliasName)); - getColumnName(pItem, pExpr->base.aliasName, sizeof(pExpr->base.aliasName) - 1); + getColumnName(pItem, pExpr->base.aliasName, pExpr->base.token,sizeof(pExpr->base.aliasName) - 1); if (finalResult) { int32_t numOfOutput = tscNumOfFields(pQueryInfo); @@ -2417,18 +2410,18 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (!requireAllFields) { if (taosArrayGetSize(pItem->pNode->pParam) < 1) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } if (taosArrayGetSize(pItem->pNode->pParam) > 1 && (pItem->aliasName != NULL && strlen(pItem->aliasName) > 0)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); } /* in first/last function, multiple columns can be add to resultset */ for (int32_t i = 0; i < taosArrayGetSize(pItem->pNode->pParam); ++i) { tSqlExprItem* pParamElem = taosArrayGet(pItem->pNode->pParam, i); if (pParamElem->pNode->tokenId != TK_ALL && pParamElem->pNode->tokenId != TK_ID) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } SColumnIndex index = COLUMN_INDEX_INITIALIZER; @@ -2437,7 +2430,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SStrToken tmpToken = pParamElem->pNode->colInfo; if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); @@ -2450,20 +2443,20 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col setResultColName(name, pItem, cvtFunc.originFuncId, &t, true); if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[j], cvtFunc, name, colIndex++, &index, finalResult) != 0) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } } else { if (getColumnIndexByName(pCmd, &pParamElem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); // functions can not be applied to tags if ((index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) || (index.columnIndex < 0)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } char name[TSDB_COL_NAME_LEN] = {0}; @@ -2473,7 +2466,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col setResultColName(name, pItem, cvtFunc.originFuncId, &pParamElem->pNode->colInfo, multiColOutput); if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, name, colIndex++, &index, finalResult) != 0) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } } @@ -2484,7 +2477,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // multicolumn selection does not support alias name if (pItem->aliasName != NULL && strlen(pItem->aliasName) > 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); } for (int32_t j = 0; j < pQueryInfo->numOfTables; ++j) { @@ -2499,7 +2492,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col setResultColName(name, pItem, cvtFunc.originFuncId, &t, true); if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[index.columnIndex], cvtFunc, name, colIndex, &index, finalResult) != 0) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } colIndex++; } @@ -2518,21 +2511,21 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // 1. valid the number of parameters if (pItem->pNode->pParam == NULL || taosArrayGetSize(pItem->pNode->pParam) != 2) { /* no parameters or more than one parameter for function */ - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } tSqlExprItem* pParamElem = taosArrayGet(pItem->pNode->pParam, 0); if (pParamElem->pNode->tokenId != TK_ID) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &pParamElem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); @@ -2540,17 +2533,17 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // functions can not be applied to tags if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } // 2. valid the column type if (!IS_NUMERIC_TYPE(pSchema->type)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } // 3. valid the parameters if (pParamElem[1].pNode->tokenId == TK_ID) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } tVariant* pVariant = &pParamElem[1].pNode->value; @@ -2566,7 +2559,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col double dp = GET_DOUBLE_VAL(val); if (dp < 0 || dp > TOP_BOTTOM_QUERY_LIMIT) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } resultSize = sizeof(double); @@ -2580,20 +2573,20 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMetaInfo->pTableMeta->id.uid); colIndex += 1; // the first column is ts - pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pQueryInfo), resultSize, false); - addExprParams(&pExpr->base, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); + pExpr = tscExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pCmd), resultSize, false); + tscExprAddParams(&pExpr->base, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); } else { tVariantDump(pVariant, val, TSDB_DATA_TYPE_BIGINT, true); int64_t nTop = GET_INT32_VAL(val); if (nTop <= 0 || nTop > 100) { // todo use macro - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } // todo REFACTOR // set the first column ts for top/bottom query SColumnIndex index1 = {index.tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, getNewResColId(pQueryInfo), + pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, getNewResColId(pCmd), TSDB_KEYSIZE, false); tstrncpy(pExpr->base.aliasName, aAggs[TSDB_FUNC_TS].name, sizeof(pExpr->base.aliasName)); @@ -2604,12 +2597,12 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col colIndex += 1; // the first column is ts - pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pQueryInfo), resultSize, false); - addExprParams(&pExpr->base, val, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t)); + pExpr = tscExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pCmd), resultSize, false); + tscExprAddParams(&pExpr->base, val, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t)); } memset(pExpr->base.aliasName, 0, tListLen(pExpr->base.aliasName)); - getColumnName(pItem, pExpr->base.aliasName, sizeof(pExpr->base.aliasName) - 1); + getColumnName(pItem, pExpr->base.aliasName, pExpr->base.token,sizeof(pExpr->base.aliasName) - 1); // todo refactor: tscColumnListInsert part SColumnList ids = createColumnList(1, index.tableIndex, index.columnIndex); @@ -2627,12 +2620,12 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col case TSDB_FUNC_TID_TAG: { pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); } // no parameters or more than one parameter for function if (pItem->pNode->pParam == NULL || taosArrayGetSize(pItem->pNode->pParam) != 1) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } tSqlExprItem* pParamItem = taosArrayGet(pItem->pNode->pParam, 0); @@ -2640,7 +2633,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &pParam->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); @@ -2649,7 +2642,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // functions can not be applied to normal columns int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); if (index.columnIndex < numOfCols && index.columnIndex != TSDB_TBNAME_COLUMN_INDEX) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } if (index.columnIndex > 0) { @@ -2665,7 +2658,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } if (colType == TSDB_DATA_TYPE_BOOL) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMetaInfo->pTableMeta->id.uid, @@ -2690,17 +2683,17 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col s.bytes = bytes; TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY); - tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TID_TAG, &index, &s, TSDB_COL_TAG); + tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TID_TAG, &index, &s, TSDB_COL_TAG, getNewResColId(pCmd)); return TSDB_CODE_SUCCESS; } case TSDB_FUNC_BLKINFO: { // no parameters or more than one parameter for function if (pItem->pNode->pParam != NULL && taosArrayGetSize(pItem->pNode->pParam) != 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } - SColumnIndex index = {.tableIndex = 0, .columnIndex = TSDB_BLOCK_DIST_COLUMN_INDEX,}; + SColumnIndex index = {.tableIndex = 0, .columnIndex = 0,}; pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); SSchema s = {.name = "block_dist", .type = TSDB_DATA_TYPE_BINARY}; @@ -2708,10 +2701,16 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col int16_t resType = 0; int16_t bytes = 0; getResultDataInfo(TSDB_DATA_TYPE_INT, 4, TSDB_FUNC_BLKINFO, 0, &resType, &bytes, &inter, 0, 0, NULL); - s.bytes = bytes; s.type = (uint8_t)resType; - SExprInfo* pExpr = tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_BLKINFO, &index, &s, TSDB_COL_TAG); + + SExprInfo* pExpr = tscExprInsert(pQueryInfo, 0, TSDB_FUNC_BLKINFO, &index, resType, + bytes, getNewResColId(pCmd), bytes, 0); + tstrncpy(pExpr->base.aliasName, s.name, sizeof(pExpr->base.aliasName)); + + SColumnList ids = createColumnList(1, index.tableIndex, index.columnIndex); + insertResultField(pQueryInfo, 0, &ids, bytes, s.type, s.name, pExpr); + pExpr->base.numOfParams = 1; pExpr->base.param[0].i64 = pTableMetaInfo->pTableMeta->tableInfo.rowSize; pExpr->base.param[0].nType = TSDB_DATA_TYPE_BIGINT; @@ -2722,28 +2721,28 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col default: { SUdfInfo* pUdfInfo = isValidUdf(pQueryInfo->pUdfInfo, pItem->pNode->operand.z, pItem->pNode->operand.n); if (pUdfInfo == NULL) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); } tSqlExprItem* pParamElem = taosArrayGet(pItem->pNode->pParam, 0);; if (pParamElem->pNode->tokenId != TK_ID) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &pParamElem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); // functions can not be applied to tags if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } int32_t inter = 0; @@ -2751,11 +2750,10 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col int16_t bytes = 0; getResultDataInfo(TSDB_DATA_TYPE_INT, 4, functionId, 0, &resType, &bytes, &inter, 0, false, pUdfInfo); - SExprInfo* pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resType, bytes, - getNewResColId(pQueryInfo), inter, false); + SExprInfo* pExpr = tscExprAppend(pQueryInfo, functionId, &index, resType, bytes, getNewResColId(pCmd), inter, false); memset(pExpr->base.aliasName, 0, tListLen(pExpr->base.aliasName)); - getColumnName(pItem, pExpr->base.aliasName, sizeof(pExpr->base.aliasName) - 1); + getColumnName(pItem, pExpr->base.aliasName, pExpr->base.token, sizeof(pExpr->base.aliasName) - 1); SSchema s = {0}; s.type = resType; @@ -2777,7 +2775,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } } - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } // todo refactor @@ -2794,12 +2792,14 @@ static SColumnList createColumnList(int32_t num, int16_t tableIndex, int32_t col return columnList; } -void getColumnName(tSqlExprItem* pItem, char* resultFieldName, int32_t nameLength) { +void getColumnName(tSqlExprItem* pItem, char* resultFieldName, char* rawName, int32_t nameLength) { + int32_t len = ((int32_t)pItem->pNode->token.n < nameLength) ? (int32_t)pItem->pNode->token.n : nameLength; + strncpy(rawName, pItem->pNode->token.z, len); + if (pItem->aliasName != NULL) { - strncpy(resultFieldName, pItem->aliasName, nameLength); + strncpy(resultFieldName, pItem->aliasName, len); } else { - int32_t len = ((int32_t)pItem->pNode->token.n < nameLength) ? (int32_t)pItem->pNode->token.n : nameLength; - strncpy(resultFieldName, pItem->pNode->token.z, len); + strncpy(resultFieldName, rawName, len); } } @@ -2811,14 +2811,6 @@ static bool isTablenameToken(SStrToken* token) { return (strncasecmp(TSQL_TBNAME_L, tmpToken.z, tmpToken.n) == 0 && tmpToken.n == strlen(TSQL_TBNAME_L)); } -static bool isTableBlockDistToken(SStrToken* token) { - SStrToken tmpToken = *token; - SStrToken tableToken = {0}; - - extractTableNameFromToken(&tmpToken, &tableToken); - - return (strncasecmp(TSQL_BLOCK_DIST, tmpToken.z, tmpToken.n) == 0 && tmpToken.n == strlen(TSQL_BLOCK_DIST_L)); -} static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SStrToken* pToken) { STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index)->pTableMeta; @@ -2848,8 +2840,6 @@ int32_t doGetColumnIndexByName(SSqlCmd* pCmd, SStrToken* pToken, SQueryInfo* pQu if (isTablenameToken(pToken)) { pIndex->columnIndex = TSDB_TBNAME_COLUMN_INDEX; - } else if (isTableBlockDistToken(pToken)) { - pIndex->columnIndex = TSDB_BLOCK_DIST_COLUMN_INDEX; } else if (strncasecmp(pToken->z, DEFAULT_PRIMARY_TIMESTAMP_COL_NAME, pToken->n) == 0) { pIndex->columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX; } else { @@ -2860,7 +2850,7 @@ int32_t doGetColumnIndexByName(SSqlCmd* pCmd, SStrToken* pToken, SQueryInfo* pQu if (colIndex != COLUMN_INDEX_INITIAL_VAL) { if (pIndex->columnIndex != COLUMN_INDEX_INITIAL_VAL) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); } else { pIndex->tableIndex = i; pIndex->columnIndex = colIndex; @@ -2875,14 +2865,14 @@ int32_t doGetColumnIndexByName(SSqlCmd* pCmd, SStrToken* pToken, SQueryInfo* pQu } if (pIndex->columnIndex == COLUMN_INDEX_INITIAL_VAL) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } } if (COLUMN_INDEX_VALIDE(*pIndex)) { return TSDB_CODE_SUCCESS; } else { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } @@ -2908,7 +2898,7 @@ int32_t getTableIndexImpl(SStrToken* pTableToken, SQueryInfo* pQueryInfo, SColum } if (pIndex->tableIndex < 0) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } return TSDB_CODE_SUCCESS; @@ -2919,7 +2909,7 @@ int32_t getTableIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIn extractTableNameFromToken(pToken, &tableToken); if (getTableIndexImpl(&tableToken, pQueryInfo, pIndex) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } return TSDB_CODE_SUCCESS; @@ -2927,13 +2917,13 @@ int32_t getTableIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIn int32_t getColumnIndexByName(SSqlCmd* pCmd, const SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { if (pQueryInfo->pTableMetaInfo == NULL || pQueryInfo->numOfTables == 0) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } SStrToken tmpToken = *pToken; if (getTableIndexByName(&tmpToken, pQueryInfo, pIndex) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } return doGetColumnIndexByName(pCmd, &tmpToken, pQueryInfo, pIndex); @@ -2941,8 +2931,7 @@ int32_t getColumnIndexByName(SSqlCmd* pCmd, const SStrToken* pToken, SQueryInfo* int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { SSqlCmd* pCmd = &pSql->cmd; - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - assert(pCmd->numOfClause == 1); + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); pCmd->command = TSDB_SQL_SHOW; @@ -2965,20 +2954,20 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (pDbPrefixToken->type != 0) { if (pDbPrefixToken->n >= TSDB_DB_NAME_LEN) { // db name is too long - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } if (pDbPrefixToken->n <= 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } if (tscValidateName(pDbPrefixToken) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pDbPrefixToken); if (ret != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } } @@ -2988,26 +2977,26 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { pPattern->n = strdequote(pPattern->z); if (pPattern->n <= 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } if (!tscValidateTableNameLength(pCmd->payloadLen)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } } } else if (showType == TSDB_MGMT_TABLE_VNODES) { if (pShowInfo->prefix.type == 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "No specified ip of dnode"); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "No specified ip of dnode"); } // show vnodes may be ip addr of dnode in payload SStrToken* pDnodeIp = &pShowInfo->prefix; if (pDnodeIp->n >= TSDB_IPv4ADDR_LEN) { // ip addr is too long - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } if (!validateIpAddress(pDnodeIp->z, pDnodeIp->n)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } } return TSDB_CODE_SUCCESS; @@ -3023,7 +3012,7 @@ int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType) { SStrToken* idStr = &(pInfo->pMiscInfo->id); if (idStr->n > TSDB_KILL_MSG_LEN) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } strncpy(pCmd->payload, idStr->z, idStr->n); @@ -3035,7 +3024,7 @@ int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType) { int32_t connId = (int32_t)strtol(connIdStr, NULL, 10); if (connId <= 0) { memset(pCmd->payload, 0, strlen(pCmd->payload)); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (killType == TSDB_SQL_KILL_CONNECTION) { @@ -3046,9 +3035,9 @@ int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType) { if (queryId <= 0) { memset(pCmd->payload, 0, strlen(pCmd->payload)); if (killType == TSDB_SQL_KILL_QUERY) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } else { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } } @@ -3070,7 +3059,7 @@ int32_t tscTansformFuncForSTableQuery(SQueryInfo* pQueryInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (pTableMetaInfo->pTableMeta == NULL || !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } assert(tscGetNumOfTags(pTableMetaInfo->pTableMeta) >= 0); @@ -3079,9 +3068,9 @@ int32_t tscTansformFuncForSTableQuery(SQueryInfo* pQueryInfo) { int16_t type = 0; int32_t interBytes = 0; - size_t size = tscSqlExprNumOfExprs(pQueryInfo); + size_t size = tscNumOfExprs(pQueryInfo); for (int32_t k = 0; k < size; ++k) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, k); + SExprInfo* pExpr = tscExprGet(pQueryInfo, k); int16_t functionId = aAggs[pExpr->base.functionId].stableFuncId; int32_t colIndex = pExpr->base.colInfo.colIndex; @@ -3092,10 +3081,10 @@ int32_t tscTansformFuncForSTableQuery(SQueryInfo* pQueryInfo) { (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) { if (getResultDataInfo(pSrcSchema->type, pSrcSchema->bytes, functionId, (int32_t)pExpr->base.param[0].i64, &type, &bytes, &interBytes, 0, true, NULL) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } - tscSqlExprUpdate(pQueryInfo, k, functionId, pExpr->base.colInfo.colIndex, TSDB_DATA_TYPE_BINARY, bytes); + tscExprUpdate(pQueryInfo, k, functionId, pExpr->base.colInfo.colIndex, TSDB_DATA_TYPE_BINARY, bytes); // todo refactor pExpr->base.interBytes = interBytes; } @@ -3112,9 +3101,9 @@ void tscRestoreFuncForSTableQuery(SQueryInfo* pQueryInfo) { return; } - size_t size = tscSqlExprNumOfExprs(pQueryInfo); + size_t size = tscNumOfExprs(pQueryInfo); for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->base.colInfo.colIndex); // the final result size and type in the same as query on single table. @@ -3149,32 +3138,32 @@ bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) const char* msg3 = "function not support for super table query"; // filter sql function not supported by metric query yet. - size_t size = tscSqlExprNumOfExprs(pQueryInfo); + size_t size = tscNumOfExprs(pQueryInfo); for (int32_t i = 0; i < size; ++i) { - int32_t functionId = tscSqlExprGet(pQueryInfo, i)->base.functionId; + int32_t functionId = tscExprGet(pQueryInfo, i)->base.functionId; if (functionId < 0) { continue; } if ((aAggs[functionId].status & TSDB_FUNCSTATE_STABLE) == 0) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); return true; } } if (tscIsTWAQuery(pQueryInfo)) { if (pQueryInfo->groupbyExpr.numOfGroupCols == 0) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); return true; } if (pQueryInfo->groupbyExpr.numOfGroupCols != 1) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); return true; } else { SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, 0); if (pColIndex->colIndex != TSDB_TBNAME_COLUMN_INDEX) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); return true; } } @@ -3207,17 +3196,28 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool int32_t prjNum = 0; int32_t aggNum = 0; - size_t numOfExpr = tscSqlExprNumOfExprs(pQueryInfo); + size_t numOfExpr = tscNumOfExprs(pQueryInfo); assert(numOfExpr > 0); + SExprInfo* pExpr = tscExprGet(pQueryInfo, startIdx); + + // ts function can be simultaneously used with any other functions. + int32_t functionID = pExpr->base.functionId; + if (functionID == TSDB_FUNC_TS || functionID == TSDB_FUNC_TS_DUMMY) { + startIdx++; + } + int32_t factor = INT32_MAX; + if (tscExprGet(pQueryInfo, 0)->base.functionId == TSDB_FUNC_LAST_ROW && (joinQuery || twQuery || !groupbyTagsOrNull(pQueryInfo))) { + return false; + } // diff function cannot be executed with other function // arithmetic function can be executed with other arithmetic functions - size_t size = tscSqlExprNumOfExprs(pQueryInfo); + size_t size = tscNumOfExprs(pQueryInfo); - for (int32_t i = startIdx; i < size; ++i) { - SExprInfo* pExpr1 = tscSqlExprGet(pQueryInfo, i); + for (int32_t i = startIdx + 1; i < size; ++i) { + SExprInfo* pExpr1 = tscExprGet(pQueryInfo, i); int16_t functionId = pExpr1->base.functionId; if (functionId < 0) { @@ -3295,11 +3295,11 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd pQueryInfo->groupbyExpr.numOfGroupCols = (int16_t)taosArrayGetSize(pList); if (pQueryInfo->groupbyExpr.numOfGroupCols > TSDB_MAX_TAGS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (pQueryInfo->numOfTables > 1) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } STableMeta* pTableMeta = NULL; @@ -3316,13 +3316,13 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &token, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } if (tableIndex == COLUMN_INDEX_INITIAL_VAL) { tableIndex = index.tableIndex; } else if (tableIndex != index.tableIndex) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); @@ -3340,14 +3340,14 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd groupTag = true; } - SSqlGroupbyExpr* pGroupExpr = &pQueryInfo->groupbyExpr; + SGroupbyExpr* pGroupExpr = &pQueryInfo->groupbyExpr; if (pGroupExpr->columnInfo == NULL) { pGroupExpr->columnInfo = taosArrayInit(4, sizeof(SColIndex)); } if (groupTag) { if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); } int32_t relIndex = index.columnIndex; @@ -3356,6 +3356,7 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd } SColIndex colIndex = { .colIndex = relIndex, .flag = TSDB_COL_TAG, .colId = pSchema->colId, }; + strncpy(colIndex.name, pSchema->name, tListLen(colIndex.name)); taosArrayPush(pGroupExpr->columnInfo, &colIndex); index.columnIndex = relIndex; @@ -3363,17 +3364,19 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd } else { // check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP || pSchema->type == TSDB_DATA_TYPE_FLOAT || pSchema->type == TSDB_DATA_TYPE_DOUBLE) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); } tscColumnListInsert(pQueryInfo->colList, index.columnIndex, pTableMeta->id.uid, pSchema); SColIndex colIndex = { .colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId }; + strncpy(colIndex.name, pSchema->name, tListLen(colIndex.name)); + taosArrayPush(pGroupExpr->columnInfo, &colIndex); pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC; if (i == 0 && num > 1) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); } } } @@ -3452,7 +3455,7 @@ static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, } if (retVal != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } switch (pExpr->tokenId) { @@ -3484,7 +3487,7 @@ static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, pColumnFilter->lowerRelOptr = TSDB_RELATION_NOTNULL; break; default: - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } return TSDB_CODE_SUCCESS; @@ -3512,7 +3515,7 @@ static int32_t tablenameListToString(tSqlExpr* pExpr, SStringBuilder* sb) { int32_t size = (int32_t) taosArrayGetSize(pList); if (size <= 0) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } if (size > 0) { @@ -3530,7 +3533,7 @@ static int32_t tablenameListToString(tSqlExpr* pExpr, SStringBuilder* sb) { } if (pVar->nLen <= 0 || !tscValidateTableNameLength(pVar->nLen)) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } @@ -3586,7 +3589,7 @@ static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SC return TSDB_CODE_TSC_OUT_OF_MEMORY; } } else { // error; - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } pColFilter->filterstr = @@ -3599,17 +3602,17 @@ static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SC && pExpr->tokenId != TK_NOTNULL && pExpr->tokenId != TK_LIKE ) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } } else { if (pExpr->tokenId == TK_LIKE) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (pSchema->type == TSDB_DATA_TYPE_BOOL) { int32_t t = pExpr->tokenId; if (t != TK_EQ && t != TK_NE && t != TK_NOTNULL && t != TK_ISNULL) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } } } @@ -3633,7 +3636,7 @@ static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* tSqlExpr* pRight = pTableCond->pRight; if (!isTablenameToken(&pLeft->colInfo)) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } int32_t ret = TSDB_CODE_SUCCESS; @@ -3642,14 +3645,14 @@ static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* ret = tablenameListToString(pRight, sb); } else if (pTableCond->tokenId == TK_LIKE) { if (pRight->tokenId != TK_STRING) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } ret = tablenameCondToString(pRight, sb); } if (ret != TSDB_CODE_SUCCESS) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); } return ret; @@ -3670,7 +3673,7 @@ static int32_t getColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSq } else { // handle leaf node SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &pExpr->pLeft->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } return extractColumnFilterInfo(pCmd, pQueryInfo, &index, pExpr, relOptr); @@ -3699,7 +3702,7 @@ static int32_t checkAndSetJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tS SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &pExpr->pLeft->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); @@ -3709,7 +3712,7 @@ static int32_t checkAndSetJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tS SJoinNode **leftNode = &pQueryInfo->tagCond.joinInfo.joinTables[index.tableIndex]; if (*leftNode == NULL) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } (*leftNode)->uid = pTableMetaInfo->pTableMeta->id.uid; @@ -3723,7 +3726,7 @@ static int32_t checkAndSetJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tS tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMeta->id.uid, pTagSchema1); if (taosArrayGetSize(pTableMetaInfo->tagColList) > 1) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } } } @@ -3732,7 +3735,7 @@ static int32_t checkAndSetJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tS index = (SColumnIndex)COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &pExpr->pRight->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); @@ -3742,7 +3745,7 @@ static int32_t checkAndSetJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tS SJoinNode **rightNode = &pQueryInfo->tagCond.joinInfo.joinTables[index.tableIndex]; if (*rightNode == NULL) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } (*rightNode)->uid = pTableMetaInfo->pTableMeta->id.uid; @@ -3755,7 +3758,7 @@ static int32_t checkAndSetJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tS tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMeta->id.uid, pTagSchema2); if (taosArrayGetSize(pTableMetaInfo->tagColList) > 1) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } } } @@ -3763,7 +3766,7 @@ static int32_t checkAndSetJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tS int16_t rightIdx = index.tableIndex; if (pTagSchema1->type != pTagSchema2->type) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } if ((*leftNode)->tagJoin == NULL) { @@ -3797,12 +3800,12 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSqlExpr* pExpr, SQueryInfo* pQuer if (*type == NON_ARITHMEIC_EXPR) { *type = NORMAL_ARITHMETIC; } else if (*type == AGG_ARIGHTMEIC) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &pExpr->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } // if column is timestamp, bool, binary, nchar, not support arithmetic, so return invalid sql @@ -3811,52 +3814,54 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSqlExpr* pExpr, SQueryInfo* pQuer if ((pSchema->type == TSDB_DATA_TYPE_TIMESTAMP) || (pSchema->type == TSDB_DATA_TYPE_BOOL) || (pSchema->type == TSDB_DATA_TYPE_BINARY) || (pSchema->type == TSDB_DATA_TYPE_NCHAR)) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } pList->ids[pList->num++] = index; } else if ((pExpr->tokenId == TK_FLOAT && (isnan(pExpr->value.dKey) || isinf(pExpr->value.dKey))) || pExpr->tokenId == TK_NULL) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } else if (pExpr->type == SQL_NODE_SQLFUNCTION) { if (*type == NON_ARITHMEIC_EXPR) { *type = AGG_ARIGHTMEIC; } else if (*type == NORMAL_ARITHMETIC) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } - int32_t outputIndex = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); + int32_t outputIndex = (int32_t)tscNumOfExprs(pQueryInfo); tSqlExprItem item = {.pNode = pExpr, .aliasName = NULL}; // sql function list in selection clause. // Append the sqlExpr into exprList of pQueryInfo structure sequentially pExpr->functionId = isValidFunction(pExpr->operand.z, pExpr->operand.n); + if (pExpr->functionId < 0) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } - int32_t code = addExprAndResultField(pCmd, pQueryInfo, outputIndex, &item, false); - if (code != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, &item, false) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; } // It is invalid in case of more than one sqlExpr, such as first(ts, k) - last(ts, k) - int32_t inc = (int32_t) tscSqlExprNumOfExprs(pQueryInfo) - outputIndex; + int32_t inc = (int32_t) tscNumOfExprs(pQueryInfo) - outputIndex; if (inc > 1) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } // Not supported data type in arithmetic expression uint64_t id = -1; for(int32_t i = 0; i < inc; ++i) { - SExprInfo* p1 = tscSqlExprGet(pQueryInfo, i + outputIndex); + SExprInfo* p1 = tscExprGet(pQueryInfo, i + outputIndex); int16_t t = p1->base.resType; if (t == TSDB_DATA_TYPE_BINARY || t == TSDB_DATA_TYPE_NCHAR || t == TSDB_DATA_TYPE_BOOL || t == TSDB_DATA_TYPE_TIMESTAMP) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } if (i == 0) { id = p1->base.uid; } else if (id != p1->base.uid) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } @@ -3901,7 +3906,7 @@ static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSqlExpr* pExpr, SQueryI // the expression not from the same table, return error if (uidLeft != uidRight && uidLeft != 0 && uidRight != 0) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } @@ -3985,14 +3990,14 @@ static bool validateJoinExprNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr } if (pExpr->tokenId != TK_EQ) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); return false; } SColumnIndex rightIndex = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &pRight->colInfo, pQueryInfo, &rightIndex) != TSDB_CODE_SUCCESS) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); return false; } @@ -4001,21 +4006,25 @@ static bool validateJoinExprNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr SSchema* pLeftSchema = tscGetTableSchema(pLeftMeterMeta->pTableMeta); int16_t leftType = pLeftSchema[pLeftIndex->columnIndex].type; + tscColumnListInsert(pQueryInfo->colList, pLeftIndex->columnIndex, pLeftMeterMeta->pTableMeta->id.uid, &pLeftSchema[pLeftIndex->columnIndex]); + STableMetaInfo* pRightMeterMeta = tscGetMetaInfo(pQueryInfo, rightIndex.tableIndex); SSchema* pRightSchema = tscGetTableSchema(pRightMeterMeta->pTableMeta); int16_t rightType = pRightSchema[rightIndex.columnIndex].type; + tscColumnListInsert(pQueryInfo->colList, rightIndex.columnIndex, pRightMeterMeta->pTableMeta->id.uid, &pRightSchema[rightIndex.columnIndex]); + if (leftType != rightType) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); return false; } else if (pLeftIndex->tableIndex == rightIndex.tableIndex) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); return false; } // table to table/ super table to super table are allowed if (UTIL_TABLE_IS_SUPER_TABLE(pLeftMeterMeta) != UTIL_TABLE_IS_SUPER_TABLE(pRightMeterMeta)) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); return false; } @@ -4037,7 +4046,7 @@ static bool validTableNameOptr(tSqlExpr* pExpr) { static int32_t setExprToCond(tSqlExpr** parent, tSqlExpr* pExpr, const char* msg, int32_t parentOptr, char* msgBuf) { if (*parent != NULL) { if (parentOptr == TK_OR && msg != NULL) { - return invalidSqlErrMsg(msgBuf, msg); + return invalidOperationMsg(msgBuf, msg); } *parent = tSqlExprCreate((*parent), pExpr, parentOptr); @@ -4053,7 +4062,7 @@ static int32_t validateNullExpr(tSqlExpr* pExpr, char* msgBuf) { tSqlExpr* pRight = pExpr->pRight; if (pRight->tokenId == TK_NULL && (!(pExpr->tokenId == TK_ISNULL || pExpr->tokenId == TK_NOTNULL))) { - return invalidSqlErrMsg(msgBuf, msg); + return invalidOperationMsg(msgBuf, msg); } return TSDB_CODE_SUCCESS; @@ -4069,12 +4078,12 @@ static int32_t validateLikeExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t if (pExpr->tokenId == TK_LIKE) { if (pRight->value.nLen > TSDB_PATTERN_STRING_MAX_LEN) { - return invalidSqlErrMsg(msgBuf, msg1); + return invalidOperationMsg(msgBuf, msg1); } SSchema* pSchema = tscGetTableSchema(pTableMeta); if ((!isTablenameToken(&pLeft->colInfo)) && !IS_VAR_DATA_TYPE(pSchema[index].type)) { - return invalidSqlErrMsg(msgBuf, msg2); + return invalidOperationMsg(msgBuf, msg2); } } @@ -4098,7 +4107,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &pLeft->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } assert(tSqlExprIsParentOfLeaf(*pExpr)); @@ -4120,7 +4129,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql if (index.columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { // query on time range if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } // set join query condition @@ -4140,11 +4149,11 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql int16_t leftIdx = index.tableIndex; if (getColumnIndexByName(pCmd, &pRight->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } if (index.tableIndex < 0 || index.tableIndex >= TSDB_MAX_JOIN_TABLE_NUM) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } SJoinNode **rightNode = &pQueryInfo->tagCond.joinInfo.joinTables[index.tableIndex]; @@ -4169,7 +4178,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql taosArrayPush((*rightNode)->tsJoin, &leftIdx); /* - * to release expression, e.g., m1.ts = m2.ts, + * To release expression, e.g., m1.ts = m2.ts, * since this expression is used to set the join query type */ tSqlExprDestroy(*pExpr); @@ -4182,17 +4191,17 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql } else if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) || index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { // query on tags, check for tag query condition if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } // in case of in operator, keep it in a seprate attribute if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { if (!validTableNameOptr(*pExpr)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); } if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (pCondExpr->pTableCond == NULL) { @@ -4200,7 +4209,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql pCondExpr->relType = parentOptr; pCondExpr->tableCondIndex = index.tableIndex; } else { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } *type = TSQL_EXPR_TBNAME; @@ -4208,7 +4217,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql } else { if (pRight != NULL && pRight->tokenId == TK_ID) { // join on tag columns for stable query if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } pQueryInfo->type |= TSDB_QUERY_TYPE_JOIN_QUERY; @@ -4227,7 +4236,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql *type = TSQL_EXPR_COLUMN; if (pRight->tokenId == TK_ID) { // other column cannot be served as the join column - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } ret = setExprToCond(&pCondExpr->pColumnCond, *pExpr, NULL, parentOptr, pQueryInfo->msg); @@ -4246,14 +4255,14 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr const char* msg1 = "query condition between different columns must use 'AND'"; if ((*pExpr)->flags & (1 << EXPR_FLAG_TS_ERROR)) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } tSqlExpr* pLeft = (*pExpr)->pLeft; tSqlExpr* pRight = (*pExpr)->pRight; if (!isValidExpr(pLeft, pRight, (*pExpr)->tokenId)) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } int32_t leftType = -1; @@ -4276,7 +4285,7 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr */ if (leftType != rightType) { if ((*pExpr)->tokenId == TK_OR && (leftType + rightType != TSQL_EXPR_TBNAME + TSQL_EXPR_TAG)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } } @@ -4287,11 +4296,11 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr exchangeExpr(*pExpr); if (pLeft->tokenId == TK_ID && pRight->tokenId == TK_TIMESTAMP && (pRight->flags & (1 << EXPR_FLAG_TIMESTAMP_VAR))) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } if ((pLeft->flags & (1 << EXPR_FLAG_TS_ERROR)) || (pRight->flags & (1 << EXPR_FLAG_TS_ERROR))) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } return handleExprInQueryCond(pCmd, pQueryInfo, pExpr, pCondExpr, type, parentOptr); @@ -4403,7 +4412,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, taosStringBuilderDestroy(&sb1); tfree(segments); - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); return ret; } @@ -4458,7 +4467,7 @@ static int32_t getTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlE if (!tSqlExprIsParentOfLeaf(pExpr)) { if (pExpr->tokenId == TK_OR) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } code = getTimeRangeFromExpr(pCmd, pQueryInfo, pExpr->pLeft); @@ -4470,7 +4479,7 @@ static int32_t getTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlE } else { SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &pExpr->pLeft->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); @@ -4480,7 +4489,7 @@ static int32_t getTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlE STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX}; if (getTimeRange(&win, pRight, pExpr->tokenId, tinfo.precision) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); } // update the timestamp query range @@ -4505,7 +4514,7 @@ static int32_t validateJoinExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr if (pQueryInfo->numOfTables == 1) { return TSDB_CODE_SUCCESS; } else { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } } @@ -4513,12 +4522,12 @@ static int32_t validateJoinExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // for stable join, tag columns // must be present for join if (pCondExpr->pJoinExpr == NULL) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } } if (!pCondExpr->tsJoin) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } return TSDB_CODE_SUCCESS; @@ -4589,7 +4598,7 @@ static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { } if (IS_ARITHMETIC_OPTR(p->_node.optr)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (!IS_RELATION_OPTR(p->_node.optr)) { @@ -4647,7 +4656,7 @@ static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { } if (retVal != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } } while (0); @@ -4708,7 +4717,7 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE taosArrayDestroy(colList); if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0 && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "filter on tag not supported for normal table"); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "filter on tag not supported for normal table"); } if (ret) { @@ -4728,7 +4737,7 @@ int32_t validateJoinNodes(SQueryInfo* pQueryInfo, SSqlObj* pSql) { SJoinNode *node = pQueryInfo->tagCond.joinInfo.joinTables[i]; if (node == NULL || node->tsJoin == NULL || taosArrayGetSize(node->tsJoin) <= 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(&pSql->cmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(&pSql->cmd), msg1); } } @@ -4738,7 +4747,7 @@ int32_t validateJoinNodes(SQueryInfo* pQueryInfo, SSqlObj* pSql) { SJoinNode *node = pQueryInfo->tagCond.joinInfo.joinTables[i]; if (node == NULL || node->tagJoin == NULL || taosArrayGetSize(node->tagJoin) <= 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(&pSql->cmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(&pSql->cmd), msg2); } } } @@ -4786,7 +4795,7 @@ int32_t mergeJoinNodes(SQueryInfo* pQueryInfo, SSqlObj* pSql) { } if (taosArrayGetSize(pQueryInfo->tagCond.joinInfo.joinTables[0]->tsJoin) != pQueryInfo->numOfTables) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(&pSql->cmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(&pSql->cmd), msg1); } STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -4807,7 +4816,7 @@ int32_t mergeJoinNodes(SQueryInfo* pQueryInfo, SSqlObj* pSql) { } if (taosArrayGetSize(pQueryInfo->tagCond.joinInfo.joinTables[0]->tagJoin) != pQueryInfo->numOfTables) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(&pSql->cmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(&pSql->cmd), msg2); } } @@ -4831,7 +4840,7 @@ int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSq SCondExpr condExpr = {0}; if ((*pExpr)->pLeft == NULL || (*pExpr)->pRight == NULL) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(&pSql->cmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(&pSql->cmd), msg1); } int32_t type = 0; @@ -4884,7 +4893,7 @@ int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSq } if (!validateFilterExpr(pQueryInfo)) { - ret = invalidSqlErrMsg(tscGetErrorMsgPayload(&pSql->cmd), msg2); + ret = invalidOperationMsg(tscGetErrorMsgPayload(&pSql->cmd), msg2); goto PARSE_WHERE_EXIT; } @@ -4918,7 +4927,7 @@ int32_t getTimeRange(STimeWindow* win, tSqlExpr* pRight, int32_t optr, int16_t t * where ts in ('2015-12-12 4:8:12') */ if (pRight->tokenId == TK_SET || optr == TK_IN) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } int64_t val = 0; @@ -4931,14 +4940,14 @@ int32_t getTimeRange(STimeWindow* win, tSqlExpr* pRight, int32_t optr, int16_t t if (taosParseTime(pRight->value.pz, &val, pRight->value.nLen, TSDB_TIME_PRECISION_MICRO, tsDaylight) == TSDB_CODE_SUCCESS) { parsed = true; } else { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } else { SStrToken token = {.z = pRight->value.pz, .n = pRight->value.nLen, .type = TK_ID}; int32_t len = tGetToken(pRight->value.pz, &token.type); if ((token.type != TK_INTEGER && token.type != TK_FLOAT) || len != pRight->value.nLen) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } } else if (pRight->tokenId == TK_INTEGER && timePrecision == TSDB_TIME_PRECISION_MILLI) { @@ -5019,7 +5028,7 @@ int32_t tsRewriteFieldNameIfNecessary(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { for (int32_t j = i + 1; j < pQueryInfo->fieldsInfo.numOfOutput; ++j) { if (strncasecmp(fieldName, tscFieldInfoGetField(&pQueryInfo->fieldsInfo, j)->name, (TSDB_COL_NAME_LEN - 1)) == 0) { const char* msg = "duplicated column name in new table"; - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } } } @@ -5044,7 +5053,7 @@ int32_t validateFillNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNo const char* msg5 = "fill only available for interval query"; if ((!isTimeWindowQuery(pQueryInfo)) && (!tscIsPointInterpQuery(pQueryInfo))) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } /* @@ -5052,12 +5061,12 @@ int32_t validateFillNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNo * the columns may be increased due to group by operation */ if (checkQueryRangeForFill(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } if (pItem->pVar.nType != TSDB_DATA_TYPE_BINARY) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } size_t numOfFields = tscNumOfFields(pQueryInfo); @@ -5088,7 +5097,7 @@ int32_t validateFillNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNo size_t num = taosArrayGetSize(pFillToken); if (num == 1) { // no actual value, return with error code - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } int32_t startPos = 1; @@ -5118,7 +5127,7 @@ int32_t validateFillNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNo tVariant* p = taosArrayGet(pFillToken, j); int32_t ret = tVariantDump(p, (char*)&pQueryInfo->fillVal[i], pField->type, true); if (ret != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } } @@ -5136,14 +5145,14 @@ int32_t validateFillNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNo } } } else { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + size_t numOfExprs = tscNumOfExprs(pQueryInfo); for(int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (pExpr->base.functionId == TSDB_FUNC_TOP || pExpr->base.functionId == TSDB_FUNC_BOTTOM) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } } @@ -5197,11 +5206,11 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq size_t size = taosArrayGetSize(pSortorder); if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { if (size > 1) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); } } else { if (size > 2) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } } @@ -5218,7 +5227,7 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // super table query if (getColumnIndexByName(pCmd, &columnName, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } bool orderByTags = false; @@ -5229,7 +5238,7 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq // it is a tag column if (pQueryInfo->groupbyExpr.columnInfo == NULL) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, 0); if (relTagIndex == pColIndex->colIndex) { @@ -5244,7 +5253,7 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq } if (!(orderByTags || orderByTS) && !isTopBottomQuery(pQueryInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } else { // order by top/bottom result value column is not supported in case of interval query. assert(!(orderByTags && orderByTS)); } @@ -5258,12 +5267,12 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq pQueryInfo->groupbyExpr.orderType = p1->sortOrder; } else if (isTopBottomQuery(pQueryInfo)) { /* order of top/bottom query in interval is not valid */ - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, 0); + SExprInfo* pExpr = tscExprGet(pQueryInfo, 0); assert(pExpr->base.functionId == TSDB_FUNC_TS); - pExpr = tscSqlExprGet(pQueryInfo, 1); + pExpr = tscExprGet(pQueryInfo, 1); if (pExpr->base.colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); @@ -5278,7 +5287,7 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq // orderby ts query on super table if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - addPrimaryTsColIntoResult(pQueryInfo); + addPrimaryTsColIntoResult(pQueryInfo, pCmd); } } } @@ -5297,11 +5306,11 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq tVariant* pVar2 = &pItem->pVar; SStrToken cname = {pVar2->nLen, pVar2->nType, pVar2->pz}; if (getColumnIndexByName(pCmd, &cname, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } else { tVariantListItem* p1 = taosArrayGet(pSortorder, 1); pQueryInfo->order.order = p1->sortOrder; @@ -5311,21 +5320,21 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq } else { // meter query if (getColumnIndexByName(pCmd, &columnName, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX && !isTopBottomQuery(pQueryInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } if (isTopBottomQuery(pQueryInfo)) { /* order of top/bottom query in interval is not valid */ - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, 0); + SExprInfo* pExpr = tscExprGet(pQueryInfo, 0); assert(pExpr->base.functionId == TSDB_FUNC_TS); - pExpr = tscSqlExprGet(pQueryInfo, 1); + pExpr = tscExprGet(pQueryInfo, 1); if (pExpr->base.colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } tVariantListItem* pItem = taosArrayGet(pSqlNode->pSortOrder, 0); @@ -5371,15 +5380,15 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { SSqlCmd* pCmd = &pSql->cmd; SAlterTableInfo* pAlterSQL = pInfo->pAlterInfo; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, DEFAULT_TABLE_INDEX); if (tscValidateName(&(pAlterSQL->name)) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } - code = tscSetTableFullName(pTableMetaInfo, &(pAlterSQL->name), pSql); + code = tscSetTableFullName(&pTableMetaInfo->name, &(pAlterSQL->name), pSql); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -5392,60 +5401,60 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; if (pAlterSQL->tableType == TSDB_SUPER_TABLE && !(UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo))) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg20); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg20); } if (pAlterSQL->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) { if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } } else if ((pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) && (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo))) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } else if ((pAlterSQL->type == TSDB_ALTER_TABLE_ADD_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_DROP_COLUMN) && UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } if (pAlterSQL->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) { SArray* pFieldList = pAlterSQL->pAddColumns; if (taosArrayGetSize(pFieldList) > 1) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } TAOS_FIELD* p = taosArrayGet(pFieldList, 0); if (!validateOneTags(pCmd, p)) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p); } else if (pAlterSQL->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) { if (tscGetNumOfTags(pTableMeta) == 1) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); } // numOfTags == 1 if (taosArrayGetSize(pAlterSQL->varList) > 1) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); } tVariantListItem* pItem = taosArrayGet(pAlterSQL->varList, 0); if (pItem->pVar.nLen >= TSDB_COL_NAME_LEN) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); } SColumnIndex index = COLUMN_INDEX_INITIALIZER; SStrToken name = {.z = pItem->pVar.pz, .n = pItem->pVar.nLen, .type = TK_STRING}; if (getColumnIndexByName(pCmd, &name, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } int32_t numOfCols = tscGetNumOfColumns(pTableMeta); if (index.columnIndex < numOfCols) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg10); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg10); } else if (index.columnIndex == numOfCols) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg11); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg11); } char name1[128] = {0}; @@ -5456,18 +5465,18 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } else if (pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) { SArray* pVarList = pAlterSQL->varList; if (taosArrayGetSize(pVarList) > 2) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } tVariantListItem* pSrcItem = taosArrayGet(pAlterSQL->varList, 0); tVariantListItem* pDstItem = taosArrayGet(pAlterSQL->varList, 1); if (pSrcItem->pVar.nLen >= TSDB_COL_NAME_LEN || pDstItem->pVar.nLen >= TSDB_COL_NAME_LEN) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); } if (pSrcItem->pVar.nType != TSDB_DATA_TYPE_BINARY || pDstItem->pVar.nType != TSDB_DATA_TYPE_BINARY) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg10); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg10); } SColumnIndex srcIndex = COLUMN_INDEX_INITIALIZER; @@ -5475,12 +5484,12 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { SStrToken srcToken = {.z = pSrcItem->pVar.pz, .n = pSrcItem->pVar.nLen, .type = TK_STRING}; if (getColumnIndexByName(pCmd, &srcToken, pQueryInfo, &srcIndex) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg17); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg17); } SStrToken destToken = {.z = pDstItem->pVar.pz, .n = pDstItem->pVar.nLen, .type = TK_STRING}; if (getColumnIndexByName(pCmd, &destToken, pQueryInfo, &destIndex) == TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg19); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg19); } tVariantListItem* pItem = taosArrayGet(pVarList, 0); @@ -5508,11 +5517,11 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER; SStrToken name = {.type = TK_STRING, .z = item->pVar.pz, .n = item->pVar.nLen}; if (getColumnIndexByName(pCmd, &name, pQueryInfo, &columnIndex) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } if (columnIndex.columnIndex < tscGetNumOfColumns(pTableMeta)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg12); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg12); } tVariantListItem* pItem = taosArrayGet(pVarList, 1); @@ -5520,7 +5529,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { pAlterSQL->tagData.data = calloc(1, pTagsSchema->bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); if (tVariantDump(&pItem->pVar, pAlterSQL->tagData.data, pTagsSchema->type, true) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg13); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg13); } pAlterSQL->tagData.dataLen = pTagsSchema->bytes; @@ -5528,7 +5537,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { // validate the length of binary if ((pTagsSchema->type == TSDB_DATA_TYPE_BINARY || pTagsSchema->type == TSDB_DATA_TYPE_NCHAR) && varDataTLen(pAlterSQL->tagData.data) > pTagsSchema->bytes) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg14); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg14); } int32_t schemaLen = sizeof(STColumn) * numOfTags; @@ -5583,23 +5592,23 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { SArray* pFieldList = pAlterSQL->pAddColumns; if (taosArrayGetSize(pFieldList) > 1) { const char* msg = "only support add one column"; - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } TAOS_FIELD* p = taosArrayGet(pFieldList, 0); if (!validateOneColumn(pCmd, p)) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p); } else if (pAlterSQL->type == TSDB_ALTER_TABLE_DROP_COLUMN) { if (tscGetNumOfColumns(pTableMeta) == TSDB_MIN_COLUMNS) { // - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg15); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg15); } size_t size = taosArrayGetSize(pAlterSQL->varList); if (size > 1) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg16); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg16); } tVariantListItem* pItem = taosArrayGet(pAlterSQL->varList, 0); @@ -5607,11 +5616,11 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER; SStrToken name = {.type = TK_STRING, .z = pItem->pVar.pz, .n = pItem->pVar.nLen}; if (getColumnIndexByName(pCmd, &name, pQueryInfo, &columnIndex) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg17); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg17); } if (columnIndex.columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg18); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg18); } char name1[TSDB_COL_NAME_LEN] = {0}; @@ -5630,14 +5639,14 @@ int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { if (pQueryInfo->interval.interval != 0 && pQueryInfo->interval.interval < 10 && pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit != 'y') { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); } size_t size = taosArrayGetSize(pQueryInfo->exprList); for (int32_t i = 0; i < size; ++i) { - int32_t functId = tscSqlExprGet(pQueryInfo, i)->base.functionId; + int32_t functId = tscExprGet(pQueryInfo, i)->base.functionId; if (!IS_STREAM_QUERY_VALID(aAggs[functId].status)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } } @@ -5652,7 +5661,7 @@ int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQu size_t size = taosArrayGetSize(pQueryInfo->exprList); for (int32_t k = 0; k < size; ++k) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, k); + SExprInfo* pExpr = tscExprGet(pQueryInfo, k); if (pExpr->base.functionId < 0) { SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * pExpr->base.functionId - 1); @@ -5668,7 +5677,7 @@ int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQu if (pExpr->base.functionId == TSDB_FUNC_PRJ && pExpr->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { bool hasSelectivity = false; for (int32_t j = 0; j < size; ++j) { - SExprInfo* pEx = tscSqlExprGet(pQueryInfo, j); + SExprInfo* pEx = tscExprGet(pQueryInfo, j); if ((aAggs[pEx->base.functionId].status & TSDB_FUNCSTATE_SELECTIVITY) == TSDB_FUNCSTATE_SELECTIVITY) { hasSelectivity = true; break; @@ -5688,10 +5697,10 @@ int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQu } if (isProjectionFunction) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } - return isProjectionFunction == true ? TSDB_CODE_TSC_INVALID_SQL : TSDB_CODE_SUCCESS; + return isProjectionFunction == true ? TSDB_CODE_TSC_INVALID_OPERATION : TSDB_CODE_SUCCESS; } typedef struct SDNodeDynConfOption { @@ -5708,12 +5717,12 @@ int32_t validateEp(char* ep) { if (NULL == pos) { int32_t val = strtol(ep, NULL, 10); if (val <= 0 || val > 65536) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } else { uint16_t port = atoi(pos + 1); if (0 == port) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } @@ -5724,7 +5733,7 @@ int32_t validateDNodeConfig(SMiscInfo* pOptions) { int32_t numOfToken = (int32_t) taosArrayGetSize(pOptions->a); if (numOfToken < 2 || numOfToken > 3) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } const int tokenLogEnd = 2; @@ -5759,7 +5768,7 @@ int32_t validateDNodeConfig(SMiscInfo* pOptions) { strdequote(pValToken->z); bool parseOk = taosCheckBalanceCfgOptions(pValToken->z, &vnodeId, &dnodeId); if (!parseOk) { - return TSDB_CODE_TSC_INVALID_SQL; // options value is invalid + return TSDB_CODE_TSC_INVALID_OPERATION; // options value is invalid } return TSDB_CODE_SUCCESS; } else if ((strncasecmp(cfgOptions[tokenMonitor].name, pOptionToken->z, pOptionToken->n) == 0) && @@ -5767,7 +5776,7 @@ int32_t validateDNodeConfig(SMiscInfo* pOptions) { SStrToken* pValToken = taosArrayGet(pOptions->a, 2); int32_t val = strtol(pValToken->z, NULL, 10); if (val != 0 && val != 1) { - return TSDB_CODE_TSC_INVALID_SQL; // options value is invalid + return TSDB_CODE_TSC_INVALID_OPERATION; // options value is invalid } return TSDB_CODE_SUCCESS; } else { @@ -5776,7 +5785,7 @@ int32_t validateDNodeConfig(SMiscInfo* pOptions) { int32_t val = strtol(pValToken->z, NULL, 10); if (val < 0 || val > 256) { /* options value is out of valid range */ - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } for (int32_t i = tokenDebugFlag; i < tokenDebugFlagEnd; ++i) { @@ -5789,13 +5798,13 @@ int32_t validateDNodeConfig(SMiscInfo* pOptions) { } } - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } int32_t validateLocalConfig(SMiscInfo* pOptions) { int32_t numOfToken = (int32_t) taosArrayGetSize(pOptions->a); if (numOfToken < 1 || numOfToken > 2) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } SDNodeDynConfOption LOCAL_DYNAMIC_CFG_OPTIONS[6] = {{"resetLog", 8}, {"rpcDebugFlag", 12}, {"tmrDebugFlag", 12}, @@ -5818,7 +5827,7 @@ int32_t validateLocalConfig(SMiscInfo* pOptions) { int32_t val = strtol(pValToken->z, NULL, 10); if (!validateDebugFlag(val)) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } for (int32_t i = 1; i < tListLen(LOCAL_DYNAMIC_CFG_OPTIONS); ++i) { @@ -5829,20 +5838,20 @@ int32_t validateLocalConfig(SMiscInfo* pOptions) { } } } - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } int32_t validateColumnName(char* name) { bool ret = taosIsKeyWordToken(name, (int32_t)strlen(name)); if (ret) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } SStrToken token = {.z = name}; token.n = tGetToken(name, &token.type); if (token.type != TK_STRING && token.type != TK_ID) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } if (token.type == TK_STRING) { @@ -5852,13 +5861,13 @@ int32_t validateColumnName(char* name) { int32_t k = tGetToken(token.z, &token.type); if (k != token.n) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } return validateColumnName(token.z); } else { if (isNumber(&token)) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } @@ -5877,7 +5886,7 @@ bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo) { return !(pQueryInfo->window.skey != pQueryInfo->window.ekey && pQueryInfo->interval.interval == 0); } -int32_t validateLimitNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIndex, SSqlNode* pSqlNode, SSqlObj* pSql) { +int32_t validateLimitNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, SSqlObj* pSql) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); const char* msg0 = "soffset/offset can not be less than 0"; @@ -5893,7 +5902,7 @@ int32_t validateLimitNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseI pQueryInfo->limit.limit, pQueryInfo->limit.offset, pQueryInfo->slimit.limit, pQueryInfo->slimit.offset); if (pQueryInfo->slimit.offset < 0 || pQueryInfo->limit.offset < 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); } if (pQueryInfo->limit.limit == 0) { @@ -5907,7 +5916,7 @@ int32_t validateLimitNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseI if (!tscQueryTags(pQueryInfo)) { // local handle the super table tag query if (tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { if (pQueryInfo->slimit.limit > 0 || pQueryInfo->slimit.offset > 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } // for projection query on super table, all queries are subqueries @@ -5930,10 +5939,7 @@ int32_t validateLimitNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseI * And then launching multiple async-queries against all qualified virtual nodes, during the first-stage * query operation. */ - int32_t code = tscGetSTableVgroupInfo(pSql, clauseIndex); - if (code != TSDB_CODE_SUCCESS) { - return code; - } +// assert(allVgroupInfoRetrieved(pQueryInfo)); // No tables included. No results generated. Query results are empty. if (pTableMetaInfo->vgroupList->numOfVgroups == 0) { @@ -5963,7 +5969,7 @@ int32_t validateLimitNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseI } } else { if (pQueryInfo->slimit.limit != -1 || pQueryInfo->slimit.offset != 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } } @@ -6001,7 +6007,7 @@ static int32_t setKeepOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDbInfo* p pMsg->daysToKeep2 = htonl((int32_t)p2->pVar.i64); break; } - default: { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } + default: { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } } } @@ -6025,7 +6031,7 @@ static int32_t setTimePrecision(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDbInfo strlen(TSDB_TIME_PRECISION_MICRO_STR) == pToken->n) { pMsg->precision = TSDB_TIME_PRECISION_MICRO; } else { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } } @@ -6057,30 +6063,30 @@ int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDbInfo* pCreateDbSql) { setCreateDBOption(pMsg, pCreateDbSql); if (setKeepOption(pCmd, pMsg, pCreateDbSql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } if (setTimePrecision(pCmd, pMsg, pCreateDbSql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } if (tscCheckCreateDbParams(pCmd, pMsg) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } return TSDB_CODE_SUCCESS; } void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClauseIndex, int32_t tableIndex) { - SQueryInfo* pParentQueryInfo = tscGetQueryInfo(&pParentObj->cmd, subClauseIndex); + SQueryInfo* pParentQueryInfo = tscGetQueryInfo(&pParentObj->cmd); if (pParentQueryInfo->groupbyExpr.numOfGroupCols > 0) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, subClauseIndex); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); SExprInfo* pExpr = NULL; size_t size = taosArrayGetSize(pQueryInfo->exprList); if (size > 0) { - pExpr = tscSqlExprGet(pQueryInfo, (int32_t)size - 1); + pExpr = tscExprGet(pQueryInfo, (int32_t)size - 1); } if (pExpr == NULL || pExpr->base.functionId != TSDB_FUNC_TAG) { @@ -6097,7 +6103,7 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau int16_t type = pTagSchema->type; int16_t bytes = pTagSchema->bytes; - pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, getNewResColId(pQueryInfo), bytes, true); + pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, getNewResColId(&pSql->cmd), bytes, true); pExpr->base.colInfo.flag = TSDB_COL_TAG; // NOTE: tag column does not add to source column list @@ -6122,16 +6128,19 @@ static void doLimitOutputNormalColOfGroupby(SExprInfo* pExpr) { pExpr->base.numOfParams = 1; } -void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) { +void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex, SSqlCmd* pCmd) { SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, tagIndex); - size_t size = tscSqlExprNumOfExprs(pQueryInfo); + size_t size = tscNumOfExprs(pQueryInfo); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->colIndex); SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = pColIndex->colIndex}; - tscAddFuncInSelectClause(pQueryInfo, (int32_t)size, TSDB_FUNC_PRJ, &colIndex, pSchema, TSDB_COL_NORMAL); + SExprInfo* pExprInfo = tscAddFuncInSelectClause(pQueryInfo, (int32_t)size, TSDB_FUNC_PRJ, &colIndex, pSchema, + TSDB_COL_NORMAL, getNewResColId(pCmd)); + + strncpy(pExprInfo->base.token, pExprInfo->base.colInfo.name, tListLen(pExprInfo->base.token)); int32_t numOfFields = tscNumOfFields(pQueryInfo); SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, numOfFields - 1); @@ -6149,7 +6158,7 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (pExpr->base.functionId == TSDB_FUNC_TAGPRJ || pExpr->base.functionId == TSDB_FUNC_TAG) { pExpr->base.functionId = TSDB_FUNC_TAG_DUMMY; tagLength += pExpr->base.resBytes; @@ -6162,7 +6171,7 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (pExpr->base.functionId < 0) { continue; } @@ -6180,7 +6189,7 @@ static int32_t doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) { size_t size = taosArrayGetSize(pQueryInfo->exprList); for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (pExpr->base.functionId == TSDB_FUNC_PRJ && (!TSDB_COL_IS_UD_COL(pExpr->base.colInfo.flag) && (pExpr->base.colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX))) { bool qualifiedCol = false; @@ -6197,7 +6206,7 @@ static int32_t doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) { // it is not a tag column/tbname column/user-defined column, return error if (!qualifiedCol) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } } @@ -6205,7 +6214,7 @@ static int32_t doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) { return TSDB_CODE_SUCCESS; } -static bool tagColumnInGroupby(SSqlGroupbyExpr* pGroupbyExpr, int16_t columnId) { +static bool tagColumnInGroupby(SGroupbyExpr* pGroupbyExpr, int16_t columnId) { for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, j); @@ -6223,7 +6232,7 @@ static bool onlyTagPrjFunction(SQueryInfo* pQueryInfo) { size_t size = taosArrayGetSize(pQueryInfo->exprList); for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (pExpr->base.functionId == TSDB_FUNC_PRJ) { hasColumnPrj = true; } else if (pExpr->base.functionId == TSDB_FUNC_TAGPRJ) { @@ -6238,9 +6247,9 @@ static bool onlyTagPrjFunction(SQueryInfo* pQueryInfo) { static bool allTagPrjInGroupby(SQueryInfo* pQueryInfo) { bool allInGroupby = true; - size_t size = tscSqlExprNumOfExprs(pQueryInfo); + size_t size = tscNumOfExprs(pQueryInfo); for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (pExpr->base.functionId != TSDB_FUNC_TAGPRJ) { continue; } @@ -6259,7 +6268,7 @@ static void updateTagPrjFunction(SQueryInfo* pQueryInfo) { size_t size = taosArrayGetSize(pQueryInfo->exprList); for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (pExpr->base.functionId == TSDB_FUNC_TAGPRJ) { pExpr->base.functionId = TSDB_FUNC_TAG; } @@ -6319,7 +6328,7 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) // When the tag projection function on tag column that is not in the group by clause, aggregation function and // selectivity function exist in select clause is not allowed. if (numOfAggregation > 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } /* @@ -6338,7 +6347,7 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) * Otherwise, return with error code. */ for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); int16_t functionId = pExpr->base.functionId; if (functionId == TSDB_FUNC_TAGPRJ || (aAggs[functionId].status & TSDB_FUNCSTATE_SELECTIVITY) == 0) { continue; @@ -6348,7 +6357,7 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) (functionId == TSDB_FUNC_LAST_DST && (pExpr->base.colInfo.flag & TSDB_COL_NULL) != 0)) { // do nothing } else { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } } @@ -6361,7 +6370,7 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) } else { if ((pQueryInfo->type & TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0) { if (numOfAggregation > 0 && pQueryInfo->groupbyExpr.numOfGroupCols == 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } if (numOfAggregation > 0 || numOfSelectivity > 0) { @@ -6406,15 +6415,16 @@ static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd, SQueryInfo* pQueryInfo } } - size_t size = tscSqlExprNumOfExprs(pQueryInfo); + size_t size = tscNumOfExprs(pQueryInfo); if (TSDB_COL_IS_TAG(pColIndex->flag)) { SColumnIndex index = {.tableIndex = pQueryInfo->groupbyExpr.tableIndex, .columnIndex = colIndex}; - SExprInfo* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, s->type, s->bytes, - getNewResColId(pQueryInfo), s->bytes, true); + SExprInfo* pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, s->type, s->bytes, + getNewResColId(pCmd), s->bytes, true); memset(pExpr->base.aliasName, 0, sizeof(pExpr->base.aliasName)); tstrncpy(pExpr->base.aliasName, s->name, sizeof(pExpr->base.aliasName)); + tstrncpy(pExpr->base.token, s->name, sizeof(pExpr->base.aliasName)); pExpr->base.colInfo.flag = TSDB_COL_TAG; @@ -6424,12 +6434,12 @@ static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd, SQueryInfo* pQueryInfo } else { // if this query is "group by" normal column, time window query is not allowed if (isTimeWindowQuery(pQueryInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } bool hasGroupColumn = false; for (int32_t j = 0; j < size; ++j) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, j); + SExprInfo* pExpr = tscExprGet(pQueryInfo, j); if ((pExpr->base.functionId == TSDB_FUNC_PRJ) && pExpr->base.colInfo.colId == pColIndex->colId) { hasGroupColumn = true; break; @@ -6438,7 +6448,7 @@ static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd, SQueryInfo* pQueryInfo //if the group by column does not required by user, add an invisible column into the final result set. if (!hasGroupColumn) { - doAddGroupColumnForSubquery(pQueryInfo, i); + doAddGroupColumnForSubquery(pQueryInfo, i, pCmd); } } } @@ -6450,10 +6460,10 @@ static int32_t doTagFunctionCheck(SQueryInfo* pQueryInfo) { bool tagProjection = false; bool tableCounting = false; - int32_t numOfCols = (int32_t) tscSqlExprNumOfExprs(pQueryInfo); + int32_t numOfCols = (int32_t) tscNumOfExprs(pQueryInfo); for (int32_t i = 0; i < numOfCols; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); int32_t functionId = pExpr->base.functionId; if (functionId == TSDB_FUNC_TAGPRJ) { @@ -6480,11 +6490,11 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { // only retrieve tags, group by is not supportted if (tscQueryTags(pQueryInfo)) { if (doTagFunctionCheck(pQueryInfo) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } if (pQueryInfo->groupbyExpr.numOfGroupCols > 0 || isTimeWindowQuery(pQueryInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } else { return TSDB_CODE_SUCCESS; } @@ -6499,9 +6509,9 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { } // check all query functions in selection clause, multi-output functions are not allowed - size_t size = tscSqlExprNumOfExprs(pQueryInfo); + size_t size = tscNumOfExprs(pQueryInfo); for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); int32_t functId = pExpr->base.functionId; /* @@ -6519,7 +6529,7 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { } if (!qualified) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } } @@ -6529,25 +6539,25 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { if (IS_MULTIOUTPUT(aAggs[functId].status) && functId != TSDB_FUNC_TOP && functId != TSDB_FUNC_BOTTOM && functId != TSDB_FUNC_TAGPRJ && functId != TSDB_FUNC_PRJ) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (functId == TSDB_FUNC_COUNT && pExpr->base.colInfo.colIndex == TSDB_TBNAME_COLUMN_INDEX) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } } if (checkUpdateTagPrjFunctions(pQueryInfo, pCmd) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } if (doAddGroupbyColumnsOnDemand(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } // projection query on super table does not compatible with "group by" syntax if (tscIsProjectionQuery(pQueryInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } return TSDB_CODE_SUCCESS; @@ -6563,7 +6573,7 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq SArray* pExprList = pSqlNode->pSelNodeList; size_t size = taosArrayGetSize(pExprList); if (size != 1) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } bool server_status = false; @@ -6574,7 +6584,7 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq if (pExpr->token.n == 1 && 0 == strncasecmp(pExpr->token.z, "1", 1)) { server_status = true; } else { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } } // TODO redefine the function @@ -6608,12 +6618,12 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq pQueryInfo->command = TSDB_SQL_CLI_VERSION;break; case 4: pQueryInfo->command = TSDB_SQL_CURRENT_USER;break; - default: { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } + default: { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } } SColumnIndex ind = {0}; - SExprInfo* pExpr1 = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG_DUMMY, &ind, TSDB_DATA_TYPE_INT, - tDataTypes[TSDB_DATA_TYPE_INT].bytes, getNewResColId(pQueryInfo), tDataTypes[TSDB_DATA_TYPE_INT].bytes, false); + SExprInfo* pExpr1 = tscExprAppend(pQueryInfo, TSDB_FUNC_TAG_DUMMY, &ind, TSDB_DATA_TYPE_INT, + tDataTypes[TSDB_DATA_TYPE_INT].bytes, getNewResColId(pCmd), tDataTypes[TSDB_DATA_TYPE_INT].bytes, false); tSqlExprItem* item = taosArrayGet(pExprList, 0); const char* name = (item->aliasName != NULL)? item->aliasName:functionsInfo[index].name; @@ -6628,69 +6638,69 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate) { if (pCreate->walLevel != -1 && (pCreate->walLevel < TSDB_MIN_WAL_LEVEL || pCreate->walLevel > TSDB_MAX_WAL_LEVEL)) { snprintf(msg, tListLen(msg), "invalid db option walLevel: %d, only 1-2 allowed", pCreate->walLevel); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } if (pCreate->replications != -1 && (pCreate->replications < TSDB_MIN_DB_REPLICA_OPTION || pCreate->replications > TSDB_MAX_DB_REPLICA_OPTION)) { snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } if (pCreate->quorum != -1 && (pCreate->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCreate->quorum > TSDB_MAX_DB_QUORUM_OPTION)) { snprintf(msg, tListLen(msg), "invalid db option quorum: %d valid range: [%d, %d]", pCreate->quorum, TSDB_MIN_DB_QUORUM_OPTION, TSDB_MAX_DB_QUORUM_OPTION); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } int32_t val = htonl(pCreate->daysPerFile); if (val != -1 && (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE)) { snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } val = htonl(pCreate->cacheBlockSize); if (val != -1 && (val < TSDB_MIN_CACHE_BLOCK_SIZE || val > TSDB_MAX_CACHE_BLOCK_SIZE)) { snprintf(msg, tListLen(msg), "invalid db option cacheBlockSize: %d valid range: [%d, %d]", val, TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } val = htonl(pCreate->maxTables); if (val != -1 && (val < TSDB_MIN_TABLES || val > TSDB_MAX_TABLES)) { snprintf(msg, tListLen(msg), "invalid db option maxSessions: %d valid range: [%d, %d]", val, TSDB_MIN_TABLES, TSDB_MAX_TABLES); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } if (pCreate->precision != TSDB_TIME_PRECISION_MILLI && pCreate->precision != TSDB_TIME_PRECISION_MICRO) { snprintf(msg, tListLen(msg), "invalid db option timePrecision: %d valid value: [%d, %d]", pCreate->precision, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } val = htonl(pCreate->commitTime); if (val != -1 && (val < TSDB_MIN_COMMIT_TIME || val > TSDB_MAX_COMMIT_TIME)) { snprintf(msg, tListLen(msg), "invalid db option commitTime: %d valid range: [%d, %d]", val, TSDB_MIN_COMMIT_TIME, TSDB_MAX_COMMIT_TIME); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } val = htonl(pCreate->fsyncPeriod); if (val != -1 && (val < TSDB_MIN_FSYNC_PERIOD || val > TSDB_MAX_FSYNC_PERIOD)) { snprintf(msg, tListLen(msg), "invalid db option fsyncPeriod: %d valid range: [%d, %d]", val, TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } if (pCreate->compression != -1 && (pCreate->compression < TSDB_MIN_COMP_LEVEL || pCreate->compression > TSDB_MAX_COMP_LEVEL)) { snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression, TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } val = (int16_t)htons(pCreate->partitions); @@ -6698,7 +6708,7 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate) { (val < TSDB_MIN_DB_PARTITON_OPTION || val > TSDB_MAX_DB_PARTITON_OPTION)) { snprintf(msg, tListLen(msg), "invalid topic option partition: %d valid range: [%d, %d]", val, TSDB_MIN_DB_PARTITON_OPTION, TSDB_MAX_DB_PARTITON_OPTION); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } @@ -6707,9 +6717,9 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate) { // for debug purpose void tscPrintSelNodeList(SSqlObj* pSql, int32_t subClauseIndex) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, subClauseIndex); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - int32_t size = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); + int32_t size = (int32_t)tscNumOfExprs(pQueryInfo); if (size == 0) { return; } @@ -6721,7 +6731,7 @@ void tscPrintSelNodeList(SSqlObj* pSql, int32_t subClauseIndex) { offset += sprintf(str, "num:%d [", size); for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); char tmpBuf[1024] = {0}; int32_t tmpLen = 0; @@ -6757,7 +6767,7 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p const char* msg1 = "invalid table name"; SSqlCmd* pCmd = &pSql->cmd; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, subClauseIndex); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; @@ -6771,17 +6781,17 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p SStrToken* pzTableName = &(pCreateTable->name); if (tscValidateName(pzTableName) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } - int32_t code = tscSetTableFullName(pTableMetaInfo, pzTableName, pSql); + int32_t code = tscSetTableFullName(&pTableMetaInfo->name, pzTableName, pSql); if(code != TSDB_CODE_SUCCESS) { return code; } if (!validateTableColumnInfo(pFieldList, pCmd) || (pTagList != NULL && !validateTagParams(pTagList, pFieldList, pCmd))) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } int32_t col = 0; @@ -6816,7 +6826,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { SSqlCmd* pCmd = &pSql->cmd; SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); // two table: the first one is for current table, and the secondary is for the super table. if (pQueryInfo->numOfTables < 2) { @@ -6835,10 +6845,10 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { SStrToken* pToken = &pCreateTableInfo->stableName; if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } - int32_t code = tscSetTableFullName(pStableMetaInfo, pToken, pSql); + int32_t code = tscSetTableFullName(&pStableMetaInfo->name, pToken, pSql); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -6874,12 +6884,12 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { if (valSize != nameSize) { tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } if (schemaSize < valSize) { tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } bool findColumnIndex = false; @@ -6903,7 +6913,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { if (pItem->pVar.nLen > pSchema->bytes) { tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } } @@ -6914,13 +6924,13 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { int16_t len = varDataTLen(tagVal); if (len > pSchema->bytes) { tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } } if (ret != TSDB_CODE_SUCCESS) { tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); @@ -6938,7 +6948,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { } else { if (schemaSize != valSize) { tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } for (int32_t i = 0; i < valSize; ++i) { @@ -6949,7 +6959,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { if (pItem->pVar.nLen > pSchema->bytes) { tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } } @@ -6960,13 +6970,13 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { int16_t len = varDataTLen(tagVal); if (len > pSchema->bytes) { tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } } if (ret != TSDB_CODE_SUCCESS) { tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); @@ -6990,11 +7000,11 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { // table name if (tscValidateName(&(pCreateTableInfo->name)) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, TABLE_INDEX); - ret = tscSetTableFullName(pTableMetaInfo, &pCreateTableInfo->name, pSql); + ret = tscSetTableFullName(&pTableMetaInfo->name, &pCreateTableInfo->name, pSql); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -7002,7 +7012,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { pCreateTableInfo->fullname = calloc(1, tNameLen(&pTableMetaInfo->name) + 1); ret = tNameExtractFullName(&pTableMetaInfo->name, pCreateTableInfo->fullname); if (ret != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } } @@ -7019,7 +7029,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { const char* msg7 = "time interval is required"; SSqlCmd* pCmd = &pSql->cmd; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); assert(pQueryInfo->numOfTables == 1); SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; @@ -7030,21 +7040,21 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { SSqlNode* pSqlNode = pCreateTable->pSelect; if (tscValidateName(pName) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } SRelationInfo* pFromInfo = pInfo->pCreateTableInfo->pSelect->from; if (pFromInfo == NULL || taosArrayGetSize(pFromInfo->list) == 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } - STableNamePair* p1 = taosArrayGet(pFromInfo->list, 0); - SStrToken srcToken = {.z = p1->name.z, .n = p1->name.n, .type = TK_STRING}; + SRelElementPair* p1 = taosArrayGet(pFromInfo->list, 0); + SStrToken srcToken = {.z = p1->tableName.z, .n = p1->tableName.n, .type = TK_STRING}; if (tscValidateName(&srcToken) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } - int32_t code = tscSetTableFullName(pTableMetaInfo, &srcToken, pSql); + int32_t code = tscSetTableFullName(&pTableMetaInfo->name, &srcToken, pSql); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -7061,46 +7071,46 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); if (validateSelectNodeList(&pSql->cmd, pQueryInfo, pSqlNode->pSelNodeList, isSTable, false, false) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } if (pSqlNode->pWhere != NULL) { // query condition in stream computing if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } // set interval value if (validateIntervalNode(pSql, pQueryInfo, pSqlNode) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } if (isTimeWindowQuery(pQueryInfo) && (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } if (!tscIsProjectionQuery(pQueryInfo) && pQueryInfo->interval.interval == 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); } // set the created table[stream] name - code = tscSetTableFullName(pTableMetaInfo, pName, pSql); + code = tscSetTableFullName(&pTableMetaInfo->name, pName, pSql); if (code != TSDB_CODE_SUCCESS) { return code; } if (pSqlNode->sqlstr.n > TSDB_MAX_SAVED_SQL_LEN) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } if (tsRewriteFieldNameIfNecessary(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; if (validateSqlFunctionInStreamSql(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } /* @@ -7109,14 +7119,14 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { */ if (pSqlNode->fillType != NULL) { if (pQueryInfo->interval.interval == 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } tVariantListItem* pItem = taosArrayGet(pSqlNode->fillType, 0); if (pItem->pVar.nType == TSDB_DATA_TYPE_BINARY) { if (!((strncmp(pItem->pVar.pz, "none", 4) == 0 && pItem->pVar.nLen == 4) || (strncmp(pItem->pVar.pz, "null", 4) == 0 && pItem->pVar.nLen == 4))) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } } } @@ -7135,7 +7145,7 @@ int32_t checkQueryRangeForFill(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { bool initialWindows = TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER); if (initialWindows) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); @@ -7155,7 +7165,7 @@ int32_t checkQueryRangeForFill(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { } // number of result is not greater than 10,000,000 if ((timeRange == 0) || (timeRange / intervalRange) >= MAX_INTERVAL_TIME_WINDOW) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } return TSDB_CODE_SUCCESS; @@ -7182,9 +7192,9 @@ int32_t tscGetExprFilters(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelect getColumnIndexByName(pCmd, pToken, pQueryInfo, &index); } - size_t numOfNodeInSel = tscSqlExprNumOfExprs(pQueryInfo); + size_t numOfNodeInSel = tscNumOfExprs(pQueryInfo); for(int32_t k = 0; k < numOfNodeInSel; ++k) { - SExprInfo* pExpr1 = tscSqlExprGet(pQueryInfo, k); + SExprInfo* pExpr1 = tscExprGet(pQueryInfo, k); if (pExpr1->base.functionId != functionId) { continue; @@ -7206,17 +7216,17 @@ int32_t tscGetExprFilters(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelect tSqlExprItem item = {.pNode = pSqlExpr, .aliasName = NULL, .distinct = false}; - int32_t outputIndex = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); + int32_t outputIndex = (int32_t)tscNumOfExprs(pQueryInfo); // ADD TRUE FOR TEST if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, &item, true) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } ++pQueryInfo->havingFieldNum; - size_t n = tscSqlExprNumOfExprs(pQueryInfo); - *pExpr = tscSqlExprGet(pQueryInfo, (int32_t)n - 1); + size_t n = tscNumOfExprs(pQueryInfo); + *pExpr = tscExprGet(pQueryInfo, (int32_t)n - 1); SInternalField* pField = taosArrayGet(pQueryInfo->fieldsInfo.internalField, n - 1); pField->visible = false; @@ -7265,7 +7275,7 @@ static int32_t handleExprInHavingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, S return TSDB_CODE_TSC_OUT_OF_MEMORY; } } else { // error; - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } pColFilter->filterstr = @@ -7278,16 +7288,16 @@ static int32_t handleExprInHavingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, S && pExpr->tokenId != TK_NOTNULL && pExpr->tokenId != TK_LIKE ) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } } else { if (pExpr->tokenId == TK_LIKE) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (expr->base.resType == TSDB_DATA_TYPE_BOOL) { if (pExpr->tokenId != TK_EQ && pExpr->tokenId != TK_NE) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } } } @@ -7324,11 +7334,11 @@ int32_t getHavingExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelectNode } if (pLeft == NULL || pRight == NULL) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (pLeft->type == pRight->type) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } exchangeExpr(pExpr); @@ -7336,15 +7346,15 @@ int32_t getHavingExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelectNode pLeft = pExpr->pLeft; pRight = pExpr->pRight; if (pLeft->type != SQL_NODE_SQLFUNCTION) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (pRight->type != SQL_NODE_VALUE) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (pExpr->tokenId >= TK_BITAND) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (pLeft->pParam) { @@ -7358,17 +7368,17 @@ int32_t getHavingExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelectNode pExpr1->tokenId != TK_STRING && pExpr1->tokenId != TK_INTEGER && pExpr1->tokenId != TK_FLOAT) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (pExpr1->tokenId == TK_ID && (pExpr1->colInfo.z == NULL && pExpr1->colInfo.n == 0)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (pExpr1->tokenId == TK_ID) { SColumnIndex index = COLUMN_INDEX_INITIALIZER; if ((getColumnIndexByName(pCmd, &pExpr1->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); @@ -7376,7 +7386,7 @@ int32_t getHavingExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelectNode if (index.columnIndex <= 0 || index.columnIndex >= tscGetNumOfColumns(pTableMeta)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } } } @@ -7384,7 +7394,7 @@ int32_t getHavingExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelectNode pLeft->functionId = isValidFunction(pLeft->operand.z, pLeft->operand.n); if (pLeft->functionId < 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } return handleExprInHavingClause(pCmd, pQueryInfo, pSelectNodeList, pExpr, parentOptr); @@ -7401,11 +7411,11 @@ int32_t validateHavingClause(SQueryInfo* pQueryInfo, tSqlExpr* pExpr, SSqlCmd* p } if (pQueryInfo->groupbyExpr.numOfGroupCols <= 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (pExpr->pLeft == NULL || pExpr->pRight == NULL) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } if (pQueryInfo->colList == NULL) { @@ -7420,41 +7430,211 @@ int32_t validateHavingClause(SQueryInfo* pQueryInfo, tSqlExpr* pExpr, SSqlCmd* p //REDO function check if (!functionCompatibleCheck(pQueryInfo, joinQuery, timeWindowQuery)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } return TSDB_CODE_SUCCESS; } -static int32_t doLoadAllTableMeta(SSqlObj* pSql, int32_t index, SSqlNode* pSqlNode, int32_t numOfTables) { +static int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, char* msgBuf, SSqlObj* pSql) { const char* msg1 = "invalid table name"; - const char* msg2 = "invalid table alias name"; - const char* msg3 = "alias name too long"; + + int32_t numOfTables = (int32_t) taosArrayGetSize(pSqlNode->from->list); + assert(pSqlNode->from->type == SQL_NODE_FROM_TABLELIST); + + for(int32_t j = 0; j < numOfTables; ++j) { + SRelElementPair* item = taosArrayGet(pSqlNode->from->list, j); + + SStrToken* t = &item->tableName; + if (t->type == TK_INTEGER || t->type == TK_FLOAT) { + return invalidOperationMsg(msgBuf, msg1); + } + + tscDequoteAndTrimToken(t); + if (tscValidateName(t) != TSDB_CODE_SUCCESS) { + return invalidOperationMsg(msgBuf, msg1); + } + + SName name = {0}; + if (tscSetTableFullName(&name, t, pSql) != TSDB_CODE_SUCCESS) { + return invalidOperationMsg(msgBuf, msg1); + } + + taosArrayPush(tableNameList, &name); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t getTableNameFromSubquery(SSqlNode* pSqlNode, SArray* tableNameList, char* msgBuf, SSqlObj* pSql) { + int32_t numOfSub = (int32_t) taosArrayGetSize(pSqlNode->from->list); + + for(int32_t j = 0; j < numOfSub; ++j) { + SRelElementPair* sub = taosArrayGet(pSqlNode->from->list, j); + + int32_t num = (int32_t)taosArrayGetSize(sub->pSubquery); + for (int32_t i = 0; i < num; ++i) { + SSqlNode* p = taosArrayGetP(sub->pSubquery, i); + if (p->from->type == SQL_NODE_FROM_TABLELIST) { + int32_t code = getTableNameFromSqlNode(p, tableNameList, msgBuf, pSql); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } else { + getTableNameFromSubquery(p, tableNameList, msgBuf, pSql); + } + } + } + + return TSDB_CODE_SUCCESS; +} + +void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); +static void freeElem(void* p) { + tfree(*(char**)p); +} + +int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { + SSqlCmd* pCmd = &pSql->cmd; + + // the table meta has already been loaded from local buffer or mnode already + if (pCmd->pTableMetaMap != NULL) { + return TSDB_CODE_SUCCESS; + } int32_t code = TSDB_CODE_SUCCESS; + SArray* tableNameList = NULL; + SArray* pVgroupList = NULL; + SArray* plist = NULL; + STableMeta* pTableMeta = NULL; + + pCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + + tableNameList = taosArrayInit(4, sizeof(SName)); + size_t size = taosArrayGetSize(pInfo->list); + for (int32_t i = 0; i < size; ++i) { + SSqlNode* pSqlNode = taosArrayGetP(pInfo->list, i); + if (pSqlNode->from == NULL) { + goto _end; + } + + // load the table meta in the from clause + if (pSqlNode->from->type == SQL_NODE_FROM_TABLELIST) { + code = getTableNameFromSqlNode(pSqlNode, tableNameList, tscGetErrorMsgPayload(pCmd), pSql); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + } else { + code = getTableNameFromSubquery(pSqlNode, tableNameList, tscGetErrorMsgPayload(pCmd), pSql); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + } + } + + uint32_t maxSize = tscGetTableMetaMaxSize(); + char name[TSDB_TABLE_FNAME_LEN] = {0}; + + char buf[80 * 1024] = {0}; + assert(maxSize < 80 * 1024); + pTableMeta = calloc(1, maxSize); + + plist = taosArrayInit(4, POINTER_BYTES); + pVgroupList = taosArrayInit(4, POINTER_BYTES); + + size_t numOfTables = taosArrayGetSize(tableNameList); + for (int32_t i = 0; i < numOfTables; ++i) { + SName* pname = taosArrayGet(tableNameList, i); + tNameExtractFullName(pname, name); + + size_t len = strlen(name); + memset(pTableMeta, 0, maxSize); + taosHashGetClone(tscTableMetaInfo, name, len, NULL, pTableMeta, -1); + + if (pTableMeta->id.uid > 0) { + if (pTableMeta->tableType == TSDB_CHILD_TABLE) { + code = tscCreateTableMetaFromSTableMeta(pTableMeta, name, buf); + + // create the child table meta from super table failed, try load it from mnode + if (code != TSDB_CODE_SUCCESS) { + char* t = strdup(name); + taosArrayPush(plist, &t); + continue; + } + } else if (pTableMeta->tableType == TSDB_SUPER_TABLE) { + // the vgroup list of a super table is not kept in local buffer, so here need retrieve it + // from the mnode each time + char* t = strdup(name); + taosArrayPush(pVgroupList, &t); + } + + STableMeta* pMeta = tscTableMetaDup(pTableMeta); + STableMetaVgroupInfo p = { .pTableMeta = pMeta }; + + const char* px = tNameGetTableName(pname); + taosHashPut(pCmd->pTableMetaMap, px, strlen(px), &p, sizeof(STableMetaVgroupInfo)); + } else { // add to the retrieve table meta array list. + char* t = strdup(name); + taosArrayPush(plist, &t); + } + } + + // load the table meta for a given table name list + if (taosArrayGetSize(plist) > 0 || taosArrayGetSize(pVgroupList) > 0) { + code = getMultiTableMetaFromMnode(pSql, plist, pVgroupList, tscTableMetaCallBack); + } + +_end: + if (plist != NULL) { + taosArrayDestroyEx(plist, freeElem); + } + + if (pVgroupList != NULL) { + taosArrayDestroyEx(pVgroupList, freeElem); + } + + if (tableNameList != NULL) { + taosArrayDestroy(tableNameList); + } + + tfree(pTableMeta); + + return code; +} + +static int32_t doLoadAllTableMeta(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, int32_t numOfTables) { + const char* msg1 = "invalid table name"; + const char* msg2 = "invalid table alias name"; + const char* msg3 = "alias name too long"; + const char* msg4 = "self join not allowed"; + + int32_t code = TSDB_CODE_SUCCESS; SSqlCmd* pCmd = &pSql->cmd; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, index); + + if (numOfTables > taosHashGetSize(pCmd->pTableMetaMap)) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); + } for (int32_t i = 0; i < numOfTables; ++i) { if (pQueryInfo->numOfTables <= i) { // more than one table tscAddEmptyMetaInfo(pQueryInfo); } - STableNamePair *item = taosArrayGet(pSqlNode->from->list, i); - SStrToken *oriName = &item->name; + SRelElementPair *item = taosArrayGet(pSqlNode->from->list, i); + SStrToken *oriName = &item->tableName; if (oriName->type == TK_INTEGER || oriName->type == TK_FLOAT) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } tscDequoteAndTrimToken(oriName); if (tscValidateName(oriName) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - code = tscSetTableFullName(pTableMetaInfo, oriName, pSql); + code = tscSetTableFullName(&pTableMetaInfo->name, oriName, pSql); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -7462,25 +7642,29 @@ static int32_t doLoadAllTableMeta(SSqlObj* pSql, int32_t index, SSqlNode* pSqlNo SStrToken* aliasName = &item->aliasName; if (TPARSER_HAS_TOKEN(*aliasName)) { if (aliasName->type == TK_INTEGER || aliasName->type == TK_FLOAT) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } tscDequoteAndTrimToken(aliasName); - if (tscValidateName(aliasName) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - if (aliasName->n >= TSDB_TABLE_NAME_LEN) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + if (tscValidateName(aliasName) != TSDB_CODE_SUCCESS || aliasName->n >= TSDB_TABLE_NAME_LEN) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } strncpy(pTableMetaInfo->aliasName, aliasName->z, aliasName->n); } else { - strncpy(pTableMetaInfo->aliasName, tNameGetTableName(&pTableMetaInfo->name), - tListLen(pTableMetaInfo->aliasName)); + strncpy(pTableMetaInfo->aliasName, tNameGetTableName(&pTableMetaInfo->name), tListLen(pTableMetaInfo->aliasName)); + } + + const char* name = tNameGetTableName(&pTableMetaInfo->name); + STableMetaVgroupInfo* p = taosHashGet(pCmd->pTableMetaMap, name, strlen(name)); + + pTableMetaInfo->pTableMeta = tscTableMetaDup(p->pTableMeta); + assert(pTableMetaInfo->pTableMeta != NULL); + + if (p->pVgroupInfo != NULL) { + pTableMetaInfo->vgroupList = tscVgroupsInfoDup(p->pVgroupInfo); } - code = tscGetTableMeta(pSql, pTableMetaInfo); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -7489,7 +7673,7 @@ static int32_t doLoadAllTableMeta(SSqlObj* pSql, int32_t index, SSqlNode* pSqlNo return TSDB_CODE_SUCCESS; } -static STableMeta* extractTempTableMetaFromNestQuery(SQueryInfo* pUpstream) { +static STableMeta* extractTempTableMetaFromSubquery(SQueryInfo* pUpstream) { int32_t numOfColumns = pUpstream->fieldsInfo.numOfOutput; STableMeta* meta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * numOfColumns); @@ -7514,31 +7698,84 @@ static STableMeta* extractTempTableMetaFromNestQuery(SQueryInfo* pUpstream) { return meta; } -int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, int32_t index) { +static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SSqlObj* pSql, SQueryInfo* pQueryInfo, char* msgBuf) { + SRelElementPair* subInfo = taosArrayGet(pSqlNode->from->list, index); + + // union all is not support currently + SSqlNode* p = taosArrayGetP(subInfo->pSubquery, 0); + + SQueryInfo* pSub = calloc(1, sizeof(SQueryInfo)); + tscInitQueryInfo(pSub); + + int32_t code = validateSqlNode(pSql, p, pSub); + assert(code != TSDB_CODE_TSC_ACTION_IN_PROGRESS); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + pSub->pDownstream = pQueryInfo; + + // create dummy table meta info + STableMetaInfo* pTableMetaInfo1 = calloc(1, sizeof(STableMetaInfo)); + pTableMetaInfo1->pTableMeta = extractTempTableMetaFromSubquery(pSub); + + if (subInfo->aliasName.n > 0) { + if (subInfo->aliasName.n >= TSDB_TABLE_FNAME_LEN) { + return invalidOperationMsg(msgBuf, "subquery alias name too long"); + } + + strncpy(pTableMetaInfo1->aliasName, subInfo->aliasName.z, subInfo->aliasName.n); + } + + taosArrayPush(pQueryInfo->pUpstream, &pSub); + + // NOTE: order mix up in subquery not support yet. + pQueryInfo->order = pSub->order; + + STableMetaInfo** tmp = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); + if (tmp == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pQueryInfo->pTableMetaInfo = tmp; + + pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables] = pTableMetaInfo1; + pQueryInfo->numOfTables += 1; + + // all columns are added into the table column list + STableMeta* pMeta = pTableMetaInfo1->pTableMeta; + int32_t startOffset = (int32_t) taosArrayGetSize(pQueryInfo->colList); + + for(int32_t i = 0; i < pMeta->tableInfo.numOfColumns; ++i) { + tscColumnListInsert(pQueryInfo->colList, i + startOffset, pMeta->id.uid, &pMeta->schema[i]); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInfo) { assert(pSqlNode != NULL && (pSqlNode->from == NULL || taosArrayGetSize(pSqlNode->from->list) > 0)); const char* msg1 = "point interpolation query needs timestamp"; const char* msg2 = "too many tables in from clause"; const char* msg3 = "start(end) time of query range required or time range too large"; + const char* msg9 = "only tag query not compatible with normal column filter"; int32_t code = TSDB_CODE_SUCCESS; SSqlCmd* pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, index); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (pTableMetaInfo == NULL) { pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); } - assert(pCmd->clauseIndex == index); - /* * handle the sql expression without from subclause - * select current_database(); + * select server_status(); * select server_version(); * select client_version(); - * select server_state(); + * select current_database(); */ if (pSqlNode->from == NULL) { assert(pSqlNode->fillType == NULL && pSqlNode->pGroupby == NULL && pSqlNode->pWhere == NULL && @@ -7547,67 +7784,51 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, int32_t index) { } if (pSqlNode->from->type == SQL_NODE_FROM_SUBQUERY) { + clearAllTableMetaInfo(pQueryInfo, false); + pQueryInfo->numOfTables = 0; + // parse the subquery in the first place - SArray* list = taosArrayGetP(pSqlNode->from->list, 0); - SSqlNode* p = taosArrayGetP(list, 0); - - code = validateSqlNode(pSql, p, 0); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - return code; + int32_t numOfSub = (int32_t) taosArrayGetSize(pSqlNode->from->list); + for(int32_t i = 0; i < numOfSub; ++i) { + code = doValidateSubquery(pSqlNode, i, pSql, pQueryInfo, tscGetErrorMsgPayload(pCmd)); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } - if (code != TSDB_CODE_SUCCESS) { - return code; + if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, false, false, false) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; } - pQueryInfo = pCmd->pQueryInfo[0]; + if (pSqlNode->pWhere != NULL) { + if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } - SQueryInfo* current = calloc(1, sizeof(SQueryInfo)); - - tscInitQueryInfo(current); - taosArrayPush(current->pUpstream, &pQueryInfo); - - STableMeta* pTableMeta = extractTempTableMetaFromNestQuery(pQueryInfo); - STableMetaInfo* pTableMetaInfo1 = calloc(1, sizeof(STableMetaInfo)); - pTableMetaInfo1->pTableMeta = pTableMeta; - - current->pTableMetaInfo = calloc(1, POINTER_BYTES); - current->pTableMetaInfo[0] = pTableMetaInfo1; - current->numOfTables = 1; - current->order = pQueryInfo->order; - - pCmd->pQueryInfo[0] = current; - pQueryInfo->pDownstream = current; - - if (validateSelectNodeList(pCmd, current, pSqlNode->pSelNodeList, false, false, false) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; + if (pTableMeta->tableInfo.precision == TSDB_TIME_PRECISION_MILLI) { + pQueryInfo->window.skey = pQueryInfo->window.skey / 1000; + pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000; + } } - } else { pQueryInfo->command = TSDB_SQL_SELECT; - size_t fromSize = taosArrayGetSize(pSqlNode->from->list); - if (fromSize > TSDB_MAX_JOIN_TABLE_NUM) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + size_t numOfTables = taosArrayGetSize(pSqlNode->from->list); + if (numOfTables > TSDB_MAX_JOIN_TABLE_NUM) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } // set all query tables, which are maybe more than one. - code = doLoadAllTableMeta(pSql, index, pSqlNode, (int32_t) fromSize); + code = doLoadAllTableMeta(pSql, pQueryInfo, pSqlNode, (int32_t) numOfTables); if (code != TSDB_CODE_SUCCESS) { return code; } bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - if (isSTable) { - code = tscGetSTableVgroupInfo(pSql, index); // TODO refactor: getTablemeta along with vgroupInfo - if (code != TSDB_CODE_SUCCESS) { - return code; - } - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_QUERY); - } else { - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TABLE_QUERY); - } + int32_t type = isSTable? TSDB_QUERY_TYPE_STABLE_QUERY:TSDB_QUERY_TYPE_TABLE_QUERY; + TSDB_QUERY_SET_TYPE(pQueryInfo->type, type); code = checkForUdf(pSql, pQueryInfo, pSqlNode->pSelNodeList); if (code != TSDB_CODE_SUCCESS) { @@ -7616,7 +7837,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, int32_t index) { // parse the group by clause in the first place if (validateGroupbyNode(pQueryInfo, pSqlNode->pGroupby, pCmd) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } // set where info @@ -7624,7 +7845,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, int32_t index) { if (pSqlNode->pWhere != NULL) { if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } pSqlNode->pWhere = NULL; @@ -7632,10 +7853,9 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, int32_t index) { pQueryInfo->window.skey = pQueryInfo->window.skey / 1000; pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000; } - } else { // set the time rang - if (taosArrayGetSize(pSqlNode->from->list) > 1) { - // If it is a join query, no where clause is not allowed. - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "condition missing for join query "); + } else { + if (taosArrayGetSize(pSqlNode->from->list) > 1) { // Cross join not allowed yet + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "cross join not supported yet"); } } @@ -7645,29 +7865,43 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, int32_t index) { if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, isSTable, joinQuery, timeWindowQuery) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } // set order by info if (validateOrderbyNode(pCmd, pQueryInfo, pSqlNode, tscGetTableSchema(pTableMetaInfo->pTableMeta)) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } // set interval value if (validateIntervalNode(pSql, pQueryInfo, pSqlNode) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } else { if (isTimeWindowQuery(pQueryInfo) && (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } + + if (tscQueryTags(pQueryInfo)) { + SExprInfo* pExpr1 = tscExprGet(pQueryInfo, 0); + + if (pExpr1->base.functionId != TSDB_FUNC_TID_TAG) { + int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryInfo->colList); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumn* pCols = taosArrayGetP(pQueryInfo->colList, i); + if (pCols->info.flist.numOfFilters > 0) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); + } + } } } // parse the having clause in the first place if (validateHavingClause(pQueryInfo, pSqlNode->pHaving, pCmd, pSqlNode->pSelNodeList, joinQuery, timeWindowQuery) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } /* @@ -7676,14 +7910,13 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, int32_t index) { */ if (isSTable) { tscTansformFuncForSTableQuery(pQueryInfo); - if (hasUnsupportFunctionsForSTableQuery(pCmd, pQueryInfo)) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } if (validateSessionNode(pCmd, pQueryInfo, pSqlNode) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } // no result due to invalid query time range @@ -7693,18 +7926,18 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, int32_t index) { } if (!hasTimestampForPointInterpQuery(pQueryInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } // in case of join query, time range is required. if (QUERY_IS_JOIN_QUERY(pQueryInfo->type)) { int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); if (timeRange == 0 && pQueryInfo->window.skey == 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } } - if ((code = validateLimitNode(pCmd, pQueryInfo, index, pSqlNode, pSql)) != TSDB_CODE_SUCCESS) { + if ((code = validateLimitNode(pCmd, pQueryInfo, pSqlNode, pSql)) != TSDB_CODE_SUCCESS) { return code; } @@ -7720,8 +7953,35 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, int32_t index) { } } - pQueryInfo->globalMerge = tscIsTwoStageSTableQuery(pCmd, pQueryInfo, 0); - pQueryInfo->arithmCalOnAgg = tscIsSecondStageQuery(pCmd, pQueryInfo); + { // set the query info + pQueryInfo->projectionQuery = tscIsProjectionQuery(pQueryInfo); + pQueryInfo->hasFilter = tscHasColumnFilter(pQueryInfo); + pQueryInfo->simpleAgg = isSimpleAggregateRv(pQueryInfo); + pQueryInfo->onlyTagQuery = onlyTagPrjFunction(pQueryInfo); + pQueryInfo->groupbyColumn = tscGroupbyColumn(pQueryInfo); + pQueryInfo->globalMerge = tscIsTwoStageSTableQuery(pCmd, pQueryInfo, 0); + + pQueryInfo->arithmeticOnAgg = tsIsArithmeticQueryOnAggResult(pQueryInfo); + + SExprInfo** p = NULL; + int32_t numOfExpr = 0; + code = createProjectionExpr(pQueryInfo, pTableMetaInfo, &p, &numOfExpr); + + if (pQueryInfo->exprList1 == NULL) { + pQueryInfo->exprList1 = taosArrayInit(4, POINTER_BYTES); + } + + taosArrayAddBatch(pQueryInfo->exprList1, (void*) p, numOfExpr); + } + +#if 0 + SQueryNode* p = qCreateQueryPlan(pQueryInfo); + char* s = queryPlanToString(p); + tfree(s); + + qDestroyQueryPlan(p); +#endif + return TSDB_CODE_SUCCESS; // Does not build query message here } @@ -7813,7 +8073,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS return TSDB_CODE_SUCCESS; } else { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "not support filter expression"); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "not support filter expression"); } } else { @@ -7833,9 +8093,9 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS if ((*pExpr)->_node.optr == TSDB_BINARY_OP_DIVIDE) { if (pRight->nodeType == TSQL_NODE_VALUE) { if (pRight->pVal->nType == TSDB_DATA_TYPE_INT && pRight->pVal->i64 == 0) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } else if (pRight->pVal->nType == TSDB_DATA_TYPE_FLOAT && pRight->pVal->dKey == 0) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } } @@ -7844,7 +8104,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS if ((*pExpr)->_node.optr != TSDB_RELATION_EQUAL && (*pExpr)->_node.optr != TSDB_RELATION_NOT_EQUAL) { if (pRight != NULL && pRight->nodeType == TSQL_NODE_VALUE) { if (pRight->pVal->nType == TSDB_DATA_TYPE_BOOL) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } } diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 2ea382132b..114fc8ee73 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -94,6 +94,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) { pTableMeta->tableType = pTableMetaMsg->tableType; pTableMeta->vgId = pTableMetaMsg->vgroup.vgId; + pTableMeta->suid = pTableMetaMsg->suid; pTableMeta->tableInfo = (STableComInfo) { .numOfTags = pTableMetaMsg->numOfTags, diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 80ed37b56b..b9b0885630 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -116,7 +116,7 @@ static void tscDumpEpSetFromVgroupInfo(SRpcEpSet *pEpSet, SNewVgroupInfo *pVgrou static void tscUpdateVgroupInfo(SSqlObj *pSql, SRpcEpSet *pEpSet) { SSqlCmd *pCmd = &pSql->cmd; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { return; } @@ -335,7 +335,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { return; } - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); if (pQueryInfo != NULL && pQueryInfo->type == TSDB_QUERY_TYPE_FREE_RESOURCE) { tscDebug("0x%"PRIx64" sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p", pSql->self, pCmd->command, pQueryInfo->type, pObj, pObj->signature); @@ -360,7 +360,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { // set the flag to denote that sql string needs to be re-parsed and build submit block with table schema if (cmd == TSDB_SQL_INSERT && rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { - pSql->cmd.submitSchema = 1; + pSql->cmd.insertParam.schemaAttached = 1; } if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) && @@ -477,7 +477,6 @@ int doBuildAndSendMsg(SSqlObj *pSql) { pCmd->command == TSDB_SQL_INSERT || pCmd->command == TSDB_SQL_CONNECT || pCmd->command == TSDB_SQL_HB || - pCmd->command == TSDB_SQL_META || pCmd->command == TSDB_SQL_RETRIEVE_FUNC || pCmd->command == TSDB_SQL_STABLEVGROUP) { pRes->code = tscBuildMsg[pCmd->command](pSql, NULL); @@ -507,7 +506,7 @@ int tscBuildAndSendRequest(SSqlObj *pSql, SQueryInfo* pQueryInfo) { uint32_t type = 0; if (pQueryInfo == NULL) { - pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); + pQueryInfo = tscGetQueryInfo(pCmd); } STableMetaInfo *pTableMetaInfo = NULL; @@ -540,7 +539,7 @@ int tscBuildAndSendRequest(SSqlObj *pSql, SQueryInfo* pQueryInfo) { int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg *) pSql->cmd.payload; - SQueryInfo *pQueryInfo = tscGetActiveQueryInfo(&pSql->cmd); + SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); pRetrieveMsg->free = htons(pQueryInfo->type); pRetrieveMsg->qId = htobe64(pSql->res.qId); @@ -582,25 +581,8 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; - - char* pMsg = pSql->cmd.payload; - - // NOTE: shell message size should not include SMsgDesc - int32_t size = pSql->cmd.payloadLen - sizeof(SMsgDesc); - - SMsgDesc* pMsgDesc = (SMsgDesc*) pMsg; - pMsgDesc->numOfVnodes = htonl(1); // always one vnode - - pMsg += sizeof(SMsgDesc); - SSubmitMsg *pShellMsg = (SSubmitMsg *)pMsg; - - pShellMsg->header.vgId = htonl(pTableMeta->vgId); - pShellMsg->header.contLen = htonl(size); // the length not includes the size of SMsgDesc - pShellMsg->length = pShellMsg->header.contLen; - - pShellMsg->numOfBlocks = htonl(pSql->cmd.numOfTablesInSubmit); // number of tables to be inserted // pSql->cmd.payloadLen is set during copying data into payload pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; @@ -609,29 +591,28 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); - tscDebug("0x%"PRIx64" build submit msg, vgId:%d numOfTables:%d numberOfEP:%d", pSql->self, pTableMeta->vgId, pSql->cmd.numOfTablesInSubmit, - pSql->epSet.numOfEps); + tscDebug("0x%"PRIx64" submit msg built, numberOfEP:%d", pSql->self, pSql->epSet.numOfEps); + return TSDB_CODE_SUCCESS; } /* * for table query, simply return the size <= 1k */ -static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql, int32_t clauseIndex) { +static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) { const static int32_t MIN_QUERY_MSG_PKT_SIZE = TSDB_MAX_BYTES_PER_ROW * 5; SSqlCmd* pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, clauseIndex); + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo)); - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + size_t numOfExprs = tscNumOfExprs(pQueryInfo); int32_t exprSize = (int32_t)(sizeof(SSqlExpr) * numOfExprs * 2); int32_t tsBufSize = (pQueryInfo->tsBuf != NULL) ? pQueryInfo->tsBuf->fileSize : 0; int32_t sqlLen = (int32_t) strlen(pSql->sqlstr) + 1; - int32_t tableSerialize = 0; STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (pTableMetaInfo->pVgroupTables != NULL) { @@ -755,7 +736,7 @@ static int32_t serializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t nu if (pColFilter->lowerRelOptr == TSDB_RELATION_INVALID && pColFilter->upperRelOptr == TSDB_RELATION_INVALID) { tscError("invalid filter info"); - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } @@ -774,7 +755,7 @@ static int32_t serializeSqlExpr(SSqlExpr* pExpr, STableMetaInfo* pTableMetaInfo, if (validateColumn && !tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) { tscError("0x%"PRIx64" table schema is not matched with parsed sql", id); - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } assert(pExpr->resColId < 0); @@ -817,14 +798,14 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; int32_t code = TSDB_CODE_SUCCESS; - int32_t size = tscEstimateQueryMsgSize(pSql, pCmd->clauseIndex); + int32_t size = tscEstimateQueryMsgSize(pSql); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { tscError("%p failed to malloc for query msg", pSql); - return TSDB_CODE_TSC_INVALID_SQL; // todo add test for this + return TSDB_CODE_TSC_INVALID_OPERATION; // todo add test for this } - SQueryInfo *pQueryInfo = tscGetActiveQueryInfo(pCmd); + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); SQueryAttr query = {{0}}; tscCreateQueryFromQueryInfo(pQueryInfo, &query, pSql); @@ -930,7 +911,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { goto _end; } - SSqlGroupbyExpr *pGroupbyExpr = query.pGroupbyExpr; + SGroupbyExpr *pGroupbyExpr = query.pGroupbyExpr; if (pGroupbyExpr->numOfGroupCols > 0) { pQueryMsg->orderByIdx = htons(pGroupbyExpr->orderIndex); pQueryMsg->orderType = htons(pGroupbyExpr->orderType); @@ -1079,8 +1060,8 @@ int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SCreateDbMsg *pCreateDbMsg = (SCreateDbMsg *)pCmd->payload; - assert(pCmd->numOfClause == 1); - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); +// assert(pCmd->numOfClause == 1); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pCreateDbMsg->db); assert(code == TSDB_CODE_SUCCESS); @@ -1212,7 +1193,7 @@ int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SDropDbMsg *pDropDbMsg = (SDropDbMsg*)pCmd->payload; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pDropDbMsg->db); assert(code == TSDB_CODE_SUCCESS && pTableMetaInfo->name.type == TSDB_DB_NAME_T); @@ -1244,9 +1225,10 @@ int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } SCMDropTableMsg *pDropTableMsg = (SCMDropTableMsg*)pCmd->payload; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); tNameExtractFullName(&pTableMetaInfo->name, pDropTableMsg->name); + pDropTableMsg->supertable = (pInfo->pMiscInfo->tableType == TSDB_SUPER_TABLE)? 1:0; pDropTableMsg->igNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_TABLE; return TSDB_CODE_SUCCESS; @@ -1301,7 +1283,7 @@ int32_t tscBuildUseDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } SUseDbMsg *pUseDbMsg = (SUseDbMsg *)pCmd->payload; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); tNameExtractFullName(&pTableMetaInfo->name, pUseDbMsg->db); pCmd->msgType = TSDB_MSG_TYPE_CM_USE_DB; @@ -1318,7 +1300,7 @@ int32_t tscBuildSyncDbReplicaMsg(SSqlObj* pSql, SSqlInfo *pInfo) { } SSyncDbMsg *pSyncMsg = (SSyncDbMsg *)pCmd->payload; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); tNameExtractFullName(&pTableMetaInfo->name, pSyncMsg->db); pCmd->msgType = TSDB_MSG_TYPE_CM_SYNC_DB; @@ -1339,6 +1321,7 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SShowInfo *pShowInfo = &pInfo->pMiscInfo->showOpt; SShowMsg *pShowMsg = (SShowMsg *)pCmd->payload; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); if (pShowInfo->showType == TSDB_MGMT_TABLE_FUNCTION) { pShowMsg->type = pShowInfo->showType; pShowMsg->payloadLen = 0; @@ -1347,9 +1330,7 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return TSDB_CODE_SUCCESS; } - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - - if (tNameIsEmpty(&pTableMetaInfo->name)) { + if (tNameIsEmpty(&pTableMetaInfo->name)) { pthread_mutex_lock(&pObj->mutex); tstrncpy(pShowMsg->db, pObj->db, sizeof(pShowMsg->db)); pthread_mutex_unlock(&pObj->mutex); @@ -1420,7 +1401,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSchema *pSchema; SSqlCmd *pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); // Reallocate the payload size @@ -1509,7 +1490,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } int tscEstimateAlterTableMsgLength(SSqlCmd *pCmd) { - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); return minMsgSize() + sizeof(SAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) + TSDB_EXTRA_PAYLOAD_SIZE; } @@ -1518,7 +1499,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int msgLen = 0; SSqlCmd *pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -1567,7 +1548,7 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload; pCmd->payloadLen = htonl(pUpdateMsg->head.contLen); - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); STableMeta *pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; SNewVgroupInfo vgroupInfo = {.vgId = -1}; @@ -1587,7 +1568,7 @@ int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SAlterDbMsg *pAlterDbMsg = (SAlterDbMsg* )pCmd->payload; pAlterDbMsg->dbType = -1; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); tNameExtractFullName(&pTableMetaInfo->name, pAlterDbMsg->db); return TSDB_CODE_SUCCESS; @@ -1603,7 +1584,7 @@ int tscBuildRetrieveFromMgmtMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg*)pCmd->payload; pRetrieveMsg->qId = htobe64(pSql->res.qId); pRetrieveMsg->free = htons(pQueryInfo->type); @@ -1627,7 +1608,7 @@ static int tscLocalResultCommonBuilder(SSqlObj *pSql, int32_t numOfRes) { pRes->row = 0; pRes->rspType = 1; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); if (tscCreateResPointerInfo(pRes, pQueryInfo) != TSDB_CODE_SUCCESS) { return pRes->code; } @@ -1651,7 +1632,7 @@ static int tscLocalResultCommonBuilder(SSqlObj *pSql, int32_t numOfRes) { int tscProcessDescribeTableRsp(SSqlObj *pSql) { SSqlCmd * pCmd = &pSql->cmd; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); @@ -1675,8 +1656,13 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { return code; } + if (pRes->pLocalMerger == NULL) { // no result from subquery, so abort here directly. + (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); + return code; + } + // global aggregation may be the upstream for parent query - SQueryInfo *pQueryInfo = tscGetActiveQueryInfo(pCmd); + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); if (pQueryInfo->pQInfo == NULL) { STableGroupInfo tableGroupInfo = {.numOfTables = 1, .pGroupList = taosArrayInit(1, POINTER_BYTES),}; tableGroupInfo.map = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); @@ -1687,21 +1673,21 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { taosArrayPush(group, &tableKeyInfo); taosArrayPush(tableGroupInfo.pGroupList, &group); - SExprInfo* list = calloc(tscSqlExprNumOfExprs(pQueryInfo), sizeof(SExprInfo)); - for(int32_t i = 0; i < tscSqlExprNumOfExprs(pQueryInfo); ++i) { - SExprInfo* pExprInfo = tscSqlExprGet(pQueryInfo, i); + // todo remove it + SExprInfo* list = calloc(tscNumOfExprs(pQueryInfo), sizeof(SExprInfo)); + for(int32_t i = 0; i < tscNumOfExprs(pQueryInfo); ++i) { + SExprInfo* pExprInfo = tscExprGet(pQueryInfo, i); list[i] = *pExprInfo; } - pQueryInfo->pQInfo = createQueryInfoFromQueryNode(pQueryInfo, list, &tableGroupInfo, NULL, NULL, pRes->pLocalMerger, MERGE_STAGE); + pQueryInfo->pQInfo = createQInfoFromQueryNode(pQueryInfo, list, &tableGroupInfo, NULL, NULL, pRes->pLocalMerger, MERGE_STAGE); + tfree(list); } uint64_t localQueryId = 0; qTableQuery(pQueryInfo->pQInfo, &localQueryId); convertQueryResult(pRes, pQueryInfo); - handleDownstreamOperator(pRes, pQueryInfo); - code = pRes->code; if (pRes->code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); @@ -1747,15 +1733,16 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { +#if 0 SSqlCmd *pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableInfoMsg *pInfoMsg = (STableInfoMsg *)pCmd->payload; int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pInfoMsg->tableFname); if (code != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } pInfoMsg->createFlag = htons(pSql->cmd.autoCreated ? 1 : 0); @@ -1768,64 +1755,40 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->payloadLen = (int32_t)(pMsg - (char*)pInfoMsg); pCmd->msgType = TSDB_MSG_TYPE_CM_TABLE_META; +#endif return TSDB_CODE_SUCCESS; } /** * multi table meta req pkg format: - * | SMgmtHead | SMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ...... - * no used 4B + * |SMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ...... + * 4B **/ -int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { -#if 0 +int tscBuildMultiTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - // copy payload content to temp buff - char *tmpData = 0; - if (pCmd->payloadLen > 0) { - if ((tmpData = calloc(1, pCmd->payloadLen + 1)) == NULL) return -1; - memcpy(tmpData, pCmd->payload, pCmd->payloadLen); - } - - // fill head info - SMgmtHead *pMgmt = (SMgmtHead *)(pCmd->payload + tsRpcHeadSize); - memset(pMgmt->db, 0, TSDB_TABLE_FNAME_LEN); // server don't need the db - - SMultiTableInfoMsg *pInfoMsg = (SMultiTableInfoMsg *)(pCmd->payload + tsRpcHeadSize + sizeof(SMgmtHead)); - pInfoMsg->numOfTables = htonl((int32_t)pCmd->count); - - if (pCmd->payloadLen > 0) { - memcpy(pInfoMsg->tableIds, tmpData, pCmd->payloadLen); - } - - tfree(tmpData); - - pCmd->payloadLen += sizeof(SMgmtHead) + sizeof(SMultiTableInfoMsg); pCmd->msgType = TSDB_MSG_TYPE_CM_TABLES_META; - assert(pCmd->payloadLen + minMsgSize() <= pCmd->allocSize); - tscDebug("0x%"PRIx64" build load multi-metermeta msg completed, numOfTables:%d, msg size:%d", pSql->self, pCmd->count, + tscDebug("0x%"PRIx64" build load multi-tablemeta msg completed, numOfTables:%d, msg size:%d", pSql->self, pCmd->count, pCmd->payloadLen); return pCmd->payloadLen; -#endif - return 0; } int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; char* pMsg = pCmd->payload; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); SSTableVgroupMsg *pStableVgroupMsg = (SSTableVgroupMsg *)pMsg; pStableVgroupMsg->numOfTables = htonl(pQueryInfo->numOfTables); pMsg += sizeof(SSTableVgroupMsg); for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, i); int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pMsg); assert(code == TSDB_CODE_SUCCESS); @@ -1842,7 +1805,7 @@ int tscBuildRetrieveFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; char *pMsg = pCmd->payload; - SQueryInfo* pQueryInfo = tscGetActiveQueryInfo(pCmd); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); int32_t numOfFuncs = (int32_t)taosArrayGetSize(pQueryInfo->pUdfInfo); SRetrieveFuncMsg *pRetrieveFuncMsg = (SRetrieveFuncMsg *)pMsg; @@ -1908,18 +1871,16 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return TSDB_CODE_SUCCESS; } -int tscProcessTableMetaRsp(SSqlObj *pSql) { - STableMetaMsg *pMetaMsg = (STableMetaMsg *)pSql->res.pRsp; - +static int32_t tableMetaMsgConvert(STableMetaMsg* pMetaMsg) { pMetaMsg->tid = htonl(pMetaMsg->tid); pMetaMsg->sversion = htons(pMetaMsg->sversion); pMetaMsg->tversion = htons(pMetaMsg->tversion); pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId); - + pMetaMsg->uid = htobe64(pMetaMsg->uid); - pMetaMsg->contLen = htons(pMetaMsg->contLen); +// pMetaMsg->contLen = htonl(pMetaMsg->contLen); pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); - + if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) && (pMetaMsg->tid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) { tscError("invalid value in table numOfEps:%d, vgId:%d tid:%d, name:%s", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId, @@ -1954,23 +1915,34 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pSchema++; } - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - assert(pTableMetaInfo->pTableMeta == NULL); - STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); - if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { - tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, tNameGetTableName(&pTableMetaInfo->name)); - return TSDB_CODE_TSC_INVALID_VALUE; + return TSDB_CODE_SUCCESS; +} + +// update the vgroupInfo if needed +static void doUpdateVgroupInfo(STableMeta *pTableMeta, SVgroupMsg *pVgroupMsg) { + if (pTableMeta->vgId > 0) { + int32_t vgId = pTableMeta->vgId; + assert(pTableMeta->tableType != TSDB_SUPER_TABLE); + + SNewVgroupInfo vgroupInfo = {.inUse = -1}; + taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); + + // vgroup info exists, compare with it + if (((vgroupInfo.inUse >= 0) && !vgroupInfoIdentical(&vgroupInfo, pVgroupMsg)) || (vgroupInfo.inUse < 0)) { + vgroupInfo = createNewVgroupInfo(pVgroupMsg); + taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(vgroupInfo)); + tscDebug("add new VgroupInfo, vgId:%d, total cached:%d", vgId, (int32_t) taosHashGetSize(tscVgroupMap)); + } } +} - assert(pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE || pTableMeta->tableType == TSDB_NORMAL_TABLE || pTableMeta->tableType == TSDB_STREAM_TABLE); - +static void doAddTableMetaToLocalBuf(STableMeta* pTableMeta, STableMetaMsg* pMetaMsg, bool updateSTable) { if (pTableMeta->tableType == TSDB_CHILD_TABLE) { - // check if super table hashmap or not + // add or update the corresponding super table meta data info int32_t len = (int32_t) strnlen(pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); - // super tableMeta data alreay exists, create it according to tableMeta and add it to hash map + // The super tableMeta already exists, create it according to tableMeta and add it to hash map STableMeta* pSupTableMeta = createSuperTableMeta(pMetaMsg); uint32_t size = tscGetTableMetaSize(pSupTableMeta); @@ -1980,155 +1952,99 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { tfree(pSupTableMeta); CChildTableMeta* cMeta = tscCreateChildMeta(pTableMeta); - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); - - taosHashPut(tscTableMetaInfo, name, strlen(name), cMeta, sizeof(CChildTableMeta)); + taosHashPut(tscTableMetaInfo, pMetaMsg->tableFname, strlen(pMetaMsg->tableFname), cMeta, sizeof(CChildTableMeta)); tfree(cMeta); } else { uint32_t s = tscGetTableMetaSize(pTableMeta); + taosHashPut(tscTableMetaInfo, pMetaMsg->tableFname, strlen(pMetaMsg->tableFname), pTableMeta, s); + } +} - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); - - taosHashPut(tscTableMetaInfo, name, strlen(name), pTableMeta, s); +int tscProcessTableMetaRsp(SSqlObj *pSql) { + STableMetaMsg *pMetaMsg = (STableMetaMsg *)pSql->res.pRsp; + int32_t code = tableMetaMsgConvert(pMetaMsg); + if (code != TSDB_CODE_SUCCESS) { + return code; } - // update the vgroupInfo if needed - if (pTableMeta->vgId > 0) { - int32_t vgId = pTableMeta->vgId; - assert(pTableMeta->tableType != TSDB_SUPER_TABLE); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); + assert(pTableMetaInfo->pTableMeta == NULL); - SNewVgroupInfo vgroupInfo = {.inUse = -1}; - taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); - - if (((vgroupInfo.inUse >= 0) && !vgroupInfoIdentical(&vgroupInfo, &pMetaMsg->vgroup)) || - (vgroupInfo.inUse < 0)) { // vgroup info exists, compare with it - vgroupInfo = createNewVgroupInfo(&pMetaMsg->vgroup); - taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(vgroupInfo)); - tscDebug("add new VgroupInfo, vgId:%d, total cached:%d", vgId, (int32_t) taosHashGetSize(tscVgroupMap)); - } + STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); + if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { + tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, tNameGetTableName(&pTableMetaInfo->name)); + return TSDB_CODE_TSC_INVALID_VALUE; } - tscDebug("0x%"PRIx64" recv table meta, uid:%" PRIu64 ", tid:%d, name:%s", pSql->self, pTableMeta->id.uid, pTableMeta->id.tid, - tNameGetTableName(&pTableMetaInfo->name)); + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + assert(strncmp(pMetaMsg->tableFname, name, tListLen(pMetaMsg->tableFname)) == 0); + + doAddTableMetaToLocalBuf(pTableMeta, pMetaMsg, true); + doUpdateVgroupInfo(pTableMeta, &pMetaMsg->vgroup); + + tscDebug("0x%"PRIx64" recv table meta, uid:%" PRIu64 ", tid:%d, name:%s, numOfCols:%d, numOfTags:%d", pSql->self, + pTableMeta->id.uid, pTableMeta->id.tid, tNameGetTableName(&pTableMetaInfo->name), pTableMeta->tableInfo.numOfColumns, + pTableMeta->tableInfo.numOfTags); free(pTableMeta); return TSDB_CODE_SUCCESS; } -/** - * multi table meta rsp pkg format: - * | STaosRsp | SMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2 - * |...... 1B 4B - **/ -int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) { -#if 0 - char *rsp = pSql->res.pRsp; +static SVgroupsInfo* createVgroupInfoFromMsg(char* pMsg, int32_t* size, uint64_t id) { + SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)pMsg; + pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups); - ieType = *rsp; - if (ieType != TSDB_IE_TYPE_META) { - tscError("invalid ie type:%d", ieType); - pSql->res.code = TSDB_CODE_TSC_INVALID_IE; - pSql->res.numOfTotal = 0; - return TSDB_CODE_TSC_APP_ERROR; - } + *size = (int32_t)(sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg)); - rsp++; + size_t vgroupsz = sizeof(SVgroupInfo) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo); + SVgroupsInfo *pVgroupInfo = calloc(1, vgroupsz); + assert(pVgroupInfo != NULL); - SMultiTableInfoMsg *pInfo = (SMultiTableInfoMsg *)rsp; - totalNum = htonl(pInfo->numOfTables); - rsp += sizeof(SMultiTableInfoMsg); + pVgroupInfo->numOfVgroups = pVgroupMsg->numOfVgroups; + if (pVgroupInfo->numOfVgroups <= 0) { + tscDebug("0x%" PRIx64 " empty vgroup info, no corresponding tables for stable", id); + } else { + for (int32_t j = 0; j < pVgroupInfo->numOfVgroups; ++j) { + // just init, no need to lock + SVgroupInfo *pVgroup = &pVgroupInfo->vgroups[j]; - for (i = 0; i < totalNum; i++) { - SMultiTableMeta *pMultiMeta = (SMultiTableMeta *)rsp; - STableMeta * pMeta = pMultiMeta->metas; + SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; + vmsg->vgId = htonl(vmsg->vgId); + vmsg->numOfEps = vmsg->numOfEps; + for (int32_t k = 0; k < vmsg->numOfEps; ++k) { + vmsg->epAddr[k].port = htons(vmsg->epAddr[k].port); + } - pMeta->sid = htonl(pMeta->sid); - pMeta->sversion = htons(pMeta->sversion); - pMeta->vgId = htonl(pMeta->vgId); - pMeta->uid = htobe64(pMeta->uid); + SNewVgroupInfo newVi = createNewVgroupInfo(vmsg); + pVgroup->numOfEps = newVi.numOfEps; + pVgroup->vgId = newVi.vgId; + for (int32_t k = 0; k < vmsg->numOfEps; ++k) { + pVgroup->epAddr[k].port = newVi.ep[k].port; + pVgroup->epAddr[k].fqdn = strndup(newVi.ep[k].fqdn, TSDB_FQDN_LEN); + } - if (pMeta->sid <= 0 || pMeta->vgId < 0) { - tscError("invalid meter vgId:%d, sid%d", pMeta->vgId, pMeta->sid); - pSql->res.code = TSDB_CODE_TSC_INVALID_VALUE; - pSql->res.numOfTotal = i; - return TSDB_CODE_TSC_APP_ERROR; + // check if current buffer contains the vgroup info. + // If not, add it + SNewVgroupInfo existVgroupInfo = {.inUse = -1}; + taosHashGetClone(tscVgroupMap, &newVi.vgId, sizeof(newVi.vgId), NULL, &existVgroupInfo, sizeof(SNewVgroupInfo)); + + if (((existVgroupInfo.inUse >= 0) && !vgroupInfoIdentical(&existVgroupInfo, vmsg)) || + (existVgroupInfo.inUse < 0)) { // vgroup info exists, compare with it + taosHashPut(tscVgroupMap, &newVi.vgId, sizeof(newVi.vgId), &newVi, sizeof(newVi)); + tscDebug("0x%" PRIx64 " add new VgroupInfo, vgId:%d, total cached:%d", id, newVi.vgId, (int32_t)taosHashGetSize(tscVgroupMap)); + } } - - // pMeta->numOfColumns = htons(pMeta->numOfColumns); - // - // if (pMeta->numOfTags > TSDB_MAX_TAGS || pMeta->numOfTags < 0) { - // tscError("invalid tag value count:%d", pMeta->numOfTags); - // pSql->res.code = TSDB_CODE_TSC_INVALID_VALUE; - // pSql->res.numOfTotal = i; - // return TSDB_CODE_TSC_APP_ERROR; - // } - // - // if (pMeta->numOfTags > TSDB_MAX_TAGS || pMeta->numOfTags < 0) { - // tscError("invalid numOfTags:%d", pMeta->numOfTags); - // pSql->res.code = TSDB_CODE_TSC_INVALID_VALUE; - // pSql->res.numOfTotal = i; - // return TSDB_CODE_TSC_APP_ERROR; - // } - // - // if (pMeta->numOfColumns > TSDB_MAX_COLUMNS || pMeta->numOfColumns < 0) { - // tscError("invalid numOfColumns:%d", pMeta->numOfColumns); - // pSql->res.code = TSDB_CODE_TSC_INVALID_VALUE; - // pSql->res.numOfTotal = i; - // return TSDB_CODE_TSC_APP_ERROR; - // } - // - // for (int j = 0; j < TSDB_REPLICA_MAX_NUM; ++j) { - // pMeta->vpeerDesc[j].vnode = htonl(pMeta->vpeerDesc[j].vnode); - // } - // - // pMeta->rowSize = 0; - // rsp += sizeof(SMultiTableMeta); - // pSchema = (SSchema *)rsp; - // - // int32_t numOfTotalCols = pMeta->numOfColumns + pMeta->numOfTags; - // for (int j = 0; j < numOfTotalCols; ++j) { - // pSchema->bytes = htons(pSchema->bytes); - // pSchema->colId = htons(pSchema->colId); - // - // // ignore the tags length - // if (j < pMeta->numOfColumns) { - // pMeta->rowSize += pSchema->bytes; - // } - // pSchema++; - // } - // - // rsp += numOfTotalCols * sizeof(SSchema); - // - // int32_t tagLen = 0; - // SSchema *pTagsSchema = tscGetTableTagSchema(pMeta); - // - // if (pMeta->tableType == TSDB_CHILD_TABLE) { - // for (int32_t j = 0; j < pMeta->numOfTags; ++j) { - // tagLen += pTagsSchema[j].bytes; - // } - // } - // - // rsp += tagLen; - // int32_t size = (int32_t)(rsp - ((char *)pMeta)); // Consistent with STableMeta in cache - // } } - - pSql->res.code = TSDB_CODE_SUCCESS; - pSql->res.numOfTotal = i; - tscDebug("0x%"PRIx64" load multi-metermeta resp from complete num:%d", pSql->self, pSql->res.numOfTotal); -#endif - - return TSDB_CODE_SUCCESS; + + return pVgroupInfo; } int tscProcessRetrieveFuncRsp(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; SUdfFuncMsg* pFuncMsg = (SUdfFuncMsg *)pSql->res.pRsp; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); - + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + pFuncMsg->num = htonl(pFuncMsg->num); assert(pFuncMsg->num == taosArrayGetSize(pQueryInfo->pUdfInfo)); @@ -2165,7 +2081,7 @@ int tscProcessRetrieveFuncRsp(SSqlObj* pSql) { return pSql->res.code; } - SQueryInfo* parQueryInfo = tscGetQueryInfo(&parent->cmd, parent->cmd.clauseIndex); + SQueryInfo* parQueryInfo = tscGetQueryInfo(&parent->cmd); assert(parent->signature == parent && (int64_t)pSql->param == parent->self); taosArrayDestroy(parQueryInfo->pUdfInfo); @@ -2175,6 +2091,91 @@ int tscProcessRetrieveFuncRsp(SSqlObj* pSql) { return TSDB_CODE_SUCCESS; } +int tscProcessMultiTableMetaRsp(SSqlObj *pSql) { + char *rsp = pSql->res.pRsp; + + SMultiTableMeta *pMultiMeta = (SMultiTableMeta *)rsp; + pMultiMeta->numOfTables = htonl(pMultiMeta->numOfTables); + pMultiMeta->numOfVgroup = htonl(pMultiMeta->numOfVgroup); + + rsp += sizeof(SMultiTableMeta); + + SSqlObj* pParentSql = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); + if(pParentSql == NULL) { + return pSql->res.code; + } + + SSqlCmd *pParentCmd = &pParentSql->cmd; + + SHashObj *pSet = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + + char* pMsg = pMultiMeta->meta; + for (int32_t i = 0; i < pMultiMeta->numOfTables; i++) { + STableMetaMsg *pMetaMsg = (STableMetaMsg *)pMsg; + int32_t code = tableMetaMsgConvert(pMetaMsg); + if (code != TSDB_CODE_SUCCESS) { + taosHashCleanup(pSet); + taosReleaseRef(tscObjRef, pParentSql->self); + return code; + } + + STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); + if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { + tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, pMetaMsg->tableFname); + taosHashCleanup(pSet); + taosReleaseRef(tscObjRef, pParentSql->self); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + SName sn = {0}; + tNameFromString(&sn, pMetaMsg->tableFname, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + const char* tableName = tNameGetTableName(&sn); + size_t keyLen = strlen(tableName); + + STableMetaVgroupInfo p = {.pTableMeta = pTableMeta,}; + taosHashPut(pParentCmd->pTableMetaMap, tableName, keyLen, &p, sizeof(STableMetaVgroupInfo)); + + bool addToBuf = false; + if (taosHashGet(pSet, &pMetaMsg->uid, sizeof(pMetaMsg->uid)) == NULL) { + addToBuf = true; + taosHashPut(pSet, &pMetaMsg->uid, sizeof(pMetaMsg->uid), "", 0); + } + + // create the tableMeta and add it into the TableMeta map + doAddTableMetaToLocalBuf(pTableMeta, pMetaMsg, addToBuf); + + // if the vgroup is not updated in current process, update it. + int64_t vgId = pMetaMsg->vgroup.vgId; + if (pTableMeta->tableType != TSDB_SUPER_TABLE && taosHashGet(pSet, &vgId, sizeof(vgId)) == NULL) { + doUpdateVgroupInfo(pTableMeta, &pMetaMsg->vgroup); + taosHashPut(pSet, &vgId, sizeof(vgId), "", 0); + } + + pMsg += pMetaMsg->contLen; + } + + for(int32_t i = 0; i < pMultiMeta->numOfVgroup; ++i) { + char* name = pMsg; + pMsg += TSDB_TABLE_NAME_LEN; + + STableMetaVgroupInfo* p = taosHashGet(pParentCmd->pTableMetaMap, name, strnlen(name, TSDB_TABLE_NAME_LEN)); + assert(p != NULL); + + int32_t size = 0; + p->pVgroupInfo = createVgroupInfoFromMsg(pMsg, &size, pSql->self); + pMsg += size; + } + + pSql->res.code = TSDB_CODE_SUCCESS; + pSql->res.numOfTotal = pMultiMeta->numOfTables; + tscDebug("0x%"PRIx64" load multi-tableMeta from mnode, numOfTables:%d", pSql->self, pMultiMeta->numOfTables); + + taosHashCleanup(pSet); + taosReleaseRef(tscObjRef, pParentSql->self); + return TSDB_CODE_SUCCESS; +} + int tscProcessSTableVgroupRsp(SSqlObj *pSql) { // master sqlObj locates in param SSqlObj* parent = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); @@ -2183,68 +2184,38 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { } assert(parent->signature == parent && (int64_t)pSql->param == parent->self); - + SSqlRes* pRes = &pSql->res; - + // NOTE: the order of several table must be preserved. SSTableVgroupRspMsg *pStableVgroup = (SSTableVgroupRspMsg *)pRes->pRsp; pStableVgroup->numOfTables = htonl(pStableVgroup->numOfTables); char *pMsg = pRes->pRsp + sizeof(SSTableVgroupRspMsg); - + SSqlCmd* pCmd = &parent->cmd; + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + for(int32_t i = 0; i < pStableVgroup->numOfTables; ++i) { - STableMetaInfo *pInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i); + char* name = pMsg; + pMsg += TSDB_TABLE_NAME_LEN; - SVgroupsMsg * pVgroupMsg = (SVgroupsMsg *) pMsg; - pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups); - - size_t size = sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg); - - size_t vgroupsz = sizeof(SVgroupInfo) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo); - pInfo->vgroupList = calloc(1, vgroupsz); - assert(pInfo->vgroupList != NULL); - - pInfo->vgroupList->numOfVgroups = pVgroupMsg->numOfVgroups; - if (pInfo->vgroupList->numOfVgroups <= 0) { - tscDebug("0x%"PRIx64" empty vgroup info, no corresponding tables for stable", pSql->self); - } else { - for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) { - // just init, no need to lock - SVgroupInfo *pVgroup = &pInfo->vgroupList->vgroups[j]; - - SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; - vmsg->vgId = htonl(vmsg->vgId); - vmsg->numOfEps = vmsg->numOfEps; - for (int32_t k = 0; k < vmsg->numOfEps; ++k) { - vmsg->epAddr[k].port = htons(vmsg->epAddr[k].port); - } - - SNewVgroupInfo newVi = createNewVgroupInfo(vmsg); - pVgroup->numOfEps = newVi.numOfEps; - pVgroup->vgId = newVi.vgId; - for (int32_t k = 0; k < vmsg->numOfEps; ++k) { - pVgroup->epAddr[k].port = newVi.ep[k].port; - pVgroup->epAddr[k].fqdn = strndup(newVi.ep[k].fqdn, TSDB_FQDN_LEN); - } - - // check if current buffer contains the vgroup info. - // If not, add it - SNewVgroupInfo existVgroupInfo = {.inUse = -1}; - taosHashGetClone(tscVgroupMap, &newVi.vgId, sizeof(newVi.vgId), NULL, &existVgroupInfo, sizeof(SNewVgroupInfo)); - - if (((existVgroupInfo.inUse >= 0) && !vgroupInfoIdentical(&existVgroupInfo, vmsg)) || - (existVgroupInfo.inUse < 0)) { // vgroup info exists, compare with it - taosHashPut(tscVgroupMap, &newVi.vgId, sizeof(newVi.vgId), &newVi, sizeof(newVi)); - tscDebug("add new VgroupInfo, vgId:%d, total cached:%d", newVi.vgId, (int32_t) taosHashGetSize(tscVgroupMap)); - } + STableMetaInfo *pInfo = NULL; + for(int32_t j = 0; j < pQueryInfo->numOfTables; ++j) { + STableMetaInfo *pInfo1 = tscGetTableMetaInfoFromCmd(pCmd, j); + if (strcmp(name, tNameGetTableName(&pInfo1->name)) != 0) { + continue; } + + pInfo = pInfo1; + break; } + int32_t size = 0; + pInfo->vgroupList = createVgroupInfoFromMsg(pMsg, &size, pSql->self); pMsg += size; } taosReleaseRef(tscObjRef, parent->self); - return pSql->res.code; } @@ -2256,7 +2227,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -2297,8 +2268,8 @@ int tscProcessShowRsp(SSqlObj *pSql) { TAOS_FIELD f = tscCreateField(pSchema->type, pSchema->name, pSchema->bytes); SInternalField* pInfo = tscFieldInfoAppend(pFieldInfo, &f); - pInfo->pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, - pTableSchema[i].type, pTableSchema[i].bytes, getNewResColId(pQueryInfo), pTableSchema[i].bytes, false); + pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, + pTableSchema[i].type, pTableSchema[i].bytes, getNewResColId(pCmd), pTableSchema[i].bytes, false); } pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; @@ -2316,7 +2287,7 @@ static void createHbObj(STscObj* pObj) { pSql->fp = tscProcessHeartBeatRsp; - SQueryInfo *pQueryInfo = tscGetQueryInfoS(&pSql->cmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfoS(&pSql->cmd); if (pQueryInfo == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; tfree(pSql); @@ -2382,7 +2353,7 @@ int tscProcessConnectRsp(SSqlObj *pSql) { int tscProcessUseDbRsp(SSqlObj *pSql) { STscObj * pObj = pSql->pTscObj; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); pthread_mutex_lock(&pObj->mutex); int ret = tNameExtractFullName(&pTableMetaInfo->name, pObj->db); @@ -2400,7 +2371,7 @@ int tscProcessDropDbRsp(SSqlObj *pSql) { } int tscProcessDropTableRsp(SSqlObj *pSql) { - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); //The cached tableMeta is expired in this case, so clean it in hash table char name[TSDB_TABLE_FNAME_LEN] = {0}; @@ -2414,7 +2385,7 @@ int tscProcessDropTableRsp(SSqlObj *pSql) { } int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); char name[TSDB_TABLE_FNAME_LEN] = {0}; tNameExtractFullName(&pTableMetaInfo->name, name); @@ -2474,7 +2445,7 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { pRes->completed = (pRetrieve->completed == 1); pRes->data = pRetrieve->data; - SQueryInfo* pQueryInfo = tscGetActiveQueryInfo(pCmd); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); if (tscCreateResPointerInfo(pRes, pQueryInfo) != TSDB_CODE_SUCCESS) { return pRes->code; } @@ -2488,8 +2459,6 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { tscSetResRawPtr(pRes, pQueryInfo); } - handleDownstreamOperator(pRes, pQueryInfo); - if (pSql->pSubscription != NULL) { int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutput; @@ -2519,53 +2488,60 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); -static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { +static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool autocreate) { SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); if (NULL == pNew) { tscError("0x%"PRIx64" malloc failed for new sqlobj to get table meta", pSql->self); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; + pNew->pTscObj = pSql->pTscObj; + pNew->signature = pNew; pNew->cmd.command = TSDB_SQL_META; tscAddQueryInfo(&pNew->cmd); - SQueryInfo *pNewQueryInfo = tscGetQueryInfoS(&pNew->cmd, 0); - - pNew->cmd.autoCreated = pSql->cmd.autoCreated; // create table if not exists + SQueryInfo *pNewQueryInfo = tscGetQueryInfoS(&pNew->cmd); if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { tscError("0x%"PRIx64" malloc failed for payload to get table meta", pSql->self); + tscFreeSqlObj(pNew); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - STableMetaInfo *pNewMeterMetaInfo = tscAddEmptyMetaInfo(pNewQueryInfo); - assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); + STableMetaInfo *pNewTableMetaInfo = tscAddEmptyMetaInfo(pNewQueryInfo); + assert(pNewQueryInfo->numOfTables == 1); - tNameAssign(&pNewMeterMetaInfo->name, &pTableMetaInfo->name); - - if (pSql->cmd.autoCreated) { - int32_t code = copyTagData(&pNew->cmd.tagData, &pSql->cmd.tagData); - if (code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" malloc failed for new tag data to get table meta", pSql->self); - tscFreeSqlObj(pNew); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } + tNameAssign(&pNewTableMetaInfo->name, &pTableMetaInfo->name); registerSqlObj(pNew); - tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get tableMeta, auto create:%d", pSql->self, pNew->self, - pNew->cmd.autoCreated); - pNew->fp = tscTableMetaCallBack; + pNew->fp = tscTableMetaCallBack; pNew->param = (void *)pSql->self; - tscDebug("0x%"PRIx64" metaRid from %" PRId64 " to %" PRId64 , pSql->self, pSql->metaRid, pNew->self); - + tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get tableMeta, auto create:%d, metaRid from %"PRId64" to %"PRId64, + pSql->self, pNew->self, autocreate, pSql->metaRid, pNew->self); pSql->metaRid = pNew->self; + { + STableInfoMsg *pInfoMsg = (STableInfoMsg *)pNew->cmd.payload; + int32_t code = tNameExtractFullName(&pNewTableMetaInfo->name, pInfoMsg->tableFname); + if (code != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + pInfoMsg->createFlag = htons(autocreate? 1 : 0); + char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg); + + // tag data exists + if (autocreate && pSql->cmd.tagData.dataLen != 0) { + pMsg = serializeTagData(&pSql->cmd.tagData, pMsg); + } + + pNew->cmd.payloadLen = (int32_t)(pMsg - (char*)pInfoMsg); + pNew->cmd.msgType = TSDB_MSG_TYPE_CM_TABLE_META; + } + int32_t code = tscBuildAndSendRequest(pNew, NULL); if (code == TSDB_CODE_SUCCESS) { code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated @@ -2574,7 +2550,77 @@ static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaIn return code; } -int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { +int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, __async_cb_func_t fp) { + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); + if (NULL == pNew) { + tscError("0x%"PRIx64" failed to allocate sqlobj to get multiple table meta", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pNew->pTscObj = pSql->pTscObj; + pNew->signature = pNew; + pNew->cmd.command = TSDB_SQL_MULTI_META; + + int32_t numOfTable = (int32_t) taosArrayGetSize(pNameList); + int32_t numOfVgroupList = (int32_t) taosArrayGetSize(pVgroupNameList); + + int32_t size = (numOfTable + numOfVgroupList) * TSDB_TABLE_FNAME_LEN + sizeof(SMultiTableInfoMsg); + if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, size)) { + tscError("0x%"PRIx64" malloc failed for payload to get table meta", pSql->self); + tscFreeSqlObj(pNew); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SMultiTableInfoMsg* pInfo = (SMultiTableInfoMsg*) pNew->cmd.payload; + pInfo->numOfTables = htonl((uint32_t) taosArrayGetSize(pNameList)); + pInfo->numOfVgroups = htonl((uint32_t) taosArrayGetSize(pVgroupNameList)); + + char* start = pInfo->tableNames; + int32_t len = 0; + for(int32_t i = 0; i < numOfTable; ++i) { + char* name = taosArrayGetP(pNameList, i); + if (i < numOfTable - 1 || numOfVgroupList > 0) { + len = sprintf(start, "%s,", name); + } else { + len = sprintf(start, "%s", name); + } + + start += len; + } + + for(int32_t i = 0; i < numOfVgroupList; ++i) { + char* name = taosArrayGetP(pVgroupNameList, i); + if (i < numOfVgroupList - 1) { + len = sprintf(start, "%s,", name); + } else { + len = sprintf(start, "%s", name); + } + + start += len; + } + + pNew->cmd.payloadLen = (int32_t) ((start - pInfo->tableNames) + sizeof(SMultiTableInfoMsg)); + pNew->cmd.msgType = TSDB_MSG_TYPE_CM_TABLES_META; + + registerSqlObj(pNew); + tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get %d tableMeta, vgroupInfo:%d, msg size:%d", pSql->self, + pNew->self, numOfTable, numOfVgroupList, pNew->cmd.payloadLen); + + pNew->fp = fp; + pNew->param = (void *)pSql->self; + + tscDebug("0x%"PRIx64" metaRid from %" PRId64 " to %" PRId64 , pSql->self, pSql->metaRid, pNew->self); + + pSql->metaRid = pNew->self; + int32_t code = tscBuildAndSendRequest(pNew, NULL); + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated + } + + return code; +} + +int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool autocreate) { assert(tIsValidName(&pTableMetaInfo->name)); uint32_t size = tscGetTableMetaMaxSize(); @@ -2582,19 +2628,16 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { pTableMetaInfo->pTableMeta = calloc(1, size); pTableMetaInfo->tableMetaSize = size; } else if (pTableMetaInfo->tableMetaSize < size) { - char *tmp = realloc(pTableMetaInfo->pTableMeta, size); - if (tmp == NULL) { + char *tmp = realloc(pTableMetaInfo->pTableMeta, size); + if (tmp == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } pTableMetaInfo->pTableMeta = (STableMeta *)tmp; - memset(pTableMetaInfo->pTableMeta, 0, size); - pTableMetaInfo->tableMetaSize = size; - } else { - //uint32_t s = tscGetTableMetaSize(pTableMetaInfo->pTableMeta); - memset(pTableMetaInfo->pTableMeta, 0, size); - pTableMetaInfo->tableMetaSize = size; } + memset(pTableMetaInfo->pTableMeta, 0, size); + pTableMetaInfo->tableMetaSize = size; + pTableMetaInfo->pTableMeta->tableType = -1; pTableMetaInfo->pTableMeta->tableInfo.numOfColumns = -1; @@ -2610,22 +2653,26 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { STableMeta* pMeta = pTableMetaInfo->pTableMeta; if (pMeta->id.uid > 0) { + // in case of child table, here only get the if (pMeta->tableType == TSDB_CHILD_TABLE) { - int32_t code = tscCreateTableMetaFromCChildMeta(pTableMetaInfo->pTableMeta, name, buf); + int32_t code = tscCreateTableMetaFromSTableMeta(pTableMetaInfo->pTableMeta, name, buf); if (code != TSDB_CODE_SUCCESS) { - return getTableMetaFromMnode(pSql, pTableMetaInfo); + return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); } } return TSDB_CODE_SUCCESS; } - return getTableMetaFromMnode(pSql, pTableMetaInfo); + return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); +} + +int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { + return tscGetTableMetaImpl(pSql, pTableMetaInfo, false); } int tscGetTableMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists) { - pSql->cmd.autoCreated = createIfNotExists; - return tscGetTableMeta(pSql, pTableMetaInfo); + return tscGetTableMetaImpl(pSql, pTableMetaInfo, createIfNotExists); } int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo) { @@ -2645,7 +2692,7 @@ int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd, 0); + SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd); pNewQueryInfo->pUdfInfo = taosArrayInit(4, sizeof(SUdfInfo)); for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) { @@ -2655,7 +2702,7 @@ int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo) { info.name = strdup(p1->name); taosArrayPush(pNewQueryInfo->pUdfInfo, &info); } - + pNew->cmd.active = pNewQueryInfo; if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { @@ -2683,7 +2730,7 @@ int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo) { } /** - * retrieve table meta from mnode, and update the local table meta hashmap. + * retrieve table meta from mnode, and then update the local table meta hashmap. * @param pSql sql object * @param tableIndex table index * @return status code @@ -2691,14 +2738,14 @@ int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo) { int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { SSqlCmd *pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); char name[TSDB_TABLE_FNAME_LEN] = {0}; int32_t code = tNameExtractFullName(&pTableMetaInfo->name, name); if (code != TSDB_CODE_SUCCESS) { tscError("0x%"PRIx64" failed to generate the table full name", pSql->self); - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; @@ -2711,11 +2758,10 @@ int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { size_t len = strlen(name); taosHashRemove(tscTableMetaInfo, name, len); - return getTableMetaFromMnode(pSql, pTableMetaInfo); + return getTableMetaFromMnode(pSql, pTableMetaInfo, false); } -static bool allVgroupInfoRetrieved(SSqlCmd* pCmd, int32_t clauseIndex) { - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, clauseIndex); +static bool allVgroupInfoRetrieved(SQueryInfo* pQueryInfo) { for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); if (pTableMetaInfo->vgroupList == NULL) { @@ -2727,11 +2773,9 @@ static bool allVgroupInfoRetrieved(SSqlCmd* pCmd, int32_t clauseIndex) { return true; } -int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { - int code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - SSqlCmd *pCmd = &pSql->cmd; - - if (allVgroupInfoRetrieved(pCmd, clauseIndex)) { +int tscGetSTableVgroupInfo(SSqlObj *pSql, SQueryInfo* pQueryInfo) { + int32_t code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + if (allVgroupInfoRetrieved(pQueryInfo)) { return TSDB_CODE_SUCCESS; } @@ -2742,13 +2786,12 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { pNew->cmd.command = TSDB_SQL_STABLEVGROUP; // TODO TEST IT - SQueryInfo *pNewQueryInfo = tscGetQueryInfoS(&pNew->cmd, 0); + SQueryInfo *pNewQueryInfo = tscGetQueryInfoS(&pNew->cmd); if (pNewQueryInfo == NULL) { tscFreeSqlObj(pNew); return code; } - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, clauseIndex); + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i); STableMeta* pTableMeta = tscTableMetaDup(pMInfo->pTableMeta); @@ -2809,9 +2852,8 @@ void tscInitMsgsFp() { tscBuildMsg[TSDB_SQL_CONNECT] = tscBuildConnectMsg; tscBuildMsg[TSDB_SQL_USE_DB] = tscBuildUseDbMsg; - tscBuildMsg[TSDB_SQL_META] = tscBuildTableMetaMsg; +// tscBuildMsg[TSDB_SQL_META] = tscBuildTableMetaMsg; tscBuildMsg[TSDB_SQL_STABLEVGROUP] = tscBuildSTableVgroupMsg; - tscBuildMsg[TSDB_SQL_MULTI_META] = tscBuildMultiMeterMetaMsg; tscBuildMsg[TSDB_SQL_RETRIEVE_FUNC] = tscBuildRetrieveFuncMsg; tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; @@ -2830,7 +2872,7 @@ void tscInitMsgsFp() { tscProcessMsgRsp[TSDB_SQL_USE_DB] = tscProcessUseDbRsp; tscProcessMsgRsp[TSDB_SQL_META] = tscProcessTableMetaRsp; tscProcessMsgRsp[TSDB_SQL_STABLEVGROUP] = tscProcessSTableVgroupRsp; - tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiMeterMetaRsp; + tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiTableMetaRsp; tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp; tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 562731b59a..d376ea1b6d 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -373,11 +373,15 @@ int taos_num_fields(TAOS_RES *res) { if (pSql == NULL || pSql->signature != pSql) return 0; int32_t num = 0; - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); if (pQueryInfo == NULL) { return num; } + while(pQueryInfo->pDownstream != NULL) { + pQueryInfo = pQueryInfo->pDownstream; + } + size_t numOfCols = tscNumOfFields(pQueryInfo); for(int32_t i = 0; i < numOfCols; ++i) { SInternalField* pInfo = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i); @@ -408,7 +412,7 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { SSqlRes *pRes = &pSql->res; if (pSql == NULL || pSql->signature != pSql) return 0; - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); if (pQueryInfo == NULL) { return NULL; } @@ -560,7 +564,7 @@ static bool tscKillQueryInDnode(SSqlObj* pSql) { return true; } - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); if ((pQueryInfo == NULL) || pQueryInfo->globalMerge) { return true; @@ -614,7 +618,7 @@ int taos_errno(TAOS_RES *tres) { * why the sql is invalid */ static bool hasAdditionalErrorInfo(int32_t code, SSqlCmd *pCmd) { - if (code != TSDB_CODE_TSC_INVALID_SQL + if (code != TSDB_CODE_TSC_INVALID_OPERATION && code != TSDB_CODE_TSC_SQL_SYNTAX_ERROR) { return false; } @@ -673,7 +677,7 @@ char *taos_get_client_info() { return version; } static void tscKillSTableQuery(SSqlObj *pSql) { SSqlCmd* pCmd = &pSql->cmd; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); if (!pQueryInfo->globalMerge) { return; @@ -724,7 +728,7 @@ void taos_stop_query(TAOS_RES *res) { // set the error code for master pSqlObj firstly pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); if (pQueryInfo->globalMerge) { assert(pSql->rpcRid <= 0); @@ -754,7 +758,7 @@ bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) { return true; } - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); if (pQueryInfo == NULL) { return true; } @@ -829,9 +833,9 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) case TSDB_DATA_TYPE_NCHAR: { int32_t charLen = varDataLen((char*)row[i] - VARSTR_HEADER_SIZE); if (fields[i].type == TSDB_DATA_TYPE_BINARY) { - assert(charLen <= fields[i].bytes); + assert(charLen <= fields[i].bytes && charLen >= 0); } else { - assert(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE); + assert(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0); } memcpy(str + len, row[i], charLen); @@ -868,15 +872,11 @@ int taos_validate_sql(TAOS *taos, const char *sql) { SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - pSql->pTscObj = taos; + pSql->pTscObj = taos; pSql->signature = pSql; - - SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; - pRes->numOfTotal = 0; - pRes->numOfClauseTotal = 0; - + pCmd->resColumnId = TSDB_RES_COL_ID; tscDebug("0x%"PRIx64" Valid SQL: %s pObj:%p", pSql->self, sql, pObj); @@ -896,10 +896,10 @@ int taos_validate_sql(TAOS *taos, const char *sql) { strtolower(pSql->sqlstr, sql); - pCmd->curSql = NULL; - if (NULL != pCmd->pTableBlockHashList) { - taosHashCleanup(pCmd->pTableBlockHashList); - pCmd->pTableBlockHashList = NULL; +// pCmd->curSql = NULL; + if (NULL != pCmd->insertParam.pTableBlockHashList) { + taosHashCleanup(pCmd->insertParam.pTableBlockHashList); + pCmd->insertParam.pTableBlockHashList = NULL; } pSql->fp = asyncCallback; @@ -921,90 +921,19 @@ int taos_validate_sql(TAOS *taos, const char *sql) { return code; } -static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t tblListLen) { - // must before clean the sqlcmd object - tscResetSqlCmd(&pSql->cmd, false); - - SSqlCmd *pCmd = &pSql->cmd; - - pCmd->command = TSDB_SQL_MULTI_META; - pCmd->count = 0; - - int code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - char *str = (char *)tblNameList; - - SQueryInfo *pQueryInfo = tscGetQueryInfoS(pCmd, pCmd->clauseIndex); - if (pQueryInfo == NULL) { - pSql->res.code = terrno; - return terrno; +void loadMultiTableMetaCallback(void *param, TAOS_RES *res, int code) { + SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)param); + if (pSql == NULL) { + return; } - STableMetaInfo *pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); + taosReleaseRef(tscObjRef, pSql->self); + pSql->res.code = code; + tsem_post(&pSql->rspSem); +} - if ((code = tscAllocPayload(pCmd, tblListLen + 16)) != TSDB_CODE_SUCCESS) { - return code; - } - - char *nextStr; - char tblName[TSDB_TABLE_FNAME_LEN]; - int payloadLen = 0; - char *pMsg = pCmd->payload; - while (1) { - nextStr = strchr(str, ','); - if (nextStr == NULL) { - break; - } - - memcpy(tblName, str, nextStr - str); - int32_t len = (int32_t)(nextStr - str); - tblName[len] = '\0'; - - str = nextStr + 1; - len = (int32_t)strtrim(tblName); - - SStrToken sToken = {.n = len, .type = TK_ID, .z = tblName}; - tGetToken(tblName, &sToken.type); - - // Check if the table name available or not - if (tscValidateName(&sToken) != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - sprintf(pCmd->payload, "table name is invalid"); - return code; - } - - if ((code = tscSetTableFullName(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) { - return code; - } - - if (++pCmd->count > TSDB_MULTI_TABLEMETA_MAX_NUM) { - code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - sprintf(pCmd->payload, "tables over the max number"); - return code; - } - - int32_t xlen = tNameLen(&pTableMetaInfo->name); - if (payloadLen + xlen + 128 >= pCmd->allocSize) { - char *pNewMem = realloc(pCmd->payload, pCmd->allocSize + tblListLen); - if (pNewMem == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - sprintf(pCmd->payload, "failed to allocate memory"); - return code; - } - - pCmd->payload = pNewMem; - pCmd->allocSize = pCmd->allocSize + tblListLen; - pMsg = pCmd->payload; - } - - char n[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, n); - payloadLen += sprintf(pMsg + payloadLen, "%s,", n); - } - - *(pMsg + payloadLen) = '\0'; - pCmd->payloadLen = payloadLen + 1; - - return TSDB_CODE_SUCCESS; +static void freeElem(void* p) { + tfree(*(char**)p); } int taos_load_table_info(TAOS *taos, const char *tableNameList) { @@ -1020,38 +949,28 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { pSql->pTscObj = taos; pSql->signature = pSql; - SSqlRes *pRes = &pSql->res; + pSql->fp = NULL; // todo set the correct callback function pointer + pSql->cmd.pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - pRes->code = 0; - pRes->numOfTotal = 0; // the number of getting table meta from server - pRes->numOfClauseTotal = 0; - - assert(pSql->fp == NULL); - tscDebug("0x%"PRIx64" tableNameList: %s pObj:%p", pSql->self, tableNameList, pObj); - - int32_t tblListLen = (int32_t)strlen(tableNameList); - if (tblListLen > MAX_TABLE_NAME_LENGTH) { - tscError("0x%"PRIx64" tableNameList too long, length:%d, maximum allowed:%d", pSql->self, tblListLen, MAX_TABLE_NAME_LENGTH); + int32_t length = (int32_t)strlen(tableNameList); + if (length > MAX_TABLE_NAME_LENGTH) { + tscError("0x%"PRIx64" tableNameList too long, length:%d, maximum allowed:%d", pSql->self, length, MAX_TABLE_NAME_LENGTH); tscFreeSqlObj(pSql); - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } - char *str = calloc(1, tblListLen + 1); + char *str = calloc(1, length + 1); if (str == NULL) { - tscError("0x%"PRIx64" failed to malloc sql string buffer", pSql->self); + tscError("0x%"PRIx64" failed to allocate sql string buffer", pSql->self); tscFreeSqlObj(pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } strtolower(str, tableNameList); - int32_t code = (uint8_t) tscParseTblNameList(pSql, str, tblListLen); + SArray* plist = taosArrayInit(4, POINTER_BYTES); + SArray* vgroupList = taosArrayInit(4, POINTER_BYTES); - /* - * set the qhandle to 0 before return in order to erase the qhandle value assigned in the previous successful query. - * If qhandle is NOT set 0, the function of taos_free_result() will send message to server by calling tscBuildAndSendRequest() - * to free connection, which may cause segment fault, when the parse phrase is not even successfully executed. - */ - pRes->qId = 0; + int32_t code = (uint8_t) tscTransferTableNameList(pSql, str, length, plist); free(str); if (code != TSDB_CODE_SUCCESS) { @@ -1059,12 +978,23 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { return code; } - tscDoQuery(pSql); + registerSqlObj(pSql); + tscDebug("0x%"PRIx64" load multiple table meta, tableNameList: %s pObj:%p", pSql->self, tableNameList, pObj); - tscDebug("0x%"PRIx64" load multi-table meta result:%d %s pObj:%p", pSql->self, pRes->code, taos_errstr(pSql), pObj); - if ((code = pRes->code) != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pSql); + code = getMultiTableMetaFromMnode(pSql, plist, vgroupList, loadMultiTableMetaCallback); + if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + code = TSDB_CODE_SUCCESS; } + taosArrayDestroyEx(plist, freeElem); + taosArrayDestroyEx(vgroupList, freeElem); + + if (code != TSDB_CODE_SUCCESS) { + tscFreeRegisteredSqlObj(pSql); + return code; + } + + tsem_wait(&pSql->rspSem); + tscFreeRegisteredSqlObj(pSql); return code; } diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 17bf575b60..8e11fd0cfb 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -37,7 +37,7 @@ static int64_t getDelayValueAfterTimewindowClosed(SSqlStream* pStream, int64_t l static bool isProjectStream(SQueryInfo* pQueryInfo) { for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - SExprInfo *pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo *pExpr = tscExprGet(pQueryInfo, i); if (pExpr->base.functionId != TSDB_FUNC_PRJ) { return false; } @@ -89,12 +89,12 @@ static void doLaunchQuery(void* param, TAOS_RES* tres, int32_t code) { return; } - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); code = tscGetTableMeta(pSql, pTableMetaInfo); if (code == 0 && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - code = tscGetSTableVgroupInfo(pSql, 0); + code = tscGetSTableVgroupInfo(pSql, pQueryInfo); } if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { @@ -138,7 +138,7 @@ static void tscProcessStreamTimer(void *handle, void *tmrId) { pStream->numOfRes = 0; // reset the numOfRes. SSqlObj *pSql = pStream->pSql; - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); tscDebug("0x%"PRIx64" timer launch query", pSql->self); if (pStream->isProject) { @@ -197,7 +197,7 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf tscError("0x%"PRIx64" stream:%p, query data failed, code:0x%08x, retry in %" PRId64 "ms", pStream->pSql->self, pStream, numOfRows, retryDelay); - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0); + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0); char name[TSDB_TABLE_FNAME_LEN] = {0}; tNameExtractFullName(&pTableMetaInfo->name, name); @@ -224,7 +224,7 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf static void tscStreamFillTimeGap(SSqlStream* pStream, TSKEY ts) { #if 0 SSqlObj * pSql = pStream->pSql; - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); if (pQueryInfo->fillType != TSDB_FILL_SET_VALUE && pQueryInfo->fillType != TSDB_FILL_NULL) { return; @@ -273,7 +273,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf return; } - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); STableMetaInfo *pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; if (numOfRows > 0) { // when reaching here the first execution of stream computing is successful. @@ -444,7 +444,7 @@ static int32_t tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) { int64_t minIntervalTime = (pStream->precision == TSDB_TIME_PRECISION_MICRO) ? tsMinIntervalTime * 1000L : tsMinIntervalTime; - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); if (!pStream->isProject && pQueryInfo->interval.interval == 0) { sprintf(pSql->cmd.payload, "the interval value is 0"); @@ -494,7 +494,7 @@ static int32_t tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) { } static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, int64_t stime) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); if (pStream->isProject) { // no data in table, flush all data till now to destination meter, 10sec delay @@ -556,7 +556,7 @@ static void tscCreateStream(void *param, TAOS_RES *res, int code) { return; } - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); @@ -614,31 +614,32 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p return NULL; } - pStream->stime = stime; - pStream->fp = fp; + pStream->stime = stime; + pStream->fp = fp; pStream->callback = callback; - pStream->param = param; - pStream->pSql = pSql; - pSql->pStream = pStream; - pSql->param = pStream; - pSql->maxRetry = TSDB_MAX_REPLICA; + pStream->param = param; + pStream->pSql = pSql; - pSql->sqlstr = calloc(1, strlen(sqlstr) + 1); + pSql->pStream = pStream; + pSql->param = pStream; + pSql->maxRetry = TSDB_MAX_REPLICA; + pSql->sqlstr = calloc(1, strlen(sqlstr) + 1); if (pSql->sqlstr == NULL) { tscError("0x%"PRIx64" failed to malloc sql string buffer", pSql->self); tscFreeSqlObj(pSql); + free(pStream); return NULL; } strtolower(pSql->sqlstr, sqlstr); + pSql->fp = tscCreateStream; + pSql->fetchFp = tscCreateStream; + pSql->cmd.resColumnId = TSDB_RES_COL_ID; + tsem_init(&pSql->rspSem, 0, 0); registerSqlObj(pSql); tscDebugL("0x%"PRIx64" SQL: %s", pSql->self, pSql->sqlstr); - tsem_init(&pSql->rspSem, 0, 0); - - pSql->fp = tscCreateStream; - pSql->fetchFp = tscCreateStream; int32_t code = tsParseSql(pSql, true); if (code == TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 6928058f23..42330aa18e 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -151,6 +151,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* strtolower(pSql->sqlstr, pSql->sqlstr); pRes->qId = 0; pRes->numOfRows = 1; + pCmd->resColumnId = TSDB_RES_COL_ID; code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); if (code != TSDB_CODE_SUCCESS) { @@ -173,7 +174,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* if (pSql->cmd.command != TSDB_SQL_SELECT && pSql->cmd.command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) { line = __LINE__; - code = TSDB_CODE_TSC_INVALID_SQL; + code = TSDB_CODE_TSC_INVALID_OPERATION; goto fail; } @@ -215,7 +216,7 @@ static void tscProcessSubscriptionTimer(void *handle, void *tmrId) { taosTmrReset(tscProcessSubscriptionTimer, pSub->interval, pSub, tscTmr, &pSub->pTimer); } - +//TODO refactor: extract table list name not simply from the sql static SArray* getTableList( SSqlObj* pSql ) { const char* p = strstr( pSql->sqlstr, " from " ); assert(p != NULL); // we are sure this is a 'select' statement @@ -224,11 +225,11 @@ static SArray* getTableList( SSqlObj* pSql ) { SSqlObj* pNew = taos_query(pSql->pTscObj, sql); if (pNew == NULL) { - tscError("0x%"PRIx64"failed to retrieve table id: cannot create new sql object.", pSql->self); + tscError("0x%"PRIx64" failed to retrieve table id: cannot create new sql object.", pSql->self); return NULL; } else if (taos_errno(pNew) != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64"failed to retrieve table id,error: %s", pSql->self, tstrerror(taos_errno(pNew))); + tscError("0x%"PRIx64" failed to retrieve table id,error: %s", pSql->self, tstrerror(taos_errno(pNew))); return NULL; } @@ -266,7 +267,7 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) { pSub->lastSyncTime = taosGetTimestampMs(); - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; SSubscriptionProgress target = {.uid = pTableMeta->id.uid, .key = 0}; @@ -284,7 +285,7 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) { } size_t numOfTables = taosArrayGetSize(tables); - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); SArray* progress = taosArrayInit(numOfTables, sizeof(SSubscriptionProgress)); for( size_t i = 0; i < numOfTables; i++ ) { STidTags* tt = taosArrayGet( tables, i ); @@ -304,7 +305,7 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) { } taosArrayDestroy(tables); - TSDB_QUERY_SET_TYPE(tscGetQueryInfo(pCmd, 0)->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); + TSDB_QUERY_SET_TYPE(tscGetQueryInfo(pCmd)->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); return 1; } @@ -503,8 +504,8 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) { SSqlObj *pSql = pSub->pSql; SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, 0); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); if (taosArrayGetSize(pSub->progress) > 0) { // fix crash in single table subscription size_t size = taosArrayGetSize(pSub->progress); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 9f67528a3f..f6560a02b8 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -66,15 +66,11 @@ static void skipRemainValue(STSBuf* pTSBuf, tVariant* tag1) { } static void subquerySetState(SSqlObj *pSql, SSubqueryState *subState, int idx, int8_t state) { - assert(idx < subState->numOfSub); - assert(subState->states); + assert(idx < subState->numOfSub && subState->states != NULL); + tscDebug("subquery:0x%"PRIx64",%d state set to %d", pSql->self, idx, state); pthread_mutex_lock(&subState->mutex); - - tscDebug("subquery:0x%"PRIx64",%d state set to %d", pSql->self, idx, state); - subState->states[idx] = state; - pthread_mutex_unlock(&subState->mutex); } @@ -87,8 +83,7 @@ static bool allSubqueryDone(SSqlObj *pParentSql) { for (int i = 0; i < subState->numOfSub; i++) { SSqlObj* pSub = pParentSql->pSubs[i]; if (0 == subState->states[i]) { - tscDebug("0x%"PRIx64" subquery:0x%"PRIx64", index: %d NOT finished, abort query completion check", pParentSql->self, - pSub->self, i); + tscDebug("0x%"PRIx64" subquery:0x%"PRIx64", index: %d NOT finished yet", pParentSql->self, pSub->self, i); done = false; break; } else { @@ -103,39 +98,31 @@ static bool allSubqueryDone(SSqlObj *pParentSql) { return done; } -static bool subAndCheckDone(SSqlObj *pSql, SSqlObj *pParentSql, int idx) { +bool subAndCheckDone(SSqlObj *pSql, SSqlObj *pParentSql, int idx) { SSubqueryState *subState = &pParentSql->subState; - assert(idx < subState->numOfSub); pthread_mutex_lock(&subState->mutex); - bool done = allSubqueryDone(pParentSql); - - if (done) { - tscDebug("0x%"PRIx64" subquery:0x%"PRIx64",%d all subs already done", pParentSql->self, - pSql->self, idx); - - pthread_mutex_unlock(&subState->mutex); - - return false; - } - - tscDebug("0x%"PRIx64" subquery:0x%"PRIx64",%d state set to 1", pParentSql->self, pSql->self, idx); +// bool done = allSubqueryDone(pParentSql); +// if (done) { +// tscDebug("0x%"PRIx64" subquery:0x%"PRIx64",%d all subs already done", pParentSql->self, pSql->self, idx); +// pthread_mutex_unlock(&subState->mutex); +// return false; +// } + tscDebug("0x%"PRIx64" subquery:0x%"PRIx64", index:%d state set to 1", pParentSql->self, pSql->self, idx); subState->states[idx] = 1; - done = allSubqueryDone(pParentSql); - + bool done = allSubqueryDone(pParentSql); pthread_mutex_unlock(&subState->mutex); - return done; } static int64_t doTSBlockIntersect(SSqlObj* pSql, STimeWindow * win) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, pSql->cmd.clauseIndex); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); win->skey = INT64_MAX; win->ekey = INT64_MIN; @@ -160,7 +147,7 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, STimeWindow * win) { for (int32_t i = 0; i < joinNum; ++i) { STSBuf* output = tsBufCreate(true, pQueryInfo->order.order); - SQueryInfo* pSubQueryInfo = tscGetQueryInfo(&pSql->pSubs[i]->cmd, 0); + SQueryInfo* pSubQueryInfo = tscGetQueryInfo(&pSql->pSubs[i]->cmd); pSubQueryInfo->tsBuf = output; @@ -405,12 +392,12 @@ SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, int32_t index) { pSupporter->pObj = pSql; pSupporter->subqueryIndex = index; - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, pSql->cmd.clauseIndex); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); memcpy(&pSupporter->interval, &pQueryInfo->interval, sizeof(pSupporter->interval)); pSupporter->limit = pQueryInfo->limit; - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, index); + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, index); pSupporter->uid = pTableMetaInfo->pTableMeta->id.uid; assert (pSupporter->uid != 0); @@ -428,14 +415,15 @@ static void tscDestroyJoinSupporter(SJoinSupporter* pSupporter) { } if (pSupporter->exprList != NULL) { - tscSqlExprInfoDestroy(pSupporter->exprList); + tscExprDestroy(pSupporter->exprList); + pSupporter->exprList = NULL; } if (pSupporter->colList != NULL) { tscColumnListDestroy(pSupporter->colList); } - tscFieldInfoClear(&pSupporter->fieldsInfo); +// tscFieldInfoClear(&pSupporter->fieldsInfo); if (pSupporter->pTSBuf != NULL) { tsBufDestroy(pSupporter->pTSBuf); @@ -449,7 +437,6 @@ static void tscDestroyJoinSupporter(SJoinSupporter* pSupporter) { pSupporter->f = NULL; } - if (pSupporter->pVgroupTables != NULL) { taosArrayDestroy(pSupporter->pVgroupTables); pSupporter->pVgroupTables = NULL; @@ -560,12 +547,12 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { continue; } - SQueryInfo *pSubQueryInfo = tscGetQueryInfo(&pPrevSub->cmd, 0); + SQueryInfo *pSubQueryInfo = tscGetQueryInfo(&pPrevSub->cmd); STSBuf *pTsBuf = pSubQueryInfo->tsBuf; pSubQueryInfo->tsBuf = NULL; // free result for async object will also free sqlObj - assert(tscSqlExprNumOfExprs(pSubQueryInfo) == 1); // ts_comp query only requires one result columns + assert(tscNumOfExprs(pSubQueryInfo) == 1); // ts_comp query only requires one result columns taos_free_result(pPrevSub); SSqlObj *pNew = createSubqueryObj(pSql, (int16_t) i, tscJoinQueryCallback, pSupporter, TSDB_SQL_SELECT, NULL); @@ -578,7 +565,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { tscClearSubqueryInfo(&pNew->cmd); pSql->pSubs[i] = pNew; - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pNew->cmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfo(&pNew->cmd); pQueryInfo->tsBuf = pTsBuf; // transfer the ownership of timestamp comp-z data to the new created object // set the second stage sub query for join process @@ -593,7 +580,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { pQueryInfo->groupbyExpr = pSupporter->groupInfo; pQueryInfo->pUpstream = taosArrayInit(4, sizeof(POINTER_BYTES)); - assert(pNew->subState.numOfSub == 0 && pNew->cmd.numOfClause == 1 && pQueryInfo->numOfTables == 1); + assert(pNew->subState.numOfSub == 0 && pQueryInfo->numOfTables == 1); tscFieldInfoUpdateOffset(pQueryInfo); @@ -601,22 +588,20 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { pTableMetaInfo->pVgroupTables = pSupporter->pVgroupTables; pSupporter->exprList = NULL; - pSupporter->colList = NULL; + pSupporter->colList = NULL; pSupporter->pVgroupTables = NULL; memset(&pSupporter->fieldsInfo, 0, sizeof(SFieldInfo)); - memset(&pSupporter->groupInfo, 0, sizeof(SSqlGroupbyExpr)); + memset(&pSupporter->groupInfo, 0, sizeof(SGroupbyExpr)); /* * When handling the projection query, the offset value will be modified for table-table join, which is changed * during the timestamp intersection. */ pSupporter->limit = pQueryInfo->limit; -// pQueryInfo->limit = pSupporter->limit; - SColumnIndex index = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; SSchema* s = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, 0); - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, 0); + SExprInfo* pExpr = tscExprGet(pQueryInfo, 0); int16_t funcId = pExpr->base.functionId; // add the invisible timestamp column @@ -625,11 +610,11 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { int16_t functionId = tscIsProjectionQuery(pQueryInfo)? TSDB_FUNC_PRJ : TSDB_FUNC_TS; - tscAddFuncInSelectClause(pQueryInfo, 0, functionId, &index, s, TSDB_COL_NORMAL); + tscAddFuncInSelectClause(pQueryInfo, 0, functionId, &index, s, TSDB_COL_NORMAL, getNewResColId(&pNew->cmd)); tscPrintSelNodeList(pNew, 0); tscFieldInfoUpdateOffset(pQueryInfo); - pExpr = tscSqlExprGet(pQueryInfo, 0); + pExpr = tscExprGet(pQueryInfo, 0); } // set the join condition tag column info, todo extract method @@ -659,11 +644,11 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { } } - subquerySetState(pPrevSub, &pSql->subState, i, 0); + subquerySetState(pNew, &pSql->subState, i, 0); size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); - tscDebug("0x%"PRIx64" subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, name:%s", - pSql->self, pNew, 0, pTableMetaInfo->vgroupIndex, pQueryInfo->type, taosArrayGetSize(pQueryInfo->exprList), + tscDebug("0x%"PRIx64" subquery:0x%"PRIx64" tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, name:%s", + pSql->self, pNew->self, 0, pTableMetaInfo->vgroupIndex, pQueryInfo->type, taosArrayGetSize(pQueryInfo->exprList), numOfCols, pQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pTableMetaInfo->name)); } @@ -682,7 +667,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { continue; } - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->pSubs[i]->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->pSubs[i]->cmd); executeQuery(pSql->pSubs[i], pQueryInfo); } @@ -708,8 +693,6 @@ void freeJoinSubqueryObj(SSqlObj* pSql) { } tfree(pSql->subState.states); - - pSql->subState.numOfSub = 0; } @@ -728,8 +711,6 @@ static int32_t quitAllSubquery(SSqlObj* pSqlSub, SSqlObj* pSqlObj, SJoinSupporte static void updateQueryTimeRange(SQueryInfo* pQueryInfo, STimeWindow* win) { assert(pQueryInfo->window.skey <= win->skey && pQueryInfo->window.ekey >= win->ekey); pQueryInfo->window = *win; - - } int32_t tidTagsCompar(const void* p1, const void* p2) { @@ -821,12 +802,14 @@ static void issueTsCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* tscClearSubqueryInfo(pCmd); tscFreeSqlResult(pSql); - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); assert(pQueryInfo->numOfTables == 1); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STimeWindow window = pQueryInfo->window; tscInitQueryInfo(pQueryInfo); + pQueryInfo->window = window; TSDB_QUERY_CLEAR_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY); TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); @@ -836,11 +819,11 @@ static void issueTsCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = 1}; SColumnIndex index = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TS_COMP, &index, &colSchema, TSDB_COL_NORMAL); + tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TS_COMP, &index, &colSchema, TSDB_COL_NORMAL, getNewResColId(pCmd)); // set the tags value for ts_comp function if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - SExprInfo *pExpr = tscSqlExprGet(pQueryInfo, 0); + SExprInfo *pExpr = tscExprGet(pQueryInfo, 0); int16_t tagColId = tscGetJoinTagColIdByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->id.uid); pExpr->base.param[0].i64 = tagColId; pExpr->base.param[0].nLen = sizeof(int64_t); @@ -868,7 +851,7 @@ static void issueTsCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* "0x%"PRIx64" subquery:0x%"PRIx64" tableIndex:%d, vgroupIndex:%d, numOfVgroups:%d, type:%d, ts_comp query to retrieve timestamps, " "numOfExpr:%" PRIzu ", colList:%" PRIzu ", numOfOutputFields:%d, name:%s", pParent->self, pSql->self, 0, pTableMetaInfo->vgroupIndex, pTableMetaInfo->vgroupList->numOfVgroups, pQueryInfo->type, - tscSqlExprNumOfExprs(pQueryInfo), numOfCols, pQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pTableMetaInfo->name)); + tscNumOfExprs(pQueryInfo), numOfCols, pQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pTableMetaInfo->name)); tscBuildAndSendRequest(pSql, NULL); } @@ -1110,7 +1093,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow SSqlCmd* pCmd = &pSql->cmd; SSqlRes* pRes = &pSql->res; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); // todo, the type may not include TSDB_QUERY_TYPE_TAG_FILTER_QUERY assert(TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)); @@ -1232,7 +1215,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow SSqlCmd* pSubCmd = &pParentSql->pSubs[m]->cmd; SArray** s = taosArrayGet(resList, m); - SQueryInfo* pQueryInfo1 = tscGetQueryInfo(pSubCmd, 0); + SQueryInfo* pQueryInfo1 = tscGetQueryInfo(pSubCmd); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo1, 0); tscBuildVgroupTableInfo(pParentSql, pTableMetaInfo, *s); @@ -1266,7 +1249,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow SSqlCmd* pCmd = &pSql->cmd; SSqlRes* pRes = &pSql->res; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); assert(!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)); if (pParentSql->res.code != TSDB_CODE_SUCCESS) { @@ -1301,7 +1284,6 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow if (pSupporter->f == NULL) { tscError("0x%"PRIx64" failed to create tmp file:%s, reason:%s", pSql->self, pSupporter->path, strerror(errno)); - pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); if (quitAllSubquery(pSql, pParentSql, pSupporter)) { @@ -1398,7 +1380,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow } // launch the query the retrieve actual results from vnode along with the filtered timestamp - SQueryInfo* pPQueryInfo = tscGetQueryInfo(&pParentSql->cmd, pParentSql->cmd.clauseIndex); + SQueryInfo* pPQueryInfo = tscGetQueryInfo(&pParentSql->cmd); updateQueryTimeRange(pPQueryInfo, &win); //update the vgroup that involved in real data query @@ -1414,7 +1396,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR SSqlCmd* pCmd = &pSql->cmd; SSqlRes* pRes = &pSql->res; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); if (pParentSql->res.code != TSDB_CODE_SUCCESS) { tscError("0x%"PRIx64" abort query due to other subquery failure. code:%d, global code:%d", pSql->self, numOfRows, pParentSql->res.code); @@ -1423,7 +1405,6 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR } tscAsyncResultOnError(pParentSql); - return; } @@ -1524,7 +1505,7 @@ void tscFetchDatablockForSubquery(SSqlObj* pSql) { SSqlRes *pRes = &pSub->res; - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSub->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSub->cmd); if (!tscHasReachLimitation(pQueryInfo, pRes)) { if (pRes->row >= pRes->numOfRows) { // no data left in current result buffer @@ -1576,7 +1557,7 @@ void tscFetchDatablockForSubquery(SSqlObj* pSql) { continue; } - SQueryInfo* p = tscGetQueryInfo(&pSub->cmd, 0); + SQueryInfo* p = tscGetQueryInfo(&pSub->cmd); orderedPrjQuery = tscNonOrderedProjectionQueryOnSTable(p, 0); if (orderedPrjQuery) { break; @@ -1600,7 +1581,7 @@ void tscFetchDatablockForSubquery(SSqlObj* pSql) { continue; } - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSub->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSub->cmd); if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && pSub->res.row >= pSub->res.numOfRows && pSub->res.completed) { @@ -1657,7 +1638,6 @@ void tscFetchDatablockForSubquery(SSqlObj* pSql) { } SSqlRes* pRes1 = &pSql1->res; - if (pRes1->row >= pRes1->numOfRows) { subquerySetState(pSql1, &pSql->subState, i, 0); } @@ -1675,7 +1655,7 @@ void tscFetchDatablockForSubquery(SSqlObj* pSql) { pSupporter = (SJoinSupporter*)pSql1->param; // wait for all subqueries completed - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd1, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd1); assert(pRes1->numOfRows >= 0 && pQueryInfo->numOfTables == 1); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -1706,9 +1686,9 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { return; } - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - int32_t numOfExprs = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); + int32_t numOfExprs = (int32_t)tscNumOfExprs(pQueryInfo); pRes->pColumnIndex = calloc(1, sizeof(SColumnIndex) * numOfExprs); if (pRes->pColumnIndex == NULL) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -1716,7 +1696,7 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { } for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); int32_t tableIndexOfSub = -1; for (int32_t j = 0; j < pQueryInfo->numOfTables; ++j) { @@ -1730,11 +1710,11 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { assert(tableIndexOfSub >= 0 && tableIndexOfSub < pQueryInfo->numOfTables); SSqlCmd* pSubCmd = &pSql->pSubs[tableIndexOfSub]->cmd; - SQueryInfo* pSubQueryInfo = tscGetQueryInfo(pSubCmd, 0); + SQueryInfo* pSubQueryInfo = tscGetQueryInfo(pSubCmd); size_t numOfSubExpr = taosArrayGetSize(pSubQueryInfo->exprList); for (int32_t k = 0; k < numOfSubExpr; ++k) { - SExprInfo* pSubExpr = tscSqlExprGet(pSubQueryInfo, k); + SExprInfo* pSubExpr = tscExprGet(pSubQueryInfo, k); if (pExpr->base.functionId == pSubExpr->base.functionId && pExpr->base.colInfo.colId == pSubExpr->base.colInfo.colId) { pRes->pColumnIndex[i] = (SColumnIndex){.tableIndex = tableIndexOfSub, .columnIndex = k}; break; @@ -1743,8 +1723,8 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { } // restore the offset value for super table query in case of final result. - tscRestoreFuncForSTableQuery(pQueryInfo); - tscFieldInfoUpdateOffset(pQueryInfo); +// tscRestoreFuncForSTableQuery(pQueryInfo); +// tscFieldInfoUpdateOffset(pQueryInfo); } void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { @@ -1754,10 +1734,10 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { SSqlObj* pParentSql = pSupporter->pObj; // There is only one subquery and table for each subquery. - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - assert(pQueryInfo->numOfTables == 1 && pSql->cmd.numOfClause == 1); + assert(pQueryInfo->numOfTables == 1); // retrieve actual query results from vnode during the second stage join subquery if (pParentSql->res.code != TSDB_CODE_SUCCESS) { @@ -1838,7 +1818,7 @@ static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsuppo int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter *pSupporter) { SSqlCmd * pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); pSql->res.qId = 0x1; assert(pSql->res.numOfRows == 0); @@ -1861,15 +1841,9 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter addGroupInfoForSubquery(pSql, pNew, 0, tableIndex); // refactor as one method - SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd, 0); + SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd); assert(pNewQueryInfo != NULL); - - // update the table index -// size_t num = taosArrayGetSize(pNewQueryInfo->colList); -// for (int32_t i = 0; i < num; ++i) { -// SColumn* pCol = taosArrayGetP(pNewQueryInfo->colList, i); -// } - + pSupporter->colList = pNewQueryInfo->colList; pNewQueryInfo->colList = NULL; @@ -1885,7 +1859,7 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter } pSupporter->groupInfo = pNewQueryInfo->groupbyExpr; - memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SSqlGroupbyExpr)); + memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SGroupbyExpr)); pNew->cmd.numOfCols = 0; pNewQueryInfo->interval.interval = 0; @@ -1898,9 +1872,12 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter pNewQueryInfo->order.orderColId = INT32_MIN; // backup the data and clear it in the sqlcmd object - memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SSqlGroupbyExpr)); - + memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SGroupbyExpr)); + + STimeWindow range = pNewQueryInfo->window; tscInitQueryInfo(pNewQueryInfo); + + pNewQueryInfo->window = range; STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0); if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // return the tableId & tag @@ -1926,21 +1903,21 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter // set get tags query type TSDB_QUERY_SET_TYPE(pNewQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY); - tscAddFuncInSelectClause(pNewQueryInfo, 0, TSDB_FUNC_TID_TAG, &colIndex, &s1, TSDB_COL_TAG); + tscAddFuncInSelectClause(pNewQueryInfo, 0, TSDB_FUNC_TID_TAG, &colIndex, &s1, TSDB_COL_TAG, getNewResColId(pCmd)); size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); tscDebug( "%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, transfer to tid_tag query to retrieve (tableId, tags), " "exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, tagIndex:%d, name:%s", - pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo), + pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscNumOfExprs(pNewQueryInfo), numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, colIndex.columnIndex, tNameGetTableName(&pNewQueryInfo->pTableMetaInfo[0]->name)); } else { SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = 1}; SColumnIndex colIndex = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscAddFuncInSelectClause(pNewQueryInfo, 0, TSDB_FUNC_TS_COMP, &colIndex, &colSchema, TSDB_COL_NORMAL); + tscAddFuncInSelectClause(pNewQueryInfo, 0, TSDB_FUNC_TS_COMP, &colIndex, &colSchema, TSDB_COL_NORMAL, getNewResColId(pCmd)); // set the tags value for ts_comp function - SExprInfo *pExpr = tscSqlExprGet(pNewQueryInfo, 0); + SExprInfo *pExpr = tscExprGet(pNewQueryInfo, 0); if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { int16_t tagColId = tscGetJoinTagColIdByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->id.uid); @@ -1967,12 +1944,12 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter tscDebug( "%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%u, transfer to ts_comp query to retrieve timestamps, " "exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, name:%s", - pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo), + pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscNumOfExprs(pNewQueryInfo), numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pNewQueryInfo->pTableMetaInfo[0]->name)); } } else { assert(0); - SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd, 0); + SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd); pNewQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY; } @@ -1983,7 +1960,7 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; SSqlRes* pRes = &pSql->res; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); assert((pQueryInfo->type & TSDB_QUERY_TYPE_SUBQUERY) == 0); int32_t code = TSDB_CODE_SUCCESS; @@ -2018,7 +1995,7 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) { } SSqlObj* pSub = pSql->pSubs[i]; - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSub->cmd, 0, 0); + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSub->cmd, 0); if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) && (pTableMetaInfo->vgroupList->numOfVgroups == 0)) { pSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; break; @@ -2106,7 +2083,7 @@ typedef struct SFirstRoundQuerySup { void doAppendData(SInterResult* pInterResult, TAOS_ROW row, int32_t numOfCols, SQueryInfo* pQueryInfo) { TSKEY key = INT64_MIN; for(int32_t i = 0; i < numOfCols; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (TSDB_COL_IS_TAG(pExpr->base.colInfo.flag) || pExpr->base.functionId == TSDB_FUNC_PRJ) { continue; } @@ -2167,7 +2144,7 @@ void tscFirstRoundRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { SFirstRoundQuerySup* pSup = param; SSqlObj* pParent = pSup->pParent; - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); int32_t code = taos_errno(pSql); if (code != TSDB_CODE_SUCCESS) { @@ -2199,7 +2176,7 @@ void tscFirstRoundRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { int32_t offset = 0; for (int32_t i = 0; i < numOfCols && offset < pSup->tagLen; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); // tag or group by column if (TSDB_COL_IS_TAG(pExpr->base.colInfo.flag) || pExpr->base.functionId == TSDB_FUNC_PRJ) { @@ -2250,7 +2227,7 @@ void tscFirstRoundRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { // set the parameters for the second round query process SSqlCmd *pPCmd = &pParent->cmd; - SQueryInfo *pQueryInfo1 = tscGetQueryInfo(pPCmd, 0); + SQueryInfo *pQueryInfo1 = tscGetQueryInfo(pPCmd); int32_t resRows = pSup->numOfRows; if (pSup->numOfRows > 0) { @@ -2300,8 +2277,8 @@ void tscFirstRoundCallback(void* param, TAOS_RES* tres, int code) { } int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); - STableMetaInfo* pTableMetaInfo1 = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); + STableMetaInfo* pTableMetaInfo1 = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); SFirstRoundQuerySup *pSup = calloc(1, sizeof(SFirstRoundQuerySup)); @@ -2313,15 +2290,20 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { SSqlObj *pNew = createSubqueryObj(pSql, 0, tscFirstRoundCallback, pSup, TSDB_SQL_SELECT, NULL); SSqlCmd *pCmd = &pNew->cmd; + SQueryInfo* pNewQueryInfo = tscGetQueryInfo(pCmd); + assert(pQueryInfo->numOfTables == 1); + + SArray* pColList = pNewQueryInfo->colList; + pNewQueryInfo->colList = NULL; + tscClearSubqueryInfo(pCmd); tscFreeSqlResult(pSql); - SQueryInfo* pNewQueryInfo = tscGetQueryInfo(pCmd, 0); - assert(pQueryInfo->numOfTables == 1); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0); tscInitQueryInfo(pNewQueryInfo); + + // add the group cond pNewQueryInfo->groupbyExpr = pQueryInfo->groupbyExpr; if (pQueryInfo->groupbyExpr.columnInfo != NULL) { pNewQueryInfo->groupbyExpr.columnInfo = taosArrayDup(pQueryInfo->groupbyExpr.columnInfo); @@ -2331,30 +2313,33 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { } } + // add the tag filter cond if (tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond) != 0) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; goto _error; } + pNewQueryInfo->window = pQueryInfo->window; pNewQueryInfo->interval = pQueryInfo->interval; + pNewQueryInfo->sessionWindow = pQueryInfo->sessionWindow; pNewQueryInfo->globalMerge = pQueryInfo->globalMerge; assert(pNewQueryInfo->globalMerge); pCmd->command = TSDB_SQL_SELECT; pNew->fp = tscFirstRoundCallback; - int32_t numOfExprs = (int32_t) tscSqlExprNumOfExprs(pQueryInfo); + int32_t numOfExprs = (int32_t) tscNumOfExprs(pQueryInfo); int32_t index = 0; for(int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (pExpr->base.functionId == TSDB_FUNC_TS && pQueryInfo->interval.interval > 0) { taosArrayPush(pSup->pColsInfo, &pExpr->base.resColId); SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; SSchema* schema = tscGetColumnSchemaById(pTableMetaInfo1->pTableMeta, pExpr->base.colInfo.colId); - SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_TS, &colIndex, schema, TSDB_COL_NORMAL); + SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_TS, &colIndex, schema, TSDB_COL_NORMAL, getNewResColId(pCmd)); p->base.resColId = pExpr->base.resColId; // update the result column id } else if (pExpr->base.functionId == TSDB_FUNC_STDDEV_DST) { taosArrayPush(pSup->pColsInfo, &pExpr->base.resColId); @@ -2363,7 +2348,7 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { SSchema schema = {.type = TSDB_DATA_TYPE_DOUBLE, .bytes = sizeof(double)}; tstrncpy(schema.name, pExpr->base.aliasName, tListLen(schema.name)); - SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_AVG, &colIndex, &schema, TSDB_COL_NORMAL); + SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_AVG, &colIndex, &schema, TSDB_COL_NORMAL, getNewResColId(pCmd)); p->base.resColId = pExpr->base.resColId; // update the result column id } else if (pExpr->base.functionId == TSDB_FUNC_TAG) { pSup->tagLen += pExpr->base.resBytes; @@ -2376,7 +2361,7 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { schema = tGetTbnameColumnSchema(); } - SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_TAG, &colIndex, schema, TSDB_COL_TAG); + SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_TAG, &colIndex, schema, TSDB_COL_TAG, getNewResColId(pCmd)); p->base.resColId = pExpr->base.resColId; } else if (pExpr->base.functionId == TSDB_FUNC_PRJ) { int32_t num = (int32_t) taosArrayGetSize(pNewQueryInfo->groupbyExpr.columnInfo); @@ -2390,7 +2375,7 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { SSchema* schema = tscGetColumnSchemaById(pTableMetaInfo1->pTableMeta, pExpr->base.colInfo.colId); //doLimitOutputNormalColOfGroupby - SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_PRJ, &colIndex, schema, TSDB_COL_NORMAL); + SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_PRJ, &colIndex, schema, TSDB_COL_NORMAL, getNewResColId(pCmd)); p->base.numOfParams = 1; p->base.param[0].i64 = 1; p->base.param[0].nType = TSDB_DATA_TYPE_INT; @@ -2400,6 +2385,21 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { } } + // add the normal column filter cond + if (pColList != NULL) { + size_t s = taosArrayGetSize(pColList); + for (int32_t i = 0; i < s; ++i) { + SColumn *pCol = taosArrayGetP(pColList, i); + + if (pCol->info.flist.numOfFilters > 0) { // copy to the pNew->cmd.colList if it is filtered. + SColumn *p = tscColumnClone(pCol); + taosArrayPush(pNewQueryInfo->colList, &p); + } + } + + tscColumnListDestroy(pColList); + } + tscInsertPrimaryTsSourceColumn(pNewQueryInfo, pTableMetaInfo->pTableMeta->id.uid); tscTansformFuncForSTableQuery(pNewQueryInfo); @@ -2407,7 +2407,7 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { "0x%"PRIx64" first round subquery:0x%"PRIx64" tableIndex:%d, vgroupIndex:%d, numOfVgroups:%d, type:%d, query to retrieve timestamps, " "numOfExpr:%" PRIzu ", colList:%d, numOfOutputFields:%d, name:%s", pSql->self, pNew->self, 0, pTableMetaInfo->vgroupIndex, pTableMetaInfo->vgroupList->numOfVgroups, pNewQueryInfo->type, - tscSqlExprNumOfExprs(pNewQueryInfo), index+1, pNewQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pTableMetaInfo->name)); + tscNumOfExprs(pNewQueryInfo), index+1, pNewQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pTableMetaInfo->name)); tscHandleMasterSTableQuery(pNew); return TSDB_CODE_SUCCESS; @@ -2432,14 +2432,12 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { tExtMemBuffer **pMemoryBuf = NULL; tOrderDescriptor *pDesc = NULL; - SColumnModel *pModel = NULL; - SColumnModel *pFinalModel = NULL; pRes->qId = 0x1; // hack the qhandle check const uint32_t nBufferSize = (1u << 16u); // 64KB - SQueryInfo *pQueryInfo = tscGetActiveQueryInfo(pCmd); + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); SSubqueryState *pState = &pSql->subState; @@ -2452,9 +2450,9 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { assert(pState->numOfSub > 0); - int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, &pFinalModel, nBufferSize); + int32_t ret = tscLocalReducerEnvCreate(pQueryInfo, &pMemoryBuf, pSql->subState.numOfSub, &pDesc, nBufferSize, pSql->self); if (ret != 0) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + pRes->code = ret; tscAsyncResultOnError(pSql); tfree(pMemoryBuf); return ret; @@ -2465,7 +2463,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { if (pSql->pSubs == NULL) { tfree(pSql->pSubs); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pFinalModel,pState->numOfSub); + tscLocalReducerEnvDestroy(pMemoryBuf, pDesc,pState->numOfSub); tscAsyncResultOnError(pSql); return ret; @@ -2508,8 +2506,6 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { trs->subqueryIndex = i; trs->pParentSql = pSql; - trs->pFinalColModel = pModel; - trs->pFFColModel = pFinalModel; SSqlObj *pNew = tscCreateSTableSubquery(pSql, trs, NULL); if (pNew == NULL) { @@ -2521,7 +2517,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { // todo handle multi-vnode situation if (pQueryInfo->tsBuf) { - SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd, 0); + SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd); pNewQueryInfo->tsBuf = tsBufClone(pQueryInfo->tsBuf); assert(pNewQueryInfo->tsBuf != NULL); } @@ -2534,13 +2530,13 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { tscError("0x%"PRIx64" failed to prepare subquery structure and launch subqueries", pSql->self); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pFinalModel, pState->numOfSub); + tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pState->numOfSub); doCleanupSubqueries(pSql, i); return pRes->code; // free all allocated resource } if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) { - tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pFinalModel, pState->numOfSub); + tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pState->numOfSub); doCleanupSubqueries(pSql, i); return pRes->code; } @@ -2608,7 +2604,7 @@ static int32_t tscReissueSubquery(SRetrieveSupport *oriTrs, SSqlObj *pSql, int32 SSqlObj *pParentSql = trsupport->pParentSql; int32_t subqueryIndex = trsupport->subqueryIndex; - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); SVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; tExtMemBufferClear(trsupport->pExtMemBuffer[subqueryIndex]); @@ -2705,13 +2701,13 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO tstrerror(pParentSql->res.code)); // release allocated resource - tscLocalReducerEnvDestroy(trsupport->pExtMemBuffer, trsupport->pOrderDescriptor, trsupport->pFinalColModel, trsupport->pFFColModel, + tscLocalReducerEnvDestroy(trsupport->pExtMemBuffer, trsupport->pOrderDescriptor, pState->numOfSub); tscFreeRetrieveSup(pSql); // in case of second stage join subquery, invoke its callback function instead of regular QueueAsyncRes - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pParentSql->cmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfo(&pParentSql->cmd); if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)) { (*pParentSql->fp)(pParentSql->param, pParentSql, pParentSql->res.code); @@ -2728,7 +2724,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p tOrderDescriptor *pDesc = trsupport->pOrderDescriptor; SSubqueryState* pState = &pParentSql->subState; - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; @@ -2777,30 +2773,40 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p tscDebug("0x%"PRIx64" retrieve from %d vnodes completed.final NumOfRows:%" PRId64 ",start to build loser tree", pParentSql->self, pState->numOfSub, pState->numOfRetrievedRows); - SQueryInfo *pPQueryInfo = tscGetQueryInfo(&pParentSql->cmd, 0); + SQueryInfo *pPQueryInfo = tscGetQueryInfo(&pParentSql->cmd); tscClearInterpInfo(pPQueryInfo); - tscCreateLocalMerger(trsupport->pExtMemBuffer, pState->numOfSub, pDesc, trsupport->pFinalColModel, trsupport->pFFColModel, pParentSql); + code = tscCreateLocalMerger(trsupport->pExtMemBuffer, pState->numOfSub, pDesc, pPQueryInfo, &pParentSql->res.pLocalMerger, pParentSql->self); + pParentSql->res.code = code; + + if (code == TSDB_CODE_SUCCESS && trsupport->pExtMemBuffer == NULL) { + pParentSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; // no result, set the result empty + } else { + pParentSql->cmd.command = TSDB_SQL_RETRIEVE_LOCALMERGE; + } + + tscCreateResPointerInfo(&pParentSql->res, pPQueryInfo); + tscDebug("0x%"PRIx64" build loser tree completed", pParentSql->self); pParentSql->res.precision = pSql->res.precision; pParentSql->res.numOfRows = 0; pParentSql->res.row = 0; - + pParentSql->res.numOfGroups = 0; + tscFreeRetrieveSup(pSql); // set the command flag must be after the semaphore been correctly set. if (pParentSql->cmd.command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) { pParentSql->cmd.command = TSDB_SQL_RETRIEVE_LOCALMERGE; - - SQueryInfo *pQueryInfo2 = tscGetQueryInfo(&pParentSql->cmd, pParentSql->cmd.clauseIndex); - size_t size = tscSqlExprNumOfExprs(pQueryInfo); + SQueryInfo *pQueryInfo2 = tscGetQueryInfo(&pParentSql->cmd); + + size_t size = tscNumOfExprs(pQueryInfo); for (int32_t j = 0; j < size; ++j) { - SQLFunctionCtx *pCtx = &pParentSql->res.pLocalMerger->pCtx[j]; - - int32_t functionId = pCtx->functionId; + SExprInfo* pExprInfo = tscExprGet(pQueryInfo2, j); + int32_t functionId = pExprInfo->base.functionId; if (functionId < 0) { SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo2->pUdfInfo, -1 * functionId - 1); code = initUdfInfo(pUdfInfo); @@ -2836,7 +2842,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR SSubqueryState* pState = &pParentSql->subState; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); SVgroupInfo *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; if (pParentSql->res.code != TSDB_CODE_SUCCESS) { @@ -2874,7 +2880,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR } SSqlRes * pRes = &pSql->res; - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); if (numOfRows > 0) { assert(pRes->numOfRows == numOfRows); @@ -2883,7 +2889,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR tscDebug("0x%"PRIx64" sub:%p retrieve numOfRows:%d totalNumOfRows:%" PRIu64 " from ep:%s, orderOfSub:%d", pParentSql->self, pSql, pRes->numOfRows, pState->numOfRetrievedRows, pSql->epSet.fqdn[pSql->epSet.inUse], idx); - if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0) && !(tscGetQueryInfo(&pParentSql->cmd, 0)->distinctTag)) { + if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0) && !(tscGetQueryInfo(&pParentSql->cmd)->distinctTag)) { tscError("0x%"PRIx64" sub:0x%"PRIx64" num of OrderedRes is too many, max allowed:%" PRId32 " , current:%" PRId64, pParentSql->self, pSql->self, tsMaxNumOfOrderedResults, num); tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_SORTED_RES_TOO_MANY); @@ -2926,7 +2932,7 @@ static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsuppo SSqlObj *pNew = createSubqueryObj(pSql, table_index, tscRetrieveDataRes, trsupport, TSDB_SQL_SELECT, prevSqlObj); if (pNew != NULL) { // the sub query of two-stage super table query - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pNew->cmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfo(&pNew->cmd); pNew->cmd.active = pQueryInfo; pQueryInfo->type |= TSDB_QUERY_TYPE_STABLE_SUBQUERY; @@ -2935,7 +2941,7 @@ static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsuppo pQueryInfo->limit.limit = -1; pQueryInfo->limit.offset = 0; - assert(pQueryInfo->numOfTables == 1 && pNew->cmd.numOfClause == 1 && trsupport->subqueryIndex < pSql->subState.numOfSub); + assert(trsupport->subqueryIndex < pSql->subState.numOfSub); // launch subquery for each vnode, so the subquery index equals to the vgroupIndex. STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, table_index); @@ -2961,10 +2967,10 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { SSqlObj* pParentSql = trsupport->pParentSql; SSqlObj* pSql = (SSqlObj *) tres; - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, 0); - assert(pSql->cmd.numOfClause == 1 && pQueryInfo->numOfTables == 1); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); + assert(pQueryInfo->numOfTables == 1); - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); SVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[trsupport->subqueryIndex]; // stable query killed or other subquery failed, all query stopped @@ -2991,7 +2997,6 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { tscError("0x%"PRIx64" sub:0x%"PRIx64" failed code:%s, retry:%d", pParentSql->self, pSql->self, tstrerror(code), trsupport->numOfRetry); int32_t sent = 0; - tscReissueSubquery(trsupport, pSql, code, &sent); if (sent) { return; @@ -3063,8 +3068,8 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) pParentObj->res.code = pSql->res.code; // set the flag in the parent sqlObj - if (pSql->cmd.submitSchema) { - pParentObj->cmd.submitSchema = 1; + if (pSql->cmd.insertParam.schemaAttached) { + pParentObj->cmd.insertParam.schemaAttached = 1; } } @@ -3099,8 +3104,8 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) // clean up tableMeta in cache tscFreeQueryInfo(&pSql->cmd, false); - SQueryInfo* pQueryInfo = tscGetQueryInfoS(&pSql->cmd, 0); - STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pParentObj->cmd, pSql->cmd.clauseIndex, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfoS(&pSql->cmd); + STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pParentObj->cmd, 0); tscAddTableMetaInfo(pQueryInfo, &pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL); subquerySetState(pSql, &pParentObj->subState, i, 0); @@ -3112,15 +3117,15 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) tscError("0x%"PRIx64" Async insertion completed, total inserted:%d rows, numOfFailed:%d, numOfTotal:%d", pParentObj->self, pParentObj->res.numOfRows, numOfFailed, numOfSub); - tscDebug("0x%"PRIx64" cleanup %d tableMeta in hashTable before reparse sql", pParentObj->self, pParentObj->cmd.numOfTables); - for(int32_t i = 0; i < pParentObj->cmd.numOfTables; ++i) { + tscDebug("0x%"PRIx64" cleanup %d tableMeta in hashTable before reparse sql", pParentObj->self, pParentObj->cmd.insertParam.numOfTables); + for(int32_t i = 0; i < pParentObj->cmd.insertParam.numOfTables; ++i) { char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(pParentObj->cmd.pTableNameList[i], name); + tNameExtractFullName(pParentObj->cmd.insertParam.pTableNameList[i], name); taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); } pParentObj->res.code = TSDB_CODE_SUCCESS; - pParentObj->cmd.parseFinished = false; +// pParentObj->cmd.parseFinished = false; tscResetSqlCmd(&pParentObj->cmd, false); @@ -3155,7 +3160,7 @@ int32_t tscHandleInsertRetry(SSqlObj* pParent, SSqlObj* pSql) { SInsertSupporter* pSupporter = (SInsertSupporter*) pSql->param; assert(pSupporter->index < pSupporter->pSql->subState.numOfSub); - STableDataBlocks* pTableDataBlock = taosArrayGetP(pParent->cmd.pDataBlocks, pSupporter->index); + STableDataBlocks* pTableDataBlock = taosArrayGetP(pParent->cmd.insertParam.pDataBlocks, pSupporter->index); int32_t code = tscCopyDataBlockToPayload(pSql, pTableDataBlock); if ((pRes->code = code)!= TSDB_CODE_SUCCESS) { @@ -3188,7 +3193,7 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { return TSDB_CODE_SUCCESS; } - pSql->subState.numOfSub = (uint16_t)taosArrayGetSize(pCmd->pDataBlocks); + pSql->subState.numOfSub = (uint16_t)taosArrayGetSize(pCmd->insertParam.pDataBlocks); assert(pSql->subState.numOfSub > 0); pRes->code = TSDB_CODE_SUCCESS; @@ -3238,7 +3243,7 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { pNew->fetchFp = pNew->fp; pSql->pSubs[numOfSub] = pNew; - STableDataBlocks* pTableDataBlock = taosArrayGetP(pCmd->pDataBlocks, numOfSub); + STableDataBlocks* pTableDataBlock = taosArrayGetP(pCmd->insertParam.pDataBlocks, numOfSub); pRes->code = tscCopyDataBlockToPayload(pNew, pTableDataBlock); if (pRes->code == TSDB_CODE_SUCCESS) { tscDebug("0x%"PRIx64" sub:%p create subObj success. orderOfSub:%d", pSql->self, pNew, numOfSub); @@ -3256,7 +3261,7 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { goto _error; } - pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); + pCmd->insertParam.pDataBlocks = tscDestroyBlockArrayList(pCmd->insertParam.pDataBlocks); // use the local variable for (int32_t j = 0; j < numOfSub; ++j) { @@ -3272,7 +3277,7 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { } static char* getResultBlockPosition(SSqlCmd* pCmd, SSqlRes* pRes, int32_t columnIndex, int16_t* bytes) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); SInternalField* pInfo = (SInternalField*) TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, columnIndex); assert(pInfo->pExpr->pExpr == NULL); @@ -3288,7 +3293,7 @@ static char* getResultBlockPosition(SSqlCmd* pCmd, SSqlRes* pRes, int32_t column static void doBuildResFromSubqueries(SSqlObj* pSql) { SSqlRes* pRes = &pSql->res; - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd, pSql->cmd.clauseIndex); + SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); int32_t numOfRes = INT32_MAX; for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { @@ -3306,7 +3311,7 @@ static void doBuildResFromSubqueries(SSqlObj* pSql) { return; } - tscRestoreFuncForSTableQuery(pQueryInfo); +// tscRestoreFuncForSTableQuery(pQueryInfo); int32_t rowSize = tscGetResRowLength(pQueryInfo->exprList); assert(numOfRes * rowSize > 0); @@ -3334,12 +3339,12 @@ static void doBuildResFromSubqueries(SSqlObj* pSql) { continue; } - SQueryInfo* pSubQueryInfo = pSub->cmd.pQueryInfo[0]; + SQueryInfo* pSubQueryInfo = pSub->cmd.pQueryInfo; tscRestoreFuncForSTableQuery(pSubQueryInfo); tscFieldInfoUpdateOffset(pSubQueryInfo); } - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + size_t numOfExprs = tscNumOfExprs(pQueryInfo); for(int32_t i = 0; i < numOfExprs; ++i) { SColumnIndex* pIndex = &pRes->pColumnIndex[i]; SSqlRes* pRes1 = &pSql->pSubs[pIndex->tableIndex]->res; @@ -3385,8 +3390,8 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { } if (pRes->tsrow == NULL) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd, pSql->cmd.clauseIndex); - pRes->numOfCols = (int16_t) tscSqlExprNumOfExprs(pQueryInfo); + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); + pRes->numOfCols = (int16_t) tscNumOfExprs(pQueryInfo); pRes->tsrow = calloc(pRes->numOfCols, POINTER_BYTES); pRes->urow = calloc(pRes->numOfCols, POINTER_BYTES); @@ -3398,8 +3403,6 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { tscAsyncResultOnError(pSql); return; } - - tscRestoreFuncForSTableQuery(pQueryInfo); } assert (pRes->row >= pRes->numOfRows); @@ -3439,7 +3442,7 @@ TAOS_ROW doSetResultRowData(SSqlObj *pSql) { return pRes->tsrow; } - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); size_t size = tscNumOfFields(pQueryInfo); @@ -3468,65 +3471,64 @@ TAOS_ROW doSetResultRowData(SSqlObj *pSql) { return pRes->tsrow; } -//static UNUSED_FUNC bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql) { -// bool hasData = true; -// SSqlCmd *pCmd = &pSql->cmd; -// -// SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); -// if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { -// bool allSubqueryExhausted = true; -// -// for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { -// if (pSql->pSubs[i] == NULL) { -// continue; -// } -// -// SSqlRes *pRes1 = &pSql->pSubs[i]->res; -// SSqlCmd *pCmd1 = &pSql->pSubs[i]->cmd; -// -// SQueryInfo *pQueryInfo1 = tscGetQueryInfo(pCmd1, pCmd1->clauseIndex); -// assert(pQueryInfo1->numOfTables == 1); -// -// STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo1, 0); -// -// /* -// * if the global limitation is not reached, and current result has not exhausted, or next more vnodes are -// * available, goes on -// */ -// if (pTableMetaInfo->vgroupIndex < pTableMetaInfo->vgroupList->numOfVgroups && pRes1->row < pRes1->numOfRows && -// (!tscHasReachLimitation(pQueryInfo1, pRes1))) { -// allSubqueryExhausted = false; -// break; -// } -// } -// -// hasData = !allSubqueryExhausted; -// } else { // otherwise, in case inner join, if any subquery exhausted, query completed. -// for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { -// if (pSql->pSubs[i] == 0) { -// continue; -// } -// -// SSqlRes * pRes1 = &pSql->pSubs[i]->res; -// SQueryInfo *pQueryInfo1 = tscGetQueryInfo(&pSql->pSubs[i]->cmd, 0); -// -// if ((pRes1->row >= pRes1->numOfRows && tscHasReachLimitation(pQueryInfo1, pRes1) && -// tscIsProjectionQuery(pQueryInfo1)) || -// (pRes1->numOfRows == 0)) { -// hasData = false; -// break; -// } -// } -// } -// -// return hasData; -//} +static UNUSED_FUNC bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql) { + bool hasData = true; + SSqlCmd *pCmd = &pSql->cmd; + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { + bool allSubqueryExhausted = true; -void* createQueryInfoFromQueryNode(SQueryInfo* pQueryInfo, SExprInfo* pExprs, STableGroupInfo* pTableGroupInfo, - SOperatorInfo* pSourceOperator, char* sql, void* merger, int32_t stage) { + for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { + if (pSql->pSubs[i] == NULL) { + continue; + } + + SSqlRes *pRes1 = &pSql->pSubs[i]->res; + SSqlCmd *pCmd1 = &pSql->pSubs[i]->cmd; + + SQueryInfo *pQueryInfo1 = tscGetQueryInfo(pCmd1); + assert(pQueryInfo1->numOfTables == 1); + + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo1, 0); + + /* + * if the global limitation is not reached, and current result has not exhausted, or next more vnodes are + * available, goes on + */ + if (pTableMetaInfo->vgroupIndex < pTableMetaInfo->vgroupList->numOfVgroups && pRes1->row < pRes1->numOfRows && + (!tscHasReachLimitation(pQueryInfo1, pRes1))) { + allSubqueryExhausted = false; + break; + } + } + + hasData = !allSubqueryExhausted; + } else { // otherwise, in case inner join, if any subquery exhausted, query completed. + for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { + if (pSql->pSubs[i] == 0) { + continue; + } + + SSqlRes * pRes1 = &pSql->pSubs[i]->res; + SQueryInfo *pQueryInfo1 = tscGetQueryInfo(&pSql->pSubs[i]->cmd); + + if ((pRes1->row >= pRes1->numOfRows && tscHasReachLimitation(pQueryInfo1, pRes1) && + tscIsProjectionQuery(pQueryInfo1)) || + (pRes1->numOfRows == 0)) { + hasData = false; + break; + } + } + } + + return hasData; +} + +// todo remove pExprs +void* createQInfoFromQueryNode(SQueryInfo* pQueryInfo, SExprInfo* pExprs, STableGroupInfo* pTableGroupInfo, + SOperatorInfo* pSourceOperator, char* sql, void* merger, int32_t stage) { assert(pQueryInfo != NULL); - int16_t numOfOutput = pQueryInfo->fieldsInfo.numOfOutput; - SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo)); if (pQInfo == NULL) { goto _cleanup; @@ -3544,13 +3546,25 @@ void* createQueryInfoFromQueryNode(SQueryInfo* pQueryInfo, SExprInfo* pExprs, ST pQueryAttr->tableGroupInfo = *pTableGroupInfo; // calculate the result row size - for (int16_t col = 0; col < numOfOutput; ++col) { - assert(pExprs[col].base.resBytes > 0); - pQueryAttr->resultRowSize += pExprs[col].base.resBytes; + SExprInfo* pEx = NULL; + int32_t num = 0; + if (pQueryAttr->pExpr3 != NULL) { + pEx = pQueryAttr->pExpr3; + num = pQueryAttr->numOfExpr3; + } else if (pQueryAttr->pExpr2 != NULL) { + pEx = pQueryAttr->pExpr2; + num = pQueryAttr->numOfExpr2; + } else { + pEx = pQueryAttr->pExpr1; + num = pQueryAttr->numOfOutput; + } + + for (int16_t col = 0; col < num; ++col) { + pQueryAttr->resultRowSize += pEx[col].base.resBytes; // keep the tag length - if (TSDB_COL_IS_TAG(pExprs[col].base.colInfo.flag)) { - pQueryAttr->tagLen += pExprs[col].base.resBytes; + if (TSDB_COL_IS_TAG(pEx[col].base.colInfo.flag)) { + pQueryAttr->tagLen += pEx[col].base.resBytes; } } @@ -3608,15 +3622,9 @@ void* createQueryInfoFromQueryNode(SQueryInfo* pQueryInfo, SExprInfo* pExprs, ST } } - for (int32_t i = 0; i < numOfOutput; ++i) { - SExprInfo* pExprInfo = &pExprs[i]; - if (pExprInfo->pExpr != NULL) { - tExprTreeDestroy(pExprInfo->pExpr, NULL); - pExprInfo->pExpr = NULL; - } - } - - tfree(pExprs); + // todo refactor: filter should not be applied here. + createFilterInfo(pQueryAttr, 0); + pQueryAttr->numOfFilterCols = 0; SArray* pa = NULL; if (stage == MASTER_SCAN) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index a2f4ddb747..3c518e01b5 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -14,10 +14,11 @@ */ #include "tscUtil.h" +#include "tsched.h" #include "hash.h" #include "os.h" -#include "texpr.h" #include "taosmsg.h" +#include "texpr.h" #include "tkey.h" #include "tmd5.h" #include "tscLocalMerge.h" @@ -30,7 +31,67 @@ #include "ttokendef.h" static void freeQueryInfoImpl(SQueryInfo* pQueryInfo); -static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, bool removeMeta); + +int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) { + int32_t n = 0; + + switch (type) { + case TSDB_DATA_TYPE_NULL: + n = sprintf(str, "null"); + break; + + case TSDB_DATA_TYPE_BOOL: + n = sprintf(str, (*(int8_t*)buf) ? "true" : "false"); + break; + + case TSDB_DATA_TYPE_TINYINT: + n = sprintf(str, "%d", *(int8_t*)buf); + break; + + case TSDB_DATA_TYPE_SMALLINT: + n = sprintf(str, "%d", *(int16_t*)buf); + break; + + case TSDB_DATA_TYPE_INT: + n = sprintf(str, "%d", *(int32_t*)buf); + break; + + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + n = sprintf(str, "%" PRId64, *(int64_t*)buf); + break; + + case TSDB_DATA_TYPE_FLOAT: + n = sprintf(str, "%f", GET_FLOAT_VAL(buf)); + break; + + case TSDB_DATA_TYPE_DOUBLE: + n = sprintf(str, "%f", GET_DOUBLE_VAL(buf)); + break; + + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + if (bufSize < 0) { + tscError("invalid buf size"); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + *str = '"'; + memcpy(str + 1, buf, bufSize); + *(str + bufSize + 1) = '"'; + n = bufSize + 2; + break; + + default: + tscError("unsupported type:%d", type); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + *len = n; + + return TSDB_CODE_SUCCESS; +} + static void tscStrToLower(char *str, int32_t n) { if (str == NULL || n <= 0) { return;} @@ -78,10 +139,10 @@ void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw) { } bool tscQueryTags(SQueryInfo* pQueryInfo) { - int32_t numOfCols = (int32_t) tscSqlExprNumOfExprs(pQueryInfo); + int32_t numOfCols = (int32_t) tscNumOfExprs(pQueryInfo); for (int32_t i = 0; i < numOfCols; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); int32_t functId = pExpr->base.functionId; // "select count(tbname)" query @@ -98,10 +159,10 @@ bool tscQueryTags(SQueryInfo* pQueryInfo) { } bool tscQueryBlockInfo(SQueryInfo* pQueryInfo) { - int32_t numOfCols = (int32_t) tscSqlExprNumOfExprs(pQueryInfo); + int32_t numOfCols = (int32_t) tscNumOfExprs(pQueryInfo); for (int32_t i = 0; i < numOfCols; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); int32_t functId = pExpr->base.functionId; if (functId == TSDB_FUNC_BLKINFO) { @@ -146,14 +207,14 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) { * 1. failed to get tableMeta from server; 2. not a super table; 3. limitation is 0; * 4. show queries, instead of a select query */ - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + size_t numOfExprs = tscNumOfExprs(pQueryInfo); if (pTableMetaInfo == NULL || !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) || pQueryInfo->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || numOfExprs == 0) { return false; } for (int32_t i = 0; i < numOfExprs; ++i) { - int32_t functionId = tscSqlExprGet(pQueryInfo, i)->base.functionId; + int32_t functionId = tscExprGet(pQueryInfo, i)->base.functionId; if (functionId < 0) { SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); @@ -198,10 +259,10 @@ bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableInde } bool tscIsProjectionQuery(SQueryInfo* pQueryInfo) { - size_t size = tscSqlExprNumOfExprs(pQueryInfo); + size_t size = tscNumOfExprs(pQueryInfo); for (int32_t i = 0; i < size; ++i) { - int32_t functionId = tscSqlExprGet(pQueryInfo, i)->base.functionId; + int32_t functionId = tscExprGet(pQueryInfo, i)->base.functionId; if (functionId < 0) { SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); @@ -221,10 +282,27 @@ bool tscIsProjectionQuery(SQueryInfo* pQueryInfo) { return true; } -bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo) { - size_t size = tscSqlExprNumOfExprs(pQueryInfo); +bool tscHasColumnFilter(SQueryInfo* pQueryInfo) { + // filter on primary timestamp column + if (pQueryInfo->window.skey != INT64_MIN || pQueryInfo->window.ekey != INT64_MAX) { + return true; + } + + size_t size = taosArrayGetSize(pQueryInfo->colList); for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SColumn* pCol = taosArrayGet(pQueryInfo->colList, i); + if (pCol->info.flist.numOfFilters > 0) { + return true; + } + } + + return false; +} + +bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo) { + size_t size = tscNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < size; ++i) { + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); assert(pExpr != NULL); int32_t functionId = pExpr->base.functionId; @@ -240,7 +318,11 @@ bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo) { return true; } -bool tscIsSecondStageQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { +bool tsIsArithmeticQueryOnAggResult(SQueryInfo* pQueryInfo) { + if (tscIsProjectionQuery(pQueryInfo)) { + return false; + } + size_t numOfOutput = tscNumOfFields(pQueryInfo); for(int32_t i = 0; i < numOfOutput; ++i) { SExprInfo* pExprInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, i)->pExpr; @@ -260,7 +342,7 @@ bool tscGroupbyColumn(SQueryInfo* pQueryInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - SSqlGroupbyExpr* pGroupbyExpr = &pQueryInfo->groupbyExpr; + SGroupbyExpr* pGroupbyExpr = &pQueryInfo->groupbyExpr; for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) { SColIndex* pIndex = taosArrayGet(pGroupbyExpr->columnInfo, k); if (!TSDB_COL_IS_TAG(pIndex->flag) && pIndex->colIndex < numOfCols) { // group by normal columns @@ -272,10 +354,10 @@ bool tscGroupbyColumn(SQueryInfo* pQueryInfo) { } bool tscIsTopBotQuery(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + size_t numOfExprs = tscNumOfExprs(pQueryInfo); for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (pExpr == NULL) { continue; } @@ -293,8 +375,8 @@ bool tscIsTopBotQuery(SQueryInfo* pQueryInfo) { } bool isTsCompQuery(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); - SExprInfo* pExpr1 = tscSqlExprGet(pQueryInfo, 0); + size_t numOfExprs = tscNumOfExprs(pQueryInfo); + SExprInfo* pExpr1 = tscExprGet(pQueryInfo, 0); if (numOfExprs != 1) { return false; } @@ -303,15 +385,15 @@ bool isTsCompQuery(SQueryInfo* pQueryInfo) { } bool hasTagValOutput(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); - SExprInfo* pExpr1 = tscSqlExprGet(pQueryInfo, 0); + size_t numOfExprs = tscNumOfExprs(pQueryInfo); + SExprInfo* pExpr1 = tscExprGet(pQueryInfo, 0); if (numOfExprs == 1 && pExpr1->base.functionId == TSDB_FUNC_TS_COMP) { return true; } for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (pExpr == NULL) { continue; } @@ -326,9 +408,9 @@ bool hasTagValOutput(SQueryInfo* pQueryInfo) { } bool timeWindowInterpoRequired(SQueryInfo *pQueryInfo) { - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + size_t numOfExprs = tscNumOfExprs(pQueryInfo); for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (pExpr == NULL) { continue; } @@ -343,9 +425,9 @@ bool timeWindowInterpoRequired(SQueryInfo *pQueryInfo) { } bool isStabledev(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + size_t numOfExprs = tscNumOfExprs(pQueryInfo); for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (pExpr == NULL) { continue; } @@ -360,9 +442,9 @@ bool isStabledev(SQueryInfo* pQueryInfo) { } bool tscIsTWAQuery(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + size_t numOfExprs = tscNumOfExprs(pQueryInfo); for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (pExpr == NULL) { continue; } @@ -376,9 +458,9 @@ bool tscIsTWAQuery(SQueryInfo* pQueryInfo) { } bool tscNeedReverseScan(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + size_t numOfExprs = tscNumOfExprs(pQueryInfo); for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (pExpr == NULL) { continue; } @@ -414,9 +496,38 @@ bool isSimpleAggregate(SQueryInfo* pQueryInfo) { return true; } - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + size_t numOfExprs = tscNumOfExprs(pQueryInfo); for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); + if (pExpr == NULL) { + continue; + } + + int32_t functionId = pExpr->base.functionId; + if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY) { + continue; + } + + if (!IS_MULTIOUTPUT(aAggs[functionId].status)) { + return true; + } + } + + return false; +} + +bool isSimpleAggregateRv(SQueryInfo* pQueryInfo) { + if (pQueryInfo->interval.interval > 0 || pQueryInfo->sessionWindow.gap > 0) { + return false; + } + + if (tscGroupbyColumn(pQueryInfo) || isTsCompQuery(pQueryInfo)) { + return false; + } + + size_t numOfExprs = tscNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < numOfExprs; ++i) { + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (pExpr == NULL) { continue; } @@ -430,7 +541,7 @@ bool isSimpleAggregate(SQueryInfo* pQueryInfo) { continue; } - + if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY) { continue; } @@ -441,12 +552,13 @@ bool isSimpleAggregate(SQueryInfo* pQueryInfo) { } return false; + } bool isBlockDistQuery(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, 0); - return (numOfExprs == 1 && pExpr->base.colInfo.colId == TSDB_BLOCK_DIST_COLUMN_INDEX); + size_t numOfExprs = tscNumOfExprs(pQueryInfo); + SExprInfo* pExpr = tscExprGet(pQueryInfo, 0); + return (numOfExprs == 1 && pExpr->base.functionId == TSDB_FUNC_BLKINFO); } void tscClearInterpInfo(SQueryInfo* pQueryInfo) { @@ -459,9 +571,9 @@ void tscClearInterpInfo(SQueryInfo* pQueryInfo) { } int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { - if (pRes->tsrow == NULL) { - pRes->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; + pRes->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; + if (pRes->tsrow == NULL) { pRes->tsrow = calloc(pRes->numOfCols, POINTER_BYTES); pRes->urow = calloc(pRes->numOfCols, POINTER_BYTES); pRes->length = calloc(pRes->numOfCols, sizeof(int32_t)); @@ -614,58 +726,237 @@ void tscSetResRawPtrRv(SSqlRes* pRes, SQueryInfo* pQueryInfo, SSDataBlock* pBloc } } -static SColumnInfo* extractColumnInfoFromResult(STableMeta* pTableMeta, SArray* pTableCols) { +static SColumnInfo* extractColumnInfoFromResult(SArray* pTableCols) { int32_t numOfCols = (int32_t) taosArrayGetSize(pTableCols); SColumnInfo* pColInfo = calloc(numOfCols, sizeof(SColumnInfo)); - - SSchema *pSchema = pTableMeta->schema; for(int32_t i = 0; i < numOfCols; ++i) { SColumn* pCol = taosArrayGetP(pTableCols, i); - int32_t index = pCol->columnIndex; - - pColInfo[i].type = pSchema[index].type; - pColInfo[i].bytes = pSchema[index].bytes; - pColInfo[i].colId = pSchema[index].colId; + pColInfo[i] = pCol->info;//[index].type; } return pColInfo; } typedef struct SDummyInputInfo { - SSDataBlock *block; - SSqlRes *pRes; // refactor: remove it + SSDataBlock *block; + SSqlObj *pSql; // refactor: remove it + int32_t numOfFilterCols; + SSingleColumnFilterInfo *pFilterInfo; } SDummyInputInfo; -SSDataBlock* doGetDataBlock(void* param, bool* newgroup) { - SOperatorInfo *pOperator = (SOperatorInfo*) param; +typedef struct SJoinStatus { + SSDataBlock* pBlock; // point to the upstream block + int32_t index; + bool completed;// current upstream is completed or not +} SJoinStatus; - SDummyInputInfo *pInput = pOperator->info; - char* pData = pInput->pRes->data; +typedef struct SJoinOperatorInfo { + SSDataBlock *pRes; + SJoinStatus *status; + int32_t numOfUpstream; + SRspResultInfo resultInfo; // todo refactor, add this info for each operator +} SJoinOperatorInfo; - SSDataBlock* pBlock = pInput->block; - pBlock->info.rows = pInput->pRes->numOfRows; - if (pBlock->info.rows == 0) { - return NULL; - } - - //TODO refactor +static void doSetupSDataBlock(SSqlRes* pRes, SSDataBlock* pBlock) { int32_t offset = 0; + char* pData = pRes->data; for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); if (pData != NULL) { pColData->pData = pData + offset * pBlock->info.rows; } else { - pColData->pData = pInput->pRes->urow[i]; + pColData->pData = pRes->urow[i]; } offset += pColData->info.bytes; } - pInput->pRes->numOfRows = 0; + pRes->numOfRows = 0; +} + +// NOTE: there is already exists data blocks before this function calls. +SSDataBlock* doGetDataBlock(void* param, bool* newgroup) { + SOperatorInfo *pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SDummyInputInfo *pInput = pOperator->info; + SSqlObj* pSql = pInput->pSql; + SSqlRes* pRes = &pSql->res; + + SSDataBlock* pBlock = pInput->block; + + pBlock->info.rows = pRes->numOfRows; + if (pRes->numOfRows != 0) { + doSetupSDataBlock(pRes, pBlock); + + if (pInput->numOfFilterCols > 0) { + doSetFilterColumnInfo(pInput->pFilterInfo, pInput->numOfFilterCols, pBlock); + int8_t* p = calloc(pBlock->info.rows, sizeof(int8_t)); + bool all = doFilterDataBlock(pInput->pFilterInfo, pInput->numOfFilterCols, pBlock->info.rows, p); + if (!all) { + doCompactSDataBlock(pBlock, pBlock->info.rows, p); + } + + tfree(p); + } + + *newgroup = false; + return pBlock; + } + + // No data block exists. So retrieve and transfer it into to SSDataBlock + TAOS_ROW pRow = NULL; + taos_fetch_block(pSql, &pRow); + + if (pRes->numOfRows == 0) { + pOperator->status = OP_EXEC_DONE; + return NULL; + } + + pBlock->info.rows = pRes->numOfRows; + doSetupSDataBlock(pRes, pBlock); *newgroup = false; return pBlock; } +SSDataBlock* doDataBlockJoin(void* param, bool* newgroup) { + SOperatorInfo *pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + assert(pOperator->numOfUpstream > 1); + + SJoinOperatorInfo* pJoinInfo = pOperator->info; + pJoinInfo->pRes->info.rows = 0; + + while(1) { + for (int32_t i = 0; i < pOperator->numOfUpstream; ++i) { + SJoinStatus* pStatus = &pJoinInfo->status[i]; + if (pStatus->pBlock == NULL || pStatus->index >= pStatus->pBlock->info.rows) { + pStatus->pBlock = pOperator->upstream[i]->exec(pOperator->upstream[i], newgroup); + pStatus->index = 0; + + if (pStatus->pBlock == NULL) { + pOperator->status = OP_EXEC_DONE; + + pJoinInfo->resultInfo.total += pJoinInfo->pRes->info.rows; + return pJoinInfo->pRes; + } + } + } + + SJoinStatus* st0 = &pJoinInfo->status[0]; + SColumnInfoData* p0 = taosArrayGet(st0->pBlock->pDataBlock, 0); + int64_t* ts0 = (int64_t*) p0->pData; + + bool prefixEqual = true; + + while(1) { + prefixEqual = true; + for (int32_t i = 1; i < pJoinInfo->numOfUpstream; ++i) { + SJoinStatus* st = &pJoinInfo->status[i]; + + SColumnInfoData* p = taosArrayGet(st->pBlock->pDataBlock, 0); + int64_t* ts = (int64_t*)p->pData; + + if (ts[st->index] < ts0[st0->index]) { // less than the first + prefixEqual = false; + if ((++(st->index)) >= st->pBlock->info.rows) { + break; + } + } else if (ts[st->index] > ts0[st0->index]) { // greater than the first; + if (prefixEqual == true) { + prefixEqual = false; + for (int32_t j = 0; j < i; ++j) { + SJoinStatus* stx = &pJoinInfo->status[j]; + if ((++(stx->index)) >= stx->pBlock->info.rows) { + break; + } + } + } else { + if ((++(st0->index)) >= st0->pBlock->info.rows) { + break; + } + } + } + } + + if (prefixEqual) { + int32_t offset = 0; + bool completed = false; + for (int32_t i = 0; i < pOperator->numOfUpstream; ++i) { + SJoinStatus* st1 = &pJoinInfo->status[i]; + int32_t rows = pJoinInfo->pRes->info.rows; + + for (int32_t j = 0; j < st1->pBlock->info.numOfCols; ++j) { + SColumnInfoData* pCol1 = taosArrayGet(pJoinInfo->pRes->pDataBlock, j + offset); + SColumnInfoData* pSrc = taosArrayGet(st1->pBlock->pDataBlock, j); + + int32_t bytes = pSrc->info.bytes; + memcpy(pCol1->pData + rows * bytes, pSrc->pData + st1->index * bytes, bytes); + } + + offset += st1->pBlock->info.numOfCols; + if ((++(st1->index)) == st1->pBlock->info.rows) { + completed = true; + } + } + + if ((++pJoinInfo->pRes->info.rows) >= pJoinInfo->resultInfo.capacity) { + pJoinInfo->resultInfo.total += pJoinInfo->pRes->info.rows; + return pJoinInfo->pRes; + } + + if (completed == true) { + break; + } + } + } +/* + while (st0->index < st0->pBlock->info.rows && st1->index < st1->pBlock->info.rows) { + SColumnInfoData* p0 = taosArrayGet(st0->pBlock->pDataBlock, 0); + SColumnInfoData* p1 = taosArrayGet(st1->pBlock->pDataBlock, 0); + + int64_t* ts0 = (int64_t*)p0->pData; + int64_t* ts1 = (int64_t*)p1->pData; + if (ts0[st0->index] == ts1[st1->index]) { // add to the final result buffer + // check if current output buffer is over the threshold to pause current loop + int32_t rows = pJoinInfo->pRes->info.rows; + for (int32_t j = 0; j < st0->pBlock->info.numOfCols; ++j) { + SColumnInfoData* pCol1 = taosArrayGet(pJoinInfo->pRes->pDataBlock, j); + SColumnInfoData* pSrc = taosArrayGet(st0->pBlock->pDataBlock, j); + + int32_t bytes = pSrc->info.bytes; + memcpy(pCol1->pData + rows * bytes, pSrc->pData + st0->index * bytes, bytes); + } + + for (int32_t j = 0; j < st1->pBlock->info.numOfCols; ++j) { + SColumnInfoData* pCol1 = taosArrayGet(pJoinInfo->pRes->pDataBlock, j + st0->pBlock->info.numOfCols); + SColumnInfoData* pSrc = taosArrayGet(st1->pBlock->pDataBlock, j); + + int32_t bytes = pSrc->info.bytes; + memcpy(pCol1->pData + rows * bytes, pSrc->pData + st1->index * bytes, bytes); + } + + st0->index++; + st1->index++; + + if ((++pJoinInfo->pRes->info.rows) >= pJoinInfo->resultInfo.capacity) { + pJoinInfo->resultInfo.total += pJoinInfo->pRes->info.rows; + return pJoinInfo->pRes; + } + } else if (ts0[st0->index] < ts1[st1->index]) { + st0->index++; + } else { + st1->index++; + } + }*/ + } +} + static void destroyDummyInputOperator(void* param, int32_t numOfOutput) { SDummyInputInfo* pInfo = (SDummyInputInfo*) param; @@ -676,15 +967,17 @@ static void destroyDummyInputOperator(void* param, int32_t numOfOutput) { } pInfo->block = destroyOutputBuf(pInfo->block); - pInfo->pRes = NULL; + pInfo->pSql = NULL; } // todo this operator servers as the adapter for Operator tree and SqlRes result, remove it later -SOperatorInfo* createDummyInputOperator(char* pResult, SSchema* pSchema, int32_t numOfCols) { +SOperatorInfo* createDummyInputOperator(SSqlObj* pSql, SSchema* pSchema, int32_t numOfCols, SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols) { assert(numOfCols > 0); SDummyInputInfo* pInfo = calloc(1, sizeof(SDummyInputInfo)); - pInfo->pRes = (SSqlRes*) pResult; + pInfo->pSql = pSql; + pInfo->pFilterInfo = pFilterInfo; + pInfo->numOfFilterCols = numOfFilterCols; pInfo->block = calloc(numOfCols, sizeof(SSDataBlock)); pInfo->block->info.numOfCols = numOfCols; @@ -710,6 +1003,52 @@ SOperatorInfo* createDummyInputOperator(char* pResult, SSchema* pSchema, int32_t return pOptr; } +static void destroyJoinOperator(void* param, int32_t numOfOutput) { + SJoinOperatorInfo* pInfo = (SJoinOperatorInfo*) param; + tfree(pInfo->status); + + pInfo->pRes = destroyOutputBuf(pInfo->pRes); +} + +SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput) { + SJoinOperatorInfo* pInfo = calloc(1, sizeof(SJoinOperatorInfo)); + + pInfo->numOfUpstream = numOfUpstream; + pInfo->status = calloc(numOfUpstream, sizeof(SJoinStatus)); + + SRspResultInfo* pResInfo = &pInfo->resultInfo; + pResInfo->capacity = 4096; + pResInfo->threshold = (int32_t) (4096 * 0.8); + + pInfo->pRes = calloc(1, sizeof(SSDataBlock)); + pInfo->pRes->info.numOfCols = numOfOutput; + pInfo->pRes->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); + for(int32_t i = 0; i < numOfOutput; ++i) { + SColumnInfoData colData = {{0}}; + colData.info.bytes = pSchema[i].bytes; + colData.info.type = pSchema[i].type; + colData.info.colId = pSchema[i].colId; + colData.pData = calloc(1, colData.info.bytes * pResInfo->capacity); + + taosArrayPush(pInfo->pRes->pDataBlock, &colData); + } + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "JoinOperator"; + pOperator->operatorType = OP_Join; + pOperator->numOfOutput = numOfOutput; + pOperator->blockingOptr = false; + pOperator->info = pInfo; + pOperator->exec = doDataBlockJoin; + pOperator->cleanup = destroyJoinOperator; + + for(int32_t i = 0; i < numOfUpstream; ++i) { + appendUpstream(pOperator, pUpstream[i]); + } + + return pOperator; +} + void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo) { // set the correct result SSDataBlock* p = pQueryInfo->pQInfo->runtimeEnv.outputBuf; @@ -724,18 +1063,18 @@ void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo) { pRes->completed = (pRes->numOfRows == 0); } -void handleDownstreamOperator(SSqlRes* pRes, SQueryInfo* pQueryInfo) { - if (pQueryInfo->pDownstream != NULL) { - // handle the following query process - SQueryInfo *px = pQueryInfo->pDownstream; - SColumnInfo* pColumnInfo = extractColumnInfoFromResult(px->pTableMetaInfo[0]->pTableMeta, px->colList); - int32_t numOfOutput = (int32_t) tscSqlExprNumOfExprs(px); +void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQueryInfo* px, SSqlRes* pOutput) { + // handle the following query process + if (px->pQInfo == NULL) { + SColumnInfo* pColumnInfo = extractColumnInfoFromResult(px->colList); - int32_t numOfCols = (int32_t) taosArrayGetSize(px->colList); - SQueriedTableInfo info = {.colList = pColumnInfo, .numOfCols = numOfCols,}; SSchema* pSchema = tscGetTableSchema(px->pTableMetaInfo[0]->pTableMeta); - STableGroupInfo tableGroupInfo = {.numOfTables = 1, .pGroupList = taosArrayInit(1, POINTER_BYTES),}; + STableGroupInfo tableGroupInfo = { + .numOfTables = 1, + .pGroupList = taosArrayInit(1, POINTER_BYTES), + }; + tableGroupInfo.map = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); STableKeyInfo tableKeyInfo = {.pTable = NULL, .lastKey = INT64_MIN}; @@ -745,18 +1084,81 @@ void handleDownstreamOperator(SSqlRes* pRes, SQueryInfo* pQueryInfo) { taosArrayPush(tableGroupInfo.pGroupList, &group); - SOperatorInfo* pSourceOptr = createDummyInputOperator((char*)pRes, pSchema, numOfCols); + // if it is a join query, create join operator here + int32_t numOfCol1 = px->pTableMetaInfo[0]->pTableMeta->tableInfo.numOfColumns; - SExprInfo *exprInfo = NULL; - /*int32_t code = */createQueryFunc(&info, numOfOutput, &exprInfo, px->exprList->pData, NULL, px->type, NULL, NULL); - px->pQInfo = createQueryInfoFromQueryNode(px, exprInfo, &tableGroupInfo, pSourceOptr, NULL, NULL, MASTER_SCAN); + int32_t numOfFilterCols = 0; + SColumnInfo* tableCols = calloc(numOfCol1, sizeof(SColumnInfo)); + for(int32_t i = 0; i < numOfCol1; ++i) { + SColumn* pCol = taosArrayGetP(px->colList, i); + if (pCol->info.flist.numOfFilters > 0) { + numOfFilterCols += 1; + } - uint64_t qId = 0; - qTableQuery(px->pQInfo, &qId); - convertQueryResult(pRes, px); + tableCols[i] = pCol->info; + } + SSingleColumnFilterInfo* pFilterInfo = NULL; + if (numOfFilterCols > 0) { + doCreateFilterInfo(tableCols, numOfCol1, numOfFilterCols, &pFilterInfo, 0); + } + + SOperatorInfo* pSourceOperator = createDummyInputOperator(pSqlObjList[0], pSchema, numOfCol1, pFilterInfo, numOfFilterCols); + + SSchema* schema = NULL; + if (px->numOfTables > 1) { + SOperatorInfo** p = calloc(px->numOfTables, POINTER_BYTES); + p[0] = pSourceOperator; + + int32_t num = (int32_t) taosArrayGetSize(px->colList); + schema = calloc(num, sizeof(SSchema)); + memcpy(schema, pSchema, numOfCol1*sizeof(SSchema)); + + int32_t offset = pSourceOperator->numOfOutput; + + for(int32_t i = 1; i < px->numOfTables; ++i) { + SSchema* pSchema1 = tscGetTableSchema(px->pTableMetaInfo[i]->pTableMeta); + int32_t n = px->pTableMetaInfo[i]->pTableMeta->tableInfo.numOfColumns; + + int32_t numOfFilterCols1 = 0; + SColumnInfo* tableCols1 = calloc(numOfCol1, sizeof(SColumnInfo)); + for(int32_t j = 0; j < numOfCol1; ++j) { + SColumn* pCol = taosArrayGetP(px->colList, j); + if (pCol->info.flist.numOfFilters > 0) { + numOfFilterCols += 1; + } + + tableCols1[j] = pCol->info; + } + + SSingleColumnFilterInfo* pFilterInfo1 = NULL; + if (numOfFilterCols1 > 0) { + doCreateFilterInfo(tableCols1, numOfCol1, numOfFilterCols1, &pFilterInfo1, 0); + } + + p[i] = createDummyInputOperator(pSqlObjList[i], pSchema1, n, pFilterInfo1, numOfFilterCols1); + memcpy(&schema[offset], pSchema1, n * sizeof(SSchema)); + offset += n; + } + + pSourceOperator = createJoinOperatorInfo(p, px->numOfTables, schema, num); + tfree(p); + } else { + size_t num = taosArrayGetSize(px->colList); + schema = calloc(num, sizeof(SSchema)); + memcpy(schema, pSchema, numOfCol1*sizeof(SSchema)); + } + + SExprInfo* exprInfo = NULL; + px->pQInfo = createQInfoFromQueryNode(px, exprInfo, &tableGroupInfo, pSourceOperator, NULL, NULL, MASTER_SCAN); tfree(pColumnInfo); + tfree(schema); + tfree(exprInfo); } + + uint64_t qId = 0; + qTableQuery(px->pQInfo, &qId); + convertQueryResult(pOutput, px); } static void tscDestroyResPointerInfo(SSqlRes* pRes) { @@ -789,24 +1191,27 @@ static void tscDestroyResPointerInfo(SSqlRes* pRes) { } void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta) { - if (pCmd == NULL || pCmd->numOfClause == 0) { + if (pCmd == NULL) { return; } - - for (int32_t i = 0; i < pCmd->numOfClause; ++i) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, i); - // recursive call it - if (taosArrayGetSize(pQueryInfo->pUpstream) > 0) { - SQueryInfo* pUp = taosArrayGetP(pQueryInfo->pUpstream, 0); - freeQueryInfoImpl(pUp); - clearAllTableMetaInfo(pUp, removeMeta); - if (pUp->pQInfo != NULL) { - qDestroyQueryInfo(pUp->pQInfo); - pUp->pQInfo = NULL; + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + + while(pQueryInfo != NULL) { + SQueryInfo* p = pQueryInfo->sibling; + + size_t numOfUpstream = taosArrayGetSize(pQueryInfo->pUpstream); + for(int32_t i = 0; i < numOfUpstream; ++i) { + SQueryInfo* pUpQueryInfo = taosArrayGetP(pQueryInfo->pUpstream, i); + freeQueryInfoImpl(pUpQueryInfo); + + clearAllTableMetaInfo(pUpQueryInfo, removeMeta); + if (pUpQueryInfo->pQInfo != NULL) { + qDestroyQueryInfo(pUpQueryInfo->pQInfo); + pUpQueryInfo->pQInfo = NULL; } - tfree(pUp); + tfree(pUpQueryInfo); } if (pCmd->subCmd) { @@ -814,7 +1219,7 @@ void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta) { } else { pQueryInfo->pUdfInfo = tscDestroyUdfArrayList(pQueryInfo->pUdfInfo); } - + freeQueryInfoImpl(pQueryInfo); clearAllTableMetaInfo(pQueryInfo, removeMeta); @@ -824,48 +1229,60 @@ void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta) { } tfree(pQueryInfo); + pQueryInfo = p; } - - pCmd->numOfClause = 0; - tfree(pCmd->pQueryInfo); + + pCmd->pQueryInfo = NULL; + pCmd->active = NULL; } void destroyTableNameList(SSqlCmd* pCmd) { - if (pCmd->numOfTables == 0) { - assert(pCmd->pTableNameList == NULL); + if (pCmd->insertParam.numOfTables == 0) { + assert(pCmd->insertParam.pTableNameList == NULL); return; } - for(int32_t i = 0; i < pCmd->numOfTables; ++i) { - tfree(pCmd->pTableNameList[i]); + for(int32_t i = 0; i < pCmd->insertParam.numOfTables; ++i) { + tfree(pCmd->insertParam.pTableNameList[i]); } - pCmd->numOfTables = 0; - tfree(pCmd->pTableNameList); + pCmd->insertParam.numOfTables = 0; + tfree(pCmd->insertParam.pTableNameList); } -void tscResetSqlCmd(SSqlCmd* pCmd, bool removeMeta) { +void tscResetSqlCmd(SSqlCmd* pCmd, bool clearCachedMeta) { pCmd->command = 0; pCmd->numOfCols = 0; pCmd->count = 0; pCmd->curSql = NULL; pCmd->msgType = 0; - pCmd->parseFinished = 0; - pCmd->autoCreated = 0; destroyTableNameList(pCmd); - pCmd->pTableBlockHashList = tscDestroyBlockHashTable(pCmd->pTableBlockHashList, removeMeta); - pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); - tscFreeQueryInfo(pCmd, removeMeta); + pCmd->insertParam.pTableBlockHashList = tscDestroyBlockHashTable(pCmd->insertParam.pTableBlockHashList, clearCachedMeta); + pCmd->insertParam.pDataBlocks = tscDestroyBlockArrayList(pCmd->insertParam.pDataBlocks); + tscFreeQueryInfo(pCmd, clearCachedMeta); + + if (pCmd->pTableMetaMap != NULL) { + STableMetaVgroupInfo* p = taosHashIterate(pCmd->pTableMetaMap, NULL); + while (p) { + tfree(p->pVgroupInfo); + tfree(p->pTableMeta); + p = taosHashIterate(pCmd->pTableMetaMap, p); + } + + taosHashCleanup(pCmd->pTableMetaMap); + pCmd->pTableMetaMap = NULL; + } } void tscFreeSqlResult(SSqlObj* pSql) { - tscDestroyLocalMerger(pSql); - SSqlRes* pRes = &pSql->res; - tscDestroyResPointerInfo(pRes); + tscDestroyLocalMerger(pRes->pLocalMerger); + pRes->pLocalMerger = NULL; + + tscDestroyResPointerInfo(pRes); memset(&pSql->res, 0, sizeof(SSqlRes)); } @@ -887,7 +1304,6 @@ void tscFreeSubobj(SSqlObj* pSql) { } tfree(pSql->subState.states); - pSql->subState.numOfSub = 0; } @@ -911,7 +1327,6 @@ void tscFreeRegisteredSqlObj(void *pSql) { tscDebug("0x%"PRIx64" free SqlObj, total in tscObj:%d, total:%d", p->self, num, total); tscFreeSqlObj(p); taosReleaseRef(tscRefId, pTscObj->rid); - } void tscFreeMetaSqlObj(int64_t *rid){ @@ -1096,14 +1511,11 @@ void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable, bool removeMeta) { int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { SSqlCmd* pCmd = &pSql->cmd; - assert(pDataBlock->pTableMeta != NULL); + assert(pDataBlock->pTableMeta != NULL && pDataBlock->size <= pDataBlock->nAllocSize && pDataBlock->size > sizeof(SMsgDesc)); - pCmd->numOfTablesInSubmit = pDataBlock->numOfTables; + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - assert(pCmd->numOfClause == 1); - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - - // todo refactor + // todo remove it later // set the correct table meta object, the table meta has been locked in pDataBlocks, so it must be in the cache if (pTableMetaInfo->pTableMeta != pDataBlock->pTableMeta) { tNameAssign(&pTableMetaInfo->name, &pDataBlock->tableName); @@ -1113,33 +1525,42 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { } pTableMetaInfo->pTableMeta = tscTableMetaDup(pDataBlock->pTableMeta); - pTableMetaInfo->tableMetaSize = tscGetTableMetaSize(pDataBlock->pTableMeta); + pTableMetaInfo->tableMetaSize = tscGetTableMetaSize(pDataBlock->pTableMeta); } /* - * the submit message consists of : [RPC header|message body|digest] - * the dataBlock only includes the RPC Header buffer and actual submit message body, space for digest needs - * additional space. + * the format of submit message is as follows [RPC header|message body|digest] + * the dataBlock only includes the RPC Header buffer and actual submit message body, + * space for digest needs additional space. */ - int ret = tscAllocPayload(pCmd, pDataBlock->size + 100); + int ret = tscAllocPayload(pCmd, pDataBlock->size); if (TSDB_CODE_SUCCESS != ret) { return ret; } - assert(pDataBlock->size <= pDataBlock->nAllocSize); memcpy(pCmd->payload, pDataBlock->pData, pDataBlock->size); - /* - * the payloadLen should be actual message body size - * the old value of payloadLen is the allocated payload size - */ + //the payloadLen should be actual message body size, the old value of payloadLen is the allocated payload size pCmd->payloadLen = pDataBlock->size; + assert(pCmd->allocSize >= (uint32_t)(pCmd->payloadLen)); - assert(pCmd->allocSize >= (uint32_t)(pCmd->payloadLen + 100) && pCmd->payloadLen > 0); + // NOTE: shell message size should not include SMsgDesc + int32_t size = pCmd->payloadLen - sizeof(SMsgDesc); + + SMsgDesc* pMsgDesc = (SMsgDesc*) pCmd->payload; + pMsgDesc->numOfVnodes = htonl(1); // always for one vnode + + SSubmitMsg *pShellMsg = (SSubmitMsg *)(pCmd->payload + sizeof(SMsgDesc)); + pShellMsg->header.vgId = htonl(pDataBlock->pTableMeta->vgId); // data in current block all routes to the same vgroup + pShellMsg->header.contLen = htonl(size); // the length not includes the size of SMsgDesc + pShellMsg->length = pShellMsg->header.contLen; + pShellMsg->numOfBlocks = htonl(pDataBlock->numOfTables); // the number of tables to be inserted + + tscDebug("0x%"PRIx64" submit msg built, vgId:%d numOfTables:%d", pSql->self, pDataBlock->pTableMeta->vgId, pDataBlock->numOfTables); return TSDB_CODE_SUCCESS; } -SQueryInfo* tscGetActiveQueryInfo(SSqlCmd* pCmd) { +SQueryInfo* tscGetQueryInfo(SSqlCmd* pCmd) { return pCmd->active; } @@ -1297,25 +1718,25 @@ static int32_t getRowExpandSize(STableMeta* pTableMeta) { } static void extractTableNameList(SSqlCmd* pCmd, bool freeBlockMap) { - pCmd->numOfTables = (int32_t) taosHashGetSize(pCmd->pTableBlockHashList); - if (pCmd->pTableNameList == NULL) { - pCmd->pTableNameList = calloc(pCmd->numOfTables, POINTER_BYTES); + pCmd->insertParam.numOfTables = (int32_t) taosHashGetSize(pCmd->insertParam.pTableBlockHashList); + if (pCmd->insertParam.pTableNameList == NULL) { + pCmd->insertParam.pTableNameList = calloc(pCmd->insertParam.numOfTables, POINTER_BYTES); } else { - memset(pCmd->pTableNameList, 0, pCmd->numOfTables * POINTER_BYTES); + memset(pCmd->insertParam.pTableNameList, 0, pCmd->insertParam.numOfTables * POINTER_BYTES); } - STableDataBlocks **p1 = taosHashIterate(pCmd->pTableBlockHashList, NULL); + STableDataBlocks **p1 = taosHashIterate(pCmd->insertParam.pTableBlockHashList, NULL); int32_t i = 0; while(p1) { STableDataBlocks* pBlocks = *p1; - tfree(pCmd->pTableNameList[i]); + tfree(pCmd->insertParam.pTableNameList[i]); - pCmd->pTableNameList[i++] = tNameDup(&pBlocks->tableName); - p1 = taosHashIterate(pCmd->pTableBlockHashList, p1); + pCmd->insertParam.pTableNameList[i++] = tNameDup(&pBlocks->tableName); + p1 = taosHashIterate(pCmd->insertParam.pTableBlockHashList, p1); } if (freeBlockMap) { - pCmd->pTableBlockHashList = tscDestroyBlockHashTable(pCmd->pTableBlockHashList, false); + pCmd->insertParam.pTableBlockHashList = tscDestroyBlockHashTable(pCmd->insertParam.pTableBlockHashList, false); } } @@ -1326,7 +1747,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, bool freeBlockMap) { void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); - STableDataBlocks** p = taosHashIterate(pCmd->pTableBlockHashList, NULL); + STableDataBlocks** p = taosHashIterate(pCmd->insertParam.pTableBlockHashList, NULL); STableDataBlocks* pOneTableBlock = *p; while(pOneTableBlock) { @@ -1382,7 +1803,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, bool freeBlockMap) { pBlocks->schemaLen = 0; // erase the empty space reserved for binary data - int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, pCmd->submitSchema); + int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, pCmd->insertParam.schemaAttached); assert(finalLen <= len); dataBuf->size += (finalLen + sizeof(SSubmitBlk)); @@ -1397,7 +1818,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, bool freeBlockMap) { tscDebug("0x%"PRIx64" table %s data block is empty", pSql->self, pOneTableBlock->tableName.tname); } - p = taosHashIterate(pCmd->pTableBlockHashList, p); + p = taosHashIterate(pCmd->insertParam.pTableBlockHashList, p); if (p == NULL) { break; } @@ -1408,7 +1829,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, bool freeBlockMap) { extractTableNameList(pCmd, freeBlockMap); // free the table data blocks; - pCmd->pDataBlocks = pVnodeDataBlockList; + pCmd->insertParam.pDataBlocks = pVnodeDataBlockList; taosHashCleanup(pVnodeDataBlockHashList); return TSDB_CODE_SUCCESS; @@ -1440,18 +1861,22 @@ bool tscIsInsertData(char* sqlstr) { } int tscAllocPayload(SSqlCmd* pCmd, int size) { - assert(size > 0); - if (pCmd->payload == NULL) { assert(pCmd->allocSize == 0); pCmd->payload = (char*)calloc(1, size); - if (pCmd->payload == NULL) return TSDB_CODE_TSC_OUT_OF_MEMORY; + if (pCmd->payload == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + pCmd->allocSize = size; } else { if (pCmd->allocSize < (uint32_t)size) { char* b = realloc(pCmd->payload, size); - if (b == NULL) return TSDB_CODE_TSC_OUT_OF_MEMORY; + if (b == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + pCmd->payload = b; pCmd->allocSize = size; } @@ -1459,7 +1884,7 @@ int tscAllocPayload(SSqlCmd* pCmd, int size) { memset(pCmd->payload, 0, pCmd->allocSize); } - assert(pCmd->allocSize >= (uint32_t)size); + assert(pCmd->allocSize >= (uint32_t)size && size > 0); return TSDB_CODE_SUCCESS; } @@ -1488,16 +1913,14 @@ SInternalField* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_F } void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); - - SExprInfo* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); - pExpr->base.offset = 0; + int32_t offset = 0; + size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for (int32_t i = 1; i < numOfExprs; ++i) { - SExprInfo* prev = taosArrayGetP(pQueryInfo->exprList, i - 1); + for (int32_t i = 0; i < numOfExprs; ++i) { SExprInfo* p = taosArrayGetP(pQueryInfo->exprList, i); - p->base.offset = prev->base.offset + prev->base.resBytes; + p->base.offset = offset; + offset += p->base.resBytes; } } @@ -1538,6 +1961,7 @@ int32_t tscFieldInfoCompare(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFi *diffSize = 1; if (pField2->bytes > pField1->bytes) { + assert(IS_VAR_DATA_TYPE(pField1->type)); pField1->bytes = pField2->bytes; } } @@ -1629,8 +2053,43 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { memset(pFieldInfo, 0, sizeof(SFieldInfo)); } -static SExprInfo* doCreateSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t resColId, int16_t interSize, int32_t colType) { +void tscFieldInfoCopy(SFieldInfo* pFieldInfo, const SFieldInfo* pSrc, const SArray* pExprList) { + assert(pFieldInfo != NULL && pSrc != NULL && pExprList != NULL); + pFieldInfo->numOfOutput = pSrc->numOfOutput; + + if (pSrc->final != NULL) { + pFieldInfo->final = calloc(pSrc->numOfOutput, sizeof(TAOS_FIELD)); + memcpy(pFieldInfo->final, pSrc->final, sizeof(TAOS_FIELD) * pSrc->numOfOutput); + } + + if (pSrc->internalField != NULL) { + size_t num = taosArrayGetSize(pSrc->internalField); + size_t numOfExpr = taosArrayGetSize(pExprList); + + for (int32_t i = 0; i < num; ++i) { + SInternalField* pfield = taosArrayGet(pSrc->internalField, i); + + SInternalField p = {.visible = pfield->visible, .field = pfield->field}; + + int32_t resColId = pfield->pExpr->base.resColId; + for(int32_t j = 0; j < numOfExpr; ++j) { + SExprInfo* pExpr = taosArrayGetP(pExprList, j); + if (pExpr->base.resColId == resColId) { + p.pExpr = pExpr; + break; + } + } +// p.pExpr = calloc(1, sizeof(SExprInfo)); + +// tscExprAssign(p.pExpr, pfield->pExpr); + taosArrayPush(pFieldInfo->internalField, &p); + } + } +} + + +SExprInfo* tscExprCreate(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, + int16_t size, int16_t resColId, int16_t interSize, int32_t colType) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex); SExprInfo* pExpr = calloc(1, sizeof(SExprInfo)); @@ -1647,16 +2106,14 @@ static SExprInfo* doCreateSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SC p->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; p->colBytes = s->bytes; p->colType = s->type; - } else if (pColIndex->columnIndex == TSDB_BLOCK_DIST_COLUMN_INDEX) { - SSchema s = tGetBlockDistColumnSchema(); - - p->colInfo.colId = TSDB_BLOCK_DIST_COLUMN_INDEX; - p->colBytes = s.bytes; - p->colType = s.type; } else if (pColIndex->columnIndex <= TSDB_UD_COLUMN_INDEX) { p->colInfo.colId = pColIndex->columnIndex; p->colBytes = size; p->colType = type; + } else if (functionId == TSDB_FUNC_BLKINFO) { + p->colInfo.colId = pColIndex->columnIndex; + p->colBytes = TSDB_MAX_BINARY_LEN; + p->colType = TSDB_DATA_TYPE_BINARY; } else { if (TSDB_COL_IS_TAG(colType)) { SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); @@ -1689,29 +2146,29 @@ static SExprInfo* doCreateSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SC return pExpr; } -SExprInfo* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, +SExprInfo* tscExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, int16_t size, int16_t resColId, int16_t interSize, bool isTagCol) { int32_t num = (int32_t)taosArrayGetSize(pQueryInfo->exprList); if (index == num) { - return tscSqlExprAppend(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); + return tscExprAppend(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); } - SExprInfo* pExpr = doCreateSqlExpr(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); + SExprInfo* pExpr = tscExprCreate(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); taosArrayInsert(pQueryInfo->exprList, index, &pExpr); return pExpr; } -SExprInfo* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, +SExprInfo* tscExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, int16_t size, int16_t resColId, int16_t interSize, bool isTagCol) { - SExprInfo* pExpr = doCreateSqlExpr(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); + SExprInfo* pExpr = tscExprCreate(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); taosArrayPush(pQueryInfo->exprList, &pExpr); return pExpr; } -SExprInfo* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, +SExprInfo* tscExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type, int16_t size) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, index); + SExprInfo* pExpr = tscExprGet(pQueryInfo, index); if (pExpr == NULL) { return NULL; } @@ -1733,9 +2190,9 @@ bool tscMultiRoundQuery(SQueryInfo* pQueryInfo, int32_t index) { return false; } - int32_t numOfExprs = (int32_t) tscSqlExprNumOfExprs(pQueryInfo); + int32_t numOfExprs = (int32_t) tscNumOfExprs(pQueryInfo); for(int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (pExpr->base.functionId == TSDB_FUNC_STDDEV_DST) { return true; } @@ -1744,11 +2201,11 @@ bool tscMultiRoundQuery(SQueryInfo* pQueryInfo, int32_t index) { return false; } -size_t tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo) { +size_t tscNumOfExprs(SQueryInfo* pQueryInfo) { return taosArrayGetSize(pQueryInfo->exprList); } -void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes) { +void tscExprAddParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes) { assert (pExpr != NULL || argument != NULL || bytes != 0); // set parameter value @@ -1759,14 +2216,14 @@ void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes) assert(pExpr->numOfParams <= 3); } -SExprInfo* tscSqlExprGet(SQueryInfo* pQueryInfo, int32_t index) { +SExprInfo* tscExprGet(SQueryInfo* pQueryInfo, int32_t index) { return taosArrayGetP(pQueryInfo->exprList, index); } /* * NOTE: Does not release SExprInfo here. */ -void tscSqlExprInfoDestroy(SArray* pExprInfo) { +void tscExprDestroy(SArray* pExprInfo) { size_t size = taosArrayGetSize(pExprInfo); for(int32_t i = 0; i < size; ++i) { @@ -1777,23 +2234,42 @@ void tscSqlExprInfoDestroy(SArray* pExprInfo) { taosArrayDestroy(pExprInfo); } -int32_t tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy) { +int32_t tscExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy) { assert(src != NULL && dst != NULL); size_t size = taosArrayGetSize(src); for (int32_t i = 0; i < size; ++i) { SExprInfo* pExpr = taosArrayGetP(src, i); - + if (pExpr->base.uid == uid) { if (deepcopy) { SExprInfo* p1 = calloc(1, sizeof(SExprInfo)); - tscSqlExprAssign(p1, pExpr); + tscExprAssign(p1, pExpr); taosArrayPush(dst, &p1); } else { taosArrayPush(dst, &pExpr); } + } + } + return 0; +} + +int32_t tscExprCopyAll(SArray* dst, const SArray* src, bool deepcopy) { + assert(src != NULL && dst != NULL); + + size_t size = taosArrayGetSize(src); + for (int32_t i = 0; i < size; ++i) { + SExprInfo* pExpr = taosArrayGetP(src, i); + + if (deepcopy) { + SExprInfo* p1 = calloc(1, sizeof(SExprInfo)); + tscExprAssign(p1, pExpr); + + taosArrayPush(dst, &p1); + } else { + taosArrayPush(dst, &pExpr); } } @@ -1826,7 +2302,7 @@ bool tscColumnExists(SArray* pColumnList, int32_t columnIndex, uint64_t uid) { return true; } -void tscSqlExprAssign(SExprInfo* dst, const SExprInfo* src) { +void tscExprAssign(SExprInfo* dst, const SExprInfo* src) { assert(dst != NULL && src != NULL); *dst = *src; @@ -1938,6 +2414,18 @@ void tscColumnListCopy(SArray* dst, const SArray* src, uint64_t tableUid) { } } +void tscColumnListCopyAll(SArray* dst, const SArray* src) { + assert(src != NULL && dst != NULL); + + size_t num = taosArrayGetSize(src); + for (int32_t i = 0; i < num; ++i) { + SColumn* pCol = taosArrayGetP(src, i); + SColumn* p = tscColumnClone(pCol); + taosArrayPush(dst, &p); + } +} + + void tscColumnListDestroy(SArray* pColumnList) { if (pColumnList == NULL) { return; @@ -1976,7 +2464,7 @@ static int32_t validateQuoteToken(SStrToken* pToken) { } if (k != pToken->n || pToken->type != TK_ID) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } return TSDB_CODE_SUCCESS; } @@ -2026,7 +2514,7 @@ void tscDequoteAndTrimToken(SStrToken* pToken) { int32_t tscValidateName(SStrToken* pToken) { if (pToken->type != TK_STRING && pToken->type != TK_ID) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } char* sep = strnchr(pToken->z, TS_PATH_DELIMITER[0], pToken->n, true); @@ -2045,14 +2533,14 @@ int32_t tscValidateName(SStrToken* pToken) { } else { sep = strnchr(pToken->z, TS_PATH_DELIMITER[0], pToken->n, true); if (sep == NULL) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } return tscValidateName(pToken); } } else { if (isNumber(pToken)) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } } } else { // two part @@ -2065,15 +2553,15 @@ int32_t tscValidateName(SStrToken* pToken) { pToken->n = tGetToken(pToken->z, &pToken->type); if (pToken->z[pToken->n] != TS_PATH_DELIMITER[0]) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } if (pToken->type != TK_STRING && pToken->type != TK_ID) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } if (pToken->type == TK_STRING && validateQuoteToken(pToken) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } int32_t firstPartLen = pToken->n; @@ -2082,11 +2570,11 @@ int32_t tscValidateName(SStrToken* pToken) { pToken->n = (uint32_t)(oldLen - (sep - pStr) - 1); int32_t len = tGetToken(pToken->z, &pToken->type); if (len != pToken->n || (pToken->type != TK_STRING && pToken->type != TK_ID)) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } if (pToken->type == TK_STRING && validateQuoteToken(pToken) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } // re-build the whole name string @@ -2121,7 +2609,7 @@ bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t return false; } - if (colId == TSDB_TBNAME_COLUMN_INDEX || colId == TSDB_BLOCK_DIST_COLUMN_INDEX || (colId <= TSDB_UD_COLUMN_INDEX && numOfParams == 2)) { + if (colId == TSDB_TBNAME_COLUMN_INDEX || (colId <= TSDB_UD_COLUMN_INDEX && numOfParams == 2)) { return true; } @@ -2240,9 +2728,9 @@ void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + size_t numOfExprs = tscNumOfExprs(pQueryInfo); for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); pColInfo[i].functionId = pExpr->base.functionId; if (TSDB_COL_IS_TAG(pExpr->base.colInfo.flag)) { @@ -2294,14 +2782,9 @@ bool tscShouldBeFreed(SSqlObj* pSql) { * @param tableIndex denote the table index for join query, where more than one table exists * @return */ -STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd* pCmd, int32_t clauseIndex, int32_t tableIndex) { - if (pCmd == NULL || pCmd->numOfClause == 0) { - return NULL; - } - - assert(clauseIndex >= 0 && clauseIndex < pCmd->numOfClause); - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, clauseIndex); +STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd* pCmd, int32_t tableIndex) { + assert(pCmd != NULL); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); return tscGetMetaInfo(pQueryInfo, tableIndex); } @@ -2318,8 +2801,8 @@ STableMetaInfo* tscGetMetaInfo(SQueryInfo* pQueryInfo, int32_t tableIndex) { return pQueryInfo->pTableMetaInfo[tableIndex]; } -SQueryInfo* tscGetQueryInfoS(SSqlCmd* pCmd, int32_t subClauseIndex) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, subClauseIndex); +SQueryInfo* tscGetQueryInfoS(SSqlCmd* pCmd) { + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); int32_t ret = TSDB_CODE_SUCCESS; while ((pQueryInfo) == NULL) { @@ -2328,7 +2811,7 @@ SQueryInfo* tscGetQueryInfoS(SSqlCmd* pCmd, int32_t subClauseIndex) { return NULL; } - pQueryInfo = tscGetQueryInfo(pCmd, subClauseIndex); + pQueryInfo = tscGetQueryInfo(pCmd); } return pQueryInfo; @@ -2352,6 +2835,7 @@ STableMetaInfo* tscGetTableMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, i return tscGetMetaInfo(pQueryInfo, k); } +// todo refactor void tscInitQueryInfo(SQueryInfo* pQueryInfo) { assert(pQueryInfo->fieldsInfo.internalField == NULL); pQueryInfo->fieldsInfo.internalField = taosArrayInit(4, sizeof(SInternalField)); @@ -2361,27 +2845,17 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) { pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); pQueryInfo->udColumnId = TSDB_UD_COLUMN_INDEX; - pQueryInfo->resColumnId = TSDB_RES_COL_ID; pQueryInfo->limit.limit = -1; pQueryInfo->limit.offset = 0; pQueryInfo->slimit.limit = -1; pQueryInfo->slimit.offset = 0; pQueryInfo->pUpstream = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->window = TSWINDOW_INITIALIZER; } int32_t tscAddQueryInfo(SSqlCmd* pCmd) { assert(pCmd != NULL); - - // todo refactor: remove this structure - size_t s = pCmd->numOfClause + 1; - char* tmp = realloc(pCmd->pQueryInfo, s * POINTER_BYTES); - if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pCmd->pQueryInfo = (SQueryInfo**)tmp; - SQueryInfo* pQueryInfo = calloc(1, sizeof(SQueryInfo)); if (pQueryInfo == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -2389,10 +2863,20 @@ int32_t tscAddQueryInfo(SSqlCmd* pCmd) { tscInitQueryInfo(pQueryInfo); - pQueryInfo->window = TSWINDOW_INITIALIZER; pQueryInfo->msg = pCmd->payload; // pointer to the parent error message buffer - pCmd->pQueryInfo[pCmd->numOfClause++] = pQueryInfo; + if (pCmd->pQueryInfo == NULL) { + pCmd->pQueryInfo = pQueryInfo; + } else { + SQueryInfo* p = pCmd->pQueryInfo; + while(p->sibling != NULL) { + p = p->sibling; + } + + p->sibling = pQueryInfo; + } + + pCmd->active = pQueryInfo; return TSDB_CODE_SUCCESS; } @@ -2400,9 +2884,14 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) { tscTagCondRelease(&pQueryInfo->tagCond); tscFieldInfoClear(&pQueryInfo->fieldsInfo); - tscSqlExprInfoDestroy(pQueryInfo->exprList); + tscExprDestroy(pQueryInfo->exprList); pQueryInfo->exprList = NULL; + if (pQueryInfo->exprList1 != NULL) { + tscExprDestroy(pQueryInfo->exprList1); + pQueryInfo->exprList1 = NULL; + } + tscColumnListDestroy(pQueryInfo->colList); pQueryInfo->colList = NULL; @@ -2422,12 +2911,94 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) { } void tscClearSubqueryInfo(SSqlCmd* pCmd) { - for (int32_t i = 0; i < pCmd->numOfClause; ++i) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, i); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + while (pQueryInfo != NULL) { + SQueryInfo* p = pQueryInfo->sibling; freeQueryInfoImpl(pQueryInfo); + pQueryInfo = p; } } +int32_t tscQueryInfoCopy(SQueryInfo* pQueryInfo, const SQueryInfo* pSrc) { + assert(pQueryInfo != NULL && pSrc != NULL); + int32_t code = TSDB_CODE_SUCCESS; + + memcpy(&pQueryInfo->interval, &pSrc->interval, sizeof(pQueryInfo->interval)); + + pQueryInfo->command = pSrc->command; + pQueryInfo->type = pSrc->type; + pQueryInfo->window = pSrc->window; + pQueryInfo->limit = pSrc->limit; + pQueryInfo->slimit = pSrc->slimit; + pQueryInfo->order = pSrc->order; + pQueryInfo->vgroupLimit = pSrc->vgroupLimit; + pQueryInfo->tsBuf = NULL; + pQueryInfo->fillType = pSrc->fillType; + pQueryInfo->fillVal = NULL; + pQueryInfo->clauseLimit = pSrc->clauseLimit; + pQueryInfo->numOfTables = 0; + pQueryInfo->window = pSrc->window; + pQueryInfo->sessionWindow = pSrc->sessionWindow; + pQueryInfo->pTableMetaInfo = NULL; + + pQueryInfo->bufLen = pSrc->bufLen; + pQueryInfo->buf = malloc(pSrc->bufLen); + if (pQueryInfo->buf == NULL) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + + if (pSrc->bufLen > 0) { + memcpy(pQueryInfo->buf, pSrc->buf, pSrc->bufLen); + } + + pQueryInfo->groupbyExpr = pSrc->groupbyExpr; + if (pSrc->groupbyExpr.columnInfo != NULL) { + pQueryInfo->groupbyExpr.columnInfo = taosArrayDup(pSrc->groupbyExpr.columnInfo); + if (pQueryInfo->groupbyExpr.columnInfo == NULL) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + } + + if (tscTagCondCopy(&pQueryInfo->tagCond, &pSrc->tagCond) != 0) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + + if (pSrc->fillType != TSDB_FILL_NONE) { + pQueryInfo->fillVal = malloc(pSrc->fieldsInfo.numOfOutput * sizeof(int64_t)); + if (pQueryInfo->fillVal == NULL) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + + memcpy(pQueryInfo->fillVal, pSrc->fillVal, pSrc->fieldsInfo.numOfOutput * sizeof(int64_t)); + } + + if (tscExprCopyAll(pQueryInfo->exprList, pSrc->exprList, true) != 0) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + + tscColumnListCopyAll(pQueryInfo->colList, pSrc->colList); + tscFieldInfoCopy(&pQueryInfo->fieldsInfo, &pSrc->fieldsInfo, pQueryInfo->exprList); + + for(int32_t i = 0; i < pSrc->numOfTables; ++i) { + STableMetaInfo* p1 = tscGetMetaInfo((SQueryInfo*) pSrc, i); + + STableMeta* pMeta = tscTableMetaDup(p1->pTableMeta); + if (pMeta == NULL) { + // todo handle the error + } + + tscAddTableMetaInfo(pQueryInfo, &p1->name, pMeta, p1->vgroupList, p1->tagColList, p1->pVgroupTables); + } + + _error: + return code; +} + void tscFreeVgroupTableInfo(SArray* pVgroupTables) { if (pVgroupTables == NULL) { return; @@ -2501,28 +3072,29 @@ void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, bool removeMeta) { if (removeMeta) { char name[TSDB_TABLE_FNAME_LEN] = {0}; tNameExtractFullName(&pTableMetaInfo->name, name); - taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); } tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables); tscClearTableMetaInfo(pTableMetaInfo); + free(pTableMetaInfo); } - + tfree(pQueryInfo->pTableMetaInfo); } STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, SName* name, STableMeta* pTableMeta, SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables) { - void* pAlloc = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); - if (pAlloc == NULL) { + void* tmp = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); + if (tmp == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; } - pQueryInfo->pTableMetaInfo = pAlloc; + pQueryInfo->pTableMetaInfo = tmp; STableMetaInfo* pTableMetaInfo = calloc(1, sizeof(STableMetaInfo)); + if (pTableMetaInfo == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; @@ -2607,8 +3179,6 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, __async_cb_func_t fp, void* param, in SSqlCmd* pCmd = &pNew->cmd; pCmd->command = cmd; - pCmd->parseFinished = 1; - pCmd->autoCreated = pSql->cmd.autoCreated; int32_t code = copyTagData(&pNew->cmd.tagData, &pSql->cmd.tagData); if (code != TSDB_CODE_SUCCESS) { @@ -2632,10 +3202,8 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, __async_cb_func_t fp, void* param, in pNew->sqlstr = NULL; pNew->maxRetry = TSDB_MAX_REPLICA; - SQueryInfo* pQueryInfo = tscGetQueryInfoS(pCmd, 0); - - assert(pSql->cmd.clauseIndex == 0); - STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfoS(pCmd); + STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); tscAddTableMetaInfo(pQueryInfo, &pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL); registerSqlObj(pNew); @@ -2644,14 +3212,14 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, __async_cb_func_t fp, void* param, in } static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pNewQueryInfo, int64_t uid) { - int32_t numOfOutput = (int32_t)tscSqlExprNumOfExprs(pNewQueryInfo); + int32_t numOfOutput = (int32_t)tscNumOfExprs(pNewQueryInfo); if (numOfOutput == 0) { return; } // set the field info in pNewQueryInfo object according to sqlExpr information for (int32_t i = 0; i < numOfOutput; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pNewQueryInfo, i); + SExprInfo* pExpr = tscExprGet(pNewQueryInfo, i); TAOS_FIELD f = tscCreateField((int8_t) pExpr->base.resType, pExpr->base.aliasName, pExpr->base.resBytes); SInternalField* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, &f); @@ -2665,7 +3233,7 @@ static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pNewQueryInfo, int64_t ui bool matched = false; for (int32_t k1 = 0; k1 < numOfOutput; ++k1) { - SExprInfo* pExpr1 = tscSqlExprGet(pNewQueryInfo, k1); + SExprInfo* pExpr1 = tscExprGet(pNewQueryInfo, k1); if (strcmp(field->name, pExpr1->base.aliasName) == 0) { // establish link according to the result field name SInternalField* pInfo = tscFieldInfoGetInternalField(&pNewQueryInfo->fieldsInfo, f); @@ -2693,7 +3261,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t return NULL; } - SQueryInfo* pQueryInfo = tscGetActiveQueryInfo(pCmd); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[tableIndex]; pNew->pTscObj = pSql->pTscObj; @@ -2708,16 +3276,14 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t pnCmd->command = cmd; pnCmd->payload = NULL; pnCmd->allocSize = 0; + pnCmd->pTableMetaMap = NULL; pnCmd->pQueryInfo = NULL; - pnCmd->numOfClause = 0; - pnCmd->clauseIndex = 0; - pnCmd->pDataBlocks = NULL; + pnCmd->insertParam.pDataBlocks = NULL; - pnCmd->numOfTables = 0; - pnCmd->parseFinished = 1; - pnCmd->pTableNameList = NULL; - pnCmd->pTableBlockHashList = NULL; + pnCmd->insertParam.numOfTables = 0; + pnCmd->insertParam.pTableNameList = NULL; + pnCmd->insertParam.pTableBlockHashList = NULL; pnCmd->tagData.data = NULL; pnCmd->tagData.dataLen = 0; @@ -2726,8 +3292,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t goto _error; } - - SQueryInfo* pNewQueryInfo = tscGetQueryInfo(pnCmd, 0); + SQueryInfo* pNewQueryInfo = tscGetQueryInfo(pnCmd); if (pQueryInfo->pUdfInfo) { pNewQueryInfo->pUdfInfo = taosArrayDup(pQueryInfo->pUdfInfo); @@ -2751,7 +3316,6 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t pNewQueryInfo->pTableMetaInfo = NULL; pNewQueryInfo->bufLen = pQueryInfo->bufLen; pNewQueryInfo->globalMerge = false; - pNewQueryInfo->arithmCalOnAgg = false; pNewQueryInfo->buf = malloc(pQueryInfo->bufLen); if (pNewQueryInfo->buf == NULL) { @@ -2798,13 +3362,13 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t // set the correct query type if (pPrevSql != NULL) { - SQueryInfo* pPrevQueryInfo = tscGetQueryInfo(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex); + SQueryInfo* pPrevQueryInfo = tscGetQueryInfo(&pPrevSql->cmd); pNewQueryInfo->type = pPrevQueryInfo->type; } else { TSDB_QUERY_SET_TYPE(pNewQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY);// it must be the subquery } - if (tscSqlExprCopy(pNewQueryInfo->exprList, pQueryInfo->exprList, uid, true) != 0) { + if (tscExprCopy(pNewQueryInfo->exprList, pQueryInfo->exprList, uid, true) != 0) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; goto _error; } @@ -2826,7 +3390,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t pTableMetaInfo->tagColList, pTableMetaInfo->pVgroupTables); } else { // transfer the ownership of pTableMeta to the newly create sql object. - STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); + STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, 0); if (pPrevInfo->pTableMeta && pPrevInfo->pTableMeta->tableType < 0) { terrno = TSDB_CODE_TSC_APP_ERROR; goto _error; @@ -2865,7 +3429,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t tscDebug("0x%"PRIx64" new subquery:0x%"PRIx64", tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%" PRIzu ", colList:%" PRIzu "," "fieldInfo:%d, name:%s, qrang:%" PRId64 " - %" PRId64 " order:%d, limit:%" PRId64, - pSql->self, pNew->self, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo), + pSql->self, pNew->self, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscNumOfExprs(pNewQueryInfo), size, pNewQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pFinalInfo->name), pNewQueryInfo->window.skey, pNewQueryInfo->window.ekey, pNewQueryInfo->order.order, pNewQueryInfo->limit.limit); @@ -2900,6 +3464,58 @@ void doExecuteQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo) { } } +void doRetrieveSubqueryData(SSchedMsg *pMsg) { + SSqlObj* pSql = (SSqlObj*) pMsg->ahandle; + if (pSql == NULL || pSql->signature != pSql) { + tscDebug("%p SqlObj is freed, not add into queue async res", pMsg->ahandle); + return; + } + + SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); + handleDownstreamOperator(pSql->pSubs, pSql->subState.numOfSub, pQueryInfo, &pSql->res); + + pSql->res.qId = -1; + if (pSql->res.code == TSDB_CODE_SUCCESS) { + (*pSql->fp)(pSql->param, pSql, pSql->res.numOfRows); + } else { + tscAsyncResultOnError(pSql); + } +} + +// NOTE: the blocking query can not be executed in the rpc message handler thread +static void tscSubqueryRetrieveCallback(void* param, TAOS_RES* tres, int code) { + // handle the pDownStream process + SRetrieveSupport* ps = param; + SSqlObj* pParentSql = ps->pParentSql; + SSqlObj* pSql = tres; + + int32_t index = ps->subqueryIndex; + bool ret = subAndCheckDone(pSql, pParentSql, index); + + // TODO refactor + tfree(ps); + pSql->param = NULL; + + if (!ret) { + tscDebug("0x%"PRIx64" sub:0x%"PRIx64" orderOfSub:%d completed, not all subquery finished", pParentSql->self, pSql->self, index); + return; + } + + pParentSql->cmd.active = pParentSql->cmd.pQueryInfo; + + SSchedMsg schedMsg = {0}; + schedMsg.fp = doRetrieveSubqueryData; + schedMsg.ahandle = (void *)pParentSql; + schedMsg.thandle = (void *)1; + schedMsg.msg = 0; + taosScheduleTask(tscQhandle, &schedMsg); +} + +// todo handle the failure +static void tscSubqueryCompleteCallback(void* param, TAOS_RES* tres, int code) { + taos_fetch_rows_a(tres, tscSubqueryRetrieveCallback, param); +} + // do execute the query according to the query execution plan void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo) { if (pSql->cmd.command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { @@ -2912,14 +3528,50 @@ void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo) { } if (taosArrayGetSize(pQueryInfo->pUpstream) > 0) { // nest query. do execute it firstly - SQueryInfo* pq = taosArrayGetP(pQueryInfo->pUpstream, 0); + pSql->subState.numOfSub = (int32_t) taosArrayGetSize(pQueryInfo->pUpstream); - pSql->cmd.active = pq; - pSql->cmd.command = TSDB_SQL_SELECT; + pSql->pSubs = calloc(pSql->subState.numOfSub, POINTER_BYTES); + pSql->subState.states = calloc(pSql->subState.numOfSub, sizeof(int8_t)); - executeQuery(pSql, pq); + for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { + SQueryInfo* pSub = taosArrayGetP(pQueryInfo->pUpstream, i); - // merge nest query result and generate final results + pSql->cmd.active = pSub; + pSql->cmd.command = TSDB_SQL_SELECT; + + // TODO handle memory failure + SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); + if (pNew == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + // return NULL; + } + + pNew->pTscObj = pSql->pTscObj; + pNew->signature = pNew; + pNew->sqlstr = strdup(pSql->sqlstr); // todo refactor + pNew->fp = tscSubqueryCompleteCallback; + + SRetrieveSupport* ps = calloc(1, sizeof(SRetrieveSupport)); // todo use object id + ps->pParentSql = pSql; + ps->subqueryIndex = i; + + pNew->param = ps; + pSql->pSubs[i] = pNew; + registerSqlObj(pNew); + + SSqlCmd* pCmd = &pNew->cmd; + pCmd->command = TSDB_SQL_SELECT; + if (tscAddQueryInfo(pCmd) != TSDB_CODE_SUCCESS) { + } + + SQueryInfo* pNewQueryInfo = tscGetQueryInfo(pCmd); + tscQueryInfoCopy(pNewQueryInfo, pSub); + + // create sub query to handle the sub query. + executeQuery(pNew, pNewQueryInfo); + } + + // merge sub query result and generate final results return; } @@ -2927,71 +3579,6 @@ void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo) { doExecuteQuery(pSql, pQueryInfo); } -/** - * todo remove it - * To decide if current is a two-stage super table query, join query, or insert. And invoke different - * procedure accordingly - * @param pSql - */ -void tscDoQuery(SSqlObj* pSql) { - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - pRes->code = TSDB_CODE_SUCCESS; - - if (pCmd->command > TSDB_SQL_LOCAL) { - tscProcessLocalCmd(pSql); - return; - } - - if (pCmd->dataSourceType == DATA_FROM_DATA_FILE) { - tscImportDataFromFile(pSql); - } else { - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); - uint16_t type = pQueryInfo->type; - - if ((pCmd->command == TSDB_SQL_SELECT) && (!TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_SUBQUERY)) && (!TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_STABLE_SUBQUERY))) { - tscAddIntoSqlList(pSql); - } - - if (TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_INSERT)) { // multi-vnodes insertion - tscHandleMultivnodeInsert(pSql); - return; - } - - if (QUERY_IS_JOIN_QUERY(type)) { - if (!TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_SUBQUERY)) { - tscHandleMasterJoinQuery(pSql); - } else { // for first stage sub query, iterate all vnodes to get all timestamp - if (!TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)) { - tscBuildAndSendRequest(pSql, NULL); - } else { // secondary stage join query. - if (pQueryInfo->globalMerge) { // super table query - tscLockByThread(&pSql->squeryLock); - tscHandleMasterSTableQuery(pSql); - tscUnlockByThread(&pSql->squeryLock); - } else { - tscBuildAndSendRequest(pSql, NULL); - } - } - } - - return; - } else if (tscMultiRoundQuery(pQueryInfo, 0) && pQueryInfo->round == 0) { - tscHandleFirstRoundStableQuery(pSql); // todo lock? - return; - } else if (pQueryInfo->globalMerge) { // super table query - tscLockByThread(&pSql->squeryLock); - tscHandleMasterSTableQuery(pSql); - tscUnlockByThread(&pSql->squeryLock); - return; - } - - pCmd->active = pQueryInfo; - tscBuildAndSendRequest(pSql, NULL); - } -} - int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid) { int32_t i = 0; while (i < TSDB_MAX_JOIN_TABLE_NUM) { @@ -3047,17 +3634,15 @@ bool tscIsQueryWithLimit(SSqlObj* pSql) { } SSqlCmd* pCmd = &pSql->cmd; - for (int32_t i = 0; i < pCmd->numOfClause; ++i) { - SQueryInfo* pqi = tscGetQueryInfoS(pCmd, i); - if (pqi == NULL) { - continue; - } - + SQueryInfo* pqi = tscGetQueryInfo(pCmd); + while(pqi != NULL) { if (pqi->limit.limit > 0) { return true; } + + pqi = pqi->sibling; } - + return false; } @@ -3099,7 +3684,7 @@ int32_t tscInvalidSQLErrMsg(char* msg, const char* additionalInfo, const char* s if (sql == NULL) { assert(additionalInfo != NULL); sprintf(msg, msgFormat1, additionalInfo); - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } char buf[64] = {0}; // only extract part of sql string @@ -3111,7 +3696,7 @@ int32_t tscInvalidSQLErrMsg(char* msg, const char* additionalInfo, const char* s sprintf(msg, msgFormat3, buf); // no additional information for invalid sql error } - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } bool tscHasReachLimitation(SQueryInfo* pQueryInfo, SSqlRes* pRes) { @@ -3133,7 +3718,7 @@ bool hasMoreVnodesToTry(SSqlObj* pSql) { } assert(pRes->completed); - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); // for normal table, no need to try any more if results are all retrieved from one vnode @@ -3151,14 +3736,15 @@ bool hasMoreVnodesToTry(SSqlObj* pSql) { } bool hasMoreClauseToTry(SSqlObj* pSql) { - return pSql->cmd.clauseIndex < pSql->cmd.numOfClause - 1; + SSqlCmd* pCmd = &pSql->cmd; + return pCmd->active->sibling != NULL; } void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) { SSqlCmd* pCmd = &pSql->cmd; SSqlRes* pRes = &pSql->res; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); /* * no result returned from the current virtual node anymore, try the next vnode if exists @@ -3214,11 +3800,7 @@ void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp) { SSqlCmd* pCmd = &pSql->cmd; SSqlRes* pRes = &pSql->res; - // current subclause is completed, try the next subclause - assert(pCmd->clauseIndex < pCmd->numOfClause - 1); - - pCmd->clauseIndex++; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd, pCmd->clauseIndex); + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); pSql->cmd.command = pQueryInfo->command; @@ -3238,7 +3820,7 @@ void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp) { pSql->subState.numOfSub = 0; pSql->fp = fp; - tscDebug("0x%"PRIx64" try data in the next subclause:%d, total subclause:%d", pSql->self, pCmd->clauseIndex, pCmd->numOfClause); + tscDebug("0x%"PRIx64" try data in the next subclause", pSql->self); if (pCmd->command > TSDB_SQL_LOCAL) { tscProcessLocalCmd(pSql); } else { @@ -3455,22 +4037,25 @@ CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta) { assert(pTableMeta != NULL); CChildTableMeta* cMeta = calloc(1, sizeof(CChildTableMeta)); + cMeta->tableType = TSDB_CHILD_TABLE; - cMeta->vgId = pTableMeta->vgId; - cMeta->id = pTableMeta->id; + cMeta->vgId = pTableMeta->vgId; + cMeta->id = pTableMeta->id; + cMeta->suid = pTableMeta->suid; tstrncpy(cMeta->sTableName, pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); return cMeta; } -int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name, void* buf) { +int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, void* buf) { assert(pChild != NULL && buf != NULL); -// uint32_t size = tscGetTableMetaMaxSize(); - STableMeta* p = buf;//calloc(1, size); - + STableMeta* p = buf; taosHashGetClone(tscTableMetaInfo, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, p, -1); - if (p->id.uid > 0) { // tableMeta exists, build child table meta and return + + // tableMeta exists, build child table meta according to the super table meta + // the uid need to be checked in addition to the general name of the super table. + if (p->id.uid > 0 && pChild->suid == p->id.uid) { pChild->sversion = p->sversion; pChild->tversion = p->tversion; @@ -3478,13 +4063,9 @@ int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name, v int32_t total = pChild->tableInfo.numOfColumns + pChild->tableInfo.numOfTags; memcpy(pChild->schema, p->schema, sizeof(SSchema) *total); - -// tfree(p); return TSDB_CODE_SUCCESS; } else { // super table has been removed, current tableMeta is also expired. remove it here taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); - -// tfree(p); return -1; } } @@ -3495,81 +4076,94 @@ uint32_t tscGetTableMetaMaxSize() { STableMeta* tscTableMetaDup(STableMeta* pTableMeta) { assert(pTableMeta != NULL); - uint32_t size = tscGetTableMetaSize(pTableMeta); + size_t size = tscGetTableMetaSize(pTableMeta); + STableMeta* p = calloc(1, size); memcpy(p, pTableMeta, size); return p; } -static int32_t createSecondaryExpr(SQueryAttr* pQueryAttr, SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo) { - if (!pQueryInfo->arithmCalOnAgg) { +SVgroupsInfo* tscVgroupsInfoDup(SVgroupsInfo* pVgroupsInfo) { + assert(pVgroupsInfo != NULL); + + size_t size = sizeof(SVgroupInfo) * pVgroupsInfo->numOfVgroups + sizeof(SVgroupsInfo); + SVgroupsInfo* pInfo = calloc(1, size); + memcpy(pInfo, pVgroupsInfo, size); + return pInfo; +} + +int32_t createProjectionExpr(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SExprInfo*** pExpr, int32_t* num) { + if (!pQueryInfo->arithmeticOnAgg) { return TSDB_CODE_SUCCESS; } - pQueryAttr->numOfExpr2 = tscNumOfFields(pQueryInfo); - pQueryAttr->pExpr2 = calloc(pQueryAttr->numOfExpr2, sizeof(SExprInfo)); - if (pQueryAttr->pExpr2 == NULL) { + *num = tscNumOfFields(pQueryInfo); + *pExpr = calloc(*(num), POINTER_BYTES); + if ((*pExpr) == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - for (int32_t i = 0; i < pQueryAttr->numOfExpr2; ++i) { + for (int32_t i = 0; i < (*num); ++i) { SInternalField* pField = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, i); - SExprInfo* pExpr = pField->pExpr; + SExprInfo* pSource = pField->pExpr; - SSqlExpr *pse = &pQueryAttr->pExpr2[i].base; + SExprInfo* px = calloc(1, sizeof(SExprInfo)); + (*pExpr)[i] = px; + + SSqlExpr *pse = &px->base; pse->uid = pTableMetaInfo->pTableMeta->id.uid; - pse->resColId = pExpr->base.resColId; + pse->resColId = pSource->base.resColId; + strncpy(pse->aliasName, pSource->base.aliasName, tListLen(pse->aliasName)); + strncpy(pse->token, pSource->base.token, tListLen(pse->token)); - if (pExpr->base.functionId != TSDB_FUNC_ARITHM) { // this should be switched to projection query + if (pSource->base.functionId != TSDB_FUNC_ARITHM) { // this should be switched to projection query pse->numOfParams = 0; // no params for projection query pse->functionId = TSDB_FUNC_PRJ; - pse->colInfo.colId = pExpr->base.resColId; + pse->colInfo.colId = pSource->base.resColId; - for (int32_t j = 0; j < pQueryAttr->numOfOutput; ++j) { - if (pQueryAttr->pExpr1[j].base.resColId == pse->colInfo.colId) { + int32_t numOfOutput = (int32_t) taosArrayGetSize(pQueryInfo->exprList); + for (int32_t j = 0; j < numOfOutput; ++j) { + SExprInfo* p = taosArrayGetP(pQueryInfo->exprList, j); + if (p->base.resColId == pse->colInfo.colId) { pse->colInfo.colIndex = j; + break; } } pse->colInfo.flag = TSDB_COL_NORMAL; - pse->resType = pExpr->base.resType; - pse->resBytes = pExpr->base.resBytes; + pse->resType = pSource->base.resType; + pse->resBytes = pSource->base.resBytes; + strncpy(pse->colInfo.name, pSource->base.aliasName, tListLen(pse->colInfo.name)); // TODO restore refactor - int32_t functionId = pExpr->base.functionId; - if (pExpr->base.functionId == TSDB_FUNC_FIRST_DST) { + int32_t functionId = pSource->base.functionId; + if (pSource->base.functionId == TSDB_FUNC_FIRST_DST) { functionId = TSDB_FUNC_FIRST; - } else if (pExpr->base.functionId == TSDB_FUNC_LAST_DST) { + } else if (pSource->base.functionId == TSDB_FUNC_LAST_DST) { functionId = TSDB_FUNC_LAST; - } else if (pExpr->base.functionId == TSDB_FUNC_STDDEV_DST) { + } else if (pSource->base.functionId == TSDB_FUNC_STDDEV_DST) { functionId = TSDB_FUNC_STDDEV; } - SUdfInfo* pUdfInfo = NULL; - - if (functionId < 0) { - pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); - } - int32_t inter = 0; - getResultDataInfo(pExpr->base.colType, pExpr->base.colBytes, functionId, 0, &pse->resType, - &pse->resBytes, &inter, 0, false, pUdfInfo); + getResultDataInfo(pSource->base.colType, pSource->base.colBytes, functionId, 0, &pse->resType, + &pse->resBytes, &inter, 0, false, NULL); pse->colType = pse->resType; pse->colBytes = pse->resBytes; } else { // arithmetic expression - pse->colInfo.colId = pExpr->base.colInfo.colId; - pse->colType = pExpr->base.colType; - pse->colBytes = pExpr->base.colBytes; + pse->colInfo.colId = pSource->base.colInfo.colId; + pse->colType = pSource->base.colType; + pse->colBytes = pSource->base.colBytes; pse->resBytes = sizeof(double); pse->resType = TSDB_DATA_TYPE_DOUBLE; - pse->functionId = pExpr->base.functionId; - pse->numOfParams = pExpr->base.numOfParams; + pse->functionId = pSource->base.functionId; + pse->numOfParams = pSource->base.numOfParams; - for (int32_t j = 0; j < pExpr->base.numOfParams; ++j) { - tVariantAssign(&pse->param[j], &pExpr->base.param[j]); - buildArithmeticExprFromMsg(&pQueryAttr->pExpr2[i], NULL); + for (int32_t j = 0; j < pSource->base.numOfParams; ++j) { + tVariantAssign(&pse->param[j], &pSource->base.param[j]); + buildArithmeticExprFromMsg(px, NULL); } } } @@ -3580,7 +4174,7 @@ static int32_t createSecondaryExpr(SQueryAttr* pQueryAttr, SQueryInfo* pQueryInf static int32_t createGlobalAggregateExpr(SQueryAttr* pQueryAttr, SQueryInfo* pQueryInfo) { assert(pQueryInfo->globalMerge); - pQueryAttr->numOfExpr3 = (int32_t) tscSqlExprNumOfExprs(pQueryInfo); + pQueryAttr->numOfExpr3 = (int32_t) tscNumOfExprs(pQueryInfo); pQueryAttr->pExpr3 = calloc(pQueryAttr->numOfExpr3, sizeof(SExprInfo)); if (pQueryAttr->pExpr3 == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -3590,7 +4184,7 @@ static int32_t createGlobalAggregateExpr(SQueryAttr* pQueryAttr, SQueryInfo* pQu SExprInfo* pExpr = &pQueryAttr->pExpr1[i]; SSqlExpr* pse = &pQueryAttr->pExpr3[i].base; - tscSqlExprAssign(&pQueryAttr->pExpr3[i], pExpr); + tscExprAssign(&pQueryAttr->pExpr3[i], pExpr); pse->colInfo.colId = pExpr->base.resColId; pse->colInfo.colIndex = i; @@ -3621,7 +4215,7 @@ static int32_t createGlobalAggregateExpr(SQueryAttr* pQueryAttr, SQueryInfo* pQu } SUdfInfo* pUdfInfo = NULL; - + if (functionId < 0) { pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); } @@ -3660,7 +4254,7 @@ static int32_t createTagColumnInfo(SQueryAttr* pQueryAttr, SQueryInfo* pQueryInf if ((pCol->columnIndex >= numOfTagColumns || pCol->columnIndex < TSDB_TBNAME_COLUMN_INDEX) || (!isValidDataType(pColSchema->type))) { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } SColumnInfo* pTagCol = &pQueryAttr->tagColList[i]; @@ -3678,7 +4272,7 @@ int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAt memset(pQueryAttr, 0, sizeof(SQueryAttr)); int16_t numOfCols = (int16_t) taosArrayGetSize(pQueryInfo->colList); - int16_t numOfOutput = (int16_t) tscSqlExprNumOfExprs(pQueryInfo); + int16_t numOfOutput = (int16_t) tscNumOfExprs(pQueryInfo); pQueryAttr->topBotQuery = tscIsTopBotQuery(pQueryInfo); pQueryAttr->hasTagResults = hasTagValOutput(pQueryInfo); @@ -3702,7 +4296,7 @@ int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAt pQueryAttr->groupbyColumn = tscGroupbyColumn(pQueryInfo); pQueryAttr->havingNum = pQueryInfo->havingFieldNum; pQueryAttr->pUdfInfo = pQueryInfo->pUdfInfo; - + if (pQueryInfo->order.order == TSDB_ORDER_ASC) { // TODO refactor pQueryAttr->window = pQueryInfo->window; } else { @@ -3714,7 +4308,7 @@ int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAt STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; - pQueryAttr->pGroupbyExpr = calloc(1, sizeof(SSqlGroupbyExpr)); + pQueryAttr->pGroupbyExpr = calloc(1, sizeof(SGroupbyExpr)); *(pQueryAttr->pGroupbyExpr) = pQueryInfo->groupbyExpr; if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) { @@ -3725,8 +4319,8 @@ int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAt pQueryAttr->pExpr1 = calloc(pQueryAttr->numOfOutput, sizeof(SExprInfo)); for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - SExprInfo* pExpr = tscSqlExprGet(pQueryInfo, i); - tscSqlExprAssign(&pQueryAttr->pExpr1[i], pExpr); + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); + tscExprAssign(&pQueryAttr->pExpr1[i], pExpr); if (pQueryAttr->pExpr1[i].base.functionId == TSDB_FUNC_ARITHM) { for (int32_t j = 0; j < pQueryAttr->pExpr1[i].base.numOfParams; ++j) { @@ -3752,13 +4346,17 @@ int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAt } // for simple table, not for super table - int32_t code = createSecondaryExpr(pQueryAttr, pQueryInfo, pTableMetaInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; + if (pQueryInfo->arithmeticOnAgg) { + pQueryAttr->numOfExpr2 = (int32_t) taosArrayGetSize(pQueryInfo->exprList1); + pQueryAttr->pExpr2 = calloc(pQueryAttr->numOfExpr2, sizeof(SExprInfo)); + for(int32_t i = 0; i < pQueryAttr->numOfExpr2; ++i) { + SExprInfo* p = taosArrayGetP(pQueryInfo->exprList1, i); + tscExprAssign(&pQueryAttr->pExpr2[i], p); + } } // tag column info - code = createTagColumnInfo(pQueryAttr, pQueryInfo, pTableMetaInfo); + int32_t code = createTagColumnInfo(pQueryAttr, pQueryInfo, pTableMetaInfo); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3783,19 +4381,106 @@ int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAt tscError("%p illegal value of numOfCols in query msg: %" PRIu64 ", table cols:%d", addr, (uint64_t)pQueryAttr->numOfCols, numOfCols); - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } if (pQueryAttr->interval.interval < 0) { tscError("%p illegal value of aggregation time interval in query msg: %" PRId64, addr, (int64_t)pQueryInfo->interval.interval); - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } if (pQueryAttr->pGroupbyExpr->numOfGroupCols < 0) { tscError("%p illegal value of numOfGroupCols in query msg: %d", addr, pQueryInfo->groupbyExpr.numOfGroupCols); - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } return TSDB_CODE_SUCCESS; } + +static int32_t doAddTableName(char* nextStr, char** str, SArray* pNameArray, SSqlObj* pSql) { + int32_t code = TSDB_CODE_SUCCESS; + SSqlCmd* pCmd = &pSql->cmd; + + char tablename[TSDB_TABLE_FNAME_LEN] = {0}; + int32_t len = 0; + + if (nextStr == NULL) { + strncpy(tablename, *str, TSDB_TABLE_FNAME_LEN); + len = (int32_t) strlen(tablename); + } else { + memcpy(tablename, *str, nextStr - (*str)); + len = (int32_t)(nextStr - (*str)); + tablename[len] = '\0'; + } + + (*str) = nextStr + 1; + len = (int32_t)strtrim(tablename); + + SStrToken sToken = {.n = len, .type = TK_ID, .z = tablename}; + tGetToken(tablename, &sToken.type); + + // Check if the table name available or not + if (tscValidateName(&sToken) != TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + sprintf(pCmd->payload, "table name is invalid"); + return code; + } + + SName name = {0}; + if ((code = tscSetTableFullName(&name, &sToken, pSql)) != TSDB_CODE_SUCCESS) { + return code; + } + + memset(tablename, 0, tListLen(tablename)); + tNameExtractFullName(&name, tablename); + + char* p = strdup(tablename); + taosArrayPush(pNameArray, &p); + return TSDB_CODE_SUCCESS; +} + +int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t length, SArray* pNameArray) { + SSqlCmd *pCmd = &pSql->cmd; + + pCmd->command = TSDB_SQL_MULTI_META; + pCmd->msgType = TSDB_MSG_TYPE_CM_TABLES_META; + + int code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + char *str = (char *)pNameList; + + SQueryInfo *pQueryInfo = tscGetQueryInfoS(pCmd); + if (pQueryInfo == NULL) { + pSql->res.code = terrno; + return terrno; + } + + char *nextStr; + while (1) { + nextStr = strchr(str, ','); + if (nextStr == NULL) { + code = doAddTableName(nextStr, &str, pNameArray, pSql); + break; + } + + code = doAddTableName(nextStr, &str, pNameArray, pSql); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (taosArrayGetSize(pNameArray) > TSDB_MULTI_TABLEMETA_MAX_NUM) { + code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + sprintf(pCmd->payload, "tables over the max number"); + return code; + } + } + + if (taosArrayGetSize(pNameArray) > TSDB_MULTI_TABLEMETA_MAX_NUM) { + code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + sprintf(pCmd->payload, "tables over the max number"); + return code; + } + + return TSDB_CODE_SUCCESS; +} + diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 02bc1c6735..801cb32568 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -242,6 +242,7 @@ typedef struct SDataCol { int len; // column data length VarDataOffsetT *dataOff; // For binary and nchar data, the offset in the data column void * pData; // Actual data pointer + TSKEY ts; // only used in last NULL column } SDataCol; static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; } diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 2f4aa4c2b2..e07c3611d7 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -142,12 +142,15 @@ extern int32_t tsMonitorInterval; extern int8_t tsEnableStream; // internal +extern int8_t tsCompactMnodeWal; extern int8_t tsPrintAuth; extern int8_t tscEmbedded; extern char configDir[]; extern char tsVnodeDir[]; extern char tsDnodeDir[]; extern char tsMnodeDir[]; +extern char tsMnodeBakDir[]; +extern char tsMnodeTmpDir[]; extern char tsDataDir[]; extern char tsLogDir[]; extern char tsScriptDir[]; diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h index 48bec7fe4d..8c97059552 100644 --- a/src/common/inc/tname.h +++ b/src/common/inc/tname.h @@ -44,8 +44,8 @@ typedef struct SResPair { // the structure for sql function in select clause typedef struct SSqlExpr { char aliasName[TSDB_COL_NAME_LEN]; // as aliasName + char token[TSDB_COL_NAME_LEN]; // original token SColIndex colInfo; - uint64_t uid; // refactor use the pointer int16_t functionId; // function id in aAgg array @@ -92,10 +92,6 @@ size_t tableIdPrefix(const char* name, char* prefix, int32_t len); void extractTableNameFromToken(SStrToken *pToken, SStrToken* pTable); -//SSchema tGetTbnameColumnSchema(); - -SSchema tGetBlockDistColumnSchema(); - SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const char* name); bool tscValidateTableNameLength(size_t len); diff --git a/src/common/src/tarithoperator.c b/src/common/src/tarithoperator.c index 1cb667d259..b37e358b9c 100644 --- a/src/common/src/tarithoperator.c +++ b/src/common/src/tarithoperator.c @@ -2569,6 +2569,7 @@ _arithmetic_operator_fn_t getArithmeticOperatorFn(int32_t arithmeticOptr) { case TSDB_BINARY_OP_REMAINDER: return vectorRemainder; default: + assert(0); return NULL; } } diff --git a/src/common/src/texpr.c b/src/common/src/texpr.c index 4334a0de26..f2dd3890a1 100644 --- a/src/common/src/texpr.c +++ b/src/common/src/texpr.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include #include "os.h" #include "texpr.h" @@ -465,27 +466,29 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { return expr; } -tExprNode* exprdup(tExprNode* pTree) { - if (pTree == NULL) { +tExprNode* exprdup(tExprNode* pNode) { + if (pNode == NULL) { return NULL; } - tExprNode* pNode = calloc(1, sizeof(tExprNode)); - if (pTree->nodeType == TSQL_NODE_EXPR) { - tExprNode* pLeft = exprdup(pTree->_node.pLeft); - tExprNode* pRight = exprdup(pTree->_node.pRight); + tExprNode* pCloned = calloc(1, sizeof(tExprNode)); + if (pNode->nodeType == TSQL_NODE_EXPR) { + tExprNode* pLeft = exprdup(pNode->_node.pLeft); + tExprNode* pRight = exprdup(pNode->_node.pRight); - pNode->nodeType = TSQL_NODE_EXPR; - pNode->_node.pLeft = pLeft; - pNode->_node.pRight = pRight; - } else if (pTree->nodeType == TSQL_NODE_VALUE) { - pNode->pVal = calloc(1, sizeof(tVariant)); - tVariantAssign(pNode->pVal, pTree->pVal); - } else if (pTree->nodeType == TSQL_NODE_COL) { - pNode->pSchema = calloc(1, sizeof(SSchema)); - *pNode->pSchema = *pTree->pSchema; + pCloned->_node.pLeft = pLeft; + pCloned->_node.pRight = pRight; + pCloned->_node.optr = pNode->_node.optr; + pCloned->_node.hasPK = pNode->_node.hasPK; + } else if (pNode->nodeType == TSQL_NODE_VALUE) { + pCloned->pVal = calloc(1, sizeof(tVariant)); + tVariantAssign(pCloned->pVal, pNode->pVal); + } else if (pNode->nodeType == TSQL_NODE_COL) { + pCloned->pSchema = calloc(1, sizeof(SSchema)); + *pCloned->pSchema = *pNode->pSchema; } - return pNode; + pCloned->nodeType = pNode->nodeType; + return pCloned; } diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index db97c3a5af..d6bbc288ad 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -176,12 +176,15 @@ int32_t tsMonitorInterval = 30; // seconds int8_t tsEnableStream = 1; // internal +int8_t tsCompactMnodeWal = 0; int8_t tsPrintAuth = 0; int8_t tscEmbedded = 0; char configDir[TSDB_FILENAME_LEN] = {0}; char tsVnodeDir[TSDB_FILENAME_LEN] = {0}; char tsDnodeDir[TSDB_FILENAME_LEN] = {0}; char tsMnodeDir[TSDB_FILENAME_LEN] = {0}; +char tsMnodeTmpDir[TSDB_FILENAME_LEN] = {0}; +char tsMnodeBakDir[TSDB_FILENAME_LEN] = {0}; char tsDataDir[TSDB_FILENAME_LEN] = {0}; char tsScriptDir[TSDB_FILENAME_LEN] = {0}; char tsTempDir[TSDB_FILENAME_LEN] = "/tmp/"; diff --git a/src/common/src/tname.c b/src/common/src/tname.c index dc868d8057..c1c6ffa4b3 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -33,15 +33,6 @@ size_t tableIdPrefix(const char* name, char* prefix, int32_t len) { return strlen(prefix); } -SSchema tGetBlockDistColumnSchema() { - SSchema s = {0}; - s.bytes = TSDB_MAX_BINARY_LEN;; - s.type = TSDB_DATA_TYPE_BINARY; - s.colId = TSDB_BLOCK_DIST_COLUMN_INDEX; - tstrncpy(s.name, TSQL_BLOCK_DIST_L, TSDB_COL_NAME_LEN); - return s; -} - SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const char* name) { SSchema s = {0}; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index d6934b8e46..179bad2ced 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -310,6 +310,16 @@ public class TSDBJNIConnector { private native int setBindTableNameImp(long stmt, String name, long conn); + public void setBindTableNameAndTags(long stmt, String tableName, int numOfTags, ByteBuffer tags, ByteBuffer typeList, ByteBuffer lengthList, ByteBuffer nullList) throws SQLException { + int code = setTableNameTagsImp(stmt, tableName, numOfTags, tags.array(), typeList.array(), lengthList.array(), + nullList.array(), this.taos); + if (code != TSDBConstants.JNI_SUCCESS) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "failed to bind table name and corresponding tags"); + } + } + + private native int setTableNameTagsImp(long stmt, String name, int numOfTags, byte[] tags, byte[] typeList, byte[] lengthList, byte[] nullList, long conn); + public void bindColumnDataArray(long stmt, ByteBuffer colDataList, ByteBuffer lengthList, ByteBuffer isNullList, int type, int bytes, int numOfRows,int columnIndex) throws SQLException { int code = bindColDataImp(stmt, colDataList.array(), lengthList.array(), isNullList.array(), type, bytes, numOfRows, columnIndex, this.taos); if (code != TSDBConstants.JNI_SUCCESS) { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java index 71e07252a3..f6810237c0 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java @@ -41,6 +41,9 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat private boolean isPrepared; private ArrayList colData; + private ArrayList tableTags; + private int tagValueLength; + private String tableName; private long nativeStmtHandle = 0; @@ -63,8 +66,8 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat if (parameterCnt > 1) { // the table name is also a parameter, so ignore it. - this.colData = new ArrayList(parameterCnt - 1); - this.colData.addAll(Collections.nCopies(parameterCnt - 1, null)); + this.colData = new ArrayList(); + this.tableTags = new ArrayList(); } } @@ -562,11 +565,109 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat } }; + private static class TableTagInfo { + private boolean isNull; + private Object value; + private int type; + public TableTagInfo(Object value, int type) { + this.value = value; + this.type = type; + } + + public static TableTagInfo createNullTag(int type) { + TableTagInfo info = new TableTagInfo(null, type); + info.isNull = true; + return info; + } + }; + public void setTableName(String name) { this.tableName = name; } + private void ensureTagCapacity(int index) { + if (this.tableTags.size() < index + 1) { + int delta = index + 1 - this.tableTags.size(); + this.tableTags.addAll(Collections.nCopies(delta, null)); + } + } + + public void setTagNull(int index, int type) { + ensureTagCapacity(index); + this.tableTags.set(index, TableTagInfo.createNullTag(type)); + } + + public void setTagBoolean(int index, boolean value) { + ensureTagCapacity(index); + this.tableTags.set(index, new TableTagInfo(value, TSDBConstants.TSDB_DATA_TYPE_BOOL)); + this.tagValueLength += Byte.BYTES; + } + + public void setTagInt(int index, int value) { + ensureTagCapacity(index); + this.tableTags.set(index, new TableTagInfo(value, TSDBConstants.TSDB_DATA_TYPE_INT)); + this.tagValueLength += Integer.BYTES; + } + + public void setTagByte(int index, byte value) { + ensureTagCapacity(index); + this.tableTags.set(index, new TableTagInfo(value, TSDBConstants.TSDB_DATA_TYPE_TINYINT)); + this.tagValueLength += Byte.BYTES; + } + + public void setTagShort(int index, short value) { + ensureTagCapacity(index); + this.tableTags.set(index, new TableTagInfo(value, TSDBConstants.TSDB_DATA_TYPE_SMALLINT)); + this.tagValueLength += Short.BYTES; + } + + public void setTagLong(int index, long value) { + ensureTagCapacity(index); + this.tableTags.set(index, new TableTagInfo(value, TSDBConstants.TSDB_DATA_TYPE_BIGINT)); + this.tagValueLength += Long.BYTES; + } + + public void setTagTimestamp(int index, long value) { + ensureTagCapacity(index); + this.tableTags.set(index, new TableTagInfo(value, TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP)); + this.tagValueLength += Long.BYTES; + } + + public void setTagFloat(int index, float value) { + ensureTagCapacity(index); + this.tableTags.set(index, new TableTagInfo(value, TSDBConstants.TSDB_DATA_TYPE_FLOAT)); + this.tagValueLength += Float.BYTES; + } + + public void setTagDouble(int index, double value) { + ensureTagCapacity(index); + this.tableTags.set(index, new TableTagInfo(value, TSDBConstants.TSDB_DATA_TYPE_DOUBLE)); + this.tagValueLength += Double.BYTES; + } + + public void setTagString(int index, String value) { + ensureTagCapacity(index); + this.tableTags.set(index, new TableTagInfo(value, TSDBConstants.TSDB_DATA_TYPE_BINARY)); + this.tagValueLength += value.getBytes().length; + } + + public void setTagNString(int index, String value) { + ensureTagCapacity(index); + this.tableTags.set(index, new TableTagInfo(value, TSDBConstants.TSDB_DATA_TYPE_NCHAR)); + + String charset = TaosGlobalConfig.getCharset(); + try { + this.tagValueLength += value.getBytes(charset).length; + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + } + public void setValueImpl(int columnIndex, ArrayList list, int type, int bytes) throws SQLException { + if (this.colData.size() == 0) { + this.colData.addAll(Collections.nCopies(this.parameters.length - 1 - this.tableTags.size(), null)); + + } ColumnInfo col = (ColumnInfo) this.colData.get(columnIndex); if (col == null) { ColumnInfo p = new ColumnInfo(); @@ -641,7 +742,122 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat TSDBJNIConnector connector = ((TSDBConnection) this.getConnection()).getConnector(); this.nativeStmtHandle = connector.prepareStmt(rawSql); - connector.setBindTableName(this.nativeStmtHandle, this.tableName); + + if (this.tableTags == null) { + connector.setBindTableName(this.nativeStmtHandle, this.tableName); + } else { + int num = this.tableTags.size(); + ByteBuffer tagDataList = ByteBuffer.allocate(this.tagValueLength); + tagDataList.order(ByteOrder.LITTLE_ENDIAN); + + ByteBuffer typeList = ByteBuffer.allocate(num); + typeList.order(ByteOrder.LITTLE_ENDIAN); + + ByteBuffer lengthList = ByteBuffer.allocate(num * Long.BYTES); + lengthList.order(ByteOrder.LITTLE_ENDIAN); + + ByteBuffer isNullList = ByteBuffer.allocate(num * Integer.BYTES); + isNullList.order(ByteOrder.LITTLE_ENDIAN); + + for (int i = 0; i < num; ++i) { + TableTagInfo tag = this.tableTags.get(i); + if (tag.isNull) { + typeList.put((byte) tag.type); + isNullList.putInt(1); + lengthList.putLong(0); + continue; + } + + switch (tag.type) { + case TSDBConstants.TSDB_DATA_TYPE_INT: { + Integer val = (Integer) tag.value; + tagDataList.putInt(val); + lengthList.putLong(Integer.BYTES); + break; + } + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: { + Byte val = (Byte) tag.value; + tagDataList.put(val); + lengthList.putLong(Byte.BYTES); + break; + } + + case TSDBConstants.TSDB_DATA_TYPE_BOOL: { + Boolean val = (Boolean) tag.value; + tagDataList.put((byte) (val ? 1 : 0)); + lengthList.putLong(Byte.BYTES); + break; + } + + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: { + Short val = (Short) tag.value; + tagDataList.putShort(val); + lengthList.putLong(Short.BYTES); + + break; + } + + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: { + Long val = (Long) tag.value; + tagDataList.putLong(val == null ? 0 : val); + lengthList.putLong(Long.BYTES); + + break; + } + + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: { + Float val = (Float) tag.value; + tagDataList.putFloat(val == null ? 0 : val); + lengthList.putLong(Float.BYTES); + + break; + } + + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: { + Double val = (Double) tag.value; + tagDataList.putDouble(val == null ? 0 : val); + lengthList.putLong(Double.BYTES); + + break; + } + + case TSDBConstants.TSDB_DATA_TYPE_NCHAR: + case TSDBConstants.TSDB_DATA_TYPE_BINARY: { + String charset = TaosGlobalConfig.getCharset(); + String val = (String) tag.value; + + byte[] b = null; + try { + if (tag.type == TSDBConstants.TSDB_DATA_TYPE_BINARY) { + b = val.getBytes(); + } else { + b = val.getBytes(charset); + } + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + + tagDataList.put(b); + lengthList.putLong(b.length); + break; + } + + case TSDBConstants.TSDB_DATA_TYPE_UTINYINT: + case TSDBConstants.TSDB_DATA_TYPE_USMALLINT: + case TSDBConstants.TSDB_DATA_TYPE_UINT: + case TSDBConstants.TSDB_DATA_TYPE_UBIGINT: { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN, "not support data types"); + } + } + + typeList.put((byte) tag.type); + isNullList.putInt(tag.isNull? 1 : 0); + } + + connector.setBindTableNameAndTags(this.nativeStmtHandle, this.tableName, this.tableTags.size(), tagDataList, + typeList, lengthList, isNullList); + } ColumnInfo colInfo = (ColumnInfo) this.colData.get(0); if (colInfo == null) { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java index 8804cc5da0..52858e7f88 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java @@ -207,10 +207,69 @@ public class TSDBPreparedStatementTest { while(rs.next()) { rows++; } - Assert.assertEquals(numOfRows, rows); + Assert.assertEquals(numOfRows, rows); } } + @Test + public void bindDataSelectColumnTest() throws SQLException { + Statement stmt = conn.createStatement(); + + int numOfRows = 1000; + + for (int loop = 0; loop < 10; loop++){ + stmt.execute("drop table if exists weather_test"); + stmt.execute("create table weather_test(ts timestamp, f1 nchar(4), f2 float, f3 double, f4 timestamp, f5 int, f6 bool, f7 binary(10))"); + + TSDBPreparedStatement s = (TSDBPreparedStatement) conn.prepareStatement("insert into ? (ts, f1, f7) values(?, ?, ?)"); + Random r = new Random(); + s.setTableName("weather_test"); + + ArrayList ts = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + ts.add(System.currentTimeMillis() + i); + } + s.setTimestamp(0, ts); + + int random = 10 + r.nextInt(5); + ArrayList s2 = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + if(i % random == 0) { + s2.add(null); + }else{ + s2.add("分支" + i % 4); + } + } + s.setNString(1, s2, 4); + + random = 10 + r.nextInt(5); + ArrayList s5 = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + if(i % random == 0) { + s5.add(null); + }else{ + s5.add("test" + i % 10); + } + } + s.setString(2, s5, 10); + + s.columnDataAddBatch(); + s.columnDataExecuteBatch(); + s.columnDataCloseBatch(); + + String sql = "select * from weather_test"; + PreparedStatement statement = conn.prepareStatement(sql); + ResultSet rs = statement.executeQuery(); + int rows = 0; + while(rs.next()) { + rows++; + } + Assert.assertEquals(numOfRows, rows); + } + } + + + @Test public void setBoolean() throws SQLException { pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); diff --git a/src/connector/nodejs/nodetaos/cinterface.js b/src/connector/nodejs/nodetaos/cinterface.js index 43a08a800a..f3961e3787 100644 --- a/src/connector/nodejs/nodetaos/cinterface.js +++ b/src/connector/nodejs/nodetaos/cinterface.js @@ -9,7 +9,7 @@ const ffi = require('ffi-napi'); const ArrayType = require('ref-array-napi'); const Struct = require('ref-struct-napi'); const FieldTypes = require('./constants'); -const errors = require ('./error'); +const errors = require('./error'); const TaosObjects = require('./taosobjects'); const { NULL_POINTER } = require('ref-napi'); @@ -22,7 +22,7 @@ function convertMicrosecondsToDatetime(time) { return new TaosObjects.TaosTimestamp(time * 0.001, true); } -function convertTimestamp(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertTimestamp(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { timestampConverter = convertMillisecondsToDatetime; if (micro == true) { timestampConverter = convertMicrosecondsToDatetime; @@ -44,14 +44,14 @@ function convertTimestamp(data, num_of_rows, nbytes = 0, offset = 0, micro=false } return res; } -function convertBool(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertBool(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = new Array(data.length); for (let i = 0; i < data.length; i++) { if (data[i] == 0) { res[i] = false; } - else if (data[i] == 1){ + else if (data[i] == 1) { res[i] = true; } else if (data[i] == FieldTypes.C_BOOL_NULL) { @@ -60,29 +60,29 @@ function convertBool(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertTinyint(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertTinyint(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; while (currOffset < data.length) { - let d = data.readIntLE(currOffset,1); + let d = data.readIntLE(currOffset, 1); res.push(d == FieldTypes.C_TINYINT_NULL ? null : d); currOffset += nbytes; } return res; } -function convertSmallint(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertSmallint(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; while (currOffset < data.length) { - let d = data.readIntLE(currOffset,2); + let d = data.readIntLE(currOffset, 2); res.push(d == FieldTypes.C_SMALLINT_NULL ? null : d); currOffset += nbytes; } return res; } -function convertInt(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertInt(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -93,7 +93,7 @@ function convertInt(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertBigint(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertBigint(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -104,7 +104,7 @@ function convertBigint(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertFloat(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertFloat(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -115,7 +115,7 @@ function convertFloat(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertDouble(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertDouble(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -126,7 +126,7 @@ function convertDouble(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -142,7 +142,7 @@ function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let dataEntry = data.slice(0, nbytes); //one entry in a row under a column; @@ -153,23 +153,23 @@ function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { // Object with all the relevant converters from pblock data to javascript readable data let convertFunctions = { - [FieldTypes.C_BOOL] : convertBool, - [FieldTypes.C_TINYINT] : convertTinyint, - [FieldTypes.C_SMALLINT] : convertSmallint, - [FieldTypes.C_INT] : convertInt, - [FieldTypes.C_BIGINT] : convertBigint, - [FieldTypes.C_FLOAT] : convertFloat, - [FieldTypes.C_DOUBLE] : convertDouble, - [FieldTypes.C_BINARY] : convertBinary, - [FieldTypes.C_TIMESTAMP] : convertTimestamp, - [FieldTypes.C_NCHAR] : convertNchar + [FieldTypes.C_BOOL]: convertBool, + [FieldTypes.C_TINYINT]: convertTinyint, + [FieldTypes.C_SMALLINT]: convertSmallint, + [FieldTypes.C_INT]: convertInt, + [FieldTypes.C_BIGINT]: convertBigint, + [FieldTypes.C_FLOAT]: convertFloat, + [FieldTypes.C_DOUBLE]: convertDouble, + [FieldTypes.C_BINARY]: convertBinary, + [FieldTypes.C_TIMESTAMP]: convertTimestamp, + [FieldTypes.C_NCHAR]: convertNchar } // Define TaosField structure var char_arr = ArrayType(ref.types.char); var TaosField = Struct({ - 'name': char_arr, - }); + 'name': char_arr, +}); TaosField.fields.name.type.size = 65; TaosField.defineProperty('type', ref.types.char); TaosField.defineProperty('bytes', ref.types.short); @@ -183,7 +183,7 @@ TaosField.defineProperty('bytes', ref.types.short); * @classdesc The CTaosInterface is the interface through which Node.JS communicates data back and forth with TDengine. It is not advised to * access this class directly and use it unless you understand what these functions do. */ -function CTaosInterface (config = null, pass = false) { +function CTaosInterface(config = null, pass = false) { ref.types.char_ptr = ref.refType(ref.types.char); ref.types.void_ptr = ref.refType(ref.types.void); ref.types.void_ptr2 = ref.refType(ref.types.void_ptr); @@ -196,64 +196,65 @@ function CTaosInterface (config = null, pass = false) { taoslibname = 'libtaos'; } this.libtaos = ffi.Library(taoslibname, { - 'taos_options': [ ref.types.int, [ ref.types.int , ref.types.void_ptr ] ], - 'taos_init': [ ref.types.void, [ ] ], + 'taos_options': [ref.types.int, [ref.types.int, ref.types.void_ptr]], + 'taos_init': [ref.types.void, []], //TAOS *taos_connect(char *ip, char *user, char *pass, char *db, int port) - 'taos_connect': [ ref.types.void_ptr, [ ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.int ] ], + 'taos_connect': [ref.types.void_ptr, [ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.int]], //void taos_close(TAOS *taos) - 'taos_close': [ ref.types.void, [ ref.types.void_ptr ] ], - //int *taos_fetch_lengths(TAOS_RES *taos); - 'taos_fetch_lengths': [ ref.types.void_ptr, [ ref.types.void_ptr ] ], + 'taos_close': [ref.types.void, [ref.types.void_ptr]], + //int *taos_fetch_lengths(TAOS_RES *res); + 'taos_fetch_lengths': [ref.types.void_ptr, [ref.types.void_ptr]], //int taos_query(TAOS *taos, char *sqlstr) - 'taos_query': [ ref.types.void_ptr, [ ref.types.void_ptr, ref.types.char_ptr ] ], - //int taos_affected_rows(TAOS *taos) - 'taos_affected_rows': [ ref.types.int, [ ref.types.void_ptr] ], + 'taos_query': [ref.types.void_ptr, [ref.types.void_ptr, ref.types.char_ptr]], + //int taos_affected_rows(TAOS_RES *res) + 'taos_affected_rows': [ref.types.int, [ref.types.void_ptr]], //int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) - 'taos_fetch_block': [ ref.types.int, [ ref.types.void_ptr, ref.types.void_ptr] ], + 'taos_fetch_block': [ref.types.int, [ref.types.void_ptr, ref.types.void_ptr]], //int taos_num_fields(TAOS_RES *res); - 'taos_num_fields': [ ref.types.int, [ ref.types.void_ptr] ], + 'taos_num_fields': [ref.types.int, [ref.types.void_ptr]], //TAOS_ROW taos_fetch_row(TAOS_RES *res) //TAOS_ROW is void **, but we set the return type as a reference instead to get the row - 'taos_fetch_row': [ ref.refType(ref.types.void_ptr2), [ ref.types.void_ptr ] ], + 'taos_fetch_row': [ref.refType(ref.types.void_ptr2), [ref.types.void_ptr]], + 'taos_print_row': [ref.types.int, [ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr, ref.types.int]], //int taos_result_precision(TAOS_RES *res) - 'taos_result_precision': [ ref.types.int, [ ref.types.void_ptr ] ], + 'taos_result_precision': [ref.types.int, [ref.types.void_ptr]], //void taos_free_result(TAOS_RES *res) - 'taos_free_result': [ ref.types.void, [ ref.types.void_ptr] ], + 'taos_free_result': [ref.types.void, [ref.types.void_ptr]], //int taos_field_count(TAOS *taos) - 'taos_field_count': [ ref.types.int, [ ref.types.void_ptr ] ], + 'taos_field_count': [ref.types.int, [ref.types.void_ptr]], //TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) - 'taos_fetch_fields': [ ref.refType(TaosField), [ ref.types.void_ptr ] ], + 'taos_fetch_fields': [ref.refType(TaosField), [ref.types.void_ptr]], //int taos_errno(TAOS *taos) - 'taos_errno': [ ref.types.int, [ ref.types.void_ptr] ], + 'taos_errno': [ref.types.int, [ref.types.void_ptr]], //char *taos_errstr(TAOS *taos) - 'taos_errstr': [ ref.types.char_ptr, [ ref.types.void_ptr] ], + 'taos_errstr': [ref.types.char_ptr, [ref.types.void_ptr]], //void taos_stop_query(TAOS_RES *res); - 'taos_stop_query': [ ref.types.void, [ ref.types.void_ptr] ], + 'taos_stop_query': [ref.types.void, [ref.types.void_ptr]], //char *taos_get_server_info(TAOS *taos); - 'taos_get_server_info': [ ref.types.char_ptr, [ ref.types.void_ptr ] ], + 'taos_get_server_info': [ref.types.char_ptr, [ref.types.void_ptr]], //char *taos_get_client_info(); - 'taos_get_client_info': [ ref.types.char_ptr, [ ] ], + 'taos_get_client_info': [ref.types.char_ptr, []], // ASYNC // void taos_query_a(TAOS *taos, char *sqlstr, void (*fp)(void *, TAOS_RES *, int), void *param) - 'taos_query_a': [ ref.types.void, [ ref.types.void_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr ] ], + 'taos_query_a': [ref.types.void, [ref.types.void_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr]], // void taos_fetch_rows_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, int numOfRows), void *param); - 'taos_fetch_rows_a': [ ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.types.void_ptr ]], + 'taos_fetch_rows_a': [ref.types.void, [ref.types.void_ptr, ref.types.void_ptr, ref.types.void_ptr]], // Subscription //TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval) - 'taos_subscribe': [ ref.types.void_ptr, [ ref.types.void_ptr, ref.types.int, ref.types.char_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr, ref.types.int] ], + 'taos_subscribe': [ref.types.void_ptr, [ref.types.void_ptr, ref.types.int, ref.types.char_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr, ref.types.int]], // TAOS_RES *taos_consume(TAOS_SUB *tsub) - 'taos_consume': [ ref.types.void_ptr, [ref.types.void_ptr] ], + 'taos_consume': [ref.types.void_ptr, [ref.types.void_ptr]], //void taos_unsubscribe(TAOS_SUB *tsub); - 'taos_unsubscribe': [ ref.types.void, [ ref.types.void_ptr ] ], + 'taos_unsubscribe': [ref.types.void, [ref.types.void_ptr]], // Continuous Query //TAOS_STREAM *taos_open_stream(TAOS *taos, char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), // int64_t stime, void *param, void (*callback)(void *)); - 'taos_open_stream': [ ref.types.void_ptr, [ ref.types.void_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.int64, ref.types.void_ptr, ref.types.void_ptr ] ], + 'taos_open_stream': [ref.types.void_ptr, [ref.types.void_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.int64, ref.types.void_ptr, ref.types.void_ptr]], //void taos_close_stream(TAOS_STREAM *tstr); - 'taos_close_stream': [ ref.types.void, [ ref.types.void_ptr ] ] + 'taos_close_stream': [ref.types.void, [ref.types.void_ptr]] }); if (pass == false) { @@ -264,7 +265,7 @@ function CTaosInterface (config = null, pass = false) { try { this._config = ref.allocCString(config); } - catch(err){ + catch (err) { throw "Attribute Error: config is expected as a str"; } } @@ -276,38 +277,38 @@ function CTaosInterface (config = null, pass = false) { return this; } CTaosInterface.prototype.config = function config() { - return this._config; - } -CTaosInterface.prototype.connect = function connect(host=null, user="root", password="taosdata", db=null, port=0) { - let _host,_user,_password,_db,_port; - try { + return this._config; +} +CTaosInterface.prototype.connect = function connect(host = null, user = "root", password = "taosdata", db = null, port = 0) { + let _host, _user, _password, _db, _port; + try { _host = host != null ? ref.allocCString(host) : ref.alloc(ref.types.char_ptr, ref.NULL); } - catch(err) { + catch (err) { throw "Attribute Error: host is expected as a str"; } try { _user = ref.allocCString(user) } - catch(err) { + catch (err) { throw "Attribute Error: user is expected as a str"; } try { _password = ref.allocCString(password); } - catch(err) { + catch (err) { throw "Attribute Error: password is expected as a str"; } try { _db = db != null ? ref.allocCString(db) : ref.alloc(ref.types.char_ptr, ref.NULL); } - catch(err) { + catch (err) { throw "Attribute Error: db is expected as a str"; } try { _port = ref.alloc(ref.types.int, port); } - catch(err) { + catch (err) { throw TypeError("port is expected as an int") } let connection = this.libtaos.taos_connect(_host, _user, _password, _db, _port); @@ -324,10 +325,10 @@ CTaosInterface.prototype.close = function close(connection) { console.log("Connection is closed"); } CTaosInterface.prototype.query = function query(connection, sql) { - return this.libtaos.taos_query(connection, ref.allocCString(sql)); + return this.libtaos.taos_query(connection, ref.allocCString(sql)); } -CTaosInterface.prototype.affectedRows = function affectedRows(connection) { - return this.libtaos.taos_affected_rows(connection); +CTaosInterface.prototype.affectedRows = function affectedRows(result) { + return this.libtaos.taos_affected_rows(result); } CTaosInterface.prototype.useResult = function useResult(result) { @@ -337,8 +338,8 @@ CTaosInterface.prototype.useResult = function useResult(result) { pfields = ref.reinterpret(pfields, this.fieldsCount(result) * 68, 0); for (let i = 0; i < pfields.length; i += 68) { //0 - 63 = name //64 - 65 = bytes, 66 - 67 = type - fields.push( { - name: ref.readCString(ref.reinterpret(pfields,65,i)), + fields.push({ + name: ref.readCString(ref.reinterpret(pfields, 65, i)), type: pfields[i + 65], bytes: pfields[i + 66] }) @@ -347,11 +348,10 @@ CTaosInterface.prototype.useResult = function useResult(result) { return fields; } CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { - //let pblock = ref.ref(ref.ref(ref.NULL)); // equal to our raw data - let pblock = this.libtaos.taos_fetch_row(result); - let num_of_rows = 1; - if (ref.isNull(pblock) == true) { - return {block:null, num_of_rows:0}; + let pblock = ref.NULL_POINTER; + let num_of_rows = this.libtaos.taos_fetch_block(result, pblock); + if (ref.isNull(pblock.deref()) == true) { + return { block: null, num_of_rows: 0 }; } var fieldL = this.libtaos.taos_fetch_lengths(result); @@ -359,10 +359,10 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { let isMicro = (this.libtaos.taos_result_precision(result) == FieldTypes.C_TIMESTAMP_MICRO); var fieldlens = []; - + if (ref.isNull(fieldL) == false) { - for (let i = 0; i < fields.length; i ++) { - let plen = ref.reinterpret(fieldL, 4, i*4); + for (let i = 0; i < fields.length; i++) { + let plen = ref.reinterpret(fieldL, 4, i * 4); let len = plen.readInt32LE(0); fieldlens.push(len); } @@ -370,21 +370,23 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { let blocks = new Array(fields.length); blocks.fill(null); - //num_of_rows = Math.abs(num_of_rows); + num_of_rows = Math.abs(num_of_rows); let offset = 0; + let ptr = pblock.deref(); + for (let i = 0; i < fields.length; i++) { - pdata = ref.reinterpret(pblock,8,i*8); - if(ref.isNull(pdata.readPointer())){ - blocks[i] = new Array(); - }else{ - pdata = ref.ref(pdata.readPointer()); - if (!convertFunctions[fields[i]['type']] ) { - throw new errors.DatabaseError("Invalid data type returned from database"); - } - blocks[i] = convertFunctions[fields[i]['type']](pdata, 1, fieldlens[i], offset, isMicro); - } + pdata = ref.reinterpret(ptr, 8, i * 8); + if (ref.isNull(pdata.readPointer())) { + blocks[i] = new Array(); + } else { + pdata = ref.ref(pdata.readPointer()); + if (!convertFunctions[fields[i]['type']]) { + throw new errors.DatabaseError("Invalid data type returned from database"); + } + blocks[i] = convertFunctions[fields[i]['type']](pdata, num_of_rows, fieldlens[i], offset, isMicro); + } } - return {blocks: blocks, num_of_rows:Math.abs(num_of_rows)} + return { blocks: blocks, num_of_rows } } CTaosInterface.prototype.fetchRow = function fetchRow(result, fields) { let row = this.libtaos.taos_fetch_row(result); @@ -414,7 +416,7 @@ CTaosInterface.prototype.errStr = function errStr(result) { // Async CTaosInterface.prototype.query_a = function query_a(connection, sql, callback, param = ref.ref(ref.NULL)) { // void taos_query_a(TAOS *taos, char *sqlstr, void (*fp)(void *param, TAOS_RES *, int), void *param) - callback = ffi.Callback(ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.types.int ], callback); + callback = ffi.Callback(ref.types.void, [ref.types.void_ptr, ref.types.void_ptr, ref.types.int], callback); this.libtaos.taos_query_a(connection, ref.allocCString(sql), callback, param); return param; } @@ -439,46 +441,46 @@ CTaosInterface.prototype.fetch_rows_a = function fetch_rows_a(result, callback, var fieldL = cti.libtaos.taos_fetch_lengths(result); var fieldlens = []; if (ref.isNull(fieldL) == false) { - - for (let i = 0; i < fields.length; i ++) { - let plen = ref.reinterpret(fieldL, 8, i*8); - let len = ref.get(plen,0,ref.types.int32); + + for (let i = 0; i < fields.length; i++) { + let plen = ref.reinterpret(fieldL, 8, i * 8); + let len = ref.get(plen, 0, ref.types.int32); fieldlens.push(len); } } - if (numOfRows2 > 0){ + if (numOfRows2 > 0) { for (let i = 0; i < fields.length; i++) { - if(ref.isNull(pdata.readPointer())){ - blocks[i] = new Array(); - }else{ - if (!convertFunctions[fields[i]['type']] ) { - throw new errors.DatabaseError("Invalid data type returned from database"); - } - let prow = ref.reinterpret(row,8,i*8); - prow = prow.readPointer(); - prow = ref.ref(prow); - blocks[i] = convertFunctions[fields[i]['type']](prow, 1, fieldlens[i], offset, isMicro); - //offset += fields[i]['bytes'] * numOfRows2; - } + if (ref.isNull(pdata.readPointer())) { + blocks[i] = new Array(); + } else { + if (!convertFunctions[fields[i]['type']]) { + throw new errors.DatabaseError("Invalid data type returned from database"); + } + let prow = ref.reinterpret(row, 8, i * 8); + prow = prow.readPointer(); + prow = ref.ref(prow); + blocks[i] = convertFunctions[fields[i]['type']](prow, 1, fieldlens[i], offset, isMicro); + //offset += fields[i]['bytes'] * numOfRows2; + } } } callback(param2, result2, numOfRows2, blocks); } - asyncCallbackWrapper = ffi.Callback(ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.types.int ], asyncCallbackWrapper); + asyncCallbackWrapper = ffi.Callback(ref.types.void, [ref.types.void_ptr, ref.types.void_ptr, ref.types.int], asyncCallbackWrapper); this.libtaos.taos_fetch_rows_a(result, asyncCallbackWrapper, param); return param; } // Fetch field meta data by result handle -CTaosInterface.prototype.fetchFields_a = function fetchFields_a (result) { +CTaosInterface.prototype.fetchFields_a = function fetchFields_a(result) { let pfields = this.fetchFields(result); let pfieldscount = this.numFields(result); let fields = []; if (ref.isNull(pfields) == false) { - pfields = ref.reinterpret(pfields, 68 * pfieldscount , 0); + pfields = ref.reinterpret(pfields, 68 * pfieldscount, 0); for (let i = 0; i < pfields.length; i += 68) { //0 - 64 = name //65 = type, 66 - 67 = bytes - fields.push( { - name: ref.readCString(ref.reinterpret(pfields,65,i)), + fields.push({ + name: ref.readCString(ref.reinterpret(pfields, 65, i)), type: pfields[i + 65], bytes: pfields[i + 66] }) @@ -488,7 +490,7 @@ CTaosInterface.prototype.fetchFields_a = function fetchFields_a (result) { } // Stop a query by result handle CTaosInterface.prototype.stopQuery = function stopQuery(result) { - if (result != null){ + if (result != null) { this.libtaos.taos_stop_query(result); } else { @@ -509,13 +511,13 @@ CTaosInterface.prototype.subscribe = function subscribe(connection, restart, top try { sql = sql != null ? ref.allocCString(sql) : ref.alloc(ref.types.char_ptr, ref.NULL); } - catch(err) { + catch (err) { throw "Attribute Error: sql is expected as a str"; } try { topic = topic != null ? ref.allocCString(topic) : ref.alloc(ref.types.char_ptr, ref.NULL); } - catch(err) { + catch (err) { throw TypeError("topic is expected as a str"); } @@ -539,8 +541,8 @@ CTaosInterface.prototype.consume = function consume(subscription) { pfields = ref.reinterpret(pfields, this.numFields(result) * 68, 0); for (let i = 0; i < pfields.length; i += 68) { //0 - 63 = name //64 - 65 = bytes, 66 - 67 = type - fields.push( { - name: ref.readCString(ref.reinterpret(pfields,64,i)), + fields.push({ + name: ref.readCString(ref.reinterpret(pfields, 64, i)), bytes: pfields[i + 64], type: pfields[i + 66] }) @@ -548,7 +550,7 @@ CTaosInterface.prototype.consume = function consume(subscription) { } let data = []; - while(true) { + while (true) { let { blocks, num_of_rows } = this.fetchBlock(result, fields); if (num_of_rows == 0) { break; @@ -559,7 +561,7 @@ CTaosInterface.prototype.consume = function consume(subscription) { for (let j = 0; j < fields.length; j++) { rowBlock[j] = blocks[j][i]; } - data[data.length-1] = (rowBlock); + data[data.length - 1] = (rowBlock); } } return { data: data, fields: fields, result: result }; @@ -570,11 +572,11 @@ CTaosInterface.prototype.unsubscribe = function unsubscribe(subscription) { } // Continuous Query -CTaosInterface.prototype.openStream = function openStream(connection, sql, callback, stime,stoppingCallback, param = ref.ref(ref.NULL)) { +CTaosInterface.prototype.openStream = function openStream(connection, sql, callback, stime, stoppingCallback, param = ref.ref(ref.NULL)) { try { sql = ref.allocCString(sql); } - catch(err) { + catch (err) { throw "Attribute Error: sql string is expected as a str"; } var cti = this; @@ -587,7 +589,7 @@ CTaosInterface.prototype.openStream = function openStream(connection, sql, callb let offset = 0; if (numOfRows2 > 0) { for (let i = 0; i < fields.length; i++) { - if (!convertFunctions[fields[i]['type']] ) { + if (!convertFunctions[fields[i]['type']]) { throw new errors.DatabaseError("Invalid data type returned from database"); } blocks[i] = convertFunctions[fields[i]['type']](row, numOfRows2, fields[i]['bytes'], offset, isMicro); @@ -596,8 +598,8 @@ CTaosInterface.prototype.openStream = function openStream(connection, sql, callb } callback(param2, result2, blocks, fields); } - asyncCallbackWrapper = ffi.Callback(ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.refType(ref.types.void_ptr2) ], asyncCallbackWrapper); - asyncStoppingCallbackWrapper = ffi.Callback( ref.types.void, [ ref.types.void_ptr ], stoppingCallback); + asyncCallbackWrapper = ffi.Callback(ref.types.void, [ref.types.void_ptr, ref.types.void_ptr, ref.refType(ref.types.void_ptr2)], asyncCallbackWrapper); + asyncStoppingCallbackWrapper = ffi.Callback(ref.types.void, [ref.types.void_ptr], stoppingCallback); let streamHandle = this.libtaos.taos_open_stream(connection, sql, asyncCallbackWrapper, stime, param, asyncStoppingCallbackWrapper); if (ref.isNull(streamHandle)) { throw new errors.TDError('Failed to open a stream with TDengine'); diff --git a/src/connector/nodejs/nodetaos/cursor.js b/src/connector/nodejs/nodetaos/cursor.js index e18e6c2500..f879d89d48 100644 --- a/src/connector/nodejs/nodetaos/cursor.js +++ b/src/connector/nodejs/nodetaos/cursor.js @@ -1,7 +1,7 @@ const ref = require('ref-napi'); require('./globalfunc.js') const CTaosInterface = require('./cinterface') -const errors = require ('./error') +const errors = require('./error') const TaosQuery = require('./taosquery') const { PerformanceObserver, performance } = require('perf_hooks'); module.exports = TDengineCursor; @@ -22,7 +22,7 @@ module.exports = TDengineCursor; * @property {fields} - Array of the field objects in order from left to right of the latest data retrieved * @since 1.0.0 */ -function TDengineCursor(connection=null) { +function TDengineCursor(connection = null) { //All parameters are store for sync queries only. this._rowcount = -1; this._connection = null; @@ -91,7 +91,7 @@ TDengineCursor.prototype.execute = function execute(operation, options, callback return null; } - if (typeof options == 'function') { + if (typeof options == 'function') { callback = options; } if (typeof options != 'object') options = {} @@ -144,10 +144,10 @@ TDengineCursor.prototype.execute = function execute(operation, options, callback } TDengineCursor.prototype._createAffectedResponse = function (num, time) { - return "Query OK, " + num + " row(s) affected (" + (time * 0.001).toFixed(8) + "s)"; + return "Query OK, " + num + " row(s) affected (" + (time * 0.001).toFixed(8) + "s)"; } TDengineCursor.prototype._createSetResponse = function (num, time) { - return "Query OK, " + num + " row(s) in set (" + (time * 0.001).toFixed(8) + "s)"; + return "Query OK, " + num + " row(s) in set (" + (time * 0.001).toFixed(8) + "s)"; } TDengineCursor.prototype.executemany = function executemany() { @@ -176,27 +176,22 @@ TDengineCursor.prototype.fetchall = function fetchall(options, callback) { throw new errors.OperationalError("Invalid use of fetchall, either result or fields from query are null. First execute a query first"); } - let data = []; + let num_of_rows = this._chandle.affectedRows(this._result); + let data = new Array(num_of_rows); + this._rowcount = 0; - //let nodetime = 0; + let time = 0; const obs = new PerformanceObserver((items) => { time += items.getEntries()[0].duration; performance.clearMarks(); }); - /* - const obs2 = new PerformanceObserver((items) => { - nodetime += items.getEntries()[0].duration; - performance.clearMarks(); - }); - obs2.observe({ entryTypes: ['measure'] }); - performance.mark('nodea'); - */ obs.observe({ entryTypes: ['measure'] }); performance.mark('A'); - while(true) { - + while (true) { let blockAndRows = this._chandle.fetchBlock(this._result, this._fields); + // console.log(blockAndRows); + // break; let block = blockAndRows.blocks; let num_of_rows = blockAndRows.num_of_rows; if (num_of_rows == 0) { @@ -205,22 +200,24 @@ TDengineCursor.prototype.fetchall = function fetchall(options, callback) { this._rowcount += num_of_rows; let numoffields = this._fields.length; for (let i = 0; i < num_of_rows; i++) { - data.push([]); - + // data.push([]); + let rowBlock = new Array(numoffields); for (let j = 0; j < numoffields; j++) { rowBlock[j] = block[j][i]; } - data[data.length-1] = (rowBlock); + data[this._rowcount - num_of_rows + i] = (rowBlock); + // data.push(rowBlock); } } + performance.mark('B'); performance.measure('query', 'A', 'B'); let response = this._createSetResponse(this._rowcount, time) console.log(response); - // this._connection._clearResultSet(); + // this._connection._clearResultSet(); let fields = this.fields; this._reset_result(); this.data = data; @@ -239,12 +236,12 @@ TDengineCursor.prototype.fetchall = function fetchall(options, callback) { * @return {number | Buffer} Number of affected rows or a Buffer that points to the results of the query * @since 1.0.0 */ -TDengineCursor.prototype.execute_a = function execute_a (operation, options, callback, param) { +TDengineCursor.prototype.execute_a = function execute_a(operation, options, callback, param) { if (operation == undefined) { throw new errors.ProgrammingError('No operation passed as argument'); return null; } - if (typeof options == 'function') { + if (typeof options == 'function') { //we expect the parameter after callback to be param param = callback; callback = options; @@ -265,14 +262,14 @@ TDengineCursor.prototype.execute_a = function execute_a (operation, options, cal } if (resCode >= 0) { -// let fieldCount = cr._chandle.numFields(res2); -// if (fieldCount == 0) { -// //cr._chandle.freeResult(res2); -// return res2; -// } -// else { -// return res2; -// } + // let fieldCount = cr._chandle.numFields(res2); + // if (fieldCount == 0) { + // //cr._chandle.freeResult(res2); + // return res2; + // } + // else { + // return res2; + // } return res2; } @@ -317,7 +314,7 @@ TDengineCursor.prototype.execute_a = function execute_a (operation, options, cal * }) */ TDengineCursor.prototype.fetchall_a = function fetchall_a(result, options, callback, param = {}) { - if (typeof options == 'function') { + if (typeof options == 'function') { //we expect the parameter after callback to be param param = callback; callback = options; @@ -360,17 +357,17 @@ TDengineCursor.prototype.fetchall_a = function fetchall_a(result, options, callb for (let k = 0; k < fields.length; k++) { rowBlock[k] = block[k][j]; } - data[data.length-1] = rowBlock; + data[data.length - 1] = rowBlock; } } cr._chandle.freeResult(result2); // free result, avoid seg faults and mem leaks! - callback(param2, result2, numOfRows2, {data:data,fields:fields}); + callback(param2, result2, numOfRows2, { data: data, fields: fields }); } } ref.writeObject(buf, 0, param); param = this._chandle.fetch_rows_a(result, asyncCallbackWrapper, buf); //returned param - return {param:param,result:result}; + return { param: param, result: result }; } /** * Stop a query given the result handle. @@ -428,7 +425,7 @@ TDengineCursor.prototype.subscribe = function subscribe(config) { */ TDengineCursor.prototype.consumeData = async function consumeData(subscription, callback) { while (true) { - let { data, fields, result} = this._chandle.consume(subscription); + let { data, fields, result } = this._chandle.consume(subscription); callback(data, fields, result); } } @@ -450,30 +447,30 @@ TDengineCursor.prototype.unsubscribe = function unsubscribe(subscription) { * @return {Buffer} A buffer pointing to the stream handle * @since 1.3.0 */ - TDengineCursor.prototype.openStream = function openStream(sql, callback, stime = 0, stoppingCallback, param = {}) { - let buf = ref.alloc('Object'); - ref.writeObject(buf, 0, param); +TDengineCursor.prototype.openStream = function openStream(sql, callback, stime = 0, stoppingCallback, param = {}) { + let buf = ref.alloc('Object'); + ref.writeObject(buf, 0, param); - let asyncCallbackWrapper = function (param2, result2, blocks, fields) { - let data = []; - let num_of_rows = blocks[0].length; - for (let j = 0; j < num_of_rows; j++) { - data.push([]); - let rowBlock = new Array(fields.length); - for (let k = 0; k < fields.length; k++) { - rowBlock[k] = blocks[k][j]; - } - data[data.length-1] = rowBlock; - } - callback(param2, result2, blocks, fields); - } - return this._chandle.openStream(this._connection._conn, sql, asyncCallbackWrapper, stime, stoppingCallback, buf); - } - /** - * Close a stream - * @param {Buffer} - A buffer pointing to the handle of the stream to be closed - * @since 1.3.0 - */ - TDengineCursor.prototype.closeStream = function closeStream(stream) { - this._chandle.closeStream(stream); - } + let asyncCallbackWrapper = function (param2, result2, blocks, fields) { + let data = []; + let num_of_rows = blocks[0].length; + for (let j = 0; j < num_of_rows; j++) { + data.push([]); + let rowBlock = new Array(fields.length); + for (let k = 0; k < fields.length; k++) { + rowBlock[k] = blocks[k][j]; + } + data[data.length - 1] = rowBlock; + } + callback(param2, result2, blocks, fields); + } + return this._chandle.openStream(this._connection._conn, sql, asyncCallbackWrapper, stime, stoppingCallback, buf); +} +/** + * Close a stream + * @param {Buffer} - A buffer pointing to the handle of the stream to be closed + * @since 1.3.0 + */ +TDengineCursor.prototype.closeStream = function closeStream(stream) { + this._chandle.closeStream(stream); +} diff --git a/src/connector/nodejs/package-lock.json b/src/connector/nodejs/package-lock.json deleted file mode 100644 index 9ca174ccd1..0000000000 --- a/src/connector/nodejs/package-lock.json +++ /dev/null @@ -1,285 +0,0 @@ -{ - "name": "td2.0-connector", - "version": "2.0.6", - "lockfileVersion": 1, - "requires": true, - "dependencies": { - "array-index": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/array-index/-/array-index-1.0.0.tgz", - "integrity": "sha1-7FanSe4QPk4Ix5C5w1PfFgVbl/k=", - "requires": { - "debug": "^2.2.0", - "es6-symbol": "^3.0.2" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - } - } - }, - "d": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", - "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==", - "requires": { - "es5-ext": "^0.10.50", - "type": "^1.0.1" - } - }, - "debug": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", - "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", - "requires": { - "ms": "2.1.2" - } - }, - "es5-ext": { - "version": "0.10.53", - "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.53.tgz", - "integrity": "sha512-Xs2Stw6NiNHWypzRTY1MtaG/uJlwCk8kH81920ma8mvN8Xq1gsfhZvpkImLQArw8AHnv8MT2I45J3c0R8slE+Q==", - "requires": { - "es6-iterator": "~2.0.3", - "es6-symbol": "~3.1.3", - "next-tick": "~1.0.0" - } - }, - "es6-iterator": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", - "integrity": "sha1-p96IkUGgWpSwhUQDstCg+/qY87c=", - "requires": { - "d": "1", - "es5-ext": "^0.10.35", - "es6-symbol": "^3.1.1" - } - }, - "es6-symbol": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz", - "integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==", - "requires": { - "d": "^1.0.1", - "ext": "^1.1.2" - } - }, - "ext": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/ext/-/ext-1.4.0.tgz", - "integrity": "sha512-Key5NIsUxdqKg3vIsdw9dSuXpPCQ297y6wBjL30edxwPgt2E44WcWBZey/ZvUc6sERLTxKdyCu4gZFmUbk1Q7A==", - "requires": { - "type": "^2.0.0" - }, - "dependencies": { - "type": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/type/-/type-2.1.0.tgz", - "integrity": "sha512-G9absDWvhAWCV2gmF1zKud3OyC61nZDwWvBL2DApaVFogI07CprggiQAOOjvp2NRjYWFzPyu7vwtDrQFq8jeSA==" - } - } - }, - "ffi-napi": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/ffi-napi/-/ffi-napi-3.1.0.tgz", - "integrity": "sha512-EsHO+sP2p/nUC/3l/l8m9niee1BLm4asUFDzkkBGR4kYVgp2KqdAYUomZhkKtzim4Fq7mcYHjpUaIHsMqs+E1g==", - "requires": { - "debug": "^4.1.1", - "get-uv-event-loop-napi-h": "^1.0.5", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1", - "ref-napi": "^2.0.1", - "ref-struct-di": "^1.1.0" - }, - "dependencies": { - "ref-napi": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-2.1.2.tgz", - "integrity": "sha512-aFl+vrIuLWUXMUTQGAwGAuSNLX3Ub5W3iVP8b7KyFFZUdn4+i4U1TXXTop0kCTUfGNu8glBGVz4lowkwMcPVVA==", - "requires": { - "debug": "^4.1.1", - "get-symbol-from-current-process-h": "^1.0.2", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1" - } - } - } - }, - "get-symbol-from-current-process-h": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/get-symbol-from-current-process-h/-/get-symbol-from-current-process-h-1.0.2.tgz", - "integrity": "sha512-syloC6fsCt62ELLrr1VKBM1ggOpMdetX9hTrdW77UQdcApPHLmf7CI7OKcN1c9kYuNxKcDe4iJ4FY9sX3aw2xw==" - }, - "get-uv-event-loop-napi-h": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/get-uv-event-loop-napi-h/-/get-uv-event-loop-napi-h-1.0.6.tgz", - "integrity": "sha512-t5c9VNR84nRoF+eLiz6wFrEp1SE2Acg0wS+Ysa2zF0eROes+LzOfuTaVHxGy8AbS8rq7FHEJzjnCZo1BupwdJg==", - "requires": { - "get-symbol-from-current-process-h": "^1.0.1" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "next-tick": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz", - "integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw=" - }, - "node-addon-api": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.2.tgz", - "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA==" - }, - "node-gyp-build": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.2.3.tgz", - "integrity": "sha512-MN6ZpzmfNCRM+3t57PTJHgHyw/h4OWnZ6mR8P5j/uZtqQr46RRuDE/P+g3n0YR/AiYXeWixZZzaip77gdICfRg==" - }, - "ref-array-napi": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ref-array-napi/-/ref-array-napi-1.2.1.tgz", - "integrity": "sha512-jQp2WWSucmxkqVfoNfm7yDlDeGu3liAbzqfwjNybL80ooLOCnCZpAK2woDInY+lxNOK/VlIVSqeDEYb4gVPuNQ==", - "requires": { - "array-index": "1", - "debug": "2", - "ref-napi": "^1.4.2" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "ref-napi": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-1.5.2.tgz", - "integrity": "sha512-hwyNmWpUkt1bDWDW4aiwCoC+SJfJO69UIdjqssNqdaS0sYJpgqzosGg/rLtk69UoQ8drZdI9yyQefM7eEMM3Gw==", - "requires": { - "debug": "^3.1.0", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1" - }, - "dependencies": { - "debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "requires": { - "ms": "^2.1.1" - } - }, - "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - } - } - } - } - }, - "ref-napi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-3.0.1.tgz", - "integrity": "sha512-W3rcb0E+tlO9u9ySFnX5vifInwwPGToOfFgTZUHJBNiOBsW0NNvgHz2zJN7ctABo/2yIlgdPQUvuqqfORIF4LA==", - "requires": { - "debug": "^4.1.1", - "get-symbol-from-current-process-h": "^1.0.2", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1" - } - }, - "ref-struct-di": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ref-struct-di/-/ref-struct-di-1.1.1.tgz", - "integrity": "sha512-2Xyn/0Qgz89VT+++WP0sTosdm9oeowLP23wRJYhG4BFdMUrLj3jhwHZNEytYNYgtPKLNTP3KJX4HEgBvM1/Y2g==", - "requires": { - "debug": "^3.1.0" - }, - "dependencies": { - "debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "requires": { - "ms": "^2.1.1" - } - } - } - }, - "ref-struct-napi": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ref-struct-napi/-/ref-struct-napi-1.1.1.tgz", - "integrity": "sha512-YgS5/d7+kT5zgtySYI5ieH0hREdv+DabgDvoczxsui0f9VLm0rrDcWEj4DHKehsH+tJnVMsLwuyctWgvdEcVRw==", - "requires": { - "debug": "2", - "ref-napi": "^1.4.2" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "ref-napi": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-1.5.2.tgz", - "integrity": "sha512-hwyNmWpUkt1bDWDW4aiwCoC+SJfJO69UIdjqssNqdaS0sYJpgqzosGg/rLtk69UoQ8drZdI9yyQefM7eEMM3Gw==", - "requires": { - "debug": "^3.1.0", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1" - }, - "dependencies": { - "debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "requires": { - "ms": "^2.1.1" - } - }, - "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - } - } - } - } - }, - "type": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", - "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==" - } - } -} diff --git a/src/connector/nodejs/package.json b/src/connector/nodejs/package.json index b39ce2c17d..d21b62108b 100644 --- a/src/connector/nodejs/package.json +++ b/src/connector/nodejs/package.json @@ -1,6 +1,6 @@ { "name": "td2.0-connector", - "version": "2.0.6", + "version": "2.0.7", "description": "A Node.js connector for TDengine.", "main": "tdengine.js", "directories": { diff --git a/src/connector/odbc/examples/c/main.c b/src/connector/odbc/examples/c/main.c index 3ad32dbe53..de01d2b85e 100644 --- a/src/connector/odbc/examples/c/main.c +++ b/src/connector/odbc/examples/c/main.c @@ -312,11 +312,7 @@ static int test_sqls_in_stmt(SQLHENV env, SQLHDBC conn, SQLHSTMT stmt, const cha size_t len = 0; ssize_t n = 0; -#ifdef _MSC_VER - n = taosGetlineImp(&line, &len, f); -#else - n = getline(&line, &len, f); -#endif + n = tgetline(&line, &len, f); if (n==-1) break; const char *p = NULL; diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index dd628ffe20..13c2886f3b 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -41,6 +41,7 @@ #include "dnodeTelemetry.h" #include "module.h" #include "qScript.h" +#include "mnode.h" #if !defined(_MODULE) || !defined(_TD_LINUX) int32_t moduleStart() { return 0; } @@ -84,7 +85,18 @@ static SStep tsDnodeSteps[] = { {"dnode-shell", dnodeInitShell, dnodeCleanupShell}, {"dnode-statustmr", dnodeInitStatusTimer,dnodeCleanupStatusTimer}, {"dnode-telemetry", dnodeInitTelemetry, dnodeCleanupTelemetry}, - {"dnode-script", scriptEnvPoolInit, scriptEnvPoolCleanup}, + {"dnode-script", scriptEnvPoolInit, scriptEnvPoolCleanup}, +}; + +static SStep tsDnodeCompactSteps[] = { + {"dnode-tfile", tfInit, tfCleanup}, + {"dnode-storage", dnodeInitStorage, dnodeCleanupStorage}, + {"dnode-eps", dnodeInitEps, dnodeCleanupEps}, + {"dnode-wal", walInit, walCleanUp}, + {"dnode-mread", dnodeInitMRead, NULL}, + {"dnode-mwrite", dnodeInitMWrite, NULL}, + {"dnode-mpeer", dnodeInitMPeer, NULL}, + {"dnode-modules", dnodeInitModules, dnodeCleanupModules}, }; static int dnodeCreateDir(const char *dir) { @@ -96,13 +108,23 @@ static int dnodeCreateDir(const char *dir) { } static void dnodeCleanupComponents() { - int32_t stepSize = sizeof(tsDnodeSteps) / sizeof(SStep); - dnodeStepCleanup(tsDnodeSteps, stepSize); + if (!tsCompactMnodeWal) { + int32_t stepSize = sizeof(tsDnodeSteps) / sizeof(SStep); + dnodeStepCleanup(tsDnodeSteps, stepSize); + } else { + int32_t stepSize = sizeof(tsDnodeCompactSteps) / sizeof(SStep); + dnodeStepCleanup(tsDnodeCompactSteps, stepSize); + } } static int32_t dnodeInitComponents() { - int32_t stepSize = sizeof(tsDnodeSteps) / sizeof(SStep); - return dnodeStepInit(tsDnodeSteps, stepSize); + if (!tsCompactMnodeWal) { + int32_t stepSize = sizeof(tsDnodeSteps) / sizeof(SStep); + return dnodeStepInit(tsDnodeSteps, stepSize); + } else { + int32_t stepSize = sizeof(tsDnodeCompactSteps) / sizeof(SStep); + return dnodeStepInit(tsDnodeCompactSteps, stepSize); + } } static int32_t dnodeInitTmr() { @@ -218,7 +240,24 @@ static int32_t dnodeInitStorage() { sprintf(tsDnodeDir, "%s/dnode", tsDataDir); // sprintf(tsVnodeBakDir, "%s/vnode_bak", tsDataDir); - //TODO(dengyihao): no need to init here + if (tsCompactMnodeWal == 1) { + sprintf(tsMnodeTmpDir, "%s/mnode_tmp", tsDataDir); + if (taosDirExist(tsMnodeTmpDir)) { + dError("mnode_tmp dir already exist in %s,quit compact job", tsMnodeTmpDir); + return -1; + } + if (dnodeCreateDir(tsMnodeTmpDir) < 0) { + dError("failed to create dir: %s, reason: %s", tsMnodeTmpDir, strerror(errno)); + return -1; + } + + sprintf(tsMnodeBakDir, "%s/mnode_bak", tsDataDir); + if (taosDirExist(tsMnodeBakDir)) { + dError("mnode_bak dir already exist in %s,quit compact job", tsMnodeBakDir); + return -1; + } + } + //TODO(dengyihao): no need to init here if (dnodeCreateDir(tsMnodeDir) < 0) { dError("failed to create dir: %s, reason: %s", tsMnodeDir, strerror(errno)); return -1; diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index e49b3eba99..ee37ffdcbb 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -42,6 +42,8 @@ int32_t main(int32_t argc, char *argv[]) { } } else if (strcmp(argv[i], "-C") == 0) { dump_config = 1; + } else if (strcmp(argv[i], "--compact-mnode-wal") == 0) { + tsCompactMnodeWal = 1; } else if (strcmp(argv[i], "-V") == 0) { #ifdef _ACCT char *versionStr = "enterprise"; diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 2495a42ba2..203ac57469 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -73,6 +73,9 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg); void mnodeProcessPeerRsp(SRpcMsg *pMsg); int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey); +int32_t mnodeCompactWal(); +int32_t mnodeCompactComponents(); + #ifdef __cplusplus } #endif diff --git a/src/inc/monitor.h b/src/inc/monitor.h index 1aefb0b848..d2e5e06487 100644 --- a/src/inc/monitor.h +++ b/src/inc/monitor.h @@ -54,7 +54,8 @@ void monCleanupSystem(); void monSaveAcctLog(SAcctMonitorObj *pMonObj); void monSaveLog(int32_t level, const char *const format, ...); void monExecuteSQL(char *sql); - +typedef void (*MonExecuteSQLCbFP)(void *param, TAOS_RES *, int code); +void monExecuteSQLWithResultCallback(char *sql, MonExecuteSQLCbFP callback, void* param); #ifdef __cplusplus } #endif diff --git a/src/inc/taos.h b/src/inc/taos.h index 6dd695b320..d27828fc36 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -112,6 +112,7 @@ typedef struct TAOS_MULTI_BIND { TAOS_STMT *taos_stmt_init(TAOS *taos); int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); +int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags); int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name); int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index d812f77580..343bd8adcf 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -33,6 +33,8 @@ extern "C" { #endif #define TSWINDOW_INITIALIZER ((STimeWindow) {INT64_MIN, INT64_MAX}) +#define TSWINDOW_DESC_INITIALIZER ((STimeWindow) {INT64_MAX, INT64_MIN}) + #define TSKEY_INITIAL_VAL INT64_MIN // Bytes for each type. @@ -246,7 +248,6 @@ do { \ #define TSDB_MAX_REPLICA 5 #define TSDB_TBNAME_COLUMN_INDEX (-1) -#define TSDB_BLOCK_DIST_COLUMN_INDEX (-2) #define TSDB_UD_COLUMN_INDEX (-1000) #define TSDB_RES_COL_ID (-5000) @@ -302,7 +303,7 @@ do { \ #define TSDB_DEFAULT_DB_UPDATE_OPTION 0 #define TSDB_MIN_DB_CACHE_LAST_ROW 0 -#define TSDB_MAX_DB_CACHE_LAST_ROW 1 +#define TSDB_MAX_DB_CACHE_LAST_ROW 3 #define TSDB_DEFAULT_CACHE_LAST_ROW 0 #define TSDB_MIN_FSYNC_PERIOD 0 @@ -353,6 +354,7 @@ do { \ #define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u #define TSDB_QUERY_TYPE_INSERT 0x100u // insert type #define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u +#define TSDB_QUERY_TYPE_FILE_INSERT 0x400u // insert data from file #define TSDB_QUERY_TYPE_STMT_INSERT 0x800u // stmt insert type #define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index b81816ca05..c84b497652 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -74,7 +74,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_REF_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x010A) //"Ref is not there") //client -#define TSDB_CODE_TSC_INVALID_SQL TAOS_DEF_ERROR_CODE(0, 0x0200) //"Invalid SQL statement") +#define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) //"Invalid Operation") #define TSDB_CODE_TSC_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0201) //"Invalid qhandle") #define TSDB_CODE_TSC_INVALID_TIME_STAMP TAOS_DEF_ERROR_CODE(0, 0x0202) //"Invalid combination of client/service time") #define TSDB_CODE_TSC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x0203) //"Invalid value in client") @@ -227,6 +227,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_VND_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0512) //"Database write operation denied") #define TSDB_CODE_VND_IS_SYNCING TAOS_DEF_ERROR_CODE(0, 0x0513) //"Database is syncing") #define TSDB_CODE_VND_INVALID_TSDB_STATE TAOS_DEF_ERROR_CODE(0, 0x0514) //"Invalid tsdb state") +#define TSDB_CODE_VND_IS_CLOSING TAOS_DEF_ERROR_CODE(0, 0x0515) //"Database is closing") // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) //"Invalid table ID") @@ -436,6 +437,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FS_INVLD_LEVEL TAOS_DEF_ERROR_CODE(0, 0x2207) //"tfs invalid level") #define TSDB_CODE_FS_NO_VALID_DISK TAOS_DEF_ERROR_CODE(0, 0x2208) //"tfs no valid disk") +// monitor +#define TSDB_CODE_MON_CONNECTION_INVALID TAOS_DEF_ERROR_CODE(0, 0x2300) //"monitor invalid monitor db connection") + #ifdef __cplusplus } #endif diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 72acd744dd..e6071d7806 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -86,7 +86,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_TABLE, "drop-table" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_ALTER_TABLE, "alter-table" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_TABLE_META, "table-meta" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_STABLE_VGROUP, "stable-vgroup" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_TABLES_META, "tables-meta" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_TABLES_META, "multiTable-meta" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_ALTER_STREAM, "alter-stream" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_SHOW, "show" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_RETRIEVE, "retrieve" ) @@ -297,6 +297,8 @@ typedef struct { typedef struct { char name[TSDB_TABLE_FNAME_LEN]; + // if user specify DROP STABLE, this flag will be set. And an error will be returned if it is not a super table + int8_t supertable; int8_t igNotExists; } SCMDropTableMsg; @@ -744,8 +746,9 @@ typedef struct { } STableInfoMsg; typedef struct { + int32_t numOfVgroups; int32_t numOfTables; - char tableIds[]; + char tableNames[]; } SMultiTableInfoMsg; typedef struct SSTableVgroupMsg { @@ -794,8 +797,9 @@ typedef struct STableMetaMsg { typedef struct SMultiTableMeta { int32_t numOfTables; + int32_t numOfVgroup; int32_t contLen; - char metas[]; + char meta[]; } SMultiTableMeta; typedef struct { diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 1ba5131f6d..376836369d 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -69,9 +69,13 @@ typedef struct { int8_t precision; int8_t compression; int8_t update; - int8_t cacheLastRow; + int8_t cacheLastRow; // 0:no cache, 1: cache last row, 2: cache last NULL column 3: 1&2 } STsdbCfg; +#define CACHE_NO_LAST(c) ((c)->cacheLastRow == 0) +#define CACHE_LAST_ROW(c) (((c)->cacheLastRow & 1) > 0) +#define CACHE_LAST_NULL_COLUMN(c) (((c)->cacheLastRow & 2) > 0) + // --------- TSDB REPOSITORY USAGE STATISTICS typedef struct { int64_t totalStorage; // total bytes occupie @@ -211,7 +215,7 @@ typedef struct SDataBlockInfo { } SDataBlockInfo; typedef struct SFileBlockInfo { - int32_t numOfRows; + int32_t numBlocksOfStep; } SFileBlockInfo; typedef struct { @@ -225,11 +229,15 @@ typedef struct { SHashObj *map; // speedup acquire the tableQueryInfo by table uid } STableGroupInfo; +#define TSDB_BLOCK_DIST_STEP_ROWS 16 typedef struct { uint16_t rowSize; uint16_t numOfFiles; uint32_t numOfTables; uint64_t totalSize; + uint64_t totalRows; + int32_t maxRows; + int32_t minRows; int32_t firstSeekTimeUs; uint32_t numOfRowsInMemTable; SArray *dataBlockInfos; @@ -261,6 +269,12 @@ TsdbQueryHandleT *tsdbQueryTables(STsdbRepo *tsdb, STsdbQueryCond *pCond, STable TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, uint64_t qId, SMemRef *pRef); + +TsdbQueryHandleT tsdbQueryCacheLast(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef); + +bool isTsdbCacheLastRow(TsdbQueryHandleT* pQueryHandle); + + /** * get the queried table object list * @param pHandle diff --git a/src/kit/taosdemo/async-sub.json b/src/kit/taosdemo/async-sub.json new file mode 100644 index 0000000000..a30a1be45c --- /dev/null +++ b/src/kit/taosdemo/async-sub.json @@ -0,0 +1,41 @@ +{ + "filetype": "subscribe", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "databases": "test", + "specified_table_query": { + "concurrent": 1, + "mode": "async", + "interval": 1000, + "restart": "yes", + "keepProgress": "yes", + "resubAfterConsume": 10, + "sqls": [ + { + "sql": "select col1 from meters where col1 > 1;", + "result": "./subscribe_res0.txt" + }, + { + "sql": "select col2 from meters where col2 > 1;", + "result": "./subscribe_res2.txt" + } + ] + }, + "super_table_query": { + "stblname": "meters", + "threads": 1, + "mode": "sync", + "interval": 1000, + "restart": "yes", + "keepProgress": "yes", + "sqls": [ + { + "sql": "select col1 from xxxx where col1 > 10;", + "result": "./subscribe_res1.txt" + } + ] + } +} diff --git a/src/kit/taosdemo/subscribe.json b/src/kit/taosdemo/subscribe.json index fd33a2e2e2..9faf03a03d 100644 --- a/src/kit/taosdemo/subscribe.json +++ b/src/kit/taosdemo/subscribe.json @@ -1,17 +1,37 @@ { - "filetype":"subscribe", + "filetype": "subscribe", "cfgdir": "/etc/taos", "host": "127.0.0.1", "port": 6030, "user": "root", "password": "taosdata", - "databases": "dbx", - "specified_table_query": - {"concurrent":1, "mode":"sync", "interval":5000, "restart":"yes", "keepProgress":"yes", - "sqls": [{"sql": "select avg(col1) from stb01 where col1 > 1;", "result": "./subscribe_res0.txt"}] - }, - "super_table_query": - {"stblname": "stb", "threads":1, "mode":"sync", "interval":10000, "restart":"yes", "keepProgress":"yes", - "sqls": [{"sql": "select col1 from xxxx where col1 > 10;", "result": "./subscribe_res1.txt"}] - } + "databases": "test", + "specified_table_query": { + "concurrent": 1, + "mode": "sync", + "interval": 1000, + "restart": "yes", + "keepProgress": "yes", + "resubAfterConsume": 10, + "sqls": [ + { + "sql": "select avg(col1) from meters where col1 > 1;", + "result": "./subscribe_res0.txt" + } + ] + }, + "super_table_query": { + "stblname": "meters", + "threads": 1, + "mode": "sync", + "interval": 1000, + "restart": "yes", + "keepProgress": "yes", + "sqls": [ + { + "sql": "select col1 from xxxx where col1 > 10;", + "result": "./subscribe_res1.txt" + } + ] + } } diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index b4689a262d..6f5d407a73 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -19,6 +19,7 @@ */ #include +#include #define _GNU_SOURCE #define CURL_STATICLIB @@ -52,6 +53,8 @@ #include "taoserror.h" #include "tutil.h" +#define STMT_IFACE_ENABLED 1 + #define REQ_EXTRA_BUF_LEN 1024 #define RESP_BUF_LEN 4096 @@ -61,6 +64,8 @@ extern char configDir[]; #define QUERY_JSON_NAME "query.json" #define SUBSCRIBE_JSON_NAME "subscribe.json" +#define STR_INSERT_INTO "INSERT INTO " + enum TEST_MODE { INSERT_TEST, // 0 QUERY_TEST, // 1 @@ -70,6 +75,8 @@ enum TEST_MODE { #define MAX_RECORDS_PER_REQ 32766 +#define HEAD_BUFF_LEN 1024*24 // 16*1024 + (192+32)*2 + insert into .. + #define MAX_SQL_SIZE 65536 #define BUFFER_SIZE (65536*2) #define COND_BUF_LEN BUFFER_SIZE - 30 @@ -114,7 +121,7 @@ typedef enum TALBE_EXISTS_EN { TBL_EXISTS_BUTT } TALBE_EXISTS_EN; -enum MODE { +enum enumSYNC_MODE { SYNC_MODE, ASYNC_MODE, MODE_BUT @@ -127,6 +134,12 @@ enum enum_TAOS_INTERFACE { INTERFACE_BUT }; +typedef enum enumQUERY_CLASS { + SPECIFIED_CLASS, + STABLE_CLASS, + CLASS_BUT +} QUERY_CLASS; + typedef enum enum_PROGRESSIVE_OR_INTERLACE { PROGRESSIVE_INSERT_MODE, INTERLACE_INSERT_MODE, @@ -190,6 +203,8 @@ typedef struct { } SColDes; /* Used by main to communicate with parse_opt. */ +static char *g_dupstr = NULL; + typedef struct SArguments_S { char * metaFile; uint32_t test_mode; @@ -217,13 +232,13 @@ typedef struct SArguments_S { uint32_t num_of_threads; uint64_t insert_interval; int64_t query_times; - uint64_t interlace_rows; - uint64_t num_of_RPR; // num_of_records_per_req + uint32_t interlace_rows; + uint32_t num_of_RPR; // num_of_records_per_req uint64_t max_sql_len; int64_t num_of_tables; int64_t num_of_DPT; int abort; - int disorderRatio; // 0: no disorder, >0: x% + uint32_t disorderRatio; // 0: no disorder, >0: x% int disorderRange; // ms or us by database precision uint32_t method_of_delete; char ** arg_list; @@ -235,23 +250,24 @@ typedef struct SColumn_S { char field[TSDB_COL_NAME_LEN + 1]; char dataType[MAX_TB_NAME_SIZE]; uint32_t dataLen; - char note[128]; + char note[128]; } StrColumn; typedef struct SSuperTable_S { char sTblName[MAX_TB_NAME_SIZE+1]; + char dataSource[MAX_TB_NAME_SIZE+1]; // rand_gen or sample + char childTblPrefix[MAX_TB_NAME_SIZE]; + char insertMode[MAX_TB_NAME_SIZE]; // taosc, rest + uint16_t childTblExists; int64_t childTblCount; - bool childTblExists; // 0: no, 1: yes uint64_t batchCreateTableNum; // 0: no batch, > 0: batch table number in one sql uint8_t autoCreateTable; // 0: create sub table, 1: auto create sub table - char childTblPrefix[MAX_TB_NAME_SIZE]; - char dataSource[MAX_TB_NAME_SIZE+1]; // rand_gen or sample - uint16_t insertMode; // 0: taosc, 1: rest, 2: stmt + uint16_t iface; // 0: taosc, 1: rest, 2: stmt int64_t childTblLimit; uint64_t childTblOffset; // int multiThreadWriteOneTbl; // 0: no, 1: yes - uint64_t interlaceRows; // + uint32_t interlaceRows; // int disorderRatio; // 0: no disorder, >0: x% int disorderRange; // ms or us by database precision uint64_t maxSqlLen; // @@ -363,16 +379,21 @@ typedef struct SDbs_S { typedef struct SpecifiedQueryInfo_S { uint64_t queryInterval; // 0: unlimit > 0 loop/s - uint64_t concurrent; - uint64_t sqlCount; + uint32_t concurrent; + int sqlCount; uint32_t asyncMode; // 0: sync, 1: async uint64_t subscribeInterval; // ms uint64_t queryTimes; - int subscribeRestart; + bool subscribeRestart; int subscribeKeepProgress; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; + int resubAfterConsume[MAX_QUERY_SQL_COUNT]; + int endAfterConsume[MAX_QUERY_SQL_COUNT]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; + char topic[MAX_QUERY_SQL_COUNT][32]; + int consumed[MAX_QUERY_SQL_COUNT]; + TAOS_RES* res[MAX_QUERY_SQL_COUNT]; uint64_t totalQueried; } SpecifiedQueryInfo; @@ -382,14 +403,16 @@ typedef struct SuperQueryInfo_S { uint32_t threadCnt; uint32_t asyncMode; // 0: sync, 1: async uint64_t subscribeInterval; // ms - int subscribeRestart; + bool subscribeRestart; int subscribeKeepProgress; uint64_t queryTimes; int64_t childTblCount; char childTblPrefix[MAX_TB_NAME_SIZE]; - uint64_t sqlCount; + int sqlCount; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; + int resubAfterConsume; + int endAfterConsume; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; char* childTblName; @@ -407,7 +430,7 @@ typedef struct SQueryMetaInfo_S { char queryMode[MAX_TB_NAME_SIZE]; // taosc, rest SpecifiedQueryInfo specifiedQueryInfo; - SuperQueryInfo superQueryInfo; + SuperQueryInfo superQueryInfo; uint64_t totalQueried; } SQueryMetaInfo; @@ -417,7 +440,8 @@ typedef struct SThreadInfo_S { int threadID; char db_name[MAX_DB_NAME_SIZE+1]; uint32_t time_precision; - char fp[4096]; + char filePath[4096]; + FILE *fp; char tb_prefix[MAX_TB_NAME_SIZE]; uint64_t start_table_from; uint64_t end_table_to; @@ -449,8 +473,10 @@ typedef struct SThreadInfo_S { uint64_t maxDelay; uint64_t minDelay; - // query + // seq of query or subscribe uint64_t querySeq; // sequence number of sql command + TAOS_SUB* tsub; + } threadInfo; #ifdef WINDOWS @@ -526,11 +552,14 @@ static int taosRandom() #endif // ifdef Windows +static void prompt(); static int createDatabasesAndStables(); static void createChildTables(); static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet); -static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port, - char* sqlstr, char *resultFile); +static int postProceSql(char *host, struct sockaddr_in *pServAddr, + uint16_t port, char* sqlstr, threadInfo *pThreadInfo); +static int64_t getTSRandTail(int64_t timeStampStep, int32_t seq, + int disorderRatio, int disorderRange); /* ************ Global variables ************ */ @@ -664,7 +693,11 @@ static void printHelp() { printf("%s%s%s%s\n", indent, "-p", indent, "The TCP/IP port number to use for the connection. Default is 0."); printf("%s%s%s%s\n", indent, "-I", indent, +#if STMT_IFACE_ENABLED == 1 "The interface (taosc, rest, and stmt) taosdemo uses. Default is 'taosc'."); +#else + "The interface (taosc, rest) taosdemo uses. Default is 'taosc'."); +#endif printf("%s%s%s%s\n", indent, "-d", indent, "Destination database. Default is 'test'."); printf("%s%s%s%s\n", indent, "-a", indent, @@ -681,8 +714,9 @@ static void printHelp() { "The data_type of columns, default: INT,INT,INT,INT."); printf("%s%s%s%s\n", indent, "-w", indent, "The length of data_type 'BINARY' or 'NCHAR'. Default is 16"); - printf("%s%s%s%s\n", indent, "-l", indent, - "The number of columns per record. Default is 4."); + printf("%s%s%s%s%d\n", indent, "-l", indent, + "The number of columns per record. Default is 4. Max values is ", + MAX_NUM_DATATYPE); printf("%s%s%s%s\n", indent, "-T", indent, "The number of threads. Default is 10."); printf("%s%s%s%s\n", indent, "-i", indent, @@ -726,7 +760,6 @@ static bool isStringNumber(char *input) } static void parse_args(int argc, char *argv[], SArguments *arguments) { - char **sptr; for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "-f") == 0) { @@ -764,8 +797,10 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->iface = TAOSC_IFACE; } else if (0 == strcasecmp(argv[i], "rest")) { arguments->iface = REST_IFACE; +#if STMT_IFACE_ENABLED == 1 } else if (0 == strcasecmp(argv[i], "stmt")) { arguments->iface = STMT_IFACE; +#endif } else { errorPrint("%s", "\n\t-I need a valid string following!\n"); exit(EXIT_FAILURE); @@ -802,7 +837,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { if ((argc == i+1) || (!isStringNumber(argv[i+1]))) { printHelp(); - errorPrint("%s", "\n\t-q need a number following!\nQuery mode -- 0: SYNC, 1: ASYNC. Default is SYNC.\n"); + errorPrint("%s", "\n\t-q need a number following!\nQuery mode -- 0: SYNC, not-0: ASYNC. Default is SYNC.\n"); exit(EXIT_FAILURE); } arguments->async_mode = atoi(argv[++i]); @@ -870,20 +905,31 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } arguments->database = argv[++i]; } else if (strcmp(argv[i], "-l") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-l need a number following!\n"); - exit(EXIT_FAILURE); + if (argc == i+1) { + if (!isStringNumber(argv[i+1])) { + printHelp(); + errorPrint("%s", "\n\t-l need a number following!\n"); + exit(EXIT_FAILURE); + } } arguments->num_of_CPR = atoi(argv[++i]); + + if (arguments->num_of_CPR > MAX_NUM_DATATYPE) { + printf("WARNING: max acceptible columns count is %d\n", MAX_NUM_DATATYPE); + prompt(); + arguments->num_of_CPR = MAX_NUM_DATATYPE; + } + + for (int col = arguments->num_of_CPR; col < MAX_NUM_DATATYPE; col++) { + arguments->datatype[col] = NULL; + } + } else if (strcmp(argv[i], "-b") == 0) { if (argc == i+1) { printHelp(); errorPrint("%s", "\n\t-b need valid string following!\n"); exit(EXIT_FAILURE); } - sptr = arguments->datatype; ++i; if (strstr(argv[i], ",") == NULL) { // only one col @@ -895,17 +941,18 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { && strcasecmp(argv[i], "BIGINT") && strcasecmp(argv[i], "DOUBLE") && strcasecmp(argv[i], "BINARY") + && strcasecmp(argv[i], "TIMESTAMP") && strcasecmp(argv[i], "NCHAR")) { printHelp(); errorPrint("%s", "-b: Invalid data_type!\n"); exit(EXIT_FAILURE); } - sptr[0] = argv[i]; + arguments->datatype[0] = argv[i]; } else { // more than one col int index = 0; - char *dupstr = strdup(argv[i]); - char *running = dupstr; + g_dupstr = strdup(argv[i]); + char *running = g_dupstr; char *token = strsep(&running, ","); while(token != NULL) { if (strcasecmp(token, "INT") @@ -916,18 +963,18 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { && strcasecmp(token, "BIGINT") && strcasecmp(token, "DOUBLE") && strcasecmp(token, "BINARY") + && strcasecmp(token, "TIMESTAMP") && strcasecmp(token, "NCHAR")) { printHelp(); - free(dupstr); + free(g_dupstr); errorPrint("%s", "-b: Invalid data_type!\n"); exit(EXIT_FAILURE); } - sptr[index++] = token; + arguments->datatype[index++] = token; token = strsep(&running, ","); if (index >= MAX_NUM_DATATYPE) break; } - free(dupstr); - sptr[index] = NULL; + arguments->datatype[index] = NULL; } } else if (strcmp(argv[i], "-w") == 0) { if ((argc == i+1) || @@ -1018,6 +1065,19 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } } + int columnCount; + for (columnCount = 0; columnCount < MAX_NUM_DATATYPE; columnCount ++) { + if (g_args.datatype[columnCount] == NULL) { + break; + } + } + + if (0 == columnCount) { + perror("data type error!"); + exit(-1); + } + g_args.num_of_CPR = columnCount; + if (((arguments->debug_print) && (arguments->metaFile == NULL)) || arguments->verbose_print) { printf("###################################################################\n"); @@ -1040,7 +1100,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } printf("# Insertion interval: %"PRIu64"\n", arguments->insert_interval); - printf("# Number of records per req: %"PRIu64"\n", + printf("# Number of records per req: %u\n", arguments->num_of_RPR); printf("# Max SQL length: %"PRIu64"\n", arguments->max_sql_len); @@ -1062,16 +1122,12 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { printf("# Print debug info: %d\n", arguments->debug_print); printf("# Print verbose info: %d\n", arguments->verbose_print); printf("###################################################################\n"); - if (!arguments->answer_yes) { - printf("Press enter key to continue\n\n"); - (void) getchar(); - } + + prompt(); } } static bool getInfoFromJsonFile(char* file); -//static int generateOneRowDataForStb(SSuperTable* stbInfo); -//static int getDataIntoMemForStb(SSuperTable* stbInfo); static void init_rand_data(); static void tmfclose(FILE *fp) { if (NULL != fp) { @@ -1103,10 +1159,11 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) { } } + verbosePrint("%s() LN%d - command: %s\n", __func__, __LINE__, command); if (code != 0) { if (!quiet) { - debugPrint("%s() LN%d - command: %s\n", __func__, __LINE__, command); - errorPrint("Failed to execute %s, reason: %s\n", command, taos_errstr(res)); + errorPrint("Failed to execute %s, reason: %s\n", + command, taos_errstr(res)); } taos_free_result(res); //taos_close(taos); @@ -1123,24 +1180,22 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) { return 0; } -static void appendResultBufToFile(char *resultBuf, char *resultFile) +static void appendResultBufToFile(char *resultBuf, threadInfo *pThreadInfo) { - FILE *fp = NULL; - if (resultFile[0] != 0) { - fp = fopen(resultFile, "at"); - if (fp == NULL) { + pThreadInfo->fp = fopen(pThreadInfo->filePath, "at"); + if (pThreadInfo->fp == NULL) { errorPrint( "%s() LN%d, failed to open result file: %s, result will not save to file\n", - __func__, __LINE__, resultFile); + __func__, __LINE__, pThreadInfo->filePath); return; - } } - fprintf(fp, "%s", resultBuf); - tmfclose(fp); + fprintf(pThreadInfo->fp, "%s", resultBuf); + tmfclose(pThreadInfo->fp); + pThreadInfo->fp = NULL; } -static void appendResultToFile(TAOS_RES *res, char* resultFile) { +static void fetchResult(TAOS_RES *res, threadInfo* pThreadInfo) { TAOS_ROW row = NULL; int num_rows = 0; int num_fields = taos_field_count(res); @@ -1158,10 +1213,11 @@ static void appendResultToFile(TAOS_RES *res, char* resultFile) { // fetch the records row by row while((row = taos_fetch_row(res))) { - if (totalLen >= 100*1024*1024 - 32000) { - appendResultBufToFile(databuf, resultFile); - totalLen = 0; - memset(databuf, 0, 100*1024*1024); + if ((strlen(pThreadInfo->filePath) > 0) + && (totalLen >= 100*1024*1024 - 32000)) { + appendResultBufToFile(databuf, pThreadInfo); + totalLen = 0; + memset(databuf, 0, 100*1024*1024); } num_rows++; int len = taos_print_row(temp, row, fields, num_fields); @@ -1171,13 +1227,17 @@ static void appendResultToFile(TAOS_RES *res, char* resultFile) { totalLen += len; } - verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", __func__, __LINE__, databuf, resultFile); - appendResultBufToFile(databuf, resultFile); + verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", + __func__, __LINE__, databuf, pThreadInfo->filePath); + if (strlen(pThreadInfo->filePath) > 0) { + appendResultBufToFile(databuf, pThreadInfo); + } free(databuf); } static void selectAndGetResult( - threadInfo *pThreadInfo, char *command, char* resultFile) { + threadInfo *pThreadInfo, char *command) +{ if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) { TAOS_RES *res = taos_query(pThreadInfo->taos, command); if (res == NULL || taos_errno(res) != 0) { @@ -1187,16 +1247,14 @@ static void selectAndGetResult( return; } - if ((resultFile) && (strlen(resultFile))) { - appendResultToFile(res, resultFile); - } + fetchResult(res, pThreadInfo); taos_free_result(res); } else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) { int retCode = postProceSql( g_queryInfo.host, &(g_queryInfo.serv_addr), g_queryInfo.port, command, - resultFile); + pThreadInfo); if (0 != retCode) { printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID); } @@ -1323,16 +1381,18 @@ static int printfInsertMeta() { printf("interface: \033[33m%s\033[0m\n", (g_args.iface==TAOSC_IFACE)?"taosc":(g_args.iface==REST_IFACE)?"rest":"stmt"); - printf("host: \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port); + printf("host: \033[33m%s:%u\033[0m\n", + g_Dbs.host, g_Dbs.port); printf("user: \033[33m%s\033[0m\n", g_Dbs.user); printf("password: \033[33m%s\033[0m\n", g_Dbs.password); printf("configDir: \033[33m%s\033[0m\n", configDir); printf("resultFile: \033[33m%s\033[0m\n", g_Dbs.resultFile); printf("thread num of insert data: \033[33m%d\033[0m\n", g_Dbs.threadCount); - printf("thread num of create table: \033[33m%d\033[0m\n", g_Dbs.threadCountByCreateTbl); + printf("thread num of create table: \033[33m%d\033[0m\n", + g_Dbs.threadCountByCreateTbl); printf("top insert interval: \033[33m%"PRIu64"\033[0m\n", g_args.insert_interval); - printf("number of records per req: \033[33m%"PRIu64"\033[0m\n", + printf("number of records per req: \033[33m%u\033[0m\n", g_args.num_of_RPR); printf("max sql length: \033[33m%"PRIu64"\033[0m\n", g_args.max_sql_len); @@ -1341,7 +1401,8 @@ static int printfInsertMeta() { for (int i = 0; i < g_Dbs.dbCount; i++) { printf("database[\033[33m%d\033[0m]:\n", i); - printf(" database[%d] name: \033[33m%s\033[0m\n", i, g_Dbs.db[i].dbName); + printf(" database[%d] name: \033[33m%s\033[0m\n", + i, g_Dbs.db[i].dbName); if (0 == g_Dbs.db[i].drop) { printf(" drop: \033[33mno\033[0m\n"); } else { @@ -1349,40 +1410,51 @@ static int printfInsertMeta() { } if (g_Dbs.db[i].dbCfg.blocks > 0) { - printf(" blocks: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.blocks); + printf(" blocks: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.blocks); } if (g_Dbs.db[i].dbCfg.cache > 0) { - printf(" cache: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.cache); + printf(" cache: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.cache); } if (g_Dbs.db[i].dbCfg.days > 0) { - printf(" days: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.days); + printf(" days: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.days); } if (g_Dbs.db[i].dbCfg.keep > 0) { - printf(" keep: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.keep); + printf(" keep: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.keep); } if (g_Dbs.db[i].dbCfg.replica > 0) { - printf(" replica: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.replica); + printf(" replica: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.replica); } if (g_Dbs.db[i].dbCfg.update > 0) { - printf(" update: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.update); + printf(" update: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.update); } if (g_Dbs.db[i].dbCfg.minRows > 0) { - printf(" minRows: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.minRows); + printf(" minRows: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.minRows); } if (g_Dbs.db[i].dbCfg.maxRows > 0) { - printf(" maxRows: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.maxRows); + printf(" maxRows: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.maxRows); } if (g_Dbs.db[i].dbCfg.comp > 0) { printf(" comp: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp); } if (g_Dbs.db[i].dbCfg.walLevel > 0) { - printf(" walLevel: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.walLevel); + printf(" walLevel: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.walLevel); } if (g_Dbs.db[i].dbCfg.fsync > 0) { - printf(" fsync: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.fsync); + printf(" fsync: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.fsync); } if (g_Dbs.db[i].dbCfg.quorum > 0) { - printf(" quorum: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.quorum); + printf(" quorum: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.quorum); } if (g_Dbs.db[i].dbCfg.precision[0] != 0) { if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) @@ -1406,7 +1478,8 @@ static int printfInsertMeta() { if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { printf(" autoCreateTable: \033[33m%s\033[0m\n", "no"); - } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + } else if (AUTO_CREATE_SUBTBL == + g_Dbs.db[i].superTbls[j].autoCreateTable) { printf(" autoCreateTable: \033[33m%s\033[0m\n", "yes"); } else { printf(" autoCreateTable: \033[33m%s\033[0m\n", "error"); @@ -1426,9 +1499,9 @@ static int printfInsertMeta() { g_Dbs.db[i].superTbls[j].childTblPrefix); printf(" dataSource: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].dataSource); - printf(" insertMode: \033[33m%s\033[0m\n", - (g_Dbs.db[i].superTbls[j].insertMode==TAOSC_IFACE)?"taosc": - (g_Dbs.db[i].superTbls[j].insertMode==REST_IFACE)?"rest":"stmt"); + printf(" iface: \033[33m%s\033[0m\n", + (g_Dbs.db[i].superTbls[j].iface==TAOSC_IFACE)?"taosc": + (g_Dbs.db[i].superTbls[j].iface==REST_IFACE)?"rest":"stmt"); if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) { printf(" childTblLimit: \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].childTblLimit); @@ -1446,7 +1519,7 @@ static int printfInsertMeta() { printf(" multiThreadWriteOneTbl: \033[33myes\033[0m\n"); } */ - printf(" interlaceRows: \033[33m%"PRIu64"\033[0m\n", + printf(" interlaceRows: \033[33m%u\033[0m\n", g_Dbs.db[i].superTbls[j].interlaceRows); if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) { @@ -1524,7 +1597,7 @@ static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, "resultFile: %s\n", g_Dbs.resultFile); fprintf(fp, "thread num of insert data: %d\n", g_Dbs.threadCount); fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl); - fprintf(fp, "number of records per req: %"PRIu64"\n", g_args.num_of_RPR); + fprintf(fp, "number of records per req: %u\n", g_args.num_of_RPR); fprintf(fp, "max sql length: %"PRIu64"\n", g_args.max_sql_len); fprintf(fp, "database count: %d\n", g_Dbs.dbCount); @@ -1576,21 +1649,26 @@ static void printfInsertMetaToFile(FILE* fp) { if (g_Dbs.db[i].dbCfg.precision[0] != 0) { if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { - fprintf(fp, " precision: %s\n", g_Dbs.db[i].dbCfg.precision); + fprintf(fp, " precision: %s\n", + g_Dbs.db[i].dbCfg.precision); } else { - fprintf(fp, " precision error: %s\n", g_Dbs.db[i].dbCfg.precision); + fprintf(fp, " precision error: %s\n", + g_Dbs.db[i].dbCfg.precision); } } - fprintf(fp, " super table count: %"PRIu64"\n", g_Dbs.db[i].superTblCount); - for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { - fprintf(fp, " super table[%"PRIu64"]:\n", j); + fprintf(fp, " super table count: %"PRIu64"\n", + g_Dbs.db[i].superTblCount); + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + fprintf(fp, " super table[%d]:\n", j); - fprintf(fp, " stbName: %s\n", g_Dbs.db[i].superTbls[j].sTblName); + fprintf(fp, " stbName: %s\n", + g_Dbs.db[i].superTbls[j].sTblName); if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { fprintf(fp, " autoCreateTable: %s\n", "no"); - } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + } else if (AUTO_CREATE_SUBTBL + == g_Dbs.db[i].superTbls[j].autoCreateTable) { fprintf(fp, " autoCreateTable: %s\n", "yes"); } else { fprintf(fp, " autoCreateTable: %s\n", "error"); @@ -1598,7 +1676,8 @@ static void printfInsertMetaToFile(FILE* fp) { if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { fprintf(fp, " childTblExists: %s\n", "no"); - } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + } else if (TBL_ALREADY_EXISTS + == g_Dbs.db[i].superTbls[j].childTblExists) { fprintf(fp, " childTblExists: %s\n", "yes"); } else { fprintf(fp, " childTblExists: %s\n", "error"); @@ -1610,12 +1689,12 @@ static void printfInsertMetaToFile(FILE* fp) { g_Dbs.db[i].superTbls[j].childTblPrefix); fprintf(fp, " dataSource: %s\n", g_Dbs.db[i].superTbls[j].dataSource); - fprintf(fp, " insertMode: %s\n", - (g_Dbs.db[i].superTbls[j].insertMode==TAOSC_IFACE)?"taosc": - (g_Dbs.db[i].superTbls[j].insertMode==REST_IFACE)?"rest":"stmt"); + fprintf(fp, " iface: %s\n", + (g_Dbs.db[i].superTbls[j].iface==TAOSC_IFACE)?"taosc": + (g_Dbs.db[i].superTbls[j].iface==REST_IFACE)?"rest":"stmt"); fprintf(fp, " insertRows: %"PRId64"\n", g_Dbs.db[i].superTbls[j].insertRows); - fprintf(fp, " interlace rows: %"PRIu64"\n", + fprintf(fp, " interlace rows: %u\n", g_Dbs.db[i].superTbls[j].interlaceRows); if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) { fprintf(fp, " stable insert interval: %"PRIu64"\n", @@ -1628,10 +1707,12 @@ static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, " multiThreadWriteOneTbl: yes\n"); } */ - fprintf(fp, " interlaceRows: %"PRIu64"\n", + fprintf(fp, " interlaceRows: %u\n", g_Dbs.db[i].superTbls[j].interlaceRows); - fprintf(fp, " disorderRange: %d\n", g_Dbs.db[i].superTbls[j].disorderRange); - fprintf(fp, " disorderRatio: %d\n", g_Dbs.db[i].superTbls[j].disorderRatio); + fprintf(fp, " disorderRange: %d\n", + g_Dbs.db[i].superTbls[j].disorderRange); + fprintf(fp, " disorderRatio: %d\n", + g_Dbs.db[i].superTbls[j].disorderRatio); fprintf(fp, " maxSqlLen: %"PRIu64"\n", g_Dbs.db[i].superTbls[j].maxSqlLen); @@ -1639,23 +1720,29 @@ static void printfInsertMetaToFile(FILE* fp) { g_Dbs.db[i].superTbls[j].timeStampStep); fprintf(fp, " startTimestamp: %s\n", g_Dbs.db[i].superTbls[j].startTimestamp); - fprintf(fp, " sampleFormat: %s\n", g_Dbs.db[i].superTbls[j].sampleFormat); - fprintf(fp, " sampleFile: %s\n", g_Dbs.db[i].superTbls[j].sampleFile); - fprintf(fp, " tagsFile: %s\n", g_Dbs.db[i].superTbls[j].tagsFile); + fprintf(fp, " sampleFormat: %s\n", + g_Dbs.db[i].superTbls[j].sampleFormat); + fprintf(fp, " sampleFile: %s\n", + g_Dbs.db[i].superTbls[j].sampleFile); + fprintf(fp, " tagsFile: %s\n", + g_Dbs.db[i].superTbls[j].tagsFile); - fprintf(fp, " columnCount: %d\n ", g_Dbs.db[i].superTbls[j].columnCount); + fprintf(fp, " columnCount: %d\n ", + g_Dbs.db[i].superTbls[j].columnCount); for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); if ((0 == strncasecmp( g_Dbs.db[i].superTbls[j].columns[k].dataType, "binary", strlen("binary"))) - || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, + || (0 == strncasecmp( + g_Dbs.db[i].superTbls[j].columns[k].dataType, "nchar", strlen("nchar")))) { fprintf(fp, "column[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); } else { - fprintf(fp, "column[%d]:%s ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType); + fprintf(fp, "column[%d]:%s ", + k, g_Dbs.db[i].superTbls[j].columns[k].dataType); } } fprintf(fp, "\n"); @@ -1668,7 +1755,8 @@ static void printfInsertMetaToFile(FILE* fp) { "binary", strlen("binary"))) || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "nchar", strlen("nchar")))) { - fprintf(fp, "tag[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType, + fprintf(fp, "tag[%d]:%s(%d) ", + k, g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); } else { fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType); @@ -1695,7 +1783,7 @@ static void printfQueryMeta() { if ((SUBSCRIBE_TEST == g_args.test_mode) || (QUERY_TEST == g_args.test_mode)) { printf("specified table query info: \n"); - printf("sqlCount: \033[33m%"PRIu64"\033[0m\n", + printf("sqlCount: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.sqlCount); if (g_queryInfo.specifiedQueryInfo.sqlCount > 0) { printf("specified tbl query times:\n"); @@ -1704,7 +1792,7 @@ static void printfQueryMeta() { printf("query interval: \033[33m%"PRIu64" ms\033[0m\n", g_queryInfo.specifiedQueryInfo.queryInterval); printf("top query times:\033[33m%"PRIu64"\033[0m\n", g_args.query_times); - printf("concurrent: \033[33m%"PRIu64"\033[0m\n", + printf("concurrent: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.concurrent); printf("mod: \033[33m%s\033[0m\n", (g_queryInfo.specifiedQueryInfo.asyncMode)?"async":"sync"); @@ -1715,15 +1803,15 @@ static void printfQueryMeta() { printf("keepProgress: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); - for (uint64_t i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { - printf(" sql[%"PRIu64"]: \033[33m%s\033[0m\n", + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { + printf(" sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.specifiedQueryInfo.sql[i]); } printf("\n"); } printf("super table query info:\n"); - printf("sqlCount: \033[33m%"PRIu64"\033[0m\n", + printf("sqlCount: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.sqlCount); if (g_queryInfo.superQueryInfo.sqlCount > 0) { @@ -2001,13 +2089,13 @@ static void printfQuerySystemInfo(TAOS * taos) { // show variables res = taos_query(taos, "show variables;"); - //appendResultToFile(res, filename); + //fetchResult(res, filename); xDumpResultToFile(filename, res); // show dnodes res = taos_query(taos, "show dnodes;"); xDumpResultToFile(filename, res); - //appendResultToFile(res, filename); + //fetchResult(res, filename); // show databases res = taos_query(taos, "show databases;"); @@ -2043,7 +2131,7 @@ static void printfQuerySystemInfo(TAOS * taos) { } static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port, - char* sqlstr, char *resultFile) + char* sqlstr, threadInfo *pThreadInfo) { char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s"; @@ -2179,8 +2267,8 @@ static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port response_buf[RESP_BUF_LEN - 1] = '\0'; printf("Response:\n%s\n", response_buf); - if (resultFile) { - appendResultBufToFile(response_buf, resultFile); + if (strlen(pThreadInfo->filePath) > 0) { + appendResultBufToFile(response_buf, pThreadInfo); } free(request_buf); @@ -2672,8 +2760,6 @@ static int createSuperTable( snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s (ts timestamp%s) tags %s", dbName, superTbl->sTblName, cols, tags); - verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, command); - if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { errorPrint( "create supertable %s failed!\n\n", superTbl->sTblName); @@ -2696,7 +2782,6 @@ static int createDatabasesAndStables() { for (int i = 0; i < g_Dbs.dbCount; i++) { if (g_Dbs.db[i].drop) { sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName); - verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command); if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { taos_close(taos); return -1; @@ -2769,7 +2854,6 @@ static int createDatabasesAndStables() { " precision \'%s\';", g_Dbs.db[i].dbCfg.precision); } - debugPrint("%s() %d command: %s\n", __func__, __LINE__, command); if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { taos_close(taos); errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName); @@ -2786,8 +2870,6 @@ static int createDatabasesAndStables() { for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName); - verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command); - ret = queryDbExec(taos, command, NO_INSERT_TYPE, true); if ((ret != 0) || (g_Dbs.db[i].drop)) { @@ -2828,8 +2910,8 @@ static void* createTable(void *sarg) int buff_len; buff_len = BUFFER_SIZE / 8; - char *buffer = calloc(buff_len, 1); - if (buffer == NULL) { + pThreadInfo->buffer = calloc(buff_len, 1); + if (pThreadInfo->buffer == NULL) { errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__); exit(-1); } @@ -2844,7 +2926,7 @@ static void* createTable(void *sarg) for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { if (0 == g_Dbs.use_metric) { - snprintf(buffer, buff_len, + snprintf(pThreadInfo->buffer, buff_len, "create table if not exists %s.%s%"PRIu64" %s;", pThreadInfo->db_name, g_args.tb_prefix, i, @@ -2853,13 +2935,13 @@ static void* createTable(void *sarg) if (superTblInfo == NULL) { errorPrint("%s() LN%d, use metric, but super table info is NULL\n", __func__, __LINE__); - free(buffer); + free(pThreadInfo->buffer); exit(-1); } else { if (0 == len) { batchNum = 0; - memset(buffer, 0, buff_len); - len += snprintf(buffer + len, + memset(pThreadInfo->buffer, 0, buff_len); + len += snprintf(pThreadInfo->buffer + len, buff_len - len, "create table "); } char* tagsValBuf = NULL; @@ -2871,10 +2953,10 @@ static void* createTable(void *sarg) i % superTblInfo->tagSampleCount); } if (NULL == tagsValBuf) { - free(buffer); + free(pThreadInfo->buffer); return NULL; } - len += snprintf(buffer + len, + len += snprintf(pThreadInfo->buffer + len, buff_len - len, "if not exists %s.%s%"PRIu64" using %s.%s tags %s ", pThreadInfo->db_name, superTblInfo->childTblPrefix, @@ -2891,10 +2973,10 @@ static void* createTable(void *sarg) } len = 0; - verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer); - if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)){ - errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer); - free(buffer); + if (0 != queryDbExec(pThreadInfo->taos, pThreadInfo->buffer, + NO_INSERT_TYPE, false)){ + errorPrint( "queryDbExec() failed. buffer:\n%s\n", pThreadInfo->buffer); + free(pThreadInfo->buffer); return NULL; } @@ -2907,18 +2989,18 @@ static void* createTable(void *sarg) } if (0 != len) { - verbosePrint("%s() %d buffer: %s\n", __func__, __LINE__, buffer); - if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)) { - errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer); + if (0 != queryDbExec(pThreadInfo->taos, pThreadInfo->buffer, + NO_INSERT_TYPE, false)) { + errorPrint( "queryDbExec() failed. buffer:\n%s\n", pThreadInfo->buffer); } } - free(buffer); + free(pThreadInfo->buffer); return NULL; } static int startMultiThreadCreateChildTable( - char* cols, int threads, uint64_t startFrom, int64_t ntables, + char* cols, int threads, uint64_t tableFrom, int64_t ntables, char* db_name, SSuperTable* superTblInfo) { pthread_t *pids = malloc(threads * sizeof(pthread_t)); @@ -2942,19 +3024,19 @@ static int startMultiThreadCreateChildTable( int64_t b = 0; b = ntables % threads; - for (int i = 0; i < threads; i++) { - threadInfo *t_info = infos + i; - t_info->threadID = i; - tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE); - t_info->superTblInfo = superTblInfo; + for (int64_t i = 0; i < threads; i++) { + threadInfo *pThreadInfo = infos + i; + pThreadInfo->threadID = i; + tstrncpy(pThreadInfo->db_name, db_name, MAX_DB_NAME_SIZE); + pThreadInfo->superTblInfo = superTblInfo; verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name); - t_info->taos = taos_connect( + pThreadInfo->taos = taos_connect( g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); - if (t_info->taos == NULL) { + if (pThreadInfo->taos == NULL) { errorPrint( "%s() LN%d, Failed to connect to TDengine, reason:%s\n", __func__, __LINE__, taos_errstr(NULL)); free(pids); @@ -2962,14 +3044,14 @@ static int startMultiThreadCreateChildTable( return -1; } - t_info->start_table_from = startFrom; - t_info->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; - startFrom = t_info->end_table_to + 1; - t_info->use_metric = true; - t_info->cols = cols; - t_info->minDelay = UINT64_MAX; - pthread_create(pids + i, NULL, createTable, t_info); + pThreadInfo->start_table_from = tableFrom; + pThreadInfo->ntables = iend_table_to = i < b ? tableFrom + a : tableFrom + a - 1; + tableFrom = pThreadInfo->end_table_to + 1; + pThreadInfo->use_metric = true; + pThreadInfo->cols = cols; + pThreadInfo->minDelay = UINT64_MAX; + pthread_create(pids + i, NULL, createTable, pThreadInfo); } for (int i = 0; i < threads; i++) { @@ -2977,8 +3059,8 @@ static int startMultiThreadCreateChildTable( } for (int i = 0; i < threads; i++) { - threadInfo *t_info = infos + i; - taos_close(t_info->taos); + threadInfo *pThreadInfo = infos + i; + taos_close(pThreadInfo->taos); } free(pids); @@ -2991,61 +3073,61 @@ static void createChildTables() { char tblColsBuf[MAX_SQL_SIZE]; int len; - for (int i = 0; i < g_Dbs.dbCount; i++) { - if (g_Dbs.use_metric) { - if (g_Dbs.db[i].superTblCount > 0) { - // with super table - for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { - if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) - || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) { - continue; - } + for (int i = 0; i < g_Dbs.dbCount; i++) { + if (g_Dbs.use_metric) { + if (g_Dbs.db[i].superTblCount > 0) { + // with super table + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) + || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) { + continue; + } + verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, + g_Dbs.db[i].superTbls[j].colsOfCreateChildTable); + uint64_t startFrom = 0; + g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount; - verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, - g_Dbs.db[i].superTbls[j].colsOfCreateChildTable); - uint64_t startFrom = 0; - g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount; + verbosePrint("%s() LN%d: create %"PRId64" child tables from %"PRIu64"\n", + __func__, __LINE__, g_totalChildTables, startFrom); - verbosePrint("%s() LN%d: create %"PRId64" child tables from %"PRIu64"\n", - __func__, __LINE__, g_totalChildTables, startFrom); - startMultiThreadCreateChildTable( - g_Dbs.db[i].superTbls[j].colsOfCreateChildTable, - g_Dbs.threadCountByCreateTbl, - startFrom, - g_Dbs.db[i].superTbls[j].childTblCount, - g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j])); + startMultiThreadCreateChildTable( + g_Dbs.db[i].superTbls[j].colsOfCreateChildTable, + g_Dbs.threadCountByCreateTbl, + startFrom, + g_Dbs.db[i].superTbls[j].childTblCount, + g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j])); + } + } + } else { + // normal table + len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP"); + for (int j = 0; j < g_args.num_of_CPR; j++) { + if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0) + || (strncasecmp(g_args.datatype[j], + "NCHAR", strlen("NCHAR")) == 0)) { + snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, + ", COL%d %s(%d)", j, g_args.datatype[j], g_args.len_of_binary); + } else { + snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, + ", COL%d %s", j, g_args.datatype[j]); + } + len = strlen(tblColsBuf); + } + + snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")"); + + verbosePrint("%s() LN%d: dbName: %s num of tb: %"PRId64" schema: %s\n", + __func__, __LINE__, + g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf); + startMultiThreadCreateChildTable( + tblColsBuf, + g_Dbs.threadCountByCreateTbl, + 0, + g_args.num_of_tables, + g_Dbs.db[i].dbName, + NULL); } - } - } else { - // normal table - len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP"); - for (int j = 0; j < g_args.num_of_CPR; j++) { - if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0) - || (strncasecmp(g_args.datatype[j], - "NCHAR", strlen("NCHAR")) == 0)) { - snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, - ", COL%d %s(%d)", j, g_args.datatype[j], g_args.len_of_binary); - } else { - snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, - ", COL%d %s", j, g_args.datatype[j]); - } - len = strlen(tblColsBuf); - } - - snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")"); - - verbosePrint("%s() LN%d: dbName: %s num of tb: %"PRId64" schema: %s\n", - __func__, __LINE__, - g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf); - startMultiThreadCreateChildTable( - tblColsBuf, - g_Dbs.threadCountByCreateTbl, - 0, - g_args.num_of_tables, - g_Dbs.db[i].dbName, - NULL); } - } } /* @@ -3439,19 +3521,6 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } g_args.interlace_rows = interlaceRows->valueint; - - // rows per table need be less than insert batch - if (g_args.interlace_rows > g_args.num_of_RPR) { - printf("NOTICE: interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n", - g_args.interlace_rows, g_args.num_of_RPR); - printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", - g_args.num_of_RPR); - if (!g_args.answer_yes) { - printf(" press Enter key to continue or Ctrl-C to stop."); - (void)getchar(); - } - g_args.interlace_rows = g_args.num_of_RPR; - } } else if (!interlaceRows) { g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req } else { @@ -3483,6 +3552,11 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { __func__, __LINE__); goto PARSE_OVER; } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) { + printf("NOTICE: number of records per request value %"PRIu64" > %d\n\n", + numRecPerReq->valueint, MAX_RECORDS_PER_REQ); + printf(" number of records per request value will be set to %d\n\n", + MAX_RECORDS_PER_REQ); + prompt(); numRecPerReq->valueint = MAX_RECORDS_PER_REQ; } g_args.num_of_RPR = numRecPerReq->valueint; @@ -3506,12 +3580,22 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_args.answer_yes = false; } } else if (!answerPrompt) { - g_args.answer_yes = false; + g_args.answer_yes = true; // default is no, mean answer_yes. } else { - printf("ERROR: failed to read json, confirm_parameter_prompt not found\n"); + errorPrint("%s", "failed to read json, confirm_parameter_prompt input mistake\n"); goto PARSE_OVER; } + // rows per table need be less than insert batch + if (g_args.interlace_rows > g_args.num_of_RPR) { + printf("NOTICE: interlace rows value %u > num_of_records_per_req %u\n\n", + g_args.interlace_rows, g_args.num_of_RPR); + printf(" interlace rows value will be set to num_of_records_per_req %u\n\n", + g_args.num_of_RPR); + prompt(); + g_args.interlace_rows = g_args.num_of_RPR; + } + cJSON* dbs = cJSON_GetObjectItem(root, "databases"); if (!dbs || dbs->type != cJSON_Array) { printf("ERROR: failed to read json, databases not found\n"); @@ -3737,36 +3821,40 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { // dbinfo cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name"); - if (!stbName || stbName->type != cJSON_String || stbName->valuestring == NULL) { + if (!stbName || stbName->type != cJSON_String + || stbName->valuestring == NULL) { errorPrint("%s() LN%d, failed to read json, stb name not found\n", __func__, __LINE__); goto PARSE_OVER; } - tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, + MAX_TB_NAME_SIZE); cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix"); if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) { printf("ERROR: failed to read json, childtable_prefix not found\n"); goto PARSE_OVER; } - tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, + MAX_DB_NAME_SIZE); - cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); // yes, no, null + cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); if (autoCreateTbl && autoCreateTbl->type == cJSON_String && autoCreateTbl->valuestring != NULL) { - if (0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) { - g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL; - } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) { - g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; - } else { - g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; - } + if ((0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) + && (TBL_ALREADY_EXISTS != g_Dbs.db[i].superTbls[j].childTblExists)) { + g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL; + } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } else { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } } else if (!autoCreateTbl) { - g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; } else { - printf("ERROR: failed to read json, auto_create_table not found\n"); - goto PARSE_OVER; + printf("ERROR: failed to read json, auto_create_table not found\n"); + goto PARSE_OVER; } cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num"); @@ -3800,6 +3888,10 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } + if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } + cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count"); if (!count || count->type != cJSON_Number || 0 >= count->valueint) { errorPrint("%s() LN%d, failed to read json, childtable_count input mistake\n", @@ -3821,22 +3913,24 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } - cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , rest, stmt - if (insertMode && insertMode->type == cJSON_String - && insertMode->valuestring != NULL) { - if (0 == strcasecmp(insertMode->valuestring, "taosc")) { - g_Dbs.db[i].superTbls[j].insertMode = TAOSC_IFACE; - } else if (0 == strcasecmp(insertMode->valuestring, "rest")) { - g_Dbs.db[i].superTbls[j].insertMode = REST_IFACE; - } else if (0 == strcasecmp(insertMode->valuestring, "stmt")) { - g_Dbs.db[i].superTbls[j].insertMode = STMT_IFACE; + cJSON *stbIface = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , rest, stmt + if (stbIface && stbIface->type == cJSON_String + && stbIface->valuestring != NULL) { + if (0 == strcasecmp(stbIface->valuestring, "taosc")) { + g_Dbs.db[i].superTbls[j].iface= TAOSC_IFACE; + } else if (0 == strcasecmp(stbIface->valuestring, "rest")) { + g_Dbs.db[i].superTbls[j].iface= REST_IFACE; +#if STMT_IFACE_ENABLED == 1 + } else if (0 == strcasecmp(stbIface->valuestring, "stmt")) { + g_Dbs.db[i].superTbls[j].iface= STMT_IFACE; +#endif } else { errorPrint("%s() LN%d, failed to read json, insert_mode %s not recognized\n", - __func__, __LINE__, insertMode->valuestring); + __func__, __LINE__, stbIface->valuestring); goto PARSE_OVER; } - } else if (!insertMode) { - g_Dbs.db[i].superTbls[j].insertMode = TAOSC_IFACE; + } else if (!stbIface) { + g_Dbs.db[i].superTbls[j].iface = TAOSC_IFACE; } else { errorPrint("%s", "failed to read json, insert_mode not found\n"); goto PARSE_OVER; @@ -3857,7 +3951,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset"); if ((childTbl_offset) && (g_Dbs.db[i].drop != true) && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) { - if (childTbl_offset->type != cJSON_Number || 0 > childTbl_offset->valueint) { + if ((childTbl_offset->type != cJSON_Number) + || (0 > childTbl_offset->valueint)) { printf("ERROR: failed to read json, childtable_offset\n"); goto PARSE_OVER; } @@ -3913,7 +4008,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file"); - if (tagsFile && tagsFile->type == cJSON_String && tagsFile->valuestring != NULL) { + if ((tagsFile && tagsFile->type == cJSON_String) + && (tagsFile->valuestring != NULL)) { tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile, tagsFile->valuestring, MAX_FILE_NAME_LEN); if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) { @@ -3973,14 +4069,12 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_Dbs.db[i].superTbls[j].interlaceRows = stbInterlaceRows->valueint; // rows per table need be less than insert batch if (g_Dbs.db[i].superTbls[j].interlaceRows > g_args.num_of_RPR) { - printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n", - i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR); - printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", + printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %u > num_of_records_per_req %u\n\n", + i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR); - if (!g_args.answer_yes) { - printf(" press Enter key to continue or Ctrl-C to stop."); - (void)getchar(); - } + printf(" interlace rows value will be set to num_of_records_per_req %u\n\n", + g_args.num_of_RPR); + prompt(); g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR; } } else if (!stbInterlaceRows) { @@ -4176,7 +4270,8 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { "query_times"); if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) { if (specifiedQueryTimes->valueint <= 0) { - errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", + errorPrint( + "%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", __func__, __LINE__, specifiedQueryTimes->valueint); goto PARSE_OVER; @@ -4193,7 +4288,8 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* concurrent = cJSON_GetObjectItem(specifiedQuery, "concurrent"); if (concurrent && concurrent->type == cJSON_Number) { if (concurrent->valueint <= 0) { - errorPrint("%s() LN%d, query sqlCount %"PRIu64" or concurrent %"PRIu64" is not correct.\n", + errorPrint( + "%s() LN%d, query sqlCount %d or concurrent %d is not correct.\n", __func__, __LINE__, g_queryInfo.specifiedQueryInfo.sqlCount, g_queryInfo.specifiedQueryInfo.concurrent); @@ -4232,15 +4328,15 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* restart = cJSON_GetObjectItem(specifiedQuery, "restart"); if (restart && restart->type == cJSON_String && restart->valuestring != NULL) { if (0 == strcmp("yes", restart->valuestring)) { - g_queryInfo.specifiedQueryInfo.subscribeRestart = 1; + g_queryInfo.specifiedQueryInfo.subscribeRestart = true; } else if (0 == strcmp("no", restart->valuestring)) { - g_queryInfo.specifiedQueryInfo.subscribeRestart = 0; + g_queryInfo.specifiedQueryInfo.subscribeRestart = false; } else { printf("ERROR: failed to read json, subscribe restart error\n"); goto PARSE_OVER; } } else { - g_queryInfo.specifiedQueryInfo.subscribeRestart = 1; + g_queryInfo.specifiedQueryInfo.subscribeRestart = true; } cJSON* keepProgress = cJSON_GetObjectItem(specifiedQuery, "keepProgress"); @@ -4260,24 +4356,28 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } // sqls - cJSON* superSqls = cJSON_GetObjectItem(specifiedQuery, "sqls"); - if (!superSqls) { + cJSON* specifiedSqls = cJSON_GetObjectItem(specifiedQuery, "sqls"); + if (!specifiedSqls) { g_queryInfo.specifiedQueryInfo.sqlCount = 0; - } else if (superSqls->type != cJSON_Array) { + } else if (specifiedSqls->type != cJSON_Array) { errorPrint("%s() LN%d, failed to read json, super sqls not found\n", __func__, __LINE__); goto PARSE_OVER; } else { - int superSqlSize = cJSON_GetArraySize(superSqls); - if (superSqlSize > MAX_QUERY_SQL_COUNT) { - errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n", - __func__, __LINE__, MAX_QUERY_SQL_COUNT); + int superSqlSize = cJSON_GetArraySize(specifiedSqls); + if (superSqlSize * g_queryInfo.specifiedQueryInfo.concurrent + > MAX_QUERY_SQL_COUNT) { + errorPrint("%s() LN%d, failed to read json, query sql(%d) * concurrent(%d) overflow, max is %d\n", + __func__, __LINE__, + superSqlSize, + g_queryInfo.specifiedQueryInfo.concurrent, + MAX_QUERY_SQL_COUNT); goto PARSE_OVER; } g_queryInfo.specifiedQueryInfo.sqlCount = superSqlSize; for (int j = 0; j < superSqlSize; ++j) { - cJSON* sql = cJSON_GetArrayItem(superSqls, j); + cJSON* sql = cJSON_GetArrayItem(specifiedSqls, j); if (sql == NULL) continue; cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); @@ -4285,13 +4385,40 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { printf("ERROR: failed to read json, sql not found\n"); goto PARSE_OVER; } - tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j], + sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + + cJSON* endAfterConsume = + cJSON_GetObjectItem(specifiedQuery, "endAfterConsume"); + if (endAfterConsume + && endAfterConsume->type == cJSON_Number) { + g_queryInfo.specifiedQueryInfo.endAfterConsume[j] + = endAfterConsume->valueint; + } else if (!endAfterConsume) { + // default value is -1, which mean infinite loop + g_queryInfo.specifiedQueryInfo.endAfterConsume[j] = -1; + } + + cJSON* resubAfterConsume = + cJSON_GetObjectItem(specifiedQuery, "resubAfterConsume"); + if ((resubAfterConsume) + && (resubAfterConsume->type == cJSON_Number) + && (resubAfterConsume->valueint >= 0)) { + g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] + = resubAfterConsume->valueint; + } else if (!resubAfterConsume) { + // default value is -1, which mean do not resub + g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] = -1; + } cJSON *result = cJSON_GetObjectItem(sql, "result"); - if (NULL != result && result->type == cJSON_String && result->valuestring != NULL) { - tstrncpy(g_queryInfo.specifiedQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN); + if ((NULL != result) && (result->type == cJSON_String) + && (result->valuestring != NULL)) { + tstrncpy(g_queryInfo.specifiedQueryInfo.result[j], + result->valuestring, MAX_FILE_NAME_LEN); } else if (NULL == result) { - memset(g_queryInfo.specifiedQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); + memset(g_queryInfo.specifiedQueryInfo.result[j], + 0, MAX_FILE_NAME_LEN); } else { printf("ERROR: failed to read json, super query result file not found\n"); goto PARSE_OVER; @@ -4398,43 +4525,66 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { if (subrestart && subrestart->type == cJSON_String && subrestart->valuestring != NULL) { if (0 == strcmp("yes", subrestart->valuestring)) { - g_queryInfo.superQueryInfo.subscribeRestart = 1; + g_queryInfo.superQueryInfo.subscribeRestart = true; } else if (0 == strcmp("no", subrestart->valuestring)) { - g_queryInfo.superQueryInfo.subscribeRestart = 0; + g_queryInfo.superQueryInfo.subscribeRestart = false; } else { printf("ERROR: failed to read json, subscribe restart error\n"); goto PARSE_OVER; } } else { - g_queryInfo.superQueryInfo.subscribeRestart = 1; + g_queryInfo.superQueryInfo.subscribeRestart = true; } - cJSON* subkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress"); - if (subkeepProgress && - subkeepProgress->type == cJSON_String - && subkeepProgress->valuestring != NULL) { - if (0 == strcmp("yes", subkeepProgress->valuestring)) { + cJSON* superkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress"); + if (superkeepProgress && + superkeepProgress->type == cJSON_String + && superkeepProgress->valuestring != NULL) { + if (0 == strcmp("yes", superkeepProgress->valuestring)) { g_queryInfo.superQueryInfo.subscribeKeepProgress = 1; - } else if (0 == strcmp("no", subkeepProgress->valuestring)) { + } else if (0 == strcmp("no", superkeepProgress->valuestring)) { g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; } else { - printf("ERROR: failed to read json, subscribe keepProgress error\n"); + printf("ERROR: failed to read json, subscribe super table keepProgress error\n"); goto PARSE_OVER; } } else { g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; } - // sqls - cJSON* subsqls = cJSON_GetObjectItem(superQuery, "sqls"); - if (!subsqls) { + cJSON* superEndAfterConsume = + cJSON_GetObjectItem(superQuery, "endAfterConsume"); + if (superEndAfterConsume + && superEndAfterConsume->type == cJSON_Number) { + g_queryInfo.superQueryInfo.endAfterConsume = + superEndAfterConsume->valueint; + } else if (!superEndAfterConsume) { + // default value is -1, which mean do not resub + g_queryInfo.superQueryInfo.endAfterConsume = -1; + } + + cJSON* superResubAfterConsume = + cJSON_GetObjectItem(superQuery, "resubAfterConsume"); + if ((superResubAfterConsume) + && (superResubAfterConsume->type == cJSON_Number) + && (superResubAfterConsume->valueint >= 0)) { + g_queryInfo.superQueryInfo.resubAfterConsume = + superResubAfterConsume->valueint; + } else if (!superResubAfterConsume) { + // default value is -1, which mean do not resub + g_queryInfo.superQueryInfo.resubAfterConsume = -1; + } + + // supert table sqls + cJSON* superSqls = cJSON_GetObjectItem(superQuery, "sqls"); + if (!superSqls) { g_queryInfo.superQueryInfo.sqlCount = 0; - } else if (subsqls->type != cJSON_Array) { + } else if (superSqls->type != cJSON_Array) { errorPrint("%s() LN%d: failed to read json, super sqls not found\n", __func__, __LINE__); goto PARSE_OVER; } else { - int superSqlSize = cJSON_GetArraySize(subsqls); + int superSqlSize = cJSON_GetArraySize(superSqls); if (superSqlSize > MAX_QUERY_SQL_COUNT) { errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n", __func__, __LINE__, MAX_QUERY_SQL_COUNT); @@ -4443,7 +4593,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { g_queryInfo.superQueryInfo.sqlCount = superSqlSize; for (int j = 0; j < superSqlSize; ++j) { - cJSON* sql = cJSON_GetArrayItem(subsqls, j); + cJSON* sql = cJSON_GetArrayItem(superSqls, j); if (sql == NULL) continue; cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); @@ -4603,16 +4753,22 @@ static int getRowDataFromSample( return dataLen; } -static int64_t generateRowData(char* recBuf, int64_t timestamp, SSuperTable* stbInfo) { +static int64_t generateStbRowData( + SSuperTable* stbInfo, + char* recBuf, int64_t timestamp) +{ int64_t dataLen = 0; char *pstr = recBuf; int64_t maxLen = MAX_DATA_SIZE; - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "(%" PRId64 ",", timestamp); + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, + "(%" PRId64 ",", timestamp); for (int i = 0; i < stbInfo->columnCount; i++) { - if ((0 == strncasecmp(stbInfo->columns[i].dataType, "BINARY", strlen("BINARY"))) - || (0 == strncasecmp(stbInfo->columns[i].dataType, "NCHAR", strlen("NCHAR")))) { + if ((0 == strncasecmp(stbInfo->columns[i].dataType, + "BINARY", strlen("BINARY"))) + || (0 == strncasecmp(stbInfo->columns[i].dataType, + "NCHAR", strlen("NCHAR")))) { if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) { errorPrint( "binary or nchar length overflow, max size:%u\n", (uint32_t)TSDB_MAX_BINARY_LEN); @@ -4628,23 +4784,23 @@ static int64_t generateRowData(char* recBuf, int64_t timestamp, SSuperTable* stb dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "\'%s\',", buf); tmfree(buf); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, - "INT", 3)) { + "INT", strlen("INT"))) { dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%d,", rand_int()); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, - "BIGINT", 6)) { + "BIGINT", strlen("BIGINT"))) { dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%"PRId64",", rand_bigint()); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, - "FLOAT", 5)) { + "FLOAT", strlen("FLOAT"))) { dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%f,", rand_float()); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, - "DOUBLE", 6)) { + "DOUBLE", strlen("DOUBLE"))) { dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%f,", rand_double()); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, - "SMALLINT", 8)) { + "SMALLINT", strlen("SMALLINT"))) { dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%d,", rand_smallint()); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, @@ -4674,46 +4830,38 @@ static int64_t generateRowData(char* recBuf, int64_t timestamp, SSuperTable* stb } static int64_t generateData(char *recBuf, char **data_type, - int num_of_cols, int64_t timestamp, int lenOfBinary) { + int64_t timestamp, int lenOfBinary) { memset(recBuf, 0, MAX_DATA_SIZE); char *pstr = recBuf; pstr += sprintf(pstr, "(%" PRId64, timestamp); - int c = 0; - for (; c < MAX_NUM_DATATYPE; c++) { - if (data_type[c] == NULL) { - break; - } - } + int columnCount = g_args.num_of_CPR; - if (0 == c) { - perror("data type error!"); - exit(-1); - } - - for (int i = 0; i < c; i++) { - if (strcasecmp(data_type[i % c], "TINYINT") == 0) { + for (int i = 0; i < columnCount; i++) { + if (strcasecmp(data_type[i % columnCount], "TINYINT") == 0) { pstr += sprintf(pstr, ",%d", rand_tinyint() ); - } else if (strcasecmp(data_type[i % c], "SMALLINT") == 0) { + } else if (strcasecmp(data_type[i % columnCount], "SMALLINT") == 0) { pstr += sprintf(pstr, ",%d", rand_smallint()); - } else if (strcasecmp(data_type[i % c], "INT") == 0) { + } else if (strcasecmp(data_type[i % columnCount], "INT") == 0) { pstr += sprintf(pstr, ",%d", rand_int()); - } else if (strcasecmp(data_type[i % c], "BIGINT") == 0) { + } else if (strcasecmp(data_type[i % columnCount], "BIGINT") == 0) { pstr += sprintf(pstr, ",%" PRId64, rand_bigint()); - } else if (strcasecmp(data_type[i % c], "FLOAT") == 0) { + } else if (strcasecmp(data_type[i % columnCount], "TIMESTAMP") == 0) { + pstr += sprintf(pstr, ",%" PRId64, rand_bigint()); + } else if (strcasecmp(data_type[i % columnCount], "FLOAT") == 0) { pstr += sprintf(pstr, ",%10.4f", rand_float()); - } else if (strcasecmp(data_type[i % c], "DOUBLE") == 0) { + } else if (strcasecmp(data_type[i % columnCount], "DOUBLE") == 0) { double t = rand_double(); pstr += sprintf(pstr, ",%20.8f", t); - } else if (strcasecmp(data_type[i % c], "BOOL") == 0) { - bool b = taosRandom() & 1; + } else if (strcasecmp(data_type[i % columnCount], "BOOL") == 0) { + bool b = rand_bool() & 1; pstr += sprintf(pstr, ",%s", b ? "true" : "false"); - } else if (strcasecmp(data_type[i % c], "BINARY") == 0) { + } else if (strcasecmp(data_type[i % columnCount], "BINARY") == 0) { char *s = malloc(lenOfBinary); rand_string(s, lenOfBinary); pstr += sprintf(pstr, ",\"%s\"", s); free(s); - } else if (strcasecmp(data_type[i % c], "NCHAR") == 0) { + } else if (strcasecmp(data_type[i % columnCount], "NCHAR") == 0) { char *s = malloc(lenOfBinary); rand_string(s, lenOfBinary); pstr += sprintf(pstr, ",\"%s\"", s); @@ -4760,171 +4908,134 @@ static int prepareSampleDataForSTable(SSuperTable *superTblInfo) { return 0; } -static int64_t execInsert(threadInfo *pThreadInfo, uint64_t k) +static int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) { - int affectedRows; - SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + int32_t affectedRows; + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - verbosePrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID, + verbosePrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->buffer); - if (superTblInfo) { - if (superTblInfo->insertMode == TAOSC_IFACE) { - affectedRows = queryDbExec( - pThreadInfo->taos, - pThreadInfo->buffer, INSERT_TYPE, false); - } else if (superTblInfo->insertMode == REST_IFACE) { - if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port, - pThreadInfo->buffer, NULL /* not set result file */)) { - affectedRows = -1; - printf("========restful return fail, threadID[%d]\n", - pThreadInfo->threadID); - } else { - affectedRows = k; - } - } else if (superTblInfo->insertMode == STMT_IFACE) { - debugPrint("%s() LN%d, stmt=%p", __func__, __LINE__, pThreadInfo->stmt); - if (0 != taos_stmt_execute(pThreadInfo->stmt)) { - errorPrint("%s() LN%d, failied to execute insert statement\n", - __func__, __LINE__); - exit(-1); - } - affectedRows = k; - } else { - errorPrint("%s() LN%d: unknown insert mode: %d\n", - __func__, __LINE__, superTblInfo->insertMode); - affectedRows = 0; + uint16_t iface; + if (superTblInfo) + iface = superTblInfo->iface; + else + iface = g_args.iface; + + debugPrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID, + __func__, __LINE__, + (g_args.iface==TAOSC_IFACE)? + "taosc":(g_args.iface==REST_IFACE)?"rest":"stmt"); + + switch(iface) { + case TAOSC_IFACE: + affectedRows = queryDbExec( + pThreadInfo->taos, + pThreadInfo->buffer, INSERT_TYPE, false); + break; + + case REST_IFACE: + if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port, + pThreadInfo->buffer, pThreadInfo)) { + affectedRows = -1; + printf("========restful return fail, threadID[%d]\n", + pThreadInfo->threadID); + } else { + affectedRows = k; + } + break; + +#if STMT_IFACE_ENABLED == 1 + case STMT_IFACE: + debugPrint("%s() LN%d, stmt=%p", __func__, __LINE__, pThreadInfo->stmt); + if (0 != taos_stmt_execute(pThreadInfo->stmt)) { + errorPrint("%s() LN%d, failied to execute insert statement\n", + __func__, __LINE__); + exit(-1); + } + affectedRows = k; + break; +#endif + + default: + errorPrint("%s() LN%d: unknown insert mode: %d\n", + __func__, __LINE__, superTblInfo->iface); + affectedRows = 0; } - } else { - affectedRows = queryDbExec(pThreadInfo->taos, pThreadInfo->buffer, INSERT_TYPE, false); - } - return affectedRows; + return affectedRows; } -static void getTableName(char *pTblName, threadInfo* pThreadInfo, uint64_t tableSeq) +static void getTableName(char *pTblName, + threadInfo* pThreadInfo, uint64_t tableSeq) { - SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - if (superTblInfo) { - if (superTblInfo->childTblLimit > 0) { - snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s", - superTblInfo->childTblName + - (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN); + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + if (superTblInfo) { + if (AUTO_CREATE_SUBTBL != superTblInfo->autoCreateTable) { + if (superTblInfo->childTblLimit > 0) { + snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s", + superTblInfo->childTblName + + (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN); + } else { + verbosePrint("[%d] %s() LN%d: from=%"PRIu64" count=%"PRId64" seq=%"PRIu64"\n", + pThreadInfo->threadID, __func__, __LINE__, + pThreadInfo->start_table_from, + pThreadInfo->ntables, tableSeq); + snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s", + superTblInfo->childTblName + tableSeq * TSDB_TABLE_NAME_LEN); + } + } else { + snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%"PRIu64"", + superTblInfo->childTblPrefix, tableSeq); + } } else { - - verbosePrint("[%d] %s() LN%d: from=%"PRIu64" count=%"PRId64" seq=%"PRIu64"\n", - pThreadInfo->threadID, __func__, __LINE__, - pThreadInfo->start_table_from, - pThreadInfo->ntables, tableSeq); - snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s", - superTblInfo->childTblName + tableSeq * TSDB_TABLE_NAME_LEN); + snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%"PRIu64"", + g_args.tb_prefix, tableSeq); } - } else { - snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%"PRIu64"", - g_args.tb_prefix, tableSeq); - } } -static int64_t generateDataTail( - SSuperTable* superTblInfo, - uint64_t batch, char* buffer, int64_t remainderBufLen, int64_t insertRows, - uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *dataLen) { - uint64_t len = 0; - uint32_t ncols_per_record = 1; // count first col ts +static int32_t generateDataTailWithoutStb( + uint32_t batch, char* buffer, + int64_t remainderBufLen, int64_t insertRows, + uint64_t recordFrom, int64_t startTime, + /* int64_t *pSamplePos, */int64_t *dataLen) { + uint64_t len = 0; char *pstr = buffer; - if (superTblInfo == NULL) { - uint32_t datatypeSeq = 0; - while(g_args.datatype[datatypeSeq]) { - datatypeSeq ++; - ncols_per_record ++; - } - } + verbosePrint("%s() LN%d batch=%d\n", __func__, __LINE__, batch); - verbosePrint("%s() LN%d batch=%"PRIu64"\n", __func__, __LINE__, batch); - - uint64_t k = 0; + int32_t k = 0; for (k = 0; k < batch;) { char data[MAX_DATA_SIZE]; memset(data, 0, MAX_DATA_SIZE); int64_t retLen = 0; - if (superTblInfo) { - if (0 == strncasecmp(superTblInfo->dataSource, - "sample", strlen("sample"))) { - retLen = getRowDataFromSample( - data, - remainderBufLen, - startTime + superTblInfo->timeStampStep * k, - superTblInfo, - pSamplePos); - } else if (0 == strncasecmp(superTblInfo->dataSource, - "rand", strlen("rand"))) { + char **data_type = g_args.datatype; + int lenOfBinary = g_args.len_of_binary; - int64_t randTail = superTblInfo->timeStampStep * k; - if (superTblInfo->disorderRatio > 0) { - int rand_num = taosRandom() % 100; - if(rand_num < superTblInfo->disorderRatio) { - randTail = (randTail + (taosRandom() % superTblInfo->disorderRange + 1)) * (-1); - debugPrint("rand data generated, back %"PRId64"\n", randTail); - } - } + retLen = generateData(data, data_type, + startTime + getTSRandTail( + (int64_t) DEFAULT_TIMESTAMP_STEP, k, + g_args.disorderRatio, + g_args.disorderRange), + lenOfBinary); - int64_t d = startTime - + randTail; - retLen = generateRowData( - data, - d, - superTblInfo); - } - - if (retLen > remainderBufLen) { - break; - } - - pstr += snprintf(pstr , retLen + 1, "%s", data); - k++; - len += retLen; - remainderBufLen -= retLen; - } else { - char **data_type = g_args.datatype; - int lenOfBinary = g_args.len_of_binary; - - int64_t randTail = DEFAULT_TIMESTAMP_STEP * k; - - if (g_args.disorderRatio != 0) { - int rand_num = taosRandom() % 100; - if (rand_num < g_args.disorderRatio) { - randTail = (randTail + (taosRandom() % g_args.disorderRange + 1)) * (-1); - - debugPrint("rand data generated, back %"PRId64"\n", randTail); - } - } else { - randTail = DEFAULT_TIMESTAMP_STEP * k; - } - - retLen = generateData(data, data_type, - ncols_per_record, - startTime + randTail, - lenOfBinary); - - if (len > remainderBufLen) + if (len > remainderBufLen) break; - pstr += sprintf(pstr, "%s", data); - k++; - len += retLen; - remainderBufLen -= retLen; - } + pstr += sprintf(pstr, "%s", data); + k++; + len += retLen; + remainderBufLen -= retLen; - verbosePrint("%s() LN%d len=%"PRIu64" k=%"PRIu64" \nbuffer=%s\n", + verbosePrint("%s() LN%d len=%"PRIu64" k=%d \nbuffer=%s\n", __func__, __LINE__, len, k, buffer); - startFrom ++; + recordFrom ++; - if (startFrom >= insertRows) { + if (recordFrom >= insertRows) { break; } } @@ -4933,17 +5044,121 @@ static int64_t generateDataTail( return k; } -static int generateSQLHead(char *tableName, int32_t tableSeq, - threadInfo* pThreadInfo, SSuperTable* superTblInfo, +static int64_t getTSRandTail(int64_t timeStampStep, int32_t seq, + int disorderRatio, int disorderRange) +{ + int64_t randTail = timeStampStep * seq; + if (disorderRatio > 0) { + int rand_num = taosRandom() % 100; + if(rand_num < disorderRatio) { + randTail = (randTail + + (taosRandom() % disorderRange + 1)) * (-1); + debugPrint("rand data generated, back %"PRId64"\n", randTail); + } + } + + return randTail; +} + +static int32_t generateStbDataTail( + SSuperTable* superTblInfo, + uint32_t batch, char* buffer, + int64_t remainderBufLen, int64_t insertRows, + uint64_t recordFrom, int64_t startTime, + int64_t *pSamplePos, int64_t *dataLen) { + uint64_t len = 0; + + char *pstr = buffer; + + bool tsRand; + if (0 == strncasecmp(superTblInfo->dataSource, "rand", strlen("rand"))) { + tsRand = true; + } else { + tsRand = false; + } + verbosePrint("%s() LN%d batch=%u\n", __func__, __LINE__, batch); + + int32_t k = 0; + for (k = 0; k < batch;) { + char data[MAX_DATA_SIZE]; + memset(data, 0, MAX_DATA_SIZE); + + int64_t retLen = 0; + + if (tsRand) { + retLen = generateStbRowData(superTblInfo, data, + startTime + getTSRandTail( + superTblInfo->timeStampStep, k, + superTblInfo->disorderRatio, + superTblInfo->disorderRange) + ); + } else { + retLen = getRowDataFromSample( + data, + remainderBufLen, + startTime + superTblInfo->timeStampStep * k, + superTblInfo, + pSamplePos); + } + + if (retLen > remainderBufLen) { + break; + } + + pstr += snprintf(pstr , retLen + 1, "%s", data); + k++; + len += retLen; + remainderBufLen -= retLen; + + verbosePrint("%s() LN%d len=%"PRIu64" k=%u \nbuffer=%s\n", + __func__, __LINE__, len, k, buffer); + + recordFrom ++; + + if (recordFrom >= insertRows) { + break; + } + } + + *dataLen = len; + return k; +} + + +static int generateSQLHeadWithoutStb(char *tableName, + char *dbName, char *buffer, int remainderBufLen) { int len; -#define HEAD_BUFF_LEN 1024*24 // 16*1024 + (192+32)*2 + insert into .. char headBuf[HEAD_BUFF_LEN]; - if (superTblInfo) { - if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { + len = snprintf( + headBuf, + HEAD_BUFF_LEN, + "%s.%s values", + dbName, + tableName); + + if (len > remainderBufLen) + return -1; + + tstrncpy(buffer, headBuf, len + 1); + + return len; +} + +static int generateStbSQLHead( + SSuperTable* superTblInfo, + char *tableName, int32_t tableSeq, + char *dbName, + char *buffer, int remainderBufLen) +{ + int len; + + char headBuf[HEAD_BUFF_LEN]; + + if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { char* tagsValBuf = NULL; if (0 == superTblInfo->tagSource) { tagsValBuf = generateTagVaulesForStb(superTblInfo, tableSeq); @@ -4962,9 +5177,9 @@ static int generateSQLHead(char *tableName, int32_t tableSeq, headBuf, HEAD_BUFF_LEN, "%s.%s using %s.%s tags %s values", - pThreadInfo->db_name, + dbName, tableName, - pThreadInfo->db_name, + dbName, superTblInfo->sTblName, tagsValBuf); tmfree(tagsValBuf); @@ -4973,22 +5188,14 @@ static int generateSQLHead(char *tableName, int32_t tableSeq, headBuf, HEAD_BUFF_LEN, "%s.%s values", - pThreadInfo->db_name, + dbName, tableName); } else { len = snprintf( headBuf, HEAD_BUFF_LEN, "%s.%s values", - pThreadInfo->db_name, - tableName); - } - } else { - len = snprintf( - headBuf, - HEAD_BUFF_LEN, - "%s.%s values", - pThreadInfo->db_name, + dbName, tableName); } @@ -5000,8 +5207,11 @@ static int generateSQLHead(char *tableName, int32_t tableSeq, return len; } -static int64_t generateInterlaceDataBuffer( - char *tableName, uint64_t batchPerTbl, uint64_t i, uint64_t batchPerTblTimes, +static int32_t generateStbInterlaceData( + SSuperTable *superTblInfo, + char *tableName, uint32_t batchPerTbl, + uint64_t i, + uint32_t batchPerTblTimes, uint64_t tableSeq, threadInfo *pThreadInfo, char *buffer, int64_t insertRows, @@ -5010,10 +5220,11 @@ static int64_t generateInterlaceDataBuffer( { assert(buffer); char *pstr = buffer; - SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - int headLen = generateSQLHead(tableName, tableSeq, pThreadInfo, - superTblInfo, pstr, *pRemainderBufLen); + int headLen = generateStbSQLHead( + superTblInfo, + tableName, tableSeq, pThreadInfo->db_name, + pstr, *pRemainderBufLen); if (headLen <= 0) { return 0; @@ -5027,29 +5238,25 @@ static int64_t generateInterlaceDataBuffer( int64_t dataLen = 0; - verbosePrint("[%d] %s() LN%d i=%"PRIu64" batchPerTblTimes=%"PRIu64" batchPerTbl = %"PRIu64"\n", + verbosePrint("[%d] %s() LN%d i=%"PRIu64" batchPerTblTimes=%u batchPerTbl = %u\n", pThreadInfo->threadID, __func__, __LINE__, i, batchPerTblTimes, batchPerTbl); - if (superTblInfo) { - if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { + if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { startTime = taosGetTimestamp(pThreadInfo->time_precision); - } - } else { - startTime = 1500000000000; } - int64_t k = generateDataTail( - superTblInfo, - batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0, - startTime, - &(pThreadInfo->samplePos), &dataLen); + int32_t k = generateStbDataTail( + superTblInfo, + batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0, + startTime, + &(pThreadInfo->samplePos), &dataLen); if (k == batchPerTbl) { pstr += dataLen; *pRemainderBufLen -= dataLen; } else { - debugPrint("%s() LN%d, generated data tail: %"PRIu64", not equal batch per table: %"PRIu64"\n", + debugPrint("%s() LN%d, generated data tail: %u, not equal batch per table: %u\n", __func__, __LINE__, k, batchPerTbl); pstr -= headLen; pstr[0] = '\0'; @@ -5059,34 +5266,363 @@ static int64_t generateInterlaceDataBuffer( return k; } -static int64_t generateProgressiveDataBuffer( - char *tableName, - int64_t tableSeq, - threadInfo *pThreadInfo, char *buffer, +static int64_t generateInterlaceDataWithoutStb( + char *tableName, uint32_t batch, + uint64_t tableSeq, + char *dbName, char *buffer, int64_t insertRows, - uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, - int64_t *pRemainderBufLen) + int64_t startTime, + uint64_t *pRemainderBufLen) { - SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + assert(buffer); + char *pstr = buffer; - int ncols_per_record = 1; // count first col ts + int headLen = generateSQLHeadWithoutStb( + tableName, dbName, + pstr, *pRemainderBufLen); - if (superTblInfo == NULL) { - int datatypeSeq = 0; - while(g_args.datatype[datatypeSeq]) { - datatypeSeq ++; - ncols_per_record ++; - } + if (headLen <= 0) { + return 0; } + pstr += headLen; + *pRemainderBufLen -= headLen; + + int64_t dataLen = 0; + + int32_t k = generateDataTailWithoutStb( + batch, pstr, *pRemainderBufLen, insertRows, 0, + startTime, + &dataLen); + + if (k == batch) { + pstr += dataLen; + *pRemainderBufLen -= dataLen; + } else { + debugPrint("%s() LN%d, generated data tail: %d, not equal batch per table: %u\n", + __func__, __LINE__, k, batch); + pstr -= headLen; + pstr[0] = '\0'; + k = 0; + } + + return k; +} + +#if STMT_IFACE_ENABLED == 1 +static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, + char *dataType, int32_t dataLen, char **ptr) +{ + if (0 == strncasecmp(dataType, + "BINARY", strlen("BINARY"))) { + if (dataLen > TSDB_MAX_BINARY_LEN) { + errorPrint( "binary length overflow, max size:%u\n", + (uint32_t)TSDB_MAX_BINARY_LEN); + return -1; + } + char *bind_binary = (char *)*ptr; + rand_string(bind_binary, dataLen); + + bind->buffer_type = TSDB_DATA_TYPE_BINARY; + bind->buffer_length = dataLen; + bind->buffer = bind_binary; + bind->length = &bind->buffer_length; + bind->is_null = NULL; + + *ptr += bind->buffer_length; + } else if (0 == strncasecmp(dataType, + "NCHAR", strlen("NCHAR"))) { + if (dataLen > TSDB_MAX_BINARY_LEN) { + errorPrint( "nchar length overflow, max size:%u\n", + (uint32_t)TSDB_MAX_BINARY_LEN); + return -1; + } + char *bind_nchar = (char *)*ptr; + rand_string(bind_nchar, dataLen); + + bind->buffer_type = TSDB_DATA_TYPE_NCHAR; + bind->buffer_length = strlen(bind_nchar); + bind->buffer = bind_nchar; + bind->length = &bind->buffer_length; + bind->is_null = NULL; + + *ptr += bind->buffer_length; + } else if (0 == strncasecmp(dataType, + "INT", strlen("INT"))) { + int32_t *bind_int = (int32_t *)*ptr; + + *bind_int = rand_int(); + bind->buffer_type = TSDB_DATA_TYPE_INT; + bind->buffer_length = sizeof(int32_t); + bind->buffer = bind_int; + bind->length = &bind->buffer_length; + bind->is_null = NULL; + + *ptr += bind->buffer_length; + } else if (0 == strncasecmp(dataType, + "BIGINT", strlen("BIGINT"))) { + int64_t *bind_bigint = (int64_t *)*ptr; + + *bind_bigint = rand_bigint(); + bind->buffer_type = TSDB_DATA_TYPE_BIGINT; + bind->buffer_length = sizeof(int64_t); + bind->buffer = bind_bigint; + bind->length = &bind->buffer_length; + bind->is_null = NULL; + + *ptr += bind->buffer_length; + } else if (0 == strncasecmp(dataType, + "FLOAT", strlen("FLOAT"))) { + float *bind_float = (float *) *ptr; + + *bind_float = rand_float(); + bind->buffer_type = TSDB_DATA_TYPE_FLOAT; + bind->buffer_length = sizeof(float); + bind->buffer = bind_float; + bind->length = &bind->buffer_length; + bind->is_null = NULL; + + *ptr += bind->buffer_length; + } else if (0 == strncasecmp(dataType, + "DOUBLE", strlen("DOUBLE"))) { + double *bind_double = (double *)*ptr; + + *bind_double = rand_double(); + bind->buffer_type = TSDB_DATA_TYPE_DOUBLE; + bind->buffer_length = sizeof(double); + bind->buffer = bind_double; + bind->length = &bind->buffer_length; + bind->is_null = NULL; + + *ptr += bind->buffer_length; + } else if (0 == strncasecmp(dataType, + "SMALLINT", strlen("SMALLINT"))) { + int16_t *bind_smallint = (int16_t *)*ptr; + + *bind_smallint = rand_smallint(); + bind->buffer_type = TSDB_DATA_TYPE_SMALLINT; + bind->buffer_length = sizeof(int16_t); + bind->buffer = bind_smallint; + bind->length = &bind->buffer_length; + bind->is_null = NULL; + + *ptr += bind->buffer_length; + } else if (0 == strncasecmp(dataType, + "TINYINT", strlen("TINYINT"))) { + int8_t *bind_tinyint = (int8_t *)*ptr; + + *bind_tinyint = rand_tinyint(); + bind->buffer_type = TSDB_DATA_TYPE_TINYINT; + bind->buffer_length = sizeof(int8_t); + bind->buffer = bind_tinyint; + bind->length = &bind->buffer_length; + bind->is_null = NULL; + *ptr += bind->buffer_length; + } else if (0 == strncasecmp(dataType, + "BOOL", strlen("BOOL"))) { + int8_t *bind_bool = (int8_t *)*ptr; + + *bind_bool = rand_bool(); + bind->buffer_type = TSDB_DATA_TYPE_BOOL; + bind->buffer_length = sizeof(int8_t); + bind->buffer = bind_bool; + bind->length = &bind->buffer_length; + bind->is_null = NULL; + + *ptr += bind->buffer_length; + } else if (0 == strncasecmp(dataType, + "TIMESTAMP", strlen("TIMESTAMP"))) { + int64_t *bind_ts2 = (int64_t *) *ptr; + + *bind_ts2 = rand_bigint(); + bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + bind->buffer_length = sizeof(int64_t); + bind->buffer = bind_ts2; + bind->length = &bind->buffer_length; + bind->is_null = NULL; + + *ptr += bind->buffer_length; + } else { + errorPrint( "No support data type: %s\n", + dataType); + return -1; + } + + return 0; +} + +static int32_t prepareStmtWithoutStb( + TAOS_STMT *stmt, + char *tableName, + uint32_t batch, + int64_t insertRows, + int64_t recordFrom, + int64_t startTime) +{ + int ret = taos_stmt_set_tbname(stmt, tableName); + if (ret != 0) { + errorPrint("failed to execute taos_stmt_set_tbname(%s). return 0x%x. reason: %s\n", + tableName, ret, taos_errstr(NULL)); + return ret; + } + + char **data_type = g_args.datatype; + + char *bindArray = malloc(sizeof(TAOS_BIND) * (g_args.num_of_CPR + 1)); + if (bindArray == NULL) { + errorPrint("Failed to allocate %d bind params\n", + (g_args.num_of_CPR + 1)); + return -1; + } + + int32_t k = 0; + for (k = 0; k < batch;) { + /* columnCount + 1 (ts) */ + char data[MAX_DATA_SIZE]; + memset(data, 0, MAX_DATA_SIZE); + + char *ptr = data; + TAOS_BIND *bind = (TAOS_BIND *)(bindArray + 0); + + int64_t *bind_ts; + + bind_ts = (int64_t *)ptr; + bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + *bind_ts = startTime + getTSRandTail( + (int64_t)DEFAULT_TIMESTAMP_STEP, k, + g_args.disorderRatio, + g_args.disorderRange); + bind->buffer_length = sizeof(int64_t); + bind->buffer = bind_ts; + bind->length = &bind->buffer_length; + bind->is_null = NULL; + + ptr += bind->buffer_length; + + for (int i = 0; i < g_args.num_of_CPR; i ++) { + bind = (TAOS_BIND *)((char *)bindArray + (sizeof(TAOS_BIND) * (i + 1))); + if ( -1 == prepareStmtBindArrayByType( + bind, + data_type[i], + g_args.len_of_binary, + &ptr)) { + return -1; + } + } + taos_stmt_bind_param(stmt, (TAOS_BIND *)bindArray); + // if msg > 3MB, break + taos_stmt_add_batch(stmt); + + k++; + recordFrom ++; + if (recordFrom >= insertRows) { + break; + } + } + + free(bindArray); + return k; +} + +static int32_t prepareStbStmt(SSuperTable *stbInfo, + TAOS_STMT *stmt, + char *tableName, uint32_t batch, + uint64_t insertRows, + uint64_t recordFrom, + int64_t startTime, char *buffer) +{ + int ret = taos_stmt_set_tbname(stmt, tableName); + if (ret != 0) { + errorPrint("failed to execute taos_stmt_set_tbname(%s). return 0x%x. reason: %s\n", + tableName, ret, taos_errstr(NULL)); + return ret; + } + + char *bindArray = malloc(sizeof(TAOS_BIND) * (stbInfo->columnCount + 1)); + if (bindArray == NULL) { + errorPrint("Failed to allocate %d bind params\n", + (stbInfo->columnCount + 1)); + return -1; + } + + bool tsRand; + if (0 == strncasecmp(stbInfo->dataSource, "rand", strlen("rand"))) { + tsRand = true; + } else { + tsRand = false; + } + + uint32_t k; + for (k = 0; k < batch;) { + /* columnCount + 1 (ts) */ + char data[MAX_DATA_SIZE]; + memset(data, 0, MAX_DATA_SIZE); + + char *ptr = data; + TAOS_BIND *bind = (TAOS_BIND *)(bindArray + 0); + + int64_t *bind_ts; + + bind_ts = (int64_t *)ptr; + bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + if (tsRand) { + *bind_ts = startTime + getTSRandTail( + stbInfo->timeStampStep, k, + stbInfo->disorderRatio, + stbInfo->disorderRange); + } else { + *bind_ts = startTime + stbInfo->timeStampStep * k; + } + bind->buffer_length = sizeof(int64_t); + bind->buffer = bind_ts; + bind->length = &bind->buffer_length; + bind->is_null = NULL; + + ptr += bind->buffer_length; + + for (int i = 0; i < stbInfo->columnCount; i ++) { + bind = (TAOS_BIND *)((char *)bindArray + (sizeof(TAOS_BIND) * (i + 1))); + if ( -1 == prepareStmtBindArrayByType( + bind, + stbInfo->columns[i].dataType, + stbInfo->columns[i].dataLen, + &ptr)) { + return -1; + } + } + taos_stmt_bind_param(stmt, (TAOS_BIND *)bindArray); + // if msg > 3MB, break + taos_stmt_add_batch(stmt); + + k++; + recordFrom ++; + if (recordFrom >= insertRows) { + break; + } + } + + free(bindArray); + return k; +} +#endif + +static int32_t generateStbProgressiveData( + SSuperTable *superTblInfo, + char *tableName, + int64_t tableSeq, + char *dbName, char *buffer, + int64_t insertRows, + uint64_t recordFrom, int64_t startTime, int64_t *pSamplePos, + int64_t *pRemainderBufLen) +{ assert(buffer != NULL); char *pstr = buffer; - int64_t k = 0; - memset(buffer, 0, *pRemainderBufLen); - int64_t headLen = generateSQLHead(tableName, tableSeq, pThreadInfo, superTblInfo, + int64_t headLen = generateStbSQLHead( + superTblInfo, + tableName, tableSeq, dbName, buffer, *pRemainderBufLen); if (headLen <= 0) { @@ -5096,29 +5632,64 @@ static int64_t generateProgressiveDataBuffer( *pRemainderBufLen -= headLen; int64_t dataLen; - k = generateDataTail(superTblInfo, - g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, startFrom, + + return generateStbDataTail(superTblInfo, + g_args.num_of_RPR, pstr, *pRemainderBufLen, + insertRows, recordFrom, startTime, pSamplePos, &dataLen); +} - return k; +static int32_t generateProgressiveDataWithoutStb( + char *tableName, + /* int64_t tableSeq, */ + threadInfo *pThreadInfo, char *buffer, + int64_t insertRows, + uint64_t recordFrom, int64_t startTime, /*int64_t *pSamplePos, */ + int64_t *pRemainderBufLen) +{ + assert(buffer != NULL); + char *pstr = buffer; + + memset(buffer, 0, *pRemainderBufLen); + + int64_t headLen = generateSQLHeadWithoutStb( + tableName, pThreadInfo->db_name, + buffer, *pRemainderBufLen); + + if (headLen <= 0) { + return 0; + } + pstr += headLen; + *pRemainderBufLen -= headLen; + + int64_t dataLen; + + return generateDataTailWithoutStb( + g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, recordFrom, + startTime, + /*pSamplePos, */&dataLen); } static void printStatPerThread(threadInfo *pThreadInfo) { - fprintf(stderr, "====thread[%d] completed total inserted rows: %"PRIu64 ", total affected rows: %"PRIu64". %.2f records/second====\n", + fprintf(stderr, "====thread[%d] completed total inserted rows: %"PRIu64 ", total affected rows: %"PRIu64". %.2f records/second====\n", pThreadInfo->threadID, pThreadInfo->totalInsertRows, pThreadInfo->totalAffectedRows, (double)(pThreadInfo->totalAffectedRows / (pThreadInfo->totalDelay/1000.0))); } +// sync write interlace data static void* syncWriteInterlace(threadInfo *pThreadInfo) { debugPrint("[%d] %s() LN%d: ### interlace write\n", pThreadInfo->threadID, __func__, __LINE__); int64_t insertRows; - uint64_t interlaceRows; + uint32_t interlaceRows; + uint64_t maxSqlLen; + int64_t nTimeStampStep; + uint64_t insert_interval; SSuperTable* superTblInfo = pThreadInfo->superTblInfo; @@ -5131,59 +5702,30 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { } else { interlaceRows = superTblInfo->interlaceRows; } + maxSqlLen = superTblInfo->maxSqlLen; + nTimeStampStep = superTblInfo->timeStampStep; + insert_interval = superTblInfo->insertInterval; } else { insertRows = g_args.num_of_DPT; interlaceRows = g_args.interlace_rows; + maxSqlLen = g_args.max_sql_len; + nTimeStampStep = DEFAULT_TIMESTAMP_STEP; + insert_interval = g_args.insert_interval; } + debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n", + pThreadInfo->threadID, __func__, __LINE__, + pThreadInfo->start_table_from, + pThreadInfo->ntables, insertRows); + if (interlaceRows > insertRows) interlaceRows = insertRows; if (interlaceRows > g_args.num_of_RPR) interlaceRows = g_args.num_of_RPR; - int progOrInterlace; - - if (interlaceRows > 0) { - progOrInterlace= INTERLACE_INSERT_MODE; - } else { - progOrInterlace = PROGRESSIVE_INSERT_MODE; - } - - uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; - pThreadInfo->buffer = calloc(maxSqlLen, 1); - if (NULL == pThreadInfo->buffer) { - errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n", - __func__, __LINE__, maxSqlLen, strerror(errno)); - return NULL; - } - - char tableName[TSDB_TABLE_NAME_LEN]; - - pThreadInfo->totalInsertRows = 0; - pThreadInfo->totalAffectedRows = 0; - - int64_t nTimeStampStep = superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP; - - uint64_t insert_interval = - superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; - uint64_t st = 0; - uint64_t et = UINT64_MAX; - - uint64_t lastPrintTime = taosGetTimestampMs(); - uint64_t startTs = taosGetTimestampMs(); - uint64_t endTs; - - uint64_t tableSeq = pThreadInfo->start_table_from; - - debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n", - pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->start_table_from, - pThreadInfo->ntables, insertRows); - - int64_t startTime = pThreadInfo->start_time; - - uint64_t batchPerTbl = interlaceRows; - uint64_t batchPerTblTimes; + uint32_t batchPerTbl = interlaceRows; + uint32_t batchPerTblTimes; if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) { batchPerTblTimes = @@ -5192,13 +5734,30 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { batchPerTblTimes = 1; } + pThreadInfo->buffer = calloc(maxSqlLen, 1); + if (NULL == pThreadInfo->buffer) { + errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n", + __func__, __LINE__, maxSqlLen, strerror(errno)); + return NULL; + } + + pThreadInfo->totalInsertRows = 0; + pThreadInfo->totalAffectedRows = 0; + + uint64_t st = 0; + uint64_t et = UINT64_MAX; + + uint64_t lastPrintTime = taosGetTimestampMs(); + uint64_t startTs = taosGetTimestampMs(); + uint64_t endTs; + + uint64_t tableSeq = pThreadInfo->start_table_from; + int64_t startTime = pThreadInfo->start_time; + uint64_t generatedRecPerTbl = 0; bool flagSleep = true; uint64_t sleepTimeTotal = 0; - char *strInsertInto = "insert into "; - int nInsertBufLen = strlen(strInsertInto); - while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) { if ((flagSleep) && (insert_interval)) { st = taosGetTimestampMs(); @@ -5210,13 +5769,16 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { char *pstr = pThreadInfo->buffer; - int len = snprintf(pstr, nInsertBufLen + 1, "%s", strInsertInto); + int len = snprintf(pstr, + strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO); pstr += len; remainderBufLen -= len; - uint64_t recOfBatch = 0; + uint32_t recOfBatch = 0; + + for (uint32_t i = 0; i < batchPerTblTimes; i ++) { + char tableName[TSDB_TABLE_NAME_LEN]; - for (uint64_t i = 0; i < batchPerTblTimes; i ++) { getTableName(tableName, pThreadInfo, tableSeq); if (0 == strlen(tableName)) { errorPrint("[%d] %s() LN%d, getTableName return null\n", @@ -5226,18 +5788,62 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { } uint64_t oldRemainderLen = remainderBufLen; - int64_t generated = generateInterlaceDataBuffer( - tableName, batchPerTbl, i, batchPerTblTimes, - tableSeq, - pThreadInfo, pstr, - insertRows, - startTime, - &remainderBufLen); - debugPrint("[%d] %s() LN%d, generated records is %"PRId64"\n", + int32_t generated; + if (superTblInfo) { + if (superTblInfo->iface == STMT_IFACE) { +#if STMT_IFACE_ENABLED == 1 + generated = prepareStbStmt(superTblInfo, + pThreadInfo->stmt, + tableName, + batchPerTbl, + insertRows, i, + startTime, + pThreadInfo->buffer); +#else + generated = -1; +#endif + } else { + generated = generateStbInterlaceData( + superTblInfo, + tableName, batchPerTbl, i, + batchPerTblTimes, + tableSeq, + pThreadInfo, pstr, + insertRows, + startTime, + &remainderBufLen); + } + } else { + if (g_args.iface == STMT_IFACE) { + debugPrint("[%d] %s() LN%d, tableName:%s, batch:%d startTime:%"PRId64"\n", + pThreadInfo->threadID, + __func__, __LINE__, + tableName, batchPerTbl, startTime); +#if STMT_IFACE_ENABLED == 1 + generated = prepareStmtWithoutStb( + pThreadInfo->stmt, tableName, + batchPerTbl, + insertRows, i, + startTime); +#else + generated = -1; +#endif + } else { + generated = generateInterlaceDataWithoutStb( + tableName, batchPerTbl, + tableSeq, + pThreadInfo->db_name, pstr, + insertRows, + startTime, + &remainderBufLen); + } + } + + debugPrint("[%d] %s() LN%d, generated records is %d\n", pThreadInfo->threadID, __func__, __LINE__, generated); if (generated < 0) { - errorPrint("[%d] %s() LN%d, generated records is %"PRId64"\n", + errorPrint("[%d] %s() LN%d, generated records is %d\n", pThreadInfo->threadID, __func__, __LINE__, generated); goto free_of_interlace; } else if (generated == 0) { @@ -5246,15 +5852,15 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { tableSeq ++; recOfBatch += batchPerTbl; + pstr += (oldRemainderLen - remainderBufLen); -// startTime += batchPerTbl * superTblInfo->timeStampStep; pThreadInfo->totalInsertRows += batchPerTbl; - verbosePrint("[%d] %s() LN%d batchPerTbl=%"PRId64" recOfBatch=%"PRId64"\n", + + verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n", pThreadInfo->threadID, __func__, __LINE__, batchPerTbl, recOfBatch); - if (progOrInterlace == INTERLACE_INSERT_MODE) { - if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { + if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { // turn to first table tableSeq = pThreadInfo->start_table_from; generatedRecPerTbl += batchPerTbl; @@ -5266,13 +5872,12 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { if (generatedRecPerTbl >= insertRows) break; - int remainRows = insertRows - generatedRecPerTbl; + int64_t remainRows = insertRows - generatedRecPerTbl; if ((remainRows > 0) && (batchPerTbl > remainRows)) batchPerTbl = remainRows; if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR) break; - } } verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%"PRId64" insertRows=%"PRId64"\n", @@ -5283,7 +5888,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { break; } - verbosePrint("[%d] %s() LN%d recOfBatch=%"PRIu64" totalInsertRows=%"PRIu64"\n", + verbosePrint("[%d] %s() LN%d recOfBatch=%d totalInsertRows=%"PRIu64"\n", pThreadInfo->threadID, __func__, __LINE__, recOfBatch, pThreadInfo->totalInsertRows); verbosePrint("[%d] %s() LN%d, buffer=%s\n", @@ -5292,7 +5897,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { startTs = taosGetTimestampMs(); if (recOfBatch == 0) { - errorPrint("[%d] %s() LN%d try inserting records of batch is %"PRIu64"\n", + errorPrint("[%d] %s() LN%d try inserting records of batch is %d\n", pThreadInfo->threadID, __func__, __LINE__, recOfBatch); errorPrint("%s\n", "\tPlease check if the batch or the buffer length is proper value!\n"); @@ -5314,7 +5919,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pThreadInfo->totalDelay += delay; if (recOfBatch != affectedRows) { - errorPrint("[%d] %s() LN%d execInsert insert %"PRIu64", affected rows: %"PRId64"\n%s\n", + errorPrint("[%d] %s() LN%d execInsert insert %d, affected rows: %"PRId64"\n%s\n", pThreadInfo->threadID, __func__, __LINE__, recOfBatch, affectedRows, pThreadInfo->buffer); goto free_of_interlace; @@ -5350,19 +5955,18 @@ free_of_interlace: return NULL; } -// sync insertion -/* - 1 thread: 100 tables * 2000 rows/s - 1 thread: 10 tables * 20000 rows/s - 6 thread: 300 tables * 2000 rows/s - - 2 taosinsertdata , 1 thread: 10 tables * 20000 rows/s -*/ +// sync insertion progressive data static void* syncWriteProgressive(threadInfo *pThreadInfo) { debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__); SSuperTable* superTblInfo = pThreadInfo->superTblInfo; uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; + int64_t timeStampStep = + superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP; + int64_t insertRows = + (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; + verbosePrint("%s() LN%d insertRows=%"PRId64"\n", + __func__, __LINE__, insertRows); pThreadInfo->buffer = calloc(maxSqlLen, 1); if (NULL == pThreadInfo->buffer) { @@ -5376,34 +5980,17 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { uint64_t startTs = taosGetTimestampMs(); uint64_t endTs; - int64_t timeStampStep = - superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP; -/* int insert_interval = - superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; - uint64_t st = 0; - uint64_t et = 0xffffffff; - */ - pThreadInfo->totalInsertRows = 0; pThreadInfo->totalAffectedRows = 0; pThreadInfo->samplePos = 0; - for (uint64_t tableSeq = - pThreadInfo->start_table_from; tableSeq <= pThreadInfo->end_table_to; - tableSeq ++) { + for (uint64_t tableSeq = pThreadInfo->start_table_from; + tableSeq <= pThreadInfo->end_table_to; + tableSeq ++) { int64_t start_time = pThreadInfo->start_time; - int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; - verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows); - for (uint64_t i = 0; i < insertRows;) { - /* - if (insert_interval) { - st = taosGetTimestampMs(); - } - */ - char tableName[TSDB_TABLE_NAME_LEN]; getTableName(tableName, pThreadInfo, tableSeq); verbosePrint("%s() LN%d: tid=%d seq=%"PRId64" tableName=%s\n", @@ -5412,18 +5999,56 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { int64_t remainderBufLen = maxSqlLen; char *pstr = pThreadInfo->buffer; - int nInsertBufLen = strlen("insert into "); - int len = snprintf(pstr, nInsertBufLen + 1, "%s", "insert into "); + int len = snprintf(pstr, + strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO); pstr += len; remainderBufLen -= len; - int64_t generated = generateProgressiveDataBuffer( - tableName, tableSeq, pThreadInfo, pstr, insertRows, - i, start_time, - &(pThreadInfo->samplePos), - &remainderBufLen); + int32_t generated; + if (superTblInfo) { + if (superTblInfo->iface == STMT_IFACE) { +#if STMT_IFACE_ENABLED == 1 + generated = prepareStbStmt( + superTblInfo, + pThreadInfo->stmt, + tableName, + g_args.num_of_RPR, + insertRows, i, start_time, pstr); +#else + generated = -1; +#endif + } else { + generated = generateStbProgressiveData( + superTblInfo, + tableName, tableSeq, pThreadInfo->db_name, pstr, + insertRows, i, start_time, + &(pThreadInfo->samplePos), + &remainderBufLen); + } + } else { + if (g_args.iface == STMT_IFACE) { +#if STMT_IFACE_ENABLED == 1 + generated = prepareStmtWithoutStb( + pThreadInfo->stmt, + tableName, + g_args.num_of_RPR, + insertRows, i, + start_time); +#else + generated = -1; +#endif + } else { + generated = generateProgressiveDataWithoutStb( + tableName, + /* tableSeq, */ + pThreadInfo, pstr, insertRows, + i, start_time, + /* &(pThreadInfo->samplePos), */ + &remainderBufLen); + } + } if (generated > 0) i += generated; else @@ -5434,13 +6059,13 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { startTs = taosGetTimestampMs(); - int64_t affectedRows = execInsert(pThreadInfo, generated); + int32_t affectedRows = execInsert(pThreadInfo, generated); endTs = taosGetTimestampMs(); uint64_t delay = endTs - startTs; performancePrint("%s() LN%d, insert execution time is %"PRId64"ms\n", __func__, __LINE__, delay); - verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n", + verbosePrint("[%d] %s() LN%d affectedRows=%d\n", pThreadInfo->threadID, __func__, __LINE__, affectedRows); @@ -5450,7 +6075,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { pThreadInfo->totalDelay += delay; if (affectedRows < 0) { - errorPrint("%s() LN%d, affected rows: %"PRId64"\n", + errorPrint("%s() LN%d, affected rows: %d\n", __func__, __LINE__, affectedRows); goto free_of_progressive; } @@ -5468,27 +6093,14 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { if (i >= insertRows) break; -/* - if (insert_interval) { - et = taosGetTimestampMs(); - - if (insert_interval > ((et - st)) ) { - int sleep_time = insert_interval - (et -st); - performancePrint("%s() LN%d sleep: %d ms for insert interval\n", - __func__, __LINE__, sleep_time); - taosMsleep(sleep_time); // ms - } - } - */ } // num_of_DPT - if (g_args.verbose_print) { - if ((tableSeq == pThreadInfo->ntables - 1) && superTblInfo && + if ((g_args.verbose_print) && + (tableSeq == pThreadInfo->ntables - 1) && (superTblInfo) && (0 == strncasecmp( superTblInfo->dataSource, "sample", strlen("sample")))) { verbosePrint("%s() LN%d samplePos=%"PRId64"\n", __func__, __LINE__, pThreadInfo->samplePos); - } } } // tableSeq @@ -5503,7 +6115,7 @@ static void* syncWrite(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - int interlaceRows; + uint32_t interlaceRows; if (superTblInfo) { if ((superTblInfo->interlaceRows == 0) @@ -5523,7 +6135,6 @@ static void* syncWrite(void *sarg) { // progressive mode return syncWriteProgressive(pThreadInfo); } - } static void callBack(void *param, TAOS_RES *res, int code) { @@ -5561,10 +6172,12 @@ static void callBack(void *param, TAOS_RES *res, int code) { int rand_num = taosRandom() % 100; if (0 != pThreadInfo->superTblInfo->disorderRatio && rand_num < pThreadInfo->superTblInfo->disorderRatio) { - int64_t d = pThreadInfo->lastTs - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1); - generateRowData(data, d, pThreadInfo->superTblInfo); + int64_t d = pThreadInfo->lastTs + - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1); + generateStbRowData(pThreadInfo->superTblInfo, data, d); } else { - generateRowData(data, pThreadInfo->lastTs += 1000, pThreadInfo->superTblInfo); + generateStbRowData(pThreadInfo->superTblInfo, + data, pThreadInfo->lastTs += 1000); } pstr += sprintf(pstr, "%s", data); pThreadInfo->counter++; @@ -5632,15 +6245,6 @@ static int convertHostToServAddr(char *host, uint16_t port, struct sockaddr_in * static void startMultiThreadInsertData(int threads, char* db_name, char* precision,SSuperTable* superTblInfo) { - //TAOS* taos; - //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { - // taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); - // if (NULL == taos) { - // printf("connect to server fail, reason: %s\n", taos_errstr(NULL)); - // exit(-1); - // } - //} - int32_t timePrec = TSDB_TIME_PRECISION_MILLI; if (0 != precision[0]) { if (0 == strncasecmp(precision, "ms", 2)) { @@ -5702,7 +6306,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, } int64_t ntables = 0; - uint64_t startFrom; + uint64_t tableFrom; if (superTblInfo) { int64_t limit; @@ -5729,25 +6333,19 @@ static void startMultiThreadInsertData(int threads, char* db_name, } ntables = limit; - startFrom = offset; + tableFrom = offset; if ((superTblInfo->childTblExists != TBL_NO_EXISTS) && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit ) > superTblInfo->childTblCount)) { printf("WARNING: specified offset + limit > child table count!\n"); - if (!g_args.answer_yes) { - printf(" Press enter key to continue or Ctrl-C to stop\n\n"); - (void)getchar(); - } + prompt(); } if ((superTblInfo->childTblExists != TBL_NO_EXISTS) && (0 == superTblInfo->childTblLimit)) { printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n"); - if (!g_args.answer_yes) { - printf(" Press enter key to continue or Ctrl-C to stop\n\n"); - (void)getchar(); - } + prompt(); } superTblInfo->childTblName = (char*)calloc(1, @@ -5767,7 +6365,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, offset); } else { ntables = g_args.num_of_tables; - startFrom = 0; + tableFrom = 0; } taos_close(taos0); @@ -5784,8 +6382,9 @@ static void startMultiThreadInsertData(int threads, char* db_name, } if ((superTblInfo) - && (superTblInfo->insertMode == REST_IFACE)) { - if (convertHostToServAddr(g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0) { + && (superTblInfo->iface == REST_IFACE)) { + if (convertHostToServAddr( + g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0) { exit(-1); } } @@ -5800,22 +6399,22 @@ static void startMultiThreadInsertData(int threads, char* db_name, memset(infos, 0, threads * sizeof(threadInfo)); for (int i = 0; i < threads; i++) { - threadInfo *t_info = infos + i; - t_info->threadID = i; - tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE); - t_info->time_precision = timePrec; - t_info->superTblInfo = superTblInfo; + threadInfo *pThreadInfo = infos + i; + pThreadInfo->threadID = i; + tstrncpy(pThreadInfo->db_name, db_name, MAX_DB_NAME_SIZE); + pThreadInfo->time_precision = timePrec; + pThreadInfo->superTblInfo = superTblInfo; - t_info->start_time = start_time; - t_info->minDelay = UINT64_MAX; + pThreadInfo->start_time = start_time; + pThreadInfo->minDelay = UINT64_MAX; if ((NULL == superTblInfo) || - (superTblInfo->insertMode != REST_IFACE)) { + (superTblInfo->iface != REST_IFACE)) { //t_info->taos = taos; - t_info->taos = taos_connect( + pThreadInfo->taos = taos_connect( g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); - if (NULL == t_info->taos) { + if (NULL == pThreadInfo->taos) { errorPrint( "%s() LN%d, connect to server fail from insert sub thread, reason: %s\n", __func__, __LINE__, @@ -5824,9 +6423,19 @@ static void startMultiThreadInsertData(int threads, char* db_name, exit(-1); } - if ((superTblInfo) && (superTblInfo->insertMode == STMT_IFACE)) { - t_info->stmt = taos_stmt_init(t_info->taos); - if (NULL == t_info->stmt) { +#if STMT_IFACE_ENABLED == 1 + if ((g_args.iface == STMT_IFACE) + || ((superTblInfo) && (superTblInfo->iface == STMT_IFACE))) { + + int columnCount; + if (superTblInfo) { + columnCount = superTblInfo->columnCount; + } else { + columnCount = g_args.num_of_CPR; + } + + pThreadInfo->stmt = taos_stmt_init(pThreadInfo->taos); + if (NULL == pThreadInfo->stmt) { errorPrint( "%s() LN%d, failed init stmt, reason: %s\n", __func__, __LINE__, @@ -5835,29 +6444,48 @@ static void startMultiThreadInsertData(int threads, char* db_name, free(infos); exit(-1); } + + char buffer[3000]; + char *pstr = buffer; + pstr += sprintf(pstr, "INSERT INTO ? values(?"); + + for (int col = 0; col < columnCount; col ++) { + pstr += sprintf(pstr, ",?"); + } + pstr += sprintf(pstr, ")"); + + int ret = taos_stmt_prepare(pThreadInfo->stmt, buffer, 0); + if (ret != 0){ + errorPrint("failed to execute taos_stmt_prepare. return 0x%x. reason: %s\n", + ret, taos_errstr(NULL)); + free(pids); + free(infos); + exit(-1); + } } +#endif } else { - t_info->taos = NULL; + pThreadInfo->taos = NULL; } /* if ((NULL == superTblInfo) || (0 == superTblInfo->multiThreadWriteOneTbl)) { */ - t_info->start_table_from = startFrom; - t_info->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; - startFrom = t_info->end_table_to + 1; + pThreadInfo->start_table_from = tableFrom; + pThreadInfo->ntables = iend_table_to = i < b ? tableFrom + a : tableFrom + a - 1; + tableFrom = pThreadInfo->end_table_to + 1; /* } else { - t_info->start_table_from = 0; - t_info->ntables = superTblInfo->childTblCount; - t_info->start_time = t_info->start_time + rand_int() % 10000 - rand_tinyint(); + pThreadInfo->start_table_from = 0; + pThreadInfo->ntables = superTblInfo->childTblCount; + pThreadInfo->start_time = pThreadInfo->start_time + rand_int() % 10000 - rand_tinyint(); } */ - tsem_init(&(t_info->lock_sem), 0, 0); + tsem_init(&(pThreadInfo->lock_sem), 0, 0); if (ASYNC_MODE == g_Dbs.asyncMode) { - pthread_create(pids + i, NULL, asyncWrite, t_info); + pthread_create(pids + i, NULL, asyncWrite, pThreadInfo); } else { - pthread_create(pids + i, NULL, syncWrite, t_info); + pthread_create(pids + i, NULL, syncWrite, pThreadInfo); } } @@ -5872,31 +6500,34 @@ static void startMultiThreadInsertData(int threads, char* db_name, double avgDelay = 0; for (int i = 0; i < threads; i++) { - threadInfo *t_info = infos + i; + threadInfo *pThreadInfo = infos + i; - tsem_destroy(&(t_info->lock_sem)); + tsem_destroy(&(pThreadInfo->lock_sem)); - if (t_info->stmt) { - taos_stmt_close(t_info->stmt); +#if STMT_IFACE_ENABLED == 1 + if (pThreadInfo->stmt) { + taos_stmt_close(pThreadInfo->stmt); } - taos_close(t_info->taos); +#endif + tsem_destroy(&(pThreadInfo->lock_sem)); + taos_close(pThreadInfo->taos); debugPrint("%s() LN%d, [%d] totalInsert=%"PRIu64" totalAffected=%"PRIu64"\n", __func__, __LINE__, - t_info->threadID, t_info->totalInsertRows, - t_info->totalAffectedRows); + pThreadInfo->threadID, pThreadInfo->totalInsertRows, + pThreadInfo->totalAffectedRows); if (superTblInfo) { - superTblInfo->totalAffectedRows += t_info->totalAffectedRows; - superTblInfo->totalInsertRows += t_info->totalInsertRows; + superTblInfo->totalAffectedRows += pThreadInfo->totalAffectedRows; + superTblInfo->totalInsertRows += pThreadInfo->totalInsertRows; } else { - g_args.totalAffectedRows += t_info->totalAffectedRows; - g_args.totalInsertRows += t_info->totalInsertRows; + g_args.totalAffectedRows += pThreadInfo->totalAffectedRows; + g_args.totalInsertRows += pThreadInfo->totalInsertRows; } - totalDelay += t_info->totalDelay; - cntDelay += t_info->cntDelay; - if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay; - if (t_info->minDelay < minDelay) minDelay = t_info->minDelay; + totalDelay += pThreadInfo->totalDelay; + cntDelay += pThreadInfo->cntDelay; + if (pThreadInfo->maxDelay > maxDelay) maxDelay = pThreadInfo->maxDelay; + if (pThreadInfo->minDelay < minDelay) minDelay = pThreadInfo->minDelay; } cntDelay -= 1; @@ -5952,26 +6583,26 @@ static void startMultiThreadInsertData(int threads, char* db_name, static void *readTable(void *sarg) { #if 1 - threadInfo *rinfo = (threadInfo *)sarg; - TAOS *taos = rinfo->taos; + threadInfo *pThreadInfo = (threadInfo *)sarg; + TAOS *taos = pThreadInfo->taos; char command[BUFFER_SIZE] = "\0"; - uint64_t sTime = rinfo->start_time; - char *tb_prefix = rinfo->tb_prefix; - FILE *fp = fopen(rinfo->fp, "a"); + uint64_t sTime = pThreadInfo->start_time; + char *tb_prefix = pThreadInfo->tb_prefix; + FILE *fp = fopen(pThreadInfo->filePath, "a"); if (NULL == fp) { - errorPrint( "fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno)); + errorPrint( "fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno)); return NULL; } int64_t num_of_DPT; -/* if (rinfo->superTblInfo) { - num_of_DPT = rinfo->superTblInfo->insertRows; // nrecords_per_table; +/* if (pThreadInfo->superTblInfo) { + num_of_DPT = pThreadInfo->superTblInfo->insertRows; // nrecords_per_table; } else { */ num_of_DPT = g_args.num_of_DPT; // } - int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; + int64_t num_of_tables = pThreadInfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; int64_t totalData = num_of_DPT * num_of_tables; bool do_aggreFunc = g_Dbs.do_aggreFunc; @@ -6024,17 +6655,17 @@ static void *readTable(void *sarg) { static void *readMetric(void *sarg) { #if 1 - threadInfo *rinfo = (threadInfo *)sarg; - TAOS *taos = rinfo->taos; + threadInfo *pThreadInfo = (threadInfo *)sarg; + TAOS *taos = pThreadInfo->taos; char command[BUFFER_SIZE] = "\0"; - FILE *fp = fopen(rinfo->fp, "a"); + FILE *fp = fopen(pThreadInfo->filePath, "a"); if (NULL == fp) { - printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno)); + printf("fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno)); return NULL; } - int64_t num_of_DPT = rinfo->superTblInfo->insertRows; - int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; + int64_t num_of_DPT = pThreadInfo->superTblInfo->insertRows; + int64_t num_of_tables = pThreadInfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; int64_t totalData = num_of_DPT * num_of_tables; bool do_aggreFunc = g_Dbs.do_aggreFunc; @@ -6095,6 +6726,13 @@ static void *readMetric(void *sarg) { return NULL; } +static void prompt() +{ + if (!g_args.answer_yes) { + printf(" Press enter key to continue or Ctrl-C to stop\n\n"); + (void)getchar(); + } +} static int insertTestProcess() { @@ -6115,10 +6753,7 @@ static int insertTestProcess() { if (g_fpOfInsertResult) printfInsertMetaToFile(g_fpOfInsertResult); - if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); - (void)getchar(); - } + prompt(); init_rand_data(); @@ -6150,7 +6785,6 @@ static int insertTestProcess() { } } - taosMsleep(1000); // create sub threads for inserting data //start = taosGetTimestampMs(); for (int i = 0; i < g_Dbs.dbCount; i++) { @@ -6229,23 +6863,22 @@ static void *specifiedTableQuery(void *sarg) { uint64_t lastPrintTime = taosGetTimestampMs(); uint64_t startTs = taosGetTimestampMs(); + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != '\0') { + sprintf(pThreadInfo->filePath, "%s-%d", + g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); + } + while(queryTimes --) { if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) < (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval) { taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval - (et - st)); // ms } - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], - pThreadInfo->threadID); - } - st = taosGetTimestampMs(); selectAndGetResult(pThreadInfo, - g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], tmpFile); + g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq]); et = taosGetTimestampMs(); printf("=thread[%"PRId64"] use %s complete one sql, Spent %10.3f s\n", @@ -6331,13 +6964,12 @@ static void *superTableQuery(void *sarg) { for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { memset(sqlstr,0,sizeof(sqlstr)); replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i); - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.superQueryInfo.result[j][0] != 0) { - sprintf(tmpFile, "%s-%d", + if (g_queryInfo.superQueryInfo.result[j][0] != '\0') { + sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.superQueryInfo.result[j], pThreadInfo->threadID); } - selectAndGetResult(pThreadInfo, sqlstr, tmpFile); + selectAndGetResult(pThreadInfo, sqlstr); totalQueried++; g_queryInfo.superQueryInfo.totalQueried ++; @@ -6390,10 +7022,7 @@ static int queryTestProcess() { &g_queryInfo.superQueryInfo.childTblCount); } - if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); - (void)getchar(); - } + prompt(); if (g_args.debug_print || g_args.verbose_print) { printfQuerySystemInfo(taos); @@ -6409,7 +7038,7 @@ static int queryTestProcess() { threadInfo *infos = NULL; //==== create sub threads for query from specify table int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent; - int nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount; + uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount; uint64_t startTs = taosGetTimestampMs(); @@ -6423,32 +7052,32 @@ static int queryTestProcess() { ERROR_EXIT("memory allocation failed for create threads\n"); } - for (int i = 0; i < nConcurrent; i++) { - for (int j = 0; j < nSqlCount; j++) { - threadInfo *t_info = infos + i * nSqlCount + j; - t_info->threadID = i * nSqlCount + j; - t_info->querySeq = j; + for (uint64_t i = 0; i < nSqlCount; i++) { + for (int j = 0; j < nConcurrent; j++) { + uint64_t seq = i * nConcurrent + j; + threadInfo *pThreadInfo = infos + seq; + pThreadInfo->threadID = seq; + pThreadInfo->querySeq = i; - if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { + if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { - char sqlStr[MAX_TB_NAME_SIZE*2]; - sprintf(sqlStr, "use %s", g_queryInfo.dbName); - verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); - if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) { - taos_close(taos); - free(infos); - free(pids); - errorPrint( "use database %s failed!\n\n", - g_queryInfo.dbName); - return -1; - } + char sqlStr[MAX_TB_NAME_SIZE*2]; + sprintf(sqlStr, "use %s", g_queryInfo.dbName); + if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) { + taos_close(taos); + free(infos); + free(pids); + errorPrint( "use database %s failed!\n\n", + g_queryInfo.dbName); + return -1; + } + } + + pThreadInfo->taos = NULL;// TODO: workaround to use separate taos connection; + + pthread_create(pids + seq, NULL, specifiedTableQuery, + pThreadInfo); } - - t_info->taos = NULL;// TODO: workaround to use separate taos connection; - - pthread_create(pids + i * nSqlCount + j, NULL, specifiedTableQuery, - t_info); - } } } else { g_queryInfo.specifiedQueryInfo.concurrent = 0; @@ -6485,17 +7114,17 @@ static int queryTestProcess() { b = ntables % threads; } - uint64_t startFrom = 0; + uint64_t tableFrom = 0; for (int i = 0; i < threads; i++) { - threadInfo *t_info = infosOfSub + i; - t_info->threadID = i; + threadInfo *pThreadInfo = infosOfSub + i; + pThreadInfo->threadID = i; - t_info->start_table_from = startFrom; - t_info->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; - startFrom = t_info->end_table_to + 1; - t_info->taos = NULL; // TODO: workaround to use separate taos connection; - pthread_create(pidsOfSub + i, NULL, superTableQuery, t_info); + pThreadInfo->start_table_from = tableFrom; + pThreadInfo->ntables = iend_table_to = i < b ? tableFrom + a : tableFrom + a - 1; + tableFrom = pThreadInfo->end_table_to + 1; + pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection; + pthread_create(pidsOfSub + i, NULL, superTableQuery, pThreadInfo); } g_queryInfo.superQueryInfo.threadCnt = threads; @@ -6533,7 +7162,8 @@ static int queryTestProcess() { return 0; } -static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { +static void stable_sub_callback( + TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { if (res == NULL || taos_errno(res) != 0) { errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n", __func__, __LINE__, code, taos_errstr(res)); @@ -6541,23 +7171,49 @@ static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int c } if (param) - appendResultToFile(res, (char*)param); + fetchResult(res, (threadInfo *)param); + // tao_unscribe() will free result. +} + +static void specified_sub_callback( + TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { + if (res == NULL || taos_errno(res) != 0) { + errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n", + __func__, __LINE__, code, taos_errstr(res)); + return; + } + + if (param) + fetchResult(res, (threadInfo *)param); // tao_unscribe() will free result. } static TAOS_SUB* subscribeImpl( - TAOS *taos, char *sql, char* topic, char* resultFileName) { + QUERY_CLASS class, + threadInfo *pThreadInfo, + char *sql, char* topic, bool restart, uint64_t interval) +{ TAOS_SUB* tsub = NULL; - if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { - tsub = taos_subscribe(taos, - g_queryInfo.specifiedQueryInfo.subscribeRestart, - topic, sql, subscribe_callback, (void*)resultFileName, + if ((SPECIFIED_CLASS == class) + && (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode)) { + tsub = taos_subscribe( + pThreadInfo->taos, + restart, + topic, sql, specified_sub_callback, (void*)pThreadInfo, g_queryInfo.specifiedQueryInfo.subscribeInterval); + } else if ((STABLE_CLASS == class) + && (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode)) { + tsub = taos_subscribe( + pThreadInfo->taos, + restart, + topic, sql, stable_sub_callback, (void*)pThreadInfo, + g_queryInfo.superQueryInfo.subscribeInterval); } else { - tsub = taos_subscribe(taos, - g_queryInfo.specifiedQueryInfo.subscribeRestart, - topic, sql, NULL, NULL, 0); + tsub = taos_subscribe( + pThreadInfo->taos, + restart, + topic, sql, NULL, NULL, interval); } if (tsub == NULL) { @@ -6572,23 +7228,25 @@ static void *superSubscribe(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; char subSqlstr[MAX_QUERY_SQL_LENGTH]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; + uint64_t tsubSeq; - if (g_queryInfo.superQueryInfo.sqlCount == 0) - return NULL; + if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) { + errorPrint("The table number(%"PRId64") of the thread is more than max query sql count: %d\n", + pThreadInfo->ntables, + MAX_QUERY_SQL_COUNT); + exit(-1); + } if (pThreadInfo->taos == NULL) { - TAOS * taos = NULL; - taos = taos_connect(g_queryInfo.host, + pThreadInfo->taos = taos_connect(g_queryInfo.host, g_queryInfo.user, g_queryInfo.password, g_queryInfo.dbName, g_queryInfo.port); - if (taos == NULL) { + if (pThreadInfo->taos == NULL) { errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", pThreadInfo->threadID, taos_errstr(NULL)); return NULL; - } else { - pThreadInfo->taos = taos; } } @@ -6604,48 +7262,99 @@ static void *superSubscribe(void *sarg) { char topic[32] = {0}; for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { - for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { - sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%d", i, j); - memset(subSqlstr,0,sizeof(subSqlstr)); - replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], subSqlstr, i); - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.superQueryInfo.result[j][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.superQueryInfo.result[j], pThreadInfo->threadID); + + tsubSeq = i - pThreadInfo->start_table_from; + verbosePrint("%s() LN%d, [%d], start=%"PRId64" end=%"PRId64" i=%"PRIu64"\n", + __func__, __LINE__, + pThreadInfo->threadID, + pThreadInfo->start_table_from, + pThreadInfo->end_table_to, i); + sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%"PRIu64"", + i, pThreadInfo->querySeq); + memset(subSqlstr, 0, sizeof(subSqlstr)); + replaceChildTblName( + g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq], + subSqlstr, i); + if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->filePath, "%s-%d", + g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); } - uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; - debugPrint("%s() LN%d, subSeq=%"PRIu64" subSqlstr: %s\n", - __func__, __LINE__, subSeq, subSqlstr); - tsub[subSeq] = subscribeImpl(pThreadInfo->taos, subSqlstr, topic, tmpFile); - if (NULL == tsub[subSeq]) { + debugPrint("%s() LN%d, [%d] subSqlstr: %s\n", + __func__, __LINE__, pThreadInfo->threadID, subSqlstr); + tsub[tsubSeq] = subscribeImpl( + STABLE_CLASS, + pThreadInfo, subSqlstr, topic, + g_queryInfo.superQueryInfo.subscribeRestart, + g_queryInfo.superQueryInfo.subscribeInterval); + if (NULL == tsub[tsubSeq]) { taos_close(pThreadInfo->taos); return NULL; } - } } // start loop to consume result + int consumed[MAX_QUERY_SQL_COUNT]; + for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) { + consumed[i] = 0; + } TAOS_RES* res = NULL; - while(1) { - for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { - for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { - if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { - continue; - } - uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; - taosMsleep(100); // ms - res = taos_consume(tsub[subSeq]); - if (res) { - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.superQueryInfo.result[j][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.superQueryInfo.result[j], - pThreadInfo->threadID); - appendResultToFile(res, tmpFile); - } - } + uint64_t st = 0, et = 0; + + while ((g_queryInfo.superQueryInfo.endAfterConsume == -1) + || (g_queryInfo.superQueryInfo.endAfterConsume < + consumed[pThreadInfo->end_table_to - pThreadInfo->start_table_from])) { + + for (uint64_t i = pThreadInfo->start_table_from; + i <= pThreadInfo->end_table_to; i++) { + tsubSeq = i - pThreadInfo->start_table_from; + if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { + continue; + } + + st = taosGetTimestampMs(); + performancePrint("st: %"PRIu64" et: %"PRIu64" st-et: %"PRIu64"\n", st, et, (st - et)); + res = taos_consume(tsub[tsubSeq]); + et = taosGetTimestampMs(); + performancePrint("st: %"PRIu64" et: %"PRIu64" delta: %"PRIu64"\n", st, et, (et - st)); + + if (res) { + if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->filePath, "%s-%d", + g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); + fetchResult(res, pThreadInfo); + } + if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->filePath, "%s-%d", + g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); + fetchResult(res, pThreadInfo); + } + consumed[tsubSeq] ++; + + if ((g_queryInfo.superQueryInfo.resubAfterConsume != -1) + && (consumed[tsubSeq] >= + g_queryInfo.superQueryInfo.resubAfterConsume)) { + printf("keepProgress:%d, resub super table query: %"PRIu64"\n", + g_queryInfo.superQueryInfo.subscribeKeepProgress, + pThreadInfo->querySeq); + taos_unsubscribe(tsub[tsubSeq], + g_queryInfo.superQueryInfo.subscribeKeepProgress); + consumed[tsubSeq]= 0; + tsub[tsubSeq] = subscribeImpl( + STABLE_CLASS, + pThreadInfo, subSqlstr, topic, + g_queryInfo.superQueryInfo.subscribeRestart, + g_queryInfo.superQueryInfo.subscribeInterval + ); + if (NULL == tsub[tsubSeq]) { + taos_close(pThreadInfo->taos); + return NULL; + } + } } } } @@ -6653,11 +7362,8 @@ static void *superSubscribe(void *sarg) { for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { - for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { - uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; - taos_unsubscribe(tsub[subSeq], - g_queryInfo.superQueryInfo.subscribeKeepProgress); - } + tsubSeq = i - pThreadInfo->start_table_from; + taos_unsubscribe(tsub[tsubSeq], 0); } taos_close(pThreadInfo->taos); @@ -6666,78 +7372,101 @@ static void *superSubscribe(void *sarg) { static void *specifiedSubscribe(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; - TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; - - if (g_queryInfo.specifiedQueryInfo.sqlCount == 0) - return NULL; +// TAOS_SUB* tsub = NULL; if (pThreadInfo->taos == NULL) { - TAOS * taos = NULL; - taos = taos_connect(g_queryInfo.host, + pThreadInfo->taos = taos_connect(g_queryInfo.host, g_queryInfo.user, g_queryInfo.password, g_queryInfo.dbName, g_queryInfo.port); - if (taos == NULL) { + if (pThreadInfo->taos == NULL) { errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", pThreadInfo->threadID, taos_errstr(NULL)); return NULL; - } else { - pThreadInfo->taos = taos; } } char sqlStr[MAX_TB_NAME_SIZE*2]; sprintf(sqlStr, "use %s", g_queryInfo.dbName); - debugPrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) { taos_close(pThreadInfo->taos); return NULL; } - char topic[32] = {0}; - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { - sprintf(topic, "taosdemo-subscribe-%d", i); - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.specifiedQueryInfo.result[i], pThreadInfo->threadID); - } - tsub[i] = subscribeImpl(pThreadInfo->taos, - g_queryInfo.specifiedQueryInfo.sql[i], topic, tmpFile); - if (NULL == tsub[i]) { - taos_close(pThreadInfo->taos); - return NULL; - } + sprintf(g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID], + "taosdemo-subscribe-%"PRIu64"-%d", + pThreadInfo->querySeq, + pThreadInfo->threadID); + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != '\0') { + sprintf(pThreadInfo->filePath, "%s-%d", + g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); } + g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = subscribeImpl( + SPECIFIED_CLASS, pThreadInfo, + g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], + g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID], + g_queryInfo.specifiedQueryInfo.subscribeRestart, + g_queryInfo.specifiedQueryInfo.subscribeInterval); + if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) { + taos_close(pThreadInfo->taos); + return NULL; + } + // start loop to consume result - TAOS_RES* res = NULL; - while(1) { - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { + + g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0; + while((g_queryInfo.specifiedQueryInfo.endAfterConsume[pThreadInfo->querySeq] == -1) + || (g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] < + g_queryInfo.specifiedQueryInfo.endAfterConsume[pThreadInfo->querySeq])) { + if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { continue; } - taosMsleep(1000); // ms - res = taos_consume(tsub[i]); - if (res) { - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.specifiedQueryInfo.result[i], pThreadInfo->threadID); - appendResultToFile(res, tmpFile); - } + g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID] = taos_consume( + g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]); + if (g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]) { + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] + != 0) { + sprintf(pThreadInfo->filePath, "%s-%d", + g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); + fetchResult( + g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID], + pThreadInfo); + } + + g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] ++; + if ((g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq] != -1) + && (g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] >= + g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) { + printf("keepProgress:%d, resub specified query: %"PRIu64"\n", + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress, + pThreadInfo->querySeq); + g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0; + taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID], + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); + g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = + subscribeImpl( + SPECIFIED_CLASS, + pThreadInfo, + g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], + g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID], + g_queryInfo.specifiedQueryInfo.subscribeRestart, + g_queryInfo.specifiedQueryInfo.subscribeInterval); + if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) { + taos_close(pThreadInfo->taos); + return NULL; + } + } } - } } - taos_free_result(res); - - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { - taos_unsubscribe(tsub[i], - g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); - } - + taos_free_result(g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]); + taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->querySeq], 0); taos_close(pThreadInfo->taos); + return NULL; } @@ -6746,10 +7475,7 @@ static int subscribeTestProcess() { printfQueryMeta(); resetAfterAnsiEscape(); - if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); - (void) getchar(); - } + prompt(); TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, @@ -6775,93 +7501,126 @@ static int subscribeTestProcess() { pthread_t *pids = NULL; threadInfo *infos = NULL; - //==== create sub threads for query for specified table + + pthread_t *pidsOfStable = NULL; + threadInfo *infosOfStable = NULL; + + //==== create threads for query for specified table if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) { - printf("%s() LN%d, sepcified query sqlCount %"PRIu64".\n", + debugPrint("%s() LN%d, sepcified query sqlCount %d.\n", __func__, __LINE__, g_queryInfo.specifiedQueryInfo.sqlCount); } else { if (g_queryInfo.specifiedQueryInfo.concurrent <= 0) { - errorPrint("%s() LN%d, sepcified query sqlCount %"PRIu64".\n", + errorPrint("%s() LN%d, sepcified query sqlCount %d.\n", __func__, __LINE__, g_queryInfo.specifiedQueryInfo.sqlCount); exit(-1); } - pids = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t)); - infos = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo)); + pids = malloc( + g_queryInfo.specifiedQueryInfo.sqlCount * + g_queryInfo.specifiedQueryInfo.concurrent * + sizeof(pthread_t)); + infos = malloc( + g_queryInfo.specifiedQueryInfo.sqlCount * + g_queryInfo.specifiedQueryInfo.concurrent * + sizeof(threadInfo)); if ((NULL == pids) || (NULL == infos)) { errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); exit(-1); } - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { - threadInfo *t_info = infos + i; - t_info->threadID = i; - t_info->taos = NULL; // TODO: workaround to use separate taos connection; - pthread_create(pids + i, NULL, specifiedSubscribe, t_info); + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { + for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) { + uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j; + threadInfo *pThreadInfo = infos + seq; + pThreadInfo->threadID = seq; + pThreadInfo->querySeq = i; + pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection; + pthread_create(pids + seq, NULL, specifiedSubscribe, pThreadInfo); + } } } - //==== create sub threads for super table query - pthread_t *pidsOfSub = NULL; - threadInfo *infosOfSub = NULL; - if ((g_queryInfo.superQueryInfo.sqlCount > 0) + //==== create threads for super table query + if (g_queryInfo.superQueryInfo.sqlCount <= 0) { + debugPrint("%s() LN%d, super table query sqlCount %d.\n", + __func__, __LINE__, + g_queryInfo.superQueryInfo.sqlCount); + } else { + if ((g_queryInfo.superQueryInfo.sqlCount > 0) && (g_queryInfo.superQueryInfo.threadCnt > 0)) { - pidsOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * + pidsOfStable = malloc( + g_queryInfo.superQueryInfo.sqlCount * + g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t)); - infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * + infosOfStable = malloc( + g_queryInfo.superQueryInfo.sqlCount * + g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo)); - if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { - errorPrint("%s() LN%d, malloc failed for create threads\n", + if ((NULL == pidsOfStable) || (NULL == infosOfStable)) { + errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); - // taos_close(taos); - exit(-1); - } + // taos_close(taos); + exit(-1); + } - int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; - int threads = g_queryInfo.superQueryInfo.threadCnt; + int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; + int threads = g_queryInfo.superQueryInfo.threadCnt; - int64_t a = ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; - } + int64_t a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } - int64_t b = 0; - if (threads != 0) { - b = ntables % threads; - } + int64_t b = 0; + if (threads != 0) { + b = ntables % threads; + } - uint64_t startFrom = 0; - for (int i = 0; i < threads; i++) { - threadInfo *t_info = infosOfSub + i; - t_info->threadID = i; + for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + uint64_t tableFrom = 0; + for (int j = 0; j < threads; j++) { + uint64_t seq = i * threads + j; + threadInfo *pThreadInfo = infosOfStable + seq; + pThreadInfo->threadID = seq; + pThreadInfo->querySeq = i; - t_info->start_table_from = startFrom; - t_info->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; - startFrom = t_info->end_table_to + 1; - t_info->taos = NULL; // TODO: workaround to use separate taos connection; - pthread_create(pidsOfSub + i, NULL, superSubscribe, t_info); - } + pThreadInfo->start_table_from = tableFrom; + pThreadInfo->ntables = jend_table_to = jend_table_to + 1; + pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection; + pthread_create(pidsOfStable + seq, + NULL, superSubscribe, pThreadInfo); + } + } - g_queryInfo.superQueryInfo.threadCnt = threads; + g_queryInfo.superQueryInfo.threadCnt = threads; - for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) { - pthread_join(pidsOfSub[i], NULL); + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + for (int j = 0; j < threads; j++) { + uint64_t seq = i * threads + j; + pthread_join(pidsOfStable[seq], NULL); + } + } } } - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { - pthread_join(pids[i], NULL); + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { + for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) { + uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j; + pthread_join(pids[seq], NULL); + } } tmfree((char*)pids); tmfree((char*)infos); - tmfree((char*)pidsOfSub); - tmfree((char*)infosOfSub); + tmfree((char*)pidsOfStable); + tmfree((char*)infosOfStable); // taos_close(taos); return 0; } @@ -6912,7 +7671,7 @@ static void setParaFromArg(){ g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; g_Dbs.dbCount = 1; - g_Dbs.db[0].drop = 1; + g_Dbs.db[0].drop = true; tstrncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE); g_Dbs.db[0].dbCfg.replica = g_args.replica; @@ -6951,7 +7710,7 @@ static void setParaFromArg(){ tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE); - g_Dbs.db[0].superTbls[0].insertMode = g_args.iface; + g_Dbs.db[0].superTbls[0].iface = g_args.iface; tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp, "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP; @@ -6974,17 +7733,21 @@ static void setParaFromArg(){ if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) { g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR; } else { - for (int i = g_Dbs.db[0].superTbls[0].columnCount; i < g_args.num_of_CPR; i++) { - tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, "INT", MAX_TB_NAME_SIZE); + for (int i = g_Dbs.db[0].superTbls[0].columnCount; + i < g_args.num_of_CPR; i++) { + tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, + "INT", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0; g_Dbs.db[0].superTbls[0].columnCount++; } } - tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, "INT", MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, + "INT", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0; - tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, "BINARY", MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, + "BINARY", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary; g_Dbs.db[0].superTbls[0].tagCount = 2; } else { @@ -7060,7 +7823,6 @@ static void querySqlFile(TAOS* taos, char* sqlFile) } memcpy(cmd + cmd_len, line, read_len); - verbosePrint("%s() LN%d cmd: %s\n", __func__, __LINE__, cmd); if (0 != queryDbExec(taos, cmd, NO_INSERT_TYPE, false)) { errorPrint("%s() LN%d, queryDbExec %s failed!\n", __func__, __LINE__, cmd); @@ -7110,47 +7872,47 @@ static void queryResult() { // query data pthread_t read_id; - threadInfo *rInfo = malloc(sizeof(threadInfo)); - assert(rInfo); - rInfo->start_time = 1500000000000; // 2017-07-14 10:40:00.000 - rInfo->start_table_from = 0; + threadInfo *pThreadInfo = malloc(sizeof(threadInfo)); + assert(pThreadInfo); + pThreadInfo->start_time = 1500000000000; // 2017-07-14 10:40:00.000 + pThreadInfo->start_table_from = 0; - //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc; + //pThreadInfo->do_aggreFunc = g_Dbs.do_aggreFunc; if (g_args.use_metric) { - rInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount; - rInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1; - rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; - tstrncpy(rInfo->tb_prefix, + pThreadInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount; + pThreadInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1; + pThreadInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; + tstrncpy(pThreadInfo->tb_prefix, g_Dbs.db[0].superTbls[0].childTblPrefix, MAX_TB_NAME_SIZE); } else { - rInfo->ntables = g_args.num_of_tables; - rInfo->end_table_to = g_args.num_of_tables -1; - tstrncpy(rInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); + pThreadInfo->ntables = g_args.num_of_tables; + pThreadInfo->end_table_to = g_args.num_of_tables -1; + tstrncpy(pThreadInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); } - rInfo->taos = taos_connect( + pThreadInfo->taos = taos_connect( g_Dbs.host, g_Dbs.user, g_Dbs.password, g_Dbs.db[0].dbName, g_Dbs.port); - if (rInfo->taos == NULL) { + if (pThreadInfo->taos == NULL) { errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); - free(rInfo); + free(pThreadInfo); exit(-1); } - tstrncpy(rInfo->fp, g_Dbs.resultFile, MAX_FILE_NAME_LEN); + tstrncpy(pThreadInfo->filePath, g_Dbs.resultFile, MAX_FILE_NAME_LEN); if (!g_Dbs.use_metric) { - pthread_create(&read_id, NULL, readTable, rInfo); + pthread_create(&read_id, NULL, readTable, pThreadInfo); } else { - pthread_create(&read_id, NULL, readMetric, rInfo); + pthread_create(&read_id, NULL, readMetric, pThreadInfo); } pthread_join(read_id, NULL); - taos_close(rInfo->taos); - free(rInfo); + taos_close(pThreadInfo->taos); + free(pThreadInfo); } static void testCmdLine() { @@ -7204,6 +7966,9 @@ int main(int argc, char *argv[]) { } else { testCmdLine(); } + + if (g_dupstr) + free(g_dupstr); } return 0; diff --git a/src/mnode/inc/mnodeAcct.h b/src/mnode/inc/mnodeAcct.h index 595dcca413..522070e909 100644 --- a/src/mnode/inc/mnodeAcct.h +++ b/src/mnode/inc/mnodeAcct.h @@ -35,6 +35,8 @@ void mnodeDropDbFromAcct(SAcctObj *pAcct, SDbObj *pDb); void mnodeAddUserToAcct(SAcctObj *pAcct, SUserObj *pUser); void mnodeDropUserFromAcct(SAcctObj *pAcct, SUserObj *pUser); +int32_t mnodeCompactAccts(); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeCluster.h b/src/mnode/inc/mnodeCluster.h index a5af544dc2..db258ae6d6 100644 --- a/src/mnode/inc/mnodeCluster.h +++ b/src/mnode/inc/mnodeCluster.h @@ -25,6 +25,8 @@ void mnodeCleanupCluster(); void mnodeUpdateClusterId(); const char* mnodeGetClusterId(); +int32_t mnodeCompactCluster(); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeDb.h b/src/mnode/inc/mnodeDb.h index da0865833d..0fa1a15e2d 100644 --- a/src/mnode/inc/mnodeDb.h +++ b/src/mnode/inc/mnodeDb.h @@ -41,6 +41,8 @@ void mnodeDecDbRef(SDbObj *pDb); bool mnodeCheckIsMonitorDB(char *db, char *monitordb); void mnodeDropAllDbs(SAcctObj *pAcct); +int32_t mnodeCompactDbs(); + // util func void mnodeAddSuperTableIntoDb(SDbObj *pDb); void mnodeRemoveSuperTableFromDb(SDbObj *pDb); diff --git a/src/mnode/inc/mnodeDnode.h b/src/mnode/inc/mnodeDnode.h index fa1995254e..2db794a173 100644 --- a/src/mnode/inc/mnodeDnode.h +++ b/src/mnode/inc/mnodeDnode.h @@ -77,6 +77,7 @@ void * mnodeGetDnodeByEp(char *ep); void mnodeUpdateDnode(SDnodeObj *pDnode); int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg); +int32_t mnodeCompactDnodes(); extern int32_t tsAccessSquence; #ifdef __cplusplus diff --git a/src/mnode/inc/mnodeMnode.h b/src/mnode/inc/mnodeMnode.h index ffdec02eb6..66e9eb1e0e 100644 --- a/src/mnode/inc/mnodeMnode.h +++ b/src/mnode/inc/mnodeMnode.h @@ -50,6 +50,7 @@ char* mnodeGetMnodeMasterEp(); void mnodeGetMnodeInfos(void *mnodes); void mnodeUpdateMnodeEpSet(SMInfos *pMnodes); +int32_t mnodeCompactMnodes(); #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index 3d997a21f3..25b841ef34 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -93,6 +93,7 @@ void sdbUpdateMnodeRoles(); int32_t sdbGetReplicaNum(); int32_t sdbInsertRow(SSdbRow *pRow); +int32_t sdbInsertCompactRow(SSdbRow *pRow); int32_t sdbDeleteRow(SSdbRow *pRow); int32_t sdbUpdateRow(SSdbRow *pRow); int32_t sdbInsertRowToQueue(SSdbRow *pRow); @@ -107,6 +108,7 @@ int32_t sdbGetId(void *pTable); uint64_t sdbGetVersion(); bool sdbCheckRowDeleted(void *pTable, void *pRow); +int32_t mnodeCompactWal(); #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeTable.h b/src/mnode/inc/mnodeTable.h index bf04f26a90..c583a60c7a 100644 --- a/src/mnode/inc/mnodeTable.h +++ b/src/mnode/inc/mnodeTable.h @@ -36,6 +36,7 @@ void mnodeCancelGetNextSuperTable(void *pIter); void mnodeDropAllChildTables(SDbObj *pDropDb); void mnodeDropAllSuperTables(SDbObj *pDropDb); void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup); +int32_t mnodeCompactTables(); #ifdef __cplusplus } diff --git a/src/mnode/inc/mnodeUser.h b/src/mnode/inc/mnodeUser.h index 156bc7aaeb..b8f0805120 100644 --- a/src/mnode/inc/mnodeUser.h +++ b/src/mnode/inc/mnodeUser.h @@ -33,6 +33,8 @@ char * mnodeGetUserFromMsg(void *pMnodeMsg); int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg); void mnodeDropAllUsers(SAcctObj *pAcct); +int32_t mnodeCompactUsers(); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h index 7b798c23f8..73b0e6ae1b 100644 --- a/src/mnode/inc/mnodeVgroup.h +++ b/src/mnode/inc/mnodeVgroup.h @@ -32,6 +32,7 @@ void mnodeDropAllDbVgroups(SDbObj *pDropDb); void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb); void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode); //void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb); +int32_t mnodeCompactVgroups(); void * mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup); void mnodeCancelGetNextVgroup(void *pIter); diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index afe474df6b..64cfa28917 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -238,6 +238,32 @@ static int32_t mnodeCreateRootAcct() { return sdbInsertRow(&row); } +int32_t mnodeCompactAccts() { + void *pIter = NULL; + SAcctObj *pAcct = NULL; + + mInfo("start to compact accts table..."); + + while (1) { + pIter = mnodeGetNextAcct(pIter, &pAcct); + if (pAcct == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsAcctSdb, + .pObj = pAcct, + }; + + mInfo("compact accts %s", pAcct->user); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact accts table..."); + + return 0; +} + #ifndef _ACCT int32_t acctInit() { return TSDB_CODE_SUCCESS; } diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 169d2ebd9d..553e8446ab 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -237,3 +237,27 @@ static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, pShow->numOfReads += numOfRows; return numOfRows; } + +int32_t mnodeCompactCluster() { + SClusterObj *pCluster = NULL; + void *pIter; + + mInfo("start to compact cluster table..."); + + pIter = mnodeGetNextCluster(NULL, &pCluster); + while (pCluster) { + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsClusterSdb, + .pObj = pCluster, + }; + + sdbInsertCompactRow(&row); + + pIter = mnodeGetNextCluster(pIter, &pCluster); + } + + mInfo("end to compact cluster table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index aac16693a5..4d99de590f 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -389,7 +389,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->compression < 0) pCfg->compression = tsCompression; if (pCfg->walLevel < 0) pCfg->walLevel = tsWAL; if (pCfg->replications < 0) pCfg->replications = tsReplications; - if (pCfg->quorum < 0) pCfg->quorum = tsQuorum; + if (pCfg->quorum < 0) pCfg->quorum = MIN(tsQuorum, pCfg->replications); if (pCfg->update < 0) pCfg->update = tsUpdate; if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = tsCacheLastRow; if (pCfg->dbType < 0) pCfg->dbType = 0; @@ -1274,3 +1274,30 @@ void mnodeDropAllDbs(SAcctObj *pAcct) { mInfo("acct:%s, all dbs:%d is dropped from sdb", pAcct->user, numOfDbs); } + +int32_t mnodeCompactDbs() { + void *pIter = NULL; + SDbObj *pDb = NULL; + + mInfo("start to compact dbs table..."); + + while (1) { + pIter = mnodeGetNextDb(pIter, &pDb); + if (pDb == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsDbSdb, + .pObj = pDb, + .rowSize = sizeof(SDbObj), + }; + + mInfo("compact dbs %s", pDb->name); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact dbs table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 3525bcac18..51f16e4bc6 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -1270,3 +1270,30 @@ char* dnodeRoles[] = { "vnode", "any" }; + +int32_t mnodeCompactDnodes() { + SDnodeObj *pDnode = NULL; + void * pIter = NULL; + + mInfo("start to compact dnodes table..."); + + while (1) { + pIter = mnodeGetNextDnode(pIter, &pDnode); + if (pDnode == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsDnodeSdb, + .pObj = pDnode, + .rowSize = sizeof(SDnodeObj), + }; + + mInfo("compact dnode %d", pDnode->dnodeId); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact dnodes table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index fca502391c..df3c49165c 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -59,6 +59,18 @@ static SStep tsMnodeSteps[] = { {"show", mnodeInitShow, mnodeCleanUpShow} }; +static SStep tsMnodeCompactSteps[] = { + {"cluster", mnodeCompactCluster, NULL}, + {"dnodes", mnodeCompactDnodes, NULL}, + {"mnodes", mnodeCompactMnodes, NULL}, + {"accts", mnodeCompactAccts, NULL}, + {"users", mnodeCompactUsers, NULL}, + {"dbs", mnodeCompactDbs, NULL}, + {"vgroups", mnodeCompactVgroups, NULL}, + {"tables", mnodeCompactTables, NULL}, + +}; + static void mnodeInitTimer(); static void mnodeCleanupTimer(); static bool mnodeNeedStart() ; @@ -73,6 +85,11 @@ static int32_t mnodeInitComponents() { return dnodeStepInit(tsMnodeSteps, stepSize); } +int32_t mnodeCompactComponents() { + int32_t stepSize = sizeof(tsMnodeCompactSteps) / sizeof(SStep); + return dnodeStepInit(tsMnodeCompactSteps, stepSize); +} + int32_t mnodeStartSystem() { if (tsMgmtIsRunning) { mInfo("mnode module already started..."); @@ -106,7 +123,7 @@ int32_t mnodeStartSystem() { int32_t mnodeInitSystem() { mnodeInitTimer(); - if (mnodeNeedStart()) { + if (mnodeNeedStart() || tsCompactMnodeWal) { return mnodeStartSystem(); } return 0; diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index ca6d6400ae..ddc9ea59c4 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -566,3 +566,30 @@ static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, vo return numOfRows; } + +int32_t mnodeCompactMnodes() { + void *pIter = NULL; + SMnodeObj *pMnode = NULL; + + mInfo("start to compact mnodes table..."); + + while (1) { + pIter = mnodeGetNextMnode(pIter, &pMnode); + if (pMnode == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsMnodeSdb, + .pObj = pMnode, + .rowSize = sizeof(SMnodeObj), + }; + + mInfo("compact mnode %d", pMnode->mnodeId); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact mnodes table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 505d3c519c..ac3e5d86ec 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -20,6 +20,7 @@ #include "tutil.h" #include "tref.h" #include "tbn.h" +#include "tfs.h" #include "tqueue.h" #include "twal.h" #include "tsync.h" @@ -450,6 +451,12 @@ int32_t sdbInit() { } tsSdbMgmt.status = SDB_STATUS_SERVING; + + if (tsCompactMnodeWal) { + mnodeCompactWal(); + exit(EXIT_SUCCESS); + } + return TSDB_CODE_SUCCESS; } @@ -683,7 +690,7 @@ static int32_t sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void * pthread_mutex_unlock(&tsSdbMgmt.mutex); // from app, row is created - if (pRow != NULL) { + if (pRow != NULL && tsCompactMnodeWal != 1) { // forward to peers pRow->processedCount = 0; int32_t syncCode = syncForwardToPeer(tsSdbMgmt.sync, pHead, pRow, TAOS_QTYPE_RPC, false); @@ -706,19 +713,21 @@ static int32_t sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void * actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version); // even it is WAL/FWD, it shall be called to update version in sync - syncForwardToPeer(tsSdbMgmt.sync, pHead, pRow, TAOS_QTYPE_RPC, false); + if (tsCompactMnodeWal != 1) { + syncForwardToPeer(tsSdbMgmt.sync, pHead, pRow, TAOS_QTYPE_RPC, false); + } // from wal or forward msg, row not created, should add into hash if (action == SDB_ACTION_INSERT) { return sdbPerformInsertAction(pHead, pTable); } else if (action == SDB_ACTION_DELETE) { - if (qtype == TAOS_QTYPE_FWD) { + //if (qtype == TAOS_QTYPE_FWD) { // Drop database/stable may take a long time and cause a timeout, so we confirm first then reput it into queue - sdbWriteFwdToQueue(1, hparam, TAOS_QTYPE_QUERY, unused); - return TSDB_CODE_SUCCESS; - } else { + // sdbWriteFwdToQueue(1, hparam, TAOS_QTYPE_QUERY, unused); + // return TSDB_CODE_SUCCESS; + //} else { return sdbPerformDeleteAction(pHead, pTable); - } + //} } else if (action == SDB_ACTION_UPDATE) { return sdbPerformUpdateAction(pHead, pTable); } else { @@ -726,6 +735,12 @@ static int32_t sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void * } } +int32_t sdbInsertCompactRow(SSdbRow *pRow) { + SSdbTable *pTable = pRow->pTable; + if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; + return sdbWriteRowToQueue(pRow, SDB_ACTION_INSERT); +} + int32_t sdbInsertRow(SSdbRow *pRow) { SSdbTable *pTable = pRow->pTable; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; @@ -1138,3 +1153,46 @@ static void *sdbWorkerFp(void *pWorker) { int32_t sdbGetReplicaNum() { return tsSdbMgmt.cfg.replica; } + +int32_t mnodeCompactWal() { + sdbInfo("vgId:1, start compact mnode wal..."); + + // close old wal + walFsync(tsSdbMgmt.wal, true); + walClose(tsSdbMgmt.wal); + + // reset version,then compacted wal log can start from version 1 + tsSdbMgmt.version = 0; + + // change wal to wal_tmp dir + SWalCfg walCfg = {.vgId = 1, .walLevel = TAOS_WAL_FSYNC, .keep = TAOS_WAL_KEEP, .fsyncPeriod = 0}; + char temp[TSDB_FILENAME_LEN] = {0}; + sprintf(temp, "%s/wal", tsMnodeTmpDir); + tsSdbMgmt.wal = walOpen(temp, &walCfg); + walRenew(tsSdbMgmt.wal); + + // compact memory tables info to wal tmp dir + if (mnodeCompactComponents() != 0) { + tfsRmdir(tsMnodeTmpDir); + return -1; + } + + // close wal + walFsync(tsSdbMgmt.wal, true); + walClose(tsSdbMgmt.wal); + + // rename old wal to wal_bak + if (taosRename(tsMnodeDir, tsMnodeBakDir) != 0) { + return -1; + } + + // rename wal_tmp to wal + if (taosRename(tsMnodeTmpDir, tsMnodeDir) != 0) { + return -1; + } + + // del wal_tmp dir + sdbInfo("vgId:1, compact mnode wal success"); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 2a8e941fcb..5710601e5c 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -966,6 +966,11 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { pMsg->rpcMsg.ahandle, pDrop->name, pSTable->uid, pSTable->numOfTables, taosHashGetSize(pSTable->vgHash)); return mnodeProcessDropSuperTableMsg(pMsg); } else { + // user specify the "DROP STABLE" sql statement, but it is actually a normal table, return error msg. + if (pDrop->supertable) { + return TSDB_CODE_MND_INVALID_TABLE_TYPE; + } + SCTableObj *pCTable = (SCTableObj *)pMsg->pTable; mInfo("msg:%p, app:%p table:%s, start to drop ctable, vgId:%d tid:%d uid:%" PRIu64, pMsg, pMsg->rpcMsg.ahandle, pDrop->name, pCTable->vgId, pCTable->tid, pCTable->uid); @@ -1189,8 +1194,8 @@ static int32_t mnodeFindSuperTableTagIndex(SSTableObj *pStable, const char *tagN static int32_t mnodeAddSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) { SSTableObj *pStable = (SSTableObj *)pMsg->pTable; - mLInfo("msg:%p, app:%p stable %s, add tag result:%s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, - tstrerror(code)); + mLInfo("msg:%p, app:%p stable %s, add tag result:%s, numOfTags:%d", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, + tstrerror(code), pStable->numOfTags); return code; } @@ -1674,12 +1679,9 @@ static int32_t mnodeSetSchemaFromSuperTable(SSchema *pSchema, SSTableObj *pTable return (pTable->numOfColumns + pTable->numOfTags) * sizeof(SSchema); } -static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) { +static int32_t mnodeDoGetSuperTableMeta(SMnodeMsg *pMsg, STableMetaMsg* pMeta) { SSTableObj *pTable = (SSTableObj *)pMsg->pTable; - STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16)); - if (pMeta == NULL) { - return TSDB_CODE_MND_OUT_OF_MEMORY; - } + pMeta->uid = htobe64(pTable->uid); pMeta->sversion = htons(pTable->sversion); pMeta->tversion = htons(pTable->tversion); @@ -1690,6 +1692,18 @@ static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) { pMeta->contLen = sizeof(STableMetaMsg) + mnodeSetSchemaFromSuperTable(pMeta->schema, pTable); tstrncpy(pMeta->tableFname, pTable->info.tableId, sizeof(pMeta->tableFname)); + return TSDB_CODE_SUCCESS; +} + +static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) { + SSTableObj *pTable = (SSTableObj *)pMsg->pTable; + STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16)); + if (pMeta == NULL) { + return TSDB_CODE_MND_OUT_OF_MEMORY; + } + + mnodeDoGetSuperTableMeta(pMsg, pMeta); + pMsg->rpcRsp.len = pMeta->contLen; pMeta->contLen = htons(pMeta->contLen); @@ -1700,11 +1714,7 @@ static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { - SSTableVgroupMsg *pInfo = pMsg->rpcMsg.pCont; - int32_t numOfTable = htonl(pInfo->numOfTables); - - // reserve space +static int32_t calculateVgroupMsgLength(SSTableVgroupMsg* pInfo, int32_t numOfTable) { int32_t contLen = sizeof(SSTableVgroupRspMsg) + 32 * sizeof(SVgroupMsg) + sizeof(SVgroupsMsg); for (int32_t i = 0; i < numOfTable; ++i) { char *stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i; @@ -1716,6 +1726,75 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { mnodeDecTableRef(pTable); } + return contLen; +} + +static char* serializeVgroupInfo(SSTableObj *pTable, char* name, char* msg, SMnodeMsg* pMsgBody, void* handle) { + SName sn = {0}; + tNameFromString(&sn, name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + const char* tableName = tNameGetTableName(&sn); + + strncpy(msg, tableName, TSDB_TABLE_NAME_LEN); + msg += TSDB_TABLE_NAME_LEN; + + if (pTable->vgHash == NULL) { + mDebug("msg:%p, app:%p stable:%s, no vgroup exist while get stable vgroup info", pMsgBody, handle, name); + mnodeDecTableRef(pTable); + + // even this super table has no corresponding table, still return + SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)msg; + pVgroupMsg->numOfVgroups = 0; + + msg += sizeof(SVgroupsMsg); + } else { + SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)msg; + mDebug("msg:%p, app:%p stable:%s, hash:%p sizeOfVgList:%d will be returned", pMsgBody, handle, + pTable->info.tableId, pTable->vgHash, taosHashGetSize(pTable->vgHash)); + + int32_t *pVgId = taosHashIterate(pTable->vgHash, NULL); + int32_t vgSize = 0; + while (pVgId) { + SVgObj *pVgroup = mnodeGetVgroup(*pVgId); + pVgId = taosHashIterate(pTable->vgHash, pVgId); + if (pVgroup == NULL) { + continue; + } + + pVgroupMsg->vgroups[vgSize].vgId = htonl(pVgroup->vgId); + pVgroupMsg->vgroups[vgSize].numOfEps = 0; + + for (int32_t vn = 0; vn < pVgroup->numOfVnodes; ++vn) { + SDnodeObj *pDnode = pVgroup->vnodeGid[vn].pDnode; + if (pDnode == NULL) break; + + tstrncpy(pVgroupMsg->vgroups[vgSize].epAddr[vn].fqdn, pDnode->dnodeFqdn, TSDB_FQDN_LEN); + pVgroupMsg->vgroups[vgSize].epAddr[vn].port = htons(pDnode->dnodePort); + + pVgroupMsg->vgroups[vgSize].numOfEps++; + } + + vgSize++; + mnodeDecVgroupRef(pVgroup); + } + + taosHashCancelIterate(pTable->vgHash, pVgId); + mnodeDecTableRef(pTable); + + pVgroupMsg->numOfVgroups = htonl(vgSize); + + // one table is done, try the next table + msg += sizeof(SVgroupsMsg) + vgSize * sizeof(SVgroupMsg); + } + + return msg; +} + +static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { + SSTableVgroupMsg *pInfo = pMsg->rpcMsg.pCont; + int32_t numOfTable = htonl(pInfo->numOfTables); + + // calculate the required space. + int32_t contLen = calculateVgroupMsgLength(pInfo, numOfTable); SSTableVgroupRspMsg *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -1726,62 +1805,16 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { for (int32_t i = 0; i < numOfTable; ++i) { char *stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i; + SSTableObj *pTable = mnodeGetSuperTable(stableName); if (pTable == NULL) { mError("msg:%p, app:%p stable:%s, not exist while get stable vgroup info", pMsg, pMsg->rpcMsg.ahandle, stableName); mnodeDecTableRef(pTable); continue; } - if (pTable->vgHash == NULL) { - mDebug("msg:%p, app:%p stable:%s, no vgroup exist while get stable vgroup info", pMsg, pMsg->rpcMsg.ahandle, - stableName); - mnodeDecTableRef(pTable); - // even this super table has no corresponding table, still return - pRsp->numOfTables++; - - SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)msg; - pVgroupMsg->numOfVgroups = 0; - - msg += sizeof(SVgroupsMsg); - } else { - SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)msg; - mDebug("msg:%p, app:%p stable:%s, hash:%p sizeOfVgList:%d will be returned", pMsg, pMsg->rpcMsg.ahandle, - pTable->info.tableId, pTable->vgHash, taosHashGetSize(pTable->vgHash)); - - int32_t *pVgId = taosHashIterate(pTable->vgHash, NULL); - int32_t vgSize = 0; - while (pVgId) { - SVgObj *pVgroup = mnodeGetVgroup(*pVgId); - pVgId = taosHashIterate(pTable->vgHash, pVgId); - if (pVgroup == NULL) continue; - - pVgroupMsg->vgroups[vgSize].vgId = htonl(pVgroup->vgId); - pVgroupMsg->vgroups[vgSize].numOfEps = 0; - - for (int32_t vn = 0; vn < pVgroup->numOfVnodes; ++vn) { - SDnodeObj *pDnode = pVgroup->vnodeGid[vn].pDnode; - if (pDnode == NULL) break; - - tstrncpy(pVgroupMsg->vgroups[vgSize].epAddr[vn].fqdn, pDnode->dnodeFqdn, TSDB_FQDN_LEN); - pVgroupMsg->vgroups[vgSize].epAddr[vn].port = htons(pDnode->dnodePort); - - pVgroupMsg->vgroups[vgSize].numOfEps++; - } - - vgSize++; - mnodeDecVgroupRef(pVgroup); - } - - taosHashCancelIterate(pTable->vgHash, pVgId); - mnodeDecTableRef(pTable); - - pVgroupMsg->numOfVgroups = htonl(vgSize); - - // one table is done, try the next table - msg += sizeof(SVgroupsMsg) + vgSize * sizeof(SVgroupMsg); - pRsp->numOfTables++; - } + msg = serializeVgroupInfo(pTable, stableName, msg, pMsg, pMsg->rpcMsg.ahandle); + pRsp->numOfTables++; } if (pRsp->numOfTables != numOfTable) { @@ -2415,9 +2448,9 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { pMeta->vgroup.numOfEps++; mnodeDecDnodeRef(pDnode); } - pMeta->vgroup.vgId = htonl(pMsg->pVgroup->vgId); - mDebug("msg:%p, app:%p table:%s, uid:%" PRIu64 " table meta is retrieved, vgId:%d sid:%d", pMsg, pMsg->rpcMsg.ahandle, + pMeta->vgroup.vgId = htonl(pMsg->pVgroup->vgId); + mDebug("msg:%p, app:%p table:%s, uid:%" PRIu64 " table meta is retrieved, vgId:%d tid:%d", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, pTable->uid, pTable->vgId, pTable->tid); return TSDB_CODE_SUCCESS; @@ -2811,56 +2844,137 @@ static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg) { static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) { SMultiTableInfoMsg *pInfo = pMsg->rpcMsg.pCont; - pInfo->numOfTables = htonl(pInfo->numOfTables); - int32_t totalMallocLen = 4 * 1024 * 1024; // first malloc 4 MB, subsequent reallocation as twice - SMultiTableMeta *pMultiMeta = rpcMallocCont(totalMallocLen); + pInfo->numOfTables = htonl(pInfo->numOfTables); + pInfo->numOfVgroups = htonl(pInfo->numOfVgroups); + + int32_t contLen = pMsg->rpcMsg.contLen - sizeof(SMultiTableInfoMsg); + + int32_t num = 0; + int32_t code = TSDB_CODE_SUCCESS; + char* str = strndup(pInfo->tableNames, contLen); + char** nameList = strsplit(str, ",", &num); + SArray* pList = taosArrayInit(4, POINTER_BYTES); + SMultiTableMeta *pMultiMeta = NULL; + + if (num != pInfo->numOfTables + pInfo->numOfVgroups) { + mError("msg:%p, app:%p, failed to get multi-tableMeta, msg inconsistent", pMsg, pMsg->rpcMsg.ahandle); + code = TSDB_CODE_MND_INVALID_TABLE_NAME; + goto _end; + } + + // first malloc 80KB, subsequent reallocation will expand the size as twice of the original size + int32_t totalMallocLen = sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16); + pMultiMeta = rpcMallocCont(totalMallocLen); if (pMultiMeta == NULL) { - return TSDB_CODE_MND_OUT_OF_MEMORY; + code = TSDB_CODE_MND_OUT_OF_MEMORY; + goto _end; } pMultiMeta->contLen = sizeof(SMultiTableMeta); pMultiMeta->numOfTables = 0; - for (int32_t t = 0; t < pInfo->numOfTables; ++t) { - char * tableId = (char *)(pInfo->tableIds + t * TSDB_TABLE_FNAME_LEN); - SCTableObj *pTable = mnodeGetChildTable(tableId); - if (pTable == NULL) continue; + int32_t t = 0; + for (; t < pInfo->numOfTables; ++t) { + char *fullName = nameList[t]; - if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableName(tableId); - if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) { - mnodeDecTableRef(pTable); - continue; + pMsg->pTable = mnodeGetTable(fullName); + if (pMsg->pTable == NULL) { + mError("msg:%p, app:%p table:%s, failed to get table meta, table not exist", pMsg, pMsg->rpcMsg.ahandle, fullName); + code = TSDB_CODE_MND_INVALID_TABLE_NAME; + goto _end; } - int availLen = totalMallocLen - pMultiMeta->contLen; - if (availLen <= sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16)) { + if (pMsg->pDb == NULL) { + pMsg->pDb = mnodeGetDbByTableName(fullName); + } + + if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) { + mnodeDecTableRef(pMsg->pTable); + code = TSDB_CODE_APP_NOT_READY; + goto _end; + } + + int remain = totalMallocLen - pMultiMeta->contLen; + if (remain <= sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16)) { totalMallocLen *= 2; pMultiMeta = rpcReallocCont(pMultiMeta, totalMallocLen); if (pMultiMeta == NULL) { - mnodeDecTableRef(pTable); - return TSDB_CODE_MND_OUT_OF_MEMORY; - } else { - t--; - mnodeDecTableRef(pTable); - continue; + mnodeDecTableRef(pMsg->pTable); + code = TSDB_CODE_MND_OUT_OF_MEMORY; + goto _end; } } - STableMetaMsg *pMeta = (STableMetaMsg *)(pMultiMeta->metas + pMultiMeta->contLen); - int32_t code = mnodeDoGetChildTableMeta(pMsg, pMeta); - if (code == TSDB_CODE_SUCCESS) { - pMultiMeta->numOfTables ++; - pMultiMeta->contLen += pMeta->contLen; + STableMetaMsg *pMeta = (STableMetaMsg *)((char*) pMultiMeta + pMultiMeta->contLen); + + if (pMsg->pTable->type == TSDB_SUPER_TABLE) { + code = mnodeDoGetSuperTableMeta(pMsg, pMeta); + taosArrayPush(pList, &fullName); // keep the full name for each super table for retrieve vgroup list + } else { + code = mnodeDoGetChildTableMeta(pMsg, pMeta); + if (pMsg->pVgroup != NULL) { + mnodeDecVgroupRef(pMsg->pVgroup); + pMsg->pVgroup = NULL; + } } - mnodeDecTableRef(pTable); + mnodeDecTableRef(pMsg->pTable); + pMsg->pTable = NULL; + + if (code == TSDB_CODE_SUCCESS) { + pMultiMeta->numOfTables++; + pMultiMeta->contLen += pMeta->contLen; + } else { + // ignore error and continue. + // Otherwise the client may found that the responding message is inconsistent. +// goto _end; + } } + char* msg = (char*) pMultiMeta + pMultiMeta->contLen; + + // add the additional super table names that needs the vgroup info + for(;t < num; ++t) { + taosArrayPush(pList, &nameList[t]); + } + + // add the pVgroupList into the pList + int32_t numOfVgroupList = (int32_t) taosArrayGetSize(pList); + pMultiMeta->numOfVgroup = htonl(numOfVgroupList); + + for(int32_t i = 0; i < numOfVgroupList; ++i) { + char* name = taosArrayGetP(pList, i); + + SSTableObj *pTable = mnodeGetSuperTable(name); + if (pTable == NULL) { + mError("msg:%p, app:%p stable:%s, not exist while get stable vgroup info", pMsg, pMsg->rpcMsg.ahandle, name); + code = TSDB_CODE_MND_INVALID_TABLE_NAME; + goto _end; + } + + msg = serializeVgroupInfo(pTable, name, msg, pMsg, pMsg->rpcMsg.ahandle); + } + + pMultiMeta->contLen = (int32_t) (msg - (char*) pMultiMeta); + + pMultiMeta->numOfTables = htonl(pMultiMeta->numOfTables); pMsg->rpcRsp.rsp = pMultiMeta; pMsg->rpcRsp.len = pMultiMeta->contLen; + code = TSDB_CODE_SUCCESS; - return TSDB_CODE_SUCCESS; + _end: + tfree(str); + tfree(nameList); + taosArrayDestroy(pList); + pMsg->pTable = NULL; + pMsg->pVgroup = NULL; + + if (code != TSDB_CODE_SUCCESS) { + rpcFreeCont(pMultiMeta); + } + + return code; } static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { @@ -3242,3 +3356,65 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro return numOfRows; } + +static int32_t mnodeCompactSuperTables() { + void *pIter = NULL; + SSTableObj *pTable = NULL; + + mInfo("start to compact super table..."); + + while (1) { + pIter = mnodeGetNextSuperTable(pIter, &pTable); + if (pTable == NULL) break; + + int32_t schemaSize = (pTable->numOfColumns + pTable->numOfTags) * sizeof(SSchema); + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsSuperTableSdb, + .pObj = pTable, + .rowSize = sizeof(SSTableObj) + schemaSize, + }; + + //mInfo("compact super %" PRIu64, pTable->uid); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact super table..."); + + return 0; +} + +static int32_t mnodeCompactChildTables() { + void *pIter = NULL; + SCTableObj *pTable = NULL; + + mInfo("start to compact child table..."); + + while (1) { + pIter = mnodeGetNextChildTable(pIter, &pTable); + if (pTable == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pObj = pTable, + .pTable = tsChildTableSdb, + }; + + //mInfo("compact child %" PRIu64 ":%d", pTable->uid, pTable->tid); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact child table..."); + + return 0; +} + +int32_t mnodeCompactTables() { + mnodeCompactSuperTables(); + + mnodeCompactChildTables(); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index e77c1b3e59..c5c54791cf 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -617,3 +617,30 @@ static int32_t mnodeProcessAuthMsg(SMnodeMsg *pMsg) { return mnodeRetriveAuth(pAuthMsg->user, &pAuthRsp->spi, &pAuthRsp->encrypt, pAuthRsp->secret, pAuthRsp->ckey); } + +int32_t mnodeCompactUsers() { + void *pIter = NULL; + SUserObj *pUser = NULL; + + mInfo("start to compact users table..."); + + while (1) { + pIter = mnodeGetNextUser(pIter, &pUser); + if (pUser == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsUserSdb, + .pObj = pUser, + .rowSize = sizeof(SUserObj), + }; + + mInfo("compact users %s", pUser->user); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact users table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 7222c8d1a0..a64d256484 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -121,7 +121,7 @@ static int32_t mnodeVgroupActionDelete(SSdbRow *pRow) { SVgObj *pVgroup = pRow->pObj; if (pVgroup->pDb == NULL) { - mError("vgId:%d, db:%s is not exist while insert into hash", pVgroup->vgId, pVgroup->dbName); + mError("vgId:%d, db:%s is not exist while delete from hash", pVgroup->vgId, pVgroup->dbName); return TSDB_CODE_MND_VGROUP_NOT_EXIST; } @@ -1302,3 +1302,30 @@ void mnodeSetVgidVer(int8_t *cver, uint64_t iver) { cver[1] = (int8_t)((int32_t)(iver % 100000) / 100); cver[2] = (int8_t)(iver % 100); } + +int32_t mnodeCompactVgroups() { + void *pIter = NULL; + SVgObj *pVgroup = NULL; + + mInfo("start to compact vgroups table..."); + + while (1) { + pIter = mnodeGetNextVgroup(pIter, &pVgroup); + if (pVgroup == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsVgroupSdb, + .pObj = pVgroup, + .rowSize = sizeof(SVgObj), + }; + + mInfo("compact vgroups %d", pVgroup->vgId); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact vgroups table..."); + + return 0; +} \ No newline at end of file diff --git a/src/os/inc/osDir.h b/src/os/inc/osDir.h index bdc65e4520..7afe126451 100644 --- a/src/os/inc/osDir.h +++ b/src/os/inc/osDir.h @@ -21,6 +21,7 @@ extern "C" { #endif void taosRemoveDir(char *rootDir); +bool taosDirExist(const char* dirname); int32_t taosMkDir(const char *pathname, mode_t mode); void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays); int32_t taosRename(char *oldName, char *newName); diff --git a/src/os/inc/osMips64.h b/src/os/inc/osMips64.h new file mode 100644 index 0000000000..ed7b08a311 --- /dev/null +++ b/src/os/inc/osMips64.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_MIPS64_H +#define TDENGINE_OS_MIPS64_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/src/detail/osDir.c b/src/os/src/detail/osDir.c index 2a2e2519b5..c467c64872 100644 --- a/src/os/src/detail/osDir.c +++ b/src/os/src/detail/osDir.c @@ -45,6 +45,10 @@ void taosRemoveDir(char *rootDir) { uInfo("dir:%s is removed", rootDir); } +bool taosDirExist(const char* dirname) { + return access(dirname, F_OK) == 0; +} + int taosMkDir(const char *path, mode_t mode) { int code = mkdir(path, 0755); if (code < 0 && errno == EEXIST) code = 0; diff --git a/src/os/src/detail/osMemory.c b/src/os/src/detail/osMemory.c index 291a54b669..d8194feab4 100644 --- a/src/os/src/detail/osMemory.c +++ b/src/os/src/detail/osMemory.c @@ -62,7 +62,7 @@ static void* taosRandomRealloc(void* ptr, size_t size, const char* file, uint32_ static char* taosRandomStrdup(const char* str, const char* file, uint32_t line) { size_t len = strlen(str); - return taosRandomAllocFail(len + 1, file, line) ? NULL : taosStrdupImp(str); + return taosRandomAllocFail(len + 1, file, line) ? NULL : tstrdup(str); } static char* taosRandomStrndup(const char* str, size_t size, const char* file, uint32_t line) { @@ -70,11 +70,11 @@ static char* taosRandomStrndup(const char* str, size_t size, const char* file, u if (len > size) { len = size; } - return taosRandomAllocFail(len + 1, file, line) ? NULL : taosStrndupImp(str, len); + return taosRandomAllocFail(len + 1, file, line) ? NULL : tstrndup(str, len); } static ssize_t taosRandomGetline(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { - return taosRandomAllocFail(*n, file, line) ? -1 : taosGetlineImp(lineptr, n, stream); + return taosRandomAllocFail(*n, file, line) ? -1 : tgetline(lineptr, n, stream); } //////////////////////////////////////////////////////////////////////////////// @@ -242,7 +242,7 @@ static char* taosStrndupDetectLeak(const char* str, size_t size, const char* fil static ssize_t taosGetlineDetectLeak(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { char* buf = NULL; size_t bufSize = 0; - ssize_t size = taosGetlineImp(&buf, &bufSize, stream); + ssize_t size = tgetline(&buf, &bufSize, stream); if (size != -1) { if (*n < size + 1) { void* p = taosReallocDetectLeak(*lineptr, size + 1, file, line); @@ -372,7 +372,7 @@ void taosFreeMem(void* ptr, const char* file, uint32_t line) { char* taosStrdupMem(const char* str, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: - return taosStrdupImp(str); + return tstrdup(str); case TAOS_ALLOC_MODE_RANDOM_FAIL: return taosRandomStrdup(str, file, line); @@ -380,13 +380,13 @@ char* taosStrdupMem(const char* str, const char* file, uint32_t line) { case TAOS_ALLOC_MODE_DETECT_LEAK: return taosStrdupDetectLeak(str, file, line); } - return taosStrdupImp(str); + return tstrdup(str); } char* taosStrndupMem(const char* str, size_t size, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: - return taosStrndupImp(str, size); + return tstrndup(str, size); case TAOS_ALLOC_MODE_RANDOM_FAIL: return taosRandomStrndup(str, size, file, line); @@ -394,13 +394,13 @@ char* taosStrndupMem(const char* str, size_t size, const char* file, uint32_t li case TAOS_ALLOC_MODE_DETECT_LEAK: return taosStrndupDetectLeak(str, size, file, line); } - return taosStrndupImp(str, size); + return tstrndup(str, size); } ssize_t taosGetlineMem(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: - return taosGetlineImp(lineptr, n, stream); + return tgetline(lineptr, n, stream); case TAOS_ALLOC_MODE_RANDOM_FAIL: return taosRandomGetline(lineptr, n, stream, file, line); @@ -408,7 +408,7 @@ ssize_t taosGetlineMem(char **lineptr, size_t *n, FILE *stream, const char* file case TAOS_ALLOC_MODE_DETECT_LEAK: return taosGetlineDetectLeak(lineptr, n, stream, file, line); } - return taosGetlineImp(lineptr, n, stream); + return tgetline(lineptr, n, stream); } static void taosCloseAllocLog() { @@ -517,4 +517,4 @@ void* taosTZfree(void* ptr) { free((void*)((char*)ptr - sizeof(size_t))); } return NULL; -} \ No newline at end of file +} diff --git a/src/os/src/detail/osSignal.c b/src/os/src/detail/osSignal.c index 4467a607b2..33cc39e112 100644 --- a/src/os/src/detail/osSignal.c +++ b/src/os/src/detail/osSignal.c @@ -25,14 +25,14 @@ typedef void (*FLinuxSignalHandler)(int32_t signum, siginfo_t *sigInfo, void *context); void taosSetSignal(int32_t signum, FSignalHandler sigfp) { - struct sigaction act = {{0}}; + struct sigaction act; memset(&act, 0, sizeof(act)); #if 1 act.sa_flags = SA_SIGINFO; act.sa_sigaction = (FLinuxSignalHandler)sigfp; -#else - act.sa_handler = sigfp; -#endif - sigaction(signum, &act, NULL); +#else + act.sa_handler = sigfp; +#endif + sigaction(signum, &act, NULL); } void taosIgnSignal(int32_t signum) { diff --git a/src/os/src/detail/osTime.c b/src/os/src/detail/osTime.c index d9d070218e..67e0c2642e 100644 --- a/src/os/src/detail/osTime.c +++ b/src/os/src/detail/osTime.c @@ -87,12 +87,12 @@ static int32_t (*parseLocaltimeFp[]) (char* timestr, int64_t* time, int32_t time int32_t taosGetTimestampSec() { return (int32_t)time(NULL); } -int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t daylight) { +int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t day_light) { /* parse datatime string in with tz */ if (strnchr(timestr, 'T', len, false) != NULL) { return parseTimeWithTz(timestr, time, timePrec); } else { - return (*parseLocaltimeFp[daylight])(timestr, time, timePrec); + return (*parseLocaltimeFp[day_light])(timestr, time, timePrec); } } diff --git a/src/os/src/windows/wSemphone.c b/src/os/src/windows/wSemphone.c index a3f0367ee1..878ceba791 100644 --- a/src/os/src/windows/wSemphone.c +++ b/src/os/src/windows/wSemphone.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE + #include "os.h" #include "taosdef.h" #include "tglobal.h" @@ -24,7 +25,7 @@ bool taosCheckPthreadValid(pthread_t thread) { return thread.p != NULL; } -void taosResetPthread(pthread_t *thread) { thread->p = 0; } +void taosResetPthread(pthread_t* thread) { thread->p = 0; } int64_t taosGetPthreadId(pthread_t thread) { #ifdef PTW32_VERSION @@ -34,27 +35,24 @@ int64_t taosGetPthreadId(pthread_t thread) { #endif } -int64_t taosGetSelfPthreadId() { - return GetCurrentThreadId(); -} +int64_t taosGetSelfPthreadId() { return GetCurrentThreadId(); } -bool taosComparePthread(pthread_t first, pthread_t second) { - return first.p == second.p; -} +bool taosComparePthread(pthread_t first, pthread_t second) { return first.p == second.p; } -int32_t taosGetPId() { - return GetCurrentProcessId(); -} +int32_t taosGetPId() { return GetCurrentProcessId(); } -int32_t taosGetCurrentAPPName(char *name, int32_t* len) { +int32_t taosGetCurrentAPPName(char* name, int32_t* len) { char filepath[1024] = {0}; GetModuleFileName(NULL, filepath, MAX_PATH); - *strrchr(filepath,'.') = '\0'; + char* sub = strrchr(filepath, '.'); + if (sub != NULL) { + *sub = '\0'; + } strcpy(name, filepath); if (len != NULL) { - *len = (int32_t) strlen(filepath); + *len = (int32_t)strlen(filepath); } return 0; diff --git a/src/plugins/http/src/httpResp.c b/src/plugins/http/src/httpResp.c index 063f2bb04e..4fcf236fce 100644 --- a/src/plugins/http/src/httpResp.c +++ b/src/plugins/http/src/httpResp.c @@ -165,7 +165,7 @@ void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char *errMsg) { } } - httpSendErrorRespImp(pContext, httpCode, "Bad Request", TSDB_CODE_TSC_INVALID_SQL & 0XFFFF, temp); + httpSendErrorRespImp(pContext, httpCode, "Bad Request", TSDB_CODE_TSC_INVALID_OPERATION & 0XFFFF, temp); } void httpSendSuccResp(HttpContext *pContext, char *desc) { diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index b345c1531f..5a0480b694 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -263,7 +263,7 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int32_t code if (code != TSDB_CODE_SUCCESS) { SSqlObj *pObj = (SSqlObj *)result; - if (code == TSDB_CODE_TSC_INVALID_SQL) { + if (code == TSDB_CODE_TSC_INVALID_OPERATION) { terrno = code; httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p, error:%s", pContext, pContext->fd, pContext->user, tstrerror(code), pObj, taos_errstr(pObj)); diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c index 399a33954d..a8031d3fd8 100644 --- a/src/plugins/http/src/httpUtil.c +++ b/src/plugins/http/src/httpUtil.c @@ -237,6 +237,11 @@ void httpFreeMultiCmds(HttpContext *pContext) { JsonBuf *httpMallocJsonBuf(HttpContext *pContext) { if (pContext->jsonBuf == NULL) { pContext->jsonBuf = (JsonBuf *)malloc(sizeof(JsonBuf)); + if (pContext->jsonBuf == NULL) { + return NULL; + } + + memset(pContext->jsonBuf, 0, sizeof(JsonBuf)); } if (!pContext->jsonBuf->pContext) { diff --git a/src/plugins/monitor/src/monMain.c b/src/plugins/monitor/src/monMain.c index 94af8e3ecd..d9f7d81ebd 100644 --- a/src/plugins/monitor/src/monMain.c +++ b/src/plugins/monitor/src/monMain.c @@ -417,3 +417,13 @@ void monExecuteSQL(char *sql) { monDebug("execute sql:%s", sql); taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "sql"); } + +void monExecuteSQLWithResultCallback(char *sql, MonExecuteSQLCbFP callback, void* param) { + if (tsMonitor.conn == NULL) { + callback(param, NULL, TSDB_CODE_MON_CONNECTION_INVALID); + return; + } + + monDebug("execute sql:%s", sql); + taos_query_a(tsMonitor.conn, sql, callback, param); +} diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 59ad2adea1..8f4e0dd261 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -71,13 +71,13 @@ typedef struct SResultRowPool { SArray* pData; // SArray } SResultRowPool; -typedef struct SSqlGroupbyExpr { +typedef struct SGroupbyExpr { int16_t tableIndex; SArray* columnInfo; // SArray, group by columns information - int16_t numOfGroupCols; + int16_t numOfGroupCols; // todo remove it int16_t orderIndex; // order by column index int16_t orderType; // order by type: asc/desc -} SSqlGroupbyExpr; +} SGroupbyExpr; typedef struct SResultRow { int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer @@ -217,7 +217,7 @@ typedef struct SQueryAttr { int32_t intermediateResultRowSize; // intermediate result row size, in case of top-k query. int32_t maxTableColumnWidth; int32_t tagLen; // tag value length of current query - SSqlGroupbyExpr* pGroupbyExpr; + SGroupbyExpr* pGroupbyExpr; SExprInfo* pExpr1; SExprInfo* pExpr2; @@ -305,6 +305,7 @@ enum OPERATOR_TYPE_E { OP_GlobalAggregate = 18, // global merge for the multi-way data sources. OP_Filter = 19, OP_Distinct = 20, + OP_Join = 21, }; typedef struct SOperatorInfo { @@ -317,7 +318,8 @@ typedef struct SOperatorInfo { SExprInfo *pExpr; SQueryRuntimeEnv *pRuntimeEnv; - struct SOperatorInfo *upstream; + struct SOperatorInfo **upstream; // upstream pointer list + int32_t numOfUpstream; // number of upstream. The value is always ONE expect for join operator __operator_fn_t exec; __optr_cleanup_fn_t cleanup; } SOperatorInfo; @@ -365,7 +367,7 @@ typedef struct SQueryParam { SColIndex *pGroupColIndex; SColumnInfo *pTagColumnInfo; - SSqlGroupbyExpr *pGroupbyExpr; + SGroupbyExpr *pGroupbyExpr; int32_t tableScanOperator; SArray *pOperator; SUdfInfo *pUdfInfo; @@ -496,9 +498,11 @@ typedef struct SMultiwayMergeInfo { bool hasPrev; bool groupMix; - SArray *udfInfo; + SArray *udfInfo; } SMultiwayMergeInfo; +void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream); + SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime); SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime); SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv); @@ -519,12 +523,20 @@ SOperatorInfo* createMultiwaySortOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SEx int32_t numOfRows, void* merger, bool groupMix); SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo); SOperatorInfo* createSLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* merger); -SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, + int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter); + +SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput); SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); SSDataBlock* doSLimit(void* param, bool* newgroup); +int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId); +void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock); +bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p); +void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p); + SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows); void* destroyOutputBuf(SSDataBlock* pBlock); @@ -542,13 +554,14 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo, SSqlExpr **pExpr, SExprInfo *prevExpr, SUdfInfo *pUdfInfo); -SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code); -SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, - SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, bool stableQuery, char* sql, uint64_t *qId, int32_t vgId, SUdfInfo* pUdfInfo); +SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code); +SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, + SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, int32_t vgId, char* sql, uint64_t *qId, SUdfInfo* pUdfInfo); int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start, int32_t prevResultLen, void* merger); +int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId); void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters); STableQueryInfo *createTableQueryInfo(SQueryAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf); diff --git a/src/query/inc/qFill.h b/src/query/inc/qFill.h index 00ac86caf4..caa0c55b3f 100644 --- a/src/query/inc/qFill.h +++ b/src/query/inc/qFill.h @@ -62,7 +62,7 @@ typedef struct SFillInfo { SFillColInfo* pFillCol; // column info for fill operations SFillTagColInfo* pTags; // tags value for filling gap - void* handle; // for dubug purpose + void* handle; // for debug purpose } SFillInfo; typedef struct SPoint { @@ -82,8 +82,6 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const struct SSDataBlock* pInput); -void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, const tFilePage* pInput); - bool taosFillHasMoreResults(SFillInfo* pFillInfo); int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows); diff --git a/src/query/inc/qHistogram.h b/src/query/inc/qHistogram.h index 7742d151a0..3b5c2b4cfb 100644 --- a/src/query/inc/qHistogram.h +++ b/src/query/inc/qHistogram.h @@ -40,7 +40,7 @@ typedef struct SHeapEntry { } SHeapEntry; typedef struct SHistogramInfo { - int32_t numOfElems; + int64_t numOfElems; int32_t numOfEntries; int32_t maxEntries; double min; diff --git a/src/query/inc/qPlan.h b/src/query/inc/qPlan.h index 8f35565e4b..30c0f8db4e 100644 --- a/src/query/inc/qPlan.h +++ b/src/query/inc/qPlan.h @@ -16,7 +16,38 @@ #ifndef TDENGINE_QPLAN_H #define TDENGINE_QPLAN_H -//TODO refactor +struct SQueryInfo; + +typedef struct SQueryNodeBasicInfo { + int32_t type; + char *name; +} SQueryNodeBasicInfo; + +typedef struct SQueryTableInfo { + char *tableName; + STableId id; +} SQueryTableInfo; + +typedef struct SQueryNode { + SQueryNodeBasicInfo info; + SQueryTableInfo tableInfo; + SSchema *pSchema; // the schema of the input SSDatablock + int32_t numOfCols; // number of input columns + SExprInfo *pExpr; // the query functions or sql aggregations + int32_t numOfOutput; // number of result columns, which is also the number of pExprs + + void *pExtInfo; // additional information + // previous operator to generated result for current node to process + // in case of join, multiple prev nodes exist. + SArray *pPrevNodes;// upstream nodes + struct SQueryNode *nextNode; +} SQueryNode; + +SQueryNode* qCreateQueryPlan(struct SQueryInfo* pQueryInfo); +void* qDestroyQueryPlan(SQueryNode* pQueryNode); + +char* queryPlanToString(SQueryNode* pQueryNode); + SArray* createTableScanPlan(SQueryAttr* pQueryAttr); SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr); SArray* createGlobalMergePlan(SQueryAttr* pQueryAttr); diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index 6f11d0aab7..63b928604c 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -107,14 +107,18 @@ typedef struct SSqlNode { struct tSqlExpr *pHaving; // having clause [optional] } SSqlNode; -typedef struct STableNamePair { - SStrToken name; +typedef struct SRelElementPair { + union { + SStrToken tableName; + SArray *pSubquery; + }; + SStrToken aliasName; -} STableNamePair; +} SRelElementPair; typedef struct SRelationInfo { int32_t type; // nested query|table name list - SArray *list; // SArray|SArray + SArray *list; // SArray } SRelationInfo; typedef struct SCreatedTableInfo { @@ -264,8 +268,9 @@ SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int SArray *tVariantListAppendToken(SArray *pList, SStrToken *pAliasToken, uint8_t sortOrder); SRelationInfo *setTableNameList(SRelationInfo* pFromInfo, SStrToken *pName, SStrToken* pAlias); -SRelationInfo *setSubquery(SRelationInfo* pFromInfo, SArray* pSqlNode); +//SRelationInfo *setSubquery(SRelationInfo* pFromInfo, SRelElementPair* p); void *destroyRelationInfo(SRelationInfo* pFromInfo); +SRelationInfo *addSubqueryElem(SRelationInfo* pRelationInfo, SArray* pSub, SStrToken* pAlias); // sql expr leaf node tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType); diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index c6ffcf7181..00a4c2572b 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -47,6 +47,9 @@ void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow, in SResultRowCellInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset); +void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr); +void* freeColumnInfo(SColumnInfo* pColumnInfo, int32_t numOfCols); + static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) { assert(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size); return pResultRowInfo->pResult[slot]; diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index 847a128396..58f7d4da26 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -519,7 +519,13 @@ distinct(X) ::= . { X.n = 0;} %type from {SRelationInfo*} %destructor from {destroyRelationInfo($$);} from(A) ::= FROM tablelist(X). {A = X;} -from(A) ::= FROM LP union(Y) RP. {A = setSubquery(NULL, Y);} +from(A) ::= FROM sub(X). {A = X;} + +%type sub {SRelationInfo*} +%destructor sub {destroyRelationInfo($$);} +sub(A) ::= LP union(Y) RP. {A = addSubqueryElem(NULL, Y, NULL);} +sub(A) ::= LP union(Y) RP ids(Z). {A = addSubqueryElem(NULL, Y, &Z);} +sub(A) ::= sub(X) COMMA LP union(Y) RP ids(Z).{A = addSubqueryElem(X, Y, &Z);} %type tablelist {SRelationInfo*} %destructor tablelist {destroyRelationInfo($$);} diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index dc32e5ee28..6379b7bbe7 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -19,6 +19,7 @@ #include "texpr.h" #include "ttype.h" #include "tsdb.h" +#include "tglobal.h" #include "qAggMain.h" #include "qFill.h" @@ -167,7 +168,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI int16_t *bytes, int32_t *interBytes, int16_t extLength, bool isSuperTable, SUdfInfo* pUdfInfo) { if (!isValidDataType(dataType)) { qError("Illegal data type %d or data type length %d", dataType, dataBytes); - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } @@ -382,7 +383,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI *interBytes = (*bytes); } else { - return TSDB_CODE_TSC_INVALID_SQL; + return TSDB_CODE_TSC_INVALID_OPERATION; } return TSDB_CODE_SUCCESS; @@ -3725,7 +3726,7 @@ char *getArithColumnData(void *param, const char* name, int32_t colId) { } } - assert(index >= 0 /*&& colId >= 0*/); + assert(index >= 0); return pSupport->data[index] + pSupport->offset * pSupport->colList[index].bytes; } @@ -4848,51 +4849,81 @@ void blockInfo_func(SQLFunctionCtx* pCtx) { pResInfo->hasResult = DATA_SET_FLAG; } -static void mergeTableBlockDist(STableBlockDist* pDist, const STableBlockDist* pSrc) { +static void mergeTableBlockDist(SResultRowCellInfo* pResInfo, const STableBlockDist* pSrc) { + STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); assert(pDist != NULL && pSrc != NULL); + pDist->numOfTables += pSrc->numOfTables; pDist->numOfRowsInMemTable += pSrc->numOfRowsInMemTable; pDist->numOfFiles += pSrc->numOfFiles; pDist->totalSize += pSrc->totalSize; + pDist->totalRows += pSrc->totalRows; - if (pDist->dataBlockInfos == NULL) { - pDist->dataBlockInfos = taosArrayInit(4, sizeof(SFileBlockInfo)); + if (pResInfo->hasResult == DATA_SET_FLAG) { + pDist->maxRows = MAX(pDist->maxRows, pSrc->maxRows); + pDist->minRows = MIN(pDist->minRows, pSrc->minRows); + } else { + pDist->maxRows = pSrc->maxRows; + pDist->minRows = pSrc->minRows; + + int32_t numSteps = tsMaxRowsInFileBlock/TSDB_BLOCK_DIST_STEP_ROWS; + pDist->dataBlockInfos = taosArrayInit(numSteps, sizeof(SFileBlockInfo)); + taosArraySetSize(pDist->dataBlockInfos, numSteps); } - taosArrayPushBatch(pDist->dataBlockInfos, pSrc->dataBlockInfos->pData, (int32_t) taosArrayGetSize(pSrc->dataBlockInfos)); + size_t steps = taosArrayGetSize(pDist->dataBlockInfos); + for (int32_t i = 0; i < steps; ++i) { + int32_t srcNumBlocks = ((SFileBlockInfo*)taosArrayGet(pSrc->dataBlockInfos, i))->numBlocksOfStep; + SFileBlockInfo* blockInfo = (SFileBlockInfo*)taosArrayGet(pDist->dataBlockInfos, i); + blockInfo->numBlocksOfStep += srcNumBlocks; + } } void block_func_merge(SQLFunctionCtx* pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); STableBlockDist info = {0}; - int32_t len = *(int32_t*) pCtx->pInput; blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info); - mergeTableBlockDist(pDist, &info); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + mergeTableBlockDist(pResInfo, &info); + + pResInfo->numOfRes = 1; + pResInfo->hasResult = DATA_SET_FLAG; } -static int32_t doGetPercentile(const SArray* pArray, double rate) { - int32_t len = (int32_t)taosArrayGetSize(pArray); - if (len <= 0) { - return 0; +void getPercentiles(STableBlockDist *pTableBlockDist, int64_t totalBlocks, int32_t numOfPercents, + double* percents, int32_t* percentiles) { + if (totalBlocks == 0) { + for (int32_t i = 0; i < numOfPercents; ++i) { + percentiles[i] = 0; + } + return; } - assert(rate >= 0 && rate <= 1.0); - int idx = (int32_t)((len - 1) * rate); + SArray *blocksInfos = pTableBlockDist->dataBlockInfos; + size_t numSteps = taosArrayGetSize(blocksInfos); + size_t cumulativeBlocks = 0; - return ((SFileBlockInfo *)(taosArrayGet(pArray, idx)))->numOfRows; -} + int percentIndex = 0; + for (int32_t indexStep = 0; indexStep < numSteps; ++indexStep) { + int32_t numStepBlocks = ((SFileBlockInfo *)taosArrayGet(blocksInfos, indexStep))->numBlocksOfStep; + if (numStepBlocks == 0) continue; + cumulativeBlocks += numStepBlocks; -static int compareBlockInfo(const void *pLeft, const void *pRight) { - int32_t left = ((SFileBlockInfo *)pLeft)->numOfRows; - int32_t right = ((SFileBlockInfo *)pRight)->numOfRows; + while (percentIndex < numOfPercents) { + double blockRank = totalBlocks * percents[percentIndex]; + if (blockRank <= cumulativeBlocks) { + percentiles[percentIndex] = indexStep; + ++percentIndex; + } else { + break; + } + } + } - if (left > right) return 1; - if (left < right) return -1; - return 0; + for (int32_t i = 0; i < numOfPercents; ++i) { + percentiles[i] = (percentiles[i]+1) * TSDB_BLOCK_DIST_STEP_ROWS - TSDB_BLOCK_DIST_STEP_ROWS/2; + } } void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) { @@ -4900,40 +4931,41 @@ void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) { return; } - int64_t min = INT64_MAX, max = INT64_MIN, avg = 0; - SArray* blockInfos= pTableBlockDist->dataBlockInfos; - int64_t totalRows = 0, totalBlocks = taosArrayGetSize(blockInfos); + SArray* blockInfos = pTableBlockDist->dataBlockInfos; + uint64_t totalRows = pTableBlockDist->totalRows; + size_t numSteps = taosArrayGetSize(blockInfos); + int64_t totalBlocks = 0; + int64_t min = -1, max = -1, avg = 0; - for (size_t i = 0; i < taosArrayGetSize(blockInfos); i++) { + for (int32_t i = 0; i < numSteps; i++) { SFileBlockInfo *blockInfo = taosArrayGet(blockInfos, i); - int64_t rows = blockInfo->numOfRows; - - min = MIN(min, rows); - max = MAX(max, rows); - totalRows += rows; + int64_t blocks = blockInfo->numBlocksOfStep; + totalBlocks += blocks; } avg = totalBlocks > 0 ? (int64_t)(totalRows/totalBlocks) : 0; - taosArraySort(blockInfos, compareBlockInfo); + min = totalBlocks > 0 ? pTableBlockDist->minRows : 0; + max = totalBlocks > 0 ? pTableBlockDist->maxRows : 0; + + double percents[] = {0.05, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90, 0.95, 0.99}; + int32_t percentiles[] = {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}; + assert(sizeof(percents)/sizeof(double) == sizeof(percentiles)/sizeof(int32_t)); + getPercentiles(pTableBlockDist, totalBlocks, sizeof(percents)/sizeof(double), percents, percentiles); uint64_t totalLen = pTableBlockDist->totalSize; int32_t rowSize = pTableBlockDist->rowSize; - + double compRatio = (totalRows>0) ? ((double)(totalLen)/(rowSize*totalRows)) : 1; int sz = sprintf(result + VARSTR_HEADER_SIZE, "summary: \n\t " "5th=[%d], 10th=[%d], 20th=[%d], 30th=[%d], 40th=[%d], 50th=[%d]\n\t " "60th=[%d], 70th=[%d], 80th=[%d], 90th=[%d], 95th=[%d], 99th=[%d]\n\t " "Min=[%"PRId64"(Rows)] Max=[%"PRId64"(Rows)] Avg=[%"PRId64"(Rows)] Stddev=[%.2f] \n\t " - "Rows=[%"PRId64"], Blocks=[%"PRId64"], Size=[%.3f(Kb)] Comp=[%.2f%%]\n\t " + "Rows=[%"PRIu64"], Blocks=[%"PRId64"], Size=[%.3f(Kb)] Comp=[%.2f]\n\t " "RowsInMem=[%d] \n\t SeekHeaderTime=[%d(us)]", - doGetPercentile(blockInfos, 0.05), doGetPercentile(blockInfos, 0.10), - doGetPercentile(blockInfos, 0.20), doGetPercentile(blockInfos, 0.30), - doGetPercentile(blockInfos, 0.40), doGetPercentile(blockInfos, 0.50), - doGetPercentile(blockInfos, 0.60), doGetPercentile(blockInfos, 0.70), - doGetPercentile(blockInfos, 0.80), doGetPercentile(blockInfos, 0.90), - doGetPercentile(blockInfos, 0.95), doGetPercentile(blockInfos, 0.99), + percentiles[0], percentiles[1], percentiles[2], percentiles[3], percentiles[4], percentiles[5], + percentiles[6], percentiles[7], percentiles[8], percentiles[9], percentiles[10], percentiles[11], min, max, avg, 0.0, - totalRows, totalBlocks, totalLen/1024.0, (double)(totalLen*100.0)/(rowSize*totalRows), + totalRows, totalBlocks, totalLen/1024.0, compRatio, pTableBlockDist->numOfRowsInMemTable, pTableBlockDist->firstSeekTimeUs); varDataSetLen(result, sz); UNUSED(sz); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 79c80557bb..a9302322de 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -34,6 +34,8 @@ #define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN) #define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) +#define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey)) + #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) #define SDATA_BLOCK_INITIALIZER (SDataBlockInfo) {{0}, 0} @@ -170,6 +172,8 @@ static void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, S static void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo); static bool hasMainOutput(SQueryAttr *pQueryAttr); +static SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols); + static int32_t setTimestampListJoinInfo(SQueryRuntimeEnv* pRuntimeEnv, tVariant* pTag, STableQueryInfo *pTableQueryInfo); static void releaseQueryBuf(size_t numOfTables); static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order); @@ -177,8 +181,6 @@ static STsdbQueryCond createTsdbQueryCond(SQueryAttr* pQueryAttr, STimeWindow* w static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo); static void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream); -static int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, - SSingleColumnFilterInfo** pFilterInfo, uint64_t qId); static void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols); static int32_t getNumOfScanTimes(SQueryAttr* pQueryAttr); @@ -192,7 +194,7 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator); static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock); -static int32_t getGroupbyColumnIndex(SSqlGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock); +static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock); static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SGroupbyOperatorInfo *pInfo, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex); static void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size); @@ -1013,7 +1015,7 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, setArithParams((SArithmeticSupport*)pCtx[i].param[1].pz, &pOperator->pExpr[i], pBlock); } else { SColIndex* pCol = &pOperator->pExpr[i].base.colInfo; - if (TSDB_COL_IS_NORMAL_COL(pCol->flag) || (pCol->colId == TSDB_BLOCK_DIST_COLUMN_INDEX) || + if (TSDB_COL_IS_NORMAL_COL(pCol->flag) || (pCtx[i].functionId == TSDB_FUNC_BLKINFO) || (TSDB_COL_IS_TAG(pCol->flag) && pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)) { SColIndex* pColIndex = &pOperator->pExpr[i].base.colInfo; SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pColIndex->colIndex); @@ -1524,7 +1526,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SGroupbyOp return TSDB_CODE_SUCCESS; } -static int32_t getGroupbyColumnIndex(SSqlGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock) { +static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock) { for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) { SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, k); if (TSDB_COL_IS_TAG(pColIndex->flag)) { @@ -1814,38 +1816,40 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf case OP_MultiTableTimeInterval: { pRuntimeEnv->proot = createMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream->info, pRuntimeEnv->proot); + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); break; } case OP_TimeWindow: { pRuntimeEnv->proot = createTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream->info, pRuntimeEnv->proot); + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); break; } case OP_Groupby: { pRuntimeEnv->proot = createGroupbyOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream->info, pRuntimeEnv->proot); + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); break; } case OP_SessionWindow: { pRuntimeEnv->proot = createSWindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream->info, pRuntimeEnv->proot); + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); break; } case OP_MultiTableAggregate: { pRuntimeEnv->proot = createMultiTableAggOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream->info, pRuntimeEnv->proot); + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); break; } case OP_Aggregate: { pRuntimeEnv->proot = createAggregateOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - if (pRuntimeEnv->proot->upstream->operatorType != OP_DummyInput) { - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream->info, pRuntimeEnv->proot); + + int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; + if (opType != OP_DummyInput && opType != OP_Join) { + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); } break; } @@ -1854,7 +1858,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf SOperatorInfo* prev = pRuntimeEnv->proot; if (i == 0) { pRuntimeEnv->proot = createArithOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - if (pRuntimeEnv->proot != NULL && pRuntimeEnv->proot->operatorType != OP_DummyInput) { // TODO refactor + if (pRuntimeEnv->proot != NULL && prev->operatorType != OP_DummyInput && prev->operatorType != OP_Join) { // TODO refactor setTableScanFilterOperatorInfo(prev->info, pRuntimeEnv->proot); } } else { @@ -1871,12 +1875,25 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf } case OP_Filter: { // todo refactor - assert(pQueryAttr->havingNum > 0); - if (pQueryAttr->stableQuery) { - pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr3, pQueryAttr->numOfExpr3); - } else { - pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - } + int32_t numOfFilterCols = 0; +// if (pQueryAttr->numOfFilterCols > 0) { +// pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, +// pQueryAttr->numOfOutput, pQueryAttr->tableCols, pQueryAttr->numOfFilterCols); +// } else { + if (pQueryAttr->stableQuery) { + SColumnInfo* pColInfo = + extractColumnFilterInfo(pQueryAttr->pExpr3, pQueryAttr->numOfExpr3, &numOfFilterCols); + pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr3, + pQueryAttr->numOfExpr3, pColInfo, numOfFilterCols); + freeColumnInfo(pColInfo, pQueryAttr->numOfExpr3); + } else { + SColumnInfo* pColInfo = + extractColumnFilterInfo(pQueryAttr->pExpr1, pQueryAttr->numOfOutput, &numOfFilterCols); + pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, + pQueryAttr->numOfOutput, pColInfo, numOfFilterCols); + freeColumnInfo(pColInfo, pQueryAttr->numOfOutput); + } +// } break; } @@ -2085,6 +2102,37 @@ static bool isFirstLastRowQuery(SQueryAttr *pQueryAttr) { return false; } +static bool isCachedLastQuery(SQueryAttr *pQueryAttr) { + for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { + int32_t functionID = pQueryAttr->pExpr1[i].base.functionId; + if (functionID == TSDB_FUNC_LAST || functionID == TSDB_FUNC_LAST_DST) { + continue; + } + + return false; + } + + if (pQueryAttr->order.order != TSDB_ORDER_DESC || !TSWINDOW_IS_EQUAL(pQueryAttr->window, TSWINDOW_DESC_INITIALIZER)) { + return false; + } + + if (pQueryAttr->groupbyColumn) { + return false; + } + + if (pQueryAttr->interval.interval > 0) { + return false; + } + + if (pQueryAttr->numOfFilterCols > 0 || pQueryAttr->havingNum > 0) { + return false; + } + + return true; +} + + + /** * The following 4 kinds of query are treated as the tags query * tagprj, tid_tag query, count(tbname), 'abc' (user defined constant value column) query @@ -2190,7 +2238,7 @@ static int32_t updateBlockLoadStatus(SQueryAttr *pQuery, int32_t status) { if (hasFirstLastFunc && status == BLK_DATA_NO_NEEDED) { if(!hasOtherFunc) { return BLK_DATA_DISCARD; - } else{ + } else { return BLK_DATA_ALL_NEEDED; } } @@ -2465,6 +2513,105 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key, bool asc return TS_JOIN_TS_EQUAL; } +bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p) { + bool all = true; + + for (int32_t i = 0; i < numOfRows; ++i) { + bool qualified = false; + + for (int32_t k = 0; k < numOfFilterCols; ++k) { + char* pElem = (char*)pFilterInfo[k].pData + pFilterInfo[k].info.bytes * i; + + qualified = false; + for (int32_t j = 0; j < pFilterInfo[k].numOfFilters; ++j) { + SColumnFilterElem* pFilterElem = &pFilterInfo[k].pFilters[j]; + + bool isnull = isNull(pElem, pFilterInfo[k].info.type); + if (isnull) { + if (pFilterElem->fp == isNullOperator) { + qualified = true; + break; + } else { + continue; + } + } else { + if (pFilterElem->fp == notNullOperator) { + qualified = true; + break; + } else if (pFilterElem->fp == isNullOperator) { + continue; + } + } + + if (pFilterElem->fp(pFilterElem, pElem, pElem, pFilterInfo[k].info.type)) { + qualified = true; + break; + } + } + + if (!qualified) { + break; + } + } + + p[i] = qualified ? 1 : 0; + if (!qualified) { + all = false; + } + } + + return all; +} + +void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p) { + int32_t len = 0; + int32_t start = 0; + for (int32_t j = 0; j < numOfRows; ++j) { + if (p[j] == 1) { + len++; + } else { + if (len > 0) { + int32_t cstart = j - len; + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pColumnInfoData = taosArrayGet(pBlock->pDataBlock, i); + + int16_t bytes = pColumnInfoData->info.bytes; + memmove(((char*)pColumnInfoData->pData) + start * bytes, pColumnInfoData->pData + cstart * bytes, + len * bytes); + } + + start += len; + len = 0; + } + } + } + + if (len > 0) { + int32_t cstart = numOfRows - len; + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pColumnInfoData = taosArrayGet(pBlock->pDataBlock, i); + + int16_t bytes = pColumnInfoData->info.bytes; + memmove(pColumnInfoData->pData + start * bytes, pColumnInfoData->pData + cstart * bytes, len * bytes); + } + + start += len; + len = 0; + } + + pBlock->info.rows = start; + pBlock->pBlockStatis = NULL; // clean the block statistics info + + if (start > 0) { + SColumnInfoData* pColumnInfoData = taosArrayGet(pBlock->pDataBlock, 0); + if (pColumnInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP && + pColumnInfoData->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + pBlock->info.window.skey = *(int64_t*)pColumnInfoData->pData; + pBlock->info.window.ekey = *(int64_t*)(pColumnInfoData->pData + TSDB_KEYSIZE * (start - 1)); + } + } +} + void filterRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock, bool ascQuery) { int32_t numOfRows = pBlock->info.rows; @@ -2497,97 +2644,11 @@ void filterRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInf // save the cursor status pRuntimeEnv->current->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); } else { - for (int32_t i = 0; i < numOfRows; ++i) { - bool qualified = false; - - for (int32_t k = 0; k < numOfFilterCols; ++k) { - char* pElem = (char*)pFilterInfo[k].pData + pFilterInfo[k].info.bytes * i; - - qualified = false; - for (int32_t j = 0; j < pFilterInfo[k].numOfFilters; ++j) { - SColumnFilterElem* pFilterElem = &pFilterInfo[k].pFilters[j]; - - bool isnull = isNull(pElem, pFilterInfo[k].info.type); - if (isnull) { - if (pFilterElem->fp == isNullOperator) { - qualified = true; - break; - } else { - continue; - } - } else { - if (pFilterElem->fp == notNullOperator) { - qualified = true; - break; - } else if (pFilterElem->fp == isNullOperator) { - continue; - } - } - - if (pFilterElem->fp(pFilterElem, pElem, pElem, pFilterInfo[k].info.type)) { - qualified = true; - break; - } - } - - if (!qualified) { - break; - } - } - - p[i] = qualified ? 1 : 0; - if (!qualified) { - all = false; - } - } + all = doFilterDataBlock(pFilterInfo, numOfFilterCols, numOfRows, p); } if (!all) { - int32_t start = 0; - int32_t len = 0; - for (int32_t j = 0; j < numOfRows; ++j) { - if (p[j] == 1) { - len++; - } else { - if (len > 0) { - int32_t cstart = j - len; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData *pColumnInfoData = taosArrayGet(pBlock->pDataBlock, i); - - int16_t bytes = pColumnInfoData->info.bytes; - memmove(((char*)pColumnInfoData->pData) + start * bytes, pColumnInfoData->pData + cstart * bytes, len * bytes); - } - - start += len; - len = 0; - } - } - } - - if (len > 0) { - int32_t cstart = numOfRows - len; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData *pColumnInfoData = taosArrayGet(pBlock->pDataBlock, i); - - int16_t bytes = pColumnInfoData->info.bytes; - memmove(pColumnInfoData->pData + start * bytes, pColumnInfoData->pData + cstart * bytes, len * bytes); - } - - start += len; - len = 0; - } - - pBlock->info.rows = start; - pBlock->pBlockStatis = NULL; // clean the block statistics info - - if (start > 0) { - SColumnInfoData* pColumnInfoData = taosArrayGet(pBlock->pDataBlock, 0); - if (pColumnInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP && - pColumnInfoData->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - pBlock->info.window.skey = *(int64_t*)pColumnInfoData->pData; - pBlock->info.window.ekey = *(int64_t*)(pColumnInfoData->pData + TSDB_KEYSIZE * (start - 1)); - } - } + doCompactSDataBlock(pBlock, numOfRows, p); } tfree(p); @@ -2620,7 +2681,7 @@ static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSData return status; } -static void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock) { +void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock) { if (numOfFilterCols > 0 && pFilterInfo[0].pData != NULL) { return; } @@ -3854,7 +3915,7 @@ void queryCostStatis(SQInfo *pQInfo) { // // int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); // -// qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64, GET_QID(pRuntimeEnv), +// qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numBlocksOfStep:%d, numOfRes:%d, lastKey:%"PRId64, GET_QID(pRuntimeEnv), // pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey); //} @@ -4058,6 +4119,15 @@ void queryCostStatis(SQInfo *pQInfo) { // return true; //} +void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream) { + if (p->upstream == NULL) { + assert(p->numOfUpstream == 0); + } + + p->upstream = realloc(p->upstream, POINTER_BYTES * (p->numOfUpstream + 1)); + p->upstream[p->numOfUpstream++] = pUpstream; +} + static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo); static int32_t setupQueryHandle(void* tsdb, SQueryRuntimeEnv* pRuntimeEnv, int64_t qId, bool isSTableQuery) { @@ -4107,6 +4177,8 @@ static int32_t setupQueryHandle(void* tsdb, SQueryRuntimeEnv* pRuntimeEnv, int64 } } } + } else if (isCachedLastQuery(pQueryAttr)) { + pRuntimeEnv->pQueryHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } else if (pQueryAttr->pointInterpQuery) { pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } else { @@ -4287,12 +4359,12 @@ static void doCloseAllTimeWindow(SQueryRuntimeEnv* pRuntimeEnv) { } static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; + SOperatorInfo *pOperator = (SOperatorInfo*) param; - STableScanInfo* pTableScanInfo = pOperator->info; - SSDataBlock* pBlock = &pTableScanInfo->block; + STableScanInfo *pTableScanInfo = pOperator->info; + SSDataBlock *pBlock = &pTableScanInfo->block; SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; STableGroupInfo *pTableGroupInfo = &pOperator->pRuntimeEnv->tableqinfoGroupInfo; *newgroup = false; @@ -4352,7 +4424,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { } if (++pTableScanInfo->current >= pTableScanInfo->times) { - if (pTableScanInfo->reverseTimes <= 0) { + if (pTableScanInfo->reverseTimes <= 0 || isTsdbCacheLastRow(pTableScanInfo->pQueryHandle)) { return NULL; } else { break; @@ -4419,7 +4491,12 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { STableBlockDist tableBlockDist = {0}; tableBlockDist.numOfTables = (int32_t)pOperator->pRuntimeEnv->tableqinfoGroupInfo.numOfTables; - tableBlockDist.dataBlockInfos = taosArrayInit(512, sizeof(SFileBlockInfo)); + + int32_t numRowSteps = tsMaxRowsInFileBlock / TSDB_BLOCK_DIST_STEP_ROWS; + tableBlockDist.dataBlockInfos = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo)); + taosArraySetSize(tableBlockDist.dataBlockInfos, numRowSteps); + tableBlockDist.maxRows = INT_MIN; + tableBlockDist.minRows = INT_MAX; tsdbGetFileBlocksDistInfo(pTableScanInfo->pQueryHandle, &tableBlockDist); tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pQueryHandle); @@ -4501,7 +4578,7 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRu SColumnInfoData infoData = {{0}}; infoData.info.type = TSDB_DATA_TYPE_BINARY; infoData.info.bytes = 1024; - infoData.info.colId = TSDB_BLOCK_DIST_COLUMN_INDEX; + infoData.info.colId = 0; taosArrayPush(pInfo->block.pDataBlock, &infoData); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); @@ -4736,13 +4813,14 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->upstream = upstream; pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doGlobalAggregate; pOperator->cleanup = destroyGlobalAggOperatorInfo; + appendUpstream(pOperator, upstream); + return pOperator; } @@ -4807,7 +4885,7 @@ static SSDataBlock* doAggregate(void* param, bool* newgroup) { SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t order = pQueryAttr->order.order; - SOperatorInfo* upstream = pOperator->upstream; + SOperatorInfo* upstream = pOperator->upstream[0]; while(1) { SSDataBlock* pBlock = upstream->exec(upstream, newgroup); @@ -4862,7 +4940,7 @@ static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) { SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t order = pQueryAttr->order.order; - SOperatorInfo* upstream = pOperator->upstream; + SOperatorInfo* upstream = pOperator->upstream[0]; while(1) { SSDataBlock* pBlock = upstream->exec(upstream, newgroup); @@ -4946,7 +5024,7 @@ static SSDataBlock* doArithmeticOperation(void* param, bool* newgroup) { bool prevVal = *newgroup; // The upstream exec may change the value of the newgroup, so use a local variable instead. - SSDataBlock* pBlock = pOperator->upstream->exec(pOperator->upstream, newgroup); + SSDataBlock* pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); if (pBlock == NULL) { assert(*newgroup == false); @@ -4980,7 +5058,7 @@ static SSDataBlock* doArithmeticOperation(void* param, bool* newgroup) { } pRes->info.rows = getNumOfResult(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); - if (pRes->info.rows >= pRuntimeEnv->resultInfo.threshold) { + if (pRes->info.rows >= 1000/*pRuntimeEnv->resultInfo.threshold*/) { break; } } @@ -5000,7 +5078,7 @@ static SSDataBlock* doLimit(void* param, bool* newgroup) { SSDataBlock* pBlock = NULL; while (1) { - pBlock = pOperator->upstream->exec(pOperator->upstream, newgroup); + pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); if (pBlock == NULL) { setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); pOperator->status = OP_EXEC_DONE; @@ -5040,27 +5118,6 @@ static SSDataBlock* doLimit(void* param, bool* newgroup) { return pBlock; } - -bool doFilterData(SColumnInfoData* p, int32_t rid, SColumnFilterElem *filterElem, __filter_func_t fp) { - char* input = p->pData + p->info.bytes * rid; - bool isnull = isNull(input, p->info.type); - if (isnull) { - return (fp == isNullOperator) ? true : false; - } else { - if (fp == notNullOperator) { - return true; - } else if (fp == isNullOperator) { - return false; - } - } - - if (fp(filterElem, input, input, p->info.type)) { - return true; - } - - return false; -} - static SSDataBlock* doFilter(void* param, bool* newgroup) { SOperatorInfo *pOperator = (SOperatorInfo *)param; if (pOperator->status == OP_EXEC_DONE) { @@ -5071,7 +5128,7 @@ static SSDataBlock* doFilter(void* param, bool* newgroup) { SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; while (1) { - SSDataBlock *pBlock = pOperator->upstream->exec(pOperator->upstream, newgroup); + SSDataBlock *pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); if (pBlock == NULL) { break; } @@ -5113,7 +5170,7 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { int32_t order = pQueryAttr->order.order; STimeWindow win = pQueryAttr->window; - SOperatorInfo* upstream = pOperator->upstream; + SOperatorInfo* upstream = pOperator->upstream[0]; while(1) { SSDataBlock* pBlock = upstream->exec(upstream, newgroup); @@ -5166,7 +5223,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t order = pQueryAttr->order.order; - SOperatorInfo* upstream = pOperator->upstream; + SOperatorInfo* upstream = pOperator->upstream[0]; while(1) { SSDataBlock* pBlock = upstream->exec(upstream, newgroup); @@ -5221,7 +5278,7 @@ static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) { int32_t order = pQueryAttr->order.order; STimeWindow win = pQueryAttr->window; - SOperatorInfo* upstream = pOperator->upstream; + SOperatorInfo* upstream = pOperator->upstream[0]; while(1) { SSDataBlock* pBlock = upstream->exec(upstream, newgroup); @@ -5272,7 +5329,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { return pInfo->binfo.pRes; } - SOperatorInfo* upstream = pOperator->upstream; + SOperatorInfo* upstream = pOperator->upstream[0]; while(1) { SSDataBlock* pBlock = upstream->exec(upstream, newgroup); @@ -5341,7 +5398,7 @@ static SSDataBlock* doFill(void* param, bool* newgroup) { } while(1) { - SSDataBlock* pBlock = pOperator->upstream->exec(pOperator->upstream, newgroup); + SSDataBlock* pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); if (*newgroup) { assert(pBlock != NULL); } @@ -5417,7 +5474,15 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { pOperator->cleanup(pOperator->info, pOperator->numOfOutput); } - destroyOperatorInfo(pOperator->upstream); + if (pOperator->upstream != NULL) { + for(int32_t i = 0; i < pOperator->numOfUpstream; ++i) { + destroyOperatorInfo(pOperator->upstream[i]); + } + + tfree(pOperator->upstream); + pOperator->numOfUpstream = 0; + } + tfree(pOperator->info); tfree(pOperator); } @@ -5442,13 +5507,14 @@ SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOpera pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->upstream = upstream; pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doAggregate; pOperator->cleanup = destroyBasicOperatorInfo; + appendUpstream(pOperator, upstream); + return pOperator; } @@ -5515,13 +5581,13 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SO pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->upstream = upstream; pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doSTableAggregate; pOperator->cleanup = destroyBasicOperatorInfo; + appendUpstream(pOperator, upstream); return pOperator; } @@ -5545,63 +5611,62 @@ SOperatorInfo* createArithOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorI pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->upstream = upstream; pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doArithmeticOperation; pOperator->cleanup = destroyArithOperatorInfo; + appendUpstream(pOperator, upstream); return pOperator; } +SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols) { + SColumnInfo* pCols = calloc(numOfOutput, sizeof(SColumnInfo)); + + int32_t numOfFilter = 0; + for(int32_t i = 0; i < numOfOutput; ++i) { + if (pExpr[i].base.flist.numOfFilters > 0) { + numOfFilter += 1; + } + + pCols[i].type = pExpr[i].base.resType; + pCols[i].bytes = pExpr[i].base.resBytes; + pCols[i].colId = pExpr[i].base.resColId; + + pCols[i].flist.numOfFilters = pExpr[i].base.flist.numOfFilters; + pCols[i].flist.filterInfo = calloc(pCols[i].flist.numOfFilters, sizeof(SColumnFilterInfo)); + memcpy(pCols[i].flist.filterInfo, pExpr[i].base.flist.filterInfo, pCols[i].flist.numOfFilters * sizeof(SColumnFilterInfo)); + } + + assert(numOfFilter > 0); + + *numOfFilterCols = numOfFilter; + return pCols; +} + SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, - int32_t numOfOutput) { + int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter) { SFilterOperatorInfo* pInfo = calloc(1, sizeof(SFilterOperatorInfo)); - { - SColumnInfo* pCols = calloc(numOfOutput, sizeof(SColumnInfo)); - - int32_t numOfFilter = 0; - for(int32_t i = 0; i < numOfOutput; ++i) { - if (pExpr[i].base.flist.numOfFilters > 0) { - numOfFilter += 1; - } - - pCols[i].type = pExpr[i].base.resType; - pCols[i].bytes = pExpr[i].base.resBytes; - pCols[i].colId = pExpr[i].base.resColId; - - pCols[i].flist.numOfFilters = pExpr[i].base.flist.numOfFilters; - pCols[i].flist.filterInfo = calloc(pCols[i].flist.numOfFilters, sizeof(SColumnFilterInfo)); - memcpy(pCols[i].flist.filterInfo, pExpr[i].base.flist.filterInfo, pCols[i].flist.numOfFilters * sizeof(SColumnFilterInfo)); - } - - assert(numOfFilter > 0); - doCreateFilterInfo(pCols, numOfOutput, numOfFilter, &pInfo->pFilterInfo, 0); - pInfo->numOfFilterCols = numOfFilter; - - for(int32_t i = 0; i < numOfOutput; ++i) { - tfree(pCols[i].flist.filterInfo); - } - - tfree(pCols); - } + assert(numOfFilter > 0 && pCols != NULL); + doCreateFilterInfo(pCols, numOfOutput, numOfFilter, &pInfo->pFilterInfo, 0); + pInfo->numOfFilterCols = numOfFilter; SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "ConditionOperator"; + pOperator->name = "FilterOperator"; pOperator->operatorType = OP_Filter; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->numOfOutput = numOfOutput; pOperator->pExpr = pExpr; - pOperator->upstream = upstream; pOperator->exec = doFilter; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->cleanup = destroyConditionOperatorInfo; + appendUpstream(pOperator, upstream); return pOperator; } @@ -5616,10 +5681,10 @@ SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorI pOperator->operatorType = OP_Limit; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; - pOperator->upstream = upstream; pOperator->exec = doLimit; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; + appendUpstream(pOperator, upstream); return pOperator; } @@ -5637,7 +5702,6 @@ SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOp pOperator->operatorType = OP_TimeWindow; pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; - pOperator->upstream = upstream; pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; @@ -5645,6 +5709,7 @@ SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOp pOperator->exec = doIntervalAgg; pOperator->cleanup = destroyBasicOperatorInfo; + appendUpstream(pOperator, upstream); return pOperator; } @@ -5662,7 +5727,6 @@ SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato pOperator->operatorType = OP_SessionWindow; pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; - pOperator->upstream = upstream; pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; @@ -5670,6 +5734,7 @@ SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato pOperator->exec = doSessionWindowAgg; pOperator->cleanup = destroyBasicOperatorInfo; + appendUpstream(pOperator, upstream); return pOperator; } @@ -5685,7 +5750,6 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRunti pOperator->operatorType = OP_MultiTableTimeInterval; pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; - pOperator->upstream = upstream; pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; @@ -5694,6 +5758,7 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRunti pOperator->exec = doSTableIntervalAgg; pOperator->cleanup = destroyBasicOperatorInfo; + appendUpstream(pOperator, upstream); return pOperator; } @@ -5710,7 +5775,6 @@ SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->operatorType = OP_Groupby; - pOperator->upstream = upstream; pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; @@ -5718,6 +5782,7 @@ SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato pOperator->exec = hashGroupbyAggregate; pOperator->cleanup = destroyGroupbyOperatorInfo; + appendUpstream(pOperator, upstream); return pOperator; } @@ -5747,8 +5812,6 @@ SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorIn pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->operatorType = OP_Fill; - - pOperator->upstream = upstream; pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; @@ -5757,6 +5820,7 @@ SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorIn pOperator->exec = doFill; pOperator->cleanup = destroySFillOperatorInfo; + appendUpstream(pOperator, upstream); return pOperator; } @@ -5795,11 +5859,12 @@ SOperatorInfo* createSLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperator pOperator->operatorType = OP_SLimit; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; - pOperator->upstream = upstream; pOperator->exec = doSLimit; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->cleanup = destroySlimitOperatorInfo; + + appendUpstream(pOperator, upstream); return pOperator; } @@ -5966,7 +6031,7 @@ static SSDataBlock* hashDistinct(void* param, bool* newgroup) { pRes->info.rows = 0; SSDataBlock* pBlock = NULL; while(1) { - pBlock = pOperator->upstream->exec(pOperator->upstream, newgroup); + pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); if (pBlock == NULL) { setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); pOperator->status = OP_EXEC_DONE; @@ -6027,12 +6092,13 @@ SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperat pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->operatorType = OP_Distinct; - pOperator->upstream = upstream; pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = hashDistinct; pOperator->cleanup = destroyDistinctOperatorInfo; + + appendUpstream(pOperator, upstream); return pOperator; } @@ -6042,10 +6108,7 @@ static int32_t getColumnIndexInSource(SQueriedTableInfo *pTableInfo, SSqlExpr *p if (TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { if (pExpr->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { return TSDB_TBNAME_COLUMN_INDEX; - } else if (pExpr->colInfo.colId == TSDB_BLOCK_DIST_COLUMN_INDEX) { - return TSDB_BLOCK_DIST_COLUMN_INDEX; } - while(j < pTableInfo->numOfTags) { if (pExpr->colInfo.colId == pTagCols[j].colId) { @@ -6730,7 +6793,7 @@ int32_t initUdfInfo(SUdfInfo* pUdfInfo) { return TSDB_CODE_SUCCESS; } -// TODO tag length should be passed from client +// TODO tag length should be passed from client, refactor int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo, SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, SUdfInfo* pUdfInfo) { *pExprInfo = NULL; @@ -6771,14 +6834,14 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp type = TSDB_DATA_TYPE_DOUBLE; bytes = tDataTypes[type].bytes; + } else if (pExprs[i].base.functionId == TSDB_FUNC_BLKINFO) { + SSchema s = {.type=TSDB_DATA_TYPE_BINARY, .bytes=TSDB_MAX_BINARY_LEN}; + type = s.type; + bytes = s.bytes; } else if (pExprs[i].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX && pExprs[i].base.functionId == TSDB_FUNC_TAGPRJ) { // parse the normal column SSchema* s = tGetTbnameColumnSchema(); type = s->type; bytes = s->bytes; - } else if (pExprs[i].base.colInfo.colId == TSDB_BLOCK_DIST_COLUMN_INDEX) { - SSchema s = tGetBlockDistColumnSchema(); - type = s.type; - bytes = s.bytes; } else if (pExprs[i].base.colInfo.colId <= TSDB_UD_COLUMN_INDEX && pExprs[i].base.colInfo.colId > TSDB_RES_COL_ID) { // it is a user-defined constant value column assert(pExprs[i].base.functionId == TSDB_FUNC_PRJ); @@ -6791,7 +6854,7 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp } else { int32_t j = getColumnIndexInSource(pTableInfo, &pExprs[i].base, pTagCols); if (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag)) { - if (j < TSDB_BLOCK_DIST_COLUMN_INDEX || j >= pTableInfo->numOfTags) { + if (j < TSDB_TBNAME_COLUMN_INDEX || j >= pTableInfo->numOfTags) { return TSDB_CODE_QRY_INVALID_MSG; } } else { @@ -6906,13 +6969,13 @@ int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg* pQueryMsg, int32_t nu return TSDB_CODE_SUCCESS; } -SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code) { +SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code) { if (pQueryMsg->numOfGroupCols == 0) { return NULL; } // using group by tag columns - SSqlGroupbyExpr *pGroupbyExpr = (SSqlGroupbyExpr *)calloc(1, sizeof(SSqlGroupbyExpr)); + SGroupbyExpr *pGroupbyExpr = (SGroupbyExpr *)calloc(1, sizeof(SGroupbyExpr)); if (pGroupbyExpr == NULL) { *code = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; @@ -6930,8 +6993,7 @@ SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex * return pGroupbyExpr; } -static int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, - SSingleColumnFilterInfo** pFilterInfo, uint64_t qId) { +int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId) { *pFilterInfo = calloc(1, sizeof(SSingleColumnFilterInfo) * numOfFilterCols); if (pFilterInfo == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; @@ -6988,7 +7050,7 @@ void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFil return NULL; } -static int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId) { +int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId) { for (int32_t i = 0; i < pQueryAttr->numOfCols; ++i) { if (pQueryAttr->tableCols[i].flist.numOfFilters > 0) { pQueryAttr->numOfFilterCols++; @@ -7028,9 +7090,6 @@ static void doUpdateExprColumnIndex(SQueryAttr *pQueryAttr) { assert(f < pQueryAttr->numOfCols); } else if (pColIndex->colId <= TSDB_UD_COLUMN_INDEX) { // do nothing for user-defined constant value result columns - } else if (pColIndex->colId == TSDB_BLOCK_DIST_COLUMN_INDEX) { - pColIndex->colIndex = 0;// only one source column, so it must be 0; - assert(pQueryAttr->numOfOutput == 1); } else { int32_t f = 0; for (f = 0; f < pQueryAttr->numOfTags; ++f) { @@ -7040,7 +7099,7 @@ static void doUpdateExprColumnIndex(SQueryAttr *pQueryAttr) { } } - assert(f < pQueryAttr->numOfTags || pColIndex->colId == TSDB_TBNAME_COLUMN_INDEX || pColIndex->colId == TSDB_BLOCK_DIST_COLUMN_INDEX); + assert(f < pQueryAttr->numOfTags || pColIndex->colId == TSDB_TBNAME_COLUMN_INDEX); } } } @@ -7073,9 +7132,9 @@ FORCE_INLINE bool checkQIdEqual(void *qHandle, uint64_t qId) { return ((SQInfo *)qHandle)->qId == qId; } -SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SSqlGroupbyExpr* pGroupbyExpr, SExprInfo* pExprs, - SExprInfo* pSecExprs, STableGroupInfo* pTableGroupInfo, SColumnInfo* pTagCols, bool stableQuery, - char* sql, uint64_t *qId, int32_t vgId, SUdfInfo* pUdfInfo) { +SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, SExprInfo* pExprs, + SExprInfo* pSecExprs, STableGroupInfo* pTableGroupInfo, SColumnInfo* pTagCols, int32_t vgId, + char* sql, uint64_t *qId, SUdfInfo* pUdfInfo) { int16_t numOfCols = pQueryMsg->numOfCols; int16_t numOfOutput = pQueryMsg->numOfOutput; @@ -7247,7 +7306,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SSqlGroupbyExpr* pGroupbyExpr colIdCheck(pQueryAttr, pQInfo->qId); // todo refactor - pQInfo->query.queryBlockDist = (numOfOutput == 1 && pExprs[0].base.colInfo.colId == TSDB_BLOCK_DIST_COLUMN_INDEX); + pQInfo->query.queryBlockDist = (numOfOutput == 1 && pExprs[0].base.functionId == TSDB_FUNC_BLKINFO); qDebug("qmsg:%p QInfo:0x%" PRIx64 "-%p created", pQueryMsg, pQInfo->qId, pQInfo); return pQInfo; @@ -7391,7 +7450,7 @@ static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo) { pTableqinfoGroupInfo->numOfTables = 0; } -static void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr) { +void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr) { if (pExprInfo == NULL) { assert(numOfExpr == 0); return NULL; @@ -7415,6 +7474,20 @@ static void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr) { return NULL; } +void* freeColumnInfo(SColumnInfo* pColumnInfo, int32_t numOfCols) { + if (pColumnInfo != NULL) { + assert(numOfCols >= 0); + + for (int32_t i = 0; i < numOfCols; i++) { + freeColumnFilterInfo(pColumnInfo[i].flist.filterInfo, pColumnInfo[i].flist.numOfFilters); + } + + tfree(pColumnInfo); + } + + return NULL; +} + void freeQInfo(SQInfo *pQInfo) { if (!isValidQInfo(pQInfo)) { return; @@ -7599,13 +7672,7 @@ void freeQueryAttr(SQueryAttr* pQueryAttr) { tfree(pQueryAttr->tagColList); tfree(pQueryAttr->pFilterInfo); - if (pQueryAttr->tableCols != NULL) { - for (int32_t i = 0; i < pQueryAttr->numOfCols; i++) { - SColumnInfo* column = pQueryAttr->tableCols + i; - freeColumnFilterInfo(column->flist.filterInfo, column->flist.numOfFilters); - } - tfree(pQueryAttr->tableCols); - } + pQueryAttr->tableCols = freeColumnInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols); if (pQueryAttr->pGroupbyExpr != NULL) { taosArrayDestroy(pQueryAttr->pGroupbyExpr->columnInfo); diff --git a/src/query/src/qFill.c b/src/query/src/qFill.c index fa572029fc..7dd73c9fe4 100644 --- a/src/query/src/qFill.c +++ b/src/query/src/qFill.c @@ -363,10 +363,6 @@ SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int3 pFillInfo->rowSize = setTagColumnInfo(pFillInfo, pFillInfo->numOfCols, pFillInfo->alloc); assert(pFillInfo->rowSize > 0); - for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - pFillInfo->pData[i] = malloc(pFillInfo->pFillCol[i].col.bytes * pFillInfo->alloc); - } - return pFillInfo; } @@ -392,10 +388,6 @@ void* taosDestroyFillInfo(SFillInfo* pFillInfo) { tfree(pFillInfo->pTags[i].tagVal); } - for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - tfree(pFillInfo->pData[i]); - } - tfree(pFillInfo->pTags); tfree(pFillInfo->pData); @@ -417,17 +409,6 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) pFillInfo->index = 0; pFillInfo->numOfRows = numOfRows; - - // ensure the space - if (pFillInfo->alloc < numOfRows) { - for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - char* tmp = realloc(pFillInfo->pData[i], numOfRows*pFillInfo->pFillCol[i].col.bytes); - assert(tmp != NULL); // todo handle error - - memset(tmp, 0, numOfRows*pFillInfo->pFillCol[i].col.bytes); - pFillInfo->pData[i] = tmp; - } - } } void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput) { @@ -435,16 +416,7 @@ void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput) SFillColInfo* pCol = &pFillInfo->pFillCol[i]; SColumnInfoData* pColData = taosArrayGet(pInput->pDataBlock, i); -// pFillInfo->pData[i] = pColData->pData; - if (pInput->info.rows > pFillInfo->alloc) { - char* t = realloc(pFillInfo->pData[i], pColData->info.bytes * pInput->info.rows); - assert(t != NULL); - - pFillInfo->pData[i] = t; - pFillInfo->alloc = pInput->info.rows; - } - - memcpy(pFillInfo->pData[i], pColData->pData, pColData->info.bytes * pInput->info.rows); + pFillInfo->pData[i] = pColData->pData; if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->col.type)*/) { // copy the tag value to tag value buffer SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex]; @@ -454,31 +426,6 @@ void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput) } } -void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, const tFilePage* pInput) { - assert(pFillInfo->numOfRows == pInput->num); - - for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - - const char* data = pInput->data + pCol->col.offset * pInput->num; - if (pInput->num > pFillInfo->alloc) { - char* t = realloc(pFillInfo->pData[i], (size_t)(pCol->col.bytes * pInput->num)); - assert(t != NULL); - - pFillInfo->pData[i] = t; - pFillInfo->alloc = (int32_t)pInput->num; - } - - memcpy(pFillInfo->pData[i], data, (size_t)(pCol->col.bytes * pInput->num)); - - if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->col.type)*/) { // copy the tag value to tag value buffer - SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex]; - assert (pTag->col.colId == pCol->col.colId); - memcpy(pTag->tagVal, data, pCol->col.bytes); // TODO not memcpy?? - } - } -} - bool taosFillHasMoreResults(SFillInfo* pFillInfo) { int32_t remain = taosNumOfRemainRows(pFillInfo); if (remain > 0) { diff --git a/src/query/src/qHistogram.c b/src/query/src/qHistogram.c index ae25a75234..5fa35d0ee5 100644 --- a/src/query/src/qHistogram.c +++ b/src/query/src/qHistogram.c @@ -446,7 +446,7 @@ void tHistogramDestroy(SHistogramInfo** pHisto) { } void tHistogramPrint(SHistogramInfo* pHisto) { - printf("total entries: %d, elements: %d\n", pHisto->numOfEntries, pHisto->numOfElems); + printf("total entries: %d, elements: %"PRId64 "\n", pHisto->numOfEntries, pHisto->numOfElems); #if defined(USE_ARRAYLIST) for (int32_t i = 0; i < pHisto->numOfEntries; ++i) { printf("%d: (%f, %" PRId64 ")\n", i + 1, pHisto->elems[i].val, pHisto->elems[i].num); diff --git a/src/query/src/qPlan.c b/src/query/src/qPlan.c index dccdc5e1d3..9079d830c9 100644 --- a/src/query/src/qPlan.c +++ b/src/query/src/qPlan.c @@ -1,42 +1,521 @@ #include "os.h" -#include "tsclient.h" +#include "tschemautil.h" +#include "qPlan.h" +#include "qExecutor.h" #include "qUtil.h" #include "texpr.h" +#include "tscUtil.h" +#include "tsclient.h" -#define QNODE_PROJECT 1 -#define QNODE_FILTER 2 -#define QNODE_RELATION 3 -#define QNODE_AGGREGATE 4 -#define QNODE_GROUPBY 5 -#define QNODE_LIMIT 6 -#define QNODE_JOIN 7 -#define QNODE_DIST 8 -#define QNODE_SORT 9 -#define QNODE_UNIONALL 10 -#define QNODE_TIMEWINDOW 11 +#define QNODE_TAGSCAN 1 +#define QNODE_TABLESCAN 2 +#define QNODE_PROJECT 3 +#define QNODE_AGGREGATE 4 +#define QNODE_GROUPBY 5 +#define QNODE_LIMIT 6 +#define QNODE_JOIN 7 +#define QNODE_DISTINCT 8 +#define QNODE_SORT 9 +#define QNODE_UNIONALL 10 +#define QNODE_TIMEWINDOW 11 +#define QNODE_SESSIONWINDOW 12 +#define QNODE_FILL 13 -typedef struct SQueryNode { - int32_t type; // the type of logic node - char *name; // the name of logic node +typedef struct SFillEssInfo { + int32_t fillType; // fill type + int64_t *val; // fill value +} SFillEssInfo; - SSchema *pSchema; // the schema of the input SSDatablock - int32_t numOfCols; // number of input columns - SExprInfo *pExpr; // the query functions or sql aggregations - int32_t numOfOutput; // number of result columns, which is also the number of pExprs +typedef struct SJoinCond { + bool tagExists; // denote if tag condition exists or not + SColumn *tagCond[2]; + SColumn *colCond[2]; +} SJoinCond; - // previous operator to generated result for current node to process - // in case of join, multiple prev nodes exist. - struct SQueryNode* prevNode; - struct SQueryNode* nextNode; -} SQueryNode; +static SQueryNode* createQueryNode(int32_t type, const char* name, SQueryNode** prev, + int32_t numOfPrev, SExprInfo** pExpr, int32_t numOfOutput, SQueryTableInfo* pTableInfo, + void* pExtInfo) { + SQueryNode* pNode = calloc(1, sizeof(SQueryNode)); + + pNode->info.type = type; + pNode->info.name = strdup(name); + + if (pTableInfo->id.uid != 0) { // it is a true table + pNode->tableInfo.id = pTableInfo->id; + pNode->tableInfo.tableName = strdup(pTableInfo->tableName); + } + + pNode->numOfOutput = numOfOutput; + pNode->pExpr = calloc(numOfOutput, sizeof(SExprInfo)); + for(int32_t i = 0; i < numOfOutput; ++i) { + tscExprAssign(&pNode->pExpr[i], pExpr[i]); + } + + pNode->pPrevNodes = taosArrayInit(4, POINTER_BYTES); + for(int32_t i = 0; i < numOfPrev; ++i) { + taosArrayPush(pNode->pPrevNodes, &prev[i]); + } + + switch(type) { + case QNODE_TABLESCAN: { + STimeWindow* window = calloc(1, sizeof(STimeWindow)); + memcpy(window, pExtInfo, sizeof(STimeWindow)); + pNode->pExtInfo = window; + break; + } + + case QNODE_TIMEWINDOW: { + SInterval* pInterval = calloc(1, sizeof(SInterval)); + pNode->pExtInfo = pInterval; + memcpy(pInterval, pExtInfo, sizeof(SInterval)); + break; + } + + case QNODE_GROUPBY: { + SGroupbyExpr* p = (SGroupbyExpr*) pExtInfo; + SGroupbyExpr* pGroupbyExpr = calloc(1, sizeof(SGroupbyExpr)); + + pGroupbyExpr->tableIndex = p->tableIndex; + pGroupbyExpr->orderType = p->orderType; + pGroupbyExpr->orderIndex = p->orderIndex; + pGroupbyExpr->numOfGroupCols = p->numOfGroupCols; + pGroupbyExpr->columnInfo = taosArrayDup(p->columnInfo); + pNode->pExtInfo = pGroupbyExpr; + break; + } + + case QNODE_FILL: { // todo !! + pNode->pExtInfo = pExtInfo; + break; + } + + case QNODE_LIMIT: { + pNode->pExtInfo = calloc(1, sizeof(SLimitVal)); + memcpy(pNode->pExtInfo, pExtInfo, sizeof(SLimitVal)); + break; + } + } + return pNode; +} + +static SQueryNode* doAddTableColumnNode(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SQueryTableInfo* info, + SArray* pExprs, SArray* tableCols) { + if (pQueryInfo->onlyTagQuery) { + int32_t num = (int32_t) taosArrayGetSize(pExprs); + SQueryNode* pNode = createQueryNode(QNODE_TAGSCAN, "TableTagScan", NULL, 0, pExprs->pData, num, info, NULL); + + if (pQueryInfo->distinctTag) { + pNode = createQueryNode(QNODE_DISTINCT, "Distinct", &pNode, 1, pExprs->pData, num, info, NULL); + } + + return pNode; + } + + STimeWindow* window = &pQueryInfo->window; + SQueryNode* pNode = createQueryNode(QNODE_TABLESCAN, "TableScan", NULL, 0, NULL, 0, + info, window); + if (pQueryInfo->projectionQuery) { + int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprs); + pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExprs->pData, numOfOutput, info, NULL); + } else { + // table source column projection, generate the projection expr + int32_t numOfCols = (int32_t) taosArrayGetSize(tableCols); + SExprInfo** pExpr = calloc(numOfCols, POINTER_BYTES); + SSchema* pSchema = pTableMetaInfo->pTableMeta->schema; + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumn* pCol = taosArrayGetP(tableCols, i); + + SColumnIndex index = {.tableIndex = 0, .columnIndex = pCol->columnIndex}; + SExprInfo* p = tscExprCreate(pQueryInfo, TSDB_FUNC_PRJ, &index, pCol->info.type, pCol->info.bytes, + pCol->info.colId, 0, TSDB_COL_NORMAL); + strncpy(p->base.aliasName, pSchema[pCol->columnIndex].name, tListLen(p->base.aliasName)); + + pExpr[i] = p; + } + + pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExpr, numOfCols, info, NULL); + for (int32_t i = 0; i < numOfCols; ++i) { + destroyQueryFuncExpr(pExpr[i], 1); + } + tfree(pExpr); + } + + return pNode; +} + +static SQueryNode* doCreateQueryPlanForOneTableImpl(SQueryInfo* pQueryInfo, SQueryNode* pNode, SQueryTableInfo* info, + SArray* pExprs) { + // check for aggregation + if (pQueryInfo->interval.interval > 0) { + int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprs); + + pNode = createQueryNode(QNODE_TIMEWINDOW, "TimeWindowAgg", &pNode, 1, pExprs->pData, numOfOutput, info, + &pQueryInfo->interval); + } else if (pQueryInfo->groupbyColumn) { + int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprs); + pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, pExprs->pData, numOfOutput, info, + &pQueryInfo->groupbyExpr); + } else if (pQueryInfo->sessionWindow.gap > 0) { + pNode = createQueryNode(QNODE_SESSIONWINDOW, "SessionWindowAgg", &pNode, 1, NULL, 0, info, NULL); + } else if (pQueryInfo->simpleAgg) { + int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprs); + pNode = createQueryNode(QNODE_AGGREGATE, "Aggregate", &pNode, 1, pExprs->pData, numOfOutput, info, NULL); + } + + if (pQueryInfo->havingFieldNum > 0 || pQueryInfo->arithmeticOnAgg) { + int32_t numOfExpr = (int32_t) taosArrayGetSize(pQueryInfo->exprList1); + pNode = + createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, info, NULL); + } + + if (pQueryInfo->fillType != TSDB_FILL_NONE) { + SFillEssInfo* pInfo = calloc(1, sizeof(SFillEssInfo)); + pInfo->fillType = pQueryInfo->fillType; + pInfo->val = calloc(pNode->numOfOutput, sizeof(int64_t)); + memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfOutput); + + pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, NULL, 0, info, pInfo); + } + + + if (pQueryInfo->limit.limit != -1 || pQueryInfo->limit.offset != 0) { + pNode = createQueryNode(QNODE_LIMIT, "Limit", &pNode, 1, NULL, 0, info, &pQueryInfo->limit); + } + + return pNode; +} + +static SQueryNode* doCreateQueryPlanForOneTable(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs, + SArray* tableCols) { + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + + SQueryTableInfo info = {.tableName = strdup(name), .id = pTableMetaInfo->pTableMeta->id,}; + + // handle the only tag query + SQueryNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, pExprs, tableCols); + if (pQueryInfo->onlyTagQuery) { + tfree(info.tableName); + return pNode; + } + + SQueryNode* pNode1 = doCreateQueryPlanForOneTableImpl(pQueryInfo, pNode, &info, pExprs); + tfree(info.tableName); + return pNode1; +} + +SArray* createQueryPlanImpl(SQueryInfo* pQueryInfo) { + SArray* upstream = NULL; + + if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) { // subquery in the from clause + upstream = taosArrayInit(4, POINTER_BYTES); + + size_t size = taosArrayGetSize(pQueryInfo->pUpstream); + for(int32_t i = 0; i < size; ++i) { + SQueryInfo* pq = taosArrayGet(pQueryInfo->pUpstream, i); + SArray* p = createQueryPlanImpl(pq); + taosArrayAddBatch(upstream, p->pData, (int32_t) taosArrayGetSize(p)); + } + } + + if (pQueryInfo->numOfTables > 1) { // it is a join query + // 1. separate the select clause according to table + upstream = taosArrayInit(5, POINTER_BYTES); + + for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[i]; + uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; + + SArray* exprList = taosArrayInit(4, POINTER_BYTES); + if (tscExprCopy(exprList, pQueryInfo->exprList, uid, true) != 0) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + exit(-1); + } + + // 2. create the query execution node + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + SQueryTableInfo info = {.tableName = strdup(name), .id = pTableMetaInfo->pTableMeta->id,}; + + // 3. get the required table column list + SArray* tableColumnList = taosArrayInit(4, sizeof(SColumn)); + tscColumnListCopy(tableColumnList, pQueryInfo->colList, uid); + + // 4. add the projection query node + SQueryNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, exprList, tableColumnList); + taosArrayPush(upstream, &pNode); + } + + // 3. add the join node here + SQueryTableInfo info = {0}; + int32_t num = (int32_t) taosArrayGetSize(pQueryInfo->exprList); + SQueryNode* pNode = createQueryNode(QNODE_JOIN, "Join", upstream->pData, pQueryInfo->numOfTables, + pQueryInfo->exprList->pData, num, &info, NULL); + + // 4. add the aggregation or projection execution node + pNode = doCreateQueryPlanForOneTableImpl(pQueryInfo, pNode, &info, pQueryInfo->exprList); + upstream = taosArrayInit(5, POINTER_BYTES); + taosArrayPush(upstream, &pNode); + } else { // only one table, normal query process + STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; + SQueryNode* pNode = doCreateQueryPlanForOneTable(pQueryInfo, pTableMetaInfo, pQueryInfo->exprList, pQueryInfo->colList); + upstream = taosArrayInit(5, POINTER_BYTES); + taosArrayPush(upstream, &pNode); + } + + return upstream; +} -// TODO create the query plan SQueryNode* qCreateQueryPlan(SQueryInfo* pQueryInfo) { + SArray* upstream = createQueryPlanImpl(pQueryInfo); + assert(taosArrayGetSize(upstream) == 1); + + SQueryNode* p = taosArrayGetP(upstream, 0); + taosArrayDestroy(upstream); + + return p; +} + +static void doDestroyQueryNode(SQueryNode* pQueryNode) { + tfree(pQueryNode->pExtInfo); + tfree(pQueryNode->pSchema); + tfree(pQueryNode->info.name); + + tfree(pQueryNode->tableInfo.tableName); + + pQueryNode->pExpr = destroyQueryFuncExpr(pQueryNode->pExpr, pQueryNode->numOfOutput); + + if (pQueryNode->pPrevNodes != NULL) { + int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pPrevNodes); + for(int32_t i = 0; i < size; ++i) { + SQueryNode* p = taosArrayGetP(pQueryNode->pPrevNodes, i); + doDestroyQueryNode(p); + } + + taosArrayDestroy(pQueryNode->pPrevNodes); + } + + tfree(pQueryNode); +} + +void* qDestroyQueryPlan(SQueryNode* pQueryNode) { + if (pQueryNode == NULL) { + return NULL; + } + + doDestroyQueryNode(pQueryNode); return NULL; } -char* queryPlanToString() { - return NULL; +bool hasAliasName(SExprInfo* pExpr) { + assert(pExpr != NULL); + return strncmp(pExpr->base.token, pExpr->base.aliasName, tListLen(pExpr->base.aliasName)) != 0; +} + +static int32_t doPrintPlan(char* buf, SQueryNode* pQueryNode, int32_t level, int32_t totalLen) { + if (level > 0) { + sprintf(buf + totalLen, "%*c", level, ' '); + totalLen += level; + } + + int32_t len1 = sprintf(buf + totalLen, "%s(", pQueryNode->info.name); + int32_t len = len1 + totalLen; + + switch(pQueryNode->info.type) { + case QNODE_TABLESCAN: { + STimeWindow* win = (STimeWindow*)pQueryNode->pExtInfo; + len1 = sprintf(buf + len, "%s #0x%" PRIx64 ") time_range: %" PRId64 " - %" PRId64 "\n", + pQueryNode->tableInfo.tableName, pQueryNode->tableInfo.id.uid, win->skey, win->ekey); + len += len1; + break; + } + + case QNODE_PROJECT: { + len1 = sprintf(buf + len, "cols: "); + len += len1; + + for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { + SSqlExpr* p = &pQueryNode->pExpr[i].base; + len1 = sprintf(buf + len, "[%s #%d]", p->aliasName, p->resColId); + len += len1; + + if (i < pQueryNode->numOfOutput - 1) { + len1 = sprintf(buf + len, ", "); + len += len1; + } + } + + len1 = sprintf(buf + len, ")"); + len += len1; + + //todo print filter info + len1 = sprintf(buf + len, " filters:(nil)\n"); + len += len1; + break; + } + + case QNODE_AGGREGATE: { + for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { + SSqlExpr* pExpr = &pQueryNode->pExpr[i].base; + if (hasAliasName(&pQueryNode->pExpr[i])) { + len1 = sprintf(buf + len,"[%s #%s]", pExpr->token, pExpr->aliasName); + } else { + len1 = sprintf(buf + len,"[%s]", pExpr->token); + } + + len += len1; + if (i < pQueryNode->numOfOutput - 1) { + len1 = sprintf(buf + len, ", "); + len += len1; + } + } + + len1 = sprintf(buf + len, ")\n"); + len += len1; + break; + } + + case QNODE_TIMEWINDOW: { + for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { + SSqlExpr* pExpr = &pQueryNode->pExpr[i].base; + if (hasAliasName(&pQueryNode->pExpr[i])) { + len1 = sprintf(buf + len,"[%s #%s]", pExpr->token, pExpr->aliasName); + } else { + len1 = sprintf(buf + len,"[%s]", pExpr->token); + } + + len += len1; + if (i < pQueryNode->numOfOutput - 1) { + len1 = sprintf(buf + len,", "); + len += len1; + } + } + + len1 = sprintf(buf + len,") "); + len += len1; + + SInterval* pInterval = pQueryNode->pExtInfo; + len1 = sprintf(buf + len, "interval:%" PRId64 "(%c), sliding:%" PRId64 "(%c), offset:%" PRId64 "\n", + pInterval->interval, pInterval->intervalUnit, pInterval->sliding, pInterval->slidingUnit, + pInterval->offset); + len += len1; + + break; + } + + case QNODE_GROUPBY: { // todo hide the invisible column + for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { + SSqlExpr* pExpr = &pQueryNode->pExpr[i].base; + + if (hasAliasName(&pQueryNode->pExpr[i])) { + len1 = sprintf(buf + len,"[%s #%s]", pExpr->token, pExpr->aliasName); + } else { + len1 = sprintf(buf + len,"[%s]", pExpr->token); + } + + len += len1; + if (i < pQueryNode->numOfOutput - 1) { + len1 = sprintf(buf + len,", "); + len += len1; + } + } + + SGroupbyExpr* pGroupbyExpr = pQueryNode->pExtInfo; + SColIndex* pIndex = taosArrayGet(pGroupbyExpr->columnInfo, 0); + + len1 = sprintf(buf + len,") groupby_col: [%s #%d]\n", pIndex->name, pIndex->colId); + len += len1; + + break; + } + + case QNODE_FILL: { + SFillEssInfo* pEssInfo = pQueryNode->pExtInfo; + len1 = sprintf(buf + len,"%d", pEssInfo->fillType); + len += len1; + + if (pEssInfo->fillType == TSDB_FILL_SET_VALUE) { + len1 = sprintf(buf + len,", val:"); + len += len1; + + // todo get the correct fill data type + for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { + len1 = sprintf(buf + len,"%"PRId64, pEssInfo->val[i]); + len += len1; + + if (i < pQueryNode->numOfOutput - 1) { + len1 = sprintf(buf + len,", "); + len += len1; + } + } + } + + len1 = sprintf(buf + len,")\n"); + len += len1; + break; + } + + case QNODE_LIMIT: { + SLimitVal* pVal = pQueryNode->pExtInfo; + len1 = sprintf(buf + len,"limit: %"PRId64", offset: %"PRId64")\n", pVal->limit, pVal->offset); + len += len1; + break; + } + + case QNODE_DISTINCT: + case QNODE_TAGSCAN: { + len1 = sprintf(buf + len,"cols: "); + len += len1; + + for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { + SSqlExpr* p = &pQueryNode->pExpr[i].base; + len1 = sprintf(buf + len,"[%s #%d]", p->aliasName, p->resColId); + len += len1; + + if (i < pQueryNode->numOfOutput - 1) { + len1 = sprintf(buf + len,", "); + len += len1; + } + } + + len1 = sprintf(buf + len,")\n"); + len += len1; + + break; + } + + case QNODE_JOIN: { + // print join condition + len1 = sprintf(buf + len, ")\n"); + len += len1; + break; + } + } + + return len; +} + +int32_t queryPlanToStringImpl(char* buf, SQueryNode* pQueryNode, int32_t level, int32_t totalLen) { + int32_t len = doPrintPlan(buf, pQueryNode, level, totalLen); + + for(int32_t i = 0; i < taosArrayGetSize(pQueryNode->pPrevNodes); ++i) { + SQueryNode* p1 = taosArrayGetP(pQueryNode->pPrevNodes, i); + int32_t len1 = queryPlanToStringImpl(buf, p1, level + 1, len); + len = len1; + } + + return len; +} + +char* queryPlanToString(SQueryNode* pQueryNode) { + assert(pQueryNode); + + char* buf = calloc(1, 4096); + + int32_t len = sprintf(buf, "===== logic plan =====\n"); + queryPlanToStringImpl(buf, pQueryNode, 0, len); + return buf; } SQueryNode* queryPlanFromString() { @@ -136,14 +615,13 @@ SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) { taosArrayPush(plan, &op); } } else { // diff/add/multiply/subtract/division - op = OP_Arithmetic; - taosArrayPush(plan, &op); - - //arithmetic on scalar function - if (pQueryAttr->pExpr2 != NULL) { + if (pQueryAttr->numOfFilterCols > 0 && pQueryAttr->vgId == 0) { // todo refactor + op = OP_Filter; + taosArrayPush(plan, &op); + } else { op = OP_Arithmetic; taosArrayPush(plan, &op); - } + } } if (pQueryAttr->limit.limit > 0 || pQueryAttr->limit.offset > 0) { diff --git a/src/query/src/qSqlParser.c b/src/query/src/qSqlParser.c index 03a74db122..847a089cc1 100644 --- a/src/query/src/qSqlParser.c +++ b/src/query/src/qSqlParser.c @@ -229,7 +229,6 @@ tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType) { pExpr->flags &= ~(1 << EXPR_FLAG_TS_ERROR); } - switch (optrType) { case TK_PLUS: { pExpr->value.i64 = pLeft->value.i64 + pRight->value.i64; @@ -325,7 +324,6 @@ static FORCE_INLINE int32_t tStrTokenCompare(SStrToken* left, SStrToken* right) return (left->type == right->type && left->n == right->n && strncasecmp(left->z, right->z, left->n) == 0) ? 0 : 1; } - int32_t tSqlExprCompare(tSqlExpr *left, tSqlExpr *right) { if ((left == NULL && right) || (left && right == NULL)) { return 1; @@ -389,8 +387,6 @@ int32_t tSqlExprCompare(tSqlExpr *left, tSqlExpr *right) { return 0; } - - tSqlExpr *tSqlExprClone(tSqlExpr *pSrc) { tSqlExpr *pExpr = calloc(1, sizeof(tSqlExpr)); @@ -536,11 +532,11 @@ SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int SRelationInfo *setTableNameList(SRelationInfo* pRelationInfo, SStrToken *pName, SStrToken* pAlias) { if (pRelationInfo == NULL) { pRelationInfo = calloc(1, sizeof(SRelationInfo)); - pRelationInfo->list = taosArrayInit(4, sizeof(STableNamePair)); + pRelationInfo->list = taosArrayInit(4, sizeof(SRelElementPair)); } pRelationInfo->type = SQL_NODE_FROM_TABLELIST; - STableNamePair p = {.name = *pName}; + SRelElementPair p = {.tableName = *pName}; if (pAlias != NULL) { p.aliasName = *pAlias; } else { @@ -551,18 +547,6 @@ SRelationInfo *setTableNameList(SRelationInfo* pRelationInfo, SStrToken *pName, return pRelationInfo; } -SRelationInfo* setSubquery(SRelationInfo* pRelationInfo, SArray* pList) { - if (pRelationInfo == NULL) { - pRelationInfo = calloc(1, sizeof(SRelationInfo)); - pRelationInfo->list = taosArrayInit(4, POINTER_BYTES); - } - - pRelationInfo->type = SQL_NODE_FROM_SUBQUERY; - taosArrayPush(pRelationInfo->list, &pList); - - return pRelationInfo; -} - void* destroyRelationInfo(SRelationInfo* pRelationInfo) { if (pRelationInfo == NULL) { return NULL; @@ -573,7 +557,7 @@ void* destroyRelationInfo(SRelationInfo* pRelationInfo) { } else { size_t size = taosArrayGetSize(pRelationInfo->list); for(int32_t i = 0; i < size; ++i) { - SArray* pa = taosArrayGetP(pRelationInfo->list, 0); + SArray* pa = taosArrayGetP(pRelationInfo->list, i); destroyAllSqlNode(pa); } taosArrayDestroy(pRelationInfo->list); @@ -583,6 +567,24 @@ void* destroyRelationInfo(SRelationInfo* pRelationInfo) { return NULL; } +SRelationInfo* addSubqueryElem(SRelationInfo* pRelationInfo, SArray* pSub, SStrToken* pAlias) { + if (pRelationInfo == NULL) { + pRelationInfo = calloc(1, sizeof(SRelationInfo)); + pRelationInfo->list = taosArrayInit(4, sizeof(SRelElementPair)); + } + + pRelationInfo->type = SQL_NODE_FROM_SUBQUERY; + + SRelElementPair p = {.pSubquery = pSub}; + if (pAlias != NULL) { + p.aliasName = *pAlias; + } else { + TPARSER_SET_NONE_TOKEN(p.aliasName); + } + + taosArrayPush(pRelationInfo->list, &p); + return pRelationInfo; +} void tSetDbName(SStrToken *pCpxName, SStrToken *pDb) { pCpxName->type = pDb->type; @@ -724,9 +726,9 @@ void tSetColumnType(TAOS_FIELD *pField, SStrToken *type) { * extract the select info out of sql string */ SSqlNode *tSetQuerySqlNode(SStrToken *pSelectToken, SArray *pSelNodeList, SRelationInfo *pFrom, tSqlExpr *pWhere, - SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, - SSessionWindowVal *pSession, SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, - SLimitVal *psLimit, tSqlExpr *pHaving) { + SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, SSessionWindowVal *pSession, + SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *psLimit, + tSqlExpr *pHaving) { assert(pSelNodeList != NULL); SSqlNode *pSqlNode = calloc(1, sizeof(SSqlNode)); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 7ff2d16962..7b08450d3b 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -581,6 +581,9 @@ void blockDistInfoToBinary(STableBlockDist* pDist, struct SBufferWriter* bw) { tbufWriteUint32(bw, pDist->numOfTables); tbufWriteUint16(bw, pDist->numOfFiles); tbufWriteUint64(bw, pDist->totalSize); + tbufWriteUint64(bw, pDist->totalRows); + tbufWriteInt32(bw, pDist->maxRows); + tbufWriteInt32(bw, pDist->minRows); tbufWriteUint32(bw, pDist->numOfRowsInMemTable); tbufWriteUint64(bw, taosArrayGetSize(pDist->dataBlockInfos)); @@ -616,13 +619,16 @@ void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDi pDist->numOfTables = tbufReadUint32(&br); pDist->numOfFiles = tbufReadUint16(&br); pDist->totalSize = tbufReadUint64(&br); + pDist->totalRows = tbufReadUint64(&br); + pDist->maxRows = tbufReadInt32(&br); + pDist->minRows = tbufReadInt32(&br); pDist->numOfRowsInMemTable = tbufReadUint32(&br); - int64_t numOfBlocks = tbufReadUint64(&br); + int64_t numSteps = tbufReadUint64(&br); bool comp = tbufReadUint8(&br); uint32_t compLen = tbufReadUint32(&br); - size_t originalLen = (size_t) (numOfBlocks*sizeof(SFileBlockInfo)); + size_t originalLen = (size_t) (numSteps *sizeof(SFileBlockInfo)); char* outputBuf = NULL; if (comp) { @@ -633,12 +639,12 @@ void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDi int32_t orignalLen = tsDecompressString(compStr, compLen, 1, outputBuf, (int32_t)originalLen , ONE_STAGE_COMP, NULL, 0); - assert(orignalLen == numOfBlocks*sizeof(SFileBlockInfo)); + assert(orignalLen == numSteps *sizeof(SFileBlockInfo)); } else { outputBuf = (char*) tbufReadBinary(&br, &originalLen); } - pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t) numOfBlocks, sizeof(SFileBlockInfo)); + pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t)numSteps, sizeof(SFileBlockInfo)); if (comp) { tfree(outputBuf); } diff --git a/src/query/src/queryMain.c b/src/query/src/queryMain.c index 2ad984add8..2ab25cadfa 100644 --- a/src/query/src/queryMain.c +++ b/src/query/src/queryMain.c @@ -162,7 +162,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi assert(pQueryMsg->stableQuery == isSTableQuery); (*pQInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo, - param.pTagColumnInfo, isSTableQuery, param.sql, qId, vgId, param.pUdfInfo); + param.pTagColumnInfo, isSTableQuery, param.sql, qId, param.pUdfInfo); if ((*pQInfo) == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; diff --git a/src/query/src/sql.c b/src/query/src/sql.c index a465b5df8c..12a0158513 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -97,27 +97,27 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 270 +#define YYNOCODE 271 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SStrToken typedef union { int yyinit; ParseTOKENTYPE yy0; - SSessionWindowVal yy15; - SIntervalVal yy42; - tSqlExpr* yy68; - SCreateAcctInfo yy77; - SArray* yy93; - int yy150; - SSqlNode* yy224; - int64_t yy279; - SLimitVal yy284; - TAOS_FIELD yy325; - SRelationInfo* yy330; - SCreateDbInfo yy372; - tVariant yy518; - SCreatedTableInfo yy528; - SCreateTableSql* yy532; + int yy112; + SCreateAcctInfo yy151; + tSqlExpr* yy166; + SCreateTableSql* yy182; + SSqlNode* yy236; + SRelationInfo* yy244; + SSessionWindowVal yy259; + SIntervalVal yy340; + TAOS_FIELD yy343; + int64_t yy369; + SCreateDbInfo yy382; + SLimitVal yy414; + SArray* yy441; + SCreatedTableInfo yy456; + tVariant yy506; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -127,17 +127,17 @@ typedef union { #define ParseARG_FETCH SSqlInfo* pInfo = yypParser->pInfo #define ParseARG_STORE yypParser->pInfo = pInfo #define YYFALLBACK 1 -#define YYNSTATE 332 -#define YYNRULE 276 +#define YYNSTATE 338 +#define YYNRULE 279 #define YYNTOKEN 192 -#define YY_MAX_SHIFT 331 -#define YY_MIN_SHIFTREDUCE 531 -#define YY_MAX_SHIFTREDUCE 806 -#define YY_ERROR_ACTION 807 -#define YY_ACCEPT_ACTION 808 -#define YY_NO_ACTION 809 -#define YY_MIN_REDUCE 810 -#define YY_MAX_REDUCE 1085 +#define YY_MAX_SHIFT 337 +#define YY_MIN_SHIFTREDUCE 538 +#define YY_MAX_SHIFTREDUCE 816 +#define YY_ERROR_ACTION 817 +#define YY_ACCEPT_ACTION 818 +#define YY_NO_ACTION 819 +#define YY_MIN_REDUCE 820 +#define YY_MAX_REDUCE 1098 /************* End control #defines *******************************************/ /* Define the yytestcase() macro to be a no-op if is not already defined @@ -203,270 +203,273 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (707) +#define YY_ACTTAB_COUNT (716) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 984, 581, 192, 581, 215, 329, 238, 17, 84, 582, - /* 10 */ 581, 582, 1066, 49, 50, 162, 53, 54, 582, 190, - /* 20 */ 226, 43, 192, 52, 273, 57, 55, 59, 56, 32, - /* 30 */ 141, 223, 1067, 48, 47, 808, 331, 46, 45, 44, - /* 40 */ 946, 947, 29, 950, 756, 957, 759, 532, 533, 534, - /* 50 */ 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, - /* 60 */ 545, 330, 220, 981, 216, 49, 50, 148, 53, 54, - /* 70 */ 148, 217, 226, 43, 960, 52, 273, 57, 55, 59, - /* 80 */ 56, 196, 975, 72, 295, 48, 47, 963, 232, 46, - /* 90 */ 45, 44, 49, 50, 258, 53, 54, 253, 315, 226, - /* 100 */ 43, 192, 52, 273, 57, 55, 59, 56, 71, 32, - /* 110 */ 222, 1067, 48, 47, 76, 238, 46, 45, 44, 49, - /* 120 */ 51, 38, 53, 54, 163, 1063, 226, 43, 274, 52, - /* 130 */ 273, 57, 55, 59, 56, 1016, 270, 268, 80, 48, - /* 140 */ 47, 948, 581, 46, 45, 44, 50, 975, 53, 54, - /* 150 */ 582, 229, 226, 43, 960, 52, 273, 57, 55, 59, - /* 160 */ 56, 763, 218, 750, 32, 48, 47, 1, 164, 46, - /* 170 */ 45, 44, 23, 293, 324, 323, 292, 291, 290, 322, - /* 180 */ 289, 321, 320, 319, 288, 318, 317, 923, 664, 911, - /* 190 */ 912, 913, 914, 915, 916, 917, 918, 919, 920, 921, - /* 200 */ 922, 924, 925, 53, 54, 18, 230, 226, 43, 960, - /* 210 */ 52, 273, 57, 55, 59, 56, 305, 304, 3, 177, - /* 220 */ 48, 47, 32, 200, 46, 45, 44, 225, 765, 769, - /* 230 */ 202, 754, 949, 757, 245, 760, 124, 123, 201, 148, - /* 240 */ 225, 765, 249, 248, 754, 231, 757, 32, 760, 32, - /* 250 */ 755, 25, 758, 1062, 931, 83, 32, 929, 930, 211, - /* 260 */ 212, 237, 932, 272, 934, 935, 933, 959, 936, 937, - /* 270 */ 963, 76, 211, 212, 23, 254, 324, 323, 38, 33, - /* 280 */ 235, 322, 12, 321, 320, 319, 86, 318, 317, 298, - /* 290 */ 688, 299, 960, 685, 960, 686, 1077, 687, 303, 32, - /* 300 */ 252, 960, 70, 57, 55, 59, 56, 1015, 208, 962, - /* 310 */ 233, 48, 47, 295, 325, 46, 45, 44, 5, 35, - /* 320 */ 166, 240, 241, 703, 66, 165, 93, 98, 89, 97, - /* 330 */ 46, 45, 44, 148, 239, 963, 58, 302, 301, 963, - /* 340 */ 284, 307, 766, 67, 960, 238, 48, 47, 762, 58, - /* 350 */ 46, 45, 44, 236, 961, 766, 297, 328, 327, 133, - /* 360 */ 87, 762, 82, 1061, 761, 731, 732, 764, 209, 109, - /* 370 */ 103, 114, 139, 137, 136, 73, 113, 761, 119, 122, - /* 380 */ 112, 184, 182, 180, 752, 111, 116, 858, 179, 128, - /* 390 */ 127, 126, 125, 176, 315, 951, 867, 859, 224, 700, - /* 400 */ 710, 81, 176, 176, 689, 24, 707, 256, 210, 716, - /* 410 */ 722, 723, 786, 143, 62, 20, 19, 63, 194, 767, - /* 420 */ 753, 195, 674, 19, 197, 276, 33, 676, 278, 33, - /* 430 */ 675, 62, 85, 28, 62, 69, 279, 663, 64, 102, - /* 440 */ 101, 692, 690, 693, 691, 121, 120, 6, 14, 13, - /* 450 */ 1026, 191, 198, 1025, 227, 108, 107, 199, 16, 15, - /* 460 */ 205, 206, 204, 189, 203, 193, 1022, 1021, 228, 250, - /* 470 */ 306, 140, 983, 41, 158, 991, 993, 142, 976, 146, - /* 480 */ 257, 958, 138, 1008, 1007, 259, 159, 285, 956, 219, - /* 490 */ 160, 161, 261, 872, 155, 153, 149, 715, 973, 281, - /* 500 */ 282, 150, 151, 266, 152, 154, 283, 156, 271, 263, - /* 510 */ 267, 68, 60, 269, 65, 265, 286, 287, 39, 260, - /* 520 */ 187, 316, 36, 296, 866, 1082, 99, 1081, 1079, 167, - /* 530 */ 300, 1076, 105, 110, 1075, 1073, 168, 892, 37, 34, - /* 540 */ 40, 188, 855, 115, 853, 117, 118, 851, 850, 242, - /* 550 */ 178, 848, 847, 846, 845, 844, 843, 842, 181, 183, - /* 560 */ 839, 837, 835, 42, 833, 185, 830, 186, 308, 255, - /* 570 */ 74, 77, 309, 262, 1009, 310, 311, 312, 313, 314, - /* 580 */ 326, 806, 243, 244, 213, 234, 280, 805, 246, 247, - /* 590 */ 804, 792, 214, 791, 251, 207, 94, 871, 870, 95, - /* 600 */ 256, 695, 275, 8, 849, 221, 75, 129, 130, 78, - /* 610 */ 841, 840, 171, 170, 893, 174, 169, 172, 173, 175, - /* 620 */ 131, 132, 832, 2, 831, 927, 717, 4, 144, 145, - /* 630 */ 720, 157, 26, 79, 264, 9, 724, 147, 939, 10, - /* 640 */ 768, 27, 7, 11, 770, 21, 22, 277, 88, 30, - /* 650 */ 90, 86, 91, 595, 627, 31, 92, 623, 621, 620, - /* 660 */ 619, 616, 585, 33, 294, 96, 61, 666, 665, 662, - /* 670 */ 611, 609, 601, 607, 100, 104, 603, 605, 599, 597, - /* 680 */ 630, 629, 106, 628, 626, 625, 624, 622, 618, 617, - /* 690 */ 62, 583, 549, 547, 810, 809, 809, 809, 134, 809, - /* 700 */ 809, 809, 809, 809, 809, 809, 135, + /* 0 */ 994, 588, 18, 588, 218, 335, 144, 224, 87, 589, + /* 10 */ 588, 589, 151, 50, 51, 151, 54, 55, 589, 195, + /* 20 */ 230, 44, 193, 53, 279, 58, 56, 60, 57, 227, + /* 30 */ 1080, 967, 973, 49, 48, 818, 337, 47, 46, 45, + /* 40 */ 956, 957, 30, 960, 47, 46, 45, 539, 540, 541, + /* 50 */ 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, + /* 60 */ 552, 336, 235, 991, 219, 50, 51, 151, 54, 55, + /* 70 */ 264, 263, 230, 44, 236, 53, 279, 58, 56, 60, + /* 80 */ 57, 1028, 985, 73, 84, 49, 48, 973, 868, 47, + /* 90 */ 46, 45, 50, 51, 179, 54, 55, 257, 195, 230, + /* 100 */ 44, 85, 53, 279, 58, 56, 60, 57, 226, 1080, + /* 110 */ 195, 301, 49, 48, 74, 242, 47, 46, 45, 50, + /* 120 */ 52, 1079, 54, 55, 165, 199, 230, 44, 72, 53, + /* 130 */ 279, 58, 56, 60, 57, 766, 1029, 769, 274, 49, + /* 140 */ 48, 321, 588, 47, 46, 45, 51, 79, 54, 55, + /* 150 */ 589, 1076, 230, 44, 39, 53, 279, 58, 56, 60, + /* 160 */ 57, 958, 877, 760, 33, 49, 48, 1075, 179, 47, + /* 170 */ 46, 45, 24, 299, 330, 329, 298, 297, 296, 328, + /* 180 */ 295, 327, 326, 325, 294, 324, 323, 933, 671, 921, + /* 190 */ 922, 923, 924, 925, 926, 927, 928, 929, 930, 931, + /* 200 */ 932, 934, 935, 54, 55, 19, 220, 230, 44, 970, + /* 210 */ 53, 279, 58, 56, 60, 57, 311, 310, 86, 1090, + /* 220 */ 49, 48, 151, 203, 47, 46, 45, 229, 775, 959, + /* 230 */ 205, 764, 985, 767, 249, 770, 127, 126, 204, 1074, + /* 240 */ 229, 775, 253, 252, 764, 13, 767, 221, 770, 89, + /* 250 */ 237, 25, 773, 212, 941, 985, 33, 939, 940, 214, + /* 260 */ 215, 241, 942, 278, 944, 945, 943, 331, 946, 947, + /* 270 */ 222, 79, 214, 215, 24, 973, 330, 329, 39, 213, + /* 280 */ 765, 328, 768, 327, 326, 325, 33, 324, 323, 276, + /* 290 */ 695, 83, 973, 692, 228, 693, 1039, 694, 233, 33, + /* 300 */ 256, 970, 71, 58, 56, 60, 57, 70, 211, 670, + /* 310 */ 301, 49, 48, 710, 33, 47, 46, 45, 5, 36, + /* 320 */ 169, 244, 245, 33, 242, 168, 96, 101, 92, 100, + /* 330 */ 33, 969, 239, 166, 243, 280, 59, 308, 307, 90, + /* 340 */ 290, 234, 776, 6, 970, 242, 49, 48, 772, 59, + /* 350 */ 47, 46, 45, 33, 971, 776, 304, 114, 699, 970, + /* 360 */ 700, 772, 1, 167, 771, 305, 321, 869, 970, 112, + /* 370 */ 106, 117, 309, 179, 961, 970, 116, 771, 122, 125, + /* 380 */ 115, 187, 185, 183, 3, 180, 119, 197, 182, 131, + /* 390 */ 130, 129, 128, 741, 742, 313, 714, 774, 970, 334, + /* 400 */ 333, 136, 258, 762, 696, 240, 34, 707, 303, 76, + /* 410 */ 142, 140, 139, 8, 77, 726, 260, 64, 732, 146, + /* 420 */ 733, 260, 63, 796, 21, 777, 67, 20, 681, 20, + /* 430 */ 282, 198, 34, 683, 34, 200, 779, 63, 65, 763, + /* 440 */ 284, 972, 682, 254, 88, 68, 63, 29, 105, 104, + /* 450 */ 285, 17, 16, 697, 194, 698, 124, 123, 15, 14, + /* 460 */ 201, 202, 208, 209, 111, 110, 207, 192, 206, 196, + /* 470 */ 1038, 231, 42, 1035, 1034, 232, 312, 143, 993, 1004, + /* 480 */ 1001, 1002, 1006, 145, 986, 261, 149, 1021, 1020, 968, + /* 490 */ 161, 162, 966, 163, 141, 164, 882, 287, 265, 291, + /* 500 */ 881, 937, 159, 725, 983, 153, 288, 152, 154, 155, + /* 510 */ 223, 156, 289, 267, 272, 277, 275, 69, 61, 292, + /* 520 */ 293, 40, 190, 37, 302, 66, 876, 273, 1095, 271, + /* 530 */ 102, 1094, 1092, 269, 170, 306, 1089, 108, 1088, 1086, + /* 540 */ 171, 902, 38, 266, 35, 41, 191, 865, 118, 43, + /* 550 */ 863, 120, 121, 861, 860, 246, 181, 858, 857, 856, + /* 560 */ 855, 854, 853, 852, 184, 186, 849, 847, 845, 843, + /* 570 */ 188, 840, 189, 322, 259, 113, 75, 80, 314, 268, + /* 580 */ 1022, 315, 316, 317, 318, 319, 320, 332, 216, 816, + /* 590 */ 238, 286, 247, 248, 815, 217, 210, 250, 97, 98, + /* 600 */ 251, 880, 814, 802, 801, 255, 260, 281, 78, 9, + /* 610 */ 225, 702, 859, 851, 178, 903, 174, 172, 173, 132, + /* 620 */ 175, 176, 133, 134, 850, 177, 135, 842, 841, 2, + /* 630 */ 26, 4, 262, 81, 727, 157, 158, 160, 147, 148, + /* 640 */ 27, 82, 949, 730, 270, 91, 734, 150, 28, 10, + /* 650 */ 11, 778, 7, 12, 780, 22, 23, 283, 31, 93, + /* 660 */ 602, 89, 94, 32, 95, 634, 630, 628, 627, 626, + /* 670 */ 623, 99, 592, 300, 34, 62, 673, 672, 618, 669, + /* 680 */ 103, 616, 608, 107, 614, 109, 610, 612, 606, 604, + /* 690 */ 637, 636, 635, 633, 632, 631, 629, 625, 624, 590, + /* 700 */ 63, 137, 556, 554, 820, 819, 819, 819, 819, 819, + /* 710 */ 819, 819, 819, 819, 819, 138, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 196, 1, 258, 1, 195, 196, 196, 258, 202, 9, - /* 10 */ 1, 9, 268, 13, 14, 205, 16, 17, 9, 258, - /* 20 */ 20, 21, 258, 23, 24, 25, 26, 27, 28, 196, - /* 30 */ 196, 267, 268, 33, 34, 193, 194, 37, 38, 39, - /* 40 */ 234, 235, 236, 237, 5, 196, 7, 45, 46, 47, + /* 0 */ 196, 1, 258, 1, 195, 196, 196, 217, 202, 9, + /* 10 */ 1, 9, 196, 13, 14, 196, 16, 17, 9, 258, + /* 20 */ 20, 21, 258, 23, 24, 25, 26, 27, 28, 268, + /* 30 */ 269, 196, 242, 33, 34, 193, 194, 37, 38, 39, + /* 40 */ 234, 235, 236, 237, 37, 38, 39, 45, 46, 47, /* 50 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, /* 60 */ 58, 59, 217, 259, 62, 13, 14, 196, 16, 17, - /* 70 */ 196, 238, 20, 21, 241, 23, 24, 25, 26, 27, - /* 80 */ 28, 258, 240, 83, 81, 33, 34, 242, 239, 37, - /* 90 */ 38, 39, 13, 14, 260, 16, 17, 255, 87, 20, - /* 100 */ 21, 258, 23, 24, 25, 26, 27, 28, 202, 196, - /* 110 */ 267, 268, 33, 34, 110, 196, 37, 38, 39, 13, - /* 120 */ 14, 117, 16, 17, 205, 258, 20, 21, 15, 23, - /* 130 */ 24, 25, 26, 27, 28, 264, 262, 266, 264, 33, - /* 140 */ 34, 235, 1, 37, 38, 39, 14, 240, 16, 17, - /* 150 */ 9, 238, 20, 21, 241, 23, 24, 25, 26, 27, - /* 160 */ 28, 122, 255, 111, 196, 33, 34, 203, 204, 37, + /* 70 */ 260, 261, 20, 21, 239, 23, 24, 25, 26, 27, + /* 80 */ 28, 265, 240, 83, 265, 33, 34, 242, 201, 37, + /* 90 */ 38, 39, 13, 14, 207, 16, 17, 255, 258, 20, + /* 100 */ 21, 243, 23, 24, 25, 26, 27, 28, 268, 269, + /* 110 */ 258, 81, 33, 34, 256, 196, 37, 38, 39, 13, + /* 120 */ 14, 269, 16, 17, 205, 258, 20, 21, 202, 23, + /* 130 */ 24, 25, 26, 27, 28, 5, 265, 7, 267, 33, + /* 140 */ 34, 87, 1, 37, 38, 39, 14, 110, 16, 17, + /* 150 */ 9, 258, 20, 21, 117, 23, 24, 25, 26, 27, + /* 160 */ 28, 235, 201, 111, 196, 33, 34, 258, 207, 37, /* 170 */ 38, 39, 94, 95, 96, 97, 98, 99, 100, 101, /* 180 */ 102, 103, 104, 105, 106, 107, 108, 216, 5, 218, /* 190 */ 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, /* 200 */ 229, 230, 231, 16, 17, 44, 238, 20, 21, 241, - /* 210 */ 23, 24, 25, 26, 27, 28, 33, 34, 199, 200, - /* 220 */ 33, 34, 196, 62, 37, 38, 39, 1, 2, 116, - /* 230 */ 69, 5, 0, 7, 140, 9, 75, 76, 77, 196, - /* 240 */ 1, 2, 148, 149, 5, 217, 7, 196, 9, 196, - /* 250 */ 5, 110, 7, 258, 216, 83, 196, 219, 220, 33, - /* 260 */ 34, 69, 224, 37, 226, 227, 228, 241, 230, 231, - /* 270 */ 242, 110, 33, 34, 94, 111, 96, 97, 117, 115, - /* 280 */ 69, 101, 110, 103, 104, 105, 114, 107, 108, 238, - /* 290 */ 2, 238, 241, 5, 241, 7, 242, 9, 238, 196, - /* 300 */ 139, 241, 141, 25, 26, 27, 28, 264, 147, 242, - /* 310 */ 217, 33, 34, 81, 217, 37, 38, 39, 63, 64, - /* 320 */ 65, 33, 34, 37, 115, 70, 71, 72, 73, 74, - /* 330 */ 37, 38, 39, 196, 142, 242, 110, 145, 146, 242, - /* 340 */ 85, 238, 116, 134, 241, 196, 33, 34, 122, 110, - /* 350 */ 37, 38, 39, 142, 205, 116, 145, 66, 67, 68, - /* 360 */ 202, 122, 243, 258, 138, 129, 130, 122, 258, 63, - /* 370 */ 64, 65, 63, 64, 65, 256, 70, 138, 72, 73, - /* 380 */ 74, 63, 64, 65, 1, 78, 80, 201, 70, 71, - /* 390 */ 72, 73, 74, 207, 87, 237, 201, 201, 61, 115, - /* 400 */ 111, 264, 207, 207, 116, 121, 120, 118, 258, 111, - /* 410 */ 111, 111, 111, 115, 115, 115, 115, 115, 258, 111, - /* 420 */ 37, 258, 111, 115, 258, 111, 115, 111, 111, 115, - /* 430 */ 111, 115, 115, 110, 115, 110, 113, 112, 136, 143, - /* 440 */ 144, 5, 5, 7, 7, 78, 79, 110, 143, 144, - /* 450 */ 233, 258, 258, 233, 233, 143, 144, 258, 143, 144, - /* 460 */ 258, 258, 258, 258, 258, 258, 233, 233, 233, 196, - /* 470 */ 233, 196, 196, 257, 244, 196, 196, 196, 240, 196, - /* 480 */ 240, 240, 61, 265, 265, 261, 196, 86, 196, 261, - /* 490 */ 196, 196, 261, 196, 247, 249, 253, 122, 254, 196, - /* 500 */ 196, 252, 251, 261, 250, 248, 196, 246, 127, 124, - /* 510 */ 126, 133, 132, 131, 135, 125, 196, 196, 196, 123, - /* 520 */ 196, 109, 196, 196, 196, 196, 196, 196, 196, 196, - /* 530 */ 196, 196, 196, 93, 196, 196, 196, 196, 196, 196, - /* 540 */ 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, + /* 210 */ 23, 24, 25, 26, 27, 28, 33, 34, 83, 242, + /* 220 */ 33, 34, 196, 62, 37, 38, 39, 1, 2, 0, + /* 230 */ 69, 5, 240, 7, 140, 9, 75, 76, 77, 258, + /* 240 */ 1, 2, 148, 149, 5, 110, 7, 255, 9, 114, + /* 250 */ 217, 110, 122, 258, 216, 240, 196, 219, 220, 33, + /* 260 */ 34, 69, 224, 37, 226, 227, 228, 217, 230, 231, + /* 270 */ 255, 110, 33, 34, 94, 242, 96, 97, 117, 258, + /* 280 */ 5, 101, 7, 103, 104, 105, 196, 107, 108, 263, + /* 290 */ 2, 265, 242, 5, 61, 7, 233, 9, 238, 196, + /* 300 */ 139, 241, 141, 25, 26, 27, 28, 110, 147, 112, + /* 310 */ 81, 33, 34, 37, 196, 37, 38, 39, 63, 64, + /* 320 */ 65, 33, 34, 196, 196, 70, 71, 72, 73, 74, + /* 330 */ 196, 241, 69, 205, 142, 15, 110, 145, 146, 202, + /* 340 */ 85, 238, 116, 110, 241, 196, 33, 34, 122, 110, + /* 350 */ 37, 38, 39, 196, 205, 116, 238, 78, 5, 241, + /* 360 */ 7, 122, 203, 204, 138, 238, 87, 201, 241, 63, + /* 370 */ 64, 65, 238, 207, 237, 241, 70, 138, 72, 73, + /* 380 */ 74, 63, 64, 65, 199, 200, 80, 258, 70, 71, + /* 390 */ 72, 73, 74, 129, 130, 238, 120, 122, 241, 66, + /* 400 */ 67, 68, 111, 1, 116, 142, 115, 115, 145, 111, + /* 410 */ 63, 64, 65, 121, 111, 111, 118, 115, 111, 115, + /* 420 */ 111, 118, 115, 111, 115, 111, 115, 115, 111, 115, + /* 430 */ 111, 258, 115, 111, 115, 258, 116, 115, 136, 37, + /* 440 */ 111, 242, 111, 196, 115, 134, 115, 110, 143, 144, + /* 450 */ 113, 143, 144, 5, 258, 7, 78, 79, 143, 144, + /* 460 */ 258, 258, 258, 258, 143, 144, 258, 258, 258, 258, + /* 470 */ 233, 233, 257, 233, 233, 233, 233, 196, 196, 196, + /* 480 */ 196, 196, 196, 196, 240, 240, 196, 266, 266, 240, + /* 490 */ 244, 196, 196, 196, 61, 196, 196, 196, 262, 86, + /* 500 */ 206, 232, 246, 122, 254, 252, 196, 253, 251, 250, + /* 510 */ 262, 249, 196, 262, 262, 127, 131, 133, 132, 196, + /* 520 */ 196, 196, 196, 196, 196, 135, 196, 126, 196, 125, + /* 530 */ 196, 196, 196, 124, 196, 196, 196, 196, 196, 196, + /* 540 */ 196, 196, 196, 123, 196, 196, 196, 196, 196, 137, /* 550 */ 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, - /* 560 */ 196, 196, 196, 137, 196, 196, 196, 196, 92, 197, - /* 570 */ 197, 197, 51, 197, 197, 89, 91, 55, 90, 88, - /* 580 */ 81, 5, 150, 5, 197, 197, 197, 5, 150, 5, - /* 590 */ 5, 96, 197, 95, 140, 197, 202, 206, 206, 202, - /* 600 */ 118, 111, 113, 110, 197, 1, 119, 198, 198, 115, - /* 610 */ 197, 197, 209, 213, 215, 211, 214, 212, 210, 208, - /* 620 */ 198, 198, 197, 203, 197, 232, 111, 199, 110, 115, - /* 630 */ 111, 245, 115, 110, 110, 128, 111, 110, 232, 128, - /* 640 */ 111, 115, 110, 110, 116, 110, 110, 113, 78, 84, - /* 650 */ 83, 114, 71, 5, 9, 84, 83, 5, 5, 5, - /* 660 */ 5, 5, 82, 115, 15, 78, 16, 5, 5, 111, - /* 670 */ 5, 5, 5, 5, 144, 144, 5, 5, 5, 5, - /* 680 */ 5, 5, 144, 5, 5, 5, 5, 5, 5, 5, - /* 690 */ 115, 82, 61, 60, 0, 269, 269, 269, 21, 269, - /* 700 */ 269, 269, 269, 269, 269, 269, 21, 269, 269, 269, - /* 710 */ 269, 269, 269, 269, 269, 269, 269, 269, 269, 269, - /* 720 */ 269, 269, 269, 269, 269, 269, 269, 269, 269, 269, - /* 730 */ 269, 269, 269, 269, 269, 269, 269, 269, 269, 269, - /* 740 */ 269, 269, 269, 269, 269, 269, 269, 269, 269, 269, - /* 750 */ 269, 269, 269, 269, 269, 269, 269, 269, 269, 269, - /* 760 */ 269, 269, 269, 269, 269, 269, 269, 269, 269, 269, - /* 770 */ 269, 269, 269, 269, 269, 269, 269, 269, 269, 269, - /* 780 */ 269, 269, 269, 269, 269, 269, 269, 269, 269, 269, - /* 790 */ 269, 269, 269, 269, 269, 269, 269, 269, 269, 269, - /* 800 */ 269, 269, 269, 269, 269, 269, 269, 269, 269, 269, - /* 810 */ 269, 269, 269, 269, 269, 269, 269, 269, 269, 269, - /* 820 */ 269, 269, 269, 269, 269, 269, 269, 269, 269, 269, - /* 830 */ 269, 269, 269, 269, 269, 269, 269, 269, 269, 269, - /* 840 */ 269, 269, 269, 269, 269, 269, 269, 269, 269, 269, - /* 850 */ 269, 269, 269, 269, 269, 269, 269, 269, 269, 269, - /* 860 */ 269, 269, 269, 269, 269, 269, 269, 269, 269, 269, - /* 870 */ 269, 269, 269, 269, 269, 269, 269, 269, 269, 269, - /* 880 */ 269, 269, 269, 269, 269, 269, 269, 269, 269, 269, - /* 890 */ 269, 269, 269, 269, 269, 269, 269, 269, 269, + /* 560 */ 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, + /* 570 */ 196, 196, 196, 109, 197, 93, 197, 197, 92, 197, + /* 580 */ 197, 51, 89, 91, 55, 90, 88, 81, 197, 5, + /* 590 */ 197, 197, 150, 5, 5, 197, 197, 150, 202, 202, + /* 600 */ 5, 206, 5, 96, 95, 140, 118, 113, 119, 110, + /* 610 */ 1, 111, 197, 197, 208, 215, 209, 214, 213, 198, + /* 620 */ 212, 210, 198, 198, 197, 211, 198, 197, 197, 203, + /* 630 */ 110, 199, 115, 115, 111, 248, 247, 245, 110, 115, + /* 640 */ 115, 110, 232, 111, 110, 78, 111, 110, 115, 128, + /* 650 */ 128, 111, 110, 110, 116, 110, 110, 113, 84, 83, + /* 660 */ 5, 114, 71, 84, 83, 9, 5, 5, 5, 5, + /* 670 */ 5, 78, 82, 15, 115, 16, 5, 5, 5, 111, + /* 680 */ 144, 5, 5, 144, 5, 144, 5, 5, 5, 5, + /* 690 */ 5, 5, 5, 5, 5, 5, 5, 5, 5, 82, + /* 700 */ 115, 21, 61, 60, 0, 270, 270, 270, 270, 270, + /* 710 */ 270, 270, 270, 270, 270, 21, 270, 270, 270, 270, + /* 720 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 730 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 740 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 750 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 760 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 770 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 780 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 790 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 800 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 810 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 820 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 830 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 840 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 850 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 860 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 870 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 880 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 890 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 900 */ 270, 270, 270, 270, 270, 270, 270, 270, }; -#define YY_SHIFT_COUNT (331) +#define YY_SHIFT_COUNT (337) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (694) +#define YY_SHIFT_MAX (704) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 161, 78, 78, 180, 180, 3, 226, 239, 9, 9, - /* 10 */ 9, 9, 9, 9, 9, 9, 9, 0, 2, 239, - /* 20 */ 288, 288, 288, 288, 141, 4, 9, 9, 9, 232, - /* 30 */ 9, 9, 9, 9, 307, 3, 11, 11, 707, 707, - /* 40 */ 707, 239, 239, 239, 239, 239, 239, 239, 239, 239, + /* 0 */ 161, 78, 78, 180, 180, 30, 226, 239, 141, 9, + /* 10 */ 9, 9, 9, 9, 9, 9, 9, 9, 0, 2, + /* 20 */ 239, 288, 288, 288, 288, 37, 37, 9, 9, 9, + /* 30 */ 229, 9, 9, 9, 9, 279, 30, 54, 54, 716, + /* 40 */ 716, 716, 239, 239, 239, 239, 239, 239, 239, 239, /* 50 */ 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, - /* 60 */ 239, 288, 288, 183, 183, 183, 183, 183, 183, 183, - /* 70 */ 9, 9, 9, 286, 9, 4, 4, 9, 9, 9, - /* 80 */ 236, 236, 284, 4, 9, 9, 9, 9, 9, 9, + /* 60 */ 239, 239, 288, 288, 183, 183, 183, 183, 183, 183, + /* 70 */ 183, 9, 9, 9, 276, 9, 9, 9, 37, 37, + /* 80 */ 9, 9, 9, 264, 264, 292, 37, 9, 9, 9, /* 90 */ 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, /* 100 */ 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, /* 110 */ 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, /* 120 */ 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, /* 130 */ 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - /* 140 */ 421, 421, 421, 375, 375, 375, 421, 375, 421, 378, - /* 150 */ 379, 380, 381, 382, 384, 390, 385, 396, 426, 421, - /* 160 */ 421, 421, 401, 401, 412, 3, 3, 421, 421, 440, - /* 170 */ 476, 521, 486, 485, 522, 488, 491, 412, 421, 499, - /* 180 */ 499, 421, 499, 421, 499, 421, 421, 707, 707, 52, - /* 190 */ 79, 106, 79, 79, 132, 187, 278, 278, 278, 278, - /* 200 */ 255, 306, 318, 313, 313, 313, 313, 192, 94, 293, - /* 210 */ 293, 39, 245, 172, 211, 291, 309, 164, 289, 298, - /* 220 */ 299, 300, 301, 308, 383, 337, 113, 302, 209, 311, - /* 230 */ 314, 316, 317, 319, 323, 296, 305, 312, 325, 315, - /* 240 */ 436, 437, 367, 576, 432, 578, 582, 438, 584, 585, - /* 250 */ 495, 498, 454, 482, 489, 493, 487, 490, 494, 515, - /* 260 */ 518, 519, 514, 523, 604, 524, 525, 527, 517, 507, - /* 270 */ 526, 511, 529, 532, 528, 533, 489, 535, 534, 536, - /* 280 */ 537, 570, 565, 567, 581, 648, 571, 573, 645, 652, - /* 290 */ 653, 654, 655, 656, 580, 649, 587, 530, 548, 548, - /* 300 */ 650, 531, 538, 548, 662, 663, 558, 548, 665, 666, - /* 310 */ 667, 668, 671, 672, 673, 674, 675, 676, 678, 679, - /* 320 */ 680, 681, 682, 683, 684, 575, 609, 677, 685, 631, - /* 330 */ 633, 694, + /* 140 */ 9, 9, 9, 433, 433, 433, 381, 381, 381, 433, + /* 150 */ 381, 433, 384, 390, 386, 388, 385, 401, 404, 409, + /* 160 */ 420, 412, 433, 433, 433, 413, 413, 464, 30, 30, + /* 170 */ 433, 433, 482, 486, 530, 493, 492, 529, 495, 498, + /* 180 */ 464, 433, 506, 506, 433, 506, 433, 506, 433, 433, + /* 190 */ 716, 716, 52, 79, 106, 79, 79, 132, 187, 278, + /* 200 */ 278, 278, 278, 255, 306, 318, 313, 313, 313, 313, + /* 210 */ 192, 94, 7, 7, 130, 275, 135, 263, 333, 347, + /* 220 */ 291, 298, 303, 304, 307, 309, 312, 314, 402, 233, + /* 230 */ 320, 302, 311, 317, 319, 322, 329, 331, 337, 305, + /* 240 */ 315, 321, 197, 308, 353, 448, 378, 584, 442, 588, + /* 250 */ 589, 447, 595, 597, 507, 509, 465, 488, 494, 499, + /* 260 */ 489, 500, 520, 517, 518, 523, 528, 532, 524, 531, + /* 270 */ 609, 534, 535, 537, 525, 521, 533, 522, 540, 542, + /* 280 */ 538, 543, 494, 545, 544, 546, 547, 567, 574, 576, + /* 290 */ 591, 655, 579, 581, 656, 661, 662, 663, 664, 665, + /* 300 */ 590, 658, 593, 536, 559, 559, 659, 539, 541, 559, + /* 310 */ 671, 672, 568, 559, 673, 676, 677, 679, 681, 682, + /* 320 */ 683, 684, 685, 686, 687, 688, 689, 690, 691, 692, + /* 330 */ 693, 585, 617, 680, 694, 641, 643, 704, }; -#define YY_REDUCE_COUNT (188) +#define YY_REDUCE_COUNT (191) #define YY_REDUCE_MIN (-256) -#define YY_REDUCE_MAX (428) +#define YY_REDUCE_MAX (432) static const short yy_reduce_ofst[] = { - /* 0 */ -158, -29, -29, 38, 38, -194, -236, -157, -167, -129, - /* 10 */ -126, -87, -32, 51, 53, 60, 103, -196, -191, -256, - /* 20 */ -155, 28, 93, 97, -166, -93, 43, 137, -151, 158, - /* 30 */ -190, -81, 149, 26, 186, -94, 195, 196, 119, -36, - /* 40 */ 19, -251, -239, -177, -133, -5, 105, 110, 150, 160, - /* 50 */ 163, 166, 193, 194, 199, 202, 203, 204, 205, 206, - /* 60 */ 207, 54, 67, 217, 220, 221, 233, 234, 235, 237, - /* 70 */ 273, 275, 276, 216, 279, 238, 240, 280, 281, 283, - /* 80 */ 218, 219, 230, 241, 290, 292, 294, 295, 297, 303, - /* 90 */ 304, 310, 320, 321, 322, 324, 326, 327, 328, 329, - /* 100 */ 330, 331, 332, 333, 334, 335, 336, 338, 339, 340, - /* 110 */ 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, - /* 120 */ 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, - /* 130 */ 361, 362, 363, 364, 365, 366, 368, 369, 370, 371, - /* 140 */ 372, 373, 374, 224, 228, 231, 376, 242, 377, 244, - /* 150 */ 243, 249, 251, 254, 246, 257, 247, 261, 386, 387, - /* 160 */ 388, 389, 391, 392, 393, 394, 397, 395, 398, 399, - /* 170 */ 402, 400, 403, 405, 408, 404, 411, 406, 407, 409, - /* 180 */ 410, 413, 422, 414, 423, 425, 427, 420, 428, + /* 0 */ -158, -29, -29, 38, 38, -194, -239, -160, -190, -32, + /* 10 */ -129, 26, 60, 103, 118, 127, 134, 157, -196, -191, + /* 20 */ -148, -210, -155, 33, 50, -8, 15, -184, -181, -165, + /* 30 */ 137, -81, 128, 149, 90, -113, -74, -39, 166, -142, + /* 40 */ 159, 185, -256, -236, -133, -107, -91, -19, -5, 21, + /* 50 */ 129, 173, 177, 196, 202, 203, 204, 205, 208, 209, + /* 60 */ 210, 211, -23, 199, 63, 237, 238, 240, 241, 242, + /* 70 */ 243, 247, 281, 282, 215, 283, 284, 285, 244, 245, + /* 80 */ 286, 287, 290, 221, 222, 246, 249, 295, 296, 297, + /* 90 */ 299, 300, 301, 310, 316, 323, 324, 325, 326, 327, + /* 100 */ 328, 330, 332, 334, 335, 336, 338, 339, 340, 341, + /* 110 */ 342, 343, 344, 345, 346, 348, 349, 350, 351, 352, + /* 120 */ 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, + /* 130 */ 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + /* 140 */ 374, 375, 376, 377, 379, 380, 236, 248, 251, 382, + /* 150 */ 252, 383, 250, 254, 253, 257, 259, 262, 387, 389, + /* 160 */ 256, 392, 391, 393, 394, 294, 395, 269, 396, 397, + /* 170 */ 398, 399, 400, 403, 405, 407, 408, 411, 414, 406, + /* 180 */ 410, 415, 421, 424, 416, 425, 427, 428, 430, 431, + /* 190 */ 426, 432, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 807, 926, 868, 938, 856, 865, 1069, 1069, 807, 807, - /* 10 */ 807, 807, 807, 807, 807, 807, 807, 985, 827, 1069, - /* 20 */ 807, 807, 807, 807, 807, 807, 807, 807, 807, 865, - /* 30 */ 807, 807, 807, 807, 875, 865, 875, 875, 980, 910, - /* 40 */ 928, 807, 807, 807, 807, 807, 807, 807, 807, 807, - /* 50 */ 807, 807, 807, 807, 807, 807, 807, 807, 807, 807, - /* 60 */ 807, 807, 807, 807, 807, 807, 807, 807, 807, 807, - /* 70 */ 807, 807, 807, 987, 990, 807, 807, 992, 807, 807, - /* 80 */ 1012, 1012, 978, 807, 807, 807, 807, 807, 807, 807, - /* 90 */ 807, 807, 807, 807, 807, 807, 807, 807, 807, 807, - /* 100 */ 807, 807, 807, 807, 807, 807, 807, 807, 807, 807, - /* 110 */ 807, 807, 807, 807, 807, 854, 807, 852, 807, 807, - /* 120 */ 807, 807, 807, 807, 807, 807, 807, 807, 807, 807, - /* 130 */ 807, 807, 807, 838, 807, 807, 807, 807, 807, 807, - /* 140 */ 829, 829, 829, 807, 807, 807, 829, 807, 829, 1019, - /* 150 */ 1023, 1017, 1005, 1013, 1004, 1000, 998, 997, 1027, 829, - /* 160 */ 829, 829, 873, 873, 869, 865, 865, 829, 829, 891, - /* 170 */ 889, 887, 879, 885, 881, 883, 877, 857, 829, 863, - /* 180 */ 863, 829, 863, 829, 863, 829, 829, 910, 928, 807, - /* 190 */ 1028, 807, 1068, 1018, 1058, 1057, 1064, 1056, 1055, 1054, - /* 200 */ 807, 807, 807, 1050, 1051, 1053, 1052, 807, 807, 1060, - /* 210 */ 1059, 807, 807, 807, 807, 807, 807, 807, 807, 807, - /* 220 */ 807, 807, 807, 807, 807, 1030, 807, 1024, 1020, 807, - /* 230 */ 807, 807, 807, 807, 807, 807, 807, 807, 940, 807, - /* 240 */ 807, 807, 807, 807, 807, 807, 807, 807, 807, 807, - /* 250 */ 807, 807, 807, 977, 807, 807, 807, 807, 988, 807, - /* 260 */ 807, 807, 807, 807, 807, 807, 807, 807, 1014, 807, - /* 270 */ 1006, 807, 807, 807, 807, 807, 952, 807, 807, 807, - /* 280 */ 807, 807, 807, 807, 807, 807, 807, 807, 807, 807, - /* 290 */ 807, 807, 807, 807, 807, 807, 807, 807, 1080, 1078, - /* 300 */ 807, 807, 807, 1074, 807, 807, 807, 1072, 807, 807, - /* 310 */ 807, 807, 807, 807, 807, 807, 807, 807, 807, 807, - /* 320 */ 807, 807, 807, 807, 807, 894, 807, 836, 834, 807, - /* 330 */ 825, 807, + /* 0 */ 817, 936, 878, 948, 866, 875, 1082, 1082, 817, 817, + /* 10 */ 817, 817, 817, 817, 817, 817, 817, 817, 995, 837, + /* 20 */ 1082, 817, 817, 817, 817, 817, 817, 817, 817, 817, + /* 30 */ 875, 817, 817, 817, 817, 885, 875, 885, 885, 990, + /* 40 */ 920, 938, 817, 817, 817, 817, 817, 817, 817, 817, + /* 50 */ 817, 817, 817, 817, 817, 817, 817, 817, 817, 817, + /* 60 */ 817, 817, 817, 817, 817, 817, 817, 817, 817, 817, + /* 70 */ 817, 817, 817, 817, 997, 1003, 1000, 817, 817, 817, + /* 80 */ 1005, 817, 817, 1025, 1025, 988, 817, 817, 817, 817, + /* 90 */ 817, 817, 817, 817, 817, 817, 817, 817, 817, 817, + /* 100 */ 817, 817, 817, 817, 817, 817, 817, 817, 817, 817, + /* 110 */ 817, 817, 817, 817, 817, 817, 817, 817, 864, 817, + /* 120 */ 862, 817, 817, 817, 817, 817, 817, 817, 817, 817, + /* 130 */ 817, 817, 817, 817, 817, 817, 848, 817, 817, 817, + /* 140 */ 817, 817, 817, 839, 839, 839, 817, 817, 817, 839, + /* 150 */ 817, 839, 1032, 1036, 1030, 1018, 1026, 1017, 1013, 1011, + /* 160 */ 1010, 1040, 839, 839, 839, 883, 883, 879, 875, 875, + /* 170 */ 839, 839, 901, 899, 897, 889, 895, 891, 893, 887, + /* 180 */ 867, 839, 873, 873, 839, 873, 839, 873, 839, 839, + /* 190 */ 920, 938, 817, 1041, 817, 1081, 1031, 1071, 1070, 1077, + /* 200 */ 1069, 1068, 1067, 817, 817, 817, 1063, 1064, 1066, 1065, + /* 210 */ 817, 817, 1073, 1072, 817, 817, 817, 817, 817, 817, + /* 220 */ 817, 817, 817, 817, 817, 817, 817, 817, 817, 1043, + /* 230 */ 817, 1037, 1033, 817, 817, 817, 817, 817, 817, 817, + /* 240 */ 817, 817, 950, 817, 817, 817, 817, 817, 817, 817, + /* 250 */ 817, 817, 817, 817, 817, 817, 817, 987, 817, 817, + /* 260 */ 817, 817, 817, 999, 998, 817, 817, 817, 817, 817, + /* 270 */ 817, 817, 817, 817, 1027, 817, 1019, 817, 817, 817, + /* 280 */ 817, 817, 962, 817, 817, 817, 817, 817, 817, 817, + /* 290 */ 817, 817, 817, 817, 817, 817, 817, 817, 817, 817, + /* 300 */ 817, 817, 817, 817, 1093, 1091, 817, 817, 817, 1087, + /* 310 */ 817, 817, 817, 1085, 817, 817, 817, 817, 817, 817, + /* 320 */ 817, 817, 817, 817, 817, 817, 817, 817, 817, 817, + /* 330 */ 817, 904, 817, 846, 844, 817, 835, 817, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1025,14 +1028,15 @@ static const char *const yyTokenName[] = { /* 258 */ "expr", /* 259 */ "as", /* 260 */ "tablelist", - /* 261 */ "tmvar", - /* 262 */ "sortlist", - /* 263 */ "sortitem", - /* 264 */ "item", - /* 265 */ "sortorder", - /* 266 */ "grouplist", - /* 267 */ "exprlist", - /* 268 */ "expritem", + /* 261 */ "sub", + /* 262 */ "tmvar", + /* 263 */ "sortlist", + /* 264 */ "sortitem", + /* 265 */ "item", + /* 266 */ "sortorder", + /* 267 */ "grouplist", + /* 268 */ "exprlist", + /* 269 */ "expritem", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -1219,103 +1223,106 @@ static const char *const yyRuleName[] = { /* 176 */ "distinct ::= DISTINCT", /* 177 */ "distinct ::=", /* 178 */ "from ::= FROM tablelist", - /* 179 */ "from ::= FROM LP union RP", - /* 180 */ "tablelist ::= ids cpxName", - /* 181 */ "tablelist ::= ids cpxName ids", - /* 182 */ "tablelist ::= tablelist COMMA ids cpxName", - /* 183 */ "tablelist ::= tablelist COMMA ids cpxName ids", - /* 184 */ "tmvar ::= VARIABLE", - /* 185 */ "interval_opt ::= INTERVAL LP tmvar RP", - /* 186 */ "interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP", - /* 187 */ "interval_opt ::=", - /* 188 */ "session_option ::=", - /* 189 */ "session_option ::= SESSION LP ids cpxName COMMA tmvar RP", - /* 190 */ "fill_opt ::=", - /* 191 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", - /* 192 */ "fill_opt ::= FILL LP ID RP", - /* 193 */ "sliding_opt ::= SLIDING LP tmvar RP", - /* 194 */ "sliding_opt ::=", - /* 195 */ "orderby_opt ::=", - /* 196 */ "orderby_opt ::= ORDER BY sortlist", - /* 197 */ "sortlist ::= sortlist COMMA item sortorder", - /* 198 */ "sortlist ::= item sortorder", - /* 199 */ "item ::= ids cpxName", - /* 200 */ "sortorder ::= ASC", - /* 201 */ "sortorder ::= DESC", - /* 202 */ "sortorder ::=", - /* 203 */ "groupby_opt ::=", - /* 204 */ "groupby_opt ::= GROUP BY grouplist", - /* 205 */ "grouplist ::= grouplist COMMA item", - /* 206 */ "grouplist ::= item", - /* 207 */ "having_opt ::=", - /* 208 */ "having_opt ::= HAVING expr", - /* 209 */ "limit_opt ::=", - /* 210 */ "limit_opt ::= LIMIT signed", - /* 211 */ "limit_opt ::= LIMIT signed OFFSET signed", - /* 212 */ "limit_opt ::= LIMIT signed COMMA signed", - /* 213 */ "slimit_opt ::=", - /* 214 */ "slimit_opt ::= SLIMIT signed", - /* 215 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", - /* 216 */ "slimit_opt ::= SLIMIT signed COMMA signed", - /* 217 */ "where_opt ::=", - /* 218 */ "where_opt ::= WHERE expr", - /* 219 */ "expr ::= LP expr RP", - /* 220 */ "expr ::= ID", - /* 221 */ "expr ::= ID DOT ID", - /* 222 */ "expr ::= ID DOT STAR", - /* 223 */ "expr ::= INTEGER", - /* 224 */ "expr ::= MINUS INTEGER", - /* 225 */ "expr ::= PLUS INTEGER", - /* 226 */ "expr ::= FLOAT", - /* 227 */ "expr ::= MINUS FLOAT", - /* 228 */ "expr ::= PLUS FLOAT", - /* 229 */ "expr ::= STRING", - /* 230 */ "expr ::= NOW", - /* 231 */ "expr ::= VARIABLE", - /* 232 */ "expr ::= PLUS VARIABLE", - /* 233 */ "expr ::= MINUS VARIABLE", - /* 234 */ "expr ::= BOOL", - /* 235 */ "expr ::= NULL", - /* 236 */ "expr ::= ID LP exprlist RP", - /* 237 */ "expr ::= ID LP STAR RP", - /* 238 */ "expr ::= expr IS NULL", - /* 239 */ "expr ::= expr IS NOT NULL", - /* 240 */ "expr ::= expr LT expr", - /* 241 */ "expr ::= expr GT expr", - /* 242 */ "expr ::= expr LE expr", - /* 243 */ "expr ::= expr GE expr", - /* 244 */ "expr ::= expr NE expr", - /* 245 */ "expr ::= expr EQ expr", - /* 246 */ "expr ::= expr BETWEEN expr AND expr", - /* 247 */ "expr ::= expr AND expr", - /* 248 */ "expr ::= expr OR expr", - /* 249 */ "expr ::= expr PLUS expr", - /* 250 */ "expr ::= expr MINUS expr", - /* 251 */ "expr ::= expr STAR expr", - /* 252 */ "expr ::= expr SLASH expr", - /* 253 */ "expr ::= expr REM expr", - /* 254 */ "expr ::= expr LIKE expr", - /* 255 */ "expr ::= expr IN LP exprlist RP", - /* 256 */ "exprlist ::= exprlist COMMA expritem", - /* 257 */ "exprlist ::= expritem", - /* 258 */ "expritem ::= expr", - /* 259 */ "expritem ::=", - /* 260 */ "cmd ::= RESET QUERY CACHE", - /* 261 */ "cmd ::= SYNCDB ids REPLICA", - /* 262 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", - /* 263 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", - /* 264 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", - /* 265 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", - /* 266 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", - /* 267 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", - /* 268 */ "cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist", - /* 269 */ "cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids", - /* 270 */ "cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist", - /* 271 */ "cmd ::= ALTER STABLE ids cpxName DROP TAG ids", - /* 272 */ "cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids", - /* 273 */ "cmd ::= KILL CONNECTION INTEGER", - /* 274 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", - /* 275 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", + /* 179 */ "from ::= FROM sub", + /* 180 */ "sub ::= LP union RP", + /* 181 */ "sub ::= LP union RP ids", + /* 182 */ "sub ::= sub COMMA LP union RP ids", + /* 183 */ "tablelist ::= ids cpxName", + /* 184 */ "tablelist ::= ids cpxName ids", + /* 185 */ "tablelist ::= tablelist COMMA ids cpxName", + /* 186 */ "tablelist ::= tablelist COMMA ids cpxName ids", + /* 187 */ "tmvar ::= VARIABLE", + /* 188 */ "interval_opt ::= INTERVAL LP tmvar RP", + /* 189 */ "interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP", + /* 190 */ "interval_opt ::=", + /* 191 */ "session_option ::=", + /* 192 */ "session_option ::= SESSION LP ids cpxName COMMA tmvar RP", + /* 193 */ "fill_opt ::=", + /* 194 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", + /* 195 */ "fill_opt ::= FILL LP ID RP", + /* 196 */ "sliding_opt ::= SLIDING LP tmvar RP", + /* 197 */ "sliding_opt ::=", + /* 198 */ "orderby_opt ::=", + /* 199 */ "orderby_opt ::= ORDER BY sortlist", + /* 200 */ "sortlist ::= sortlist COMMA item sortorder", + /* 201 */ "sortlist ::= item sortorder", + /* 202 */ "item ::= ids cpxName", + /* 203 */ "sortorder ::= ASC", + /* 204 */ "sortorder ::= DESC", + /* 205 */ "sortorder ::=", + /* 206 */ "groupby_opt ::=", + /* 207 */ "groupby_opt ::= GROUP BY grouplist", + /* 208 */ "grouplist ::= grouplist COMMA item", + /* 209 */ "grouplist ::= item", + /* 210 */ "having_opt ::=", + /* 211 */ "having_opt ::= HAVING expr", + /* 212 */ "limit_opt ::=", + /* 213 */ "limit_opt ::= LIMIT signed", + /* 214 */ "limit_opt ::= LIMIT signed OFFSET signed", + /* 215 */ "limit_opt ::= LIMIT signed COMMA signed", + /* 216 */ "slimit_opt ::=", + /* 217 */ "slimit_opt ::= SLIMIT signed", + /* 218 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", + /* 219 */ "slimit_opt ::= SLIMIT signed COMMA signed", + /* 220 */ "where_opt ::=", + /* 221 */ "where_opt ::= WHERE expr", + /* 222 */ "expr ::= LP expr RP", + /* 223 */ "expr ::= ID", + /* 224 */ "expr ::= ID DOT ID", + /* 225 */ "expr ::= ID DOT STAR", + /* 226 */ "expr ::= INTEGER", + /* 227 */ "expr ::= MINUS INTEGER", + /* 228 */ "expr ::= PLUS INTEGER", + /* 229 */ "expr ::= FLOAT", + /* 230 */ "expr ::= MINUS FLOAT", + /* 231 */ "expr ::= PLUS FLOAT", + /* 232 */ "expr ::= STRING", + /* 233 */ "expr ::= NOW", + /* 234 */ "expr ::= VARIABLE", + /* 235 */ "expr ::= PLUS VARIABLE", + /* 236 */ "expr ::= MINUS VARIABLE", + /* 237 */ "expr ::= BOOL", + /* 238 */ "expr ::= NULL", + /* 239 */ "expr ::= ID LP exprlist RP", + /* 240 */ "expr ::= ID LP STAR RP", + /* 241 */ "expr ::= expr IS NULL", + /* 242 */ "expr ::= expr IS NOT NULL", + /* 243 */ "expr ::= expr LT expr", + /* 244 */ "expr ::= expr GT expr", + /* 245 */ "expr ::= expr LE expr", + /* 246 */ "expr ::= expr GE expr", + /* 247 */ "expr ::= expr NE expr", + /* 248 */ "expr ::= expr EQ expr", + /* 249 */ "expr ::= expr BETWEEN expr AND expr", + /* 250 */ "expr ::= expr AND expr", + /* 251 */ "expr ::= expr OR expr", + /* 252 */ "expr ::= expr PLUS expr", + /* 253 */ "expr ::= expr MINUS expr", + /* 254 */ "expr ::= expr STAR expr", + /* 255 */ "expr ::= expr SLASH expr", + /* 256 */ "expr ::= expr REM expr", + /* 257 */ "expr ::= expr LIKE expr", + /* 258 */ "expr ::= expr IN LP exprlist RP", + /* 259 */ "exprlist ::= exprlist COMMA expritem", + /* 260 */ "exprlist ::= expritem", + /* 261 */ "expritem ::= expr", + /* 262 */ "expritem ::=", + /* 263 */ "cmd ::= RESET QUERY CACHE", + /* 264 */ "cmd ::= SYNCDB ids REPLICA", + /* 265 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", + /* 266 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", + /* 267 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", + /* 268 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", + /* 269 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", + /* 270 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", + /* 271 */ "cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist", + /* 272 */ "cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids", + /* 273 */ "cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist", + /* 274 */ "cmd ::= ALTER STABLE ids cpxName DROP TAG ids", + /* 275 */ "cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids", + /* 276 */ "cmd ::= KILL CONNECTION INTEGER", + /* 277 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", + /* 278 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", }; #endif /* NDEBUG */ @@ -1443,51 +1450,52 @@ static void yy_destructor( case 248: /* fill_opt */ case 250: /* groupby_opt */ case 251: /* orderby_opt */ - case 262: /* sortlist */ - case 266: /* grouplist */ + case 263: /* sortlist */ + case 267: /* grouplist */ { -taosArrayDestroy((yypminor->yy93)); +taosArrayDestroy((yypminor->yy441)); } break; case 236: /* create_table_list */ { -destroyCreateTableSql((yypminor->yy532)); +destroyCreateTableSql((yypminor->yy182)); } break; case 240: /* select */ { -destroySqlNode((yypminor->yy224)); +destroySqlNode((yypminor->yy236)); } break; case 243: /* selcollist */ case 256: /* sclp */ - case 267: /* exprlist */ + case 268: /* exprlist */ { -tSqlExprListDestroy((yypminor->yy93)); +tSqlExprListDestroy((yypminor->yy441)); } break; case 244: /* from */ case 260: /* tablelist */ + case 261: /* sub */ { -destroyRelationInfo((yypminor->yy330)); +destroyRelationInfo((yypminor->yy244)); } break; case 245: /* where_opt */ case 252: /* having_opt */ case 258: /* expr */ - case 268: /* expritem */ + case 269: /* expritem */ { -tSqlExprDestroy((yypminor->yy68)); +tSqlExprDestroy((yypminor->yy166)); } break; case 255: /* union */ { -destroyAllSqlNode((yypminor->yy93)); +destroyAllSqlNode((yypminor->yy441)); } break; - case 263: /* sortitem */ + case 264: /* sortitem */ { -tVariantDestroy(&(yypminor->yy518)); +tVariantDestroy(&(yypminor->yy506)); } break; /********* End destructor definitions *****************************************/ @@ -1960,103 +1968,106 @@ static const struct { { 257, -1 }, /* (176) distinct ::= DISTINCT */ { 257, 0 }, /* (177) distinct ::= */ { 244, -2 }, /* (178) from ::= FROM tablelist */ - { 244, -4 }, /* (179) from ::= FROM LP union RP */ - { 260, -2 }, /* (180) tablelist ::= ids cpxName */ - { 260, -3 }, /* (181) tablelist ::= ids cpxName ids */ - { 260, -4 }, /* (182) tablelist ::= tablelist COMMA ids cpxName */ - { 260, -5 }, /* (183) tablelist ::= tablelist COMMA ids cpxName ids */ - { 261, -1 }, /* (184) tmvar ::= VARIABLE */ - { 246, -4 }, /* (185) interval_opt ::= INTERVAL LP tmvar RP */ - { 246, -6 }, /* (186) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ - { 246, 0 }, /* (187) interval_opt ::= */ - { 247, 0 }, /* (188) session_option ::= */ - { 247, -7 }, /* (189) session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ - { 248, 0 }, /* (190) fill_opt ::= */ - { 248, -6 }, /* (191) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ - { 248, -4 }, /* (192) fill_opt ::= FILL LP ID RP */ - { 249, -4 }, /* (193) sliding_opt ::= SLIDING LP tmvar RP */ - { 249, 0 }, /* (194) sliding_opt ::= */ - { 251, 0 }, /* (195) orderby_opt ::= */ - { 251, -3 }, /* (196) orderby_opt ::= ORDER BY sortlist */ - { 262, -4 }, /* (197) sortlist ::= sortlist COMMA item sortorder */ - { 262, -2 }, /* (198) sortlist ::= item sortorder */ - { 264, -2 }, /* (199) item ::= ids cpxName */ - { 265, -1 }, /* (200) sortorder ::= ASC */ - { 265, -1 }, /* (201) sortorder ::= DESC */ - { 265, 0 }, /* (202) sortorder ::= */ - { 250, 0 }, /* (203) groupby_opt ::= */ - { 250, -3 }, /* (204) groupby_opt ::= GROUP BY grouplist */ - { 266, -3 }, /* (205) grouplist ::= grouplist COMMA item */ - { 266, -1 }, /* (206) grouplist ::= item */ - { 252, 0 }, /* (207) having_opt ::= */ - { 252, -2 }, /* (208) having_opt ::= HAVING expr */ - { 254, 0 }, /* (209) limit_opt ::= */ - { 254, -2 }, /* (210) limit_opt ::= LIMIT signed */ - { 254, -4 }, /* (211) limit_opt ::= LIMIT signed OFFSET signed */ - { 254, -4 }, /* (212) limit_opt ::= LIMIT signed COMMA signed */ - { 253, 0 }, /* (213) slimit_opt ::= */ - { 253, -2 }, /* (214) slimit_opt ::= SLIMIT signed */ - { 253, -4 }, /* (215) slimit_opt ::= SLIMIT signed SOFFSET signed */ - { 253, -4 }, /* (216) slimit_opt ::= SLIMIT signed COMMA signed */ - { 245, 0 }, /* (217) where_opt ::= */ - { 245, -2 }, /* (218) where_opt ::= WHERE expr */ - { 258, -3 }, /* (219) expr ::= LP expr RP */ - { 258, -1 }, /* (220) expr ::= ID */ - { 258, -3 }, /* (221) expr ::= ID DOT ID */ - { 258, -3 }, /* (222) expr ::= ID DOT STAR */ - { 258, -1 }, /* (223) expr ::= INTEGER */ - { 258, -2 }, /* (224) expr ::= MINUS INTEGER */ - { 258, -2 }, /* (225) expr ::= PLUS INTEGER */ - { 258, -1 }, /* (226) expr ::= FLOAT */ - { 258, -2 }, /* (227) expr ::= MINUS FLOAT */ - { 258, -2 }, /* (228) expr ::= PLUS FLOAT */ - { 258, -1 }, /* (229) expr ::= STRING */ - { 258, -1 }, /* (230) expr ::= NOW */ - { 258, -1 }, /* (231) expr ::= VARIABLE */ - { 258, -2 }, /* (232) expr ::= PLUS VARIABLE */ - { 258, -2 }, /* (233) expr ::= MINUS VARIABLE */ - { 258, -1 }, /* (234) expr ::= BOOL */ - { 258, -1 }, /* (235) expr ::= NULL */ - { 258, -4 }, /* (236) expr ::= ID LP exprlist RP */ - { 258, -4 }, /* (237) expr ::= ID LP STAR RP */ - { 258, -3 }, /* (238) expr ::= expr IS NULL */ - { 258, -4 }, /* (239) expr ::= expr IS NOT NULL */ - { 258, -3 }, /* (240) expr ::= expr LT expr */ - { 258, -3 }, /* (241) expr ::= expr GT expr */ - { 258, -3 }, /* (242) expr ::= expr LE expr */ - { 258, -3 }, /* (243) expr ::= expr GE expr */ - { 258, -3 }, /* (244) expr ::= expr NE expr */ - { 258, -3 }, /* (245) expr ::= expr EQ expr */ - { 258, -5 }, /* (246) expr ::= expr BETWEEN expr AND expr */ - { 258, -3 }, /* (247) expr ::= expr AND expr */ - { 258, -3 }, /* (248) expr ::= expr OR expr */ - { 258, -3 }, /* (249) expr ::= expr PLUS expr */ - { 258, -3 }, /* (250) expr ::= expr MINUS expr */ - { 258, -3 }, /* (251) expr ::= expr STAR expr */ - { 258, -3 }, /* (252) expr ::= expr SLASH expr */ - { 258, -3 }, /* (253) expr ::= expr REM expr */ - { 258, -3 }, /* (254) expr ::= expr LIKE expr */ - { 258, -5 }, /* (255) expr ::= expr IN LP exprlist RP */ - { 267, -3 }, /* (256) exprlist ::= exprlist COMMA expritem */ - { 267, -1 }, /* (257) exprlist ::= expritem */ - { 268, -1 }, /* (258) expritem ::= expr */ - { 268, 0 }, /* (259) expritem ::= */ - { 194, -3 }, /* (260) cmd ::= RESET QUERY CACHE */ - { 194, -3 }, /* (261) cmd ::= SYNCDB ids REPLICA */ - { 194, -7 }, /* (262) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - { 194, -7 }, /* (263) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - { 194, -7 }, /* (264) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - { 194, -7 }, /* (265) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - { 194, -8 }, /* (266) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - { 194, -9 }, /* (267) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - { 194, -7 }, /* (268) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ - { 194, -7 }, /* (269) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ - { 194, -7 }, /* (270) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ - { 194, -7 }, /* (271) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ - { 194, -8 }, /* (272) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ - { 194, -3 }, /* (273) cmd ::= KILL CONNECTION INTEGER */ - { 194, -5 }, /* (274) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - { 194, -5 }, /* (275) cmd ::= KILL QUERY INTEGER COLON INTEGER */ + { 244, -2 }, /* (179) from ::= FROM sub */ + { 261, -3 }, /* (180) sub ::= LP union RP */ + { 261, -4 }, /* (181) sub ::= LP union RP ids */ + { 261, -6 }, /* (182) sub ::= sub COMMA LP union RP ids */ + { 260, -2 }, /* (183) tablelist ::= ids cpxName */ + { 260, -3 }, /* (184) tablelist ::= ids cpxName ids */ + { 260, -4 }, /* (185) tablelist ::= tablelist COMMA ids cpxName */ + { 260, -5 }, /* (186) tablelist ::= tablelist COMMA ids cpxName ids */ + { 262, -1 }, /* (187) tmvar ::= VARIABLE */ + { 246, -4 }, /* (188) interval_opt ::= INTERVAL LP tmvar RP */ + { 246, -6 }, /* (189) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ + { 246, 0 }, /* (190) interval_opt ::= */ + { 247, 0 }, /* (191) session_option ::= */ + { 247, -7 }, /* (192) session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ + { 248, 0 }, /* (193) fill_opt ::= */ + { 248, -6 }, /* (194) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + { 248, -4 }, /* (195) fill_opt ::= FILL LP ID RP */ + { 249, -4 }, /* (196) sliding_opt ::= SLIDING LP tmvar RP */ + { 249, 0 }, /* (197) sliding_opt ::= */ + { 251, 0 }, /* (198) orderby_opt ::= */ + { 251, -3 }, /* (199) orderby_opt ::= ORDER BY sortlist */ + { 263, -4 }, /* (200) sortlist ::= sortlist COMMA item sortorder */ + { 263, -2 }, /* (201) sortlist ::= item sortorder */ + { 265, -2 }, /* (202) item ::= ids cpxName */ + { 266, -1 }, /* (203) sortorder ::= ASC */ + { 266, -1 }, /* (204) sortorder ::= DESC */ + { 266, 0 }, /* (205) sortorder ::= */ + { 250, 0 }, /* (206) groupby_opt ::= */ + { 250, -3 }, /* (207) groupby_opt ::= GROUP BY grouplist */ + { 267, -3 }, /* (208) grouplist ::= grouplist COMMA item */ + { 267, -1 }, /* (209) grouplist ::= item */ + { 252, 0 }, /* (210) having_opt ::= */ + { 252, -2 }, /* (211) having_opt ::= HAVING expr */ + { 254, 0 }, /* (212) limit_opt ::= */ + { 254, -2 }, /* (213) limit_opt ::= LIMIT signed */ + { 254, -4 }, /* (214) limit_opt ::= LIMIT signed OFFSET signed */ + { 254, -4 }, /* (215) limit_opt ::= LIMIT signed COMMA signed */ + { 253, 0 }, /* (216) slimit_opt ::= */ + { 253, -2 }, /* (217) slimit_opt ::= SLIMIT signed */ + { 253, -4 }, /* (218) slimit_opt ::= SLIMIT signed SOFFSET signed */ + { 253, -4 }, /* (219) slimit_opt ::= SLIMIT signed COMMA signed */ + { 245, 0 }, /* (220) where_opt ::= */ + { 245, -2 }, /* (221) where_opt ::= WHERE expr */ + { 258, -3 }, /* (222) expr ::= LP expr RP */ + { 258, -1 }, /* (223) expr ::= ID */ + { 258, -3 }, /* (224) expr ::= ID DOT ID */ + { 258, -3 }, /* (225) expr ::= ID DOT STAR */ + { 258, -1 }, /* (226) expr ::= INTEGER */ + { 258, -2 }, /* (227) expr ::= MINUS INTEGER */ + { 258, -2 }, /* (228) expr ::= PLUS INTEGER */ + { 258, -1 }, /* (229) expr ::= FLOAT */ + { 258, -2 }, /* (230) expr ::= MINUS FLOAT */ + { 258, -2 }, /* (231) expr ::= PLUS FLOAT */ + { 258, -1 }, /* (232) expr ::= STRING */ + { 258, -1 }, /* (233) expr ::= NOW */ + { 258, -1 }, /* (234) expr ::= VARIABLE */ + { 258, -2 }, /* (235) expr ::= PLUS VARIABLE */ + { 258, -2 }, /* (236) expr ::= MINUS VARIABLE */ + { 258, -1 }, /* (237) expr ::= BOOL */ + { 258, -1 }, /* (238) expr ::= NULL */ + { 258, -4 }, /* (239) expr ::= ID LP exprlist RP */ + { 258, -4 }, /* (240) expr ::= ID LP STAR RP */ + { 258, -3 }, /* (241) expr ::= expr IS NULL */ + { 258, -4 }, /* (242) expr ::= expr IS NOT NULL */ + { 258, -3 }, /* (243) expr ::= expr LT expr */ + { 258, -3 }, /* (244) expr ::= expr GT expr */ + { 258, -3 }, /* (245) expr ::= expr LE expr */ + { 258, -3 }, /* (246) expr ::= expr GE expr */ + { 258, -3 }, /* (247) expr ::= expr NE expr */ + { 258, -3 }, /* (248) expr ::= expr EQ expr */ + { 258, -5 }, /* (249) expr ::= expr BETWEEN expr AND expr */ + { 258, -3 }, /* (250) expr ::= expr AND expr */ + { 258, -3 }, /* (251) expr ::= expr OR expr */ + { 258, -3 }, /* (252) expr ::= expr PLUS expr */ + { 258, -3 }, /* (253) expr ::= expr MINUS expr */ + { 258, -3 }, /* (254) expr ::= expr STAR expr */ + { 258, -3 }, /* (255) expr ::= expr SLASH expr */ + { 258, -3 }, /* (256) expr ::= expr REM expr */ + { 258, -3 }, /* (257) expr ::= expr LIKE expr */ + { 258, -5 }, /* (258) expr ::= expr IN LP exprlist RP */ + { 268, -3 }, /* (259) exprlist ::= exprlist COMMA expritem */ + { 268, -1 }, /* (260) exprlist ::= expritem */ + { 269, -1 }, /* (261) expritem ::= expr */ + { 269, 0 }, /* (262) expritem ::= */ + { 194, -3 }, /* (263) cmd ::= RESET QUERY CACHE */ + { 194, -3 }, /* (264) cmd ::= SYNCDB ids REPLICA */ + { 194, -7 }, /* (265) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + { 194, -7 }, /* (266) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + { 194, -7 }, /* (267) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + { 194, -7 }, /* (268) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + { 194, -8 }, /* (269) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + { 194, -9 }, /* (270) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + { 194, -7 }, /* (271) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ + { 194, -7 }, /* (272) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ + { 194, -7 }, /* (273) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ + { 194, -7 }, /* (274) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ + { 194, -8 }, /* (275) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ + { 194, -3 }, /* (276) cmd ::= KILL CONNECTION INTEGER */ + { 194, -5 }, /* (277) cmd ::= KILL STREAM INTEGER COLON INTEGER */ + { 194, -5 }, /* (278) cmd ::= KILL QUERY INTEGER COLON INTEGER */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -2315,13 +2326,13 @@ static void yy_reduce( break; case 46: /* cmd ::= ALTER DATABASE ids alter_db_optr */ case 47: /* cmd ::= ALTER TOPIC ids alter_topic_optr */ yytestcase(yyruleno==47); -{ SStrToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy372, &t);} +{ SStrToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy382, &t);} break; case 48: /* cmd ::= ALTER ACCOUNT ids acct_optr */ -{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy77);} +{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy151);} break; case 49: /* cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ -{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy77);} +{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy151);} break; case 50: /* ids ::= ID */ case 51: /* ids ::= STRING */ yytestcase(yyruleno==51); @@ -2343,17 +2354,17 @@ static void yy_reduce( { setDCLSqlElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &yymsp[0].minor.yy0);} break; case 57: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ -{ setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy77);} +{ setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy151);} break; case 58: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */ case 59: /* cmd ::= CREATE TOPIC ifnotexists ids topic_optr */ yytestcase(yyruleno==59); -{ setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy372, &yymsp[-2].minor.yy0);} +{ setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy382, &yymsp[-2].minor.yy0);} break; case 60: /* cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ -{ setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &yymsp[-5].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy325, &yymsp[0].minor.yy0, 1);} +{ setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &yymsp[-5].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy343, &yymsp[0].minor.yy0, 1);} break; case 61: /* cmd ::= CREATE AGGREGATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ -{ setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &yymsp[-5].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy325, &yymsp[0].minor.yy0, 2);} +{ setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &yymsp[-5].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy343, &yymsp[0].minor.yy0, 2);} break; case 62: /* cmd ::= CREATE USER ids PASS ids */ { setCreateUserSql(pInfo, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);} @@ -2384,20 +2395,20 @@ static void yy_reduce( break; case 83: /* acct_optr ::= pps tseries storage streams qtime dbs users conns state */ { - yylhsminor.yy77.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; - yylhsminor.yy77.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; - yylhsminor.yy77.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; - yylhsminor.yy77.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; - yylhsminor.yy77.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; - yylhsminor.yy77.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy77.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy77.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; - yylhsminor.yy77.stat = yymsp[0].minor.yy0; + yylhsminor.yy151.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; + yylhsminor.yy151.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; + yylhsminor.yy151.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; + yylhsminor.yy151.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; + yylhsminor.yy151.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; + yylhsminor.yy151.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; + yylhsminor.yy151.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; + yylhsminor.yy151.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; + yylhsminor.yy151.stat = yymsp[0].minor.yy0; } - yymsp[-8].minor.yy77 = yylhsminor.yy77; + yymsp[-8].minor.yy151 = yylhsminor.yy151; break; case 84: /* keep ::= KEEP tagitemlist */ -{ yymsp[-1].minor.yy93 = yymsp[0].minor.yy93; } +{ yymsp[-1].minor.yy441 = yymsp[0].minor.yy441; } break; case 85: /* cache ::= CACHE INTEGER */ case 86: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==86); @@ -2417,234 +2428,234 @@ static void yy_reduce( { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } break; case 100: /* db_optr ::= */ -{setDefaultCreateDbOption(&yymsp[1].minor.yy372); yymsp[1].minor.yy372.dbType = TSDB_DB_TYPE_DEFAULT;} +{setDefaultCreateDbOption(&yymsp[1].minor.yy382); yymsp[1].minor.yy382.dbType = TSDB_DB_TYPE_DEFAULT;} break; case 101: /* db_optr ::= db_optr cache */ -{ yylhsminor.yy372 = yymsp[-1].minor.yy372; yylhsminor.yy372.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy372 = yylhsminor.yy372; +{ yylhsminor.yy382 = yymsp[-1].minor.yy382; yylhsminor.yy382.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy382 = yylhsminor.yy382; break; case 102: /* db_optr ::= db_optr replica */ case 119: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==119); -{ yylhsminor.yy372 = yymsp[-1].minor.yy372; yylhsminor.yy372.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy372 = yylhsminor.yy372; +{ yylhsminor.yy382 = yymsp[-1].minor.yy382; yylhsminor.yy382.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy382 = yylhsminor.yy382; break; case 103: /* db_optr ::= db_optr quorum */ case 120: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==120); -{ yylhsminor.yy372 = yymsp[-1].minor.yy372; yylhsminor.yy372.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy372 = yylhsminor.yy372; +{ yylhsminor.yy382 = yymsp[-1].minor.yy382; yylhsminor.yy382.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy382 = yylhsminor.yy382; break; case 104: /* db_optr ::= db_optr days */ -{ yylhsminor.yy372 = yymsp[-1].minor.yy372; yylhsminor.yy372.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy372 = yylhsminor.yy372; +{ yylhsminor.yy382 = yymsp[-1].minor.yy382; yylhsminor.yy382.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy382 = yylhsminor.yy382; break; case 105: /* db_optr ::= db_optr minrows */ -{ yylhsminor.yy372 = yymsp[-1].minor.yy372; yylhsminor.yy372.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } - yymsp[-1].minor.yy372 = yylhsminor.yy372; +{ yylhsminor.yy382 = yymsp[-1].minor.yy382; yylhsminor.yy382.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } + yymsp[-1].minor.yy382 = yylhsminor.yy382; break; case 106: /* db_optr ::= db_optr maxrows */ -{ yylhsminor.yy372 = yymsp[-1].minor.yy372; yylhsminor.yy372.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } - yymsp[-1].minor.yy372 = yylhsminor.yy372; +{ yylhsminor.yy382 = yymsp[-1].minor.yy382; yylhsminor.yy382.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } + yymsp[-1].minor.yy382 = yylhsminor.yy382; break; case 107: /* db_optr ::= db_optr blocks */ case 122: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==122); -{ yylhsminor.yy372 = yymsp[-1].minor.yy372; yylhsminor.yy372.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy372 = yylhsminor.yy372; +{ yylhsminor.yy382 = yymsp[-1].minor.yy382; yylhsminor.yy382.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy382 = yylhsminor.yy382; break; case 108: /* db_optr ::= db_optr ctime */ -{ yylhsminor.yy372 = yymsp[-1].minor.yy372; yylhsminor.yy372.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy372 = yylhsminor.yy372; +{ yylhsminor.yy382 = yymsp[-1].minor.yy382; yylhsminor.yy382.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy382 = yylhsminor.yy382; break; case 109: /* db_optr ::= db_optr wal */ case 124: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==124); -{ yylhsminor.yy372 = yymsp[-1].minor.yy372; yylhsminor.yy372.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy372 = yylhsminor.yy372; +{ yylhsminor.yy382 = yymsp[-1].minor.yy382; yylhsminor.yy382.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy382 = yylhsminor.yy382; break; case 110: /* db_optr ::= db_optr fsync */ case 125: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==125); -{ yylhsminor.yy372 = yymsp[-1].minor.yy372; yylhsminor.yy372.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy372 = yylhsminor.yy372; +{ yylhsminor.yy382 = yymsp[-1].minor.yy382; yylhsminor.yy382.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy382 = yylhsminor.yy382; break; case 111: /* db_optr ::= db_optr comp */ case 123: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==123); -{ yylhsminor.yy372 = yymsp[-1].minor.yy372; yylhsminor.yy372.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy372 = yylhsminor.yy372; +{ yylhsminor.yy382 = yymsp[-1].minor.yy382; yylhsminor.yy382.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy382 = yylhsminor.yy382; break; case 112: /* db_optr ::= db_optr prec */ -{ yylhsminor.yy372 = yymsp[-1].minor.yy372; yylhsminor.yy372.precision = yymsp[0].minor.yy0; } - yymsp[-1].minor.yy372 = yylhsminor.yy372; +{ yylhsminor.yy382 = yymsp[-1].minor.yy382; yylhsminor.yy382.precision = yymsp[0].minor.yy0; } + yymsp[-1].minor.yy382 = yylhsminor.yy382; break; case 113: /* db_optr ::= db_optr keep */ case 121: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==121); -{ yylhsminor.yy372 = yymsp[-1].minor.yy372; yylhsminor.yy372.keep = yymsp[0].minor.yy93; } - yymsp[-1].minor.yy372 = yylhsminor.yy372; +{ yylhsminor.yy382 = yymsp[-1].minor.yy382; yylhsminor.yy382.keep = yymsp[0].minor.yy441; } + yymsp[-1].minor.yy382 = yylhsminor.yy382; break; case 114: /* db_optr ::= db_optr update */ case 126: /* alter_db_optr ::= alter_db_optr update */ yytestcase(yyruleno==126); -{ yylhsminor.yy372 = yymsp[-1].minor.yy372; yylhsminor.yy372.update = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy372 = yylhsminor.yy372; +{ yylhsminor.yy382 = yymsp[-1].minor.yy382; yylhsminor.yy382.update = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy382 = yylhsminor.yy382; break; case 115: /* db_optr ::= db_optr cachelast */ case 127: /* alter_db_optr ::= alter_db_optr cachelast */ yytestcase(yyruleno==127); -{ yylhsminor.yy372 = yymsp[-1].minor.yy372; yylhsminor.yy372.cachelast = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy372 = yylhsminor.yy372; +{ yylhsminor.yy382 = yymsp[-1].minor.yy382; yylhsminor.yy382.cachelast = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy382 = yylhsminor.yy382; break; case 116: /* topic_optr ::= db_optr */ case 128: /* alter_topic_optr ::= alter_db_optr */ yytestcase(yyruleno==128); -{ yylhsminor.yy372 = yymsp[0].minor.yy372; yylhsminor.yy372.dbType = TSDB_DB_TYPE_TOPIC; } - yymsp[0].minor.yy372 = yylhsminor.yy372; +{ yylhsminor.yy382 = yymsp[0].minor.yy382; yylhsminor.yy382.dbType = TSDB_DB_TYPE_TOPIC; } + yymsp[0].minor.yy382 = yylhsminor.yy382; break; case 117: /* topic_optr ::= topic_optr partitions */ case 129: /* alter_topic_optr ::= alter_topic_optr partitions */ yytestcase(yyruleno==129); -{ yylhsminor.yy372 = yymsp[-1].minor.yy372; yylhsminor.yy372.partitions = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy372 = yylhsminor.yy372; +{ yylhsminor.yy382 = yymsp[-1].minor.yy382; yylhsminor.yy382.partitions = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy382 = yylhsminor.yy382; break; case 118: /* alter_db_optr ::= */ -{ setDefaultCreateDbOption(&yymsp[1].minor.yy372); yymsp[1].minor.yy372.dbType = TSDB_DB_TYPE_DEFAULT;} +{ setDefaultCreateDbOption(&yymsp[1].minor.yy382); yymsp[1].minor.yy382.dbType = TSDB_DB_TYPE_DEFAULT;} break; case 130: /* typename ::= ids */ { yymsp[0].minor.yy0.type = 0; - tSetColumnType (&yylhsminor.yy325, &yymsp[0].minor.yy0); + tSetColumnType (&yylhsminor.yy343, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy325 = yylhsminor.yy325; + yymsp[0].minor.yy343 = yylhsminor.yy343; break; case 131: /* typename ::= ids LP signed RP */ { - if (yymsp[-1].minor.yy279 <= 0) { + if (yymsp[-1].minor.yy369 <= 0) { yymsp[-3].minor.yy0.type = 0; - tSetColumnType(&yylhsminor.yy325, &yymsp[-3].minor.yy0); + tSetColumnType(&yylhsminor.yy343, &yymsp[-3].minor.yy0); } else { - yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy279; // negative value of name length - tSetColumnType(&yylhsminor.yy325, &yymsp[-3].minor.yy0); + yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy369; // negative value of name length + tSetColumnType(&yylhsminor.yy343, &yymsp[-3].minor.yy0); } } - yymsp[-3].minor.yy325 = yylhsminor.yy325; + yymsp[-3].minor.yy343 = yylhsminor.yy343; break; case 132: /* typename ::= ids UNSIGNED */ { yymsp[-1].minor.yy0.type = 0; yymsp[-1].minor.yy0.n = ((yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z); - tSetColumnType (&yylhsminor.yy325, &yymsp[-1].minor.yy0); + tSetColumnType (&yylhsminor.yy343, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy325 = yylhsminor.yy325; + yymsp[-1].minor.yy343 = yylhsminor.yy343; break; case 133: /* signed ::= INTEGER */ -{ yylhsminor.yy279 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[0].minor.yy279 = yylhsminor.yy279; +{ yylhsminor.yy369 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[0].minor.yy369 = yylhsminor.yy369; break; case 134: /* signed ::= PLUS INTEGER */ -{ yymsp[-1].minor.yy279 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } +{ yymsp[-1].minor.yy369 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; case 135: /* signed ::= MINUS INTEGER */ -{ yymsp[-1].minor.yy279 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} +{ yymsp[-1].minor.yy369 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} break; case 139: /* cmd ::= CREATE TABLE create_table_list */ -{ pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = yymsp[0].minor.yy532;} +{ pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = yymsp[0].minor.yy182;} break; case 140: /* create_table_list ::= create_from_stable */ { SCreateTableSql* pCreateTable = calloc(1, sizeof(SCreateTableSql)); pCreateTable->childTableInfo = taosArrayInit(4, sizeof(SCreatedTableInfo)); - taosArrayPush(pCreateTable->childTableInfo, &yymsp[0].minor.yy528); + taosArrayPush(pCreateTable->childTableInfo, &yymsp[0].minor.yy456); pCreateTable->type = TSQL_CREATE_TABLE_FROM_STABLE; - yylhsminor.yy532 = pCreateTable; + yylhsminor.yy182 = pCreateTable; } - yymsp[0].minor.yy532 = yylhsminor.yy532; + yymsp[0].minor.yy182 = yylhsminor.yy182; break; case 141: /* create_table_list ::= create_table_list create_from_stable */ { - taosArrayPush(yymsp[-1].minor.yy532->childTableInfo, &yymsp[0].minor.yy528); - yylhsminor.yy532 = yymsp[-1].minor.yy532; + taosArrayPush(yymsp[-1].minor.yy182->childTableInfo, &yymsp[0].minor.yy456); + yylhsminor.yy182 = yymsp[-1].minor.yy182; } - yymsp[-1].minor.yy532 = yylhsminor.yy532; + yymsp[-1].minor.yy182 = yylhsminor.yy182; break; case 142: /* create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ { - yylhsminor.yy532 = tSetCreateTableInfo(yymsp[-1].minor.yy93, NULL, NULL, TSQL_CREATE_TABLE); - setSqlInfo(pInfo, yylhsminor.yy532, NULL, TSDB_SQL_CREATE_TABLE); + yylhsminor.yy182 = tSetCreateTableInfo(yymsp[-1].minor.yy441, NULL, NULL, TSQL_CREATE_TABLE); + setSqlInfo(pInfo, yylhsminor.yy182, NULL, TSDB_SQL_CREATE_TABLE); yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; setCreatedTableName(pInfo, &yymsp[-4].minor.yy0, &yymsp[-5].minor.yy0); } - yymsp[-5].minor.yy532 = yylhsminor.yy532; + yymsp[-5].minor.yy182 = yylhsminor.yy182; break; case 143: /* create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ { - yylhsminor.yy532 = tSetCreateTableInfo(yymsp[-5].minor.yy93, yymsp[-1].minor.yy93, NULL, TSQL_CREATE_STABLE); - setSqlInfo(pInfo, yylhsminor.yy532, NULL, TSDB_SQL_CREATE_TABLE); + yylhsminor.yy182 = tSetCreateTableInfo(yymsp[-5].minor.yy441, yymsp[-1].minor.yy441, NULL, TSQL_CREATE_STABLE); + setSqlInfo(pInfo, yylhsminor.yy182, NULL, TSDB_SQL_CREATE_TABLE); yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; setCreatedTableName(pInfo, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); } - yymsp[-9].minor.yy532 = yylhsminor.yy532; + yymsp[-9].minor.yy182 = yylhsminor.yy182; break; case 144: /* create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; - yylhsminor.yy528 = createNewChildTableInfo(&yymsp[-5].minor.yy0, NULL, yymsp[-1].minor.yy93, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); + yylhsminor.yy456 = createNewChildTableInfo(&yymsp[-5].minor.yy0, NULL, yymsp[-1].minor.yy441, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); } - yymsp[-9].minor.yy528 = yylhsminor.yy528; + yymsp[-9].minor.yy456 = yylhsminor.yy456; break; case 145: /* create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP */ { yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; yymsp[-11].minor.yy0.n += yymsp[-10].minor.yy0.n; - yylhsminor.yy528 = createNewChildTableInfo(&yymsp[-8].minor.yy0, yymsp[-5].minor.yy93, yymsp[-1].minor.yy93, &yymsp[-11].minor.yy0, &yymsp[-12].minor.yy0); + yylhsminor.yy456 = createNewChildTableInfo(&yymsp[-8].minor.yy0, yymsp[-5].minor.yy441, yymsp[-1].minor.yy441, &yymsp[-11].minor.yy0, &yymsp[-12].minor.yy0); } - yymsp[-12].minor.yy528 = yylhsminor.yy528; + yymsp[-12].minor.yy456 = yylhsminor.yy456; break; case 146: /* tagNamelist ::= tagNamelist COMMA ids */ -{taosArrayPush(yymsp[-2].minor.yy93, &yymsp[0].minor.yy0); yylhsminor.yy93 = yymsp[-2].minor.yy93; } - yymsp[-2].minor.yy93 = yylhsminor.yy93; +{taosArrayPush(yymsp[-2].minor.yy441, &yymsp[0].minor.yy0); yylhsminor.yy441 = yymsp[-2].minor.yy441; } + yymsp[-2].minor.yy441 = yylhsminor.yy441; break; case 147: /* tagNamelist ::= ids */ -{yylhsminor.yy93 = taosArrayInit(4, sizeof(SStrToken)); taosArrayPush(yylhsminor.yy93, &yymsp[0].minor.yy0);} - yymsp[0].minor.yy93 = yylhsminor.yy93; +{yylhsminor.yy441 = taosArrayInit(4, sizeof(SStrToken)); taosArrayPush(yylhsminor.yy441, &yymsp[0].minor.yy0);} + yymsp[0].minor.yy441 = yylhsminor.yy441; break; case 148: /* create_table_args ::= ifnotexists ids cpxName AS select */ { - yylhsminor.yy532 = tSetCreateTableInfo(NULL, NULL, yymsp[0].minor.yy224, TSQL_CREATE_STREAM); - setSqlInfo(pInfo, yylhsminor.yy532, NULL, TSDB_SQL_CREATE_TABLE); + yylhsminor.yy182 = tSetCreateTableInfo(NULL, NULL, yymsp[0].minor.yy236, TSQL_CREATE_STREAM); + setSqlInfo(pInfo, yylhsminor.yy182, NULL, TSDB_SQL_CREATE_TABLE); yymsp[-3].minor.yy0.n += yymsp[-2].minor.yy0.n; setCreatedTableName(pInfo, &yymsp[-3].minor.yy0, &yymsp[-4].minor.yy0); } - yymsp[-4].minor.yy532 = yylhsminor.yy532; + yymsp[-4].minor.yy182 = yylhsminor.yy182; break; case 149: /* columnlist ::= columnlist COMMA column */ -{taosArrayPush(yymsp[-2].minor.yy93, &yymsp[0].minor.yy325); yylhsminor.yy93 = yymsp[-2].minor.yy93; } - yymsp[-2].minor.yy93 = yylhsminor.yy93; +{taosArrayPush(yymsp[-2].minor.yy441, &yymsp[0].minor.yy343); yylhsminor.yy441 = yymsp[-2].minor.yy441; } + yymsp[-2].minor.yy441 = yylhsminor.yy441; break; case 150: /* columnlist ::= column */ -{yylhsminor.yy93 = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(yylhsminor.yy93, &yymsp[0].minor.yy325);} - yymsp[0].minor.yy93 = yylhsminor.yy93; +{yylhsminor.yy441 = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(yylhsminor.yy441, &yymsp[0].minor.yy343);} + yymsp[0].minor.yy441 = yylhsminor.yy441; break; case 151: /* column ::= ids typename */ { - tSetColumnInfo(&yylhsminor.yy325, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy325); + tSetColumnInfo(&yylhsminor.yy343, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy343); } - yymsp[-1].minor.yy325 = yylhsminor.yy325; + yymsp[-1].minor.yy343 = yylhsminor.yy343; break; case 152: /* tagitemlist ::= tagitemlist COMMA tagitem */ -{ yylhsminor.yy93 = tVariantListAppend(yymsp[-2].minor.yy93, &yymsp[0].minor.yy518, -1); } - yymsp[-2].minor.yy93 = yylhsminor.yy93; +{ yylhsminor.yy441 = tVariantListAppend(yymsp[-2].minor.yy441, &yymsp[0].minor.yy506, -1); } + yymsp[-2].minor.yy441 = yylhsminor.yy441; break; case 153: /* tagitemlist ::= tagitem */ -{ yylhsminor.yy93 = tVariantListAppend(NULL, &yymsp[0].minor.yy518, -1); } - yymsp[0].minor.yy93 = yylhsminor.yy93; +{ yylhsminor.yy441 = tVariantListAppend(NULL, &yymsp[0].minor.yy506, -1); } + yymsp[0].minor.yy441 = yylhsminor.yy441; break; case 154: /* tagitem ::= INTEGER */ case 155: /* tagitem ::= FLOAT */ yytestcase(yyruleno==155); case 156: /* tagitem ::= STRING */ yytestcase(yyruleno==156); case 157: /* tagitem ::= BOOL */ yytestcase(yyruleno==157); -{ toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy518, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy518 = yylhsminor.yy518; +{ toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy506, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy506 = yylhsminor.yy506; break; case 158: /* tagitem ::= NULL */ -{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy518, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy518 = yylhsminor.yy518; +{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy506, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy506 = yylhsminor.yy506; break; case 159: /* tagitem ::= MINUS INTEGER */ case 160: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==160); @@ -2654,56 +2665,56 @@ static void yy_reduce( yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = yymsp[0].minor.yy0.type; toTSDBType(yymsp[-1].minor.yy0.type); - tVariantCreate(&yylhsminor.yy518, &yymsp[-1].minor.yy0); + tVariantCreate(&yylhsminor.yy506, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy518 = yylhsminor.yy518; + yymsp[-1].minor.yy506 = yylhsminor.yy506; break; case 163: /* select ::= SELECT selcollist from where_opt interval_opt session_option fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ { - yylhsminor.yy224 = tSetQuerySqlNode(&yymsp[-12].minor.yy0, yymsp[-11].minor.yy93, yymsp[-10].minor.yy330, yymsp[-9].minor.yy68, yymsp[-4].minor.yy93, yymsp[-3].minor.yy93, &yymsp[-8].minor.yy42, &yymsp[-7].minor.yy15, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy93, &yymsp[0].minor.yy284, &yymsp[-1].minor.yy284, yymsp[-2].minor.yy68); + yylhsminor.yy236 = tSetQuerySqlNode(&yymsp[-12].minor.yy0, yymsp[-11].minor.yy441, yymsp[-10].minor.yy244, yymsp[-9].minor.yy166, yymsp[-4].minor.yy441, yymsp[-3].minor.yy441, &yymsp[-8].minor.yy340, &yymsp[-7].minor.yy259, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy441, &yymsp[0].minor.yy414, &yymsp[-1].minor.yy414, yymsp[-2].minor.yy166); } - yymsp[-12].minor.yy224 = yylhsminor.yy224; + yymsp[-12].minor.yy236 = yylhsminor.yy236; break; case 164: /* select ::= LP select RP */ -{yymsp[-2].minor.yy224 = yymsp[-1].minor.yy224;} +{yymsp[-2].minor.yy236 = yymsp[-1].minor.yy236;} break; case 165: /* union ::= select */ -{ yylhsminor.yy93 = setSubclause(NULL, yymsp[0].minor.yy224); } - yymsp[0].minor.yy93 = yylhsminor.yy93; +{ yylhsminor.yy441 = setSubclause(NULL, yymsp[0].minor.yy236); } + yymsp[0].minor.yy441 = yylhsminor.yy441; break; case 166: /* union ::= union UNION ALL select */ -{ yylhsminor.yy93 = appendSelectClause(yymsp[-3].minor.yy93, yymsp[0].minor.yy224); } - yymsp[-3].minor.yy93 = yylhsminor.yy93; +{ yylhsminor.yy441 = appendSelectClause(yymsp[-3].minor.yy441, yymsp[0].minor.yy236); } + yymsp[-3].minor.yy441 = yylhsminor.yy441; break; case 167: /* cmd ::= union */ -{ setSqlInfo(pInfo, yymsp[0].minor.yy93, NULL, TSDB_SQL_SELECT); } +{ setSqlInfo(pInfo, yymsp[0].minor.yy441, NULL, TSDB_SQL_SELECT); } break; case 168: /* select ::= SELECT selcollist */ { - yylhsminor.yy224 = tSetQuerySqlNode(&yymsp[-1].minor.yy0, yymsp[0].minor.yy93, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); + yylhsminor.yy236 = tSetQuerySqlNode(&yymsp[-1].minor.yy0, yymsp[0].minor.yy441, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } - yymsp[-1].minor.yy224 = yylhsminor.yy224; + yymsp[-1].minor.yy236 = yylhsminor.yy236; break; case 169: /* sclp ::= selcollist COMMA */ -{yylhsminor.yy93 = yymsp[-1].minor.yy93;} - yymsp[-1].minor.yy93 = yylhsminor.yy93; +{yylhsminor.yy441 = yymsp[-1].minor.yy441;} + yymsp[-1].minor.yy441 = yylhsminor.yy441; break; case 170: /* sclp ::= */ - case 195: /* orderby_opt ::= */ yytestcase(yyruleno==195); -{yymsp[1].minor.yy93 = 0;} + case 198: /* orderby_opt ::= */ yytestcase(yyruleno==198); +{yymsp[1].minor.yy441 = 0;} break; case 171: /* selcollist ::= sclp distinct expr as */ { - yylhsminor.yy93 = tSqlExprListAppend(yymsp[-3].minor.yy93, yymsp[-1].minor.yy68, yymsp[-2].minor.yy0.n? &yymsp[-2].minor.yy0:0, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); + yylhsminor.yy441 = tSqlExprListAppend(yymsp[-3].minor.yy441, yymsp[-1].minor.yy166, yymsp[-2].minor.yy0.n? &yymsp[-2].minor.yy0:0, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); } - yymsp[-3].minor.yy93 = yylhsminor.yy93; + yymsp[-3].minor.yy441 = yylhsminor.yy441; break; case 172: /* selcollist ::= sclp STAR */ { tSqlExpr *pNode = tSqlExprCreateIdValue(NULL, TK_ALL); - yylhsminor.yy93 = tSqlExprListAppend(yymsp[-1].minor.yy93, pNode, 0, 0); + yylhsminor.yy441 = tSqlExprListAppend(yymsp[-1].minor.yy441, pNode, 0, 0); } - yymsp[-1].minor.yy93 = yylhsminor.yy93; + yymsp[-1].minor.yy441 = yylhsminor.yy441; break; case 173: /* as ::= AS ids */ { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } @@ -2720,332 +2731,340 @@ static void yy_reduce( yymsp[0].minor.yy0 = yylhsminor.yy0; break; case 178: /* from ::= FROM tablelist */ -{yymsp[-1].minor.yy330 = yymsp[0].minor.yy330;} + case 179: /* from ::= FROM sub */ yytestcase(yyruleno==179); +{yymsp[-1].minor.yy244 = yymsp[0].minor.yy244;} break; - case 179: /* from ::= FROM LP union RP */ -{yymsp[-3].minor.yy330 = setSubquery(NULL, yymsp[-1].minor.yy93);} + case 180: /* sub ::= LP union RP */ +{yymsp[-2].minor.yy244 = addSubqueryElem(NULL, yymsp[-1].minor.yy441, NULL);} break; - case 180: /* tablelist ::= ids cpxName */ + case 181: /* sub ::= LP union RP ids */ +{yymsp[-3].minor.yy244 = addSubqueryElem(NULL, yymsp[-2].minor.yy441, &yymsp[0].minor.yy0);} + break; + case 182: /* sub ::= sub COMMA LP union RP ids */ +{yylhsminor.yy244 = addSubqueryElem(yymsp[-5].minor.yy244, yymsp[-2].minor.yy441, &yymsp[0].minor.yy0);} + yymsp[-5].minor.yy244 = yylhsminor.yy244; + break; + case 183: /* tablelist ::= ids cpxName */ { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy330 = setTableNameList(NULL, &yymsp[-1].minor.yy0, NULL); + yylhsminor.yy244 = setTableNameList(NULL, &yymsp[-1].minor.yy0, NULL); } - yymsp[-1].minor.yy330 = yylhsminor.yy330; + yymsp[-1].minor.yy244 = yylhsminor.yy244; break; - case 181: /* tablelist ::= ids cpxName ids */ + case 184: /* tablelist ::= ids cpxName ids */ { yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy330 = setTableNameList(NULL, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); + yylhsminor.yy244 = setTableNameList(NULL, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy330 = yylhsminor.yy330; + yymsp[-2].minor.yy244 = yylhsminor.yy244; break; - case 182: /* tablelist ::= tablelist COMMA ids cpxName */ + case 185: /* tablelist ::= tablelist COMMA ids cpxName */ { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy330 = setTableNameList(yymsp[-3].minor.yy330, &yymsp[-1].minor.yy0, NULL); + yylhsminor.yy244 = setTableNameList(yymsp[-3].minor.yy244, &yymsp[-1].minor.yy0, NULL); } - yymsp[-3].minor.yy330 = yylhsminor.yy330; + yymsp[-3].minor.yy244 = yylhsminor.yy244; break; - case 183: /* tablelist ::= tablelist COMMA ids cpxName ids */ + case 186: /* tablelist ::= tablelist COMMA ids cpxName ids */ { yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy330 = setTableNameList(yymsp[-4].minor.yy330, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); + yylhsminor.yy244 = setTableNameList(yymsp[-4].minor.yy244, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } - yymsp[-4].minor.yy330 = yylhsminor.yy330; + yymsp[-4].minor.yy244 = yylhsminor.yy244; break; - case 184: /* tmvar ::= VARIABLE */ + case 187: /* tmvar ::= VARIABLE */ {yylhsminor.yy0 = yymsp[0].minor.yy0;} yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 185: /* interval_opt ::= INTERVAL LP tmvar RP */ -{yymsp[-3].minor.yy42.interval = yymsp[-1].minor.yy0; yymsp[-3].minor.yy42.offset.n = 0;} + case 188: /* interval_opt ::= INTERVAL LP tmvar RP */ +{yymsp[-3].minor.yy340.interval = yymsp[-1].minor.yy0; yymsp[-3].minor.yy340.offset.n = 0;} break; - case 186: /* interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ -{yymsp[-5].minor.yy42.interval = yymsp[-3].minor.yy0; yymsp[-5].minor.yy42.offset = yymsp[-1].minor.yy0;} + case 189: /* interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ +{yymsp[-5].minor.yy340.interval = yymsp[-3].minor.yy0; yymsp[-5].minor.yy340.offset = yymsp[-1].minor.yy0;} break; - case 187: /* interval_opt ::= */ -{memset(&yymsp[1].minor.yy42, 0, sizeof(yymsp[1].minor.yy42));} + case 190: /* interval_opt ::= */ +{memset(&yymsp[1].minor.yy340, 0, sizeof(yymsp[1].minor.yy340));} break; - case 188: /* session_option ::= */ -{yymsp[1].minor.yy15.col.n = 0; yymsp[1].minor.yy15.gap.n = 0;} + case 191: /* session_option ::= */ +{yymsp[1].minor.yy259.col.n = 0; yymsp[1].minor.yy259.gap.n = 0;} break; - case 189: /* session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ + case 192: /* session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - yymsp[-6].minor.yy15.col = yymsp[-4].minor.yy0; - yymsp[-6].minor.yy15.gap = yymsp[-1].minor.yy0; + yymsp[-6].minor.yy259.col = yymsp[-4].minor.yy0; + yymsp[-6].minor.yy259.gap = yymsp[-1].minor.yy0; } break; - case 190: /* fill_opt ::= */ -{ yymsp[1].minor.yy93 = 0; } + case 193: /* fill_opt ::= */ +{ yymsp[1].minor.yy441 = 0; } break; - case 191: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + case 194: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ { tVariant A = {0}; toTSDBType(yymsp[-3].minor.yy0.type); tVariantCreate(&A, &yymsp[-3].minor.yy0); - tVariantListInsert(yymsp[-1].minor.yy93, &A, -1, 0); - yymsp[-5].minor.yy93 = yymsp[-1].minor.yy93; + tVariantListInsert(yymsp[-1].minor.yy441, &A, -1, 0); + yymsp[-5].minor.yy441 = yymsp[-1].minor.yy441; } break; - case 192: /* fill_opt ::= FILL LP ID RP */ + case 195: /* fill_opt ::= FILL LP ID RP */ { toTSDBType(yymsp[-1].minor.yy0.type); - yymsp[-3].minor.yy93 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + yymsp[-3].minor.yy441 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); } break; - case 193: /* sliding_opt ::= SLIDING LP tmvar RP */ + case 196: /* sliding_opt ::= SLIDING LP tmvar RP */ {yymsp[-3].minor.yy0 = yymsp[-1].minor.yy0; } break; - case 194: /* sliding_opt ::= */ + case 197: /* sliding_opt ::= */ {yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = NULL; yymsp[1].minor.yy0.type = 0; } break; - case 196: /* orderby_opt ::= ORDER BY sortlist */ -{yymsp[-2].minor.yy93 = yymsp[0].minor.yy93;} + case 199: /* orderby_opt ::= ORDER BY sortlist */ +{yymsp[-2].minor.yy441 = yymsp[0].minor.yy441;} break; - case 197: /* sortlist ::= sortlist COMMA item sortorder */ + case 200: /* sortlist ::= sortlist COMMA item sortorder */ { - yylhsminor.yy93 = tVariantListAppend(yymsp[-3].minor.yy93, &yymsp[-1].minor.yy518, yymsp[0].minor.yy150); + yylhsminor.yy441 = tVariantListAppend(yymsp[-3].minor.yy441, &yymsp[-1].minor.yy506, yymsp[0].minor.yy112); } - yymsp[-3].minor.yy93 = yylhsminor.yy93; + yymsp[-3].minor.yy441 = yylhsminor.yy441; break; - case 198: /* sortlist ::= item sortorder */ + case 201: /* sortlist ::= item sortorder */ { - yylhsminor.yy93 = tVariantListAppend(NULL, &yymsp[-1].minor.yy518, yymsp[0].minor.yy150); + yylhsminor.yy441 = tVariantListAppend(NULL, &yymsp[-1].minor.yy506, yymsp[0].minor.yy112); } - yymsp[-1].minor.yy93 = yylhsminor.yy93; + yymsp[-1].minor.yy441 = yylhsminor.yy441; break; - case 199: /* item ::= ids cpxName */ + case 202: /* item ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - tVariantCreate(&yylhsminor.yy518, &yymsp[-1].minor.yy0); + tVariantCreate(&yylhsminor.yy506, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy518 = yylhsminor.yy518; + yymsp[-1].minor.yy506 = yylhsminor.yy506; break; - case 200: /* sortorder ::= ASC */ -{ yymsp[0].minor.yy150 = TSDB_ORDER_ASC; } + case 203: /* sortorder ::= ASC */ +{ yymsp[0].minor.yy112 = TSDB_ORDER_ASC; } break; - case 201: /* sortorder ::= DESC */ -{ yymsp[0].minor.yy150 = TSDB_ORDER_DESC;} + case 204: /* sortorder ::= DESC */ +{ yymsp[0].minor.yy112 = TSDB_ORDER_DESC;} break; - case 202: /* sortorder ::= */ -{ yymsp[1].minor.yy150 = TSDB_ORDER_ASC; } + case 205: /* sortorder ::= */ +{ yymsp[1].minor.yy112 = TSDB_ORDER_ASC; } break; - case 203: /* groupby_opt ::= */ -{ yymsp[1].minor.yy93 = 0;} + case 206: /* groupby_opt ::= */ +{ yymsp[1].minor.yy441 = 0;} break; - case 204: /* groupby_opt ::= GROUP BY grouplist */ -{ yymsp[-2].minor.yy93 = yymsp[0].minor.yy93;} + case 207: /* groupby_opt ::= GROUP BY grouplist */ +{ yymsp[-2].minor.yy441 = yymsp[0].minor.yy441;} break; - case 205: /* grouplist ::= grouplist COMMA item */ + case 208: /* grouplist ::= grouplist COMMA item */ { - yylhsminor.yy93 = tVariantListAppend(yymsp[-2].minor.yy93, &yymsp[0].minor.yy518, -1); + yylhsminor.yy441 = tVariantListAppend(yymsp[-2].minor.yy441, &yymsp[0].minor.yy506, -1); } - yymsp[-2].minor.yy93 = yylhsminor.yy93; + yymsp[-2].minor.yy441 = yylhsminor.yy441; break; - case 206: /* grouplist ::= item */ + case 209: /* grouplist ::= item */ { - yylhsminor.yy93 = tVariantListAppend(NULL, &yymsp[0].minor.yy518, -1); + yylhsminor.yy441 = tVariantListAppend(NULL, &yymsp[0].minor.yy506, -1); } - yymsp[0].minor.yy93 = yylhsminor.yy93; + yymsp[0].minor.yy441 = yylhsminor.yy441; break; - case 207: /* having_opt ::= */ - case 217: /* where_opt ::= */ yytestcase(yyruleno==217); - case 259: /* expritem ::= */ yytestcase(yyruleno==259); -{yymsp[1].minor.yy68 = 0;} + case 210: /* having_opt ::= */ + case 220: /* where_opt ::= */ yytestcase(yyruleno==220); + case 262: /* expritem ::= */ yytestcase(yyruleno==262); +{yymsp[1].minor.yy166 = 0;} break; - case 208: /* having_opt ::= HAVING expr */ - case 218: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==218); -{yymsp[-1].minor.yy68 = yymsp[0].minor.yy68;} + case 211: /* having_opt ::= HAVING expr */ + case 221: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==221); +{yymsp[-1].minor.yy166 = yymsp[0].minor.yy166;} break; - case 209: /* limit_opt ::= */ - case 213: /* slimit_opt ::= */ yytestcase(yyruleno==213); -{yymsp[1].minor.yy284.limit = -1; yymsp[1].minor.yy284.offset = 0;} + case 212: /* limit_opt ::= */ + case 216: /* slimit_opt ::= */ yytestcase(yyruleno==216); +{yymsp[1].minor.yy414.limit = -1; yymsp[1].minor.yy414.offset = 0;} break; - case 210: /* limit_opt ::= LIMIT signed */ - case 214: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==214); -{yymsp[-1].minor.yy284.limit = yymsp[0].minor.yy279; yymsp[-1].minor.yy284.offset = 0;} + case 213: /* limit_opt ::= LIMIT signed */ + case 217: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==217); +{yymsp[-1].minor.yy414.limit = yymsp[0].minor.yy369; yymsp[-1].minor.yy414.offset = 0;} break; - case 211: /* limit_opt ::= LIMIT signed OFFSET signed */ -{ yymsp[-3].minor.yy284.limit = yymsp[-2].minor.yy279; yymsp[-3].minor.yy284.offset = yymsp[0].minor.yy279;} + case 214: /* limit_opt ::= LIMIT signed OFFSET signed */ +{ yymsp[-3].minor.yy414.limit = yymsp[-2].minor.yy369; yymsp[-3].minor.yy414.offset = yymsp[0].minor.yy369;} break; - case 212: /* limit_opt ::= LIMIT signed COMMA signed */ -{ yymsp[-3].minor.yy284.limit = yymsp[0].minor.yy279; yymsp[-3].minor.yy284.offset = yymsp[-2].minor.yy279;} + case 215: /* limit_opt ::= LIMIT signed COMMA signed */ +{ yymsp[-3].minor.yy414.limit = yymsp[0].minor.yy369; yymsp[-3].minor.yy414.offset = yymsp[-2].minor.yy369;} break; - case 215: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ -{yymsp[-3].minor.yy284.limit = yymsp[-2].minor.yy279; yymsp[-3].minor.yy284.offset = yymsp[0].minor.yy279;} + case 218: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ +{yymsp[-3].minor.yy414.limit = yymsp[-2].minor.yy369; yymsp[-3].minor.yy414.offset = yymsp[0].minor.yy369;} break; - case 216: /* slimit_opt ::= SLIMIT signed COMMA signed */ -{yymsp[-3].minor.yy284.limit = yymsp[0].minor.yy279; yymsp[-3].minor.yy284.offset = yymsp[-2].minor.yy279;} + case 219: /* slimit_opt ::= SLIMIT signed COMMA signed */ +{yymsp[-3].minor.yy414.limit = yymsp[0].minor.yy369; yymsp[-3].minor.yy414.offset = yymsp[-2].minor.yy369;} break; - case 219: /* expr ::= LP expr RP */ -{yylhsminor.yy68 = yymsp[-1].minor.yy68; yylhsminor.yy68->token.z = yymsp[-2].minor.yy0.z; yylhsminor.yy68->token.n = (yymsp[0].minor.yy0.z - yymsp[-2].minor.yy0.z + 1);} - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 222: /* expr ::= LP expr RP */ +{yylhsminor.yy166 = yymsp[-1].minor.yy166; yylhsminor.yy166->token.z = yymsp[-2].minor.yy0.z; yylhsminor.yy166->token.n = (yymsp[0].minor.yy0.z - yymsp[-2].minor.yy0.z + 1);} + yymsp[-2].minor.yy166 = yylhsminor.yy166; break; - case 220: /* expr ::= ID */ -{ yylhsminor.yy68 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_ID);} - yymsp[0].minor.yy68 = yylhsminor.yy68; + case 223: /* expr ::= ID */ +{ yylhsminor.yy166 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_ID);} + yymsp[0].minor.yy166 = yylhsminor.yy166; break; - case 221: /* expr ::= ID DOT ID */ -{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy68 = tSqlExprCreateIdValue(&yymsp[-2].minor.yy0, TK_ID);} - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 224: /* expr ::= ID DOT ID */ +{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy166 = tSqlExprCreateIdValue(&yymsp[-2].minor.yy0, TK_ID);} + yymsp[-2].minor.yy166 = yylhsminor.yy166; break; - case 222: /* expr ::= ID DOT STAR */ -{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy68 = tSqlExprCreateIdValue(&yymsp[-2].minor.yy0, TK_ALL);} - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 225: /* expr ::= ID DOT STAR */ +{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy166 = tSqlExprCreateIdValue(&yymsp[-2].minor.yy0, TK_ALL);} + yymsp[-2].minor.yy166 = yylhsminor.yy166; break; - case 223: /* expr ::= INTEGER */ -{ yylhsminor.yy68 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_INTEGER);} - yymsp[0].minor.yy68 = yylhsminor.yy68; + case 226: /* expr ::= INTEGER */ +{ yylhsminor.yy166 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_INTEGER);} + yymsp[0].minor.yy166 = yylhsminor.yy166; break; - case 224: /* expr ::= MINUS INTEGER */ - case 225: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==225); -{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy68 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_INTEGER);} - yymsp[-1].minor.yy68 = yylhsminor.yy68; + case 227: /* expr ::= MINUS INTEGER */ + case 228: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==228); +{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy166 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_INTEGER);} + yymsp[-1].minor.yy166 = yylhsminor.yy166; break; - case 226: /* expr ::= FLOAT */ -{ yylhsminor.yy68 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_FLOAT);} - yymsp[0].minor.yy68 = yylhsminor.yy68; + case 229: /* expr ::= FLOAT */ +{ yylhsminor.yy166 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_FLOAT);} + yymsp[0].minor.yy166 = yylhsminor.yy166; break; - case 227: /* expr ::= MINUS FLOAT */ - case 228: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==228); -{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy68 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_FLOAT);} - yymsp[-1].minor.yy68 = yylhsminor.yy68; + case 230: /* expr ::= MINUS FLOAT */ + case 231: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==231); +{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy166 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_FLOAT);} + yymsp[-1].minor.yy166 = yylhsminor.yy166; break; - case 229: /* expr ::= STRING */ -{ yylhsminor.yy68 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_STRING);} - yymsp[0].minor.yy68 = yylhsminor.yy68; + case 232: /* expr ::= STRING */ +{ yylhsminor.yy166 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_STRING);} + yymsp[0].minor.yy166 = yylhsminor.yy166; break; - case 230: /* expr ::= NOW */ -{ yylhsminor.yy68 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_NOW); } - yymsp[0].minor.yy68 = yylhsminor.yy68; + case 233: /* expr ::= NOW */ +{ yylhsminor.yy166 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_NOW); } + yymsp[0].minor.yy166 = yylhsminor.yy166; break; - case 231: /* expr ::= VARIABLE */ -{ yylhsminor.yy68 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_VARIABLE);} - yymsp[0].minor.yy68 = yylhsminor.yy68; + case 234: /* expr ::= VARIABLE */ +{ yylhsminor.yy166 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_VARIABLE);} + yymsp[0].minor.yy166 = yylhsminor.yy166; break; - case 232: /* expr ::= PLUS VARIABLE */ - case 233: /* expr ::= MINUS VARIABLE */ yytestcase(yyruleno==233); -{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_VARIABLE; yylhsminor.yy68 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_VARIABLE);} - yymsp[-1].minor.yy68 = yylhsminor.yy68; + case 235: /* expr ::= PLUS VARIABLE */ + case 236: /* expr ::= MINUS VARIABLE */ yytestcase(yyruleno==236); +{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_VARIABLE; yylhsminor.yy166 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_VARIABLE);} + yymsp[-1].minor.yy166 = yylhsminor.yy166; break; - case 234: /* expr ::= BOOL */ -{ yylhsminor.yy68 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_BOOL);} - yymsp[0].minor.yy68 = yylhsminor.yy68; + case 237: /* expr ::= BOOL */ +{ yylhsminor.yy166 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_BOOL);} + yymsp[0].minor.yy166 = yylhsminor.yy166; break; - case 235: /* expr ::= NULL */ -{ yylhsminor.yy68 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_NULL);} - yymsp[0].minor.yy68 = yylhsminor.yy68; + case 238: /* expr ::= NULL */ +{ yylhsminor.yy166 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_NULL);} + yymsp[0].minor.yy166 = yylhsminor.yy166; break; - case 236: /* expr ::= ID LP exprlist RP */ -{ yylhsminor.yy68 = tSqlExprCreateFunction(yymsp[-1].minor.yy93, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } - yymsp[-3].minor.yy68 = yylhsminor.yy68; + case 239: /* expr ::= ID LP exprlist RP */ +{ yylhsminor.yy166 = tSqlExprCreateFunction(yymsp[-1].minor.yy441, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } + yymsp[-3].minor.yy166 = yylhsminor.yy166; break; - case 237: /* expr ::= ID LP STAR RP */ -{ yylhsminor.yy68 = tSqlExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } - yymsp[-3].minor.yy68 = yylhsminor.yy68; + case 240: /* expr ::= ID LP STAR RP */ +{ yylhsminor.yy166 = tSqlExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } + yymsp[-3].minor.yy166 = yylhsminor.yy166; break; - case 238: /* expr ::= expr IS NULL */ -{yylhsminor.yy68 = tSqlExprCreate(yymsp[-2].minor.yy68, NULL, TK_ISNULL);} - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 241: /* expr ::= expr IS NULL */ +{yylhsminor.yy166 = tSqlExprCreate(yymsp[-2].minor.yy166, NULL, TK_ISNULL);} + yymsp[-2].minor.yy166 = yylhsminor.yy166; break; - case 239: /* expr ::= expr IS NOT NULL */ -{yylhsminor.yy68 = tSqlExprCreate(yymsp[-3].minor.yy68, NULL, TK_NOTNULL);} - yymsp[-3].minor.yy68 = yylhsminor.yy68; + case 242: /* expr ::= expr IS NOT NULL */ +{yylhsminor.yy166 = tSqlExprCreate(yymsp[-3].minor.yy166, NULL, TK_NOTNULL);} + yymsp[-3].minor.yy166 = yylhsminor.yy166; break; - case 240: /* expr ::= expr LT expr */ -{yylhsminor.yy68 = tSqlExprCreate(yymsp[-2].minor.yy68, yymsp[0].minor.yy68, TK_LT);} - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 243: /* expr ::= expr LT expr */ +{yylhsminor.yy166 = tSqlExprCreate(yymsp[-2].minor.yy166, yymsp[0].minor.yy166, TK_LT);} + yymsp[-2].minor.yy166 = yylhsminor.yy166; break; - case 241: /* expr ::= expr GT expr */ -{yylhsminor.yy68 = tSqlExprCreate(yymsp[-2].minor.yy68, yymsp[0].minor.yy68, TK_GT);} - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 244: /* expr ::= expr GT expr */ +{yylhsminor.yy166 = tSqlExprCreate(yymsp[-2].minor.yy166, yymsp[0].minor.yy166, TK_GT);} + yymsp[-2].minor.yy166 = yylhsminor.yy166; break; - case 242: /* expr ::= expr LE expr */ -{yylhsminor.yy68 = tSqlExprCreate(yymsp[-2].minor.yy68, yymsp[0].minor.yy68, TK_LE);} - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 245: /* expr ::= expr LE expr */ +{yylhsminor.yy166 = tSqlExprCreate(yymsp[-2].minor.yy166, yymsp[0].minor.yy166, TK_LE);} + yymsp[-2].minor.yy166 = yylhsminor.yy166; break; - case 243: /* expr ::= expr GE expr */ -{yylhsminor.yy68 = tSqlExprCreate(yymsp[-2].minor.yy68, yymsp[0].minor.yy68, TK_GE);} - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 246: /* expr ::= expr GE expr */ +{yylhsminor.yy166 = tSqlExprCreate(yymsp[-2].minor.yy166, yymsp[0].minor.yy166, TK_GE);} + yymsp[-2].minor.yy166 = yylhsminor.yy166; break; - case 244: /* expr ::= expr NE expr */ -{yylhsminor.yy68 = tSqlExprCreate(yymsp[-2].minor.yy68, yymsp[0].minor.yy68, TK_NE);} - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 247: /* expr ::= expr NE expr */ +{yylhsminor.yy166 = tSqlExprCreate(yymsp[-2].minor.yy166, yymsp[0].minor.yy166, TK_NE);} + yymsp[-2].minor.yy166 = yylhsminor.yy166; break; - case 245: /* expr ::= expr EQ expr */ -{yylhsminor.yy68 = tSqlExprCreate(yymsp[-2].minor.yy68, yymsp[0].minor.yy68, TK_EQ);} - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 248: /* expr ::= expr EQ expr */ +{yylhsminor.yy166 = tSqlExprCreate(yymsp[-2].minor.yy166, yymsp[0].minor.yy166, TK_EQ);} + yymsp[-2].minor.yy166 = yylhsminor.yy166; break; - case 246: /* expr ::= expr BETWEEN expr AND expr */ -{ tSqlExpr* X2 = tSqlExprClone(yymsp[-4].minor.yy68); yylhsminor.yy68 = tSqlExprCreate(tSqlExprCreate(yymsp[-4].minor.yy68, yymsp[-2].minor.yy68, TK_GE), tSqlExprCreate(X2, yymsp[0].minor.yy68, TK_LE), TK_AND);} - yymsp[-4].minor.yy68 = yylhsminor.yy68; + case 249: /* expr ::= expr BETWEEN expr AND expr */ +{ tSqlExpr* X2 = tSqlExprClone(yymsp[-4].minor.yy166); yylhsminor.yy166 = tSqlExprCreate(tSqlExprCreate(yymsp[-4].minor.yy166, yymsp[-2].minor.yy166, TK_GE), tSqlExprCreate(X2, yymsp[0].minor.yy166, TK_LE), TK_AND);} + yymsp[-4].minor.yy166 = yylhsminor.yy166; break; - case 247: /* expr ::= expr AND expr */ -{yylhsminor.yy68 = tSqlExprCreate(yymsp[-2].minor.yy68, yymsp[0].minor.yy68, TK_AND);} - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 250: /* expr ::= expr AND expr */ +{yylhsminor.yy166 = tSqlExprCreate(yymsp[-2].minor.yy166, yymsp[0].minor.yy166, TK_AND);} + yymsp[-2].minor.yy166 = yylhsminor.yy166; break; - case 248: /* expr ::= expr OR expr */ -{yylhsminor.yy68 = tSqlExprCreate(yymsp[-2].minor.yy68, yymsp[0].minor.yy68, TK_OR); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 251: /* expr ::= expr OR expr */ +{yylhsminor.yy166 = tSqlExprCreate(yymsp[-2].minor.yy166, yymsp[0].minor.yy166, TK_OR); } + yymsp[-2].minor.yy166 = yylhsminor.yy166; break; - case 249: /* expr ::= expr PLUS expr */ -{yylhsminor.yy68 = tSqlExprCreate(yymsp[-2].minor.yy68, yymsp[0].minor.yy68, TK_PLUS); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 252: /* expr ::= expr PLUS expr */ +{yylhsminor.yy166 = tSqlExprCreate(yymsp[-2].minor.yy166, yymsp[0].minor.yy166, TK_PLUS); } + yymsp[-2].minor.yy166 = yylhsminor.yy166; break; - case 250: /* expr ::= expr MINUS expr */ -{yylhsminor.yy68 = tSqlExprCreate(yymsp[-2].minor.yy68, yymsp[0].minor.yy68, TK_MINUS); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 253: /* expr ::= expr MINUS expr */ +{yylhsminor.yy166 = tSqlExprCreate(yymsp[-2].minor.yy166, yymsp[0].minor.yy166, TK_MINUS); } + yymsp[-2].minor.yy166 = yylhsminor.yy166; break; - case 251: /* expr ::= expr STAR expr */ -{yylhsminor.yy68 = tSqlExprCreate(yymsp[-2].minor.yy68, yymsp[0].minor.yy68, TK_STAR); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 254: /* expr ::= expr STAR expr */ +{yylhsminor.yy166 = tSqlExprCreate(yymsp[-2].minor.yy166, yymsp[0].minor.yy166, TK_STAR); } + yymsp[-2].minor.yy166 = yylhsminor.yy166; break; - case 252: /* expr ::= expr SLASH expr */ -{yylhsminor.yy68 = tSqlExprCreate(yymsp[-2].minor.yy68, yymsp[0].minor.yy68, TK_DIVIDE);} - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 255: /* expr ::= expr SLASH expr */ +{yylhsminor.yy166 = tSqlExprCreate(yymsp[-2].minor.yy166, yymsp[0].minor.yy166, TK_DIVIDE);} + yymsp[-2].minor.yy166 = yylhsminor.yy166; break; - case 253: /* expr ::= expr REM expr */ -{yylhsminor.yy68 = tSqlExprCreate(yymsp[-2].minor.yy68, yymsp[0].minor.yy68, TK_REM); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 256: /* expr ::= expr REM expr */ +{yylhsminor.yy166 = tSqlExprCreate(yymsp[-2].minor.yy166, yymsp[0].minor.yy166, TK_REM); } + yymsp[-2].minor.yy166 = yylhsminor.yy166; break; - case 254: /* expr ::= expr LIKE expr */ -{yylhsminor.yy68 = tSqlExprCreate(yymsp[-2].minor.yy68, yymsp[0].minor.yy68, TK_LIKE); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 257: /* expr ::= expr LIKE expr */ +{yylhsminor.yy166 = tSqlExprCreate(yymsp[-2].minor.yy166, yymsp[0].minor.yy166, TK_LIKE); } + yymsp[-2].minor.yy166 = yylhsminor.yy166; break; - case 255: /* expr ::= expr IN LP exprlist RP */ -{yylhsminor.yy68 = tSqlExprCreate(yymsp[-4].minor.yy68, (tSqlExpr*)yymsp[-1].minor.yy93, TK_IN); } - yymsp[-4].minor.yy68 = yylhsminor.yy68; + case 258: /* expr ::= expr IN LP exprlist RP */ +{yylhsminor.yy166 = tSqlExprCreate(yymsp[-4].minor.yy166, (tSqlExpr*)yymsp[-1].minor.yy441, TK_IN); } + yymsp[-4].minor.yy166 = yylhsminor.yy166; break; - case 256: /* exprlist ::= exprlist COMMA expritem */ -{yylhsminor.yy93 = tSqlExprListAppend(yymsp[-2].minor.yy93,yymsp[0].minor.yy68,0, 0);} - yymsp[-2].minor.yy93 = yylhsminor.yy93; + case 259: /* exprlist ::= exprlist COMMA expritem */ +{yylhsminor.yy441 = tSqlExprListAppend(yymsp[-2].minor.yy441,yymsp[0].minor.yy166,0, 0);} + yymsp[-2].minor.yy441 = yylhsminor.yy441; break; - case 257: /* exprlist ::= expritem */ -{yylhsminor.yy93 = tSqlExprListAppend(0,yymsp[0].minor.yy68,0, 0);} - yymsp[0].minor.yy93 = yylhsminor.yy93; + case 260: /* exprlist ::= expritem */ +{yylhsminor.yy441 = tSqlExprListAppend(0,yymsp[0].minor.yy166,0, 0);} + yymsp[0].minor.yy441 = yylhsminor.yy441; break; - case 258: /* expritem ::= expr */ -{yylhsminor.yy68 = yymsp[0].minor.yy68;} - yymsp[0].minor.yy68 = yylhsminor.yy68; + case 261: /* expritem ::= expr */ +{yylhsminor.yy166 = yymsp[0].minor.yy166;} + yymsp[0].minor.yy166 = yylhsminor.yy166; break; - case 260: /* cmd ::= RESET QUERY CACHE */ + case 263: /* cmd ::= RESET QUERY CACHE */ { setDCLSqlElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} break; - case 261: /* cmd ::= SYNCDB ids REPLICA */ + case 264: /* cmd ::= SYNCDB ids REPLICA */ { setDCLSqlElems(pInfo, TSDB_SQL_SYNC_DB_REPLICA, 1, &yymsp[-1].minor.yy0);} break; - case 262: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + case 265: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy93, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, -1); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy441, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 263: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + case 266: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3056,14 +3075,14 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 264: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + case 267: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy93, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, -1); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy441, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 265: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + case 268: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3074,7 +3093,7 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 266: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + case 269: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -3088,26 +3107,26 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 267: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + case 270: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ { yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; toTSDBType(yymsp[-2].minor.yy0.type); SArray* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - A = tVariantListAppend(A, &yymsp[0].minor.yy518, -1); + A = tVariantListAppend(A, &yymsp[0].minor.yy506, -1); SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 268: /* cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ + case 271: /* cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy93, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, TSDB_SUPER_TABLE); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy441, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 269: /* cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ + case 272: /* cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3118,14 +3137,14 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 270: /* cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ + case 273: /* cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy93, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, TSDB_SUPER_TABLE); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy441, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 271: /* cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ + case 274: /* cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3136,7 +3155,7 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 272: /* cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ + case 275: /* cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -3150,13 +3169,13 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 273: /* cmd ::= KILL CONNECTION INTEGER */ + case 276: /* cmd ::= KILL CONNECTION INTEGER */ {setKillSql(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);} break; - case 274: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ + case 277: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSql(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);} break; - case 275: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ + case 278: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSql(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);} break; default: diff --git a/src/query/tests/unitTest.cpp b/src/query/tests/unitTest.cpp index d2b058cf7c..33ba8200d3 100644 --- a/src/query/tests/unitTest.cpp +++ b/src/query/tests/unitTest.cpp @@ -99,47 +99,47 @@ TEST(testCase, db_table_name) { EXPECT_EQ(testValidateName(t4), TSDB_CODE_SUCCESS); char t5[] = "table.'def'"; - EXPECT_EQ(testValidateName(t5), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t5), TSDB_CODE_TSC_INVALID_OPERATION); char t6[] = "'table'.'def'"; - EXPECT_EQ(testValidateName(t6), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t6), TSDB_CODE_TSC_INVALID_OPERATION); char t7[] = "'_ab1234'.'def'"; EXPECT_EQ(testValidateName(t7), TSDB_CODE_SUCCESS); printf("%s\n", t7); char t8[] = "'_ab&^%1234'.'def'"; - EXPECT_EQ(testValidateName(t8), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t8), TSDB_CODE_TSC_INVALID_OPERATION); char t9[] = "'_123'.'gtest中文'"; - EXPECT_EQ(testValidateName(t9), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t9), TSDB_CODE_TSC_INVALID_OPERATION); char t10[] = "abc.'gtest中文'"; - EXPECT_EQ(testValidateName(t10), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t10), TSDB_CODE_TSC_INVALID_OPERATION); char t10_1[] = "abc.'中文gtest'"; - EXPECT_EQ(testValidateName(t10_1), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t10_1), TSDB_CODE_TSC_INVALID_OPERATION); char t11[] = "'192.168.0.1'.abc"; - EXPECT_EQ(testValidateName(t11), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t11), TSDB_CODE_TSC_INVALID_OPERATION); char t12[] = "192.168.0.1.abc"; - EXPECT_EQ(testValidateName(t12), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t12), TSDB_CODE_TSC_INVALID_OPERATION); char t13[] = "abc."; - EXPECT_EQ(testValidateName(t13), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t13), TSDB_CODE_TSC_INVALID_OPERATION); char t14[] = ".abc"; - EXPECT_EQ(testValidateName(t14), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t14), TSDB_CODE_TSC_INVALID_OPERATION); char t15[] = ".'abc'"; - EXPECT_EQ(testValidateName(t15), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t15), TSDB_CODE_TSC_INVALID_OPERATION); char t16[] = ".abc'"; - EXPECT_EQ(testValidateName(t16), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t16), TSDB_CODE_TSC_INVALID_OPERATION); char t17[] = "123a.\"abc\""; - EXPECT_EQ(testValidateName(t17), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t17), TSDB_CODE_TSC_INVALID_OPERATION); printf("%s\n", t17); char t18[] = "a.\"abc\""; @@ -147,13 +147,13 @@ TEST(testCase, db_table_name) { printf("%s\n", t18); char t19[] = "'_ab1234'.'def'.'ab123'"; - EXPECT_EQ(testValidateName(t19), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t19), TSDB_CODE_TSC_INVALID_OPERATION); char t20[] = "'_ab1234*&^'"; - EXPECT_EQ(testValidateName(t20), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t20), TSDB_CODE_TSC_INVALID_OPERATION); char t21[] = "'1234_abc'"; - EXPECT_EQ(testValidateName(t21), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t21), TSDB_CODE_TSC_INVALID_OPERATION); // =======Containing capital letters================= @@ -167,10 +167,10 @@ TEST(testCase, db_table_name) { EXPECT_EQ(testValidateName(t32), TSDB_CODE_SUCCESS); char t33[] = "'ABC.def"; - EXPECT_EQ(testValidateName(t33), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t33), TSDB_CODE_TSC_INVALID_OPERATION); char t33_0[] = "abc.DEF'"; - EXPECT_EQ(testValidateName(t33_0), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t33_0), TSDB_CODE_TSC_INVALID_OPERATION); char t34[] = "'ABC.def'"; //int32_t tmp0 = testValidateName(t34); @@ -193,136 +193,136 @@ TEST(testCase, db_table_name) { // do not use key words char t39[] = "table.'DEF'"; - EXPECT_EQ(testValidateName(t39), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t39), TSDB_CODE_TSC_INVALID_OPERATION); char t40[] = "'table'.'DEF'"; - EXPECT_EQ(testValidateName(t40), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t40), TSDB_CODE_TSC_INVALID_OPERATION); char t41[] = "'_abXYZ1234'.'deFF'"; EXPECT_EQ(testValidateName(t41), TSDB_CODE_SUCCESS); char t42[] = "'_abDEF&^%1234'.'DIef'"; - EXPECT_EQ(testValidateName(t42), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t42), TSDB_CODE_TSC_INVALID_OPERATION); char t43[] = "'_123'.'Gtest中文'"; - EXPECT_EQ(testValidateName(t43), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t43), TSDB_CODE_TSC_INVALID_OPERATION); char t44[] = "'aABC'.'Gtest中文'"; - EXPECT_EQ(testValidateName(t44), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t44), TSDB_CODE_TSC_INVALID_OPERATION); char t45[] = "'ABC'."; - EXPECT_EQ(testValidateName(t45), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t45), TSDB_CODE_TSC_INVALID_OPERATION); char t46[] = ".'ABC'"; - EXPECT_EQ(testValidateName(t46), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t46), TSDB_CODE_TSC_INVALID_OPERATION); char t47[] = "a.\"aTWc\""; EXPECT_EQ(testValidateName(t47), TSDB_CODE_SUCCESS); // ================has space ================= char t60[] = " ABC "; - EXPECT_EQ(testValidateName(t60), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t60), TSDB_CODE_TSC_INVALID_OPERATION); char t60_1[] = " ABC "; - EXPECT_EQ(testValidateName(t60_1), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t60_1), TSDB_CODE_TSC_INVALID_OPERATION); char t61[] = "' ABC '"; - EXPECT_EQ(testValidateName(t61), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t61), TSDB_CODE_TSC_INVALID_OPERATION); char t61_1[] = "' ABC '"; - EXPECT_EQ(testValidateName(t61_1), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t61_1), TSDB_CODE_TSC_INVALID_OPERATION); char t62[] = " ABC . def "; - EXPECT_EQ(testValidateName(t62), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t62), TSDB_CODE_TSC_INVALID_OPERATION); char t63[] = "' ABC . def "; - EXPECT_EQ(testValidateName(t63), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t63), TSDB_CODE_TSC_INVALID_OPERATION); char t63_0[] = " abc . DEF ' "; - EXPECT_EQ(testValidateName(t63_0), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t63_0), TSDB_CODE_TSC_INVALID_OPERATION); char t64[] = " ' ABC . def ' "; //int32_t tmp1 = testValidateName(t64); - EXPECT_EQ(testValidateName(t64), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t64), TSDB_CODE_TSC_INVALID_OPERATION); char t65[] = " ' ABC '. def "; - EXPECT_EQ(testValidateName(t65), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t65), TSDB_CODE_TSC_INVALID_OPERATION); char t66[] = "' ABC '.' DEF '"; - EXPECT_EQ(testValidateName(t66), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t66), TSDB_CODE_TSC_INVALID_OPERATION); char t67[] = "abc . ' DEF '"; - EXPECT_EQ(testValidateName(t67), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t67), TSDB_CODE_TSC_INVALID_OPERATION); char t68[] = "' abc '.' DEF '"; - EXPECT_EQ(testValidateName(t68), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t68), TSDB_CODE_TSC_INVALID_OPERATION); // do not use key words char t69[] = "table.'DEF'"; - EXPECT_EQ(testValidateName(t69), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t69), TSDB_CODE_TSC_INVALID_OPERATION); char t70[] = "'table'.'DEF'"; - EXPECT_EQ(testValidateName(t70), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t70), TSDB_CODE_TSC_INVALID_OPERATION); char t71[] = "'_abXYZ1234 '.' deFF '"; - EXPECT_EQ(testValidateName(t71), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t71), TSDB_CODE_TSC_INVALID_OPERATION); char t72[] = "'_abDEF&^%1234'.' DIef'"; - EXPECT_EQ(testValidateName(t72), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t72), TSDB_CODE_TSC_INVALID_OPERATION); char t73[] = "'_123'.' Gtest中文'"; - EXPECT_EQ(testValidateName(t73), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t73), TSDB_CODE_TSC_INVALID_OPERATION); char t74[] = "' aABC'.'Gtest中文'"; - EXPECT_EQ(testValidateName(t74), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t74), TSDB_CODE_TSC_INVALID_OPERATION); char t75[] = "' ABC '."; - EXPECT_EQ(testValidateName(t75), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t75), TSDB_CODE_TSC_INVALID_OPERATION); char t76[] = ".' ABC'"; - EXPECT_EQ(testValidateName(t76), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t76), TSDB_CODE_TSC_INVALID_OPERATION); char t77[] = " a . \"aTWc\" "; - EXPECT_EQ(testValidateName(t77), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t77), TSDB_CODE_TSC_INVALID_OPERATION); char t78[] = " a.\"aTWc \""; - EXPECT_EQ(testValidateName(t78), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t78), TSDB_CODE_TSC_INVALID_OPERATION); // ===============muti string by space =================== // There's no such case. //char t160[] = "A BC"; - //EXPECT_EQ(testValidateName(t160), TSDB_CODE_TSC_INVALID_SQL); + //EXPECT_EQ(testValidateName(t160), TSDB_CODE_TSC_INVALID_OPERATION); //printf("end:%s\n", t160); // There's no such case. //char t161[] = "' A BC '"; - //EXPECT_EQ(testValidateName(t161), TSDB_CODE_TSC_INVALID_SQL); + //EXPECT_EQ(testValidateName(t161), TSDB_CODE_TSC_INVALID_OPERATION); char t162[] = " AB C . de f "; - EXPECT_EQ(testValidateName(t162), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t162), TSDB_CODE_TSC_INVALID_OPERATION); char t163[] = "' AB C . de f "; - EXPECT_EQ(testValidateName(t163), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t163), TSDB_CODE_TSC_INVALID_OPERATION); char t163_0[] = " ab c . DE F ' "; - EXPECT_EQ(testValidateName(t163_0), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t163_0), TSDB_CODE_TSC_INVALID_OPERATION); char t164[] = " ' AB C . de f ' "; //int32_t tmp2 = testValidateName(t164); - EXPECT_EQ(testValidateName(t164), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t164), TSDB_CODE_TSC_INVALID_OPERATION); char t165[] = " ' A BC '. de f "; - EXPECT_EQ(testValidateName(t165), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t165), TSDB_CODE_TSC_INVALID_OPERATION); char t166[] = "' AB C '.' DE F '"; - EXPECT_EQ(testValidateName(t166), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t166), TSDB_CODE_TSC_INVALID_OPERATION); char t167[] = "ab c . ' D EF '"; - EXPECT_EQ(testValidateName(t167), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t167), TSDB_CODE_TSC_INVALID_OPERATION); char t168[] = "' a bc '.' DE F '"; - EXPECT_EQ(testValidateName(t168), TSDB_CODE_TSC_INVALID_SQL); + EXPECT_EQ(testValidateName(t168), TSDB_CODE_TSC_INVALID_OPERATION); } diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 3d8f54ad9b..493d5f117a 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -709,7 +709,7 @@ static void syncChooseMaster(SSyncNode *pNode) { } static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { - int32_t onlineNum = 0; + int32_t onlineNum = 0, arbOnlineNum = 0; int32_t masterIndex = -1; int32_t replica = pNode->replica; @@ -723,13 +723,15 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { SSyncPeer *pArb = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]; if (pArb && pArb->role != TAOS_SYNC_ROLE_OFFLINE) { onlineNum++; + ++arbOnlineNum; replica = pNode->replica + 1; } if (onlineNum <= replica * 0.5) { if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) { - if (nodeRole == TAOS_SYNC_ROLE_MASTER && onlineNum == replica * 0.5 && onlineNum >= 1) { + if (nodeRole == TAOS_SYNC_ROLE_MASTER && onlineNum == replica * 0.5 && ((replica > 2 && onlineNum - arbOnlineNum > 1) || pNode->replica < 3)) { sInfo("vgId:%d, self keep work as master, online:%d replica:%d", pNode->vgId, onlineNum, replica); + masterIndex = pNode->selfIndex; } else { nodeRole = TAOS_SYNC_ROLE_UNSYNCED; sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica); @@ -1002,6 +1004,7 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) { if (nodeRole == TAOS_SYNC_ROLE_SLAVE) { // nodeVersion = pHead->version; code = (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL); + syncConfirmForward(pNode->rid, pHead->version, code, false); } else { if (nodeSStatus != TAOS_SYNC_STATUS_INIT) { code = syncSaveIntoBuffer(pPeer, pHead); @@ -1404,7 +1407,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) { pthread_mutex_lock(&pNode->mutex); for (int32_t i = 0; i < pSyncFwds->fwds; ++i) { SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % SYNC_MAX_FWDS; - if (ABS(time - pFwdInfo->time) < 2000) break; + if (ABS(time - pFwdInfo->time) < 10000) break; sDebug("vgId:%d, forward info expired, hver:%" PRIu64 " curtime:%" PRIu64 " savetime:%" PRIu64, pNode->vgId, pFwdInfo->version, time, pFwdInfo->time); diff --git a/src/tsdb/inc/tsdbCommit.h b/src/tsdb/inc/tsdbCommit.h index 5e740081d1..6bd5dc4325 100644 --- a/src/tsdb/inc/tsdbCommit.h +++ b/src/tsdb/inc/tsdbCommit.h @@ -33,6 +33,7 @@ void tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn); int tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord); void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord); void *tsdbCommitData(STsdbRepo *pRepo); +int tsdbApplyRtn(STsdbRepo *pRepo); static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) { if (fid >= pRtn->maxFid) { diff --git a/src/tsdb/inc/tsdbMeta.h b/src/tsdb/inc/tsdbMeta.h index 7484071ce3..45bbd5a7c6 100644 --- a/src/tsdb/inc/tsdbMeta.h +++ b/src/tsdb/inc/tsdbMeta.h @@ -36,6 +36,12 @@ typedef struct STable { char* sql; void* cqhandle; SRWLatch latch; // TODO: implementa latch functions + + SDataCol *lastCols; + int16_t maxColNum; + int16_t restoreColumnNum; + bool hasRestoreLastColumn; + int lastColSVersion; T_REF_DECLARE() } STable; @@ -78,6 +84,11 @@ void tsdbUnRefTable(STable* pTable); void tsdbUpdateTableSchema(STsdbRepo* pRepo, STable* pTable, STSchema* pSchema, bool insertAct); int tsdbRestoreTable(STsdbRepo* pRepo, void* cont, int contLen); void tsdbOrgMeta(STsdbRepo* pRepo); +int tsdbInitColIdCacheWithSchema(STable* pTable, STSchema* pSchema); +int16_t tsdbGetLastColumnsIndexByColId(STable* pTable, int16_t colId); +int tsdbUpdateLastColSchema(STable *pTable, STSchema *pNewSchema); +STSchema* tsdbGetTableLatestSchema(STable *pTable); +void tsdbFreeLastColumns(STable* pTable); static FORCE_INLINE int tsdbCompareSchemaVersion(const void *key1, const void *key2) { if (*(int16_t *)key1 < schemaVersion(*(STSchema **)key2)) { diff --git a/src/tsdb/inc/tsdbint.h b/src/tsdb/inc/tsdbint.h index 4d62164df9..945b74af35 100644 --- a/src/tsdb/inc/tsdbint.h +++ b/src/tsdb/inc/tsdbint.h @@ -75,6 +75,9 @@ struct STsdbRepo { STsdbCfg save_config; // save apply config bool config_changed; // config changed flag pthread_mutex_t save_mutex; // protect save config + + uint8_t hasCachedLastRow; + uint8_t hasCachedLastColumn; STsdbAppH appH; STsdbStat stat; @@ -83,6 +86,7 @@ struct STsdbRepo { SMemTable* mem; SMemTable* imem; STsdbFS* fs; + SRtn rtn; tsem_t readyToCommit; pthread_mutex_t mutex; bool repoLocked; @@ -100,6 +104,7 @@ int tsdbUnlockRepo(STsdbRepo* pRepo); STsdbMeta* tsdbGetMeta(STsdbRepo* pRepo); int tsdbCheckCommit(STsdbRepo* pRepo); int tsdbRestoreInfo(STsdbRepo* pRepo); +int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg); void tsdbGetRootDir(int repoid, char dirName[]); void tsdbGetDataDir(int repoid, char dirName[]); diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 24f1c11628..f2bbe347bd 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -86,10 +86,12 @@ static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError); static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo); static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, TSKEY maxKey, int maxRows, int8_t update); -static int tsdbApplyRtn(STsdbRepo *pRepo); static int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn); void *tsdbCommitData(STsdbRepo *pRepo) { + if (pRepo->imem == NULL) { + return NULL; + } tsdbStartCommit(pRepo); // Commit to update meta file @@ -1149,7 +1151,7 @@ static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const return -1; } - if (pSubBlocks && taosArrayPushBatch(pCommith->aSubBlk, pSubBlocks, nSubBlocks) == NULL) { + if (pSubBlocks && taosArrayAddBatch(pCommith->aSubBlk, pSubBlocks, nSubBlocks) == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; } @@ -1428,7 +1430,7 @@ static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *p return false; } -static int tsdbApplyRtn(STsdbRepo *pRepo) { +int tsdbApplyRtn(STsdbRepo *pRepo) { SRtn rtn; SFSIter fsiter; STsdbFS * pfs = REPO_FS(pRepo); diff --git a/src/tsdb/src/tsdbCommitQueue.c b/src/tsdb/src/tsdbCommitQueue.c index e753a3211e..abea79bc4f 100644 --- a/src/tsdb/src/tsdbCommitQueue.c +++ b/src/tsdb/src/tsdbCommitQueue.c @@ -113,11 +113,15 @@ int tsdbScheduleCommit(STsdbRepo *pRepo) { } static void tsdbApplyRepoConfig(STsdbRepo *pRepo) { + pthread_mutex_lock(&pRepo->save_mutex); + pRepo->config_changed = false; STsdbCfg * pSaveCfg = &pRepo->save_config; - + STsdbCfg oldCfg; int32_t oldTotalBlocks = pRepo->config.totalBlocks; + memcpy(&oldCfg, &(pRepo->config), sizeof(STsdbCfg)); + pRepo->config.compression = pRepo->save_config.compression; pRepo->config.keep = pRepo->save_config.keep; pRepo->config.keep1 = pRepo->save_config.keep1; @@ -125,10 +129,12 @@ static void tsdbApplyRepoConfig(STsdbRepo *pRepo) { pRepo->config.cacheLastRow = pRepo->save_config.cacheLastRow; pRepo->config.totalBlocks = pRepo->save_config.totalBlocks; - tsdbInfo("vgId:%d apply new config: compression(%d), keep(%d,%d,%d), totalBlocks(%d), cacheLastRow(%d),totalBlocks(%d)", + pthread_mutex_unlock(&pRepo->save_mutex); + + tsdbInfo("vgId:%d apply new config: compression(%d), keep(%d,%d,%d), totalBlocks(%d), cacheLastRow(%d->%d),totalBlocks(%d->%d)", REPO_ID(pRepo), pSaveCfg->compression, pSaveCfg->keep,pSaveCfg->keep1, pSaveCfg->keep2, - pSaveCfg->totalBlocks, pSaveCfg->cacheLastRow, pSaveCfg->totalBlocks); + pSaveCfg->totalBlocks, oldCfg.cacheLastRow, pSaveCfg->cacheLastRow, oldTotalBlocks, pSaveCfg->totalBlocks); int err = tsdbExpendPool(pRepo, oldTotalBlocks); if (!TAOS_SUCCEEDED(err)) { @@ -136,6 +142,12 @@ static void tsdbApplyRepoConfig(STsdbRepo *pRepo) { REPO_ID(pRepo), oldTotalBlocks, pSaveCfg->totalBlocks, tstrerror(err)); } + if (oldCfg.cacheLastRow != pRepo->config.cacheLastRow) { + if (tsdbLockRepo(pRepo) < 0) return; + tsdbCacheLastData(pRepo, &oldCfg); + tsdbUnlockRepo(pRepo); + } + } static void *tsdbLoopCommit(void *arg) { @@ -165,10 +177,8 @@ static void *tsdbLoopCommit(void *arg) { pRepo = ((SCommitReq *)pNode->data)->pRepo; // check if need to apply new config - if (pRepo->config_changed) { - pthread_mutex_lock(&pRepo->save_mutex); + if (pRepo->config_changed) { tsdbApplyRepoConfig(pRepo); - pthread_mutex_unlock(&pRepo->save_mutex); } tsdbCommitData(pRepo); diff --git a/src/tsdb/src/tsdbFS.c b/src/tsdb/src/tsdbFS.c index fd9b5e77e3..54372ae8c2 100644 --- a/src/tsdb/src/tsdbFS.c +++ b/src/tsdb/src/tsdbFS.c @@ -33,7 +33,9 @@ static int tsdbScanDataDir(STsdbRepo *pRepo); static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf); static int tsdbRestoreCurrent(STsdbRepo *pRepo); static int tsdbComparTFILE(const void *arg1, const void *arg2); -static void tsdbScanAndTryFixDFilesHeader(STsdbRepo *pRepo); +static void tsdbScanAndTryFixDFilesHeader(STsdbRepo *pRepo, int32_t *nExpired); +static int tsdbProcessExpiredFS(STsdbRepo *pRepo); +static int tsdbCreateMeta(STsdbRepo *pRepo); // ================== CURRENT file header info static int tsdbEncodeFSHeader(void **buf, SFSHeader *pHeader) { @@ -212,6 +214,8 @@ STsdbFS *tsdbNewFS(STsdbCfg *pCfg) { return NULL; } + pfs->intxn = false; + pfs->nstatus = tsdbNewFSStatus(maxFSet); if (pfs->nstatus == NULL) { tsdbFreeFS(pfs); @@ -234,22 +238,84 @@ void *tsdbFreeFS(STsdbFS *pfs) { return NULL; } +static int tsdbProcessExpiredFS(STsdbRepo *pRepo) { + tsdbStartFSTxn(pRepo, 0, 0); + if (tsdbCreateMeta(pRepo) < 0) { + tsdbError("vgId:%d failed to create meta since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + if (tsdbApplyRtn(pRepo) < 0) { + tsdbEndFSTxnWithError(REPO_FS(pRepo)); + tsdbError("vgId:%d failed to apply rtn since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + if (tsdbEndFSTxn(pRepo) < 0) { + tsdbError("vgId:%d failed to end fs txn since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + return 0; +} + +static int tsdbCreateMeta(STsdbRepo *pRepo) { + STsdbFS *pfs = REPO_FS(pRepo); + SMFile * pOMFile = pfs->cstatus->pmf; + SMFile mf; + SDiskID did; + + if (pOMFile != NULL) { + // keep the old meta file + tsdbUpdateMFile(pfs, pOMFile); + return 0; + } + + // Create a new meta file + did.level = TFS_PRIMARY_LEVEL; + did.id = TFS_PRIMARY_ID; + tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); + + if (tsdbCreateMFile(&mf, true) < 0) { + tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + tsdbInfo("vgId:%d meta file %s is created", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf)); + + if (tsdbUpdateMFileHeader(&mf) < 0) { + tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno)); + tsdbApplyMFileChange(&mf, pOMFile); + return -1; + } + + TSDB_FILE_FSYNC(&mf); + tsdbCloseMFile(&mf); + tsdbUpdateMFile(pfs, &mf); + + return 0; +} + int tsdbOpenFS(STsdbRepo *pRepo) { STsdbFS *pfs = REPO_FS(pRepo); char current[TSDB_FILENAME_LEN] = "\0"; + int nExpired = 0; ASSERT(pfs != NULL); tsdbGetTxnFname(REPO_ID(pRepo), TSDB_TXN_CURR_FILE, current); + tsdbGetRtnSnap(pRepo, &pRepo->rtn); if (access(current, F_OK) == 0) { if (tsdbOpenFSFromCurrent(pRepo) < 0) { tsdbError("vgId:%d failed to open FS since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; } - tsdbScanAndTryFixDFilesHeader(pRepo); + tsdbScanAndTryFixDFilesHeader(pRepo, &nExpired); + if (nExpired > 0) { + tsdbProcessExpiredFS(pRepo); + } } else { + // should skip expired fileset inside of the function if (tsdbRestoreCurrent(pRepo) < 0) { tsdbError("vgId:%d failed to restore current file since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; @@ -1110,6 +1176,11 @@ static int tsdbRestoreDFileSet(STsdbRepo *pRepo) { ASSERT(tvid == REPO_ID(pRepo)); + if (tfid < pRepo->rtn.minFid) { // skip file expired + ++index; + continue; + } + if (ftype == 0) { fset.fid = tfid; } else { @@ -1206,7 +1277,7 @@ static int tsdbComparTFILE(const void *arg1, const void *arg2) { } } -static void tsdbScanAndTryFixDFilesHeader(STsdbRepo *pRepo) { +static void tsdbScanAndTryFixDFilesHeader(STsdbRepo *pRepo, int32_t *nExpired) { STsdbFS * pfs = REPO_FS(pRepo); SFSStatus *pStatus = pfs->cstatus; SDFInfo info; @@ -1214,7 +1285,9 @@ static void tsdbScanAndTryFixDFilesHeader(STsdbRepo *pRepo) { for (size_t i = 0; i < taosArrayGetSize(pStatus->df); i++) { SDFileSet fset; tsdbInitDFileSetEx(&fset, (SDFileSet *)taosArrayGet(pStatus->df, i)); - + if (fset.fid < pRepo->rtn.minFid) { + ++*nExpired; + } tsdbDebug("vgId:%d scan DFileSet %d header", REPO_ID(pRepo), fset.fid); if (tsdbOpenDFileSet(&fset, O_RDWR) < 0) { diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index fd02a3c8b9..afbedd5b2f 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -26,6 +26,8 @@ static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH); static void tsdbFreeRepo(STsdbRepo *pRepo); static void tsdbStartStream(STsdbRepo *pRepo); static void tsdbStopStream(STsdbRepo *pRepo); +static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh); +static int tsdbRestoreLastRow(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh, SBlockIdx *pIdx); // Function declaration int32_t tsdbCreateRepo(int repoid) { @@ -267,6 +269,10 @@ int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) { repo->config_changed = true; pthread_mutex_unlock(&repo->save_mutex); + + // schedule a commit msg then the new config will be applied immediatly + tsdbAsyncCommit(repo); + return 0; #if 0 STsdbRepo *pRepo = (STsdbRepo *)repo; @@ -511,8 +517,10 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { if (pCfg->update != 0) pCfg->update = 1; // update cacheLastRow - if (pCfg->cacheLastRow != 0) pCfg->cacheLastRow = 1; - + if (pCfg->cacheLastRow != 0) { + if (pCfg->cacheLastRow > 3) + pCfg->cacheLastRow = 1; + } return 0; } @@ -545,6 +553,8 @@ static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { return NULL; } pRepo->config_changed = false; + atomic_store_8(&pRepo->hasCachedLastRow, 0); + atomic_store_8(&pRepo->hasCachedLastColumn, 0); code = tsem_init(&(pRepo->readyToCommit), 0, 1); if (code != 0) { @@ -614,13 +624,180 @@ static void tsdbStopStream(STsdbRepo *pRepo) { } } +static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh) { + //tsdbInfo("tsdbRestoreLastColumns of table %s", pTable->name->data); + + STSchema *pSchema = tsdbGetTableLatestSchema(pTable); + if (pSchema == NULL) { + tsdbError("tsdbGetTableLatestSchema of table %s fail", pTable->name->data); + return 0; + } + + SBlock* pBlock; + int numColumns; + int32_t blockIdx; + SDataStatis* pBlockStatis = NULL; + SDataRow row = NULL; + // restore last column data with last schema + + int err = 0; + + numColumns = schemaNCols(pSchema); + if (numColumns <= pTable->restoreColumnNum) { + pTable->hasRestoreLastColumn = true; + return 0; + } + if (pTable->lastColSVersion != schemaVersion(pSchema)) { + if (tsdbInitColIdCacheWithSchema(pTable, pSchema) < 0) { + return -1; + } + } + + row = taosTMalloc(dataRowMaxBytesFromSchema(pSchema)); + if (row == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + err = -1; + goto out; + } + tdInitDataRow(row, pSchema); + + // first load block index info + if (tsdbLoadBlockInfo(pReadh, NULL) < 0) { + err = -1; + goto out; + } + + pBlockStatis = calloc(numColumns, sizeof(SDataStatis)); + if (pBlockStatis == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + err = -1; + goto out; + } + memset(pBlockStatis, 0, numColumns * sizeof(SDataStatis)); + for(int32_t i = 0; i < numColumns; ++i) { + STColumn *pCol = schemaColAt(pSchema, i); + pBlockStatis[i].colId = pCol->colId; + } + + // load block from backward + SBlockIdx *pIdx = pReadh->pBlkIdx; + blockIdx = (int32_t)(pIdx->numOfBlocks - 1); + + while (numColumns > pTable->restoreColumnNum && blockIdx >= 0) { + bool loadStatisData = false; + pBlock = pReadh->pBlkInfo->blocks + blockIdx; + blockIdx -= 1; + + // load block data + if (tsdbLoadBlockData(pReadh, pBlock, NULL) < 0) { + err = -1; + goto out; + } + + // file block with sub-blocks has no statistics data + if (pBlock->numOfSubBlocks <= 1) { + tsdbLoadBlockStatis(pReadh, pBlock); + tsdbGetBlockStatis(pReadh, pBlockStatis, (int)numColumns); + loadStatisData = true; + } + + for (int16_t i = 0; i < numColumns && numColumns > pTable->restoreColumnNum; ++i) { + STColumn *pCol = schemaColAt(pSchema, i); + // ignore loaded columns + if (pTable->lastCols[i].bytes != 0) { + continue; + } + + // ignore block which has no not-null colId column + if (loadStatisData && pBlockStatis[i].numOfNull == pBlock->numOfRows) { + continue; + } + + // OK,let's load row from backward to get not-null column + for (int32_t rowId = pBlock->numOfRows - 1; rowId >= 0; rowId--) { + SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; + tdAppendColVal(row, tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->bytes, pCol->offset); + //SDataCol *pDataCol = readh.pDCols[0]->cols + j; + void* value = tdGetRowDataOfCol(row, (int8_t)pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); + if (isNull(value, pCol->type)) { + continue; + } + + int16_t idx = tsdbGetLastColumnsIndexByColId(pTable, pCol->colId); + if (idx == -1) { + tsdbError("tsdbRestoreLastColumns restore vgId:%d,table:%s cache column %d fail", REPO_ID(pRepo), pTable->name->data, pCol->colId); + continue; + } + // save not-null column + SDataCol *pLastCol = &(pTable->lastCols[idx]); + pLastCol->pData = malloc(pCol->bytes); + pLastCol->bytes = pCol->bytes; + pLastCol->colId = pCol->colId; + memcpy(pLastCol->pData, value, pCol->bytes); + + // save row ts(in column 0) + pDataCol = pReadh->pDCols[0]->cols + 0; + pCol = schemaColAt(pSchema, 0); + tdAppendColVal(row, tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->bytes, pCol->offset); + pLastCol->ts = dataRowKey(row); + + pTable->restoreColumnNum += 1; + + tsdbDebug("tsdbRestoreLastColumns restore vgId:%d,table:%s cache column %d, %" PRId64, REPO_ID(pRepo), pTable->name->data, pLastCol->colId, pLastCol->ts); + break; + } + } + } + +out: + taosTZfree(row); + tfree(pBlockStatis); + + if (err == 0 && numColumns <= pTable->restoreColumnNum) { + pTable->hasRestoreLastColumn = true; + } + + return err; +} + +static int tsdbRestoreLastRow(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh, SBlockIdx *pIdx) { + ASSERT(pTable->lastRow == NULL); + if (tsdbLoadBlockInfo(pReadh, NULL) < 0) { + return -1; + } + + SBlock* pBlock = pReadh->pBlkInfo->blocks + pIdx->numOfBlocks - 1; + + if (tsdbLoadBlockData(pReadh, pBlock, NULL) < 0) { + return -1; + } + + // Get the data in row + + STSchema *pSchema = tsdbGetTableSchema(pTable); + pTable->lastRow = taosTMalloc(dataRowMaxBytesFromSchema(pSchema)); + if (pTable->lastRow == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + tdInitDataRow(pTable->lastRow, pSchema); + for (int icol = 0; icol < schemaNCols(pSchema); icol++) { + STColumn *pCol = schemaColAt(pSchema, icol); + SDataCol *pDataCol = pReadh->pDCols[0]->cols + icol; + tdAppendColVal(pTable->lastRow, tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, pCol->bytes, + pCol->offset); + } + + return 0; +} + int tsdbRestoreInfo(STsdbRepo *pRepo) { SFSIter fsiter; SReadH readh; SDFileSet *pSet; STsdbMeta *pMeta = pRepo->tsdbMeta; STsdbCfg * pCfg = REPO_CFG(pRepo); - SBlock * pBlock; if (tsdbInitReadH(&readh, pRepo) < 0) { return -1; @@ -628,6 +805,14 @@ int tsdbRestoreInfo(STsdbRepo *pRepo) { tsdbFSIterInit(&fsiter, REPO_FS(pRepo), TSDB_FS_ITER_BACKWARD); + if (CACHE_LAST_NULL_COLUMN(pCfg)) { + for (int i = 1; i < pMeta->maxTables; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable == NULL) continue; + pTable->restoreColumnNum = 0; + } + } + while ((pSet = tsdbFSIterNext(&fsiter)) != NULL) { if (tsdbSetAndOpenReadFSet(&readh, pSet) < 0) { tsdbDestroyReadH(&readh); @@ -643,6 +828,8 @@ int tsdbRestoreInfo(STsdbRepo *pRepo) { STable *pTable = pMeta->tables[i]; if (pTable == NULL) continue; + //tsdbInfo("tsdbRestoreInfo restore vgId:%d,table:%s", REPO_ID(pRepo), pTable->name->data); + if (tsdbSetReadTable(&readh, pTable) < 0) { tsdbDestroyReadH(&readh); return -1; @@ -653,42 +840,155 @@ int tsdbRestoreInfo(STsdbRepo *pRepo) { if (pIdx && lastKey < pIdx->maxKey) { pTable->lastKey = pIdx->maxKey; - if (pCfg->cacheLastRow) { - if (tsdbLoadBlockInfo(&readh, NULL) < 0) { - tsdbDestroyReadH(&readh); - return -1; - } - - pBlock = readh.pBlkInfo->blocks + pIdx->numOfBlocks - 1; - - if (tsdbLoadBlockData(&readh, pBlock, NULL) < 0) { - tsdbDestroyReadH(&readh); - return -1; - } - - // Get the data in row - ASSERT(pTable->lastRow == NULL); - STSchema *pSchema = tsdbGetTableSchema(pTable); - pTable->lastRow = taosTMalloc(dataRowMaxBytesFromSchema(pSchema)); - if (pTable->lastRow == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbDestroyReadH(&readh); - return -1; - } - - tdInitDataRow(pTable->lastRow, pSchema); - for (int icol = 0; icol < schemaNCols(pSchema); icol++) { - STColumn *pCol = schemaColAt(pSchema, icol); - SDataCol *pDataCol = readh.pDCols[0]->cols + icol; - tdAppendColVal(pTable->lastRow, tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, pCol->bytes, - pCol->offset); - } + if (CACHE_LAST_ROW(pCfg) && tsdbRestoreLastRow(pRepo, pTable, &readh, pIdx) != 0) { + tsdbDestroyReadH(&readh); + return -1; } } + // restore NULL columns + if (pIdx && CACHE_LAST_NULL_COLUMN(pCfg) && !pTable->hasRestoreLastColumn) { + if (tsdbRestoreLastColumns(pRepo, pTable, &readh) != 0) { + tsdbDestroyReadH(&readh); + return -1; + } + } } } tsdbDestroyReadH(&readh); + if (CACHE_LAST_ROW(pCfg)) { + atomic_store_8(&pRepo->hasCachedLastRow, 1); + } + if (CACHE_LAST_NULL_COLUMN(pCfg)) { + atomic_store_8(&pRepo->hasCachedLastColumn, 1); + } + return 0; } + +int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg) { + bool cacheLastRow = false, cacheLastCol = false; + SFSIter fsiter; + SReadH readh; + SDFileSet *pSet; + STsdbMeta *pMeta = pRepo->tsdbMeta; + int tableNum = 0; + int maxTableIdx = 0; + int cacheLastRowTableNum = 0; + int cacheLastColTableNum = 0; + + bool need_free_last_row = CACHE_LAST_ROW(oldCfg) && !CACHE_LAST_ROW(&(pRepo->config)); + bool need_free_last_col = CACHE_LAST_NULL_COLUMN(oldCfg) && !CACHE_LAST_NULL_COLUMN(&(pRepo->config)); + + if (CACHE_LAST_ROW(&(pRepo->config)) || CACHE_LAST_NULL_COLUMN(&(pRepo->config))) { + tsdbInfo("tsdbCacheLastData cache last data since cacheLast option changed"); + cacheLastRow = !CACHE_LAST_ROW(oldCfg) && CACHE_LAST_ROW(&(pRepo->config)); + cacheLastCol = !CACHE_LAST_NULL_COLUMN(oldCfg) && CACHE_LAST_NULL_COLUMN(&(pRepo->config)); + } + + // calc max table idx and table num + for (int i = 1; i < pMeta->maxTables; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable == NULL) continue; + tableNum += 1; + maxTableIdx = i; + if (cacheLastCol) { + pTable->restoreColumnNum = 0; + } + } + + // if close last option,need to free data + if (need_free_last_row || need_free_last_col) { + if (need_free_last_row) { + atomic_store_8(&pRepo->hasCachedLastRow, 0); + } + if (need_free_last_col) { + atomic_store_8(&pRepo->hasCachedLastColumn, 0); + } + tsdbInfo("free cache last data since cacheLast option changed"); + for (int i = 1; i < maxTableIdx; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable == NULL) continue; + if (need_free_last_row) { + taosTZfree(pTable->lastRow); + pTable->lastRow = NULL; + pTable->lastKey = TSKEY_INITIAL_VAL; + } + if (need_free_last_col) { + tsdbFreeLastColumns(pTable); + } + } + } + + if (!cacheLastRow && !cacheLastCol) { + return 0; + } + + cacheLastRowTableNum = cacheLastRow ? tableNum : 0; + cacheLastColTableNum = cacheLastCol ? tableNum : 0; + + if (tsdbInitReadH(&readh, pRepo) < 0) { + return -1; + } + + tsdbFSIterInit(&fsiter, REPO_FS(pRepo), TSDB_FS_ITER_BACKWARD); + + while ((pSet = tsdbFSIterNext(&fsiter)) != NULL && (cacheLastRowTableNum > 0 || cacheLastColTableNum > 0)) { + if (tsdbSetAndOpenReadFSet(&readh, pSet) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + if (tsdbLoadBlockIdx(&readh) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + for (int i = 1; i <= maxTableIdx; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable == NULL) continue; + + //tsdbInfo("tsdbRestoreInfo restore vgId:%d,table:%s", REPO_ID(pRepo), pTable->name->data); + + if (tsdbSetReadTable(&readh, pTable) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + SBlockIdx *pIdx = readh.pBlkIdx; + + if (pIdx && cacheLastRowTableNum > 0 && pTable->lastRow == NULL) { + pTable->lastKey = pIdx->maxKey; + + if (tsdbRestoreLastRow(pRepo, pTable, &readh, pIdx) != 0) { + tsdbDestroyReadH(&readh); + return -1; + } + cacheLastRowTableNum -= 1; + } + + // restore NULL columns + if (pIdx && cacheLastColTableNum > 0 && !pTable->hasRestoreLastColumn) { + if (tsdbRestoreLastColumns(pRepo, pTable, &readh) != 0) { + tsdbDestroyReadH(&readh); + return -1; + } + if (pTable->hasRestoreLastColumn) { + cacheLastColTableNum -= 1; + } + } + } + } + + tsdbDestroyReadH(&readh); + + if (cacheLastRow) { + atomic_store_8(&pRepo->hasCachedLastRow, 1); + } + if (cacheLastCol) { + atomic_store_8(&pRepo->hasCachedLastColumn, 1); + } + + return 0; +} \ No newline at end of file diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index c6fcf55686..79dbb8be5d 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -274,7 +274,7 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { int tsdbAsyncCommit(STsdbRepo *pRepo) { tsem_wait(&(pRepo->readyToCommit)); - ASSERT(pRepo->imem == NULL); + //ASSERT(pRepo->imem == NULL); if (pRepo->mem == NULL) { tsem_post(&(pRepo->readyToCommit)); return 0; @@ -964,6 +964,49 @@ static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter) { } } +static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SDataRow row) { + tsdbDebug("vgId:%d updateTableLatestColumn, %s row version:%d", REPO_ID(pRepo), pTable->name->data, dataRowVersion(row)); + + STSchema* pSchema = tsdbGetTableLatestSchema(pTable); + if (tsdbUpdateLastColSchema(pTable, pSchema) < 0) { + return; + } + + pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); + if (pSchema == NULL) { + return; + } + + SDataCol *pLatestCols = pTable->lastCols; + + for (int16_t j = 0; j < schemaNCols(pSchema); j++) { + STColumn *pTCol = schemaColAt(pSchema, j); + // ignore not exist colId + int16_t idx = tsdbGetLastColumnsIndexByColId(pTable, pTCol->colId); + if (idx == -1) { + continue; + } + + void* value = tdGetRowDataOfCol(row, (int8_t)pTCol->type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset); + if (isNull(value, pTCol->type)) { + continue; + } + + SDataCol *pDataCol = &(pLatestCols[idx]); + if (pDataCol->pData == NULL) { + pDataCol->pData = malloc(pSchema->columns[j].bytes); + pDataCol->bytes = pSchema->columns[j].bytes; + } else if (pDataCol->bytes < pSchema->columns[j].bytes) { + pDataCol->pData = realloc(pDataCol->pData, pSchema->columns[j].bytes); + pDataCol->bytes = pSchema->columns[j].bytes; + } + + memcpy(pDataCol->pData, value, pDataCol->bytes); + //tsdbInfo("updateTableLatestColumn vgId:%d cache column %d for %d,%s", REPO_ID(pRepo), j, pDataCol->bytes, (char*)pDataCol->pData); + pDataCol->ts = dataRowKey(row); + } +} + static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow row) { STsdbCfg *pCfg = &pRepo->config; @@ -977,7 +1020,7 @@ static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow } if (tsdbGetTableLastKeyImpl(pTable) < dataRowKey(row)) { - if (pCfg->cacheLastRow || pTable->lastRow != NULL) { + if (CACHE_LAST_ROW(pCfg) || pTable->lastRow != NULL) { SDataRow nrow = pTable->lastRow; if (taosTSizeof(nrow) < dataRowLen(row)) { SDataRow orow = nrow; @@ -1002,7 +1045,10 @@ static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow } else { pTable->lastKey = dataRowKey(row); } - } + if (CACHE_LAST_NULL_COLUMN(pCfg)) { + updateTableLatestColumn(pRepo, pTable, row); + } + } return 0; } diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index e6cbc4da9e..324a7c79c5 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -589,6 +589,131 @@ void tsdbUnRefTable(STable *pTable) { } } +void tsdbFreeLastColumns(STable* pTable) { + if (pTable->lastCols == NULL) { + return; + } + + for (int i = 0; i < pTable->maxColNum; ++i) { + if (pTable->lastCols[i].bytes == 0) { + continue; + } + tfree(pTable->lastCols[i].pData); + pTable->lastCols[i].bytes = 0; + pTable->lastCols[i].pData = NULL; + } + tfree(pTable->lastCols); + pTable->lastCols = NULL; + pTable->maxColNum = 0; + pTable->lastColSVersion = -1; + pTable->restoreColumnNum = 0; +} + +int16_t tsdbGetLastColumnsIndexByColId(STable* pTable, int16_t colId) { + if (pTable->lastCols == NULL) { + return -1; + } + for (int16_t i = 0; i < pTable->maxColNum; ++i) { + if (pTable->lastCols[i].colId == colId) { + return i; + } + } + + return -1; +} + +int tsdbInitColIdCacheWithSchema(STable* pTable, STSchema* pSchema) { + ASSERT(pTable->lastCols == NULL); + + int16_t numOfColumn = pSchema->numOfCols; + + pTable->lastCols = (SDataCol*)malloc(numOfColumn * sizeof(SDataCol)); + if (pTable->lastCols == NULL) { + return -1; + } + + for (int16_t i = 0; i < numOfColumn; ++i) { + STColumn *pCol = schemaColAt(pSchema, i); + SDataCol* pDataCol = &(pTable->lastCols[i]); + pDataCol->bytes = 0; + pDataCol->pData = NULL; + pDataCol->colId = pCol->colId; + } + + pTable->lastColSVersion = schemaVersion(pSchema); + pTable->maxColNum = numOfColumn; + pTable->restoreColumnNum = 0; + return 0; +} + +STSchema* tsdbGetTableLatestSchema(STable *pTable) { + return tsdbGetTableSchemaByVersion(pTable, -1); +} + +int tsdbUpdateLastColSchema(STable *pTable, STSchema *pNewSchema) { + if (pTable->lastColSVersion == schemaVersion(pNewSchema)) { + return 0; + } + + tsdbInfo("tsdbUpdateLastColSchema:%s,%d->%d", pTable->name->data, pTable->lastColSVersion, schemaVersion(pNewSchema)); + + int16_t numOfCols = pNewSchema->numOfCols; + SDataCol *lastCols = (SDataCol*)malloc(numOfCols * sizeof(SDataCol)); + if (lastCols == NULL) { + return -1; + } + + TSDB_WLOCK_TABLE(pTable); + + for (int16_t i = 0; i < numOfCols; ++i) { + STColumn *pCol = schemaColAt(pNewSchema, i); + int16_t idx = tsdbGetLastColumnsIndexByColId(pTable, pCol->colId); + + SDataCol* pDataCol = &(lastCols[i]); + if (idx != -1) { + // move col data to new last column array + SDataCol* pOldDataCol = &(pTable->lastCols[idx]); + memcpy(pDataCol, pOldDataCol, sizeof(SDataCol)); + } else { + // init new colid data + pDataCol->colId = pCol->colId; + pDataCol->bytes = 0; + pDataCol->pData = NULL; + } + } + + SDataCol *oldLastCols = pTable->lastCols; + int16_t oldLastColNum = pTable->maxColNum; + + pTable->lastColSVersion = schemaVersion(pNewSchema); + pTable->lastCols = lastCols; + pTable->maxColNum = numOfCols; + + if (oldLastCols == NULL) { + TSDB_WUNLOCK_TABLE(pTable); + return 0; + } + + // free old schema last column datas + for (int16_t i = 0; i < oldLastColNum; ++i) { + SDataCol* pDataCol = &(oldLastCols[i]); + if (pDataCol->bytes == 0) { + continue; + } + int16_t idx = tsdbGetLastColumnsIndexByColId(pTable, pDataCol->colId); + if (idx != -1) { + continue; + } + + // free not exist column data + tfree(pDataCol->pData); + } + TSDB_WUNLOCK_TABLE(pTable); + tfree(oldLastCols); + + return 0; +} + void tsdbUpdateTableSchema(STsdbRepo *pRepo, STable *pTable, STSchema *pSchema, bool insertAct) { ASSERT(TABLE_TYPE(pTable) != TSDB_STREAM_TABLE && TABLE_TYPE(pTable) != TSDB_SUPER_TABLE); STsdbMeta *pMeta = pRepo->tsdbMeta; @@ -672,6 +797,10 @@ static STable *tsdbNewTable() { pTable->lastKey = TSKEY_INITIAL_VAL; + pTable->lastCols = NULL; + pTable->restoreColumnNum = 0; + pTable->maxColNum = 0; + pTable->lastColSVersion = -1; return pTable; } @@ -785,8 +914,10 @@ static void tsdbFreeTable(STable *pTable) { kvRowFree(pTable->tagVal); tSkipListDestroy(pTable->pIndex); - taosTZfree(pTable->lastRow); + taosTZfree(pTable->lastRow); tfree(pTable->sql); + + tsdbFreeLastColumns(pTable); free(pTable); } } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 9df25409de..fd2b403184 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -62,6 +62,7 @@ typedef struct SLoadCompBlockInfo { int32_t fileId; } SLoadCompBlockInfo; + typedef struct STableCheckInfo { STableId tableId; TSKEY lastKey; @@ -107,7 +108,7 @@ typedef struct STsdbQueryHandle { SArray* pTableCheckInfo; // SArray int32_t activeIndex; bool checkFiles; // check file stage - bool cachelastrow; // check if last row cached + int8_t cachelastrow; // check if last row cached bool loadExternalRow; // load time window external data rows bool currentLoadExternalRows; // current load external rows int32_t loadType; // block load type @@ -117,7 +118,6 @@ typedef struct STsdbQueryHandle { SFSIter fileIter; SReadH rhelper; STableBlockInfo* pDataBlockInfo; - SDataCols *pDataCols; // in order to hold current file data block int32_t allocSize; // allocated data block size SMemRef *pMemRef; @@ -138,6 +138,7 @@ typedef struct STableGroupSupporter { static STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList); static int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList); +static int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle); static int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey); static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); @@ -512,6 +513,8 @@ void tsdbResetQueryHandleForNewTable(TsdbQueryHandleT queryHandle, STsdbQueryCon pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); } + + TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef) { pCond->twindow = updateLastrowForEachGroup(groupList); @@ -528,10 +531,30 @@ TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STable } assert(pCond->order == TSDB_ORDER_ASC && pCond->twindow.skey <= pCond->twindow.ekey); - pQueryHandle->type = TSDB_QUERY_TYPE_LAST; + if (pQueryHandle->cachelastrow) { + pQueryHandle->type = TSDB_QUERY_TYPE_LAST; + } + return pQueryHandle; } + +TsdbQueryHandleT tsdbQueryCacheLast(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef) { + STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); + int32_t code = checkForCachedLast(pQueryHandle); + if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0 + terrno = code; + return NULL; + } + + if (pQueryHandle->cachelastrow) { + pQueryHandle->type = TSDB_QUERY_TYPE_LAST; + } + + return pQueryHandle; +} + + SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) { assert(pHandle != NULL); @@ -2105,6 +2128,7 @@ int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) queryHandle; pTableBlockInfo->totalSize = 0; + pTableBlockInfo->totalRows = 0; STsdbFS* pFileHandle = REPO_FS(pQueryHandle->pTsdb); // find the start data block in file @@ -2178,7 +2202,12 @@ int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist pTableBlockInfo->totalSize += pBlock[j].len; int32_t numOfRows = pBlock[j].numOfRows; - taosArrayPush(pTableBlockInfo->dataBlockInfos, &numOfRows); + pTableBlockInfo->totalRows += numOfRows; + if (numOfRows > pTableBlockInfo->maxRows) pTableBlockInfo->maxRows = numOfRows; + if (numOfRows < pTableBlockInfo->minRows) pTableBlockInfo->minRows = numOfRows; + int32_t stepIndex = (numOfRows-1)/TSDB_BLOCK_DIST_STEP_ROWS; + SFileBlockInfo *blockInfo = (SFileBlockInfo*)taosArrayGet(pTableBlockInfo->dataBlockInfos, stepIndex); + blockInfo->numBlocksOfStep++; } } } @@ -2460,6 +2489,159 @@ static bool loadCachedLastRow(STsdbQueryHandle* pQueryHandle) { return false; } + + +static bool loadCachedLast(STsdbQueryHandle* pQueryHandle) { + // the last row is cached in buffer, return it directly. + // here note that the pQueryHandle->window must be the TS_INITIALIZER + int32_t tgNumOfCols = (int32_t)QH_GET_NUM_OF_COLS(pQueryHandle); + size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + int32_t numOfRows = 0; + assert(numOfTables > 0 && tgNumOfCols > 0); + SQueryFilePos* cur = &pQueryHandle->cur; + TSKEY priKey = TSKEY_INITIAL_VAL; + int32_t priIdx = -1; + SColumnInfoData* pColInfo = NULL; + + while (++pQueryHandle->activeIndex < numOfTables) { + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + STable* pTable = pCheckInfo->pTableObj; + char* pData = NULL; + + int32_t numOfCols = pTable->maxColNum; + + if (pTable->lastCols == NULL || pTable->maxColNum <= 0) { + tsdbWarn("no last cached for table, uid:%" PRIu64 ",tid:%d", pTable->tableId.uid, pTable->tableId.tid); + continue; + } + + int32_t i = 0, j = 0; + while(i < tgNumOfCols && j < numOfCols) { + pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + if (pTable->lastCols[j].colId < pColInfo->info.colId) { + j++; + continue; + } else if (pTable->lastCols[j].colId > pColInfo->info.colId) { + i++; + continue; + } + + pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; + + if (pTable->lastCols[j].bytes > 0) { + void* value = pTable->lastCols[j].pData; + switch (pColInfo->info.type) { + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + memcpy(pData, value, varDataTLen(value)); + break; + case TSDB_DATA_TYPE_NULL: + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + *(uint8_t *)pData = *(uint8_t *)value; + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + *(uint16_t *)pData = *(uint16_t *)value; + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + *(uint32_t *)pData = *(uint32_t *)value; + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + *(uint64_t *)pData = *(uint64_t *)value; + break; + case TSDB_DATA_TYPE_FLOAT: + SET_FLOAT_PTR(pData, value); + break; + case TSDB_DATA_TYPE_DOUBLE: + SET_DOUBLE_PTR(pData, value); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + priKey = tdGetKey(*(TKEY *)value); + priIdx = i; + + i++; + j++; + continue; + } else { + *(TSKEY *)pData = *(TSKEY *)value; + } + break; + default: + memcpy(pData, value, pColInfo->info.bytes); + } + + for (int32_t n = 0; n < tgNumOfCols; ++n) { + if (n == i) { + continue; + } + + pColInfo = taosArrayGet(pQueryHandle->pColumns, n); + pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; + + if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + *(TSKEY *)pData = pTable->lastCols[j].ts; + continue; + } + + if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { + setVardataNull(pData, pColInfo->info.type); + } else { + setNull(pData, pColInfo->info.type, pColInfo->info.bytes); + } + } + + numOfRows++; + assert(numOfRows < pQueryHandle->outputCapacity); + } + + i++; + j++; + } + + // leave the real ts column as the last row, because last function only (not stable) use the last row as res + if (priKey != TSKEY_INITIAL_VAL) { + pColInfo = taosArrayGet(pQueryHandle->pColumns, priIdx); + pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; + + *(TSKEY *)pData = priKey; + + for (int32_t n = 0; n < tgNumOfCols; ++n) { + if (n == priIdx) { + continue; + } + + pColInfo = taosArrayGet(pQueryHandle->pColumns, n); + pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; + + assert (pColInfo->info.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX); + + if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { + setVardataNull(pData, pColInfo->info.type); + } else { + setNull(pData, pColInfo->info.type, pColInfo->info.bytes); + } + } + + numOfRows++; + } + + if (numOfRows > 0) { + cur->rows = numOfRows; + cur->mixBlock = true; + + return true; + } + } + + return false; +} + + static bool loadDataBlockFromTableSeq(STsdbQueryHandle* pQueryHandle) { size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); assert(numOfTables > 0); @@ -2496,8 +2678,12 @@ bool tsdbNextDataBlock(TsdbQueryHandleT pHandle) { int64_t stime = taosGetTimestampUs(); int64_t elapsedTime = stime; - if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST && pQueryHandle->cachelastrow) { - return loadCachedLastRow(pQueryHandle); + if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST) { + if (pQueryHandle->cachelastrow == 1) { + return loadCachedLastRow(pQueryHandle); + } else if (pQueryHandle->cachelastrow == 2) { + return loadCachedLast(pQueryHandle); + } } if (pQueryHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { @@ -2695,6 +2881,10 @@ int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey) { return TSDB_CODE_SUCCESS; } +bool isTsdbCacheLastRow(TsdbQueryHandleT* pQueryHandle) { + return ((STsdbQueryHandle *)pQueryHandle)->cachelastrow > 0; +} + int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList) { assert(pQueryHandle != NULL && groupList != NULL); @@ -2706,11 +2896,15 @@ int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *g STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(group, 0); - int32_t code = tsdbGetCachedLastRow(pInfo->pTable, &pRow, &key); - if (code != TSDB_CODE_SUCCESS) { - pQueryHandle->cachelastrow = false; - } else { - pQueryHandle->cachelastrow = (pRow != NULL); + int32_t code = 0; + + if (((STable*)pInfo->pTable)->lastRow) { + code = tsdbGetCachedLastRow(pInfo->pTable, &pRow, &key); + if (code != TSDB_CODE_SUCCESS) { + pQueryHandle->cachelastrow = 0; + } else { + pQueryHandle->cachelastrow = 1; + } } // update the tsdb query time range @@ -2724,6 +2918,26 @@ int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *g return code; } +int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle) { + assert(pQueryHandle != NULL); + + int32_t code = 0; + + if (pQueryHandle->pTsdb && atomic_load_8(&pQueryHandle->pTsdb->hasCachedLastColumn)){ + pQueryHandle->cachelastrow = 2; + } + + // update the tsdb query time range + if (pQueryHandle->cachelastrow) { + pQueryHandle->window = TSWINDOW_INITIALIZER; + pQueryHandle->checkFiles = false; + pQueryHandle->activeIndex = -1; // start from -1 + } + + return code; +} + + STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { STimeWindow window = {INT64_MAX, INT64_MIN}; diff --git a/src/tsdb/src/tsdbSync.c b/src/tsdb/src/tsdbSync.c index 5a2756537e..edcb84d091 100644 --- a/src/tsdb/src/tsdbSync.c +++ b/src/tsdb/src/tsdbSync.c @@ -424,24 +424,42 @@ static int32_t tsdbSyncRecvDFileSetArray(SSyncH *pSynch) { } if (tsdbSendDecision(pSynch, false) < 0) { - tsdbError("vgId:%d, filed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno)); + tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; } } else { // Need to copy from remote - tsdbInfo("vgId:%d, fileset:%d will be received", REPO_ID(pRepo), pSynch->pdf->fid); - - // Notify remote to send there file here - if (tsdbSendDecision(pSynch, true) < 0) { - tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; + int fidLevel = tsdbGetFidLevel(pSynch->pdf->fid, &(pSynch->rtn)); + if (fidLevel < 0) { // expired fileset + tsdbInfo("vgId:%d, fileset:%d will be skipped as expired", REPO_ID(pRepo), pSynch->pdf->fid); + if (tsdbSendDecision(pSynch, false) < 0) { + tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + // Move forward + if (tsdbRecvDFileSetInfo(pSynch) < 0) { + tsdbError("vgId:%d, failed to recv fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + if (pLSet) { + pLSet = tsdbFSIterNext(&fsiter); + } + // Next loop + continue; + } else { + tsdbInfo("vgId:%d, fileset:%d will be received", REPO_ID(pRepo), pSynch->pdf->fid); + // Notify remote to send there file here + if (tsdbSendDecision(pSynch, true) < 0) { + tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } } // Create local files and copy from remote SDiskID did; SDFileSet fset; - tfsAllocDisk(tsdbGetFidLevel(pSynch->pdf->fid, &(pSynch->rtn)), &(did.level), &(did.id)); + tfsAllocDisk(fidLevel, &(did.level), &(did.id)); if (did.level == TFS_UNDECIDED_LEVEL) { terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; tsdbError("vgId:%d, failed allc disk since %s", REPO_ID(pRepo), tstrerror(terrno)); @@ -548,6 +566,13 @@ static int32_t tsdbSyncSendDFileSet(SSyncH *pSynch, SDFileSet *pSet) { STsdbRepo *pRepo = pSynch->pRepo; bool toSend = false; + // skip expired fileset + if (pSet && tsdbGetFidLevel(pSet->fid, &(pSynch->rtn)) < 0) { + tsdbInfo("vgId:%d, don't sync send since fileset:%d smaller than minFid:%d", REPO_ID(pRepo), pSet->fid, + pSynch->rtn.minFid); + return 0; + } + if (tsdbSendDFileSetInfo(pSynch, pSet) < 0) { tsdbError("vgId:%d, failed to send fileset:%d info since %s", REPO_ID(pRepo), pSet ? pSet->fid : -1, tstrerror(terrno)); return -1; diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index b6b49693f6..cd4850e47e 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -148,6 +148,7 @@ int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj); size_t taosHashGetMemSize(const SHashObj *pHashObj); void *taosHashIterate(SHashObj *pHashObj, void *p); + void taosHashCancelIterate(SHashObj *pHashObj, void *p); #ifdef __cplusplus diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index f2e268c2d4..fc7b6b8584 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -50,7 +50,15 @@ void* taosArrayInit(size_t size, size_t elemSize); * @param nEles * @return */ -void *taosArrayPushBatch(SArray *pArray, const void *pData, int nEles); +void *taosArrayAddBatch(SArray *pArray, const void *pData, int nEles); + +/** + * add all element from the source array list into the destination + * @param pArray + * @param pInput + * @return + */ +void* taosArrayAddAll(SArray* pArray, const SArray* pInput); /** * @@ -59,7 +67,7 @@ void *taosArrayPushBatch(SArray *pArray, const void *pData, int nEles); * @return */ static FORCE_INLINE void* taosArrayPush(SArray* pArray, const void* pData) { - return taosArrayPushBatch(pArray, pData, 1); + return taosArrayAddBatch(pArray, pData, 1); } /** @@ -98,6 +106,14 @@ void* taosArrayGetLast(const SArray* pArray); */ size_t taosArrayGetSize(const SArray* pArray); +/** + * set the size of array + * @param pArray + * @param size size of the array + * @return + */ +void taosArraySetSize(SArray* pArray, size_t size); + /** * insert data into array * @param pArray diff --git a/src/util/inc/ttoken.h b/src/util/inc/ttoken.h index d5e45e60a1..3bf030a9eb 100644 --- a/src/util/inc/ttoken.h +++ b/src/util/inc/ttoken.h @@ -37,8 +37,6 @@ typedef struct SStrToken { char *z; } SStrToken; -extern const char escapeChar[]; - /** * check if it is a number or not * @param pToken @@ -47,8 +45,6 @@ extern const char escapeChar[]; #define isNumber(tk) \ ((tk)->type == TK_INTEGER || (tk)->type == TK_FLOAT || (tk)->type == TK_HEX || (tk)->type == TK_BIN) -#define GET_ESCAPE_CHAR(c) (escapeChar[(uint8_t)(c)]) - /** * tokenizer for sql string * @param z diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index f4587b33e7..5e7d9d14da 100644 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -56,7 +56,7 @@ static int32_t taosArrayResize(SArray* pArray) { return 0; } -void* taosArrayPushBatch(SArray* pArray, const void* pData, int nEles) { +void* taosArrayAddBatch(SArray* pArray, const void* pData, int nEles) { if (pArray == NULL || pData == NULL) { return NULL; } @@ -82,6 +82,10 @@ void* taosArrayPushBatch(SArray* pArray, const void* pData, int nEles) { return dst; } +void* taosArrayAddAll(SArray* pArray, const SArray* pInput) { + return taosArrayAddBatch(pArray, pInput->pData, (int32_t) taosArrayGetSize(pInput)); +} + void* taosArrayPop(SArray* pArray) { assert( pArray != NULL ); @@ -111,6 +115,11 @@ void* taosArrayGetLast(const SArray* pArray) { size_t taosArrayGetSize(const SArray* pArray) { return pArray->size; } +void taosArraySetSize(SArray* pArray, size_t size) { + assert(size <= pArray->capacity); + pArray->size = size; +} + void* taosArrayInsert(SArray* pArray, size_t index, void* pData) { if (pArray == NULL || pData == NULL) { return NULL; diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 57d45cc8c0..569f9b01bd 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -613,7 +613,7 @@ void doCleanupDataCache(SCacheObj *pCacheObj) { // todo memory leak if there are object with refcount greater than 0 in hash table? taosHashCleanup(pCacheObj->pHashTable); - taosTrashcanEmpty(pCacheObj, true); + taosTrashcanEmpty(pCacheObj, false); __cache_lock_destroy(pCacheObj); diff --git a/src/util/src/tcrc32c.c b/src/util/src/tcrc32c.c index 4009973a9f..d2b63eb9ee 100644 --- a/src/util/src/tcrc32c.c +++ b/src/util/src/tcrc32c.c @@ -18,7 +18,7 @@ 3. This notice may not be removed or altered from any source distribution. */ -#ifndef _TD_ARM_ +#if !defined(_TD_ARM_) && !defined(_TD_MIPS_) #include #endif diff --git a/src/util/src/terror.c b/src/util/src/terror.c index c8666aa0b6..7f66aa7a4c 100644 --- a/src/util/src/terror.c +++ b/src/util/src/terror.c @@ -83,7 +83,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_REF_ALREADY_EXIST, "Ref is already there" TAOS_DEFINE_ERROR(TSDB_CODE_REF_NOT_EXIST, "Ref is not there") //client -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_SQL, "Invalid SQL statement") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_OPERATION, "Invalid operation") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_QHANDLE, "Invalid qhandle") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TIME_STAMP, "Invalid combination of client/service time") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_VALUE, "Invalid value in client") diff --git a/src/util/src/ttokenizer.c b/src/util/src/ttokenizer.c index 871d84cb66..81b65ef308 100644 --- a/src/util/src/ttokenizer.c +++ b/src/util/src/ttokenizer.c @@ -237,18 +237,6 @@ static const char isIdChar[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ }; -const char escapeChar[] = { - /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, /* 0x */ - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, /* 1x */ - 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, /* 2x */ - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0x3E, 0x3F, /* 3x */ - 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F,/* 4x */ - 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F,/* 5x */ - 0x60, 0x07, 0x08, 0x63, 0x64, 0x65, 0x0C, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x0A, 0x6F,/* 6x */ - 0x70, 0x71, 0x0D, 0x73, 0x09, 0x75, 0x0B, 0x77, 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F,/* 7x */ -}; - static void* keywordHashTable = NULL; static void doInitKeywordsTable(void) { @@ -598,7 +586,6 @@ SStrToken tscReplaceStrToken(char **str, SStrToken *token, const char* newToken) return ntoken; } - SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) { SStrToken t0 = {0}; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index ca3746c598..fdf215a966 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -120,12 +120,14 @@ int32_t vnodeDrop(int32_t vgId) { vDebug("vgId:%d, failed to drop, vnode not find", vgId); return TSDB_CODE_VND_INVALID_VGROUP_ID; } + if (pVnode->dropped) { + vnodeRelease(pVnode); + return TSDB_CODE_SUCCESS; + } vInfo("vgId:%d, vnode will be dropped, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); pVnode->dropped = 1; - // remove from hash, so new messages wont be consumed - vnodeRemoveFromHash(pVnode); vnodeRelease(pVnode); vnodeCleanupInMWorker(pVnode); @@ -425,6 +427,10 @@ int32_t vnodeOpen(int32_t vgId) { int32_t vnodeClose(int32_t vgId) { SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) return 0; + if (pVnode->dropped) { + vnodeRelease(pVnode); + return 0; + } vDebug("vgId:%d, vnode will be closed, pVnode:%p", pVnode->vgId, pVnode); vnodeRemoveFromHash(pVnode); @@ -448,7 +454,11 @@ void vnodeDestroy(SVnodeObj *pVnode) { } if (pVnode->tsdb) { - code = tsdbCloseRepo(pVnode->tsdb, 1); + // the deleted vnode does not need to commit, so as to speed up the deletion + int toCommit = 1; + if (pVnode->dropped) toCommit = 0; + + code = tsdbCloseRepo(pVnode->tsdb, toCommit); pVnode->tsdb = NULL; } @@ -510,6 +520,8 @@ void vnodeCleanUp(SVnodeObj *pVnode) { vnodeSetClosingStatus(pVnode); + vnodeRemoveFromHash(pVnode); + // stop replication module if (pVnode->sync > 0) { int64_t sync = pVnode->sync; diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index b28bdbf130..3964975e35 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -117,14 +117,17 @@ static SVReadMsg *vnodeBuildVReadMsg(SVnodeObj *pVnode, void *pCont, int32_t con } int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam) { + SVnodeObj *pVnode = vparam; + if (pVnode->dropped) { + return TSDB_CODE_APP_NOT_READY; + } + SVReadMsg *pRead = vnodeBuildVReadMsg(vparam, pCont, contLen, qtype, rparam); if (pRead == NULL) { assert(terrno != 0); return terrno; } - SVnodeObj *pVnode = vparam; - int32_t code = vnodeCheckRead(pVnode); if (code != TSDB_CODE_SUCCESS) { taosFreeQitem(pRead); @@ -354,7 +357,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { // kill current query and free corresponding resources. if (pRetrieve->free == 1) { - vWarn("vgId:%d, QInfo:%"PRIu64 "-%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, pRetrieve->qId, *handle); + vWarn("vgId:%d, QInfo:%"PRIx64 "-%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, pRetrieve->qId, *handle); qKillQuery(*handle); qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); diff --git a/src/vnode/src/vnodeStatus.c b/src/vnode/src/vnodeStatus.c index c482d1fd1a..1eaddc3d25 100644 --- a/src/vnode/src/vnodeStatus.c +++ b/src/vnode/src/vnodeStatus.c @@ -66,6 +66,9 @@ static bool vnodeSetClosingStatusImp(SVnodeObj* pVnode) { } bool vnodeSetClosingStatus(SVnodeObj* pVnode) { + if (pVnode->status == TAOS_VN_STATUS_CLOSING) + return true; + while (!vnodeSetClosingStatusImp(pVnode)) { taosMsleep(1); } diff --git a/src/vnode/src/vnodeSync.c b/src/vnode/src/vnodeSync.c index 05af34a34f..4197428fec 100644 --- a/src/vnode/src/vnodeSync.c +++ b/src/vnode/src/vnodeSync.c @@ -55,6 +55,11 @@ void vnodeNotifyRole(int32_t vgId, int8_t role) { vTrace("vgId:%d, vnode not found while notify role", vgId); return; } + if (pVnode->dropped) { + vTrace("vgId:%d, vnode dropped while notify role", vgId); + vnodeRelease(pVnode); + return; + } vInfo("vgId:%d, sync role changed from %s to %s", pVnode->vgId, syncRole[pVnode->role], syncRole[role]); pVnode->role = role; @@ -75,6 +80,11 @@ void vnodeCtrlFlow(int32_t vgId, int32_t level) { vTrace("vgId:%d, vnode not found while flow ctrl", vgId); return; } + if (pVnode->dropped) { + vTrace("vgId:%d, vnode dropped while flow ctrl", vgId); + vnodeRelease(pVnode); + return; + } if (pVnode->flowctrlLevel != level) { vDebug("vgId:%d, set flowctrl level from %d to %d", pVnode->vgId, pVnode->flowctrlLevel, level); @@ -116,11 +126,16 @@ void vnodeStopSyncFile(int32_t vgId, uint64_t fversion) { } void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code) { - void *pVnode = vnodeAcquire(vgId); + SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) { vError("vgId:%d, vnode not found while confirm forward", vgId); } + if (code == TSDB_CODE_SYN_CONFIRM_EXPIRED && pVnode->status == TAOS_VN_STATUS_CLOSING) { + vDebug("vgId:%d, db:%s, vnode is closing while confirm forward", vgId, pVnode->db); + code = TSDB_CODE_VND_IS_CLOSING; + } + dnodeSendRpcVWriteRsp(pVnode, wparam, code); vnodeRelease(pVnode); } @@ -129,6 +144,7 @@ int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rpara SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) { vError("vgId:%d, vnode not found while write to cache", vgId); + vnodeRelease(pVnode); return TSDB_CODE_VND_INVALID_VGROUP_ID; } diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 36516d81df..16089c8e91 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -340,8 +340,11 @@ static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) { if (pWrite->processedCount >= 100) { vError("vgId:%d, msg:%p, failed to process since %s, retry:%d", pVnode->vgId, pWrite, tstrerror(code), pWrite->processedCount); - pWrite->processedCount = 1; - dnodeSendRpcVWriteRsp(pWrite->pVnode, pWrite, code); + void *handle = pWrite->rpcMsg.handle; + taosFreeQitem(pWrite); + vnodeRelease(pVnode); + SRpcMsg rpcRsp = {.handle = handle, .code = code}; + rpcSendResponse(&rpcRsp); } else { code = vnodePerformFlowCtrl(pWrite); if (code == 0) { @@ -386,4 +389,6 @@ void vnodeWaitWriteCompleted(SVnodeObj *pVnode) { vTrace("vgId:%d, queued wmsg num:%d", pVnode->vgId, pVnode->queuedWMsg); taosMsleep(10); } + + taosMsleep(900); } diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index f865870d47..b884546a08 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -430,6 +430,8 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch pWal->vgId, fileId, pHead->version, pWal->version, pHead->len, offset); pWal->version = pHead->version; + + //wInfo("writeFp: %ld", offset); (*writeFp)(pVnode, pHead, TAOS_QTYPE_WAL, NULL); } diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index 709b39d077..2f2b9f6933 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -37,7 +37,7 @@ pipeline { stage('Parallel test stage') { parallel { stage('pytest') { - agent{label '184'} + agent{label 'slad1'} steps { pre_test() sh ''' @@ -62,7 +62,7 @@ pipeline { } stage('test_crash_gen') { - agent{label "185"} + agent{label "slad2"} steps { pre_test() sh ''' @@ -149,7 +149,7 @@ pipeline { } stage('test_valgrind') { - agent{label "186"} + agent{label "slad3"} steps { pre_test() diff --git a/tests/examples/c/CMakeLists.txt b/tests/examples/c/CMakeLists.txt index b9bafc9563..906ca2dd41 100644 --- a/tests/examples/c/CMakeLists.txt +++ b/tests/examples/c/CMakeLists.txt @@ -4,7 +4,9 @@ IF (TD_LINUX) INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc ${TD_COMMUNITY_DIR}/inc) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(demo apitest.c) - TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread lua) + TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread ) + ADD_EXECUTABLE(subscribe subscribe.c) + TARGET_LINK_LIBRARIES(subscribe taos_static trpc tutil pthread ) ADD_EXECUTABLE(epoll epoll.c) TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua) ENDIF () diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index f20c0321c4..b2411d1212 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -7,7 +7,6 @@ #include #include - static void prepare_data(TAOS* taos) { TAOS_RES *result; result = taos_query(taos, "drop database if exists test;"); @@ -69,7 +68,6 @@ static void prepare_data(TAOS* taos) { usleep(1000000); } - static int print_result(TAOS_RES* res, int blockFetch) { TAOS_ROW row = NULL; int num_fields = taos_num_fields(res); @@ -99,7 +97,6 @@ static int print_result(TAOS_RES* res, int blockFetch) { return nRows; } - static void check_row_count(int line, TAOS_RES* res, int expected) { int actual = print_result(res, expected % 2); if (actual != expected) { @@ -109,7 +106,6 @@ static void check_row_count(int line, TAOS_RES* res, int expected) { } } - static void verify_query(TAOS* taos) { prepare_data(taos); @@ -153,7 +149,6 @@ static void verify_query(TAOS* taos) { taos_free_result(res); } - void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { int rows = print_result(res, *(int*)param); printf("%d rows consumed in subscribe_callback\n", rows); @@ -235,10 +230,10 @@ static void verify_subscribe(TAOS* taos) { taos_unsubscribe(tsub, 0); } - void verify_prepare(TAOS* taos) { TAOS_RES* result = taos_query(taos, "drop database if exists test;"); taos_free_result(result); + usleep(100000); result = taos_query(taos, "create database test;"); @@ -248,6 +243,7 @@ void verify_prepare(TAOS* taos) { taos_free_result(result); return; } + taos_free_result(result); usleep(100000); @@ -369,6 +365,7 @@ void verify_prepare(TAOS* taos) { taos_stmt_add_batch(stmt); } if (taos_stmt_execute(stmt) != 0) { + taos_stmt_close(stmt); printf("\033[31mfailed to execute insert statement.\033[0m\n"); return; } @@ -380,6 +377,230 @@ void verify_prepare(TAOS* taos) { v.v1 = 5; v.v2 = 15; taos_stmt_bind_param(stmt, params + 2); + if (taos_stmt_execute(stmt) != 0) { + taos_stmt_close(stmt); + printf("\033[31mfailed to execute select statement.\033[0m\n"); + return; + } + + result = taos_stmt_use_result(stmt); + + TAOS_ROW row; + int rows = 0; + int num_fields = taos_num_fields(result); + TAOS_FIELD *fields = taos_fetch_fields(result); + char temp[256]; + + // fetch the records row by row + while ((row = taos_fetch_row(result))) { + rows++; + taos_print_row(temp, row, fields, num_fields); + printf("%s\n", temp); + } + + taos_free_result(result); + taos_stmt_close(stmt); +} + + + + + +void verify_prepare2(TAOS* taos) { + TAOS_RES* result = taos_query(taos, "drop database if exists test;"); + taos_free_result(result); + usleep(100000); + result = taos_query(taos, "create database test;"); + + int code = taos_errno(result); + if (code != 0) { + printf("\033[31mfailed to create database, reason:%s\033[0m\n", taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); + + usleep(100000); + taos_select_db(taos, "test"); + + // create table + const char* sql = "create table m1 (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), blob nchar(10))"; + result = taos_query(taos, sql); + code = taos_errno(result); + if (code != 0) { + printf("\033[31mfailed to create table, reason:%s\033[0m\n", taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); + + // insert 10 records + struct { + int64_t ts[10]; + int8_t b[10]; + int8_t v1[10]; + int16_t v2[10]; + int32_t v4[10]; + int64_t v8[10]; + float f4[10]; + double f8[10]; + char bin[10][40]; + char blob[10][80]; + } v; + + int32_t *t8_len = malloc(sizeof(int32_t) * 10); + int32_t *t16_len = malloc(sizeof(int32_t) * 10); + int32_t *t32_len = malloc(sizeof(int32_t) * 10); + int32_t *t64_len = malloc(sizeof(int32_t) * 10); + int32_t *float_len = malloc(sizeof(int32_t) * 10); + int32_t *double_len = malloc(sizeof(int32_t) * 10); + int32_t *bin_len = malloc(sizeof(int32_t) * 10); + int32_t *blob_len = malloc(sizeof(int32_t) * 10); + + TAOS_STMT* stmt = taos_stmt_init(taos); + TAOS_MULTI_BIND params[10]; + char is_null[10] = {0}; + + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(v.ts[0]); + params[0].buffer = v.ts; + params[0].length = t64_len; + params[0].is_null = is_null; + params[0].num = 10; + + params[1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[1].buffer_length = sizeof(v.b[0]); + params[1].buffer = v.b; + params[1].length = t8_len; + params[1].is_null = is_null; + params[1].num = 10; + + params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[2].buffer_length = sizeof(v.v1[0]); + params[2].buffer = v.v1; + params[2].length = t8_len; + params[2].is_null = is_null; + params[2].num = 10; + + params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[3].buffer_length = sizeof(v.v2[0]); + params[3].buffer = v.v2; + params[3].length = t16_len; + params[3].is_null = is_null; + params[3].num = 10; + + params[4].buffer_type = TSDB_DATA_TYPE_INT; + params[4].buffer_length = sizeof(v.v4[0]); + params[4].buffer = v.v4; + params[4].length = t32_len; + params[4].is_null = is_null; + params[4].num = 10; + + params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[5].buffer_length = sizeof(v.v8[0]); + params[5].buffer = v.v8; + params[5].length = t64_len; + params[5].is_null = is_null; + params[5].num = 10; + + params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[6].buffer_length = sizeof(v.f4[0]); + params[6].buffer = v.f4; + params[6].length = float_len; + params[6].is_null = is_null; + params[6].num = 10; + + params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[7].buffer_length = sizeof(v.f8[0]); + params[7].buffer = v.f8; + params[7].length = double_len; + params[7].is_null = is_null; + params[7].num = 10; + + params[8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[8].buffer_length = sizeof(v.bin[0]); + params[8].buffer = v.bin; + params[8].length = bin_len; + params[8].is_null = is_null; + params[8].num = 10; + + params[9].buffer_type = TSDB_DATA_TYPE_NCHAR; + params[9].buffer_length = sizeof(v.blob[0]); + params[9].buffer = v.blob; + params[9].length = blob_len; + params[9].is_null = is_null; + params[9].num = 10; + + + sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; + code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("\033[31mfailed to execute taos_stmt_prepare. code:0x%x\033[0m\n", code); + } + + code = taos_stmt_set_tbname(stmt, "m1"); + if (code != 0){ + printf("\033[31mfailed to execute taos_stmt_prepare. code:0x%x\033[0m\n", code); + } + + int64_t ts = 1591060628000; + for (int i = 0; i < 10; ++i) { + v.ts[i] = ts++; + is_null[i] = 0; + + v.b[i] = (int8_t)i % 2; + v.v1[i] = (int8_t)i; + v.v2[i] = (int16_t)(i * 2); + v.v4[i] = (int32_t)(i * 4); + v.v8[i] = (int64_t)(i * 8); + v.f4[i] = (float)(i * 40); + v.f8[i] = (double)(i * 80); + for (int j = 0; j < sizeof(v.bin[0]) - 1; ++j) { + v.bin[i][j] = (char)(i + '0'); + } + strcpy(v.blob[i], "一二三四五六七八九十"); + + t8_len[i] = sizeof(int8_t); + t16_len[i] = sizeof(int16_t); + t32_len[i] = sizeof(int32_t); + t64_len[i] = sizeof(int64_t); + float_len[i] = sizeof(float); + double_len[i] = sizeof(double); + bin_len[i] = sizeof(v.bin[0]); + blob_len[i] = (int32_t)strlen(v.blob[i]); + } + + taos_stmt_bind_param_batch(stmt, params); + taos_stmt_add_batch(stmt); + + if (taos_stmt_execute(stmt) != 0) { + printf("\033[31mfailed to execute insert statement.\033[0m\n"); + return; + } + taos_stmt_close(stmt); + + + + // query the records + stmt = taos_stmt_init(taos); + taos_stmt_prepare(stmt, "SELECT * FROM m1 WHERE v1 > ? AND v2 < ?", 0); + TAOS_BIND qparams[2]; + + int8_t v1 = 5; + int16_t v2 = 15; + qparams[0].buffer_type = TSDB_DATA_TYPE_TINYINT; + qparams[0].buffer_length = sizeof(v1); + qparams[0].buffer = &v1; + qparams[0].length = &qparams[0].buffer_length; + qparams[0].is_null = NULL; + + qparams[1].buffer_type = TSDB_DATA_TYPE_SMALLINT; + qparams[1].buffer_length = sizeof(v2); + qparams[1].buffer = &v2; + qparams[1].length = &qparams[1].buffer_length; + qparams[1].is_null = NULL; + + taos_stmt_bind_param(stmt, qparams); if (taos_stmt_execute(stmt) != 0) { printf("\033[31mfailed to execute select statement.\033[0m\n"); return; @@ -404,6 +625,248 @@ void verify_prepare(TAOS* taos) { taos_stmt_close(stmt); } + + +void verify_prepare3(TAOS* taos) { + TAOS_RES* result = taos_query(taos, "drop database if exists test;"); + taos_free_result(result); + usleep(100000); + result = taos_query(taos, "create database test;"); + + int code = taos_errno(result); + if (code != 0) { + printf("\033[31mfailed to create database, reason:%s\033[0m\n", taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); + + usleep(100000); + taos_select_db(taos, "test"); + + // create table + const char* sql = "create stable st1 (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), blob nchar(10)) tags (id1 int, id2 binary(40))"; + result = taos_query(taos, sql); + code = taos_errno(result); + if (code != 0) { + printf("\033[31mfailed to create table, reason:%s\033[0m\n", taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); + + TAOS_BIND tags[2]; + + int32_t id1 = 1; + char id2[40] = "abcdefghijklmnopqrstuvwxyz0123456789"; + uintptr_t id2_len = strlen(id2); + + tags[0].buffer_type = TSDB_DATA_TYPE_INT; + tags[0].buffer_length = sizeof(int); + tags[0].buffer = &id1; + tags[0].length = NULL; + tags[0].is_null = NULL; + + tags[1].buffer_type = TSDB_DATA_TYPE_BINARY; + tags[1].buffer_length = sizeof(id2); + tags[1].buffer = id2; + tags[1].length = &id2_len; + tags[1].is_null = NULL; + + + // insert 10 records + struct { + int64_t ts[10]; + int8_t b[10]; + int8_t v1[10]; + int16_t v2[10]; + int32_t v4[10]; + int64_t v8[10]; + float f4[10]; + double f8[10]; + char bin[10][40]; + char blob[10][80]; + } v; + + int32_t *t8_len = malloc(sizeof(int32_t) * 10); + int32_t *t16_len = malloc(sizeof(int32_t) * 10); + int32_t *t32_len = malloc(sizeof(int32_t) * 10); + int32_t *t64_len = malloc(sizeof(int32_t) * 10); + int32_t *float_len = malloc(sizeof(int32_t) * 10); + int32_t *double_len = malloc(sizeof(int32_t) * 10); + int32_t *bin_len = malloc(sizeof(int32_t) * 10); + int32_t *blob_len = malloc(sizeof(int32_t) * 10); + + TAOS_STMT* stmt = taos_stmt_init(taos); + TAOS_MULTI_BIND params[10]; + char is_null[10] = {0}; + + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(v.ts[0]); + params[0].buffer = v.ts; + params[0].length = t64_len; + params[0].is_null = is_null; + params[0].num = 10; + + params[1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[1].buffer_length = sizeof(v.b[0]); + params[1].buffer = v.b; + params[1].length = t8_len; + params[1].is_null = is_null; + params[1].num = 10; + + params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[2].buffer_length = sizeof(v.v1[0]); + params[2].buffer = v.v1; + params[2].length = t8_len; + params[2].is_null = is_null; + params[2].num = 10; + + params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[3].buffer_length = sizeof(v.v2[0]); + params[3].buffer = v.v2; + params[3].length = t16_len; + params[3].is_null = is_null; + params[3].num = 10; + + params[4].buffer_type = TSDB_DATA_TYPE_INT; + params[4].buffer_length = sizeof(v.v4[0]); + params[4].buffer = v.v4; + params[4].length = t32_len; + params[4].is_null = is_null; + params[4].num = 10; + + params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[5].buffer_length = sizeof(v.v8[0]); + params[5].buffer = v.v8; + params[5].length = t64_len; + params[5].is_null = is_null; + params[5].num = 10; + + params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[6].buffer_length = sizeof(v.f4[0]); + params[6].buffer = v.f4; + params[6].length = float_len; + params[6].is_null = is_null; + params[6].num = 10; + + params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[7].buffer_length = sizeof(v.f8[0]); + params[7].buffer = v.f8; + params[7].length = double_len; + params[7].is_null = is_null; + params[7].num = 10; + + params[8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[8].buffer_length = sizeof(v.bin[0]); + params[8].buffer = v.bin; + params[8].length = bin_len; + params[8].is_null = is_null; + params[8].num = 10; + + params[9].buffer_type = TSDB_DATA_TYPE_NCHAR; + params[9].buffer_length = sizeof(v.blob[0]); + params[9].buffer = v.blob; + params[9].length = blob_len; + params[9].is_null = is_null; + params[9].num = 10; + + + sql = "insert into ? using st1 tags(?,?) values(?,?,?,?,?,?,?,?,?,?)"; + code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("\033[31mfailed to execute taos_stmt_prepare. code:0x%x\033[0m\n", code); + } + + code = taos_stmt_set_tbname_tags(stmt, "m1", tags); + if (code != 0){ + printf("\033[31mfailed to execute taos_stmt_prepare. code:0x%x\033[0m\n", code); + } + + int64_t ts = 1591060628000; + for (int i = 0; i < 10; ++i) { + v.ts[i] = ts++; + is_null[i] = 0; + + v.b[i] = (int8_t)i % 2; + v.v1[i] = (int8_t)i; + v.v2[i] = (int16_t)(i * 2); + v.v4[i] = (int32_t)(i * 4); + v.v8[i] = (int64_t)(i * 8); + v.f4[i] = (float)(i * 40); + v.f8[i] = (double)(i * 80); + for (int j = 0; j < sizeof(v.bin[0]) - 1; ++j) { + v.bin[i][j] = (char)(i + '0'); + } + strcpy(v.blob[i], "一二三四五六七八九十"); + + t8_len[i] = sizeof(int8_t); + t16_len[i] = sizeof(int16_t); + t32_len[i] = sizeof(int32_t); + t64_len[i] = sizeof(int64_t); + float_len[i] = sizeof(float); + double_len[i] = sizeof(double); + bin_len[i] = sizeof(v.bin[0]); + blob_len[i] = (int32_t)strlen(v.blob[i]); + } + + + taos_stmt_bind_param_batch(stmt, params); + taos_stmt_add_batch(stmt); + + if (taos_stmt_execute(stmt) != 0) { + printf("\033[31mfailed to execute insert statement.\033[0m\n"); + return; + } + taos_stmt_close(stmt); + + // query the records + stmt = taos_stmt_init(taos); + taos_stmt_prepare(stmt, "SELECT * FROM m1 WHERE v1 > ? AND v2 < ?", 0); + + TAOS_BIND qparams[2]; + + int8_t v1 = 5; + int16_t v2 = 15; + qparams[0].buffer_type = TSDB_DATA_TYPE_TINYINT; + qparams[0].buffer_length = sizeof(v1); + qparams[0].buffer = &v1; + qparams[0].length = &qparams[0].buffer_length; + qparams[0].is_null = NULL; + + qparams[1].buffer_type = TSDB_DATA_TYPE_SMALLINT; + qparams[1].buffer_length = sizeof(v2); + qparams[1].buffer = &v2; + qparams[1].length = &qparams[1].buffer_length; + qparams[1].is_null = NULL; + + taos_stmt_bind_param(stmt, qparams); + if (taos_stmt_execute(stmt) != 0) { + printf("\033[31mfailed to execute select statement.\033[0m\n"); + return; + } + + result = taos_stmt_use_result(stmt); + + TAOS_ROW row; + int rows = 0; + int num_fields = taos_num_fields(result); + TAOS_FIELD *fields = taos_fetch_fields(result); + char temp[256]; + + // fetch the records row by row + while ((row = taos_fetch_row(result))) { + rows++; + taos_print_row(temp, row, fields, num_fields); + printf("%s\n", temp); + } + + taos_free_result(result); + taos_stmt_close(stmt); +} + + + void retrieve_callback(void *param, TAOS_RES *tres, int numOfRows) { if (numOfRows > 0) { @@ -495,6 +958,12 @@ int main(int argc, char *argv[]) { printf("************ verify prepare *************\n"); verify_prepare(taos); + printf("************ verify prepare2 *************\n"); + verify_prepare2(taos); + + printf("************ verify prepare3 *************\n"); + verify_prepare3(taos); + printf("************ verify stream *************\n"); verify_stream(taos); printf("done\n"); diff --git a/tests/mas/Jenkinsfile b/tests/mas/Jenkinsfile new file mode 100644 index 0000000000..b2a1a5e116 --- /dev/null +++ b/tests/mas/Jenkinsfile @@ -0,0 +1,309 @@ +def pre_test(){ + + sh ''' + sudo rmtaos||echo 'no taosd installed' + ''' + sh ''' + cd ${WKC} + git reset --hard + git checkout $BRANCH_NAME + git pull + git submodule update + cd ${WK} + git reset --hard + git checkout $BRANCH_NAME + git pull + export TZ=Asia/Harbin + date + rm -rf ${WK}/debug + mkdir debug + cd debug + cmake .. > /dev/null + make > /dev/null + make install > /dev/null + pip3 install ${WKC}/src/connector/python/linux/python3/ + ''' + return 1 +} +pipeline { + agent none + environment{ + + WK = '/var/lib/jenkins/workspace/TDinternal' + WKC= '/var/lib/jenkins/workspace/TDinternal/community' + } + + stages { + stage('Parallel test stage') { + parallel { + stage('pytest') { + agent{label 'slam1'} + steps { + pre_test() + sh ''' + cd ${WKC}/tests + find pytest -name '*'sql|xargs rm -rf + ./test-all.sh pytest + date''' + } + } + stage('test_b1') { + agent{label 'slam2'} + steps { + pre_test() + + sh ''' + cd ${WKC}/tests + ./test-all.sh b1 + date''' + + + } + } + + stage('test_crash_gen') { + agent{label "slam3"} + steps { + pre_test() + sh ''' + cd ${WKC}/tests/pytest + ''' + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/pytest + ./crash_gen.sh -a -p -t 4 -s 2000 + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/pytest + rm -rf /var/lib/taos/* + rm -rf /var/log/taos/* + ./handle_crash_gen_val_log.sh + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/pytest + rm -rf /var/lib/taos/* + rm -rf /var/log/taos/* + ./handle_taosd_val_log.sh + ''' + } + + sh''' + systemctl start taosd + sleep 10 + ''' + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/gotest + bash batchtest.sh + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/examples/python/PYTHONConnectorChecker + python3 PythonChecker.py + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/examples/JDBC/JDBCDemo/ + mvn clean package assembly:single -DskipTests >/dev/null + java -jar target/JDBCDemo-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/src/connector/jdbc + mvn clean package -Dmaven.test.skip=true >/dev/null + cd ${WKC}/tests/examples/JDBC/JDBCDemo/ + java --class-path=../../../../src/connector/jdbc/target:$JAVA_HOME/jre/lib/ext -jar target/JDBCDemo-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cp -rf ${WKC}/tests/examples/nodejs ${JENKINS_HOME}/workspace/ + cd ${JENKINS_HOME}/workspace/nodejs + node nodejsChecker.js host=localhost + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC# + dotnet run + ''' + } + sh ''' + systemctl stop taosd + cd ${WKC}/tests + ./test-all.sh b2 + date + ''' + sh ''' + cd ${WKC}/tests + ./test-all.sh full unit + date''' + } + } + + stage('test_valgrind') { + agent{label "slam4"} + + steps { + pre_test() + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/pytest + nohup taosd >/dev/null & + sleep 10 + python3 concurrent_inquiry.py -c 1 + + ''' + } + sh ''' + cd ${WKC}/tests + ./test-all.sh full jdbc + date''' + sh ''' + cd ${WKC}/tests/pytest + ./valgrind-test.sh 2>&1 > mem-error-out.log + ./handle_val_log.sh + + date + cd ${WKC}/tests + ./test-all.sh b3 + date''' + sh ''' + date + cd ${WKC}/tests + ./test-all.sh full example + date''' + } + } + + stage('arm64_build'){ + agent{label 'arm64'} + steps{ + sh ''' + cd ${WK} + git fetch + git checkout develop + git pull + cd ${WKC} + git fetch + git checkout develop + git pull + git submodule update + cd ${WKC}/packaging + ./release.sh -v cluster -c aarch64 -n 2.0.0.0 -m 2.0.0.0 + + ''' + } + } + stage('arm32_build'){ + agent{label 'arm32'} + steps{ + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WK} + git fetch + git checkout develop + git pull + cd ${WKC} + git fetch + git checkout develop + git pull + git submodule update + cd ${WKC}/packaging + ./release.sh -v cluster -c aarch32 -n 2.0.0.0 -m 2.0.0.0 + + ''' + } + + } + } + } + } + + } + post { + success { + emailext ( + subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", + body: ''' + + + + + + + + + + + + +

+ 构建信息 +
+
    +
    +
  • 构建名称>>分支：${PROJECT_NAME}
    • +
  • 构建结果： Successful
    • +
  • 构建编号：${BUILD_NUMBER}
    • +
  • 触发用户：${CAUSE}
    • +
  • 变更概要：${CHANGES}
    • +
  • 构建地址：${BUILD_URL}
    • +
  • 构建日志：${BUILD_URL}console
    • +
  • 变更集：${JELLY_SCRIPT}
    • +
    +
+
+ + ''', + to: "yqliu@taosdata.com,pxiao@taosdata.com", + from: "support@taosdata.com" + ) + } + failure { + emailext ( + subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", + body: ''' + + + + + + + + + + + + +

+ 构建信息 +
+
    +
    +
  • 构建名称>>分支：${PROJECT_NAME}
    • +
  • 构建结果： Successful
    • +
  • 构建编号：${BUILD_NUMBER}
    • +
  • 触发用户：${CAUSE}
    • +
  • 变更概要：${CHANGES}
    • +
  • 构建地址：${BUILD_URL}
    • +
  • 构建日志：${BUILD_URL}console
    • +
  • 变更集：${JELLY_SCRIPT}
    • +
    +
+
+ + ''', + to: "yqliu@taosdata.com,pxiao@taosdata.com", + from: "support@taosdata.com" + ) + } + } +} \ No newline at end of file diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh index 9a16084683..bf62f401bf 100755 --- a/tests/perftest-scripts/perftest-query.sh +++ b/tests/perftest-scripts/perftest-query.sh @@ -64,18 +64,25 @@ function runQueryPerfTest { [ -f $PERFORMANCE_TEST_REPORT ] && rm $PERFORMANCE_TEST_REPORT nohup $WORK_DIR/TDengine/debug/build/bin/taosd -c /etc/taosperf/ > /dev/null 2>&1 & echoInfo "Wait TDengine to start" - sleep 300 + sleep 60 echoInfo "Run Performance Test" cd $WORK_DIR/TDengine/tests/pytest python3 query/queryPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT + mkdir -p /var/lib/perf/ + mkdir -p /var/log/perf/ + rm -rf /var/lib/perf/* + rm -rf /var/log/perf/* + nohup $WORK_DIR/TDengine/debug/build/bin/taosd -c /etc/perf/ > /dev/null 2>&1 & + echoInfo "Wait TDengine to start" + sleep 10 + echoInfo "Run Performance Test" + cd $WORK_DIR/TDengine/tests/pytest + python3 insert/insertFromCSVPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT - - python3 perfbenchmark/joinPerformance.py | tee -a $PERFORMANCE_TEST_REPORT - } diff --git a/tests/pytest/crash_gen/service_manager.py b/tests/pytest/crash_gen/service_manager.py index 95507f0142..1cd65c1dde 100644 --- a/tests/pytest/crash_gen/service_manager.py +++ b/tests/pytest/crash_gen/service_manager.py @@ -22,7 +22,7 @@ from queue import Queue, Empty from .shared.config import Config from .shared.db import DbTarget, DbConn from .shared.misc import Logging, Helper, CrashGenError, Status, Progress, Dice -from .shared.types import DirPath +from .shared.types import DirPath, IpcStream # from crash_gen.misc import CrashGenError, Dice, Helper, Logging, Progress, Status # from crash_gen.db import DbConn, DbTarget @@ -177,13 +177,12 @@ quorum 2 return "127.0.0.1" def getServiceCmdLine(self): # to start the instance - cmdLine = [] if Config.getConfig().track_memory_leaks: Logging.info("Invoking VALGRIND on service...") - cmdLine = ['valgrind', '--leak-check=yes'] - # TODO: move "exec -c" into Popen(), we can both "use shell" and NOT fork so ask to lose kill control - cmdLine += ["exec " + self.getExecFile(), '-c', self.getCfgDir()] # used in subproce.Popen() - return cmdLine + return ['exec /usr/bin/valgrind', '--leak-check=yes', self.getExecFile(), '-c', self.getCfgDir()] + else: + # TODO: move "exec -c" into Popen(), we can both "use shell" and NOT fork so ask to lose kill control + return ["exec " + self.getExecFile(), '-c', self.getCfgDir()] # used in subproce.Popen() def _getDnodes(self, dbc): dbc.query("show dnodes") @@ -281,16 +280,16 @@ class TdeSubProcess: return '[TdeSubProc: pid = {}, status = {}]'.format( self.getPid(), self.getStatus() ) - def getStdOut(self) -> BinaryIO : + def getIpcStdOut(self) -> IpcStream : if self._popen.universal_newlines : # alias of text_mode raise CrashGenError("We need binary mode for STDOUT IPC") # Logging.info("Type of stdout is: {}".format(type(self._popen.stdout))) - return typing.cast(BinaryIO, self._popen.stdout) + return typing.cast(IpcStream, self._popen.stdout) - def getStdErr(self) -> BinaryIO : + def getIpcStdErr(self) -> IpcStream : if self._popen.universal_newlines : # alias of text_mode raise CrashGenError("We need binary mode for STDERR IPC") - return typing.cast(BinaryIO, self._popen.stderr) + return typing.cast(IpcStream, self._popen.stderr) # Now it's always running, since we matched the life cycle # def isRunning(self): @@ -301,11 +300,6 @@ class TdeSubProcess: def _start(self, cmdLine) -> Popen : ON_POSIX = 'posix' in sys.builtin_module_names - - # Sanity check - # if self.subProcess: # already there - # raise RuntimeError("Corrupt process state") - # Prepare environment variables for coverage information # Ref: https://stackoverflow.com/questions/2231227/python-subprocess-popen-with-a-modified-environment @@ -314,9 +308,8 @@ class TdeSubProcess: # print(myEnv) # print("Starting TDengine with env: ", myEnv.items()) - # print("Starting TDengine via Shell: {}".format(cmdLineStr)) + print("Starting TDengine: {}".format(cmdLine)) - # useShell = True # Needed to pass environments into it return Popen( ' '.join(cmdLine), # ' '.join(cmdLine) if useShell else cmdLine, shell=True, # Always use shell, since we need to pass ENV vars @@ -732,19 +725,19 @@ class ServiceManagerThread: self._ipcQueue = Queue() # type: Queue self._thread = threading.Thread( # First thread captures server OUTPUT target=self.svcOutputReader, - args=(subProc.getStdOut(), self._ipcQueue, logDir)) + args=(subProc.getIpcStdOut(), self._ipcQueue, logDir)) self._thread.daemon = True # thread dies with the program self._thread.start() time.sleep(0.01) if not self._thread.is_alive(): # What happened? - Logging.info("Failed to started process to monitor STDOUT") + Logging.info("Failed to start process to monitor STDOUT") self.stop() raise CrashGenError("Failed to start thread to monitor STDOUT") Logging.info("Successfully started process to monitor STDOUT") self._thread2 = threading.Thread( # 2nd thread captures server ERRORs target=self.svcErrorReader, - args=(subProc.getStdErr(), self._ipcQueue, logDir)) + args=(subProc.getIpcStdErr(), self._ipcQueue, logDir)) self._thread2.daemon = True # thread dies with the program self._thread2.start() time.sleep(0.01) @@ -887,14 +880,19 @@ class ServiceManagerThread: print("\nNon-UTF8 server output: {}\n".format(bChunk.decode('cp437'))) return None - def _textChunkGenerator(self, streamIn: BinaryIO, logDir: str, logFile: str + def _textChunkGenerator(self, streamIn: IpcStream, logDir: str, logFile: str ) -> Generator[TextChunk, None, None]: ''' - Take an input stream with binary data, produced a generator of decoded - "text chunks", and also save the original binary data in a log file. + Take an input stream with binary data (likely from Popen), produced a generator of decoded + "text chunks". + + Side effect: it also save the original binary data in a log file. ''' os.makedirs(logDir, exist_ok=True) logF = open(os.path.join(logDir, logFile), 'wb') + if logF is None: + Logging.error("Failed to open log file (binary write): {}/{}".format(logDir, logFile)) + return for bChunk in iter(streamIn.readline, b''): logF.write(bChunk) # Write to log file immediately tChunk = self._decodeBinaryChunk(bChunk) # decode @@ -902,14 +900,14 @@ class ServiceManagerThread: yield tChunk # TODO: split into actual text lines # At the end... - streamIn.close() # Close the stream - logF.close() # Close the output file + streamIn.close() # Close the incoming stream + logF.close() # Close the log file - def svcOutputReader(self, stdOut: BinaryIO, queue, logDir: str): + def svcOutputReader(self, ipcStdOut: IpcStream, queue, logDir: str): ''' The infinite routine that processes the STDOUT stream for the sub process being managed. - :param stdOut: the IO stream object used to fetch the data from + :param ipcStdOut: the IO stream object used to fetch the data from :param queue: the queue where we dump the roughly parsed chunk-by-chunk text data :param logDir: where we should dump a verbatim output file ''' @@ -917,7 +915,7 @@ class ServiceManagerThread: # Important Reference: https://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python # print("This is the svcOutput Reader...") # stdOut.readline() # Skip the first output? TODO: remove? - for tChunk in self._textChunkGenerator(stdOut, logDir, 'stdout.log') : + for tChunk in self._textChunkGenerator(ipcStdOut, logDir, 'stdout.log') : queue.put(tChunk) # tChunk garanteed not to be None self._printProgress("_i") @@ -940,12 +938,12 @@ class ServiceManagerThread: Logging.info("EOF found TDengine STDOUT, marking the process as terminated") self.setStatus(Status.STATUS_STOPPED) - def svcErrorReader(self, stdErr: BinaryIO, queue, logDir: str): + def svcErrorReader(self, ipcStdErr: IpcStream, queue, logDir: str): # os.makedirs(logDir, exist_ok=True) # logFile = os.path.join(logDir,'stderr.log') # fErr = open(logFile, 'wb') # for line in iter(err.readline, b''): - for tChunk in self._textChunkGenerator(stdErr, logDir, 'stderr.log') : + for tChunk in self._textChunkGenerator(ipcStdErr, logDir, 'stderr.log') : queue.put(tChunk) # tChunk garanteed not to be None # fErr.write(line) Logging.info("TDengine STDERR: {}".format(tChunk)) diff --git a/tests/pytest/crash_gen/shared/types.py b/tests/pytest/crash_gen/shared/types.py index 814a821917..42fd2a1617 100644 --- a/tests/pytest/crash_gen/shared/types.py +++ b/tests/pytest/crash_gen/shared/types.py @@ -1,4 +1,4 @@ -from typing import Any, List, Dict, NewType +from typing import Any, BinaryIO, List, Dict, NewType from enum import Enum DirPath = NewType('DirPath', str) @@ -26,3 +26,5 @@ class TdDataType(Enum): TdColumns = Dict[str, TdDataType] TdTags = Dict[str, TdDataType] + +IpcStream = NewType('IpcStream', BinaryIO) \ No newline at end of file diff --git a/tests/pytest/crash_gen/valgrind_taos.supp b/tests/pytest/crash_gen/valgrind_taos.supp index a00b2d830c..5f6604ba77 100644 --- a/tests/pytest/crash_gen/valgrind_taos.supp +++ b/tests/pytest/crash_gen/valgrind_taos.supp @@ -17496,4 +17496,25 @@ obj:/usr/bin/python3.8 obj:/usr/bin/python3.8 fun:PyVectorcall_Call +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:__libc_alloc_buffer_allocate + fun:alloc_buffer_allocate + fun:__resolv_conf_allocate + fun:__resolv_conf_load + fun:__resolv_conf_get_current + fun:__res_vinit + fun:maybe_init + fun:context_get + fun:context_get + fun:__resolv_context_get + fun:gaih_inet.constprop.0 + fun:getaddrinfo + fun:taosGetFqdn + fun:taosCheckGlobalCfg + fun:taos_init_imp } \ No newline at end of file diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index a748c9dd2d..be2cfee04b 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -38,6 +38,8 @@ python3 ./test.py -f table/boundary.py python3 ./test.py -f table/create.py python3 ./test.py -f table/del_stable.py +#stable +python3 ./test.py -f stable/insert.py # tag python3 ./test.py -f tag_lite/filter.py @@ -183,7 +185,7 @@ python3 ./test.py -f stable/query_after_reset.py # perfbenchmark python3 ./test.py -f perfbenchmark/bug3433.py #python3 ./test.py -f perfbenchmark/bug3589.py - +python3 ./test.py -f perfbenchmark/taosdemoInsert.py #query python3 ./test.py -f query/filter.py diff --git a/tests/pytest/insert/insertFromCSVPerformance.py b/tests/pytest/insert/insertFromCSVPerformance.py index 84fd1d7cca..e1f741bd12 100644 --- a/tests/pytest/insert/insertFromCSVPerformance.py +++ b/tests/pytest/insert/insertFromCSVPerformance.py @@ -31,7 +31,7 @@ class insertFromCSVPerformace: self.host = "127.0.0.1" self.user = "root" self.password = "taosdata" - self.config = "/etc/taosperf" + self.config = "/etc/perf" self.conn = taos.connect( self.host, self.user, diff --git a/tests/pytest/perfbenchmark/taosdemoInsert.py b/tests/pytest/perfbenchmark/taosdemoInsert.py new file mode 100644 index 0000000000..59a8143d5a --- /dev/null +++ b/tests/pytest/perfbenchmark/taosdemoInsert.py @@ -0,0 +1,387 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import taos +import sys +import os +import json +import argparse +import subprocess +import datetime +import re + + +from multiprocessing import cpu_count +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.dnodes import TDDnode + +class Taosdemo: + def __init__(self, clearCache, dbName, keep): + self.clearCache = clearCache + self.dbname = dbName + self.drop = "yes" + self.keep = keep + self.host = "127.0.0.1" + self.user = "root" + self.password = "taosdata" + # self.config = "/etc/taosperf" + # self.conn = taos.connect( + # self.host, + # self.user, + # self.password, + # self.config) + + # env config + def getBuildPath(self) -> str: + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/debug/build/bin")] + break + return buildPath + + def getExeToolsDir(self) -> str: + self.debugdir = self.getBuildPath() + "/debug/build/bin" + return self.debugdir + + def getCfgDir(self) -> str: + self.config = self.getBuildPath() + "/sim/dnode1/cfg" + return self.config + + # taodemo insert file config + def dbinfocfg(self) -> dict: + return { + "name": self.dbname, + "drop": self.drop, + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": self.keep, + "minRows": 100, + "maxRows": 4096, + "comp": 2, + "walLevel": 1, + "cachelast": 0, + "quorum": 1, + "fsync": 3000, + "update": 0 + } + + def type_check(func): + def wrapper(self, **kwargs): + num_types = ["int", "float", "bigint", "tinyint", "smallint", "double"] + str_types = ["binary", "nchar"] + for k ,v in kwargs.items(): + if k.lower() not in num_types and k.lower() not in str_types: + return f"args {k} type error, not allowed" + elif not isinstance(v, (int, list, tuple)): + return f"value {v} type error, not allowed" + elif k.lower() in num_types and not isinstance(v, int): + return f"arg {v} takes 1 positional argument must be type int " + elif isinstance(v, (list,tuple)) and len(v) > 2: + return f"arg {v} takes from 1 to 2 positional arguments but more than 2 were given " + elif isinstance(v,(list,tuple)) and [ False for _ in v if not isinstance(_, int) ]: + return f"arg {v} takes from 1 to 2 positional arguments must be type int " + else: + pass + return func(self, **kwargs) + return wrapper + + @type_check + def column_tag_count(self, **column_tag) -> list : + init_column_tag = [] + for k, v in column_tag.items(): + if re.search(k, "int, float, bigint, tinyint, smallint, double", re.IGNORECASE): + init_column_tag.append({"type": k, "count": v}) + elif re.search(k, "binary, nchar", re.IGNORECASE): + if isinstance(v, int): + init_column_tag.append({"type": k, "count": v, "len":8}) + elif len(v) == 1: + init_column_tag.append({"type": k, "count": v[0], "len": 8}) + else: + init_column_tag.append({"type": k, "count": v[0], "len": v[1]}) + return init_column_tag + + def stbcfg(self, stb: str, child_tab_count: int, rows: int, prechildtab: str, columns: dict, tags: dict) -> dict: + return { + "name": stb, + "child_table_exists": "no", + "childtable_count": child_tab_count, + "childtable_prefix": prechildtab, + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": rows, + "childtable_limit": 0, + "childtable_offset": 0, + "rows_per_tbl": 1, + "max_sql_len": 65480, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 10, + "start_timestamp": f"{datetime.datetime.now():%F %X}", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": self.column_tag_count(**columns), + "tags": self.column_tag_count(**tags) + } + + def schemecfg(self,intcount=1,floatcount=0,bcount=0,tcount=0,scount=0,doublecount=0,binarycount=0,ncharcount=0): + return { + "INT": intcount, + "FLOAT": floatcount, + "BIGINT": bcount, + "TINYINT": tcount, + "SMALLINT": scount, + "DOUBLE": doublecount, + "BINARY": binarycount, + "NCHAR": ncharcount + } + + def insertcfg(self,db: dict, stbs: list) -> dict: + return { + "filetype": "insert", + "cfgdir": self.config, + "host": self.host, + "port": 6030, + "user": self.user, + "password": self.password, + "thread_count": cpu_count(), + "thread_count_create_tbl": cpu_count(), + "result_file": "/tmp/insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 100, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": db, + "super_tables": stbs + }] + } + + def createinsertfile(self,db: dict, stbs: list) -> str: + date = datetime.datetime.now() + file_create_table = f"/tmp/insert_{date:%F-%H%M}.json" + + with open(file_create_table, 'w') as f: + json.dump(self.insertcfg(db, stbs), f) + + return file_create_table + + # taosdemo query file config + def querysqls(self, sql: str) -> list: + return [{"sql":sql,"result":""}] + + def querycfg(self, sql: str) -> dict: + return { + "filetype": "query", + "cfgdir": self.config, + "host": self.host, + "port": 6030, + "user": self.user, + "password": self.password, + "confirm_parameter_prompt": "yes", + "query_times": 10, + "query_mode": "taosc", + "databases": self.dbname, + "specified_table_query": { + "query_interval": 0, + "concurrent": cpu_count(), + "sqls": self.querysqls(sql) + } + } + + def createqueryfile(self, sql: str): + date = datetime.datetime.now() + file_query_table = f"/tmp/query_{date:%F-%H%M}.json" + + with open(file_query_table,"w") as f: + json.dump(self.querycfg(sql), f) + + return file_query_table + + # Execute taosdemo, and delete temporary files when finished + def taosdemotable(self, filepath: str, resultfile="/dev/null"): + taosdemopath = self.getBuildPath() + "/debug/build/bin" + with open(filepath,"r") as f: + filetype = json.load(f)["filetype"] + if filetype == "insert": + taosdemo_table_cmd = f"{taosdemopath}/taosdemo -f {filepath} > {resultfile} 2>&1" + else: + taosdemo_table_cmd = f"yes | {taosdemopath}/taosdemo -f {filepath} > {resultfile} 2>&1" + try: + _ = subprocess.check_output(taosdemo_table_cmd, shell=True).decode("utf-8") + except subprocess.CalledProcessError as e: + _ = e.output + + def droptmpfile(self, filepath: str): + drop_file_cmd = f"[ -f {filepath} ] && rm -f {filepath}" + try: + _ = subprocess.check_output(drop_file_cmd, shell=True).decode("utf-8") + except subprocess.CalledProcessError as e: + _ = e.output + + # TODO:需要完成TD-4153的数据插入和客户端请求的性能查询。 + def td4153insert(self): + + tdLog.printNoPrefix("========== start to create table and insert data ==========") + self.dbname = "td4153" + db = self.dbinfocfg() + stblist = [] + + columntype = self.schemecfg(intcount=1, ncharcount=100) + tagtype = self.schemecfg(intcount=1) + stbname = "stb1" + prechild = "t1" + stable = self.stbcfg( + stb=stbname, + prechildtab=prechild, + child_tab_count=2, + rows=10000, + columns=columntype, + tags=tagtype + ) + stblist.append(stable) + insertfile = self.createinsertfile(db=db, stbs=stblist) + + nmon_file = f"/tmp/insert_{datetime.datetime.now():%F-%H%M}.nmon" + cmd = f"nmon -s5 -F {nmon_file} -m /tmp/" + try: + _ = subprocess.check_output(cmd, shell=True).decode("utf-8") + except subprocess.CalledProcessError as e: + _ = e.output + + self.taosdemotable(insertfile) + self.droptmpfile(insertfile) + self.droptmpfile("/tmp/insert_res.txt") + + # In order to prevent too many performance files from being generated, the nmon file is deleted. + # and the delete statement can be cancelled during the actual test. + self.droptmpfile(nmon_file) + + cmd = f"ps -ef|grep -w nmon| grep -v grep | awk '{{print $2}}'" + try: + time.sleep(10) + _ = subprocess.check_output(cmd,shell=True).decode("utf-8") + except BaseException as e: + raise e + + def td4153query(self): + tdLog.printNoPrefix("========== start to query operation ==========") + + sqls = { + "select_all": "select * from stb1", + "select_join": "select * from t10, t11 where t10.ts=t11.ts" + } + for type, sql in sqls.items(): + result_file = f"/tmp/queryResult_{type}.log" + query_file = self.createqueryfile(sql) + try: + self.taosdemotable(query_file, resultfile=result_file) + except subprocess.CalledProcessError as e: + out_put = e.output + if result_file: + print(f"execute rows {type.split('_')[1]} sql, the sql is: {sql}") + max_sql_time_cmd = f''' + grep -o Spent.*s {result_file} |awk 'NR==1{{max=$2;next}}{{max=max>$2?max:$2}}END{{print "Max=",max,"s"}}' + ''' + max_sql_time = subprocess.check_output(max_sql_time_cmd, shell=True).decode("UTF-8") + print(f"{type.split('_')[1]} rows sql time : {max_sql_time}") + + min_sql_time_cmd = f''' + grep -o Spent.*s {result_file} |awk 'NR==1{{min=$2;next}}{{min=min<$2?min:$2}}END{{print "Min=",min,"s"}}' + ''' + min_sql_time = subprocess.check_output(min_sql_time_cmd, shell=True).decode("UTF-8") + print(f"{type.split('_')[1]} rows sql time : {min_sql_time}") + + avg_sql_time_cmd = f''' + grep -o Spent.*s {result_file} |awk '{{sum+=$2}}END{{print "Average=",sum/NR,"s"}}' + ''' + avg_sql_time = subprocess.check_output(avg_sql_time_cmd, shell=True).decode("UTF-8") + print(f"{type.split('_')[1]} rows sql time : {avg_sql_time}") + + self.droptmpfile(query_file) + self.droptmpfile(result_file) + + drop_query_tmt_file_cmd = " find ./ -name 'querySystemInfo-*' -type f -exec rm {} \; " + try: + _ = subprocess.check_output(drop_query_tmt_file_cmd, shell=True).decode("utf-8") + except subprocess.CalledProcessError as e: + _ = e.output + pass + + def td4153(self): + self.td4153insert() + self.td4153query() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + '-r', + '--remove-cache', + action='store_true', + default=False, + help='clear cache before query (default: False)') + parser.add_argument( + '-d', + '--database-name', + action='store', + default='db', + type=str, + help='Database name to be created (default: db)') + parser.add_argument( + '-k', + '--keep-time', + action='store', + default=3650, + type=int, + help='Database keep parameters (default: 3650)') + + args = parser.parse_args() + taosdemo = Taosdemo(args.remove_cache, args.database_name, args.keep_time) + # taosdemo.conn = taos.connect( + # taosdemo.host, + # taosdemo.user, + # taosdemo.password, + # taosdemo.config + # ) + + debugdir = taosdemo.getExeToolsDir() + cfgdir = taosdemo.getCfgDir() + cmd = f"{debugdir}/taosd -c {cfgdir} >/dev/null 2>&1 &" + try: + _ = subprocess.check_output(cmd, shell=True).decode("utf-8") + except subprocess.CalledProcessError as e: + _ = e.output + + if taosdemo.clearCache: + # must be root permission + subprocess.check_output("echo 3 > /proc/sys/vm/drop_caches", shell=True).decode("utf-8") + + taosdemo.td4153() diff --git a/tests/pytest/stable/insert.py b/tests/pytest/stable/insert.py index 0ef816da8d..ef5635c77c 100644 --- a/tests/pytest/stable/insert.py +++ b/tests/pytest/stable/insert.py @@ -72,6 +72,9 @@ class TDTestCase: tdSql.query("describe db.stb") tdSql.checkRows(3) + tdSql.error("drop stable if exists db.dev_01") + tdSql.error("drop stable if exists db.dev_02") + tdSql.execute("alter stable db.stb add tag t1 int") tdSql.query("describe db.stb") tdSql.checkRows(4) @@ -80,6 +83,13 @@ class TDTestCase: tdSql.query("show stables") tdSql.checkRows(1) + tdSql.error("drop stable if exists db.dev_001") + tdSql.error("drop stable if exists db.dev_002") + + for i in range(10): + tdSql.execute("drop stable if exists db.stb") + tdSql.query("show stables") + tdSql.checkRows(1) def stop(self): tdSql.close() diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index b50180e2b3..a45393e222 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -24,7 +24,7 @@ class taosdemoPerformace: self.host = "127.0.0.1" self.user = "root" self.password = "taosdata" - self.config = "/etc/taosperf" + self.config = "/etc/perf" self.conn = taos.connect( self.host, self.user, @@ -77,7 +77,7 @@ class taosdemoPerformace: insert_data = { "filetype": "insert", - "cfgdir": "/etc/taosperf", + "cfgdir": "/etc/perf", "host": "127.0.0.1", "port": 6030, "user": "root", @@ -104,7 +104,7 @@ class taosdemoPerformace: return output def insertData(self): - os.system("taosdemo -f %s > taosdemoperf.txt" % self.generateJson()) + os.system("taosdemo -f %s > taosdemoperf.txt 2>&1" % self.generateJson()) self.createTableTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'") self.insertRecordsTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'") self.recordsPerSecond = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'") diff --git a/tests/pytest/tools/taosdemoTestWithJson.py b/tests/pytest/tools/taosdemoTestWithJson.py index f57af9ce5c..b2ecd54976 100644 --- a/tests/pytest/tools/taosdemoTestWithJson.py +++ b/tests/pytest/tools/taosdemoTestWithJson.py @@ -23,32 +23,32 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] + if "community" in selfPath: + projPath = selfPath[: selfPath.find("community")] else: - projPath = selfPath[:selfPath.find("tests")] + projPath = selfPath[: selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if "taosd" in files: rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + if "packaging" not in rootRealPath: + buildPath = root[: len(root) - len("/build/bin")] break return buildPath - + def run(self): tdSql.prepare() buildPath = self.getBuildPath() - if (buildPath == ""): + if buildPath == "": tdLog.exit("taosd not found!") else: tdLog.info("taosd found in %s" % buildPath) - binPath = buildPath+ "/build/bin/" - os.system("yes | %staosdemo -f tools/insert.json" % binPath) + binPath = buildPath + "/build/bin/" + os.system("%staosdemo -f tools/insert.json -y" % binPath) tdSql.execute("use db01") tdSql.query("select count(*) from stb01") diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index db48632cb7..55eb62d9bb 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -840,6 +840,1026 @@ int stmt_func3(TAOS_STMT *stmt) { } + +//1 tables 10 records +int stmt_funcb_autoctb1(TAOS_STMT *stmt) { + struct { + int64_t *ts; + int8_t b[10]; + int8_t v1[10]; + int16_t v2[10]; + int32_t v4[10]; + int64_t v8[10]; + float f4[10]; + double f8[10]; + char bin[10][40]; + } v = {0}; + + v.ts = malloc(sizeof(int64_t) * 1 * 10); + + int *lb = malloc(10 * sizeof(int)); + + TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + +// int one_null = 1; + int one_not_null = 0; + + char* is_null = malloc(sizeof(char) * 10); + char* no_null = malloc(sizeof(char) * 10); + + for (int i = 0; i < 10; ++i) { + lb[i] = 40; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v.b[i] = (int8_t)(i % 2); + v.v1[i] = (int8_t)((i+1) % 2); + v.v2[i] = (int16_t)i; + v.v4[i] = (int32_t)(i+1); + v.v8[i] = (int64_t)(i+2); + v.f4[i] = (float)(i+3); + v.f8[i] = (double)(i+4); + memset(v.bin[i], '0'+i%10, 40); + } + + for (int i = 0; i < 10; i+=10) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v.ts[10*i/10]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = 10; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v.b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = 10; + + params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[i+2].buffer_length = sizeof(int8_t); + params[i+2].buffer = v.v1; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = 10; + + params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+3].buffer_length = sizeof(int16_t); + params[i+3].buffer = v.v2; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = 10; + + params[i+4].buffer_type = TSDB_DATA_TYPE_INT; + params[i+4].buffer_length = sizeof(int32_t); + params[i+4].buffer = v.v4; + params[i+4].length = NULL; + params[i+4].is_null = is_null; + params[i+4].num = 10; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+5].buffer_length = sizeof(int64_t); + params[i+5].buffer = v.v8; + params[i+5].length = NULL; + params[i+5].is_null = is_null; + params[i+5].num = 10; + + params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+6].buffer_length = sizeof(float); + params[i+6].buffer = v.f4; + params[i+6].length = NULL; + params[i+6].is_null = is_null; + params[i+6].num = 10; + + params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+7].buffer_length = sizeof(double); + params[i+7].buffer = v.f8; + params[i+7].length = NULL; + params[i+7].is_null = is_null; + params[i+7].num = 10; + + params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+8].buffer_length = 40; + params[i+8].buffer = v.bin; + params[i+8].length = lb; + params[i+8].is_null = is_null; + params[i+8].num = 10; + + params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+9].buffer_length = 40; + params[i+9].buffer = v.bin; + params[i+9].length = lb; + params[i+9].is_null = is_null; + params[i+9].num = 10; + + } + + int64_t tts = 1591060628000; + for (int i = 0; i < 10; ++i) { + v.ts[i] = tts + i; + } + + + for (int i = 0; i < 1; ++i) { + tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; + tags[i+0].buffer = v.v4; + tags[i+0].is_null = &one_not_null; + tags[i+0].length = NULL; + + tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + tags[i+1].buffer = v.b; + tags[i+1].is_null = &one_not_null; + tags[i+1].length = NULL; + + tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + tags[i+2].buffer = v.v1; + tags[i+2].is_null = &one_not_null; + tags[i+2].length = NULL; + + tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + tags[i+3].buffer = v.v2; + tags[i+3].is_null = &one_not_null; + tags[i+3].length = NULL; + + tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; + tags[i+4].buffer = v.v8; + tags[i+4].is_null = &one_not_null; + tags[i+4].length = NULL; + + tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; + tags[i+5].buffer = v.f4; + tags[i+5].is_null = &one_not_null; + tags[i+5].length = NULL; + + tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; + tags[i+6].buffer = v.f8; + tags[i+6].is_null = &one_not_null; + tags[i+6].length = NULL; + + tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; + tags[i+7].buffer = v.bin; + tags[i+7].is_null = &one_not_null; + tags[i+7].length = (uintptr_t *)lb; + + tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; + tags[i+8].buffer = v.bin; + tags[i+8].is_null = &one_not_null; + tags[i+8].length = (uintptr_t *)lb; + } + + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + exit(1); + } + + int id = 0; + for (int zz = 0; zz < 1; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname_tags(stmt, buf, tags); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname_tags. code:0x%x\n", code); + } + + taos_stmt_bind_param_batch(stmt, params + id * 10); + taos_stmt_add_batch(stmt); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + + ++id; + + unsigned long long endtime = getCurrentTime(); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); + + free(v.ts); + free(lb); + free(params); + free(is_null); + free(no_null); + free(tags); + + return 0; +} + + + + +//1 tables 10 records +int stmt_funcb_autoctb2(TAOS_STMT *stmt) { + struct { + int64_t *ts; + int8_t b[10]; + int8_t v1[10]; + int16_t v2[10]; + int32_t v4[10]; + int64_t v8[10]; + float f4[10]; + double f8[10]; + char bin[10][40]; + } v = {0}; + + v.ts = malloc(sizeof(int64_t) * 1 * 10); + + int *lb = malloc(10 * sizeof(int)); + + TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + +// int one_null = 1; + int one_not_null = 0; + + char* is_null = malloc(sizeof(char) * 10); + char* no_null = malloc(sizeof(char) * 10); + + for (int i = 0; i < 10; ++i) { + lb[i] = 40; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v.b[i] = (int8_t)(i % 2); + v.v1[i] = (int8_t)((i+1) % 2); + v.v2[i] = (int16_t)i; + v.v4[i] = (int32_t)(i+1); + v.v8[i] = (int64_t)(i+2); + v.f4[i] = (float)(i+3); + v.f8[i] = (double)(i+4); + memset(v.bin[i], '0'+i%10, 40); + } + + for (int i = 0; i < 10; i+=10) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v.ts[10*i/10]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = 10; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v.b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = 10; + + params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[i+2].buffer_length = sizeof(int8_t); + params[i+2].buffer = v.v1; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = 10; + + params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+3].buffer_length = sizeof(int16_t); + params[i+3].buffer = v.v2; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = 10; + + params[i+4].buffer_type = TSDB_DATA_TYPE_INT; + params[i+4].buffer_length = sizeof(int32_t); + params[i+4].buffer = v.v4; + params[i+4].length = NULL; + params[i+4].is_null = is_null; + params[i+4].num = 10; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+5].buffer_length = sizeof(int64_t); + params[i+5].buffer = v.v8; + params[i+5].length = NULL; + params[i+5].is_null = is_null; + params[i+5].num = 10; + + params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+6].buffer_length = sizeof(float); + params[i+6].buffer = v.f4; + params[i+6].length = NULL; + params[i+6].is_null = is_null; + params[i+6].num = 10; + + params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+7].buffer_length = sizeof(double); + params[i+7].buffer = v.f8; + params[i+7].length = NULL; + params[i+7].is_null = is_null; + params[i+7].num = 10; + + params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+8].buffer_length = 40; + params[i+8].buffer = v.bin; + params[i+8].length = lb; + params[i+8].is_null = is_null; + params[i+8].num = 10; + + params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+9].buffer_length = 40; + params[i+9].buffer = v.bin; + params[i+9].length = lb; + params[i+9].is_null = is_null; + params[i+9].num = 10; + + } + + int64_t tts = 1591060628000; + for (int i = 0; i < 10; ++i) { + v.ts[i] = tts + i; + } + + + for (int i = 0; i < 1; ++i) { + tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; + tags[i+0].buffer = v.v4; + tags[i+0].is_null = &one_not_null; + tags[i+0].length = NULL; + + tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + tags[i+1].buffer = v.b; + tags[i+1].is_null = &one_not_null; + tags[i+1].length = NULL; + + tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + tags[i+2].buffer = v.v1; + tags[i+2].is_null = &one_not_null; + tags[i+2].length = NULL; + + tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + tags[i+3].buffer = v.v2; + tags[i+3].is_null = &one_not_null; + tags[i+3].length = NULL; + + tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; + tags[i+4].buffer = v.v8; + tags[i+4].is_null = &one_not_null; + tags[i+4].length = NULL; + + tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; + tags[i+5].buffer = v.f4; + tags[i+5].is_null = &one_not_null; + tags[i+5].length = NULL; + + tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; + tags[i+6].buffer = v.f8; + tags[i+6].is_null = &one_not_null; + tags[i+6].length = NULL; + + tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; + tags[i+7].buffer = v.bin; + tags[i+7].is_null = &one_not_null; + tags[i+7].length = (uintptr_t *)lb; + + tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; + tags[i+8].buffer = v.bin; + tags[i+8].is_null = &one_not_null; + tags[i+8].length = (uintptr_t *)lb; + } + + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? using stb1 tags(1,true,2,3,4,5.0,6.0,'a','b') values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + exit(1); + } + + int id = 0; + for (int zz = 0; zz < 1; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname_tags(stmt, buf, tags); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname_tags. code:0x%x\n", code); + } + + taos_stmt_bind_param_batch(stmt, params + id * 10); + taos_stmt_add_batch(stmt); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + + ++id; + + unsigned long long endtime = getCurrentTime(); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); + + free(v.ts); + free(lb); + free(params); + free(is_null); + free(no_null); + free(tags); + + return 0; +} + + + + + +//1 tables 10 records +int stmt_funcb_autoctb3(TAOS_STMT *stmt) { + struct { + int64_t *ts; + int8_t b[10]; + int8_t v1[10]; + int16_t v2[10]; + int32_t v4[10]; + int64_t v8[10]; + float f4[10]; + double f8[10]; + char bin[10][40]; + } v = {0}; + + v.ts = malloc(sizeof(int64_t) * 1 * 10); + + int *lb = malloc(10 * sizeof(int)); + + TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + +// int one_null = 1; + int one_not_null = 0; + + char* is_null = malloc(sizeof(char) * 10); + char* no_null = malloc(sizeof(char) * 10); + + for (int i = 0; i < 10; ++i) { + lb[i] = 40; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v.b[i] = (int8_t)(i % 2); + v.v1[i] = (int8_t)((i+1) % 2); + v.v2[i] = (int16_t)i; + v.v4[i] = (int32_t)(i+1); + v.v8[i] = (int64_t)(i+2); + v.f4[i] = (float)(i+3); + v.f8[i] = (double)(i+4); + memset(v.bin[i], '0'+i%10, 40); + } + + for (int i = 0; i < 10; i+=10) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v.ts[10*i/10]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = 10; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v.b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = 10; + + params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[i+2].buffer_length = sizeof(int8_t); + params[i+2].buffer = v.v1; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = 10; + + params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+3].buffer_length = sizeof(int16_t); + params[i+3].buffer = v.v2; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = 10; + + params[i+4].buffer_type = TSDB_DATA_TYPE_INT; + params[i+4].buffer_length = sizeof(int32_t); + params[i+4].buffer = v.v4; + params[i+4].length = NULL; + params[i+4].is_null = is_null; + params[i+4].num = 10; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+5].buffer_length = sizeof(int64_t); + params[i+5].buffer = v.v8; + params[i+5].length = NULL; + params[i+5].is_null = is_null; + params[i+5].num = 10; + + params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+6].buffer_length = sizeof(float); + params[i+6].buffer = v.f4; + params[i+6].length = NULL; + params[i+6].is_null = is_null; + params[i+6].num = 10; + + params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+7].buffer_length = sizeof(double); + params[i+7].buffer = v.f8; + params[i+7].length = NULL; + params[i+7].is_null = is_null; + params[i+7].num = 10; + + params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+8].buffer_length = 40; + params[i+8].buffer = v.bin; + params[i+8].length = lb; + params[i+8].is_null = is_null; + params[i+8].num = 10; + + params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+9].buffer_length = 40; + params[i+9].buffer = v.bin; + params[i+9].length = lb; + params[i+9].is_null = is_null; + params[i+9].num = 10; + + } + + int64_t tts = 1591060628000; + for (int i = 0; i < 10; ++i) { + v.ts[i] = tts + i; + } + + + for (int i = 0; i < 1; ++i) { + tags[i+0].buffer_type = TSDB_DATA_TYPE_BOOL; + tags[i+0].buffer = v.b; + tags[i+0].is_null = &one_not_null; + tags[i+0].length = NULL; + + tags[i+1].buffer_type = TSDB_DATA_TYPE_SMALLINT; + tags[i+1].buffer = v.v2; + tags[i+1].is_null = &one_not_null; + tags[i+1].length = NULL; + + tags[i+2].buffer_type = TSDB_DATA_TYPE_FLOAT; + tags[i+2].buffer = v.f4; + tags[i+2].is_null = &one_not_null; + tags[i+2].length = NULL; + + tags[i+3].buffer_type = TSDB_DATA_TYPE_BINARY; + tags[i+3].buffer = v.bin; + tags[i+3].is_null = &one_not_null; + tags[i+3].length = (uintptr_t *)lb; + } + + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? using stb1 tags(1,?,2,?,4,?,6.0,?,'b') values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + exit(1); + } + + int id = 0; + for (int zz = 0; zz < 1; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname_tags(stmt, buf, tags); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname_tags. code:0x%x\n", code); + } + + taos_stmt_bind_param_batch(stmt, params + id * 10); + taos_stmt_add_batch(stmt); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + + ++id; + + unsigned long long endtime = getCurrentTime(); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); + + free(v.ts); + free(lb); + free(params); + free(is_null); + free(no_null); + free(tags); + + return 0; +} + + + + + + +//1 tables 10 records +int stmt_funcb_autoctb_e1(TAOS_STMT *stmt) { + struct { + int64_t *ts; + int8_t b[10]; + int8_t v1[10]; + int16_t v2[10]; + int32_t v4[10]; + int64_t v8[10]; + float f4[10]; + double f8[10]; + char bin[10][40]; + } v = {0}; + + v.ts = malloc(sizeof(int64_t) * 1 * 10); + + int *lb = malloc(10 * sizeof(int)); + + TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + +// int one_null = 1; + int one_not_null = 0; + + char* is_null = malloc(sizeof(char) * 10); + char* no_null = malloc(sizeof(char) * 10); + + for (int i = 0; i < 10; ++i) { + lb[i] = 40; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v.b[i] = (int8_t)(i % 2); + v.v1[i] = (int8_t)((i+1) % 2); + v.v2[i] = (int16_t)i; + v.v4[i] = (int32_t)(i+1); + v.v8[i] = (int64_t)(i+2); + v.f4[i] = (float)(i+3); + v.f8[i] = (double)(i+4); + memset(v.bin[i], '0'+i%10, 40); + } + + for (int i = 0; i < 10; i+=10) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v.ts[10*i/10]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = 10; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v.b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = 10; + + params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[i+2].buffer_length = sizeof(int8_t); + params[i+2].buffer = v.v1; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = 10; + + params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+3].buffer_length = sizeof(int16_t); + params[i+3].buffer = v.v2; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = 10; + + params[i+4].buffer_type = TSDB_DATA_TYPE_INT; + params[i+4].buffer_length = sizeof(int32_t); + params[i+4].buffer = v.v4; + params[i+4].length = NULL; + params[i+4].is_null = is_null; + params[i+4].num = 10; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+5].buffer_length = sizeof(int64_t); + params[i+5].buffer = v.v8; + params[i+5].length = NULL; + params[i+5].is_null = is_null; + params[i+5].num = 10; + + params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+6].buffer_length = sizeof(float); + params[i+6].buffer = v.f4; + params[i+6].length = NULL; + params[i+6].is_null = is_null; + params[i+6].num = 10; + + params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+7].buffer_length = sizeof(double); + params[i+7].buffer = v.f8; + params[i+7].length = NULL; + params[i+7].is_null = is_null; + params[i+7].num = 10; + + params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+8].buffer_length = 40; + params[i+8].buffer = v.bin; + params[i+8].length = lb; + params[i+8].is_null = is_null; + params[i+8].num = 10; + + params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+9].buffer_length = 40; + params[i+9].buffer = v.bin; + params[i+9].length = lb; + params[i+9].is_null = is_null; + params[i+9].num = 10; + + } + + int64_t tts = 1591060628000; + for (int i = 0; i < 10; ++i) { + v.ts[i] = tts + i; + } + + + for (int i = 0; i < 1; ++i) { + tags[i+0].buffer_type = TSDB_DATA_TYPE_BOOL; + tags[i+0].buffer = v.b; + tags[i+0].is_null = &one_not_null; + tags[i+0].length = NULL; + + tags[i+1].buffer_type = TSDB_DATA_TYPE_SMALLINT; + tags[i+1].buffer = v.v2; + tags[i+1].is_null = &one_not_null; + tags[i+1].length = NULL; + + tags[i+2].buffer_type = TSDB_DATA_TYPE_FLOAT; + tags[i+2].buffer = v.f4; + tags[i+2].is_null = &one_not_null; + tags[i+2].length = NULL; + + tags[i+3].buffer_type = TSDB_DATA_TYPE_BINARY; + tags[i+3].buffer = v.bin; + tags[i+3].is_null = &one_not_null; + tags[i+3].length = (uintptr_t *)lb; + } + + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? using stb1 (id1,id2,id3,id4,id5,id6,id7,id8,id9) tags(1,?,2,?,4,?,6.0,?,'b') values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + return -1; + } + + int id = 0; + for (int zz = 0; zz < 1; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname_tags(stmt, buf, tags); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname_tags. code:0x%x\n", code); + } + + taos_stmt_bind_param_batch(stmt, params + id * 10); + taos_stmt_add_batch(stmt); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + + ++id; + + unsigned long long endtime = getCurrentTime(); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); + + free(v.ts); + free(lb); + free(params); + free(is_null); + free(no_null); + free(tags); + + return 0; +} + + + + + +//1 tables 10 records +int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { + struct { + int64_t *ts; + int8_t b[10]; + int8_t v1[10]; + int16_t v2[10]; + int32_t v4[10]; + int64_t v8[10]; + float f4[10]; + double f8[10]; + char bin[10][40]; + } v = {0}; + + v.ts = malloc(sizeof(int64_t) * 1 * 10); + + int *lb = malloc(10 * sizeof(int)); + + TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + +// int one_null = 1; + int one_not_null = 0; + + char* is_null = malloc(sizeof(char) * 10); + char* no_null = malloc(sizeof(char) * 10); + + for (int i = 0; i < 10; ++i) { + lb[i] = 40; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v.b[i] = (int8_t)(i % 2); + v.v1[i] = (int8_t)((i+1) % 2); + v.v2[i] = (int16_t)i; + v.v4[i] = (int32_t)(i+1); + v.v8[i] = (int64_t)(i+2); + v.f4[i] = (float)(i+3); + v.f8[i] = (double)(i+4); + memset(v.bin[i], '0'+i%10, 40); + } + + for (int i = 0; i < 10; i+=10) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v.ts[10*i/10]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = 10; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v.b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = 10; + + params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[i+2].buffer_length = sizeof(int8_t); + params[i+2].buffer = v.v1; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = 10; + + params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+3].buffer_length = sizeof(int16_t); + params[i+3].buffer = v.v2; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = 10; + + params[i+4].buffer_type = TSDB_DATA_TYPE_INT; + params[i+4].buffer_length = sizeof(int32_t); + params[i+4].buffer = v.v4; + params[i+4].length = NULL; + params[i+4].is_null = is_null; + params[i+4].num = 10; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+5].buffer_length = sizeof(int64_t); + params[i+5].buffer = v.v8; + params[i+5].length = NULL; + params[i+5].is_null = is_null; + params[i+5].num = 10; + + params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+6].buffer_length = sizeof(float); + params[i+6].buffer = v.f4; + params[i+6].length = NULL; + params[i+6].is_null = is_null; + params[i+6].num = 10; + + params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+7].buffer_length = sizeof(double); + params[i+7].buffer = v.f8; + params[i+7].length = NULL; + params[i+7].is_null = is_null; + params[i+7].num = 10; + + params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+8].buffer_length = 40; + params[i+8].buffer = v.bin; + params[i+8].length = lb; + params[i+8].is_null = is_null; + params[i+8].num = 10; + + params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+9].buffer_length = 40; + params[i+9].buffer = v.bin; + params[i+9].length = lb; + params[i+9].is_null = is_null; + params[i+9].num = 10; + + } + + int64_t tts = 1591060628000; + for (int i = 0; i < 10; ++i) { + v.ts[i] = tts + i; + } + + + for (int i = 0; i < 1; ++i) { + tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; + tags[i+0].buffer = v.v4; + tags[i+0].is_null = &one_not_null; + tags[i+0].length = NULL; + + tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + tags[i+1].buffer = v.b; + tags[i+1].is_null = &one_not_null; + tags[i+1].length = NULL; + + tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + tags[i+2].buffer = v.v1; + tags[i+2].is_null = &one_not_null; + tags[i+2].length = NULL; + + tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + tags[i+3].buffer = v.v2; + tags[i+3].is_null = &one_not_null; + tags[i+3].length = NULL; + + tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; + tags[i+4].buffer = v.v8; + tags[i+4].is_null = &one_not_null; + tags[i+4].length = NULL; + + tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; + tags[i+5].buffer = v.f4; + tags[i+5].is_null = &one_not_null; + tags[i+5].length = NULL; + + tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; + tags[i+6].buffer = v.f8; + tags[i+6].is_null = &one_not_null; + tags[i+6].length = NULL; + + tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; + tags[i+7].buffer = v.bin; + tags[i+7].is_null = &one_not_null; + tags[i+7].length = (uintptr_t *)lb; + + tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; + tags[i+8].buffer = v.bin; + tags[i+8].is_null = &one_not_null; + tags[i+8].length = (uintptr_t *)lb; + } + + + unsigned long long starttime = getCurrentTime(); + + char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + exit(1); + } + + int id = 0; + for (int zz = 0; zz < 1; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname_tags(stmt, buf, NULL); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname_tags. code:0x%x\n", code); + return -1; + } + + taos_stmt_bind_param_batch(stmt, params + id * 10); + taos_stmt_add_batch(stmt); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + + ++id; + + unsigned long long endtime = getCurrentTime(); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); + + free(v.ts); + free(lb); + free(params); + free(is_null); + free(no_null); + free(tags); + + return 0; +} + + + //300 tables 60 records int stmt_funcb1(TAOS_STMT *stmt) { struct { @@ -2540,14 +3560,14 @@ int sql_s_perf1(TAOS *taos) { } -void prepare(TAOS *taos, int bigsize) { +void prepare(TAOS *taos, int bigsize, int createChildTable) { TAOS_RES *result; int code; result = taos_query(taos, "drop database demo"); taos_free_result(result); - result = taos_query(taos, "create database demo"); + result = taos_query(taos, "create database demo keep 36500"); code = taos_errno(result); if (code != 0) { printf("failed to create database, reason:%s\n", taos_errstr(result)); @@ -2558,15 +3578,34 @@ void prepare(TAOS *taos, int bigsize) { result = taos_query(taos, "use demo"); taos_free_result(result); - - // create table - for (int i = 0 ; i < 300; i++) { + + if (createChildTable) { + // create table + for (int i = 0 ; i < 300; i++) { + char buf[1024]; + if (bigsize) { + sprintf(buf, "create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), bin2 binary(40))", i) ; + } else { + sprintf(buf, "create table m%d (ts timestamp, b int)", i) ; + } + result = taos_query(taos, buf); + code = taos_errno(result); + if (code != 0) { + printf("failed to create table, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + exit(1); + } + taos_free_result(result); + } + } else { char buf[1024]; if (bigsize) { - sprintf(buf, "create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), bin2 binary(40))", i) ; + sprintf(buf, "create stable stb1 (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), bin2 binary(40))" + " tags(id1 int, id2 bool, id3 tinyint, id4 smallint, id5 bigint, id6 float, id7 double, id8 binary(40), id9 nchar(40))") ; } else { - sprintf(buf, "create table m%d (ts timestamp, b int)", i) ; + sprintf(buf, "create stable stb1 (ts timestamp, b int) tags(id1 int, id2 bool, id3 tinyint, id4 smallint, id5 bigint, id6 float, id7 double, id8 binary(40), id9 nchar(40))") ; } + result = taos_query(taos, buf); code = taos_errno(result); if (code != 0) { @@ -2593,7 +3632,7 @@ void preparem(TAOS *taos, int bigsize, int idx) { result = taos_query(taos, sql); taos_free_result(result); - sprintf(sql, "create database %s", dbname); + sprintf(sql, "create database %s keep 36500", dbname); result = taos_query(taos, sql); code = taos_errno(result); if (code != 0) { @@ -2640,8 +3679,8 @@ void* runcase(void *par) { (void)idx; -#if 0 - prepare(taos, 1); +#if 1 + prepare(taos, 1, 1); stmt = taos_stmt_init(taos); @@ -2664,8 +3703,8 @@ void* runcase(void *par) { #endif -#if 0 - prepare(taos, 1); +#if 1 + prepare(taos, 1, 1); stmt = taos_stmt_init(taos); @@ -2679,8 +3718,8 @@ void* runcase(void *par) { #endif -#if 0 - prepare(taos, 1); +#if 1 + prepare(taos, 1, 1); stmt = taos_stmt_init(taos); @@ -2699,7 +3738,7 @@ void* runcase(void *par) { #endif #if 1 - prepare(taos, 1); + prepare(taos, 1, 1); stmt = taos_stmt_init(taos); @@ -2720,12 +3759,11 @@ void* runcase(void *par) { printf("check result end\n"); taos_stmt_close(stmt); - return NULL; #endif #if 1 - prepare(taos, 1); + prepare(taos, 1, 1); stmt = taos_stmt_init(taos); @@ -2749,7 +3787,7 @@ void* runcase(void *par) { #endif #if 1 - prepare(taos, 1); + prepare(taos, 1, 1); stmt = taos_stmt_init(taos); @@ -2773,7 +3811,7 @@ void* runcase(void *par) { #endif #if 1 - prepare(taos, 1); + prepare(taos, 1, 1); stmt = taos_stmt_init(taos); @@ -2798,7 +3836,7 @@ void* runcase(void *par) { #if 1 - prepare(taos, 1); + prepare(taos, 1, 1); stmt = taos_stmt_init(taos); @@ -2813,11 +3851,88 @@ void* runcase(void *par) { check_result(taos, "m299", 0, 180000); printf("check result end\n"); taos_stmt_close(stmt); + +#endif + +#if 1 + prepare(taos, 1, 0); + + stmt = taos_stmt_init(taos); + + printf("1t+10r+bm+autoctb start\n"); + stmt_funcb_autoctb1(stmt); + printf("1t+10r+bm+autoctb end\n"); + printf("check result start\n"); + check_result(taos, "m0", 1, 10); + printf("check result end\n"); + taos_stmt_close(stmt); + +#endif + +#if 1 + prepare(taos, 1, 0); + + stmt = taos_stmt_init(taos); + + printf("1t+10r+bm+autoctb start\n"); + stmt_funcb_autoctb2(stmt); + printf("1t+10r+bm+autoctb end\n"); + printf("check result start\n"); + check_result(taos, "m0", 1, 10); + printf("check result end\n"); + taos_stmt_close(stmt); + +#endif + + +#if 1 + prepare(taos, 1, 0); + + stmt = taos_stmt_init(taos); + + printf("1t+10r+bm+autoctb start\n"); + stmt_funcb_autoctb3(stmt); + printf("1t+10r+bm+autoctb end\n"); + printf("check result start\n"); + check_result(taos, "m0", 1, 10); + printf("check result end\n"); + taos_stmt_close(stmt); + +#endif + +#if 1 + prepare(taos, 1, 0); + + stmt = taos_stmt_init(taos); + + printf("1t+10r+bm+autoctb+e1 start\n"); + stmt_funcb_autoctb_e1(stmt); + printf("1t+10r+bm+autoctb+e1 end\n"); + printf("check result start\n"); + //check_result(taos, "m0", 1, 0); + printf("check result end\n"); + taos_stmt_close(stmt); + +#endif + +#if 1 + prepare(taos, 1, 0); + + stmt = taos_stmt_init(taos); + + printf("1t+10r+bm+autoctb+e2 start\n"); + stmt_funcb_autoctb_e2(stmt); + printf("1t+10r+bm+autoctb+e2 end\n"); + printf("check result start\n"); + //check_result(taos, "m0", 1, 0); + printf("check result end\n"); + taos_stmt_close(stmt); + #endif #if 1 - prepare(taos, 1); + prepare(taos, 1, 1); stmt = taos_stmt_init(taos); @@ -2836,7 +3951,7 @@ void* runcase(void *par) { #endif #if 1 - prepare(taos, 1); + prepare(taos, 1, 1); stmt = taos_stmt_init(taos); @@ -2856,7 +3971,7 @@ void* runcase(void *par) { #if 1 - prepare(taos, 1); + prepare(taos, 1, 1); stmt = taos_stmt_init(taos); @@ -2874,7 +3989,7 @@ void* runcase(void *par) { #endif #if 1 - prepare(taos, 1); + prepare(taos, 1, 1); stmt = taos_stmt_init(taos); @@ -2890,7 +4005,7 @@ void* runcase(void *par) { #if 1 - prepare(taos, 1); + prepare(taos, 1, 1); stmt = taos_stmt_init(taos); @@ -2908,7 +4023,7 @@ void* runcase(void *par) { #endif #if 1 - prepare(taos, 1); + prepare(taos, 1, 1); stmt = taos_stmt_init(taos); @@ -2927,7 +4042,7 @@ void* runcase(void *par) { #endif #if 1 - prepare(taos, 1); + prepare(taos, 1, 1); stmt = taos_stmt_init(taos); @@ -2952,7 +4067,7 @@ void* runcase(void *par) { #if 1 - prepare(taos, 1); + prepare(taos, 1, 1); stmt = taos_stmt_init(taos); @@ -2972,7 +4087,7 @@ void* runcase(void *par) { #if 1 - prepare(taos, 1); + prepare(taos, 1, 1); (void)stmt; printf("120t+60r+sql start\n"); @@ -2988,7 +4103,7 @@ void* runcase(void *par) { #endif #if 1 - prepare(taos, 1); + prepare(taos, 1, 1); (void)stmt; printf("1t+60r+sql start\n"); diff --git a/tests/script/api/stmtBatchTest.c b/tests/script/api/stmtBatchTest.c index 3ce2fefb0d..8bd296db61 100644 --- a/tests/script/api/stmtBatchTest.c +++ b/tests/script/api/stmtBatchTest.c @@ -37,7 +37,7 @@ typedef struct { //void taosMsleep(int mseconds); -int g_rows = 0; +int g_runTimes = 5; unsigned long long getCurrentTime(){ @@ -1336,7 +1336,7 @@ static void checkResult(TAOS *taos, char *tname, int printr, int expected) { result = taos_query(taos, sql); int code = taos_errno(result); if (code != 0) { - printf("failed to query table, reason:%s\n", taos_errstr(result)); + printf("failed to query table: %s, reason:%s\n", tname, taos_errstr(result)); taos_free_result(result); return; } @@ -1358,9 +1358,9 @@ static void checkResult(TAOS *taos, char *tname, int printr, int expected) { } if (rows == expected) { - printf("%d rows are fetched as expectation\n", rows); + printf("%d rows are fetched as expectation from %s\n", rows, tname); } else { - printf("!!!expect %d rows, but %d rows are fetched\n", expected, rows); + printf("!!!expect %d rows, but %d rows are fetched from %s\n", expected, rows, tname); return; } @@ -1451,15 +1451,52 @@ static void prepareV_long(TAOS *taos, int schemaCase, int tableNum, int lenO -//void runcase(TAOS *taos, int idx) { -static void* runCase(void *para) { - ThreadInfo* tInfo = (ThreadInfo *)para; - TAOS *taos = tInfo->taos; - int idx = tInfo->idx; - - TAOS_STMT *stmt = NULL; +static void prepareVcolumn(TAOS *taos, int schemaCase, int tableNum, int lenOfBinaryDef, char* dbName) { + TAOS_RES *result; + int code; + char sqlstr[1024] = {0}; + sprintf(sqlstr, "drop database if exists %s;", dbName); + result = taos_query(taos, sqlstr); + taos_free_result(result); - (void)idx; + sprintf(sqlstr, "create database %s;", dbName); + result = taos_query(taos, sqlstr); + code = taos_errno(result); + if (code != 0) { + printf("failed to create database, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); + + sprintf(sqlstr, "use %s;", dbName); + result = taos_query(taos, sqlstr); + taos_free_result(result); + + // create table + for (int i = 0 ; i < tableNum; i++) { + char buf[1024]; + if (schemaCase) { + sprintf(buf, "create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, br binary(%d), nr nchar(%d), ts2 timestamp)", i, lenOfBinaryDef, lenOfBinaryDef) ; + } else { + sprintf(buf, "create table m%d (ts timestamp, b int)", i) ; + } + + result = taos_query(taos, buf); + code = taos_errno(result); + if (code != 0) { + printf("failed to create table, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); + } + +} + +//void runcase(TAOS *taos, int idx) { +static void runCase(TAOS *taos) { + TAOS_STMT *stmt = NULL; int tableNum; int lenOfBinaryDef; @@ -1975,7 +2012,7 @@ static void* runCase(void *para) { } #endif - return NULL; + return ; } @@ -2166,14 +2203,8 @@ static int stmt_bind_case_001_long(TAOS_STMT *stmt, int tableNum, int rowsOfPerC return 0; } -static void* runCase_long(void *para) { - ThreadInfo* tInfo = (ThreadInfo *)para; - TAOS *taos = tInfo->taos; - int idx = tInfo->idx; - +static void runCase_long(TAOS *taos) { TAOS_STMT *stmt = NULL; - - (void)idx; int tableNum; int lenOfBinaryDef; @@ -2228,7 +2259,7 @@ static void* runCase_long(void *para) { prepareV_long(taos, 1, tableNum, lenOfBinaryDef); totalRowsPerTbl = 0; - for (int i = 0; i < 30000; i++) { + for (int i = 0; i < g_runTimes; i++) { stmt = taos_stmt_init(taos); stmt_bind_case_001_long(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum, &startTs); @@ -2245,17 +2276,895 @@ static void* runCase_long(void *para) { } #endif - return NULL; + return; } +/*=======================*/ +/* +test scene: insert into tb1 (ts,f1) values (?,?) +*/ +static int stmt_specifyCol_bind_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { + sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + + int totalRowsPerTbl = rowsOfPerColum * bingNum; + + v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + + int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + + int64_t tts = 1591060628000; + + for (int i = 0; i < rowsOfPerColum; ++i) { + lb[i] = lenOfBinaryAct; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v->b[i] = (int8_t)(i % 2); + v->v1[i] = (int8_t)((i+1) % 2); + v->v2[i] = (int16_t)i; + v->v4[i] = (int32_t)(i+1); + v->v8[i] = (int64_t)(i+2); + v->f4[i] = (float)(i+3); + v->f8[i] = (double)(i+4); + char tbuf[MAX_BINARY_DEF_LEN]; + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "binary-%d-0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", i%10); + memcpy(v->br + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "nchar-%d-0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", i%10); + memcpy(v->nr + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + v->ts2[i] = tts + i; + } + + int i = 0; + for (int j = 0; j < bingNum * tableNum; j++) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v->ts[j*rowsOfPerColum]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = rowsOfPerColum; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v->b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = rowsOfPerColum; + + params[i+2].buffer_type = TSDB_DATA_TYPE_INT; + params[i+2].buffer_length = sizeof(int32_t); + params[i+2].buffer = v->v4; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = rowsOfPerColum; + + params[i+3].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+3].buffer_length = sizeof(float); + params[i+3].buffer = v->f4; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = rowsOfPerColum; + + params[i+4].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+4].buffer_length = (uintptr_t)lenOfBinaryDef; + params[i+4].buffer = v->br; + params[i+4].length = lb; + params[i+4].is_null = is_null; + params[i+4].num = rowsOfPerColum; + + i+=columnNum; + } + + //int64_t tts = 1591060628000; + for (int i = 0; i < totalRowsPerTbl * tableNum; ++i) { + v->ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + +// create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, br binary(%d), nr nchar(%d), ts2 timestamp) + char *sql = "insert into m0 (ts,b,v4,f4,br) values(?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + int id = 0; + for (int l = 0; l < bingNum; l++) { + for (int zz = 0; zz < tableNum; zz++) { + //char buf[32]; + //sprintf(buf, "m%d", zz); + //code = taos_stmt_set_tbname(stmt, buf); + //if (code != 0){ + // printf("failed to execute taos_stmt_set_tbname. code:0x%x[%s]\n", code, tstrerror(code)); + // return -1; + //} + + for (int col=0; col < columnNum; ++col) { + code = taos_stmt_bind_single_param_batch(stmt, params + id, col); + if (code != 0){ + printf("failed to execute taos_stmt_bind_single_param_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + id++; + } + + code = taos_stmt_add_batch(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_add_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_execute. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + unsigned long long endtime = getCurrentTime(); + unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); + printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); + + free(v->ts); + free(v->br); + free(v->nr); + free(v); + free(lb); + free(params); + free(is_null); + free(no_null); + + return 0; +} + +/*=======================*/ +/* +test scene: insert into ? (ts,f1) values (?,?) +*/ +static int stmt_specifyCol_bind_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { + sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + + int totalRowsPerTbl = rowsOfPerColum * bingNum; + + v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + + int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + + int64_t tts = 1591060628000; + + for (int i = 0; i < rowsOfPerColum; ++i) { + lb[i] = lenOfBinaryAct; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v->b[i] = (int8_t)(i % 2); + v->v1[i] = (int8_t)((i+1) % 2); + v->v2[i] = (int16_t)i; + v->v4[i] = (int32_t)(i+1); + v->v8[i] = (int64_t)(i+2); + v->f4[i] = (float)(i+3); + v->f8[i] = (double)(i+4); + char tbuf[MAX_BINARY_DEF_LEN]; + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "binary-%d", i%10); + memcpy(v->br + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "nchar-%d", i%10); + memcpy(v->nr + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + v->ts2[i] = tts + i; + } + + int i = 0; + for (int j = 0; j < bingNum * tableNum; j++) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v->ts[j*rowsOfPerColum]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = rowsOfPerColum; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v->b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = rowsOfPerColum; + + params[i+2].buffer_type = TSDB_DATA_TYPE_INT; + params[i+2].buffer_length = sizeof(int32_t); + params[i+2].buffer = v->v4; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = rowsOfPerColum; + + params[i+3].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+3].buffer_length = sizeof(float); + params[i+3].buffer = v->f4; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = rowsOfPerColum; + + params[i+4].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+4].buffer_length = (uintptr_t)lenOfBinaryDef; + params[i+4].buffer = v->br; + params[i+4].length = lb; + params[i+4].is_null = is_null; + params[i+4].num = rowsOfPerColum; + + i+=columnNum; + } + + //int64_t tts = 1591060628000; + for (int i = 0; i < totalRowsPerTbl * tableNum; ++i) { + v->ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + +// create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, br binary(%d), nr nchar(%d), ts2 timestamp) + char *sql = "insert into ? (ts,b,v4,f4,br) values(?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + int id = 0; + for (int l = 0; l < bingNum; l++) { + for (int zz = 0; zz < tableNum; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + for (int col=0; col < columnNum; ++col) { + code = taos_stmt_bind_single_param_batch(stmt, params + id, col); + if (code != 0){ + printf("failed to execute taos_stmt_bind_single_param_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + id++; + } + + code = taos_stmt_add_batch(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_add_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_execute. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + unsigned long long endtime = getCurrentTime(); + unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); + printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); + + free(v->ts); + free(v->br); + free(v->nr); + free(v); + free(lb); + free(params); + free(is_null); + free(no_null); + + return 0; +} + +/*=======================*/ +/* +test scene: insert into tb1 (ts,f1) values (?,?) +*/ +static int stmt_specifyCol_bind_case_001_maxRows(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { + sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + + int totalRowsPerTbl = rowsOfPerColum * bingNum; + + v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + + int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + + int64_t tts = 1591060628000; + + for (int i = 0; i < rowsOfPerColum; ++i) { + lb[i] = lenOfBinaryAct; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v->b[i] = (int8_t)(i % 2); + v->v1[i] = (int8_t)((i+1) % 2); + v->v2[i] = (int16_t)i; + v->v4[i] = (int32_t)(i+1); + v->v8[i] = (int64_t)(i+2); + v->f4[i] = (float)(i+3); + v->f8[i] = (double)(i+4); + char tbuf[MAX_BINARY_DEF_LEN]; + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "binary-%d-0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", i%10); + memcpy(v->br + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "nchar-%d-0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", i%10); + memcpy(v->nr + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + v->ts2[i] = tts + i; + } + + int i = 0; + for (int j = 0; j < bingNum * tableNum; j++) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v->ts[j*rowsOfPerColum]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = rowsOfPerColum; + + params[i+1].buffer_type = TSDB_DATA_TYPE_INT; + params[i+1].buffer_length = sizeof(int32_t); + params[i+1].buffer = v->v4; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = rowsOfPerColum; + + i+=columnNum; + } + + //int64_t tts = 1591060628000; + for (int i = 0; i < totalRowsPerTbl * tableNum; ++i) { + v->ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + +// create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, br binary(%d), nr nchar(%d), ts2 timestamp) + char *sql = "insert into m0 (ts,b,v4,f4,br) values(?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + int id = 0; + for (int l = 0; l < bingNum; l++) { + for (int zz = 0; zz < tableNum; zz++) { + //char buf[32]; + //sprintf(buf, "m%d", zz); + //code = taos_stmt_set_tbname(stmt, buf); + //if (code != 0){ + // printf("failed to execute taos_stmt_set_tbname. code:0x%x[%s]\n", code, tstrerror(code)); + // return -1; + //} + + for (int col=0; col < columnNum; ++col) { + code = taos_stmt_bind_single_param_batch(stmt, params + id, col); + if (code != 0){ + printf("failed to execute taos_stmt_bind_single_param_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + id++; + } + + code = taos_stmt_add_batch(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_add_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_execute. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + unsigned long long endtime = getCurrentTime(); + unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); + printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); + + free(v->ts); + free(v->br); + free(v->nr); + free(v); + free(lb); + free(params); + free(is_null); + free(no_null); + + return 0; +} + +static void SpecifyColumnBatchCase(TAOS *taos) { + TAOS_STMT *stmt = NULL; + + int tableNum; + int lenOfBinaryDef; + int rowsOfPerColum; + int bingNum; + int lenOfBinaryAct; + int columnNum; + + int totalRowsPerTbl; + +//=======================================================================// +//=============================== single table ==========================// +//========== case 1: ======================// +#if 1 +{ + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 1; + bingNum = 1; + lenOfBinaryDef = 40; + lenOfBinaryAct = 8; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db1"); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 1 check result end\n\n"); +} +#endif + + //========== case 2: ======================// +#if 1 +{ + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 5; + bingNum = 1; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 15; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db2"); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + //checkResult(taos, "m1", 0, totalRowsPerTbl); + //checkResult(taos, "m2", 0, totalRowsPerTbl); + //checkResult(taos, "m3", 0, totalRowsPerTbl); + //checkResult(taos, "m4", 0, totalRowsPerTbl); + //checkResult(taos, "m5", 0, totalRowsPerTbl); + //checkResult(taos, "m6", 0, totalRowsPerTbl); + //checkResult(taos, "m7", 0, totalRowsPerTbl); + //checkResult(taos, "m8", 0, totalRowsPerTbl); + //checkResult(taos, "m9", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 2 check result end\n\n"); +} +#endif + + //========== case 2-1: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 32767; + bingNum = 1; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 15; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db2_1"); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + //checkResult(taos, "m1", 0, totalRowsPerTbl); + //checkResult(taos, "m2", 0, totalRowsPerTbl); + //checkResult(taos, "m3", 0, totalRowsPerTbl); + //checkResult(taos, "m4", 0, totalRowsPerTbl); + //checkResult(taos, "m5", 0, totalRowsPerTbl); + //checkResult(taos, "m6", 0, totalRowsPerTbl); + //checkResult(taos, "m7", 0, totalRowsPerTbl); + //checkResult(taos, "m8", 0, totalRowsPerTbl); + //checkResult(taos, "m9", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 2-1 check result end\n\n"); + } +#endif + //========== case 2-2: ======================// +#if 1 + { + printf("====case 2-2 error test start\n"); + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 32768; + bingNum = 1; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 15; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db2_2"); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + //checkResult(taos, "m1", 0, totalRowsPerTbl); + //checkResult(taos, "m2", 0, totalRowsPerTbl); + //checkResult(taos, "m3", 0, totalRowsPerTbl); + //checkResult(taos, "m4", 0, totalRowsPerTbl); + //checkResult(taos, "m5", 0, totalRowsPerTbl); + //checkResult(taos, "m6", 0, totalRowsPerTbl); + //checkResult(taos, "m7", 0, totalRowsPerTbl); + //checkResult(taos, "m8", 0, totalRowsPerTbl); + //checkResult(taos, "m9", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("====case 2-2 check result end\n\n"); + } +#endif + + + //========== case 3: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 1; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 20; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db3"); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + //checkResult(taos, "m1", 0, totalRowsPerTbl); + //checkResult(taos, "m2", 0, totalRowsPerTbl); + //checkResult(taos, "m3", 0, totalRowsPerTbl); + //checkResult(taos, "m4", 0, totalRowsPerTbl); + //checkResult(taos, "m5", 0, totalRowsPerTbl); + //checkResult(taos, "m6", 0, totalRowsPerTbl); + //checkResult(taos, "m7", 0, totalRowsPerTbl); + //checkResult(taos, "m8", 0, totalRowsPerTbl); + //checkResult(taos, "m9", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 3 check result end\n\n"); + } +#endif + + //========== case 4: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 5; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 33; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db4"); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + //checkResult(taos, "m1", 0, totalRowsPerTbl); + //checkResult(taos, "m2", 0, totalRowsPerTbl); + //checkResult(taos, "m3", 0, totalRowsPerTbl); + //checkResult(taos, "m4", 0, totalRowsPerTbl); + //checkResult(taos, "m5", 0, totalRowsPerTbl); + //checkResult(taos, "m6", 0, totalRowsPerTbl); + //checkResult(taos, "m7", 0, totalRowsPerTbl); + //checkResult(taos, "m8", 0, totalRowsPerTbl); + //checkResult(taos, "m9", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 4 check result end\n\n"); + } +#endif + +//=======================================================================// +//=============================== multi tables ==========================// + //========== case 5: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 1; + bingNum = 1; + lenOfBinaryDef = 40; + lenOfBinaryAct = 16; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db5"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 5 check result end\n\n"); + } +#endif + + //========== case 6: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 5; + bingNum = 1; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 20; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db6"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 6 check result end\n\n"); + } +#endif + + //========== case 7: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 1; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 33; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db7"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 7 check result end\n\n"); + } +#endif + + //========== case 8: ======================// +#if 1 +{ + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 5; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 40; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db8"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 8 check result end\n\n"); +} +#endif + + //=======================================================================// + //=============================== multi-rows to single table ==========================// + //========== case 9: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 23740; + bingNum = 1; + lenOfBinaryDef = 40; + lenOfBinaryAct = 8; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db9"); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 9 check result end\n\n"); + } +#endif + + //========== case 10: ======================// +#if 0 + { + printf("====case 10 error test start\n"); + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 23741; // WAL size exceeds limit + bingNum = 1; + lenOfBinaryDef = 40; + lenOfBinaryAct = 8; + columnNum = 5; + + prepareV(taos, 1, tableNum, lenOfBinaryDef); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("====case 10 check result end\n\n"); + } +#endif + + + //=======================================================================// + //=============================== multi tables, multi bind one same table ==========================// + //========== case 13: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 1; + bingNum = 5; + lenOfBinaryDef = 40; + lenOfBinaryAct = 28; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db13"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 13 check result end\n\n"); + } +#endif + + //========== case 14: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 5; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 33; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db14"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 14 check result end\n\n"); + } +#endif + + + //========== case 15: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 1000; + rowsOfPerColum = 10; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 8; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db15"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m111", 0, totalRowsPerTbl); + checkResult(taos, "m222", 0, totalRowsPerTbl); + checkResult(taos, "m333", 0, totalRowsPerTbl); + checkResult(taos, "m500", 0, totalRowsPerTbl); + checkResult(taos, "m999", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 15 check result end\n\n"); + } +#endif + + //========== case 17: ======================// +#if 1 + { + //printf("case 17 test start\n"); + stmt = taos_stmt_init(taos); + + tableNum = 10; + rowsOfPerColum = 100; + bingNum = 1; + lenOfBinaryDef = 100; + lenOfBinaryAct = 8; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db17"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m5", 0, totalRowsPerTbl); + checkResult(taos, "m8", 0, totalRowsPerTbl); + checkResult(taos, "m9", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 17 check result end\n\n"); + } +#endif + + return ; + +} int main(int argc, char *argv[]) { TAOS *taos; char host[32] = "127.0.0.1"; char* serverIp = NULL; - int threadNum = 2; + int threadNum = 1; // connect to server if (argc == 1) { @@ -2268,9 +3177,12 @@ int main(int argc, char *argv[]) } else if (argc == 4) { serverIp = argv[1]; threadNum = atoi(argv[2]); - g_rows = atoi(argv[3]); + g_runTimes = atoi(argv[3]); } + printf("server:%s, runTimes:%d\n\n", serverIp, g_runTimes); + +#if 0 printf("server:%s, threadNum:%d, rows:%d\n\n", serverIp, threadNum, g_rows); pthread_t *pThreadList = (pthread_t *) calloc(sizeof(pthread_t), (size_t)threadNum); @@ -2287,7 +3199,8 @@ int main(int argc, char *argv[]) tInfo->taos = taos; tInfo->idx = i; if (0 == i) { - pthread_create(&(pThreadList[0]), NULL, runCase, (void *)tInfo); + //pthread_create(&(pThreadList[0]), NULL, runCase, (void *)tInfo); + pthread_create(&(pThreadList[0]), NULL, SpecifyColumnBatchCase, (void *)tInfo); } else if (1 == i){ pthread_create(&(pThreadList[0]), NULL, runCase_long, (void *)tInfo); } @@ -2300,6 +3213,18 @@ int main(int argc, char *argv[]) free(pThreadList); free(threadInfo); +#endif + + taos = taos_connect(serverIp, "root", "taosdata", NULL, 0); + if (taos == NULL) { + printf("failed to connect to TDengine, reason:%s\n", taos_errstr(taos)); + return -1; + } + + runCase(taos); + runCase_long(taos); + SpecifyColumnBatchCase(taos); + return 0; } diff --git a/tests/script/fullGeneralSuite.sim b/tests/script/fullGeneralSuite.sim index cde51ebdbf..2cd2236200 100644 --- a/tests/script/fullGeneralSuite.sim +++ b/tests/script/fullGeneralSuite.sim @@ -33,6 +33,7 @@ run general/compute/percentile.sim run general/compute/stddev.sim run general/compute/sum.sim run general/compute/top.sim +run general/compute/block_dist.sim run general/db/alter_option.sim run general/db/alter_tables_d2.sim run general/db/alter_tables_v1.sim diff --git a/tests/script/general/compute/block_dist.sim b/tests/script/general/compute/block_dist.sim new file mode 100644 index 0000000000..51cf903654 --- /dev/null +++ b/tests/script/general/compute/block_dist.sim @@ -0,0 +1,94 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/exec.sh -n dnode1 -s start +sleep 2000 +sql connect + +$dbPrefix = m_di_db +$tbPrefix = m_di_tb +$mtPrefix = m_di_mt +$ntPrefix = m_di_nt +$tbNum = 1 +$rowNum = 2000 + +print =============== step1 +$i = 0 +$db = $dbPrefix . $i +$mt = $mtPrefix . $i +$nt = $ntPrefix . $i + +sql drop database $db -x step1 +step1: +sql create database $db +sql use $db +sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol int) + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $mt tags( $i ) + + $x = 0 + while $x < $rowNum + $cc = $x * 60000 + $ms = 1601481600000 + $cc + sql insert into $tb values ($ms , $x ) + $x = $x + 1 + endw + + $i = $i + 1 +endw + +sql create table $nt (ts timestamp, tbcol int) +$x = 0 +while $x < $rowNum + $cc = $x * 60000 + $ms = 1601481600000 + $cc + sql insert into $nt values ($ms , $x ) + $x = $x + 1 +endw + +sleep 100 + +print =============== step2 +$i = 0 +$tb = $tbPrefix . $i + +sql select _block_dist() from $tb + +if $rows != 1 then + print expect 1, actual:$rows + return -1 +endi + +print =============== step3 +$i = 0 +$mt = $mtPrefix . $i +sql select _block_dist() from $mt + +if $rows != 1 then + print expect 1, actual:$rows + return -1 +endi + +print =============== step4 +$i = 0 +$nt = $ntPrefix . $i + +sql select _block_dist() from $nt + +if $rows != 1 then + print expect 1, actual:$rows + return -1 +endi + +print =============== clear +sql drop database $db +sql show databases +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/compute/testSuite.sim b/tests/script/general/compute/testSuite.sim index 6cd6badaee..91bf4bf0cd 100644 --- a/tests/script/general/compute/testSuite.sim +++ b/tests/script/general/compute/testSuite.sim @@ -14,3 +14,4 @@ run general/compute/percentile.sim run general/compute/stddev.sim run general/compute/sum.sim run general/compute/top.sim +run general/compute/block_dist.sim diff --git a/tests/script/general/parser/binary_escapeCharacter.sim b/tests/script/general/parser/binary_escapeCharacter.sim index b5bb10284b..176edd4e29 100644 --- a/tests/script/general/parser/binary_escapeCharacter.sim +++ b/tests/script/general/parser/binary_escapeCharacter.sim @@ -1,3 +1,4 @@ + system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 diff --git a/tests/script/general/parser/commit.sim b/tests/script/general/parser/commit.sim index dfe521b92b..7c4c883fb1 100644 --- a/tests/script/general/parser/commit.sim +++ b/tests/script/general/parser/commit.sim @@ -68,7 +68,7 @@ while $loop <= $loops while $i < 10 sql select count(*) from $stb where t1 = $i if $data00 != $rowNum then - print expect $rowNum, actual: $data00 + print expect $rowNum , actual: $data00 return -1 endi $i = $i + 1 diff --git a/tests/script/general/parser/function.sim b/tests/script/general/parser/function.sim index 65058333fb..56ce15c36f 100644 --- a/tests/script/general/parser/function.sim +++ b/tests/script/general/parser/function.sim @@ -809,3 +809,5 @@ endi if $data00 != 1 then return -1 endi + +print ====================> TODO stddev + normal column filter diff --git a/tests/script/general/parser/last_cache.sim b/tests/script/general/parser/last_cache.sim new file mode 100644 index 0000000000..9d7da9ddba --- /dev/null +++ b/tests/script/general/parser/last_cache.sim @@ -0,0 +1,71 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 +system sh/exec.sh -n dnode1 -s start + +sleep 100 +sql connect +print ======================== dnode1 start + +$db = testdb + +sql create database $db cachelast 2 +sql use $db + +sql create stable st2 (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) tags (id int) + +sql create table tb1 using st2 tags (1); +sql create table tb2 using st2 tags (2); +sql create table tb3 using st2 tags (3); +sql create table tb4 using st2 tags (4); +sql create table tb5 using st2 tags (1); +sql create table tb6 using st2 tags (2); +sql create table tb7 using st2 tags (3); +sql create table tb8 using st2 tags (4); +sql create table tb9 using st2 tags (5); +sql create table tba using st2 tags (5); +sql create table tbb using st2 tags (5); +sql create table tbc using st2 tags (5); +sql create table tbd using st2 tags (5); +sql create table tbe using st2 tags (5); + +sql insert into tb1 values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000) +sql insert into tb1 values ("2021-05-10 10:10:11", 4, 5.0, NULL, -2000) +sql insert into tb1 values ("2021-05-12 10:10:12", 6,NULL, NULL, -3000) + +sql insert into tb2 values ("2021-05-09 10:11:13",-1,-2.0,'-3', -1001) +sql insert into tb2 values ("2021-05-10 10:11:14",-4,-5.0, NULL, -2001) +sql insert into tb2 values ("2021-05-11 10:11:15",-6, -7, '-8', -3001) + +sql insert into tb3 values ("2021-05-09 10:12:17", 7, 8.0, '9' , -1002) +sql insert into tb3 values ("2021-05-09 10:12:17",10,11.0, NULL, -2002) +sql insert into tb3 values ("2021-05-09 10:12:18",12,NULL, NULL, -3002) + +sql insert into tb4 values ("2021-05-09 10:12:19",13,14.0,'15' , -1003) +sql insert into tb4 values ("2021-05-10 10:12:20",16,17.0, NULL, -2003) +sql insert into tb4 values ("2021-05-11 10:12:21",18,NULL, NULL, -3003) + +sql insert into tb5 values ("2021-05-09 10:12:22",19, 20, '21', -1004) +sql insert into tb6 values ("2021-05-11 10:12:23",22, 23, NULL, -2004) +sql insert into tb7 values ("2021-05-10 10:12:24",24,NULL, '25', -3004) +sql insert into tb8 values ("2021-05-11 10:12:25",26,NULL, '27', -4004) + +sql insert into tb9 values ("2021-05-09 10:12:26",28, 29, '30', -1005) +sql insert into tba values ("2021-05-10 10:12:27",31, 32, NULL, -2005) +sql insert into tbb values ("2021-05-10 10:12:28",33,NULL, '35', -3005) +sql insert into tbc values ("2021-05-11 10:12:29",36, 37, NULL, -4005) +sql insert into tbd values ("2021-05-11 10:12:29",NULL,NULL,NULL,NULL ) + +run general/parser/last_cache_query.sim + +system sh/exec.sh -n dnode1 -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s start + +run general/parser/last_cache_query.sim + +system sh/exec.sh -n dnode1 -s stop -x SIGINT + + + diff --git a/tests/script/general/parser/last_cache_query.sim b/tests/script/general/parser/last_cache_query.sim new file mode 100644 index 0000000000..2acd000585 --- /dev/null +++ b/tests/script/general/parser/last_cache_query.sim @@ -0,0 +1,416 @@ + +sleep 100 +sql connect + +$db = testdb + +sql use $db + +print "test tb1" + +sql select last(ts) from tb1 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi + + +sql select last(f1) from tb1 +if $rows != 1 then + return -1 +endi +if $data00 != 6 then + print $data00 + return -1 +endi + +sql select last(*) from tb1 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 5.000000000 then + print $data02 + return -1 +endi +if $data03 != 3 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi + + +sql select last(tb1.*,ts,f4) from tb1 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 5.000000000 then + print $data02 + return -1 +endi +if $data03 != 3 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi +if $data05 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data06 != @70-01-01 07:59:57.000@ then + return -1 +endi + + + + +print "test tb2" + +sql select last(ts) from tb2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-11 10:11:15.000@ then + print $data00 + return -1 +endi + + +sql select last(f1) from tb2 +if $rows != 1 then + return -1 +endi +if $data00 != -6 then + print $data00 + return -1 +endi + +sql select last(*) from tb2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-11 10:11:15.000@ then + print $data00 + return -1 +endi +if $data01 != -6 then + return -1 +endi +if $data02 != -7.000000000 then + print $data02 + return -1 +endi +if $data03 != -8 then + return -1 +endi +if $data04 != @70-01-01 07:59:56.999@ then + if $data04 != @70-01-01 07:59:57.-01@ then + return -1 + endi +endi + + +sql select last(tb2.*,ts,f4) from tb2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-11 10:11:15.000@ then + print $data00 + return -1 +endi +if $data01 != -6 then + return -1 +endi +if $data02 != -7.000000000 then + print $data02 + return -1 +endi +if $data03 != -8 then + return -1 +endi +if $data04 != @70-01-01 07:59:56.999@ then + if $data04 != @70-01-01 07:59:57.-01@ then + return -1 + endi +endi +if $data05 != @21-05-11 10:11:15.000@ then + print $data00 + return -1 +endi +if $data06 != @70-01-01 07:59:56.999@ then + if $data04 != @70-01-01 07:59:57.-01@ then + return -1 + endi +endi + + + + + + + +print "test tbd" +sql select last(*) from tbd +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-11 10:12:29.000@ then + print $data00 + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data02 != NULL then + print $data02 + return -1 +endi +if $data03 != NULL then + return -1 +endi +if $data04 != NULL then + return -1 +endi + + + +print "test tbe" +sql select last(*) from tbe +if $rows != 0 then + return -1 +endi + + + + + +print "test stable" +sql select last(ts) from st2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi + + +sql select last(f1) from st2 +if $rows != 1 then + return -1 +endi +if $data00 != 6 then + print $data00 + return -1 +endi + +sql select last(*) from st2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 37.000000000 then + print $data02 + return -1 +endi +if $data03 != 27 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi + + +sql select last(st2.*,ts,f4) from st2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 37.000000000 then + print $data02 + return -1 +endi +if $data03 != 27 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi +if $data05 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data06 != @70-01-01 07:59:57.000@ then + return -1 +endi + + +sql select last(*) from st2 group by id +if $rows != 5 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 5.000000000 then + print $data02 + return -1 +endi +if $data03 != 21 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data10 != @21-05-11 10:12:23.000@ then + return -1 +endi +if $data11 != 22 then + return -1 +endi +if $data12 != 23.000000000 then + print $data02 + return -1 +endi +if $data13 != -8 then + return -1 +endi +if $data14 != @70-01-01 07:59:57.996@ then +if $data14 != @70-01-01 07:59:58.-04@ then + print $data14 + return -1 +endi +endi +if $data15 != 2 then + return -1 +endi +if $data20 != @21-05-10 10:12:24.000@ then + return -1 +endi +if $data21 != 24 then + return -1 +endi +if $data22 != 8.000000000 then + print $data02 + return -1 +endi +if $data23 != 25 then + return -1 +endi +if $data24 != @70-01-01 07:59:56.996@ then +if $data24 != @70-01-01 07:59:57.-04@ then + return -1 +endi +endi +if $data25 != 3 then + return -1 +endi +if $data30 != @21-05-11 10:12:25.000@ then + return -1 +endi +if $data31 != 26 then + return -1 +endi +if $data32 != 17.000000000 then + print $data02 + return -1 +endi +if $data33 != 27 then + return -1 +endi +if $data34 != @70-01-01 07:59:55.996@ then +if $data34 != @70-01-01 07:59:56.-04@ then + return -1 +endi +endi +if $data35 != 4 then + return -1 +endi +if $data40 != @21-05-11 10:12:29.000@ then + return -1 +endi +if $data41 != 36 then + return -1 +endi +if $data42 != 37.000000000 then + print $data02 + return -1 +endi +if $data43 != 35 then + return -1 +endi +if $data44 != @70-01-01 07:59:55.995@ then +if $data44 != @70-01-01 07:59:56.-05@ then + return -1 +endi +endi +if $data45 != 5 then + return -1 +endi + + +print "test tbn" +sql create table tbn (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) +sql insert into tbn values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000) +sql insert into tbn values ("2021-05-10 10:10:11", 4, 5.0, NULL, -2000) +sql insert into tbn values ("2021-05-12 10:10:12", 6,NULL, NULL, -3000) +sql insert into tbn values ("2021-05-13 10:10:12", NULL,NULL, NULL,NULL) + +sql select last(*) from tbn; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-13 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 5.000000000 then + print $data02 + return -1 +endi +if $data03 != 3 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi + diff --git a/tests/script/general/parser/nchar.sim b/tests/script/general/parser/nchar.sim index 786cee651b..84719efcab 100644 --- a/tests/script/general/parser/nchar.sim +++ b/tests/script/general/parser/nchar.sim @@ -1,6 +1,5 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/general/parser/nestquery.sim b/tests/script/general/parser/nestquery.sim new file mode 100644 index 0000000000..9e2736833f --- /dev/null +++ b/tests/script/general/parser/nestquery.sim @@ -0,0 +1,207 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/exec.sh -n dnode1 -s start + +sleep 100 +sql connect + +print ======================== dnode1 start + +$dbPrefix = nest_query +$tbPrefix = nest_tb +$mtPrefix = nest_mt +$tbNum = 10 +$rowNum = 10000 +$totalNum = $tbNum * $rowNum + +print =============== nestquery.sim + +$i = 0 +$db = $dbPrefix . $i +$mt = $mtPrefix . $i + +sql drop database if exists $db +sql create database if not exists $db + +sql use $db +sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int) + +$half = $tbNum / 2 + +$i = 0 +while $i < $half + $tb = $tbPrefix . $i + + $nextSuffix = $i + $half + $tb1 = $tbPrefix . $nextSuffix + + sql create table $tb using $mt tags( $i ) + sql create table $tb1 using $mt tags( $nextSuffix ) + + $x = 0 + while $x < $rowNum + $y = $x * 60000 + $ms = 1600099200000 + $y + $c = $x / 100 + $c = $c * 100 + $c = $x - $c + $binary = 'binary . $c + $binary = $binary . ' + $nchar = 'nchar . $c + $nchar = $nchar . ' + sql insert into $tb values ($ms , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) $tb1 values ($ms , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) + $x = $x + 1 + endw + + $i = $i + 1 +endw + +sleep 100 + +$i = 1 +$tb = $tbPrefix . $i + +print ==============> simple nest query test +sql select count(*) from (select count(*) from nest_mt0) +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +sql select count(*) from (select count(*) from nest_mt0 group by tbname) +if $rows != 1 then + return -1 +endi + +if $data00 != 10 then + return -1 +endi + +sql select count(*) from (select count(*) from nest_mt0 interval(10h) group by tbname) +if $rows != 1 then + return -1 +endi + +if $data00 != 170 then + return -1 +endi + +sql select sum(a) from (select count(*) a from nest_mt0 interval(10h) group by tbname) +if $rows != 1 then + return -1 +endi + +if $data00 != 100000 then + return -1 +endi + +print =================> alias name test +sql select ts from (select count(*) a from nest_tb0 interval(1h)) +if $rows != 167 then + return -1 +endi + +if $data00 != @20-09-15 00:00:00.000@ then + return -1 +endi + +sql select count(a) from (select count(*) a from nest_tb0 interval(1h)) +if $rows != 1 then + return -1 +endi + +if $data00 != 167 then + return -1 +endi + +print ================>master query + filter +sql select t.* from (select count(*) a from nest_tb0 interval(10h)) t where t.a <= 520; +if $rows != 2 then + return -1 +endi + + +print ===================> nest query interval + + + +print ===================> complex query + + + +print ===================> group by + having + + + + + +print =========================> nest query join +sql select a.ts,a.k,b.ts from (select count(*) k from nest_tb0 interval(30a)) a, (select count(*) f from nest_tb1 interval(30a)) b where a.ts = b.ts ; +if $rows != 10000 then + return -1 +endi + +if $data00 != @20-09-15 00:00:00.000@ then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != @20-09-15 00:00:00.000@ then + return -1 +endi + +if $data10 != @20-09-15 00:01:00.000@ then + return -1 +endi + +if $data11 != 1 then + return -1 +endi + +if $data12 != @20-09-15 00:01:00.000@ then + return -1 +endi + +sql select sum(a.k), sum(b.f) from (select count(*) k from nest_tb0 interval(30a)) a, (select count(*) f from nest_tb1 interval(30a)) b where a.ts = b.ts ; +if $rows != 1 then + return -1 +endi + +if $data00 != 10000 then + return -1 +endi + +if $data01 != 10000 then + return -1 +endi + +sql select a.ts,a.k,b.ts,c.ts,c.ts,c.x from (select count(*) k from nest_tb0 interval(30a)) a, (select count(*) f from nest_tb1 interval(30a)) b, (select count(*) x from nest_tb2 interval(30a)) c where a.ts = b.ts and a.ts = c.ts +if $rows != 10000 then + return -1 +endi + +if $data00 != @20-09-15 00:00:00.000@ then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != @20-09-15 00:00:00.000@ then + return -1 +endi + +if $data03 != @20-09-15 00:00:00.000@ then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/select_with_tags.sim b/tests/script/general/parser/select_with_tags.sim index 7e5f31f759..9d5214bfaa 100644 --- a/tests/script/general/parser/select_with_tags.sim +++ b/tests/script/general/parser/select_with_tags.sim @@ -887,10 +887,16 @@ sql_error select tbname, t1 from select_tags_mt0 interval(1y); #valid sql: select first(c1), last(c2), count(*) from select_tags_mt0 group by tbname, t1; #valid sql: select first(c1), tbname, t1 from select_tags_mt0 group by t2; +print ==================================>TD-4231 +sql_error select t1,tbname from select_tags_mt0 where c1<0 +sql_error select t1,tbname from select_tags_mt0 where c1<0 and tbname in ('select_tags_tb12') + +sql select tbname from select_tags_mt0 where tbname in ('select_tags_tb12'); + sql_error select first(c1), last(c2), t1 from select_tags_mt0 group by tbname; sql_error select first(c1), last(c2), tbname, t2 from select_tags_mt0 group by tbname; sql_error select first(c1), count(*), t2, t1, tbname from select_tags_mt0 group by tbname; -# this sql is valid: select first(c1), t2 from select_tags_mt0 group by tbname; +#valid sql: select first(c1), t2 from select_tags_mt0 group by tbname; #sql select first(ts), tbname from select_tags_mt0 group by tbname; #sql select count(c1) from select_tags_mt0 where c1=99 group by tbname; diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index 8904642344..da91255e3f 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -54,11 +54,12 @@ run general/parser/timestamp.sim run general/parser/sliding.sim run general/parser/function.sim run general/parser/stableOp.sim - run general/parser/having.sim run general/parser/having_child.sim run general/parser/slimit_alter_tags.sim run general/parser/binary_escapeCharacter.sim +run general/parser/between_and.sim +run general/parser/last_cache.sim run general/parser/slimit_alter_tags.sim run general/parser/udf.sim run general/parser/udf_dll.sim diff --git a/tests/script/regressionSuite.sim b/tests/script/regressionSuite.sim index e5e2194e87..d5742cd98f 100644 --- a/tests/script/regressionSuite.sim +++ b/tests/script/regressionSuite.sim @@ -32,6 +32,7 @@ run general/compute/percentile.sim run general/compute/stddev.sim run general/compute/sum.sim run general/compute/top.sim +run general/compute/block_dist.sim run general/db/alter_option.sim run general/db/alter_tables_d2.sim run general/db/alter_tables_v1.sim diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim index 8e15c4f527..b9ee508f78 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim @@ -158,7 +158,7 @@ if $dnode4Vtatus != offline then sleep 2000 goto wait_dnode4_vgroup_offline endi -if $dnode3Vtatus != master then +if $dnode3Vtatus != unsynced then sleep 2000 goto wait_dnode4_vgroup_offline endi diff --git a/tests/script/unique/cluster/cache.sim b/tests/script/unique/cluster/cache.sim index 33aaea425c..740eddfb0d 100644 --- a/tests/script/unique/cluster/cache.sim +++ b/tests/script/unique/cluster/cache.sim @@ -41,7 +41,7 @@ sql create dnode $hostname2 sleep 10000 sql show log.tables; -if $rows != 5 then +if $rows > 6 then return -1 endi diff --git a/tests/script/unique/dnode/monitor.sim b/tests/script/unique/dnode/monitor.sim index b9b5e41889..0b41a4137c 100644 --- a/tests/script/unique/dnode/monitor.sim +++ b/tests/script/unique/dnode/monitor.sim @@ -56,7 +56,7 @@ print $data30 print $data40 print $data50 -if $rows != 5 then +if $rows > 6 then return -1 endi diff --git a/tests/script/unique/dnode/monitor_bug.sim b/tests/script/unique/dnode/monitor_bug.sim index efdf5e94b9..60c6524d9c 100644 --- a/tests/script/unique/dnode/monitor_bug.sim +++ b/tests/script/unique/dnode/monitor_bug.sim @@ -19,7 +19,7 @@ sleep 3000 sql show dnodes print dnode1 openVnodes $data2_1 -if $data2_1 != 1 then +if $data2_1 > 2 then return -1 endi @@ -41,7 +41,7 @@ print dnode2 openVnodes $data2_2 if $data2_1 != 0 then goto show2 endi -if $data2_2 != 1 then +if $data2_2 > 2 then goto show2 endi @@ -55,7 +55,7 @@ print $data30 print $data40 print $data50 -if $rows != 4 then +if $rows > 5 then return -1 endi