Merge pull request #3 from taosdata/develop

合并315内容至1380分支

CMakeLists.txt

@@ -16,6 +16,7 @@ SET(TD_GRANT FALSE)
 SET(TD_MQTT FALSE)
 SET(TD_TSDB_PLUGINS FALSE)
 SET(TD_STORAGE FALSE)
+SET(TD_TOPIC FALSE)
 
 SET(TD_COVER FALSE)
 SET(TD_MEM_CHECK FALSE)

Jenkinsfile

@@ -46,6 +46,7 @@ def pre_test(){
     git fetch origin +refs/pull/${CHANGE_ID}/merge
     git checkout -qf FETCH_HEAD
     git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD develop)|grep -v -E '.*md|//src//connector|Jenkinsfile' || exit 0
+    find ${WKC}/tests/pytest -name \'*\'.sql -exec rm -rf {} \\;
     cd ${WK}
     git reset --hard HEAD~10
     git checkout develop
@@ -115,7 +116,6 @@ pipeline {
               sh '''
               date
               cd ${WKC}/tests
-              find pytest -name '*'sql|xargs rm -rf
               ./test-all.sh p1
               date'''
             }
@@ -131,7 +131,6 @@ pipeline {
                 sh '''
                 date
                 cd ${WKC}/tests
-                find pytest -name '*'sql|xargs rm -rf
                 ./test-all.sh p2
                 date'''
             }

cmake/define.inc

@@ -25,6 +25,10 @@ IF (TD_STORAGE)
   ADD_DEFINITIONS(-D_STORAGE)
 ENDIF ()
 
+IF (TD_TOPIC)
+  ADD_DEFINITIONS(-D_TOPIC)
+ENDIF ()
+
 IF (TD_GODLL)
   ADD_DEFINITIONS(-D_TD_GO_DLL_)
 ENDIF ()

cmake/input.inc

@@ -9,6 +9,14 @@ ELSEIF (${ACCOUNT} MATCHES "false")
   MESSAGE(STATUS "Build without account plugins")
 ENDIF ()
 
+IF (${TOPIC} MATCHES "true")
+  SET(TD_TOPIC TRUE)
+  MESSAGE(STATUS "Build with topic plugins")
+ELSEIF (${TOPIC} MATCHES "false")
+  SET(TD_TOPIC FALSE)
+  MESSAGE(STATUS "Build without topic plugins")
+ENDIF ()
+
 IF (${COVER} MATCHES "true")
   SET(TD_COVER TRUE)
   MESSAGE(STATUS "Build with test coverage")

cmake/version.inc

@@ -4,7 +4,7 @@ PROJECT(TDengine)
 IF (DEFINED VERNUMBER)
   SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "2.0.16.0")
+  SET(TD_VER_NUMBER "2.0.17.0")
 ENDIF ()
 
 IF (DEFINED VERCOMPATIBLE)

documentation20/cn/03.architecture/docs.md

@@ -166,7 +166,7 @@ TDengine 分布式架构的逻辑结构图如下：
 
 **虚拟节点(vnode)**: 为更好的支持数据分片、负载均衡，防止数据过热或倾斜，数据节点被虚拟化成多个虚拟节点(vnode，图中V2, V3, V4等)。每个 vnode 都是一个相对独立的工作单元，是时序数据存储的基本单元，具有独立的运行线程、内存空间与持久化存储的路径。一个 vnode 包含一定数量的表（数据采集点）。当创建一张新表时，系统会检查是否需要创建新的 vnode。一个数据节点上能创建的 vnode 的数量取决于该数据节点所在物理节点的硬件资源。一个 vnode 只属于一个DB，但一个DB可以有多个 vnode。一个 vnode 除存储的时序数据外，也保存有所包含的表的schema、标签值等。一个虚拟节点由所属的数据节点的EP，以及所属的VGroup ID在系统内唯一标识，由管理节点创建并管理。
 
-**管理节点(mnode):** 一个虚拟的逻辑单元，负责所有数据节点运行状态的监控和维护，以及节点之间的负载均衡(图中M)。同时，管理节点也负责元数据(包括用户、数据库、表、静态标签等)的存储和管理，因此也称为 Meta Node。TDengine 集群中可配置多个(最多不超过5个) mnode，它们自动构建成为一个虚拟管理节点组(图中M0, M1, M2)。mnode 间采用 master/slave 的机制进行管理，而且采取强一致方式进行数据同步, 任何数据更新操作只能在 Master 上进行。mnode 集群的创建由系统自动完成，无需人工干预。每个dnode上至多有一个mnode，由所属的数据节点的EP来唯一标识。每个dnode通过内部消息交互自动获取整个集群中所有 mnode 所在的 dnode 的EP。
+**管理节点(mnode):** 一个虚拟的逻辑单元，负责所有数据节点运行状态的监控和维护，以及节点之间的负载均衡(图中M)。同时，管理节点也负责元数据(包括用户、数据库、表、静态标签等)的存储和管理，因此也称为 Meta Node。TDengine 集群中可配置多个(开源版最多不超过3个) mnode，它们自动构建成为一个虚拟管理节点组(图中M0, M1, M2)。mnode 间采用 master/slave 的机制进行管理，而且采取强一致方式进行数据同步, 任何数据更新操作只能在 Master 上进行。mnode 集群的创建由系统自动完成，无需人工干预。每个dnode上至多有一个mnode，由所属的数据节点的EP来唯一标识。每个dnode通过内部消息交互自动获取整个集群中所有 mnode 所在的 dnode 的EP。
 
 **虚拟节点组(VGroup):** 不同数据节点上的 vnode 可以组成一个虚拟节点组(vnode group)来保证系统的高可靠。虚拟节点组内采取master/slave的方式进行管理。写操作只能在 master vnode 上进行，系统采用异步复制的方式将数据同步到 slave vnode，这样确保了一份数据在多个物理节点上有拷贝。一个 vgroup 里虚拟节点个数就是数据的副本数。如果一个DB的副本数为N，系统必须有至少N个数据节点。副本数在创建DB时通过参数 replica 可以指定，缺省为1。使用 TDengine 的多副本特性，可以不再需要昂贵的磁盘阵列等存储设备，就可以获得同样的数据高可靠性。虚拟节点组由管理节点创建、管理，并且由管理节点分配一个系统唯一的ID，VGroup ID。如果两个虚拟节点的vnode group ID相同，说明他们属于同一个组，数据互为备份。虚拟节点组里虚拟节点的个数是可以动态改变的，容许只有一个，也就是没有数据复制。VGroup ID是永远不变的，即使一个虚拟节点组被删除，它的ID也不会被收回重复利用。
 

documentation20/cn/10.cluster/docs.md

@@ -227,5 +227,11 @@ SHOW MNODES;
 
 如果副本数为偶数，当一个 vnode group 里一半 vnode 不工作时，是无法从中选出 master 的。同理，一半 mnode 不工作时，是无法选出 mnode 的 master 的，因为存在“split brain”问题。为解决这个问题，TDengine 引入了 Arbitrator 的概念。Arbitrator 模拟一个 vnode 或 mnode 在工作，但只简单的负责网络连接，不处理任何数据插入或访问。只要包含 Arbitrator 在内，超过半数的 vnode 或 mnode 工作，那么该 vnode group 或 mnode 组就可以正常的提供数据插入或查询服务。比如对于副本数为 2 的情形，如果一个节点 A 离线，但另外一个节点 B 正常，而且能连接到 Arbitrator，那么节点 B 就能正常工作。
 
-TDengine提供一个执行程序，名为 tarbitrator，找任何一台Linux服务器运行它即可。请点击[安装包下载](https://www.taosdata.com/cn/all-downloads/)，在TDengine Arbitrator Linux一节中，选择适合的版本下载并安装。该程序对系统资源几乎没有要求，只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号，缺省是6042。配置每个taosd实例时，可以在配置文件taos.cfg里将参数arbitrator设置为Arbitrator的End Point。如果该参数配置了，当副本数为偶数时，系统将自动连接配置的Arbitrator。如果副本数为奇数，即使配置了Arbitrator，系统也不会去建立连接。
+总之，在目前版本下，TDengine 建议在双副本环境要配置 Arbitrator，以提升系统的可用性。
+
+Arbitrator 的执行程序名为 tarbitrator。该程序对系统资源几乎没有要求，只需要保证有网络连接，找任何一台 Linux 服务器运行它即可。以下简要描述安装配置的步骤：
+1. 请点击 [安装包下载](https://www.taosdata.com/cn/all-downloads/)，在 TDengine Arbitrator Linux 一节中，选择合适的版本下载并安装。
+2. 该应用的命令行参数 `-p` 可以指定其对外服务的端口号，缺省是 6042。
+3. 修改每个 taosd 实例的配置文件，在 taos.cfg 里将参数 arbitrator 设置为 tarbitrator 程序所对应的 End Point。（如果该参数配置了，当副本数为偶数时，系统将自动连接配置的 Arbitrator。如果副本数为奇数，即使配置了 Arbitrator，系统也不会去建立连接。）
+4. 在配置文件中配置了的 Arbitrator，会出现在 `SHOW DNODES;` 指令的返回结果中，对应的 role 列的值会是“arb”。
 

documentation20/cn/11.administrator/docs.md

@@ -9,7 +9,7 @@
 每个 DB 可以创建固定数目的 vgroup，默认与 CPU 核数相同，可通过 maxVgroupsPerDb 配置；vgroup 中的每个副本会是一个 vnode；每个 vnode 会占用固定大小的内存（大小与数据库的配置参数 blocks 和 cache 有关)；每个 Table 会占用与标签总长度有关的内存；此外，系统会有一些固定的内存开销。因此，每个 DB 需要的系统内存可通过如下公式计算：
 
 ```
-Memory Size = maxVgroupsPerDb * (blocks * cache + 10Mb) + numOfTables * (tagSizePerTable + 0.5Kb)
+Memory Size = maxVgroupsPerDb * (blocks * cache + 10MB) + numOfTables * (tagSizePerTable + 0.5KB)
 ```
 
 示例：假设是 4 核机器，cache 是缺省大小 16M, blocks 是缺省值 6，假设有 10 万张表，标签总长度是 256 字节，则总的内存需求为：4 \* (16 \* 6 + 10) + 100000 \* (0.25 + 0.5) / 1000 = 499M。
@@ -437,7 +437,7 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下
 - 表的列名：不能包含特殊字符，不能超过 64 个字符
 - 数据库名、表名、列名，都不能以数字开头
 - 表的列数：不能超过 1024 列
-- 记录的最大长度：包括时间戳8 byte，不能超过16KB
+- 记录的最大长度：包括时间戳 8 byte，不能超过 16KB（每个 BINARY/NCHAR 类型的列还会额外占用 2 个 byte 的存储位置）
 - 单条 SQL 语句默认最大字符串长度：65480 byte
 - 数据库副本数：不能超过 3
 - 用户名：不能超过 23 个 byte
@@ -453,39 +453,41 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下
 
 | 关键字列表 |             |              |            |           |
 | ---------- | ----------- | ------------ | ---------- | --------- |
-| ABLOCKS    | CONNECTION  | GROUP        | MINUS      | SLASH     |
+| ABLOCKS    | CONNECTIONS | GT           | MNODES     | SLIDING   |
-| ABORT      | CONNECTIONS | GT           | MNODES     | SLIDING   |
+| ABORT      | COPY        | ID           | MODULES    | SLIMIT    |
-| ACCOUNT    | COPY        | ID           | MODULES    | SMALLINT  |
+| ACCOUNT    | COUNT       | IF           | NCHAR      | SMALLINT  |
-| ACCOUNTS   | COUNT       | IF           | NCHAR      | SPREAD    |
+| ACCOUNTS   | CREATE      | IGNORE       | NE         | SPREAD    |
-| ADD        | CREATE      | IGNORE       | NE         | STABLE    |
+| ADD        | CTIME       | IMMEDIATE    | NONE       | STABLE    |
-| AFTER      | CTIME       | IMMEDIATE    | NONE       | STABLES   |
+| AFTER      | DATABASE    | IMPORT       | NOT        | STABLES   |
-| ALL        | DATABASE    | IMPORT       | NOT        | STAR      |
+| ALL        | DATABASES   | IN           | NOTNULL    | STAR      |
-| ALTER      | DATABASES   | IN           | NOTNULL    | STATEMENT |
+| ALTER      | DAYS        | INITIALLY    | NOW        | STATEMENT |
-| AND        | DAYS        | INITIALLY    | NOW        | STDDEV    |
+| AND        | DEFERRED    | INSERT       | OF         | STDDEV    |
-| AS         | DEFERRED    | INSERT       | OF         | STREAM    |
+| AS         | DELIMITERS  | INSTEAD      | OFFSET     | STREAM    |
-| ASC        | DELIMITERS  | INSTEAD      | OFFSET     | STREAMS   |
+| ASC        | DESC        | INTEGER      | OR         | STREAMS   |
-| ATTACH     | DESC        | INTEGER      | OR         | STRING    |
+| ATTACH     | DESCRIBE    | INTERVAL     | ORDER      | STRING    |
-| AVG        | DESCRIBE    | INTERVAL     | ORDER      | SUM       |
+| AVG        | DETACH      | INTO         | PASS       | SUM       |
-| BEFORE     | DETACH      | INTO         | PASS       | TABLE     |
+| BEFORE     | DIFF        | IP           | PERCENTILE | TABLE     |
-| BEGIN      | DIFF        | IP           | PERCENTILE | TABLES    |
+| BEGIN      | DISTINCT    | IS           | PLUS       | TABLES    |
-| BETWEEN    | DISTINCT    | IS           | PLUS       | TAG       |
+| BETWEEN    | DIVIDE      | ISNULL       | PRAGMA     | TAG       |
-| BIGINT     | DIVIDE      | ISNULL       | PRAGMA     | TAGS      |
+| BIGINT     | DNODE       | JOIN         | PREV       | TAGS      |
-| BINARY     | DNODE       | JOIN         | PREV       | TBLOCKS   |
+| BINARY     | DNODES      | KEEP         | PRIVILEGE  | TBLOCKS   |
-| BITAND     | DNODES      | KEEP         | PRIVILEGE  | TBNAME    |
+| BITAND     | DOT         | KEY          | QUERIES    | TBNAME    |
-| BITNOT     | DOT         | KEY          | QUERIES    | TIMES     |
+| BITNOT     | DOUBLE      | KILL         | QUERY      | TIMES     |
-| BITOR      | DOUBLE      | KILL         | QUERY      | TIMESTAMP |
+| BITOR      | DROP        | LAST         | RAISE      | TIMESTAMP |
-| BOOL       | DROP        | LAST         | RAISE      | TINYINT   |
+| BOOL       | EACH        | LE           | REM        | TINYINT   |
-| BOTTOM     | EACH        | LE           | REM        | TOP       |
+| BOTTOM     | END         | LEASTSQUARES | REPLACE    | TOP       |
-| BY         | END         | LEASTSQUARES | REPLACE    | TRIGGER   |
+| BY         | EQ          | LIKE         | REPLICA    | TRIGGER   |
-| CACHE      | EQ          | LIKE         | REPLICA    | UMINUS    |
+| CACHE      | EXISTS      | LIMIT        | RESET      | UMINUS    |
-| CASCADE    | EXISTS      | LIMIT        | RESET      | UPLUS     |
+| CASCADE    | EXPLAIN     | LINEAR       | RESTRICT   | UPLUS     |
-| CHANGE     | EXPLAIN     | LINEAR       | RESTRICT   | USE       |
+| CHANGE     | FAIL        | LOCAL        | ROW        | USE       |
-| CLOG       | FAIL        | LOCAL        | ROW        | USER      |
+| CLOG       | FILL        | LP           | ROWS       | USER      |
-| CLUSTER    | FILL        | LP           | ROWS       | USERS     |
+| CLUSTER    | FIRST       | LSHIFT       | RP         | USERS     |
-| COLON      | FIRST       | LSHIFT       | RP         | USING     |
+| COLON      | FLOAT       | LT           | RSHIFT     | USING     |
-| COLUMN     | FLOAT       | LT           | RSHIFT     | VALUES    |
+| COLUMN     | FOR         | MATCH        | SCORES     | VALUES    |
-| COMMA      | FOR         | MATCH        | SCORES     | VARIABLE  |
+| COMMA      | FROM        | MAX          | SELECT     | VARIABLE  |
-| COMP       | FROM        | MAX          | SELECT     | VGROUPS   |
+| COMP       | GE          | METRIC       | SEMI       | VGROUPS   |
-| CONCAT     | GE          | METRIC       | SEMI       | VIEW      |
+| CONCAT     | GLOB        | METRICS      | SET        | VIEW      |
-| CONFIGS    | GLOB        | METRICS      | SET        | WAVG      |
+| CONFIGS    | GRANTS      | MIN          | SHOW       | WAVG      |
-| CONFLICT   | GRANTS      | MIN          | SHOW       | WHERE     |
+| CONFLICT   | GROUP       | MINUS        | SLASH      | WHERE     |
+| CONNECTION |             |              |            |           |
+

documentation20/cn/12.taos-sql/docs.md

@@ -4,6 +4,8 @@
 
 TAOS SQL 是用户对 TDengine 进行数据写入和查询的主要工具。TAOS SQL 为了便于用户快速上手，在一定程度上提供类似于标准 SQL 类似的风格和模式。严格意义上，TAOS SQL 并不是也不试图提供 SQL 标准的语法。此外，由于 TDengine 针对的时序性结构化数据不提供删除功能，因此在 TAO SQL 中不提供数据删除的相关功能。
 
+TAOS SQL 不支持关键字的缩写，例如 DESCRIBE 不能缩写为 DESC。 
+
 本章节 SQL 语法遵循如下约定：
 
 - < > 里的内容是用户需要输入的，但不要输入 <> 本身
@@ -113,7 +115,7 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
     ```mysql
     ALTER DATABASE db_name QUORUM 2;
     ```
-    QUORUM 参数是指数据写入成功所需要的确认数，取值范围 [1, 3]。对于异步复制，quorum 设为 1，具有 master 角色的虚拟节点自己确认即可。对于同步复制，需要至少大于等于 2。原则上，Quorum >= 1 并且 Quorum <= replica(副本数)，这个参数在启动一个同步模块实例时需要提供。
+    QUORUM 参数是指数据写入成功所需要的确认数，取值范围 [1, 2]。对于异步复制，quorum 设为 1，具有 master 角色的虚拟节点自己确认即可。对于同步复制，需要至少大于等于 2。原则上，Quorum >= 1 并且 Quorum <= replica(副本数)，这个参数在启动一个同步模块实例时需要提供。
 
     ```mysql
     ALTER DATABASE db_name BLOCKS 100;
@@ -146,7 +148,7 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
 
     2) 表名最大长度为 192；
 
-    3) 表的每行长度不能超过16k个字符;
+    3) 表的每行长度不能超过 16k 个字符;（注意：每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置）
 
     4) 子表名只能由字母、数字和下划线组成，且不能以数字开头
 
@@ -402,8 +404,8 @@ SELECT select_expr [, select_expr ...]
     FROM {tb_name_list}
     [WHERE where_condition]
     [INTERVAL (interval_val [, interval_offset])]
+    [SLIDING sliding_val]
     [FILL fill_val]
-    [SLIDING fill_val]
     [GROUP BY col_list]
     [ORDER BY col_list { DESC | ASC }]
     [SLIMIT limit_val [, SOFFSET offset_val]]
@@ -620,9 +622,10 @@ Query OK, 1 row(s) in set (0.001091s)
 ```
 
 - 可以使用 * 返回所有列，或指定列名。可以对数字列进行四则运算，可以给输出的列取列名
-- where语句可以使用各种逻辑判断来过滤数字值，或使用通配符来过滤字符串
+- WHERE 语句可以使用各种逻辑判断来过滤数字值，或使用通配符来过滤字符串
 - 输出结果缺省按首列时间戳升序排序，但可以指定按降序排序( _c0 指首列时间戳)。使用 ORDER BY 对其他字段进行排序为非法操作。
 - 参数 LIMIT 控制输出条数，OFFSET 指定从第几条开始输出。LIMIT/OFFSET 对结果集的执行顺序在 ORDER BY 之后。
+- 参数 SLIMIT 控制由 GROUP BY 指令划分的每个分组中的输出条数。
 - 通过”>>"输出结果可以导出到指定文件
 
 ### 支持的条件过滤操作
@@ -1162,17 +1165,20 @@ TDengine支持按时间段进行聚合，可以将表中数据按照时间段进
 SELECT function_list FROM tb_name
   [WHERE where_condition]
   INTERVAL (interval [, offset])
+  [SLIDING sliding]
   [FILL ({NONE | VALUE | PREV | NULL | LINEAR})]
 
 SELECT function_list FROM stb_name
   [WHERE where_condition]
   INTERVAL (interval [, offset])
+  [SLIDING sliding]
   [FILL ({ VALUE | PREV | NULL | LINEAR})]
   [GROUP BY tags]
 ```
 
 - 聚合时间段的长度由关键词INTERVAL指定，最短时间间隔10毫秒（10a），并且支持偏移（偏移必须小于间隔）。聚合查询中，能够同时执行的聚合和选择函数仅限于单个输出的函数：count、avg、sum 、stddev、leastsquares、percentile、min、max、first、last，不能使用具有多行输出结果的函数（例如：top、bottom、diff以及四则运算）。
 - WHERE语句可以指定查询的起止时间和其他过滤条件
+- SLIDING语句用于指定聚合时间段的前向增量
 - FILL语句指定某一时间区间数据缺失的情况下的填充模式。填充模式包括以下几种：
   * 不进行填充：NONE(默认填充模式)。
   * VALUE填充：固定值填充，此时需要指定填充的数值。例如：fill(value, 1.23)。
@@ -1184,6 +1190,8 @@ SELECT function_list FROM stb_name
   2. 在时间维度聚合中，返回的结果中时间序列严格单调递增。
   3. 如果查询对象是超级表，则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用group by语句，则返回的结果按照时间序列严格单调递增；如果查询中使用了group by语句分组，则返回结果中每个group内不按照时间序列严格单调递增。
 
+时间聚合也常被用于连续查询场景，可以参考文档 [连续查询(Continuous Query)](https://www.taosdata.com/cn/documentation/advanced-features#continuous-query)。
+
 **示例:** 智能电表的建表语句如下：
 
 ```mysql
@@ -1203,7 +1211,7 @@ SELECT AVG(current), MAX(current), LEASTSQUARES(current, start_val, step_val), P
 ## <a class="anchor" id="limitation"></a>TAOS SQL 边界限制
 
 - 数据库名最大长度为 32
-- 表名最大长度为192，每行数据最大长度16k个字符
+- 表名最大长度为 192，每行数据最大长度 16k 个字符（注意：数据行内每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置）
 - 列名最大长度为 64，最多允许 1024 列，最少需要 2 列，第一列必须是时间戳
 - 标签最多允许 128 个，可以 1 个，标签总长度不超过 16k 个字符
 - SQL 语句最大长度 65480 个字符，但可通过系统配置参数 maxSQLLength 修改，最长可配置为 1M
@@ -1222,3 +1230,4 @@ TAOS SQL支持表之间按主键时间戳来join两张表的列，暂不支持
 **is not null与不为空的表达式适用范围**
 
 is not null支持所有类型的列。不为空的表达式为 <>""，仅对非数值类型的列适用。
+

documentation20/cn/13.faq/docs.md

@@ -92,6 +92,8 @@ TDengine 目前尚不支持删除功能，未来根据用户需求可能会支
 
 从 2.0.8.0 开始，TDengine 支持更新已经写入数据的功能。使用更新功能需要在创建数据库时使用 UPDATE 1 参数，之后可以使用 INSERT INTO 命令更新已经写入的相同时间戳数据。UPDATE 参数不支持 ALTER DATABASE 命令修改。没有使用 UPDATE 1 参数创建的数据库，写入相同时间戳的数据不会修改之前的数据，也不会报错。
 
+另需注意，在 UPDATE 设置为 0 时，后发送的相同时间戳的数据会被直接丢弃，但并不会报错，而且仍然会被计入 affected rows （所以不能利用 INSERT 指令的返回信息进行时间戳查重）。这样设计的主要原因是，TDengine 把写入的数据看做一个数据流，无论时间戳是否出现冲突，TDengine 都认为产生数据的原始设备真实地产生了这样的数据。UPDATE 参数只是控制这样的流数据在进行持久化时要怎样处理——UPDATE 为 0 时，表示先写入的数据覆盖后写入的数据；而 UPDATE 为 1 时，表示后写入的数据覆盖先写入的数据。这种覆盖关系如何选择，取决于对数据的后续使用和统计中，希望以先还是后生成的数据为准。
+
 ## 10. 我怎么创建超过1024列的表？
 
 使用2.0及其以上版本，默认支持1024列；2.0之前的版本，TDengine最大允许创建250列的表。但是如果确实超过限值，建议按照数据特性，逻辑地将这个宽表分解成几个小表。

packaging/docker/Dockerfile

@@ -12,9 +12,13 @@ RUN tar -zxf ${pkgFile}
 WORKDIR /root/${dirName}/
 RUN /bin/bash install.sh -e no
 
+RUN apt-get clean && apt-get update && apt-get install -y locales
+RUN locale-gen en_US.UTF-8
 ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib"
-ENV LANG=C.UTF-8
+ENV LC_CTYPE=en_US.UTF-8
-ENV LC_ALL=C.UTF-8
+ENV LANG=en_US.UTF-8
+ENV LC_ALL=en_US.UTF-8
+
 EXPOSE 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042
 CMD ["taosd"]
 VOLUME [ "/var/lib/taos", "/var/log/taos","/etc/taos/" ]

packaging/docker/dockerManifest.sh

@@ -36,10 +36,10 @@ done
 echo "verNumber=${verNumber}"
 
 #docker manifest create -a tdengine/tdengine:${verNumber} tdengine/tdengine-amd64:${verNumber} tdengine/tdengine-aarch64:${verNumber} tdengine/tdengine-aarch32:${verNumber}
-docker manifest create -a tdengine/tdengine tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest
+docker manifest create -a tdengine/tdengine:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest
 
 docker login -u tdengine -p ${passWord}  #replace the docker registry username and password
 
-docker manifest push tdengine/tdengine
+docker manifest push tdengine/tdengine:latest
 
 # how set latest version ???

snap/snapcraft.yaml

@@ -1,6 +1,6 @@
 name: tdengine
 base: core18
-version: '2.0.16.0'
+version: '2.0.17.0'
 icon: snap/gui/t-dengine.svg
 summary: an open-source big data platform designed and optimized for IoT. 
 description: |
@@ -72,7 +72,7 @@ parts:
       - usr/bin/taosd
       - usr/bin/taos
       - usr/bin/taosdemo
-      - usr/lib/libtaos.so.2.0.16.0
+      - usr/lib/libtaos.so.2.0.17.0
       - usr/lib/libtaos.so.1
       - usr/lib/libtaos.so
 

src/client/inc/tscUtil.h

@@ -123,6 +123,7 @@ int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, i
 bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo);
 bool tscIsTWAQuery(SQueryInfo* pQueryInfo);
 bool tscIsSecondStageQuery(SQueryInfo* pQueryInfo);
+bool tscGroupbyColumn(SQueryInfo* pQueryInfo);
 
 bool tscNonOrderedProjectionQueryOnSTable(SQueryInfo *pQueryInfo, int32_t tableIndex);
 bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex);
@@ -133,6 +134,7 @@ bool tscIsProjectionQuery(SQueryInfo* pQueryInfo);
 bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex);
 bool tscQueryTags(SQueryInfo* pQueryInfo);
 bool tscMultiRoundQuery(SQueryInfo* pQueryInfo, int32_t tableIndex);
+bool tscQueryBlockInfo(SQueryInfo* pQueryInfo);
 
 SSqlExpr* tscAddFuncInSelectClause(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId,
                                        SColumnIndex* pIndex, SSchema* pColSchema, int16_t colType);
@@ -152,7 +154,6 @@ SInternalField* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_F
 SInternalField* tscFieldInfoGetInternalField(SFieldInfo* pFieldInfo, int32_t index);
 TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index);
 
-void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo);
 void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo);
 
 int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index);

src/client/inc/tsclient.h

@@ -198,9 +198,10 @@ typedef struct STableDataBlocks {
 typedef struct SQueryInfo {
   int16_t          command;       // the command may be different for each subclause, so keep it seperately.
   uint32_t         type;          // query/insert type
+  STimeWindow      window;        // the whole query time window
 
-  STimeWindow      window;        // query time window
+  SInterval        interval;      // tumble time window
-  SInterval        interval;
+  SSessionWindow   sessionWindow; // session time window
 
   SSqlGroupbyExpr  groupbyExpr;   // group by tags info
   SArray *         colList;       // SArray<SColumn*>
@@ -232,6 +233,7 @@ typedef struct SQueryInfo {
 typedef struct {
   int     command;
   uint8_t msgType;
+  char    reserve1[3];        // fix bus error on arm32
   bool    autoCreated;        // create table if it is not existed during retrieve table meta in mnode
 
   union {
@@ -244,8 +246,10 @@ typedef struct {
 
   char *       curSql;       // current sql, resume position of sql after parsing paused
   int8_t       parseFinished;
+  char    reserve2[3];        // fix bus error on arm32
 
   int16_t      numOfCols;
+  char    reserve3[2];        // fix bus error on arm32
   uint32_t     allocSize;
   char *       payload;
   int32_t      payloadLen;
@@ -255,7 +259,9 @@ typedef struct {
   int32_t      numOfParams;
 
   int8_t       dataSourceType;     // load data from file or not
+  char    reserve4[3];        // fix bus error on arm32
   int8_t       submitSchema;   // submit block is built with table schema
+  char    reserve5[3];        // fix bus error on arm32
   STagData     tagData;        // NOTE: pTagData->data is used as a variant length array
 
   SName      **pTableNameList; // all involved tableMeta list of current insert sql statement.
@@ -397,7 +403,6 @@ typedef struct SSqlStream {
 
 void tscSetStreamDestTable(SSqlStream* pStream, const char* dstTable);
 
-
 int  tscAcquireRpc(const char *key, const char *user, const char *secret,void **pRpcObj);
 void tscReleaseRpc(void *param);
 void tscInitMsgsFp();

src/client/src/tscLocalMerge.c

@@ -100,6 +100,10 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalMerger *pReducer, tOrderDescr
     } else if (functionId == TSDB_FUNC_APERCT) {
       pCtx->param[0].i64 = pExpr->param[0].i64;
       pCtx->param[0].nType  = pExpr->param[0].nType;
+    } else if (functionId == TSDB_FUNC_BLKINFO) {
+      pCtx->param[0].i64 = pExpr->param[0].i64;
+      pCtx->param[0].nType = pExpr->param[0].nType;
+      pCtx->numOfParams = 1;
     }
 
     pCtx->interBufBytes = pExpr->interBytes;
@@ -951,10 +955,10 @@ static void doFillResult(SSqlObj *pSql, SLocalMerger *pLocalMerge, bool doneOutp
   // todo extract function
   int64_t actualETime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey: pQueryInfo->window.skey;
 
-  tFilePage **pResPages = malloc(POINTER_BYTES * pQueryInfo->fieldsInfo.numOfOutput);
+  void** pResPages = malloc(POINTER_BYTES * pQueryInfo->fieldsInfo.numOfOutput);
   for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
     TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
-    pResPages[i] = calloc(1, sizeof(tFilePage) + pField->bytes * pLocalMerge->resColModel->capacity);
+    pResPages[i] = calloc(1, pField->bytes * pLocalMerge->resColModel->capacity);
   }
 
   while (1) {
@@ -966,7 +970,7 @@ static void doFillResult(SSqlObj *pSql, SLocalMerger *pLocalMerge, bool doneOutp
       if (pQueryInfo->limit.offset > 0) {
         for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
           TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
-          memmove(pResPages[i]->data, pResPages[i]->data + pField->bytes * pQueryInfo->limit.offset,
+          memmove(pResPages[i], ((char*)pResPages[i]) + pField->bytes * pQueryInfo->limit.offset,
                   (size_t)(newRows * pField->bytes));
         }
       }
@@ -1010,7 +1014,7 @@ static void doFillResult(SSqlObj *pSql, SLocalMerger *pLocalMerge, bool doneOutp
     int32_t offset = 0;
     for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
       TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
-      memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, (size_t)(pField->bytes * pRes->numOfRows));
+      memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i], (size_t)(pField->bytes * pRes->numOfRows));
       offset += pField->bytes;
     }
 

src/client/src/tscParseInsert.c

@@ -307,7 +307,8 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SStrToken *pToken, char *payload,
           return tscInvalidSQLErrMsg(msg, "illegal float data", pToken->z);
         }
 
-        *((float *)payload) = (float)dv;
+//        *((float *)payload) = (float)dv;
+        SET_FLOAT_VAL(payload, dv);
       }
       break;
 

src/client/src/tscServer.c

@@ -497,8 +497,6 @@ int tscProcessSql(SSqlObj *pSql) {
       return pSql->res.code;
     }
   } else if (pCmd->command >= TSDB_SQL_LOCAL) {
-    //pSql->epSet = tscMgmtEpSet;
-//  } else {  // local handler
     return (*tscProcessMsgRsp[pCmd->command])(pSql);
   }
   
@@ -645,7 +643,6 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
     }
 
     pSql->epSet.inUse = rand()%pSql->epSet.numOfEps;
-
     pQueryMsg->head.vgId = htonl(vgId);
 
     STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg;
@@ -660,8 +657,6 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
     int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables);
     assert(index >= 0 && index < numOfVgroups);
 
-    tscDebug("%p query on stable, vgIndex:%d, numOfVgroups:%d", pSql, index, numOfVgroups);
-
     SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index);
 
     // set the vgroup info 
@@ -671,6 +666,9 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
     int32_t numOfTables = (int32_t)taosArrayGetSize(pTableIdList->itemList);
     pQueryMsg->numOfTables = htonl(numOfTables);  // set the number of tables
 
+    tscDebug("%p query on stable, vgId:%d, numOfTables:%d, vgIndex:%d, numOfVgroups:%d", pSql,
+             pTableIdList->vgInfo.vgId, numOfTables, index, numOfVgroups);
+
     // serialize each table id info
     for(int32_t i = 0; i < numOfTables; ++i) {
       STableIdInfo* pItem = taosArrayGet(pTableIdList->itemList, i);
@@ -705,7 +703,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
 
   size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList);
-  if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) {
+  if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo) && !tscQueryBlockInfo(pQueryInfo)) {
     tscError("%p illegal value of numOfCols in query msg: %" PRIu64 ", table cols:%d", pSql, (uint64_t)numOfSrcCols,
         tscGetNumOfColumns(pTableMeta));
 
@@ -756,6 +754,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   pQueryMsg->vgroupLimit    = htobe64(pQueryInfo->vgroupLimit);
   pQueryMsg->sqlstrLen      = htonl(sqlLen);
   pQueryMsg->prevResultLen  = htonl(pQueryInfo->bufLen);
+  pQueryMsg->sw.gap         = htobe64(pQueryInfo->sessionWindow.gap);
+  pQueryMsg->sw.primaryColId = htonl(PRIMARYKEY_TIMESTAMP_COL_INDEX);
 
   size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo);
   pQueryMsg->numOfOutput = htons((int16_t)numOfOutput);  // this is the stage one output column number
@@ -835,13 +835,31 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     pSqlFuncExpr->colInfo.colIndex = htons(pExpr->colInfo.colIndex);
     pSqlFuncExpr->colInfo.flag     = htons(pExpr->colInfo.flag);
 
+    if (TSDB_COL_IS_UD_COL(pExpr->colInfo.flag)) {
+      pSqlFuncExpr->colType  = htons(pExpr->resType);
+      pSqlFuncExpr->colBytes = htons(pExpr->resBytes);
+    } else if (pExpr->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) {
+      SSchema *s = tGetTbnameColumnSchema();
+
+      pSqlFuncExpr->colType = htons(s->type);
+      pSqlFuncExpr->colBytes = htons(s->bytes);
+    } else if (pExpr->colInfo.colId == TSDB_BLOCK_DIST_COLUMN_INDEX) {
+      SSchema s = tGetBlockDistColumnSchema();
+
+      pSqlFuncExpr->colType = htons(s.type);
+      pSqlFuncExpr->colBytes = htons(s.bytes);
+    } else {
+      SSchema* s = tscGetColumnSchemaById(pTableMeta, pExpr->colInfo.colId);
+      pSqlFuncExpr->colType  = htons(s->type);
+      pSqlFuncExpr->colBytes = htons(s->bytes);
+    }
+
     pSqlFuncExpr->functionId  = htons(pExpr->functionId);
     pSqlFuncExpr->numOfParams = htons(pExpr->numOfParams);
     pSqlFuncExpr->resColId    = htons(pExpr->resColId);
     pMsg += sizeof(SSqlFuncMsg);
 
-    for (int32_t j = 0; j < pExpr->numOfParams; ++j) {
+    for (int32_t j = 0; j < pExpr->numOfParams; ++j) { // todo add log
-      // todo add log
       pSqlFuncExpr->arg[j].argType = htons((uint16_t)pExpr->param[j].nType);
       pSqlFuncExpr->arg[j].argBytes = htons(pExpr->param[j].nLen);
 
@@ -866,6 +884,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     for (int32_t i = 0; i < output; ++i) {
       SInternalField* pField = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, i);
       SSqlExpr *pExpr = pField->pSqlExpr;
+
+      // this should be switched to projection query
       if (pExpr != NULL) {
         // the queried table has been removed and a new table with the same name has already been created already
         // return error msg
@@ -879,27 +899,25 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
           return TSDB_CODE_TSC_INVALID_SQL;
         }
 
-        pSqlFuncExpr1->colInfo.colId    = htons(pExpr->colInfo.colId);
+        pSqlFuncExpr1->numOfParams = 0;  // no params for projection query
-        pSqlFuncExpr1->colInfo.colIndex = htons(pExpr->colInfo.colIndex);
+        pSqlFuncExpr1->functionId  = htons(TSDB_FUNC_PRJ);
-        pSqlFuncExpr1->colInfo.flag     = htons(pExpr->colInfo.flag);
+        pSqlFuncExpr1->colInfo.colId = htons(pExpr->resColId);
+        pSqlFuncExpr1->colInfo.flag = htons(TSDB_COL_NORMAL);
 
-        pSqlFuncExpr1->functionId  = htons(pExpr->functionId);
+        bool assign = false;
-        pSqlFuncExpr1->numOfParams = htons(pExpr->numOfParams);
+        for (int32_t f = 0; f < tscSqlExprNumOfExprs(pQueryInfo); ++f) {
+          SSqlExpr *pe = tscSqlExprGet(pQueryInfo, f);
+          if (pe == pExpr) {
+            pSqlFuncExpr1->colInfo.colIndex = htons(f);
+            pSqlFuncExpr1->colType = htons(pe->resType);
+            pSqlFuncExpr1->colBytes = htons(pe->resBytes);
+            assign = true;
+            break;
+          }
+        }
+
+        assert(assign);
         pMsg += sizeof(SSqlFuncMsg);
-
-        for (int32_t j = 0; j < pExpr->numOfParams; ++j) {
-          // todo add log
-          pSqlFuncExpr1->arg[j].argType = htons((uint16_t)pExpr->param[j].nType);
-          pSqlFuncExpr1->arg[j].argBytes = htons(pExpr->param[j].nLen);
-
-          if (pExpr->param[j].nType == TSDB_DATA_TYPE_BINARY) {
-            memcpy(pMsg, pExpr->param[j].pz, pExpr->param[j].nLen);
-            pMsg += pExpr->param[j].nLen;
-          } else {
-            pSqlFuncExpr1->arg[j].argValue.i64 = htobe64(pExpr->param[j].i64);
-          }
-        }
-
         pSqlFuncExpr1 = (SSqlFuncMsg *)pMsg;
       } else {
         assert(pField->pArithExprInfo != NULL);
@@ -1055,7 +1073,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   SSqlCmd *pCmd = &pSql->cmd;
   pCmd->payloadLen = sizeof(SCreateDbMsg);
-  pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_DB;
+  
+  pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_CM_CREATE_DB : TSDB_MSG_TYPE_CM_CREATE_TP;
 
   SCreateDbMsg *pCreateDbMsg = (SCreateDbMsg *)pCmd->payload;
 
@@ -1187,7 +1206,7 @@ int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
   pDropDbMsg->ignoreNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0;
 
-  pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DB;
+  pCmd->msgType = (pInfo->pMiscInfo->dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_CM_DROP_DB : TSDB_MSG_TYPE_CM_DROP_TP;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1514,9 +1533,11 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) {
 int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   SSqlCmd *pCmd = &pSql->cmd;
   pCmd->payloadLen = sizeof(SAlterDbMsg);
-  pCmd->msgType = TSDB_MSG_TYPE_CM_ALTER_DB;
+  pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_CM_ALTER_DB : TSDB_MSG_TYPE_CM_ALTER_TP;
 
   SAlterDbMsg *pAlterDbMsg = (SAlterDbMsg* )pCmd->payload;
+  pAlterDbMsg->dbType = -1;
+  
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
   tNameExtractFullName(&pTableMetaInfo->name, pAlterDbMsg->db);
 

src/client/src/tscSub.c

@@ -503,9 +503,19 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) {
   SSqlCmd *pCmd = &pSql->cmd;
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
-  if (taosArrayGetSize(pSub->progress) > 0) { // fix crash in single tabel subscription
+  if (taosArrayGetSize(pSub->progress) > 0) { // fix crash in single table subscription
-    pQueryInfo->window.skey = ((SSubscriptionProgress*)taosArrayGet(pSub->progress, 0))->key;
+
-    tscDebug("subscribe:%s set subscribe skey:%"PRId64, pSub->topic, pQueryInfo->window.skey);
+    size_t size = taosArrayGetSize(pSub->progress);
+    TSKEY s = INT64_MAX;
+    for(int32_t i = 0; i < size; ++i) {
+      TSKEY k = ((SSubscriptionProgress*)taosArrayGet(pSub->progress, i))->key;
+      if (s > k) {
+        s = k;
+      }
+    }
+
+    pQueryInfo->window.skey = s;
+    tscDebug("subscribe:%s set next round subscribe skey:%"PRId64, pSub->topic, pQueryInfo->window.skey);
   }
 
   if (pSub->pTimer == NULL) {

src/client/src/tscSubquery.c

@@ -74,14 +74,14 @@ static bool allSubqueryDone(SSqlObj *pParentSql) {
   SSubqueryState *subState = &pParentSql->subState;
 
   //lock in caller
-  
+  tscDebug("%p total subqueries: %d", pParentSql, subState->numOfSub);
   for (int i = 0; i < subState->numOfSub; i++) {
     if (0 == subState->states[i]) {
-      tscDebug("%p subquery:%p,%d is NOT finished, total:%d", pParentSql, pParentSql->pSubs[i], i, subState->numOfSub);
+      tscDebug("%p subquery:%p, index: %d NOT finished, abort query completion check", pParentSql, pParentSql->pSubs[i], i);
       done = false;
       break;
     } else {
-      tscDebug("%p subquery:%p,%d is finished, total:%d", pParentSql, pParentSql->pSubs[i], i, subState->numOfSub);
+      tscDebug("%p subquery:%p, index: %d finished", pParentSql, pParentSql->pSubs[i], i);
     }
   }
 
@@ -453,7 +453,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
     pSubQueryInfo->tsBuf = NULL;
   
     // free result for async object will also free sqlObj
-    assert(tscSqlExprNumOfExprs(pSubQueryInfo) == 1); // ts_comp query only requires one resutl columns
+    assert(tscSqlExprNumOfExprs(pSubQueryInfo) == 1); // ts_comp query only requires one result columns
     taos_free_result(pPrevSub);
   
     SSqlObj *pNew = createSubqueryObj(pSql, (int16_t) i, tscJoinQueryCallback, pSupporter, TSDB_SQL_SELECT, NULL);
@@ -507,6 +507,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
     SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, 0);
     int16_t funcId = pExpr->functionId;
 
+    // add the invisible timestamp column
     if ((pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) ||
         (funcId != TSDB_FUNC_TS && funcId != TSDB_FUNC_TS_DUMMY && funcId != TSDB_FUNC_PRJ)) {
 
@@ -847,6 +848,8 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
   SSqlRes* pRes = &pSql->res;
 
   SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
+
+  // todo, the type may not include TSDB_QUERY_TYPE_TAG_FILTER_QUERY
   assert(TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY));
 
   if (pParentSql->res.code != TSDB_CODE_SUCCESS) {
@@ -1059,7 +1062,6 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
       tscError("%p invalid ts comp file from vnode, abort subquery, file size:%d", pSql, numOfRows);
 
       pParentSql->res.code = TAOS_SYSTEM_ERROR(errno);
-
       if (quitAllSubquery(pSql, pParentSql, pSupporter)){
         return;
       }
@@ -1880,6 +1882,13 @@ void doAppendData(SInterResult* pInterResult, TAOS_ROW row, int32_t numOfCols, S
       }
     }
 
+    if (p && taosArrayGetSize(p) > 0) {
+      SResPair *l = taosArrayGetLast(p);
+      if (l->key == key && key == INT64_MIN) {
+        continue;
+      }
+    }
+
     //append a new column
     if (p == NULL) {
       SStddevInterResult t = {.colId = id, .pResult = taosArrayInit(10, sizeof(SResPair)),};
@@ -1941,7 +1950,11 @@ void tscFirstRoundRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) {
 
           // tag or group by column
           if (TSDB_COL_IS_TAG(pExpr->colInfo.flag) || pExpr->functionId == TSDB_FUNC_PRJ) {
+            if (row[i] == NULL) {
+              setNull(p + offset, pExpr->resType, pExpr->resBytes);
+            } else {
               memcpy(p + offset, row[i], length[i]);
+            }
             offset += pExpr->resBytes;
           }
         }
@@ -2639,6 +2652,11 @@ static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsuppo
     SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0);
 
     pQueryInfo->type |= TSDB_QUERY_TYPE_STABLE_SUBQUERY;
+
+    // clear the limit/offset info, since it should not be sent to vnode to be executed.
+    pQueryInfo->limit.limit = -1;
+    pQueryInfo->limit.offset = 0;
+
     assert(pQueryInfo->numOfTables == 1 && pNew->cmd.numOfClause == 1 && trsupport->subqueryIndex < pSql->subState.numOfSub);
     
     // launch subquery for each vnode, so the subquery index equals to the vgroupIndex.
@@ -3098,30 +3116,6 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) {
   }
 }
 
-static UNUSED_FUNC void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pField) {
-  SSqlRes *pRes = &pSql->res;
-  
-  if (pRes->tsrow[columnIndex] != NULL && pField->type == TSDB_DATA_TYPE_NCHAR) {
-    // convert unicode to native code in a temporary buffer extra one byte for terminated symbol
-    if (pRes->buffer[columnIndex] == NULL) {
-      pRes->buffer[columnIndex] = malloc(pField->bytes + TSDB_NCHAR_SIZE);
-    }
-    
-    /* string terminated char for binary data*/
-    memset(pRes->buffer[columnIndex], 0, pField->bytes + TSDB_NCHAR_SIZE);
-    
-    int32_t length = taosUcs4ToMbs(pRes->tsrow[columnIndex], pRes->length[columnIndex], pRes->buffer[columnIndex]);
-    if ( length >= 0 ) {
-      pRes->tsrow[columnIndex] = (unsigned char*)pRes->buffer[columnIndex];
-      pRes->length[columnIndex] = length;
-    } else {
-      tscError("%p charset:%s to %s. val:%s convert failed.", pSql, DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)pRes->tsrow[columnIndex]);
-      pRes->tsrow[columnIndex] = NULL;
-      pRes->length[columnIndex] = 0;
-    }
-  }
-}
-
 char *getArithmeticInputSrc(void *param, const char *name, int32_t colId) {
   SArithmeticSupport *pSupport = (SArithmeticSupport *) param;
 

src/client/src/tscUtil.c

@@ -97,6 +97,22 @@ bool tscQueryTags(SQueryInfo* pQueryInfo) {
   return true;
 }
 
+bool tscQueryBlockInfo(SQueryInfo* pQueryInfo) {
+  int32_t numOfCols = (int32_t) tscSqlExprNumOfExprs(pQueryInfo);
+
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
+    int32_t functId = pExpr->functionId;
+
+    // "select count(tbname)" query
+    if (functId == TSDB_FUNC_BLKINFO) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
 bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) {
   if (pQueryInfo == NULL) {
     return false;
@@ -223,6 +239,21 @@ bool tscIsSecondStageQuery(SQueryInfo* pQueryInfo) {
   return false;
 }
 
+bool tscGroupbyColumn(SQueryInfo* pQueryInfo) {
+  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+  int32_t         numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
+
+  SSqlGroupbyExpr* pGroupbyExpr = &pQueryInfo->groupbyExpr;
+  for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) {
+    SColIndex* pIndex = taosArrayGet(pGroupbyExpr->columnInfo, k);
+    if (!TSDB_COL_IS_TAG(pIndex->flag) && pIndex->colIndex < numOfCols) {  // group by normal columns
+      return true;
+    }
+  }
+
+  return false;
+}
+
 bool tscIsTWAQuery(SQueryInfo* pQueryInfo) {
   size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
   for (int32_t i = 0; i < numOfExprs; ++i) {
@@ -1726,6 +1757,11 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) {
   pQueryInfo->colList     = taosArrayInit(4, POINTER_BYTES);
   pQueryInfo->udColumnId  = TSDB_UD_COLUMN_INDEX;
   pQueryInfo->resColumnId = -1000;
+  pQueryInfo->limit.limit = -1;
+  pQueryInfo->limit.offset = 0;
+
+  pQueryInfo->slimit.limit = -1;
+  pQueryInfo->slimit.offset = 0;
 }
 
 int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) {

src/common/inc/tdataformat.h

@@ -283,12 +283,37 @@ typedef struct {
 #define keyCol(pCols) (&((pCols)->cols[0]))  // Key column
 #define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)]
 #define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx))
-#define dataColsTKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, 0))
+static FORCE_INLINE TKEY dataColsTKeyFirst(SDataCols *pCols) {
-#define dataColsKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, 0))
+  if (pCols->numOfRows) {
-#define dataColsTKeyLast(pCols) \
+    return dataColsTKeyAt(pCols, 0);
-  (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, (pCols)->numOfRows - 1))
+  } else {
-#define dataColsKeyLast(pCols) \
+    return TKEY_INVALID;
-  (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, (pCols)->numOfRows - 1))
+  }
+}
+
+static FORCE_INLINE TSKEY dataColsKeyFirst(SDataCols *pCols) {
+  if (pCols->numOfRows) {
+    return dataColsKeyAt(pCols, 0);
+  } else {
+    return TSDB_DATA_TIMESTAMP_NULL;
+  }
+}
+
+static FORCE_INLINE TKEY dataColsTKeyLast(SDataCols *pCols) {
+  if (pCols->numOfRows) {
+    return dataColsTKeyAt(pCols, pCols->numOfRows - 1);
+  } else {
+    return TKEY_INVALID;
+  }
+}
+
+static FORCE_INLINE TSKEY dataColsKeyLast(SDataCols *pCols) {
+  if (pCols->numOfRows) {
+    return dataColsKeyAt(pCols, pCols->numOfRows - 1);
+  } else {
+    return TSDB_DATA_TIMESTAMP_NULL;
+  }
+}
 
 SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
 void       tdResetDataCols(SDataCols *pCols);

src/common/inc/tglobal.h

@@ -95,6 +95,7 @@ extern int8_t  tsCompression;
 extern int8_t  tsWAL;
 extern int32_t tsFsyncPeriod;
 extern int32_t tsReplications;
+extern int16_t tsPartitons;
 extern int32_t tsQuorum;
 extern int8_t  tsUpdate;
 extern int8_t  tsCacheLastRow;

src/common/inc/tname.h

@@ -33,7 +33,7 @@ typedef struct SDataStatis {
 
 typedef struct SColumnInfoData {
   SColumnInfo info;
-  void* pData;    // the corresponding block data in memory
+  char* pData;    // the corresponding block data in memory
 } SColumnInfoData;
 
 typedef struct SResPair {

src/common/src/tglobal.c

@@ -126,8 +126,9 @@ int8_t  tsWAL           = TSDB_DEFAULT_WAL_LEVEL;
 int32_t tsFsyncPeriod   = TSDB_DEFAULT_FSYNC_PERIOD;
 int32_t tsReplications  = TSDB_DEFAULT_DB_REPLICA_OPTION;
 int32_t tsQuorum        = TSDB_DEFAULT_DB_QUORUM_OPTION;
+int16_t tsPartitons     = TSDB_DEFAULT_DB_PARTITON_OPTION;
 int8_t  tsUpdate        = TSDB_DEFAULT_DB_UPDATE_OPTION;
-int8_t  tsCacheLastRow  = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
+int8_t  tsCacheLastRow  = TSDB_DEFAULT_CACHE_LAST_ROW;
 int32_t tsMaxVgroupsPerDb  = 0;
 int32_t tsMinTablePerVnode = TSDB_TABLES_STEP;
 int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES;
@@ -853,6 +854,16 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "partitions";
+  cfg.ptr = &tsPartitons;
+  cfg.valType = TAOS_CFG_VTYPE_INT16;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = TSDB_MIN_DB_PARTITON_OPTION;
+  cfg.maxValue = TSDB_MAX_DB_PARTITON_OPTION;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   cfg.option = "quorum";
   cfg.ptr = &tsQuorum;
   cfg.valType = TAOS_CFG_VTYPE_INT32;

src/connector/jdbc/pom.xml

@@ -102,6 +102,7 @@
                         <include>**/*Test.java</include>
                     </includes>
                     <excludes>
+                        <exclude>**/DatetimeBefore1970Test.java</exclude>
                         <exclude>**/AppMemoryLeakTest.java</exclude>
                         <exclude>**/AuthenticationTest.java</exclude>
                         <exclude>**/TaosInfoMonitorTest.java</exclude>

src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDriver.java

@@ -1,76 +1,12 @@
 package com.taosdata.jdbc;
 
-import java.io.*;
 import java.sql.Driver;
 import java.sql.DriverPropertyInfo;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Properties;
 import java.util.StringTokenizer;
 
 public abstract class AbstractDriver implements Driver {
 
-    private static final String TAOS_CFG_FILENAME = "taos.cfg";
-
-    /**
-     * @param cfgDirPath
-     * @return return the config dir
-     **/
-    protected File loadConfigDir(String cfgDirPath) {
-        if (cfgDirPath == null)
-            return loadDefaultConfigDir();
-        File cfgDir = new File(cfgDirPath);
-        if (!cfgDir.exists())
-            return loadDefaultConfigDir();
-        return cfgDir;
-    }
-
-    /**
-     * @return search the default config dir, if the config dir is not exist will return null
-     */
-    protected File loadDefaultConfigDir() {
-        File cfgDir;
-        File cfgDir_linux = new File("/etc/taos");
-        cfgDir = cfgDir_linux.exists() ? cfgDir_linux : null;
-        File cfgDir_windows = new File("C:\\TDengine\\cfg");
-        cfgDir = (cfgDir == null && cfgDir_windows.exists()) ? cfgDir_windows : cfgDir;
-        return cfgDir;
-    }
-
-    protected List<String> loadConfigEndpoints(File cfgFile) {
-        List<String> endpoints = new ArrayList<>();
-        try (BufferedReader reader = new BufferedReader(new FileReader(cfgFile))) {
-            String line = null;
-            while ((line = reader.readLine()) != null) {
-                if (line.trim().startsWith("firstEp") || line.trim().startsWith("secondEp")) {
-                    endpoints.add(line.substring(line.indexOf('p') + 1).trim());
-                }
-                if (endpoints.size() > 1)
-                    break;
-            }
-        } catch (FileNotFoundException e) {
-            e.printStackTrace();
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-        return endpoints;
-    }
-
-    protected void loadTaosConfig(Properties info) {
-        if ((info.getProperty(TSDBDriver.PROPERTY_KEY_HOST) == null ||
-                info.getProperty(TSDBDriver.PROPERTY_KEY_HOST).isEmpty()) && (
-                info.getProperty(TSDBDriver.PROPERTY_KEY_PORT) == null ||
-                        info.getProperty(TSDBDriver.PROPERTY_KEY_PORT).isEmpty())) {
-            File cfgDir = loadConfigDir(info.getProperty(TSDBDriver.PROPERTY_KEY_CONFIG_DIR));
-            File cfgFile = cfgDir.listFiles((dir, name) -> TAOS_CFG_FILENAME.equalsIgnoreCase(name))[0];
-            List<String> endpoints = loadConfigEndpoints(cfgFile);
-            if (!endpoints.isEmpty()) {
-                info.setProperty(TSDBDriver.PROPERTY_KEY_HOST, endpoints.get(0).split(":")[0]);
-                info.setProperty(TSDBDriver.PROPERTY_KEY_PORT, endpoints.get(0).split(":")[1]);
-            }
-        }
-    }
-
     protected DriverPropertyInfo[] getPropertyInfo(Properties info) {
         DriverPropertyInfo hostProp = new DriverPropertyInfo(TSDBDriver.PROPERTY_KEY_HOST, info.getProperty(TSDBDriver.PROPERTY_KEY_HOST));
         hostProp.required = false;
@@ -156,6 +92,4 @@ public abstract class AbstractDriver implements Driver {
         return urlProps;
     }
 
-
-
 }

src/connector/jdbc/src/main/java/com/taosdata/jdbc/ColumnMetaData.java

@@ -52,4 +52,14 @@ public class ColumnMetaData {
     public void setColIndex(int colIndex) {
         this.colIndex = colIndex;
     }
+
+    @Override
+    public String toString() {
+        return "ColumnMetaData{" +
+                "colType=" + colType +
+                ", colName='" + colName + '\'' +
+                ", colSize=" + colSize +
+                ", colIndex=" + colIndex +
+                '}';
+    }
 }

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java

@@ -87,11 +87,10 @@ public class TSDBConnection extends AbstractConnection {
     }
 
     public void close() throws SQLException {
-        if (isClosed()) {
+        if (isClosed)
-            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+            return;
-        }
-        this.isClosed = true;
         this.connector.closeConnection();
+        this.isClosed = true;
     }
 
     public boolean isClosed() throws SQLException {

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java

@@ -112,8 +112,6 @@ public class TSDBDriver extends AbstractDriver {
         if ((props = parseURL(url, info)) == null) {
             return null;
         }
-        //load taos.cfg start
-        loadTaosConfig(info);
 
         try {
             TSDBJNIConnector.init((String) props.get(PROPERTY_KEY_CONFIG_DIR), (String) props.get(PROPERTY_KEY_LOCALE),

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java

@@ -1,4 +1,5 @@
-/***************************************************************************
+/**
+ * *************************************************************************
  * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
  *
  * This program is free software: you can use, redistribute, and/or modify
@@ -20,6 +21,9 @@ import java.sql.SQLException;
 import java.sql.SQLWarning;
 import java.util.List;
 
+/**
+ * JNI connector
+ * */
 public class TSDBJNIConnector {
     private static volatile Boolean isInitialized = false;
 

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java

@@ -20,18 +20,16 @@ import java.util.ArrayList;
 import java.util.List;
 
 public class TSDBResultSet extends AbstractResultSet implements ResultSet {
-    private TSDBJNIConnector jniConnector;
+    private final TSDBJNIConnector jniConnector;
-
     private final TSDBStatement statement;
-    private long resultSetPointer = 0L;
+    private final long resultSetPointer;
     private List<ColumnMetaData> columnMetaDataList = new ArrayList<>();
-
+    private final TSDBResultSetRowData rowData;
-    private TSDBResultSetRowData rowData;
+    private final TSDBResultSetBlockData blockData;
-    private TSDBResultSetBlockData blockData;
 
     private boolean batchFetch = false;
     private boolean lastWasNull = false;
-    private final int COLUMN_INDEX_START_VALUE = 1;
+    private boolean isClosed;
 
     public void setBatchFetch(boolean batchFetch) {
         this.batchFetch = batchFetch;
@@ -56,13 +54,13 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet {
 
         int code = this.jniConnector.getSchemaMetaData(this.resultSetPointer, this.columnMetaDataList);
         if (code == TSDBConstants.JNI_CONNECTION_NULL) {
-            throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL);
         }
         if (code == TSDBConstants.JNI_RESULT_SET_NULL) {
-            throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL));
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_RESULT_SET_NULL);
         }
         if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) {
-            throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0));
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_NUM_OF_FIELDS_0);
         }
         this.rowData = new TSDBResultSetRowData(this.columnMetaDataList.size());
         this.blockData = new TSDBResultSetBlockData(this.columnMetaDataList, this.columnMetaDataList.size());
@@ -78,16 +76,12 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet {
             this.blockData.reset();
 
             if (code == TSDBConstants.JNI_CONNECTION_NULL) {
-                throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL);
             } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) {
-                throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL));
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_RESULT_SET_NULL);
             } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) {
-                throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0));
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_NUM_OF_FIELDS_0);
-            } else if (code == TSDBConstants.JNI_FETCH_END) {
+            } else return code != TSDBConstants.JNI_FETCH_END;
-                return false;
-            }
-
-            return true;
         } else {
             if (rowData != null) {
                 this.rowData.clear();
@@ -95,11 +89,11 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet {
 
             int code = this.jniConnector.fetchRow(this.resultSetPointer, this.rowData);
             if (code == TSDBConstants.JNI_CONNECTION_NULL) {
-                throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL);
             } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) {
-                throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL));
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_RESULT_SET_NULL);
             } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) {
-                throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0));
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_NUM_OF_FIELDS_0);
             } else if (code == TSDBConstants.JNI_FETCH_END) {
                 return false;
             } else {
@@ -109,14 +103,17 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet {
     }
 
     public void close() throws SQLException {
+        if (isClosed)
+            return;
         if (this.jniConnector != null) {
             int code = this.jniConnector.freeResultSet(this.resultSetPointer);
             if (code == TSDBConstants.JNI_CONNECTION_NULL) {
-                throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL);
             } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) {
-                throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL));
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_RESULT_SET_NULL);
             }
         }
+        isClosed = true;
     }
 
     public boolean wasNull() throws SQLException {
@@ -415,8 +412,8 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet {
     }
 
     public boolean isClosed() throws SQLException {
-        //TODO: check if need release resources
+        if (isClosed)
-        boolean isClosed = true;
+            return true;
         if (jniConnector != null) {
             isClosed = jniConnector.isResultsetClosed();
         }
@@ -429,14 +426,12 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet {
     }
 
     private int getTrueColumnIndex(int columnIndex) throws SQLException {
-        if (columnIndex < this.COLUMN_INDEX_START_VALUE) {
+        if (columnIndex < 1)
-            throw new SQLException("Column Index out of range, " + columnIndex + " < " + this.COLUMN_INDEX_START_VALUE);
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE, "columnIndex(" + columnIndex + "): < 1");
-        }
 
         int numOfCols = this.columnMetaDataList.size();
-        if (columnIndex > numOfCols) {
+        if (columnIndex > numOfCols)
-            throw new SQLException("Column Index out of range, " + columnIndex + " > " + numOfCols);
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE, "columnIndex: " + columnIndex);
-        }
         return columnIndex - 1;
     }
 }

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java

@@ -73,12 +73,12 @@ public class TSDBStatement extends AbstractStatement {
     }
 
     public void close() throws SQLException {
-        if (!isClosed) {
+        if (isClosed)
-            if (this.resultSet != null)
+            return;
+        if (this.resultSet != null && !this.resultSet.isClosed())
             this.resultSet.close();
         isClosed = true;
     }
-    }
 
     public boolean execute(String sql) throws SQLException {
         // check if closed

src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBJNIConnectorTest.java

@@ -0,0 +1,89 @@
+package com.taosdata.jdbc;
+
+import org.junit.Test;
+
+import java.sql.SQLException;
+import java.sql.SQLWarning;
+import java.util.ArrayList;
+import java.util.List;
+
+public class TSDBJNIConnectorTest {
+
+    private static TSDBResultSetRowData rowData;
+
+    @Test
+    public void test() {
+        try {
+            // init
+            TSDBJNIConnector.init(null, null, null, null);
+            // connect
+            TSDBJNIConnector connector = new TSDBJNIConnector();
+            connector.connect("127.0.0.1", 6030, null, "root", "taosdata");
+            // executeQuery
+            long pSql = connector.executeQuery("show variables");
+            if (connector.isUpdateQuery(pSql)) {
+                connector.freeResultSet(pSql);
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_WITH_EXECUTEQUERY);
+            }
+            // get schema
+            List<ColumnMetaData> columnMetaDataList = new ArrayList<>();
+            int code = connector.getSchemaMetaData(pSql, columnMetaDataList);
+            if (code == TSDBConstants.JNI_CONNECTION_NULL) {
+                throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
+            }
+            if (code == TSDBConstants.JNI_RESULT_SET_NULL) {
+                throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL));
+            }
+            if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) {
+                throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0));
+            }
+            int columnSize = columnMetaDataList.size();
+            // print metadata
+            for (int i = 0; i < columnSize; i++) {
+                System.out.println(columnMetaDataList.get(i));
+            }
+            rowData = new TSDBResultSetRowData(columnSize);
+            // iterate resultSet
+            for (int i = 0; next(connector, pSql); i++) {
+                System.out.println("col[" + i + "] size: " + rowData.getColSize());
+                rowData.getData().stream().forEach(col -> System.out.print(col + "\t"));
+                System.out.println();
+            }
+            // close resultSet
+            code = connector.freeResultSet(pSql);
+            if (code == TSDBConstants.JNI_CONNECTION_NULL) {
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL);
+            } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) {
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_RESULT_SET_NULL);
+            }
+            // close statement
+
+            // close connection
+            connector.closeConnection();
+
+        } catch (SQLWarning throwables) {
+            throwables.printStackTrace();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+    private static boolean next(TSDBJNIConnector connector, long pSql) throws SQLException {
+        if (rowData != null)
+            rowData.clear();
+
+        int code = connector.fetchRow(pSql, rowData);
+        if (code == TSDBConstants.JNI_CONNECTION_NULL) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL);
+        } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_RESULT_SET_NULL);
+        } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_NUM_OF_FIELDS_0);
+        } else if (code == TSDBConstants.JNI_FETCH_END) {
+            return false;
+        } else {
+            return true;
+        }
+    }
+
+}

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DatetimeBefore1970Test.java

@@ -0,0 +1,68 @@
+package com.taosdata.jdbc.cases;
+
+import com.taosdata.jdbc.utils.TimestampUtil;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.sql.*;
+
+public class DatetimeBefore1970Test {
+
+    private static Connection conn;
+
+    @Test
+    public void test() {
+        try (Statement stmt = conn.createStatement()) {
+            stmt.executeUpdate("insert into weather(ts) values('1969-12-31 23:59:59.999')");
+            stmt.executeUpdate("insert into weather(ts) values('1970-01-01 00:00:00.000')");
+            stmt.executeUpdate("insert into weather(ts) values('1970-01-01 08:00:00.000')");
+            stmt.executeUpdate("insert into weather(ts) values('1970-01-01 07:59:59.999')");
+
+            ResultSet rs = stmt.executeQuery("select * from weather");
+            while (rs.next()) {
+                Timestamp ts = rs.getTimestamp("ts");
+                System.out.println("long: " + ts.getTime() + ", string: " + TimestampUtil.longToDatetime(ts.getTime()));
+            }
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+    public static void main(String[] args) {
+        System.out.println("timestamp: " + Long.MAX_VALUE + ", string: " + TimestampUtil.longToDatetime(Long.MAX_VALUE));
+        System.out.println("timestamp: " + Long.MIN_VALUE + ", string: " + TimestampUtil.longToDatetime(Long.MIN_VALUE));
+        System.out.println("timestamp: " + 0 + ", string: " + TimestampUtil.longToDatetime(0));
+        System.out.println("timestamp: " + -1 + ", string: " + TimestampUtil.longToDatetime(-1));
+        String datetime = "1970-01-01 00:00:00.000";
+        System.out.println("timestamp: " + TimestampUtil.datetimeToLong(datetime) + ", string: " + datetime);
+        datetime = "1969-12-31 23:59:59.999";
+        System.out.println("timestamp: " + TimestampUtil.datetimeToLong(datetime) + ", string: " + datetime);
+    }
+
+    @BeforeClass
+    public static void beforeClass() {
+        try {
+            Class.forName("com.taosdata.jdbc.TSDBDriver");
+            conn = DriverManager.getConnection("jdbc:TAOS://127.0.0.1:6030/?user=root&password=taosdata");
+            Statement stmt = conn.createStatement();
+            stmt.execute("drop database if exists test_timestamp");
+            stmt.execute("create database if not exists test_timestamp keep 36500");
+            stmt.execute("use test_timestamp");
+            stmt.execute("create table weather(ts timestamp,f1 float)");
+            stmt.close();
+        } catch (ClassNotFoundException | SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+    @AfterClass
+    public static void afterClass() {
+        try {
+            if (conn != null)
+                conn.close();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+}

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/FailOverTest.java

@@ -4,7 +4,6 @@ import org.junit.Test;
 
 import java.sql.*;
 import java.text.SimpleDateFormat;
-import java.util.Date;
 import java.util.concurrent.TimeUnit;
 
 public class FailOverTest {
@@ -18,13 +17,17 @@ public class FailOverTest {
 
         long end = System.currentTimeMillis() + 1000 * 60 * 5;
         while (System.currentTimeMillis() < end) {
-            try (Connection conn = DriverManager.getConnection(url)) {
+            try (Connection conn = DriverManager.getConnection(url); Statement stmt = conn.createStatement()) {
-                Statement stmt = conn.createStatement();
+                ResultSet rs = stmt.executeQuery("show dnodes");
-                ResultSet resultSet = stmt.executeQuery("select server_status()");
+                ResultSetMetaData meta = rs.getMetaData();
-                resultSet.next();
+                while (rs.next()) {
-                int status = resultSet.getInt("server_status()");
+                    for (int i = 1; i <= meta.getColumnCount(); i++) {
-                System.out.println(">>>>>>>>>" + sdf.format(new Date()) + " status : " + status);
+                        System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t");
-                stmt.close();
+                    }
+                    System.out.println();
+                }
+                System.out.println("=======================");
+                rs.close();
                 TimeUnit.SECONDS.sleep(5);
             } catch (SQLException | InterruptedException e) {
                 e.printStackTrace();

src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/TimestampUtil.java

@@ -4,7 +4,7 @@ import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 
-public class TimeStampUtil {
+public class TimestampUtil {
 
     private static final String datetimeFormat = "yyyy-MM-dd HH:mm:ss.SSS";
 

src/connector/python/linux/python2/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.6",
+    version="2.0.7",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",

src/connector/python/linux/python2/taos/cinterface.py

@@ -22,10 +22,10 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False):
 
     if num_of_rows > 0:
         return list(map(_timestamp_converter, ctypes.cast(
-            data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]))
+            data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]))
     else:
         return list(map(_timestamp_converter, ctypes.cast(
-            data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]))
+            data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]))
 
 
 def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False):
@@ -145,10 +145,10 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """
     if num_of_rows > 0:
         return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(
-            data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]]
+            data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]]
     else:
         return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(
-            data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]]
+            data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]]
 
 
 def _crow_bigint_unsigned_to_python(
@@ -162,13 +162,13 @@ def _crow_bigint_unsigned_to_python(
         return [
             None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast(
                 data, ctypes.POINTER(
-                    ctypes.c_ulong))[
+                    ctypes.c_uint64))[
                 :abs(num_of_rows)]]
     else:
         return [
             None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast(
                 data, ctypes.POINTER(
-                    ctypes.c_ulong))[
+                    ctypes.c_uint64))[
                 :abs(num_of_rows)]]
 
 
@@ -600,7 +600,7 @@ class CTaosInterface(object):
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_INT):
     #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[0]
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BIGINT):
-    #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[0]
+    #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_int64))[0]
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_FLOAT):
     #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[0]
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_DOUBLE):
@@ -608,7 +608,7 @@ class CTaosInterface(object):
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BINARY):
     #         return (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char))[0:byte]).rstrip('\x00')
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TIMESTAMP):
-    #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[0]
+    #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_int64))[0]
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_NCHAR):
     #         return (ctypes.cast(data,  ctypes.c_char_p).value).rstrip('\x00')
 

src/connector/python/linux/python3/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.5",
+    version="2.0.7",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",

src/connector/python/linux/python3/taos/cinterface.py

@@ -22,10 +22,10 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False):
 
     if num_of_rows > 0:
         return list(map(_timestamp_converter, ctypes.cast(
-            data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]))
+            data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]))
     else:
         return list(map(_timestamp_converter, ctypes.cast(
-            data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]))
+            data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]))
 
 
 def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False):
@@ -145,10 +145,10 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """
     if num_of_rows > 0:
         return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(
-            data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]]
+            data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]]
     else:
         return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(
-            data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]]
+            data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]]
 
 
 def _crow_bigint_unsigned_to_python(
@@ -162,13 +162,13 @@ def _crow_bigint_unsigned_to_python(
         return [
             None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast(
                 data, ctypes.POINTER(
-                    ctypes.c_ulong))[
+                    ctypes.c_uint64))[
                 :abs(num_of_rows)]]
     else:
         return [
             None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast(
                 data, ctypes.POINTER(
-                    ctypes.c_ulong))[
+                    ctypes.c_uint64))[
                 :abs(num_of_rows)]]
 
 
@@ -600,7 +600,7 @@ class CTaosInterface(object):
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_INT):
     #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[0]
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BIGINT):
-    #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[0]
+    #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_int64))[0]
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_FLOAT):
     #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[0]
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_DOUBLE):
@@ -608,7 +608,7 @@ class CTaosInterface(object):
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BINARY):
     #         return (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char))[0:byte]).rstrip('\x00')
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TIMESTAMP):
-    #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[0]
+    #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_int64))[0]
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_NCHAR):
     #         return (ctypes.cast(data,  ctypes.c_char_p).value).rstrip('\x00')
 

src/connector/python/linux/python3/taos/cursor.py

@@ -1,7 +1,6 @@
 from .cinterface import CTaosInterface
 from .error import *
 from .constants import FieldType
-import threading
 
 # querySeqNum = 0
 
@@ -38,7 +37,6 @@ class TDengineCursor(object):
         self._block_iter = 0
         self._affected_rows = 0
         self._logfile = ""
-        self._threadId = threading.get_ident()
 
         if connection is not None:
             self._connection = connection
@@ -105,12 +103,6 @@ class TDengineCursor(object):
     def execute(self, operation, params=None):
         """Prepare and execute a database operation (query or command).
         """
-        # if threading.get_ident() != self._threadId:
-        #     info ="Cursor execute:Thread ID not match,creater:"+str(self._threadId)+" caller:"+str(threading.get_ident())
-        #     raise OperationalError(info)
-        # print(info)
-        # return None
-
         if not operation:
             return None
 
@@ -280,12 +272,6 @@ class TDengineCursor(object):
     def _handle_result(self):
         """Handle the return result from query.
         """
-        # if threading.get_ident() != self._threadId:
-        #     info = "Cursor handleresult:Thread ID not match,creater:"+str(self._threadId)+" caller:"+str(threading.get_ident())
-        #     raise OperationalError(info)
-        # print(info)
-        # return None
-
         self._description = []
         for ele in self._fields:
             self._description.append(

src/connector/python/osx/python3/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.5",
+    version="2.0.7",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",

src/connector/python/osx/python3/taos/cinterface.py

@@ -22,10 +22,10 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False):
 
     if num_of_rows > 0:
         return list(map(_timestamp_converter, ctypes.cast(
-            data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]))
+            data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]))
     else:
         return list(map(_timestamp_converter, ctypes.cast(
-            data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]))
+            data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]))
 
 
 def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False):
@@ -145,10 +145,10 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """
     if num_of_rows > 0:
         return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(
-            data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]]
+            data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]]
     else:
         return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(
-            data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]]
+            data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]]
 
 
 def _crow_bigint_unsigned_to_python(
@@ -162,13 +162,13 @@ def _crow_bigint_unsigned_to_python(
         return [
             None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast(
                 data, ctypes.POINTER(
-                    ctypes.c_ulong))[
+                    ctypes.c_uint64))[
                 :abs(num_of_rows)]]
     else:
         return [
             None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast(
                 data, ctypes.POINTER(
-                    ctypes.c_ulong))[
+                    ctypes.c_uint64))[
                 :abs(num_of_rows)]]
 
 
@@ -600,7 +600,7 @@ class CTaosInterface(object):
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_INT):
     #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[0]
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BIGINT):
-    #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[0]
+    #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_int64))[0]
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_FLOAT):
     #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[0]
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_DOUBLE):
@@ -608,7 +608,7 @@ class CTaosInterface(object):
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BINARY):
     #         return (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char))[0:byte]).rstrip('\x00')
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TIMESTAMP):
-    #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[0]
+    #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_int64))[0]
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_NCHAR):
     #         return (ctypes.cast(data,  ctypes.c_char_p).value).rstrip('\x00')
 

src/connector/python/osx/python3/taos/cursor.py

@@ -1,7 +1,6 @@
 from .cinterface import CTaosInterface
 from .error import *
 from .constants import FieldType
-import threading
 
 # querySeqNum = 0
 
@@ -38,7 +37,6 @@ class TDengineCursor(object):
         self._block_iter = 0
         self._affected_rows = 0
         self._logfile = ""
-        self._threadId = threading.get_ident()
 
         if connection is not None:
             self._connection = connection
@@ -105,12 +103,6 @@ class TDengineCursor(object):
     def execute(self, operation, params=None):
         """Prepare and execute a database operation (query or command).
         """
-        # if threading.get_ident() != self._threadId:
-        #     info ="Cursor execute:Thread ID not match,creater:"+str(self._threadId)+" caller:"+str(threading.get_ident())
-        #     raise OperationalError(info)
-        # print(info)
-        # return None
-
         if not operation:
             return None
 
@@ -280,12 +272,6 @@ class TDengineCursor(object):
     def _handle_result(self):
         """Handle the return result from query.
         """
-        # if threading.get_ident() != self._threadId:
-        #     info = "Cursor handleresult:Thread ID not match,creater:"+str(self._threadId)+" caller:"+str(threading.get_ident())
-        #     raise OperationalError(info)
-        # print(info)
-        # return None
-
         self._description = []
         for ele in self._fields:
             self._description.append(

src/connector/python/windows/python2/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.4",
+    version="2.0.7",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",

src/connector/python/windows/python2/taos/cinterface.py

@@ -22,10 +22,10 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False):
 
     if num_of_rows > 0:
         return list(map(_timestamp_converter, ctypes.cast(
-            data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]))
+            data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]))
     else:
         return list(map(_timestamp_converter, ctypes.cast(
-            data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]))
+            data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]))
 
 
 def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False):
@@ -145,10 +145,10 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """
     if num_of_rows > 0:
         return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(
-            data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]]
+            data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]]
     else:
         return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(
-            data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]]
+            data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]]
 
 
 def _crow_bigint_unsigned_to_python(
@@ -162,13 +162,13 @@ def _crow_bigint_unsigned_to_python(
         return [
             None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast(
                 data, ctypes.POINTER(
-                    ctypes.c_ulong))[
+                    ctypes.c_uint64))[
                 :abs(num_of_rows)]]
     else:
         return [
             None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast(
                 data, ctypes.POINTER(
-                    ctypes.c_ulong))[
+                    ctypes.c_uint64))[
                 :abs(num_of_rows)]]
 
 
@@ -600,7 +600,7 @@ class CTaosInterface(object):
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_INT):
     #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[0]
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BIGINT):
-    #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[0]
+    #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_int64))[0]
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_FLOAT):
     #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[0]
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_DOUBLE):
@@ -608,7 +608,7 @@ class CTaosInterface(object):
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BINARY):
     #         return (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char))[0:byte]).rstrip('\x00')
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TIMESTAMP):
-    #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[0]
+    #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_int64))[0]
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_NCHAR):
     #         return (ctypes.cast(data,  ctypes.c_char_p).value).rstrip('\x00')
 

src/connector/python/windows/python2/taos/cursor.py

@@ -1,7 +1,6 @@
 from .cinterface import CTaosInterface
 from .error import *
 from .constants import FieldType
-import threading
 
 # querySeqNum = 0
 
@@ -38,7 +37,6 @@ class TDengineCursor(object):
         self._block_iter = 0
         self._affected_rows = 0
         self._logfile = ""
-        self._threadId = threading.get_ident()
 
         if connection is not None:
             self._connection = connection

src/connector/python/windows/python3/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.4",
+    version="2.0.7",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",

src/connector/python/windows/python3/taos/cinterface.py

@@ -22,10 +22,10 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False):
 
     if num_of_rows > 0:
         return list(map(_timestamp_converter, ctypes.cast(
-            data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]))
+            data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]))
     else:
         return list(map(_timestamp_converter, ctypes.cast(
-            data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]))
+            data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]))
 
 
 def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False):
@@ -145,10 +145,10 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """
     if num_of_rows > 0:
         return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(
-            data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]]
+            data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]]
     else:
         return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(
-            data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]]
+            data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]]
 
 
 def _crow_bigint_unsigned_to_python(
@@ -162,13 +162,13 @@ def _crow_bigint_unsigned_to_python(
         return [
             None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast(
                 data, ctypes.POINTER(
-                    ctypes.c_ulong))[
+                    ctypes.c_uint64))[
                 :abs(num_of_rows)]]
     else:
         return [
             None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast(
                 data, ctypes.POINTER(
-                    ctypes.c_ulong))[
+                    ctypes.c_uint64))[
                 :abs(num_of_rows)]]
 
 
@@ -600,7 +600,7 @@ class CTaosInterface(object):
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_INT):
     #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[0]
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BIGINT):
-    #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[0]
+    #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_int64))[0]
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_FLOAT):
     #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[0]
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_DOUBLE):
@@ -608,7 +608,7 @@ class CTaosInterface(object):
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BINARY):
     #         return (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char))[0:byte]).rstrip('\x00')
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TIMESTAMP):
-    #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[0]
+    #         return ctypes.cast(data,  ctypes.POINTER(ctypes.c_int64))[0]
     #     elif (dtype == CTaosInterface.TSDB_DATA_TYPE_NCHAR):
     #         return (ctypes.cast(data,  ctypes.c_char_p).value).rstrip('\x00')
 

src/connector/python/windows/python3/taos/cursor.py

@@ -1,7 +1,6 @@
 from .cinterface import CTaosInterface
 from .error import *
 from .constants import FieldType
-import threading
 
 # querySeqNum = 0
 
@@ -38,7 +37,6 @@ class TDengineCursor(object):
         self._block_iter = 0
         self._affected_rows = 0
         self._logfile = ""
-        self._threadId = threading.get_ident()
 
         if connection is not None:
             self._connection = connection

src/dnode/CMakeLists.txt

@@ -31,6 +31,10 @@ IF (TD_MQTT)
   TARGET_LINK_LIBRARIES(taosd mqtt)
 ENDIF ()
 
+IF (TD_TOPIC)
+  TARGET_LINK_LIBRARIES(taosd topic)
+ENDIF ()
+
 SET(PREPARE_ENV_CMD "prepare_env_cmd")
 SET(PREPARE_ENV_TARGET "prepare_env_target")
 ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD}

src/dnode/src/dnodeMain.c

@@ -189,6 +189,11 @@ static void dnodeCheckDataDirOpenned(char *dir) {
 }
 
 static int32_t dnodeInitStorage() {
+  if (tsDiskCfgNum == 1 && dnodeCreateDir(tsDataDir) < 0) {
+    dError("failed to create dir: %s, reason: %s", tsDataDir, strerror(errno));
+    return -1;
+  } 
+  
   if (tfsInit(tsDiskCfg, tsDiskCfgNum) < 0) {
     dError("failed to init TFS since %s", tstrerror(terrno));
     return -1;

src/dnode/src/dnodeShell.c

@@ -47,8 +47,11 @@ int32_t dnodeInitShell() {
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= dnodeDispatchToMWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE]  = dnodeDispatchToMWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB]   = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TP]   = dnodeDispatchToMWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB]     = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TP]     = dnodeDispatchToMWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB]    = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TP]    = dnodeDispatchToMWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= dnodeDispatchToMWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE]  = dnodeDispatchToMWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = dnodeDispatchToMWriteQueue;

src/dnode/src/dnodeVnodes.c

@@ -198,6 +198,14 @@ void dnodeCleanupVnodes() {
 static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
   if (pMsg->code != TSDB_CODE_SUCCESS) {
     dError("status rsp is received, error:%s", tstrerror(pMsg->code));
+    if (pMsg->code == TSDB_CODE_MND_DNODE_NOT_EXIST) {
+      char clusterId[TSDB_CLUSTER_ID_LEN];
+      dnodeGetClusterId(clusterId);
+      if (clusterId[0] != '\0') {
+	dError("exit zombie dropped dnode");
+	exit(EXIT_FAILURE);
+      }
+    }
     taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
     return;
   }

src/inc/taosdef.h

@@ -222,6 +222,9 @@ do { \
 #define TSDB_MQTT_TOPIC_LEN       64
 #define TSDB_MQTT_CLIENT_ID_LEN   32
 
+#define TSDB_DB_TYPE_DEFAULT      0
+#define TSDB_DB_TYPE_TOPIC        1
+
 #define TSDB_DEFAULT_PKT_SIZE     65480  //same as RPC_MAX_UDP_SIZE
 
 #define TSDB_PAYLOAD_SIZE         TSDB_DEFAULT_PKT_SIZE
@@ -306,6 +309,10 @@ do { \
 #define TSDB_MAX_DB_REPLICA_OPTION      3
 #define TSDB_DEFAULT_DB_REPLICA_OPTION  1
 
+#define TSDB_MIN_DB_PARTITON_OPTION     0
+#define TSDB_MAX_DB_PARTITON_OPTION     1000
+#define TSDB_DEFAULT_DB_PARTITON_OPTION 4
+
 #define TSDB_MIN_DB_QUORUM_OPTION       1
 #define TSDB_MAX_DB_QUORUM_OPTION       2
 #define TSDB_DEFAULT_DB_QUORUM_OPTION   1

src/inc/taoserror.h

@@ -185,6 +185,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_INVALID_DB_OPTION_DAYS    TAOS_DEF_ERROR_CODE(0, 0x0390)  //"Invalid database option: days out of range")
 #define TSDB_CODE_MND_INVALID_DB_OPTION_KEEP    TAOS_DEF_ERROR_CODE(0, 0x0391)  //"Invalid database option: keep >= keep1 >= keep0 >= days")
 
+#define TSDB_CODE_MND_INVALID_TOPIC             TAOS_DEF_ERROR_CODE(0, 0x0392)  //"Invalid topic name)
+#define TSDB_CODE_MND_INVALID_TOPIC_OPTION      TAOS_DEF_ERROR_CODE(0, 0x0393)  //"Invalid topic option)
+
 // dnode
 #define TSDB_CODE_DND_MSG_NOT_PROCESSED         TAOS_DEF_ERROR_CODE(0, 0x0400)  //"Message not processed")
 #define TSDB_CODE_DND_OUT_OF_MEMORY             TAOS_DEF_ERROR_CODE(0, 0x0401)  //"Dnode out of memory")
@@ -235,6 +238,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO     TAOS_DEF_ERROR_CODE(0, 0x0612)  //"Invalid information to create table")
 #define TSDB_CODE_TDB_NO_AVAIL_DISK             TAOS_DEF_ERROR_CODE(0, 0x0613)  //"No available disk")
 #define TSDB_CODE_TDB_MESSED_MSG                TAOS_DEF_ERROR_CODE(0, 0x0614)  //"TSDB messed message")
+#define TSDB_CODE_TDB_IVLD_TAG_VAL              TAOS_DEF_ERROR_CODE(0, 0x0615)  //"TSDB invalid tag value")
 
 // query
 #define TSDB_CODE_QRY_INVALID_QHANDLE           TAOS_DEF_ERROR_CODE(0, 0x0700)  //"Invalid handle")
@@ -249,7 +253,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_QRY_IN_EXEC                   TAOS_DEF_ERROR_CODE(0, 0x0709)  //"Multiple retrieval of this query")
 #define TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW       TAOS_DEF_ERROR_CODE(0, 0x070A)  //"Too many time window in query")
 #define TSDB_CODE_QRY_NOT_ENOUGH_BUFFER         TAOS_DEF_ERROR_CODE(0, 0x070B)  //"Query buffer limit has reached")
-#define TSDB_CODE_QRY_INCONSISTAN               TAOS_DEF_ERROR_CODE(0, 0x070C)  //"File inconsistance in replica")
+#define TSDB_CODE_QRY_INCONSISTAN               TAOS_DEF_ERROR_CODE(0, 0x070C)  //"File inconsistency in replica")
 
 
 // grant

src/inc/taosmsg.h

@@ -107,6 +107,12 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY13, "dummy13" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY14, "dummy14" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "nettest" )
 
+// message for topic
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_TP, "create-tp" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_TP, "drop-tp" )	  
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_USE_TP, "use-tp" )	 
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_ALTER_TP, "alter-tp" )
+
 #ifndef TAOS_MESSAGE_C
   TSDB_MSG_TYPE_MAX  // 105
 #endif
@@ -141,6 +147,7 @@ enum _mgmt_table {
   TSDB_MGMT_TABLE_VNODES,
   TSDB_MGMT_TABLE_STREAMTABLES,
   TSDB_MGMT_TABLE_CLUSTER,
+  TSDB_MGMT_TABLE_TP,
   TSDB_MGMT_TABLE_MAX,
 };
 
@@ -387,7 +394,7 @@ typedef struct SColIndex {
   int16_t  colId;      // column id
   int16_t  colIndex;   // column index in colList if it is a normal column or index in tagColList if a tag
   uint16_t flag;       // denote if it is a tag or a normal column
-  char     name[TSDB_COL_NAME_LEN];
+  char     name[TSDB_COL_NAME_LEN];  // TODO remove it
 } SColIndex;
 
 /* sql function msg, to describe the message to vnode about sql function
@@ -395,7 +402,10 @@ typedef struct SColIndex {
 typedef struct SSqlFuncMsg {
   int16_t functionId;
   int16_t numOfParams;
+
   int16_t resColId;      // result column id, id of the current output column
+  int16_t colType;
+  int16_t colBytes;
 
   SColIndex colInfo;
   struct ArgElem {
@@ -475,6 +485,7 @@ typedef struct {
   int16_t     orderColId;
   int16_t     numOfCols;        // the number of columns will be load from vnode
   SInterval   interval;
+  SSessionWindow sw;            // session window
   uint16_t    tagCondLen;       // tag length in current query
   uint32_t    tbnameCondLen;    // table name filter condition string length
   int16_t     numOfGroupCols;   // num of group by columns
@@ -555,7 +566,9 @@ typedef struct {
   int8_t   ignoreExist;
   int8_t   update;
   int8_t   cacheLastRow;
-  int8_t   reserve[8];
+  int8_t   dbType;
+  int16_t  partitions;
+  int8_t   reserve[5];
 } SCreateDbMsg, SAlterDbMsg;
 
 typedef struct {
@@ -616,6 +629,7 @@ typedef struct {
   int32_t  maxtablesPerVnode;
   int32_t  maxVgroupsPerDb;
   char     arbitrator[TSDB_EP_LEN];   // tsArbitrator
+  char     reserve[2];                // to solve arm32 bus error
   char     timezone[64];              // tsTimezone
   int64_t  checkTime;                 // 1970-01-01 00:00:00.000
   char     locale[TSDB_LOCALE_LEN];   // tsLocale
@@ -674,7 +688,8 @@ typedef struct {
   int8_t   cacheLastRow;
   int32_t  vgCfgVersion;
   int8_t   dbReplica;
-  int8_t   reserved[9];
+  int8_t   dbType;
+  int8_t   reserved[8];
 } SVnodeCfg;
 
 typedef struct {

src/inc/tp.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TP
+#define TDENGINE_TP
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t tpInit();
+void    tpCleanUp();
+void    tpUpdateTs(int32_t *seq, void *pMsg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/inc/tsdb.h

@@ -158,13 +158,18 @@ int32_t tsdbInsertData(STsdbRepo *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pR
 
 typedef void *TsdbQueryHandleT;  // Use void to hide implementation details
 
-// query condition to build vnode iterator
+#define BLOCK_LOAD_OFFSET_SEQ_ORDER   1
+#define BLOCK_LOAD_TABLE_SEQ_ORDER    2
+#define BLOCK_LOAD_TABLE_RR_ORDER     3
+
+// query condition to build multi-table data block iterator
 typedef struct STsdbQueryCond {
   STimeWindow  twindow;
   int32_t      order;             // desc|asc order to iterate the data block
   int32_t      numOfCols;
   SColumnInfo *colList;
   bool         loadExternalRows;  // load external rows or not
+  int32_t      type;              // data block load type:
 } STsdbQueryCond;
 
 typedef struct SMemRef {
@@ -181,17 +186,31 @@ typedef struct SDataBlockInfo {
   int32_t     tid;
 } SDataBlockInfo;
 
+typedef struct SFileBlockInfo {
+  int32_t numOfRows;
+} SFileBlockInfo;
+
 typedef struct {
   void *pTable;
   TSKEY lastKey;
 } STableKeyInfo;
 
 typedef struct {
-  size_t    numOfTables;
+  uint32_t  numOfTables;
   SArray *  pGroupList;
   SHashObj *map;  // speedup acquire the tableQueryInfo by table uid
 } STableGroupInfo;
 
+typedef struct {
+  uint16_t  rowSize;
+  uint16_t  numOfFiles;
+  uint32_t  numOfTables;
+  uint64_t  totalSize;
+  int32_t   firstSeekTimeUs;
+  uint32_t  numOfRowsInMemTable;
+  SArray   *dataBlockInfos;
+} STableBlockDist;
+
 /**
  * Get the data block iterator, starting from position according to the query condition
  *
@@ -252,16 +271,7 @@ int64_t tsdbGetNumOfRowsInMemTable(TsdbQueryHandleT* pHandle);
  * @param pQueryHandle
  * @return
  */
-bool tsdbNextDataBlock(TsdbQueryHandleT *pQueryHandle);
+bool tsdbNextDataBlock(TsdbQueryHandleT pQueryHandle);
-/**
- * move to next block if exists but not merge data in memtable 
- *
- * @param pQueryHandle
- * @return
- */
-bool tsdbNextDataBlockWithoutMerge(TsdbQueryHandleT *pQueryHandle);
-
-SArray* tsdbGetExternalRow(TsdbQueryHandleT *pHandle, SMemRef* pMemRef, int16_t type);
 
 /**
  * Get current data block information
@@ -306,7 +316,7 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo *tsdb, uint64_t uid, TSKEY key, const
                                  SColIndex *pColIndex, int32_t numOfCols);
 
 /**
- * destory the created table group list, which is generated by tag query
+ * destroy the created table group list, which is generated by tag query
  * @param pGroupList
  */
 void tsdbDestroyTableGroup(STableGroupInfo *pGroupList);
@@ -336,6 +346,12 @@ int32_t tsdbGetTableGroupFromIdList(STsdbRepo *tsdb, SArray *pTableIdList, STabl
  */
 void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle);
 
+void tsdbResetQueryHandle(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond);
+
+void tsdbResetQueryHandleForNewTable(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList);
+
+int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist* pTableBlockInfo);
+
 /**
  * get the statistics of repo usage
  * @param repo. point to the tsdbrepo

src/inc/ttokendef.h

@@ -62,175 +62,148 @@
 #define TK_BITNOT                          43
 #define TK_SHOW                            44
 #define TK_DATABASES                       45
-#define TK_MNODES                          46
+#define TK_TOPICS                          46
-#define TK_DNODES                          47
+#define TK_MNODES                          47
-#define TK_ACCOUNTS                        48
+#define TK_DNODES                          48
-#define TK_USERS                           49
+#define TK_ACCOUNTS                        49
-#define TK_MODULES                         50
+#define TK_USERS                           50
-#define TK_QUERIES                         51
+#define TK_MODULES                         51
-#define TK_CONNECTIONS                     52
+#define TK_QUERIES                         52
-#define TK_STREAMS                         53
+#define TK_CONNECTIONS                     53
-#define TK_VARIABLES                       54
+#define TK_STREAMS                         54
-#define TK_SCORES                          55
+#define TK_VARIABLES                       55
-#define TK_GRANTS                          56
+#define TK_SCORES                          56
-#define TK_VNODES                          57
+#define TK_GRANTS                          57
-#define TK_IPTOKEN                         58
+#define TK_VNODES                          58
-#define TK_DOT                             59
+#define TK_IPTOKEN                         59
-#define TK_CREATE                          60
+#define TK_DOT                             60
-#define TK_TABLE                           61
+#define TK_CREATE                          61
-#define TK_DATABASE                        62
+#define TK_TABLE                           62
-#define TK_TABLES                          63
+#define TK_DATABASE                        63
-#define TK_STABLES                         64
+#define TK_TABLES                          64
-#define TK_VGROUPS                         65
+#define TK_STABLES                         65
-#define TK_DROP                            66
+#define TK_VGROUPS                         66
-#define TK_STABLE                          67
+#define TK_DROP                            67
-#define TK_DNODE                           68
+#define TK_STABLE                          68
-#define TK_USER                            69
+#define TK_TOPIC                           69
-#define TK_ACCOUNT                         70
+#define TK_DNODE                           70
-#define TK_USE                             71
+#define TK_USER                            71
-#define TK_DESCRIBE                        72
+#define TK_ACCOUNT                         72
-#define TK_ALTER                           73
+#define TK_USE                             73
-#define TK_PASS                            74
+#define TK_DESCRIBE                        74
-#define TK_PRIVILEGE                       75
+#define TK_ALTER                           75
-#define TK_LOCAL                           76
+#define TK_PASS                            76
-#define TK_IF                              77
+#define TK_PRIVILEGE                       77
-#define TK_EXISTS                          78
+#define TK_LOCAL                           78
-#define TK_PPS                             79
+#define TK_IF                              79
-#define TK_TSERIES                         80
+#define TK_EXISTS                          80
-#define TK_DBS                             81
+#define TK_PPS                             81
-#define TK_STORAGE                         82
+#define TK_TSERIES                         82
-#define TK_QTIME                           83
+#define TK_DBS                             83
-#define TK_CONNS                           84
+#define TK_STORAGE                         84
-#define TK_STATE                           85
+#define TK_QTIME                           85
-#define TK_KEEP                            86
+#define TK_CONNS                           86
-#define TK_CACHE                           87
+#define TK_STATE                           87
-#define TK_REPLICA                         88
+#define TK_KEEP                            88
-#define TK_QUORUM                          89
+#define TK_CACHE                           89
-#define TK_DAYS                            90
+#define TK_REPLICA                         90
-#define TK_MINROWS                         91
+#define TK_QUORUM                          91
-#define TK_MAXROWS                         92
+#define TK_DAYS                            92
-#define TK_BLOCKS                          93
+#define TK_MINROWS                         93
-#define TK_CTIME                           94
+#define TK_MAXROWS                         94
-#define TK_WAL                             95
+#define TK_BLOCKS                          95
-#define TK_FSYNC                           96
+#define TK_CTIME                           96
-#define TK_COMP                            97
+#define TK_WAL                             97
-#define TK_PRECISION                       98
+#define TK_FSYNC                           98
-#define TK_UPDATE                          99
+#define TK_COMP                            99
-#define TK_CACHELAST                      100
+#define TK_PRECISION                      100
-#define TK_LP                             101
+#define TK_UPDATE                         101
-#define TK_RP                             102
+#define TK_CACHELAST                      102
-#define TK_UNSIGNED                       103
+#define TK_PARTITIONS                     103
-#define TK_TAGS                           104
+#define TK_LP                             104
-#define TK_USING                          105
+#define TK_RP                             105
-#define TK_COMMA                          106
+#define TK_UNSIGNED                       106
-#define TK_AS                             107
+#define TK_TAGS                           107
-#define TK_NULL                           108
+#define TK_USING                          108
-#define TK_SELECT                         109
+#define TK_COMMA                          109
-#define TK_UNION                          110
+#define TK_AS                             110
-#define TK_ALL                            111
+#define TK_NULL                           111
-#define TK_DISTINCT                       112
+#define TK_SELECT                         112
-#define TK_FROM                           113
+#define TK_UNION                          113
-#define TK_VARIABLE                       114
+#define TK_ALL                            114
-#define TK_INTERVAL                       115
+#define TK_DISTINCT                       115
-#define TK_FILL                           116
+#define TK_FROM                           116
-#define TK_SLIDING                        117
+#define TK_VARIABLE                       117
-#define TK_ORDER                          118
+#define TK_INTERVAL                       118
-#define TK_BY                             119
+#define TK_SESSION                        119
-#define TK_ASC                            120
+#define TK_FILL                           120
-#define TK_DESC                           121
+#define TK_SLIDING                        121
-#define TK_GROUP                          122
+#define TK_ORDER                          122
-#define TK_HAVING                         123
+#define TK_BY                             123
-#define TK_LIMIT                          124
+#define TK_ASC                            124
-#define TK_OFFSET                         125
+#define TK_DESC                           125
-#define TK_SLIMIT                         126
+#define TK_GROUP                          126
-#define TK_SOFFSET                        127
+#define TK_HAVING                         127
-#define TK_WHERE                          128
+#define TK_LIMIT                          128
-#define TK_NOW                            129
+#define TK_OFFSET                         129
-#define TK_RESET                          130
+#define TK_SLIMIT                         130
-#define TK_QUERY                          131
+#define TK_SOFFSET                        131
-#define TK_ADD                            132
+#define TK_WHERE                          132
-#define TK_COLUMN                         133
+#define TK_NOW                            133
-#define TK_TAG                            134
+#define TK_RESET                          134
-#define TK_CHANGE                         135
+#define TK_QUERY                          135
-#define TK_SET                            136
+#define TK_ADD                            136
-#define TK_KILL                           137
+#define TK_COLUMN                         137
-#define TK_CONNECTION                     138
+#define TK_TAG                            138
-#define TK_STREAM                         139
+#define TK_CHANGE                         139
-#define TK_COLON                          140
+#define TK_SET                            140
-#define TK_ABORT                          141
+#define TK_KILL                           141
-#define TK_AFTER                          142
+#define TK_CONNECTION                     142
-#define TK_ATTACH                         143
+#define TK_STREAM                         143
-#define TK_BEFORE                         144
+#define TK_COLON                          144
-#define TK_BEGIN                          145
+#define TK_ABORT                          145
-#define TK_CASCADE                        146
+#define TK_AFTER                          146
-#define TK_CLUSTER                        147
+#define TK_ATTACH                         147
-#define TK_CONFLICT                       148
+#define TK_BEFORE                         148
-#define TK_COPY                           149
+#define TK_BEGIN                          149
-#define TK_DEFERRED                       150
+#define TK_CASCADE                        150
-#define TK_DELIMITERS                     151
+#define TK_CLUSTER                        151
-#define TK_DETACH                         152
+#define TK_CONFLICT                       152
-#define TK_EACH                           153
+#define TK_COPY                           153
-#define TK_END                            154
+#define TK_DEFERRED                       154
-#define TK_EXPLAIN                        155
+#define TK_DELIMITERS                     155
-#define TK_FAIL                           156
+#define TK_DETACH                         156
-#define TK_FOR                            157
+#define TK_EACH                           157
-#define TK_IGNORE                         158
+#define TK_END                            158
-#define TK_IMMEDIATE                      159
+#define TK_EXPLAIN                        159
-#define TK_INITIALLY                      160
+#define TK_FAIL                           160
-#define TK_INSTEAD                        161
+#define TK_FOR                            161
-#define TK_MATCH                          162
+#define TK_IGNORE                         162
-#define TK_KEY                            163
+#define TK_IMMEDIATE                      163
-#define TK_OF                             164
+#define TK_INITIALLY                      164
-#define TK_RAISE                          165
+#define TK_INSTEAD                        165
-#define TK_REPLACE                        166
+#define TK_MATCH                          166
-#define TK_RESTRICT                       167
+#define TK_KEY                            167
-#define TK_ROW                            168
+#define TK_OF                             168
-#define TK_STATEMENT                      169
+#define TK_RAISE                          169
-#define TK_TRIGGER                        170
+#define TK_REPLACE                        170
-#define TK_VIEW                           171
+#define TK_RESTRICT                       171
-#define TK_COUNT                          172
+#define TK_ROW                            172
-#define TK_SUM                            173
+#define TK_STATEMENT                      173
-#define TK_AVG                            174
+#define TK_TRIGGER                        174
-#define TK_MIN                            175
+#define TK_VIEW                           175
-#define TK_MAX                            176
+#define TK_SEMI                           176
-#define TK_FIRST                          177
+#define TK_NONE                           177
-#define TK_LAST                           178
+#define TK_PREV                           178
-#define TK_TOP                            179
+#define TK_LINEAR                         179
-#define TK_BOTTOM                         180
+#define TK_IMPORT                         180
-#define TK_STDDEV                         181
+#define TK_METRIC                         181
-#define TK_PERCENTILE                     182
+#define TK_TBNAME                         182
-#define TK_APERCENTILE                    183
+#define TK_JOIN                           183
-#define TK_LEASTSQUARES                   184
+#define TK_METRICS                        184
-#define TK_HISTOGRAM                      185
+#define TK_INSERT                         185
-#define TK_DIFF                           186
+#define TK_INTO                           186
-#define TK_SPREAD                         187
+#define TK_VALUES                         187
-#define TK_TWA                            188
-#define TK_INTERP                         189
-#define TK_LAST_ROW                       190
-#define TK_RATE                           191
-#define TK_IRATE                          192
-#define TK_SUM_RATE                       193
-#define TK_SUM_IRATE                      194
-#define TK_AVG_RATE                       195
-#define TK_AVG_IRATE                      196
-#define TK_TBID                           197
-#define TK_SEMI                           198
-#define TK_NONE                           199
-#define TK_PREV                           200
-#define TK_LINEAR                         201
-#define TK_IMPORT                         202
-#define TK_METRIC                         203
-#define TK_TBNAME                         204
-#define TK_JOIN                           205
-#define TK_METRICS                        206
-#define TK_INSERT                         207
-#define TK_INTO                           208
-#define TK_VALUES                         209
-
-
-
-
-
 
 
 #define TK_SPACE                          300

src/kit/taosdemo/insert.json

@@ -11,6 +11,7 @@
     "confirm_parameter_prompt": "no",
     "insert_interval": 0,
     "num_of_records_per_req": 100,
+    "max_sql_len": 1024000,
 	"databases": [{
 		"dbinfo": {
 			"name": "db",
@@ -38,7 +39,9 @@
 			"auto_create_table": "no",
 			"data_source": "rand",
 			"insert_mode": "taosc",
-			"insert_rows": 100000,
+            "childtable_limit": 33,
+            "childtable_offset": 33,
+			"insert_rows": 1000,
 			"multi_thread_write_one_tbl": "no",
 			"number_of_tbl_in_one_sql": 0,
 			"rows_per_tbl": 100,

src/mnode/inc/mnodeDef.h

@@ -175,7 +175,9 @@ typedef struct {
   int8_t  quorum;
   int8_t  update;
   int8_t  cacheLastRow;
-  int8_t  reserved[10];
+  int8_t  dbType;
+  int16_t partitions;
+  int8_t  reserved[7];
 } SDbCfg;
 
 typedef struct SDbObj {

src/mnode/src/mnodeDb.c

@@ -22,6 +22,7 @@
 #include "tname.h"
 #include "tbn.h"
 #include "tdataformat.h"
+#include "tp.h"
 #include "mnode.h"
 #include "mnodeDef.h"
 #include "mnodeInt.h"
@@ -39,7 +40,7 @@
 
 #define VG_LIST_SIZE 8
 int64_t tsDbRid = -1;
-static void *  tsDbSdb = NULL;
+void *  tsDbSdb = NULL;
 static int32_t tsDbUpdateSize;
 
 static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg *pMsg);
@@ -48,8 +49,14 @@ static int32_t mnodeSetDbDropping(SDbObj *pDb);
 static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 static int32_t mnodeProcessCreateDbMsg(SMnodeMsg *pMsg);
-static int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg);
 static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg);
+int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg);
+
+#ifndef _TOPIC
+int32_t tpInit() { return 0; }
+void    tpCleanUp() {}
+void    tpUpdateTs(int32_t *seq, void *pMsg) {}
+#endif
 
 static void mnodeDestroyDb(SDbObj *pDb) {
   pthread_mutex_destroy(&pDb->mutex);
@@ -176,7 +183,7 @@ int32_t mnodeInitDbs() {
   mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_DB, mnodeCancelGetNextDb);
   
   mDebug("table:dbs table is created");
-  return 0;
+  return tpInit();
 }
 
 void *mnodeGetNextDb(void *pIter, SDbObj **pDb) {
@@ -332,6 +339,17 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) {
     return TSDB_CODE_MND_INVALID_DB_OPTION;
   }
 
+  if (pCfg->dbType < 0 || pCfg->dbType > 1) {
+    mError("invalid db option dbType:%d valid range: [%d, %d]", pCfg->dbType, 0, 1);
+    return TSDB_CODE_MND_INVALID_DB_OPTION;
+  }
+
+  if (pCfg->partitions < TSDB_MIN_DB_PARTITON_OPTION || pCfg->partitions > TSDB_MAX_DB_PARTITON_OPTION) {
+    mError("invalid db option partitions:%d valid range: [%d, %d]", pCfg->partitions, TSDB_MIN_DB_PARTITON_OPTION,
+           TSDB_MAX_DB_PARTITON_OPTION);
+    return TSDB_CODE_MND_INVALID_DB_OPTION;
+  }
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -354,6 +372,8 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) {
   if (pCfg->quorum < 0) pCfg->quorum = tsQuorum;
   if (pCfg->update < 0) pCfg->update = tsUpdate;
   if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = tsCacheLastRow;
+  if (pCfg->dbType < 0) pCfg->dbType = 0;
+  if (pCfg->partitions < 0) pCfg->partitions = tsPartitons;
 }
 
 static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) {
@@ -408,7 +428,9 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg *
     .replications        = pCreate->replications,
     .quorum              = pCreate->quorum,
     .update              = pCreate->update,
-    .cacheLastRow        = pCreate->cacheLastRow
+    .cacheLastRow        = pCreate->cacheLastRow,
+    .dbType              = pCreate->dbType,
+    .partitions          = pCreate->partitions
   };
 
   mnodeSetDefaultDbCfg(&pDb->cfg);
@@ -501,6 +523,7 @@ void mnodeRemoveVgroupFromDb(SVgObj *pVgroup) {
 }
 
 void mnodeCleanupDbs() {
+  tpCleanUp();
   sdbCloseTable(tsDbRid);
   tsDbSdb = NULL;
 }
@@ -660,7 +683,7 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
   return 0;
 }
 
-static char *mnodeGetDbStr(char *src) {
+char *mnodeGetDbStr(char *src) {
   char *pos = strstr(src, TS_PATH_DELIMITER);
   if (pos != NULL) ++pos;
 
@@ -852,6 +875,7 @@ static int32_t mnodeProcessCreateDbMsg(SMnodeMsg *pMsg) {
   pCreate->daysToKeep2     = htonl(pCreate->daysToKeep2);
   pCreate->commitTime      = htonl(pCreate->commitTime);
   pCreate->fsyncPeriod     = htonl(pCreate->fsyncPeriod);
+  pCreate->partitions      = htons(pCreate->partitions);
   pCreate->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock);
   pCreate->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock);
   
@@ -887,6 +911,8 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) {
   int8_t  precision      = pAlter->precision;
   int8_t  update         = pAlter->update;
   int8_t  cacheLastRow   = pAlter->cacheLastRow;
+  int8_t  dbType         = pAlter->dbType;
+  int16_t partitions     = htons(pAlter->partitions);
   
   terrno = TSDB_CODE_SUCCESS;
 
@@ -1004,6 +1030,16 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) {
     newCfg.cacheLastRow = cacheLastRow;
   }
 
+  if (dbType >= 0 && dbType != pDb->cfg.dbType) {
+    mDebug("db:%s, dbType:%d change to %d", pDb->name, pDb->cfg.dbType, dbType);
+    newCfg.dbType = dbType;
+  }
+
+  if (partitions >= 0 && partitions != pDb->cfg.partitions) {
+    mDebug("db:%s, partitions:%d change to %d", pDb->name, pDb->cfg.partitions, partitions);
+    newCfg.partitions = partitions;
+  }
+
   return newCfg;
 }
 
@@ -1031,6 +1067,8 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) {
 }
 
 static int32_t mnodeAlterDb(SDbObj *pDb, SAlterDbMsg *pAlter, void *pMsg) {
+  mDebug("db:%s, type:%d do alter operation", pDb->name, pDb->cfg.dbType);
+
   SDbCfg newCfg = mnodeGetAlterDbOption(pDb, pAlter);
   if (terrno != TSDB_CODE_SUCCESS) {
     return terrno;
@@ -1061,9 +1099,9 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SAlterDbMsg *pAlter, void *pMsg) {
   return code;
 }
 
-static int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg) {
+int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg) {
   SAlterDbMsg *pAlter = pMsg->rpcMsg.pCont;
-  mDebug("db:%s, alter db msg is received from thandle:%p", pAlter->db, pMsg->rpcMsg.handle);
+  mDebug("db:%s, alter db msg is received from thandle:%p, dbType:%d", pAlter->db, pMsg->rpcMsg.handle, pAlter->dbType);
 
   if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pAlter->db);
   if (pMsg->pDb == NULL) {

src/mnode/src/mnodeShow.c

@@ -109,6 +109,7 @@ static char *mnodeGetShowType(int32_t showType) {
     case TSDB_MGMT_TABLE_VNODES:  return "show vnodes";
     case TSDB_MGMT_TABLE_CLUSTER: return "show clusters";
     case TSDB_MGMT_TABLE_STREAMTABLES : return "show streamtables";
+    case TSDB_MGMT_TABLE_TP:      return "show topics";
     default:                      return "undefined";
   }
 }

src/mnode/src/mnodeVgroup.c

@@ -367,6 +367,11 @@ static int32_t mnodeAllocVgroupIdPool(SVgObj *pInputVgroup) {
     maxIdPoolSize = MAX(maxIdPoolSize, idPoolSize);
   }
 
+  // create one table each vnode
+  if (pDb->cfg.dbType == TSDB_DB_TYPE_TOPIC) {
+    maxIdPoolSize = 1;
+  }
+
   // new vgroup
   if (pInputVgroup->idPool == NULL) {
     pInputVgroup->idPool = taosInitIdPool(maxIdPoolSize);
@@ -379,6 +384,11 @@ static int32_t mnodeAllocVgroupIdPool(SVgObj *pInputVgroup) {
     }
   }
 
+  // create one table each vnode
+  if (pDb->cfg.dbType == TSDB_DB_TYPE_TOPIC) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   // realloc all vgroups in db
   int32_t newIdPoolSize;
   if (minIdPoolSize * 4 < tsTableIncStepPerVnode) {
@@ -449,6 +459,10 @@ int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSi
     maxVgroupsPerDb = MIN(maxVgroupsPerDb, TSDB_MAX_VNODES_PER_DB);
   }
 
+  if (pDb->cfg.dbType == TSDB_DB_TYPE_TOPIC) {
+    maxVgroupsPerDb = TSDB_MAX_DB_PARTITON_OPTION;
+  }
+
   int32_t code = TSDB_CODE_MND_NO_ENOUGH_DNODES;
   if (pDb->numOfVgroups < maxVgroupsPerDb) {
     mDebug("msg:%p, app:%p db:%s, try to create a new vgroup, numOfVgroups:%d maxVgroupsPerDb:%d", pMsg,
@@ -881,6 +895,7 @@ static SCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
   pCfg->update              = pDb->cfg.update;
   pCfg->cacheLastRow        = pDb->cfg.cacheLastRow;
   pCfg->dbReplica           = pDb->cfg.replications;
+  pCfg->dbType              = pDb->cfg.dbType;
   
   SVnodeDesc *pNodes = pVnode->nodes;
   for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) {

src/os/inc/osTime.h

@@ -72,6 +72,11 @@ typedef struct SInterval {
   int64_t offset;
 } SInterval;
 
+typedef struct SSessionWindow {
+  int64_t gap;             // gap between two session window(in microseconds)
+  int32_t primaryColId;    // primary timestamp column
+} SSessionWindow;
+
 int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision);
 int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision);
 int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision);

src/plugins/http/inc/httpJson.h

@@ -63,9 +63,11 @@ void httpJsonString(JsonBuf* buf, char* sVal, int32_t len);
 void httpJsonOriginString(JsonBuf* buf, char* sVal, int32_t len);
 void httpJsonStringForTransMean(JsonBuf* buf, char* SVal, int32_t maxLen);
 void httpJsonInt64(JsonBuf* buf, int64_t num);
+void httpJsonUInt64(JsonBuf* buf, uint64_t num);
 void httpJsonTimestamp(JsonBuf* buf, int64_t t, bool us);
 void httpJsonUtcTimestamp(JsonBuf* buf, int64_t t, bool us);
 void httpJsonInt(JsonBuf* buf, int32_t num);
+void httpJsonUInt(JsonBuf* buf, uint32_t num);
 void httpJsonFloat(JsonBuf* buf, float num);
 void httpJsonDouble(JsonBuf* buf, double num);
 void httpJsonNull(JsonBuf* buf);

src/plugins/http/src/httpJson.c

@@ -256,6 +256,12 @@ void httpJsonInt64(JsonBuf* buf, int64_t num) {
   buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%" PRId64, num);
 }
 
+void httpJsonUInt64(JsonBuf* buf, uint64_t num) {
+  httpJsonItemToken(buf);
+  httpJsonTestBuf(buf, MAX_NUM_STR_SZ);
+  buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%" PRIu64, num);
+}
+
 void httpJsonTimestamp(JsonBuf* buf, int64_t t, bool us) {
   char       ts[35] = {0};
   struct tm* ptm;
@@ -303,6 +309,12 @@ void httpJsonInt(JsonBuf* buf, int32_t num) {
   buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%d", num);
 }
 
+void httpJsonUInt(JsonBuf* buf, uint32_t num) {
+  httpJsonItemToken(buf);
+  httpJsonTestBuf(buf, MAX_NUM_STR_SZ);
+  buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%u", num);
+}
+
 void httpJsonFloat(JsonBuf* buf, float num) {
   httpJsonItemToken(buf);
   httpJsonTestBuf(buf, MAX_NUM_STR_SZ);

src/plugins/http/src/httpRestJson.c

@@ -162,6 +162,18 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result,
         case TSDB_DATA_TYPE_BIGINT:
           httpJsonInt64(jsonBuf, *((int64_t *)row[i]));
           break;
+        case TSDB_DATA_TYPE_UTINYINT:
+          httpJsonUInt(jsonBuf, *((uint8_t *)row[i]));
+          break;
+        case TSDB_DATA_TYPE_USMALLINT:
+          httpJsonUInt(jsonBuf, *((uint16_t *)row[i]));
+          break;
+        case TSDB_DATA_TYPE_UINT:
+          httpJsonUInt(jsonBuf, *((uint32_t *)row[i]));
+          break;
+        case TSDB_DATA_TYPE_UBIGINT:
+          httpJsonUInt64(jsonBuf, *((uint64_t *)row[i]));
+          break;
         case TSDB_DATA_TYPE_FLOAT:
           httpJsonFloat(jsonBuf, GET_FLOAT_VAL(row[i]));
           break;

src/query/inc/qAggMain.h

@@ -26,6 +26,7 @@ extern "C" {
 #include "taosdef.h"
 #include "trpc.h"
 #include "tvariant.h"
+#include "tsdb.h"
 
 #define TSDB_FUNC_INVALID_ID  -1
 #define TSDB_FUNC_COUNT        0
@@ -70,15 +71,17 @@ extern "C" {
 #define TSDB_FUNC_AVG_IRATE    34
 
 #define TSDB_FUNC_TID_TAG      35
-#define TSDB_FUNC_HISTOGRAM    36
+#define TSDB_FUNC_BLKINFO      36
-#define TSDB_FUNC_HLL          37
+
-#define TSDB_FUNC_MODE         38
+#define TSDB_FUNC_HISTOGRAM    37
-#define TSDB_FUNC_SAMPLE       39
+#define TSDB_FUNC_HLL          38
-#define TSDB_FUNC_CEIL         40
+#define TSDB_FUNC_MODE         39
-#define TSDB_FUNC_FLOOR        41
+#define TSDB_FUNC_SAMPLE       40
-#define TSDB_FUNC_ROUND        42
+#define TSDB_FUNC_CEIL         41
-#define TSDB_FUNC_MAVG         43
+#define TSDB_FUNC_FLOOR        42
-#define TSDB_FUNC_CSUM         44
+#define TSDB_FUNC_ROUND        43
+#define TSDB_FUNC_MAVG         44
+#define TSDB_FUNC_CSUM         45
 
 
 #define TSDB_FUNCSTATE_SO           0x1u    // single output
@@ -214,13 +217,14 @@ typedef struct SAggFunctionInfo {
   void (*xFinalize)(SQLFunctionCtx *pCtx);
   void (*mergeFunc)(SQLFunctionCtx *pCtx);
 
-  int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId);
+  int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId);
 } SAggFunctionInfo;
 
 #define GET_RES_INFO(ctx) ((ctx)->resultInfo)
 
 int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type,
                           int16_t *len, int32_t *interBytes, int16_t extLength, bool isSuperTable);
+int32_t isValidFunction(const char* name, int32_t len);
 
 #define IS_STREAM_QUERY_VALID(x)  (((x)&TSDB_FUNCSTATE_STREAM) != 0)
 #define IS_MULTIOUTPUT(x)         (((x)&TSDB_FUNCSTATE_MO) != 0)
@@ -242,12 +246,16 @@ typedef struct STwaInfo {
   STimeWindow win;
 } STwaInfo;
 
+struct SBufferWriter;
+void blockDistInfoToBinary(STableBlockDist* pDist, struct SBufferWriter* bw);
+void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDist);
+
 /* global sql function array */
 extern struct SAggFunctionInfo aAggs[];
 
 extern int32_t functionCompatList[]; // compatible check array list
 
-bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval);
+bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval);
 
 /**
  * the numOfRes should be kept, since it may be used later
@@ -258,14 +266,14 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const cha
     (_r)->initialized = false; \
   } while (0)
 
-static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, uint32_t bufLen) {
+static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, int32_t bufLen) {
   pResInfo->initialized = true;  // the this struct has been initialized flag
   
   pResInfo->complete = false;
   pResInfo->hasResult = false;
   pResInfo->numOfRes = 0;
   
-  memset(GET_ROWCELL_INTERBUF(pResInfo), 0, (size_t)bufLen);
+  memset(GET_ROWCELL_INTERBUF(pResInfo), 0, bufLen);
 }
 
 #ifdef __cplusplus

src/query/inc/qExecutor.h

@@ -12,8 +12,8 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TDENGINE_QUERYEXECUTOR_H
+#ifndef TDENGINE_QEXECUTOR_H
-#define TDENGINE_QUERYEXECUTOR_H
+#define TDENGINE_QEXECUTOR_H
 
 #include "os.h"
 
@@ -37,30 +37,24 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int
 #define Q_STATUS_EQUAL(p, s)  (((p) & (s)) != 0u)
 #define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP)
 
-#define SET_STABLE_QUERY_OVER(_q) ((_q)->tableIndex = (int32_t)((_q)->tableqinfoGroupInfo.numOfTables))
-#define IS_STASBLE_QUERY_OVER(_q) ((_q)->tableIndex >= (int32_t)((_q)->tableqinfoGroupInfo.numOfTables))
-
 #define GET_TABLEGROUP(q, _index)   ((SArray*) taosArrayGetP((q)->tableqinfoGroupInfo.pGroupList, (_index)))
 
+#define GET_NUM_OF_RESULTS(_r) (((_r)->outputBuf) == NULL? 0:((_r)->outputBuf)->info.rows)
+
 enum {
   // when query starts to execute, this status will set
       QUERY_NOT_COMPLETED = 0x1u,
 
-  /* result output buffer is full, current query is paused.
-   * this status is only exist in group-by clause and diff/add/division/multiply/ query.
-   */
-      QUERY_RESBUF_FULL = 0x2u,
-
   /* query is over
    * 1. this status is used in one row result query process, e.g., count/sum/first/last/ avg...etc.
    * 2. when all data within queried time window, it is also denoted as query_completed
    */
-      QUERY_COMPLETED = 0x4u,
+      QUERY_COMPLETED = 0x2u,
 
   /* when the result is not completed return to client, this status will be
    * usually used in case of interval query with interpolation option
    */
-      QUERY_OVER = 0x8u,
+      QUERY_OVER = 0x4u,
 };
 
 typedef struct SResultRowPool {
@@ -86,13 +80,13 @@ typedef struct SSqlGroupbyExpr {
 
 typedef struct SResultRow {
   int32_t       pageId;      // pageId & rowId is the position of current result in disk-based output buffer
-  int32_t       rowId:29;    // row index in buffer page
+  int32_t       offset:29;    // row index in buffer page
   bool          startInterp; // the time window start timestamp has done the interpolation already.
   bool          endInterp;   // the time window end timestamp has done the interpolation already.
   bool          closed;      // this result status: closed or opened
   uint32_t      numOfRows;   // number of rows of current time window
   SResultRowCellInfo*  pCellInfo;  // For each result column, there is a resultInfo
-  union {STimeWindow win; char* key;};  // start key of current time window
+  union {STimeWindow win; char* key;};  // start key of current result row
 } SResultRow;
 
 typedef struct SGroupResInfo {
@@ -106,12 +100,11 @@ typedef struct SGroupResInfo {
  * If the number of generated results is greater than this value,
  * query query will be halt and return results to client immediate.
  */
-typedef struct SResultRec {
+typedef struct SRspResultInfo {
   int64_t total;      // total generated result size in rows
-  int64_t rows;       // current result set size in rows
+  int32_t capacity;   // capacity of current result output buffer
-  int64_t capacity;   // capacity of current result output buffer
   int32_t threshold;  // result size threshold in rows.
-} SResultRec;
+} SRspResultInfo;
 
 typedef struct SResultRowInfo {
   SResultRow** pResult;    // result list
@@ -138,7 +131,6 @@ typedef struct SSingleColumnFilterInfo {
 typedef struct STableQueryInfo {
   TSKEY       lastKey;
   int32_t     groupIndex;     // group id in table list
-  int16_t     queryRangeSet;  // denote if the query range is set, only available for interval query
   tVariant    tag;
   STimeWindow win;
   STSCursor   cur;
@@ -179,72 +171,77 @@ typedef struct {
   SArray* pResult;  // SArray<SStddevInterResult>
 } SInterResult;
 
+typedef struct SSDataBlock {
+  SDataStatis *pBlockStatis;
+  SArray      *pDataBlock;
+  SDataBlockInfo info;
+} SSDataBlock;
+
 typedef struct SQuery {
-  int16_t          numOfCols;
-  int16_t          numOfTags;
-  SOrderVal        order;
-  STimeWindow      window;
-  SInterval        interval;
-  int16_t          precision;
-  int16_t          numOfOutput;
-  int16_t          fillType;
-  int16_t          checkResultBuf;   // check if the buffer is full during scan each block
   SLimitVal        limit;
 
-  int32_t          srcRowSize;       // todo extract struct
-  int32_t          resultRowSize;
-  int32_t          maxSrcColumnSize;
-  int32_t          tagLen;           // tag value length of current query
-
-  SSqlGroupbyExpr* pGroupbyExpr;
-  SExprInfo*       pExpr1;
-  SExprInfo*       pExpr2;
-  int32_t          numOfExpr2;
-
-  SColumnInfo*     colList;
-  SColumnInfo*     tagColList;
-  int32_t          numOfFilterCols;
-  int64_t*         fillVal;
-  uint32_t         status;             // query status
-  SResultRec       rec;
-  int32_t          pos;
-  tFilePage**      sdata;
-  STableQueryInfo* current;
-  int32_t          numOfCheckedBlocks; // number of check data blocks
-
-  SOrderedPrjQueryInfo prjInfo;        // limit value for each vgroup, only available in global order projection query.
-  SSingleColumnFilterInfo* pFilterInfo;
-} SQuery;
-
-typedef struct SQueryRuntimeEnv {
-  jmp_buf              env;
-  SQuery*              pQuery;
-  SQLFunctionCtx*      pCtx;
-  int32_t              numOfRowsPerPage;
-  uint16_t*            offset;
-  uint16_t             scanFlag;         // denotes reversed scan of data or not
-  SFillInfo*           pFillInfo;
-  SResultRowInfo       resultRowInfo;
-
-  SQueryCostInfo       summary;
-  void*                pQueryHandle;
-  void*                pSecQueryHandle;  // another thread for
   bool             stableQuery;      // super table query or not
   bool             topBotQuery;      // TODO used bitwise flag
   bool             groupbyColumn;    // denote if this is a groupby normal column query
   bool             hasTagResults;    // if there are tag values in final result or not
   bool             timeWindowInterpo;// if the time window start/end required interpolation
-  bool                 queryWindowIdentical; // all query time windows are identical for all tables in one group
   bool             queryBlockDist;    // if query data block distribution
   bool             stabledev;        // super table stddev query
   int32_t          interBufSize;     // intermediate buffer sizse
+
+  SOrderVal        order;
+  int16_t          numOfCols;
+  int16_t          numOfTags;
+
+  STimeWindow      window;
+  SInterval        interval;
+  SSessionWindow   sw;
+  int16_t          precision;
+  int16_t          numOfOutput;
+  int16_t          fillType;
+  int16_t          checkResultBuf;   // check if the buffer is full during scan each block
+
+  int32_t          srcRowSize;       // todo extract struct
+  int32_t          resultRowSize;
+  int32_t          intermediateResultRowSize; // intermediate result row size, in case of top-k query.
+  int32_t          maxSrcColumnSize;
+  int32_t          tagLen;           // tag value length of current query
+  SSqlGroupbyExpr* pGroupbyExpr;
+  SExprInfo*       pExpr1;
+  SExprInfo*       pExpr2;
+  int32_t          numOfExpr2;
+  SColumnInfo*     colList;
+  SColumnInfo*     tagColList;
+  int32_t          numOfFilterCols;
+  int64_t*         fillVal;
+  SOrderedPrjQueryInfo prjInfo;        // limit value for each vgroup, only available in global order projection query.
+  SSingleColumnFilterInfo* pFilterInfo;
+
+  STableQueryInfo* current;
+  void*            tsdb;
+  SMemRef          memRef;
+  STableGroupInfo  tableGroupInfo;       // table <tid, last_key> list  SArray<STableKeyInfo>
+  int32_t          vgId;
+} SQuery;
+
+typedef SSDataBlock* (*__operator_fn_t)(void* param);
+typedef void (*__optr_cleanup_fn_t)(void* param, int32_t num);
+
+struct SOperatorInfo;
+
+typedef struct SQueryRuntimeEnv {
+  jmp_buf               env;
+  SQuery*               pQuery;
+  uint32_t              status;           // query status
+  void*                 qinfo;
+  uint8_t               scanFlag;         // denotes reversed scan of data or not
+  void*                 pQueryHandle;
+
   int32_t               prevGroupId;      // previous executed group id
   SDiskbasedResultBuf*  pResultBuf;       // query result buffer based on blocked-wised disk file
   SHashObj*             pResultRowHashTable; // quick locate the window object for each result
   char*                 keyBuf;           // window key buffer
   SResultRowPool*       pool;             // window result object pool
-
-  int32_t*             rowCellInfoOffset;// offset value for each row result cell info
   char**                prevRow;
 
   SArray*               prevResult;       // intermediate result, SArray<SInterResult>
@@ -253,8 +250,57 @@ typedef struct SQueryRuntimeEnv {
 
   char*                 tagVal;           // tag value of current data block
   SArithmeticSupport   *sasArray;
+
+  SSDataBlock          *outputBuf;
+  STableGroupInfo       tableqinfoGroupInfo;  // this is a group array list, including SArray<STableQueryInfo*> structure
+  struct SOperatorInfo *proot;
+  struct SOperatorInfo *pTableScanner;   // table scan operator
+  SGroupResInfo         groupResInfo;
+  int64_t               currentOffset;   // dynamic offset value
+
+  SRspResultInfo        resultInfo;
+  SHashObj             *pTableRetrieveTsMap;
 } SQueryRuntimeEnv;
 
+enum {
+  OP_IN_EXECUTING   = 1,
+  OP_RES_TO_RETURN  = 2,
+  OP_EXEC_DONE      = 3,
+};
+
+enum OPERATOR_TYPE_E {
+  OP_TableScan         = 1,
+  OP_DataBlocksOptScan = 2,
+  OP_TableSeqScan      = 3,
+  OP_TagScan           = 4,
+  OP_TableBlockInfoScan= 5,
+  OP_Aggregate         = 6,
+  OP_Arithmetic        = 7,
+  OP_Groupby           = 8,
+  OP_Limit             = 9,
+  OP_Offset            = 10,
+  OP_TimeWindow        = 11,
+  OP_SessionWindow     = 12,
+  OP_Fill              = 13,
+  OP_MultiTableAggregate     = 14,
+  OP_MultiTableTimeInterval  = 15,
+};
+
+typedef struct SOperatorInfo {
+  uint8_t               operatorType;
+  bool                  blockingOptr;  // block operator or not
+  uint8_t               status;        // denote if current operator is completed
+  int32_t               numOfOutput;   // number of columns of the current operator results
+  char                 *name;          // name, used to show the query execution plan
+  void                 *info;          // extension attribution
+  SExprInfo            *pExpr;
+  SQueryRuntimeEnv     *pRuntimeEnv;
+
+  struct SOperatorInfo *upstream;
+  __operator_fn_t       exec;
+  __optr_cleanup_fn_t   cleanup;
+} SOperatorInfo;
+
 enum {
   QUERY_RESULT_NOT_READY = 1,
   QUERY_RESULT_READY     = 2,
@@ -264,21 +310,9 @@ typedef struct SQInfo {
   void*            signature;
   int32_t          code;        // error code to returned to client
   int64_t          owner;       // if it is in execution
-  void*            tsdb;
-  SMemRef          memRef; 
-  int32_t          vgId;
-  STableGroupInfo  tableGroupInfo;       // table <tid, last_key> list  SArray<STableKeyInfo>
-  STableGroupInfo  tableqinfoGroupInfo;  // this is a group array list, including SArray<STableQueryInfo*> structure
-  SQueryRuntimeEnv runtimeEnv;
-  SHashObj*        arrTableIdInfo;
-  int32_t          groupIndex;
 
-  /*
+  SQueryRuntimeEnv runtimeEnv;
-   * the query is executed position on which meter of the whole list.
+  SQuery           query;
-   * when the index reaches the last one of the list, it means the query is completed.
-   */
-  int32_t          tableIndex;
-  SGroupResInfo    groupResInfo;
   void*            pBuf;        // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables;
 
   pthread_mutex_t  lock;        // used to synchronize the rsp/query threads
@@ -287,6 +321,7 @@ typedef struct SQInfo {
   void*            rspContext;  // response context
   int64_t          startExecTs; // start to exec timestamp
   char*            sql;         // query sql string
+  SQueryCostInfo   summary;
 } SQInfo;
 
 typedef struct SQueryParam {
@@ -305,10 +340,93 @@ typedef struct SQueryParam {
   SSqlGroupbyExpr *pGroupbyExpr;
 } SQueryParam;
 
+typedef struct STableScanInfo {
+  void           *pQueryHandle;
+  int32_t         numOfBlocks;
+  int32_t         numOfSkipped;
+  int32_t         numOfBlockStatis;
+  int64_t         numOfRows;
+                 
+  int32_t         order;        // scan order
+  int32_t         times;        // repeat counts
+  int32_t         current;
+  int32_t         reverseTimes; // 0 by default
+
+  SQLFunctionCtx *pCtx;         // next operator query context
+  SResultRowInfo *pResultRowInfo;
+  int32_t        *rowCellInfoOffset;
+  SExprInfo      *pExpr;
+  SSDataBlock     block;
+  bool            loadExternalRows; // load external rows (prev & next rows)
+  int32_t         numOfOutput;
+  int64_t         elapsedTime;
+
+  int32_t         tableIndex;
+} STableScanInfo;
+
+typedef struct STagScanInfo {
+  SColumnInfo* pCols;
+  SSDataBlock* pRes;
+  int32_t      totalTables;
+  int32_t      currentIndex;
+} STagScanInfo;
+
+typedef struct SOptrBasicInfo {
+  SResultRowInfo    resultRowInfo;
+  int32_t          *rowCellInfoOffset;  // offset value for each row result cell info
+  SQLFunctionCtx   *pCtx;
+  SSDataBlock      *pRes;
+} SOptrBasicInfo;
+
+typedef struct SOptrBasicInfo STableIntervalOperatorInfo;
+
+typedef struct SAggOperatorInfo {
+  SOptrBasicInfo binfo;
+  uint32_t       seed;
+} SAggOperatorInfo;
+
+typedef struct SArithOperatorInfo {
+  SOptrBasicInfo binfo;
+  int32_t        bufCapacity;
+  uint32_t       seed;
+} SArithOperatorInfo;
+
+typedef struct SLimitOperatorInfo {
+  int64_t limit;
+  int64_t total;
+} SLimitOperatorInfo;
+
+typedef struct SOffsetOperatorInfo {
+  int64_t offset;
+} SOffsetOperatorInfo;
+
+typedef struct SFillOperatorInfo {
+  SFillInfo   *pFillInfo;
+  SSDataBlock *pRes;
+  int64_t      totalInputRows;
+} SFillOperatorInfo;
+
+typedef struct SGroupbyOperatorInfo {
+  SOptrBasicInfo binfo;
+  int32_t        colIndex;
+  char          *prevData;   // previous group by value
+} SGroupbyOperatorInfo;
+
+typedef struct SSWindowOperatorInfo {
+  SOptrBasicInfo binfo;
+  STimeWindow    curWindow;  // current time window
+  TSKEY          prevTs;     // previous timestamp
+  int32_t        numOfRows;  // number of rows
+  int32_t        start;      // start row index
+} SSWindowOperatorInfo;
+
 void freeParam(SQueryParam *param);
 int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param);
 int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo, SSqlFuncMsg **pExprMsg,
                                    SColumnInfo* pTagCols);
+int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo,
+                                           SSqlFuncMsg **pExprMsg, SExprInfo *prevExpr);
+
 SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code);
 SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs,
                         SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, bool stableQuery, char* sql);
@@ -318,13 +436,9 @@ void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters);
 bool isQueryKilled(SQInfo *pQInfo);
 int32_t checkForQueryBuf(size_t numOfTables);
 bool doBuildResCheck(SQInfo* pQInfo);
-void setQueryStatus(SQuery *pQuery, int8_t status);
+void setQueryStatus(SQueryRuntimeEnv *pRuntimeEnv, int8_t status);
 
 bool onlyQueryTags(SQuery* pQuery);
-void buildTagQueryResult(SQInfo *pQInfo);
-void stableQueryImpl(SQInfo *pQInfo);
-void buildTableBlockDistResult(SQInfo *pQInfo);
-void tableQueryImpl(SQInfo *pQInfo);
 bool isValidQInfo(void *param);
 
 int32_t doDumpQueryResult(SQInfo *pQInfo, char *data);
@@ -336,4 +450,4 @@ void freeQInfo(SQInfo *pQInfo);
 
 int32_t getMaximumIdleDurationSec();
 
-#endif  // TDENGINE_QUERYEXECUTOR_H
+#endif  // TDENGINE_QEXECUTOR_H

src/query/inc/qFill.h

@@ -24,6 +24,8 @@ extern "C" {
 #include "qExtbuffer.h"
 #include "taosdef.h"
 
+struct SSDataBlock;
+
 typedef struct {
   STColumn col;             // column info
   int16_t  functionId;      // sql function id
@@ -78,7 +80,7 @@ void* taosDestroyFillInfo(SFillInfo *pFillInfo);
 
 void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);
 
-void taosFillSetDataBlockFromFilePage(SFillInfo* pFillInfo, const tFilePage** pInput);
+void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const struct SSDataBlock* pInput);
 
 void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, const tFilePage* pInput);
 
@@ -88,7 +90,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t
 
 int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType);
 
-int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity);
+int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity);
 
 #ifdef __cplusplus
 }

src/query/inc/qResultbuf.h

@@ -55,7 +55,6 @@ typedef struct SResultBufStatis {
 } SResultBufStatis;
 
 typedef struct SDiskbasedResultBuf {
-  int32_t   numOfRowsPerPage;
   int32_t   numOfPages;
   int64_t   totalBufSize;
   int64_t   fileSize;            // disk file size
@@ -77,7 +76,7 @@ typedef struct SDiskbasedResultBuf {
   SResultBufStatis statis;
 } SDiskbasedResultBuf;
 
-#define DEFAULT_INTERN_BUF_PAGE_SIZE  (256L)                          // in bytes
+#define DEFAULT_INTERN_BUF_PAGE_SIZE  (1024L)                          // in bytes
 #define PAGE_INFO_INITIALIZER         (SPageDiskInfo){-1, -1}
 
 /**
@@ -89,8 +88,7 @@ typedef struct SDiskbasedResultBuf {
  * @param handle
  * @return
  */
-int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t rowSize, int32_t pagesize,
+int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, const void* handle);
-                                    int32_t inMemBufSize, const void* handle);
 
 /**
  *
@@ -101,13 +99,6 @@ int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t ro
  */
 tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId);
 
-/**
- *
- * @param pResultBuf
- * @return
- */
-size_t getNumOfRowsPerPage(const SDiskbasedResultBuf* pResultBuf);
-
 /**
  *
  * @param pResultBuf

src/query/inc/qSqlparser.h

@@ -58,14 +58,19 @@ typedef struct SIntervalVal {
   SStrToken offset;
 } SIntervalVal;
 
+typedef struct SSessionWindowVal {
+  SStrToken col;
+  SStrToken gap;
+} SSessionWindowVal;
+
 typedef struct SQuerySQL {
   struct tSQLExprList *pSelection;   // select clause
   SArray *             from;         // from clause  SArray<tVariantListItem>
   struct tSQLExpr *    pWhere;       // where clause [optional]
   SArray *             pGroupby;     // groupby clause, only for tags[optional], SArray<tVariantListItem>
   SArray *             pSortOrder;   // orderby [optional], SArray<tVariantListItem>
-  SStrToken            interval;     // interval [optional]
+  SIntervalVal         interval;     // (interval, interval_offset) [optional]
-  SStrToken            offset;       // offset window [optional]
+  SSessionWindowVal    sessionVal;   // session window [optional]
   SStrToken            sliding;      // sliding window [optional]
   SLimitVal            limit;        // limit offset [optional]
   SLimitVal            slimit;       // group limit offset [optional]
@@ -125,6 +130,8 @@ typedef struct SCreateDbInfo {
   int8_t    update;
   int8_t    cachelast; 
   SArray    *keep;
+  int8_t    dbType;
+  int16_t   partitions;
 } SCreateDbInfo;
 
 typedef struct SCreateAcctInfo {
@@ -155,6 +162,7 @@ typedef struct SUserInfo {
 typedef struct SMiscInfo {
   SArray    *a;         // SArray<SStrToken>
   bool       existsCheck;
+  int16_t    dbType;
   int16_t    tableType;
   SUserInfo  user;
   union {
@@ -182,19 +190,32 @@ typedef struct SSqlInfo {
   };
 } SSqlInfo;
 
-typedef struct tSQLExpr {
+#define NON_ARITHMEIC_EXPR 0
-  uint32_t         nSQLOptr;    // TK_FUNCTION: sql function, TK_LE: less than(binary expr)
+#define NORMAL_ARITHMETIC  1
+#define AGG_ARIGHTMEIC     2
 
-  // the full sql string of function(col, param), which is actually the raw
+enum SQL_NODE_TYPE {
-  // field name, since the function name is kept in nSQLOptr already
+  SQL_NODE_TABLE_COLUMN= 1,
+  SQL_NODE_SQLFUNCTION = 2,
+  SQL_NODE_VALUE       = 3,
+  SQL_NODE_EXPR        = 4,
+};
+
+typedef struct tSQLExpr {
+  uint16_t         type;       // sql node type
+  uint32_t         tokenId;    // TK_FUNCTION: sql function, TK_LE: less than(binary expr)
+
+  // the whole string of the function(col, param), while the function name is kept in token
   SStrToken        operand;
-  SStrToken        colInfo;     // field id
+  uint32_t         functionId;  // function id
-  tVariant         val;         // value only for string, float, int
+
+  SStrToken        colInfo;     // table column info
+  tVariant         value;       // the use input value
   SStrToken        token;       // original sql expr string
 
   struct tSQLExpr *pLeft;       // left child
   struct tSQLExpr *pRight;      // right child
-  struct tSQLExprList *pParam;  // function parameters
+  struct tSQLExprList *pParam;  // function parameters list
 } tSQLExpr;
 
 // used in select clause. select <tSQLExprList> from xxx
@@ -240,8 +261,8 @@ tSQLExprList *tSqlExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken
 
 void tSqlExprListDestroy(tSQLExprList *pList);
 
-SQuerySQL *tSetQuerySqlElems(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere,
+SQuerySQL *tSetQuerySqlNode(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere,
-                             SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval,
+                             SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, SSessionWindowVal *pSession,
                              SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *pGLimit);
 
 SCreateTableSQL *tSetCreateSqlElems(SArray *pCols, SArray *pTags, SQuerySQL *pSelect, int32_t type);
@@ -265,7 +286,7 @@ void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pTableNameToken, SStrToken
 void SqlInfoDestroy(SSqlInfo *pInfo);
 
 void setDCLSQLElems(SSqlInfo *pInfo, int32_t type, int32_t nParams, ...);
-void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck,int16_t tableType);
+void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck,int16_t dbType,int16_t tableType);
 void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns);
 
 void setCreateDbInfo(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDbInfo *pDB, SStrToken *pIgExists);
@@ -276,6 +297,7 @@ void setKillSql(SSqlInfo *pInfo, int32_t type, SStrToken *ip);
 void setAlterUserSql(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege);
 
 void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo);
+void setDefaultCreateTopicOption(SCreateDbInfo *pDBInfo);
 
 // prefix show db.tables;
 void setDbName(SStrToken *pCpxName, SStrToken *pDb);
@@ -290,16 +312,6 @@ void tSqlSetColumnType(TAOS_FIELD *pField, SStrToken *type);
 
 void *ParseAlloc(void *(*mallocProc)(size_t));
 
-enum {
-  TSQL_NODE_TYPE_EXPR  = 0x1,
-  TSQL_NODE_TYPE_ID    = 0x2,
-  TSQL_NODE_TYPE_VALUE = 0x4,
-};
-
-#define NON_ARITHMEIC_EXPR 0
-#define NORMAL_ARITHMETIC  1
-#define AGG_ARIGHTMEIC     2
-
 SSqlInfo qSQLParse(const char *str);
 
 #ifdef __cplusplus

src/query/inc/qTsbuf.h

@@ -113,12 +113,10 @@ STSBuf* tsBufClone(STSBuf* pTSBuf);
 STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id);
 
 void    tsBufFlush(STSBuf* pTSBuf);
-
 void    tsBufResetPos(STSBuf* pTSBuf);
-STSElem tsBufGetElem(STSBuf* pTSBuf);
-
 bool    tsBufNextPos(STSBuf* pTSBuf);
 
+STSElem tsBufGetElem(STSBuf* pTSBuf);
 STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, tVariant* tag);
 
 STSCursor tsBufGetCursor(STSBuf* pTSBuf);

src/query/inc/qUtil.h

@@ -27,7 +27,7 @@
 #define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t))
 
 #define curTimeWindowIndex(_winres)        ((_winres)->curIndex)
-#define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pExpr1[1].base.arg->argValue.i64:1)
+#define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!(sq)))? (_q)->pExpr1[1].base.arg->argValue.i64:1)
 
 int32_t getOutputInterResultBufSize(SQuery* pQuery);
 
@@ -44,22 +44,18 @@ void    closeResultRow(SResultRowInfo* pResultRowInfo, int32_t slot);
 bool    isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot);
 void    clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow, int16_t type);
 
-SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index);
+SResultRowCellInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset);
 
 static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) {
   assert(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size);
   return pResultRowInfo->pResult[slot];
 }
 
-static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SResultRow *pResult,
+static FORCE_INLINE char *getPosInResultPage(SQuery *pQuery, tFilePage* page, int32_t rowOffset, int16_t offset) {
-    tFilePage* page) {
+  assert(rowOffset >= 0 && pQuery != NULL);
-  assert(pResult != NULL && pRuntimeEnv != NULL);
 
-  SQuery *pQuery = pRuntimeEnv->pQuery;
+  int32_t numOfRows = (int32_t)GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pQuery->topBotQuery, pQuery->stableQuery);
-
+  return ((char *)page->data) + rowOffset + offset * numOfRows;
-  int32_t realRowId = (int32_t)(pResult->rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery));
-  return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * pRuntimeEnv->numOfRowsPerPage +
-      pQuery->pExpr1[columnIndex].bytes * realRowId;
 }
 
 bool isNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type);
@@ -74,8 +70,6 @@ void* destroyResultRowPool(SResultRowPool* p);
 int32_t getNumOfAllocatedResultRows(SResultRowPool* p);
 int32_t getNumOfUsedResultRows(SResultRowPool* p);
 
-bool isPointInterpoQuery(SQuery *pQuery);
-
 typedef struct {
   SArray* pResult;     // SArray<SResPair>
   int32_t colId;
@@ -85,12 +79,14 @@ void interResToBinary(SBufferWriter* bw, SArray* pRes, int32_t tagLen);
 SArray* interResFromBinary(const char* data, int32_t len);
 void freeInterResult(void* param);
 
-void    initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo, int32_t offset);
+void    initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo);
 void    cleanupGroupResInfo(SGroupResInfo* pGroupResInfo);
+bool    hasRemainDataInCurrentGroup(SGroupResInfo* pGroupResInfo);
 bool    hasRemainData(SGroupResInfo* pGroupResInfo);
+
 bool    incNextGroup(SGroupResInfo* pGroupResInfo);
 int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo);
 
-int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQInfo *pQInfo);
+int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQueryRuntimeEnv *pRuntimeEnv, int32_t* offset);
 
 #endif  // TDENGINE_QUERYUTIL_H

src/query/inc/sql.y

@@ -64,6 +64,7 @@ program ::= cmd.    {}
 
 //////////////////////////////////THE SHOW STATEMENT///////////////////////////////////////////
 cmd ::= SHOW DATABASES.  { setShowOptions(pInfo, TSDB_MGMT_TABLE_DB, 0, 0);}
+cmd ::= SHOW TOPICS.     { setShowOptions(pInfo, TSDB_MGMT_TABLE_TP, 0, 0);}
 cmd ::= SHOW MNODES.     { setShowOptions(pInfo, TSDB_MGMT_TABLE_MNODE, 0, 0);}
 cmd ::= SHOW DNODES.     { setShowOptions(pInfo, TSDB_MGMT_TABLE_DNODE, 0, 0);}
 cmd ::= SHOW ACCOUNTS.   { setShowOptions(pInfo, TSDB_MGMT_TABLE_ACCT, 0, 0);}
@@ -131,16 +132,18 @@ cmd ::= SHOW dbPrefix(X) VGROUPS ids(Y).    {
 //drop configure for tables
 cmd ::= DROP TABLE ifexists(Y) ids(X) cpxName(Z).   {
     X.n += Z.n;
-    setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y, -1);
+    setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y, -1, -1);
 }
 
 //drop stable
 cmd ::= DROP STABLE ifexists(Y) ids(X) cpxName(Z).   {
     X.n += Z.n;
-    setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y, TSDB_SUPER_TABLE);
+    setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y, -1, TSDB_SUPER_TABLE);
 }
 
-cmd ::= DROP DATABASE ifexists(Y) ids(X).    { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, -1); }
+cmd ::= DROP DATABASE ifexists(Y) ids(X).    { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, TSDB_DB_TYPE_DEFAULT, -1); }
+cmd ::= DROP TOPIC ifexists(Y) ids(X).    { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, TSDB_DB_TYPE_TOPIC, -1); }
+
 cmd ::= DROP DNODE ids(X).       { setDCLSQLElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &X);    }
 cmd ::= DROP USER ids(X).        { setDCLSQLElems(pInfo, TSDB_SQL_DROP_USER, 1, &X);     }
 cmd ::= DROP ACCOUNT ids(X).     { setDCLSQLElems(pInfo, TSDB_SQL_DROP_ACCT, 1, &X);  }
@@ -162,6 +165,7 @@ cmd ::= ALTER DNODE ids(X) ids(Y) ids(Z).       { setDCLSQLElems(pInfo, TSDB_SQL
 cmd ::= ALTER LOCAL ids(X).                     { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &X);              }
 cmd ::= ALTER LOCAL ids(X) ids(Y).              { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &X, &Y);          }
 cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SStrToken t = {0};  setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);}
+cmd ::= ALTER TOPIC ids(X) alter_topic_optr(Y). { SStrToken t = {0};  setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);}
 
 cmd ::= ALTER ACCOUNT ids(X) acct_optr(Z).      { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, NULL, &Z);}
 cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z).      { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, &Y, &Z);}
@@ -187,6 +191,7 @@ cmd ::= CREATE DNODE   ids(X).     { setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE
 cmd ::= CREATE ACCOUNT ids(X) PASS ids(Y) acct_optr(Z).
                                 { setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &X, &Y, &Z);}
 cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y).  { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);}
+cmd ::= CREATE TOPIC ifnotexists(Z) ids(X) topic_optr(Y).  { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);}
 cmd ::= CREATE USER ids(X) PASS ids(Y).     { setCreateUserSql(pInfo, &X, &Y);}
 
 pps(Y) ::= .                                { Y.n = 0;   }
@@ -247,9 +252,10 @@ comp(Y)    ::= COMP INTEGER(X).               { Y = X; }
 prec(Y)    ::= PRECISION STRING(X).           { Y = X; }
 update(Y)  ::= UPDATE INTEGER(X).             { Y = X; }     
 cachelast(Y) ::= CACHELAST INTEGER(X).        { Y = X; }
+partitions(Y) ::= PARTITIONS INTEGER(X).      { Y = X; }
 
 %type db_optr {SCreateDbInfo}
-db_optr(Y) ::= . {setDefaultCreateDbOption(&Y);}
+db_optr(Y) ::= . {setDefaultCreateDbOption(&Y); Y.dbType = TSDB_DB_TYPE_DEFAULT;}
 
 db_optr(Y) ::= db_optr(Z) cache(X).          { Y = Z; Y.cacheBlockSize = strtol(X.z, NULL, 10); }
 db_optr(Y) ::= db_optr(Z) replica(X).        { Y = Z; Y.replica = strtol(X.z, NULL, 10); }
@@ -267,8 +273,13 @@ db_optr(Y) ::= db_optr(Z) keep(X).           { Y = Z; Y.keep = X; }
 db_optr(Y) ::= db_optr(Z) update(X).         { Y = Z; Y.update = strtol(X.z, NULL, 10); }
 db_optr(Y) ::= db_optr(Z) cachelast(X).      { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); }
 
+%type topic_optr {SCreateDbInfo}
+
+topic_optr(Y) ::= db_optr(Z).                       { Y = Z; Y.dbType = TSDB_DB_TYPE_TOPIC; }
+topic_optr(Y) ::= topic_optr(Z) partitions(X).      { Y = Z; Y.partitions = strtol(X.z, NULL, 10); }
+
 %type alter_db_optr {SCreateDbInfo}
-alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y);}
+alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y); Y.dbType = TSDB_DB_TYPE_DEFAULT;}
 
 alter_db_optr(Y) ::= alter_db_optr(Z) replica(X).     { Y = Z; Y.replica = strtol(X.z, NULL, 10); }
 alter_db_optr(Y) ::= alter_db_optr(Z) quorum(X).      { Y = Z; Y.quorum = strtol(X.z, NULL, 10); }
@@ -280,6 +291,11 @@ alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X).       { Y = Z; Y.fsyncPeriod = s
 alter_db_optr(Y) ::= alter_db_optr(Z) update(X).      { Y = Z; Y.update = strtol(X.z, NULL, 10); }
 alter_db_optr(Y) ::= alter_db_optr(Z) cachelast(X).   { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); }
 
+%type alter_topic_optr {SCreateDbInfo}
+
+alter_topic_optr(Y) ::= alter_db_optr(Z).                       { Y = Z; Y.dbType = TSDB_DB_TYPE_TOPIC; }
+alter_topic_optr(Y) ::= alter_topic_optr(Z) partitions(X).      { Y = Z; Y.partitions = strtol(X.z, NULL, 10); }
+
 %type typename {TAOS_FIELD}
 typename(A) ::= ids(X). { 
   X.type = 0;
@@ -436,8 +452,8 @@ tagitem(A) ::= PLUS(X) FLOAT(Y).  {
 //////////////////////// The SELECT statement /////////////////////////////////
 %type select {SQuerySQL*}
 %destructor select {doDestroyQuerySql($$);}
-select(A) ::= SELECT(T) selcollist(W) from(X) where_opt(Y) interval_opt(K) fill_opt(F) sliding_opt(S) groupby_opt(P) orderby_opt(Z) having_opt(N) slimit_opt(G) limit_opt(L). {
+select(A) ::= SELECT(T) selcollist(W) from(X) where_opt(Y) interval_opt(K) session_option(H) fill_opt(F) sliding_opt(S) groupby_opt(P) orderby_opt(Z) having_opt(N) slimit_opt(G) limit_opt(L). {
-  A = tSetQuerySqlElems(&T, W, X, Y, P, Z, &K, &S, F, &L, &G);
+  A = tSetQuerySqlNode(&T, W, X, Y, P, Z, &K, &H, &S, F, &L, &G);
 }
 
 %type union {SSubclauseInfo*}
@@ -455,7 +471,7 @@ cmd ::= union(X). { setSqlInfo(pInfo, X, NULL, TSDB_SQL_SELECT); }
 // select server_version(), select client_version(),
 // select server_state();
 select(A) ::= SELECT(T) selcollist(W). {
-  A = tSetQuerySqlElems(&T, W, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
+  A = tSetQuerySqlNode(&T, W, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
 }
 
 // selcollist is a list of expressions that are to become the return
@@ -530,10 +546,18 @@ tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z) ids(F). {
 tmvar(A) ::= VARIABLE(X).   {A = X;}
 
 %type interval_opt {SIntervalVal}
-interval_opt(N) ::= INTERVAL LP tmvar(E) RP.    {N.interval = E; N.offset.n = 0; N.offset.z = NULL; N.offset.type = 0;}
+interval_opt(N) ::= INTERVAL LP tmvar(E) RP.    {N.interval = E; N.offset.n = 0;}
-interval_opt(N) ::= INTERVAL LP tmvar(E) COMMA tmvar(O) RP.    {N.interval = E; N.offset = O;}
+interval_opt(N) ::= INTERVAL LP tmvar(E) COMMA tmvar(X) RP.    {N.interval = E; N.offset = X;}
 interval_opt(N) ::= .                           {memset(&N, 0, sizeof(N));}
 
+%type session_option {SSessionWindowVal}
+session_option(X) ::= .                                                  {X.col.n = 0; X.gap.n = 0;}
+session_option(X) ::= SESSION LP ids(V) cpxName(Z) COMMA tmvar(Y) RP.    {
+   V.n += Z.n;
+   X.col = V;
+   X.gap = Y;
+}
+
 %type fill_opt {SArray*}
 %destructor fill_opt {taosArrayDestroy($$);}
 fill_opt(N) ::= .                                           { N = 0;     }
@@ -818,6 +842,6 @@ cmd ::= KILL QUERY INTEGER(X) COLON(Z) INTEGER(Y).        {X.n += (Z.n + Y.n); s
 %fallback ID ABORT AFTER ASC ATTACH BEFORE BEGIN CASCADE CLUSTER CONFLICT COPY DATABASE DEFERRED
   DELIMITERS DESC DETACH EACH END EXPLAIN FAIL FOR GLOB IGNORE IMMEDIATE INITIALLY INSTEAD
   LIKE MATCH KEY OF OFFSET RAISE REPLACE RESTRICT ROW STATEMENT TRIGGER VIEW ALL
-  COUNT SUM AVG MIN MAX FIRST LAST TOP BOTTOM STDDEV PERCENTILE APERCENTILE LEASTSQUARES HISTOGRAM DIFF
+  NOW IPTOKEN SEMI NONE PREV LINEAR IMPORT
-  SPREAD TWA INTERP LAST_ROW RATE IRATE SUM_RATE SUM_IRATE AVG_RATE AVG_IRATE TBID NOW IPTOKEN SEMI NONE PREV LINEAR IMPORT
   METRIC TBNAME JOIN METRICS STABLE NULL INSERT INTO VALUES.
+

src/query/src/qAggMain.c

@@ -18,6 +18,7 @@
 #include "taosmsg.h"
 #include "texpr.h"
 #include "ttype.h"
+#include "tsdb.h"
 
 #include "qAggMain.h"
 #include "qFill.h"
@@ -26,11 +27,9 @@
 #include "qTsbuf.h"
 #include "queryLog.h"
 
-//#define GET_INPUT_DATA_LIST(x) (((char *)((x)->pInput)) + ((x)->startOffset) * ((x)->inputBytes))
 #define GET_INPUT_DATA_LIST(x) ((char *)((x)->pInput))
 #define GET_INPUT_DATA(x, y) (GET_INPUT_DATA_LIST(x) + (y) * (x)->inputBytes)
 
-//#define GET_TS_LIST(x)    ((TSKEY*)&((x)->ptsList[(x)->startOffset]))
 #define GET_TS_LIST(x)    ((TSKEY*)((x)->ptsList))
 #define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)])
 
@@ -191,6 +190,11 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
     *bytes = (int16_t)(dataBytes + sizeof(int16_t) + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t) + VARSTR_HEADER_SIZE);
     *interBytes = 0;
     return TSDB_CODE_SUCCESS;
+  } else if (functionId == TSDB_FUNC_BLKINFO) {
+    *type = TSDB_DATA_TYPE_BINARY;
+    *bytes = 16384;
+    *interBytes = 0;
+    return TSDB_CODE_SUCCESS;
   }
   
   if (functionId == TSDB_FUNC_COUNT) {
@@ -209,7 +213,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
   
   if (functionId == TSDB_FUNC_TS_COMP) {
     *type = TSDB_DATA_TYPE_BINARY;
-    *bytes = sizeof(int32_t);  // this results is compressed ts data
+    *bytes = 1;  // this results is compressed ts data, only one byte
     *interBytes = POINTER_BYTES;
     return TSDB_CODE_SUCCESS;
   }
@@ -355,6 +359,22 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
   return TSDB_CODE_SUCCESS;
 }
 
+// TODO use hash table
+int32_t isValidFunction(const char* name, int32_t len) {
+  for(int32_t i = 0; i <= TSDB_FUNC_BLKINFO; ++i) {
+    int32_t nameLen = (int32_t) strlen(aAggs[i].name);
+    if (len != nameLen) {
+      continue;
+    }
+
+    if (strncasecmp(aAggs[i].name, name, len) == 0) {
+      return i;
+    }
+  }
+
+  return -1;
+}
+
 // set the query flag to denote that query is completed
 static void no_next_step(SQLFunctionCtx *pCtx) {
   SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
@@ -458,7 +478,7 @@ static void count_func_merge(SQLFunctionCtx *pCtx) {
  * @param filterCols
  * @return
  */
-int32_t count_load_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) {
+int32_t countRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) {
   if (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
     return BLK_DATA_NO_NEEDED;
   } else {
@@ -466,7 +486,7 @@ int32_t count_load_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32
   }
 }
 
-int32_t no_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) {
+int32_t noDataRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) {
   return BLK_DATA_NO_NEEDED;
 }
 
@@ -674,16 +694,16 @@ static void sum_func_merge(SQLFunctionCtx *pCtx) {
   }
 }
 
-static int32_t statisRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) {
+static int32_t statisRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) {
   return BLK_DATA_STATIS_NEEDED;
 }
 
-static int32_t dataBlockRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) {
+static int32_t dataBlockRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) {
   return BLK_DATA_ALL_NEEDED;
 }
 
 // todo: if column in current data block are null, opt for this case
-static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) {
+static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) {
   if (pCtx->order == TSDB_ORDER_DESC) {
     return BLK_DATA_NO_NEEDED;
   }
@@ -696,7 +716,7 @@ static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, i
   }
 }
 
-static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) {
+static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) {
   if (pCtx->order != pCtx->param[0].i64) {
     return BLK_DATA_NO_NEEDED;
   }
@@ -708,7 +728,7 @@ static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, in
   }
 }
 
-static int32_t firstDistFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) {
+static int32_t firstDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) {
   if (pCtx->order == TSDB_ORDER_DESC) {
     return BLK_DATA_NO_NEEDED;
   }
@@ -724,11 +744,11 @@ static int32_t firstDistFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY en
   if (pInfo->hasResult != DATA_SET_FLAG) {
     return BLK_DATA_ALL_NEEDED;
   } else {  // data in current block is not earlier than current result
-    return (pInfo->ts <= start) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED;
+    return (pInfo->ts <= w->skey) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED;
   }
 }
 
-static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) {
+static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) {
   if (pCtx->order != pCtx->param[0].i64) {
     return BLK_DATA_NO_NEEDED;
   }
@@ -744,7 +764,7 @@ static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end
   if (pInfo->hasResult != DATA_SET_FLAG) {
     return BLK_DATA_ALL_NEEDED;
   } else {
-    return (pInfo->ts > end) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED;
+    return (pInfo->ts > w->ekey) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED;
   }
 }
 
@@ -1359,7 +1379,21 @@ static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   }
 
 static void stddev_function(SQLFunctionCtx *pCtx) {
-  SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo);
+
+  if (pCtx->currentStage == REPEAT_SCAN && pStd->stage == 0) {
+    pStd->stage++;
+    avg_finalizer(pCtx);
+
+    pResInfo->initialized = true; // set it initialized to avoid re-initialization
+
+    // save average value into tmpBuf, for second stage scan
+    SAvgInfo *pAvg = GET_ROWCELL_INTERBUF(pResInfo);
+
+    pStd->avg = GET_DOUBLE_VAL(pCtx->pOutput);
+    assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum));
+  }
   
   if (pStd->stage == 0) {
     // the first stage is to calculate average value
@@ -1433,6 +1467,19 @@ static void stddev_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
   SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo);
 
+  if (pCtx->currentStage == REPEAT_SCAN && pStd->stage == 0) {
+    pStd->stage++;
+    avg_finalizer(pCtx);
+
+    pResInfo->initialized = true; // set it initialized to avoid re-initialization
+
+    // save average value into tmpBuf, for second stage scan
+    SAvgInfo *pAvg = GET_ROWCELL_INTERBUF(pResInfo);
+
+    pStd->avg = GET_DOUBLE_VAL(pCtx->pOutput);
+    assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum));
+  }
+
   /* the first stage is to calculate average value */
   if (pStd->stage == 0) {
     avg_function_f(pCtx, index);
@@ -1776,7 +1823,7 @@ static bool first_last_function_setup(SQLFunctionCtx *pCtx) {
 
 // todo opt for null block
 static void first_function(SQLFunctionCtx *pCtx) {
-  if (pCtx->order == TSDB_ORDER_DESC || pCtx->preAggVals.dataBlockLoaded == false) {
+  if (pCtx->order == TSDB_ORDER_DESC /*|| pCtx->preAggVals.dataBlockLoaded == false*/) {
     return;
   }
   
@@ -1850,7 +1897,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) {
    * 1. data block that are not loaded
    * 2. scan data files in desc order
    */
-  if (pCtx->order == TSDB_ORDER_DESC || pCtx->preAggVals.dataBlockLoaded == false) {
+  if (pCtx->order == TSDB_ORDER_DESC/* || pCtx->preAggVals.dataBlockLoaded == false*/) {
     return;
   }
   
@@ -1921,7 +1968,7 @@ static void first_dist_func_merge(SQLFunctionCtx *pCtx) {
  *    least one data in this block that is not null.(TODO opt for this case)
  */
 static void last_function(SQLFunctionCtx *pCtx) {
-  if (pCtx->order != pCtx->param[0].i64 || pCtx->preAggVals.dataBlockLoaded == false) {
+  if (pCtx->order != pCtx->param[0].i64/* || pCtx->preAggVals.dataBlockLoaded == false*/) {
     return;
   }
   
@@ -1934,6 +1981,7 @@ static void last_function(SQLFunctionCtx *pCtx) {
         continue; 
       }
     }
+
     memcpy(pCtx->pOutput, data, pCtx->inputBytes);
     
     TSKEY ts = GET_TS_DATA(pCtx, i);
@@ -2013,13 +2061,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) {
     return;
   }
 
-  // data block is discard, not loaded, do not need to check it
-  if (!pCtx->preAggVals.dataBlockLoaded) {
-    return;
-  }
-
   int32_t notNullElems = 0;
-  
   for (int32_t i = pCtx->size - 1; i >= 0; --i) {
     char *data = GET_INPUT_DATA(pCtx, i);
     if (pCtx->hasNull && isNull(data, pCtx->inputType)) {
@@ -2125,12 +2167,7 @@ static void last_row_finalizer(SQLFunctionCtx *pCtx) {
   // do nothing at the first stage
   SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
   if (pResInfo->hasResult != DATA_SET_FLAG) {
-    if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
-      setVardataNull(pCtx->pOutput, pCtx->outputType);
-    } else {
     setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes);
-    }
-    
     return;
   }
   
@@ -2445,7 +2482,7 @@ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) {
   }
 }
 
-bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval) {
+bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval) {
   SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
   if (pResInfo == NULL) {
     return true;
@@ -2460,7 +2497,7 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const cha
   
   tValuePair **pRes = (tValuePair**) pTopBotInfo->res;
   
-  if (functionId == TSDB_FUNC_TOP) {
+  if (pCtx->functionId == TSDB_FUNC_TOP) {
     switch (pCtx->inputType) {
       case TSDB_DATA_TYPE_TINYINT:
         return GET_INT8_VAL(maxval) > pRes[0]->v.i64;
@@ -2535,6 +2572,10 @@ static void top_function(SQLFunctionCtx *pCtx) {
   STopBotInfo *pRes = getTopBotOutputInfo(pCtx);
   assert(pRes->num >= 0);
 
+  if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i64)) {
+    buildTopBotStruct(pRes, pCtx);
+  }
+  
   for (int32_t i = 0; i < pCtx->size; ++i) {
     char *data = GET_INPUT_DATA(pCtx, i);
     TSKEY ts = GET_TS_DATA(pCtx, i);
@@ -2729,6 +2770,17 @@ static void percentile_function(SQLFunctionCtx *pCtx) {
   SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
   SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
 
+  if (pCtx->currentStage == REPEAT_SCAN && pInfo->stage == 0) {
+    // all data are null, set it completed
+    if (pInfo->numOfElems == 0) {
+      pResInfo->complete = true;
+    } else {
+      pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval);
+    }
+
+    pInfo->stage += 1;
+  }
+
   // the first stage, only acquire the min/max value
   if (pInfo->stage == 0) {
     if (pCtx->preAggVals.isSet) {
@@ -2802,10 +2854,20 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   }
 
   SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
-
   SPercentileInfo *pInfo = (SPercentileInfo *)GET_ROWCELL_INTERBUF(pResInfo);
-  if (pInfo->stage == 0) {
 
+  if (pCtx->currentStage == REPEAT_SCAN && pInfo->stage == 0) {
+    // all data are null, set it completed
+    if (pInfo->numOfElems == 0) {
+      pResInfo->complete = true;
+    } else {
+      pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval);
+    }
+
+    pInfo->stage += 1;
+  }
+
+  if (pInfo->stage == 0) {
     double v = 0;
     GET_TYPED_DATA(v, double, pCtx->inputType, pData);
 
@@ -3240,8 +3302,6 @@ static void col_project_function(SQLFunctionCtx *pCtx) {
              pCtx->inputBytes);
     }
   }
-
-  pCtx->pOutput += pCtx->size * pCtx->outputBytes;
 }
 
 static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) {
@@ -3547,9 +3607,6 @@ static void diff_function(SQLFunctionCtx *pCtx) {
     int32_t forwardStep = (isFirstBlock) ? notNullElems - 1 : notNullElems;
     
     GET_RES_INFO(pCtx)->numOfRes += forwardStep;
-    
-    pCtx->pOutput += forwardStep * pCtx->outputBytes;
-    pCtx->ptsOutputBuf = (char*)pCtx->ptsOutputBuf + forwardStep * TSDB_KEYSIZE;
   }
 }
 
@@ -3631,7 +3688,7 @@ char *getArithColumnData(void *param, const char* name, int32_t colId) {
     }
   }
   
-  assert(index >= 0 && colId >= 0);
+  assert(index >= 0 /*&& colId >= 0*/);
   return pSupport->data[index] + pSupport->offset * pSupport->colList[index].bytes;
 }
 
@@ -3640,7 +3697,6 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) {
   SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz;
   
   arithmeticTreeTraverse(sas->pArithExpr->pExpr, pCtx->size, pCtx->pOutput, sas, pCtx->order, getArithColumnData);
-  pCtx->pOutput += pCtx->outputBytes * pCtx->size;
 }
 
 static void arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) {
@@ -4172,17 +4228,13 @@ static void interp_function_impl(SQLFunctionCtx *pCtx) {
 
   if (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) {
     *(TSKEY *)pCtx->pOutput = pCtx->startTs;
-  } else {
+  } else if (type == TSDB_FILL_NULL) {
-    if (pCtx->start.key == INT64_MIN) {
-      assert(pCtx->end.key == INT64_MIN);
-      return;
-    }
-
-    if (type == TSDB_FILL_NULL) {
     setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes);
   } else if (type == TSDB_FILL_SET_VALUE) {
     tVariantDump(&pCtx->param[1], pCtx->pOutput, pCtx->inputType, true);
-    } else if (type == TSDB_FILL_PREV) {
+  } else {
+    if (pCtx->start.key != INT64_MIN && pCtx->start.key < pCtx->startTs && pCtx->end.key > pCtx->startTs) {
+      if (type == TSDB_FILL_PREV) {
         if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) {
           SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->start.val);
         } else {
@@ -4210,11 +4262,53 @@ static void interp_function_impl(SQLFunctionCtx *pCtx) {
           setNull(pCtx->pOutput, srcType, pCtx->inputBytes);
         }
       }
+    } else {
+      // no data generated yet
+      if (pCtx->size == 1) {
+        return;
+      }
+
+      // check the timestamp in input buffer
+      TSKEY skey = GET_TS_DATA(pCtx, 0);
+      TSKEY ekey = GET_TS_DATA(pCtx, 1);
+
+      // no data generated yet
+      if (!(skey < pCtx->startTs && ekey > pCtx->startTs)) {
+        return;
+      }
+
+      assert(pCtx->start.key == INT64_MIN && skey < pCtx->startTs && ekey > pCtx->startTs);
+
+      if (type == TSDB_FILL_PREV) {
+        assignVal(pCtx->pOutput, pCtx->pInput, pCtx->outputBytes, pCtx->inputType);
+      } else if (type == TSDB_FILL_NEXT) {
+        char* val = ((char*)pCtx->pInput) + pCtx->inputBytes;
+        assignVal(pCtx->pOutput, val, pCtx->outputBytes, pCtx->inputType);
+      } else if (type == TSDB_FILL_LINEAR) {
+        char *start = GET_INPUT_DATA(pCtx, 0);
+        char *end = GET_INPUT_DATA(pCtx, 1);
+
+        SPoint point1 = {.key = skey, .val = start};
+        SPoint point2 = {.key = ekey, .val = end};
+        SPoint point = {.key = pCtx->startTs, .val = pCtx->pOutput};
+
+        int32_t srcType = pCtx->inputType;
+        if (IS_NUMERIC_TYPE(srcType)) {  // TODO should find the not null data?
+          if (isNull(start, srcType) || isNull(end, srcType)) {
+            setNull(pCtx->pOutput, srcType, pCtx->inputBytes);
+          } else {
+            taosGetLinearInterpolationVal(&point, pCtx->outputType, &point1, &point2, srcType);
+          }
+        } else {
+          setNull(pCtx->pOutput, srcType, pCtx->inputBytes);
+        }
+      }
+    }
   }
 
   SET_VAL(pCtx, 1, 1);
-
 }
+
 static void interp_function(SQLFunctionCtx *pCtx) {
   // at this point, the value is existed, return directly
   if (pCtx->size > 0) {
@@ -4289,11 +4383,22 @@ static void ts_comp_finalize(SQLFunctionCtx *pCtx) {
   STSBuf *     pTSbuf = pInfo->pTSBuf;
   
   tsBufFlush(pTSbuf);
+  qDebug("total timestamp :%"PRId64, pTSbuf->numOfTotal);
 
+  // TODO refactor transfer ownership of current file
   *(FILE **)pCtx->pOutput = pTSbuf->f;
 
+  pResInfo->complete = true;
+
+  // get the file size
+  struct stat fStat;
+  if ((fstat(fileno(pTSbuf->f), &fStat) == 0)) {
+    pResInfo->numOfRes = fStat.st_size;
+  }
+
   pTSbuf->remainOpen = true;
   tsBufDestroy(pTSbuf);
+
   doFinalizer(pCtx);
 }
 
@@ -4637,10 +4742,126 @@ static void sumrate_finalizer(SQLFunctionCtx *pCtx) {
   doFinalizer(pCtx);
 }
 
+void blockInfo_func(SQLFunctionCtx* pCtx) {
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo);
+
+  int32_t len = *(int32_t*) pCtx->pInput;
+  blockDistInfoFromBinary((char*)pCtx->pInput + sizeof(int32_t), len, pDist);
+  pDist->rowSize = (int16_t) pCtx->param[0].i64;
+
+  memcpy(pCtx->pOutput, pCtx->pInput, sizeof(int32_t) + len);
+
+  pResInfo->numOfRes  = 1;
+  pResInfo->hasResult = DATA_SET_FLAG;
+}
+
+static void mergeTableBlockDist(STableBlockDist* pDist, const STableBlockDist* pSrc) {
+  assert(pDist != NULL && pSrc != NULL);
+  pDist->numOfTables += pSrc->numOfTables;
+  pDist->numOfRowsInMemTable += pSrc->numOfRowsInMemTable;
+  pDist->numOfFiles += pSrc->numOfFiles;
+  pDist->totalSize += pSrc->totalSize;
+
+  if (pDist->dataBlockInfos == NULL) {
+    pDist->dataBlockInfos = taosArrayInit(4, sizeof(SFileBlockInfo));
+  }
+
+  taosArrayPushBatch(pDist->dataBlockInfos, pSrc->dataBlockInfos->pData, (int32_t) taosArrayGetSize(pSrc->dataBlockInfos));
+}
+
+void block_func_merge(SQLFunctionCtx* pCtx) {
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+
+  STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo);
+  STableBlockDist info = {0};
+
+  int32_t len = *(int32_t*) pCtx->pInput;
+  blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info);
+
+  mergeTableBlockDist(pDist, &info);
+}
+
+static int32_t doGetPercentile(const SArray* pArray, double rate) {
+  int32_t len = (int32_t)taosArrayGetSize(pArray);
+  if (len <= 0) {
+    return 0;
+  }
+
+  assert(rate >= 0 && rate <= 1.0);
+  int idx = (int32_t)((len - 1) * rate);
+
+  return ((SFileBlockInfo *)(taosArrayGet(pArray, idx)))->numOfRows;
+}
+
+static int compareBlockInfo(const void *pLeft, const void *pRight) {
+  int32_t left  = ((SFileBlockInfo *)pLeft)->numOfRows;
+  int32_t right = ((SFileBlockInfo *)pRight)->numOfRows;
+
+  if (left > right) return 1;
+  if (left < right) return -1;
+  return 0;
+}
+
+void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) {
+  if (pTableBlockDist == NULL) {
+    return;
+  }
+
+  int64_t min = INT64_MAX, max = INT64_MIN, avg = 0;
+  SArray* blockInfos= pTableBlockDist->dataBlockInfos;
+  int64_t totalRows = 0, totalBlocks = taosArrayGetSize(blockInfos);
+
+  for (size_t i = 0; i < taosArrayGetSize(blockInfos); i++) {
+    SFileBlockInfo *blockInfo = taosArrayGet(blockInfos, i);
+    int64_t rows = blockInfo->numOfRows;
+
+    min = MIN(min, rows);
+    max = MAX(max, rows);
+    totalRows += rows;
+  }
+
+  avg = totalBlocks > 0 ? (int64_t)(totalRows/totalBlocks) : 0;
+  taosArraySort(blockInfos, compareBlockInfo);
+
+  uint64_t totalLen = pTableBlockDist->totalSize;
+  int32_t rowSize = pTableBlockDist->rowSize;
+
+  int sz = sprintf(result + VARSTR_HEADER_SIZE,
+                   "summary: \n\t "
+                   "5th=[%d], 10th=[%d], 20th=[%d], 30th=[%d], 40th=[%d], 50th=[%d]\n\t "
+                   "60th=[%d], 70th=[%d], 80th=[%d], 90th=[%d], 95th=[%d], 99th=[%d]\n\t "
+                   "Min=[%"PRId64"(Rows)] Max=[%"PRId64"(Rows)] Avg=[%"PRId64"(Rows)] Stddev=[%.2f] \n\t "
+                   "Rows=[%"PRId64"], Blocks=[%"PRId64"], Size=[%.3f(Kb)] Comp=[%.2f%%]\n\t "
+                   "RowsInMem=[%d] \n\t SeekHeaderTime=[%d(us)]",
+                   doGetPercentile(blockInfos, 0.05), doGetPercentile(blockInfos, 0.10),
+                   doGetPercentile(blockInfos, 0.20), doGetPercentile(blockInfos, 0.30),
+                   doGetPercentile(blockInfos, 0.40), doGetPercentile(blockInfos, 0.50),
+                   doGetPercentile(blockInfos, 0.60), doGetPercentile(blockInfos, 0.70),
+                   doGetPercentile(blockInfos, 0.80), doGetPercentile(blockInfos, 0.90),
+                   doGetPercentile(blockInfos, 0.95), doGetPercentile(blockInfos, 0.99),
+                   min, max, avg, 0.0,
+                   totalRows, totalBlocks, totalLen/1024.0, (double)(totalLen*100.0)/(rowSize*totalRows),
+                   pTableBlockDist->numOfRowsInMemTable, pTableBlockDist->firstSeekTimeUs);
+  varDataSetLen(result, sz);
+  UNUSED(sz);
+}
+
+void blockinfo_func_finalizer(SQLFunctionCtx* pCtx) {
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo);
+
+  pDist->rowSize = (int16_t)pCtx->param[0].i64;
+  generateBlockDistResult(pDist, pCtx->pOutput);
+
+  // cannot set the numOfIteratedElems again since it is set during previous iteration
+  pResInfo->numOfRes  = 1;
+  pResInfo->hasResult = DATA_SET_FLAG;
+
+  doFinalizer(pCtx);
+}
 
 /////////////////////////////////////////////////////////////////////////////////////////////
-
-
 /*
  * function compatible list.
  * tag and ts are not involved in the compatibility check
@@ -4659,8 +4880,8 @@ int32_t functionCompatList[] = {
     4,         -1,       -1,         1,        1,      1,          1,           1,        1,     -1,
     //  tag,    colprj,   tagprj,    arithmetic, diff, first_dist, last_dist,   interp    rate    irate
     1,          1,        1,         1,       -1,      1,          1,           5,        1,      1,
-    // sum_rate, sum_irate, avg_rate, avg_irate
+    // sum_rate, sum_irate, avg_rate, avg_irate, tid_tag, blk_info
-    1,          1,        1,         1,
+    1,          1,        1,         1,          6,       7
 };
 
 SAggFunctionInfo aAggs[] = {{
@@ -4675,7 +4896,7 @@ SAggFunctionInfo aAggs[] = {{
                               no_next_step,
                               doFinalizer,
                               count_func_merge,
-                              count_load_data_info,
+                              countRequired,
                           },
                           {
                               // 1
@@ -4860,7 +5081,7 @@ SAggFunctionInfo aAggs[] = {{
                               no_next_step,
                               spread_function_finalizer,
                               spread_func_merge,
-                              count_load_data_info,
+                              countRequired,
                           },
                           {
                               // 14
@@ -4902,7 +5123,7 @@ SAggFunctionInfo aAggs[] = {{
                               no_next_step,
                               doFinalizer,
                               copy_function,
-                              no_data_info,
+                              noDataRequired,
                           },
                           {
                               // 17
@@ -4930,7 +5151,7 @@ SAggFunctionInfo aAggs[] = {{
                               no_next_step,
                               doFinalizer,
                               copy_function,
-                              no_data_info,
+                              noDataRequired,
                           },
                           {
                               // 19
@@ -4958,7 +5179,7 @@ SAggFunctionInfo aAggs[] = {{
                               no_next_step,
                               doFinalizer,
                               copy_function,
-                              no_data_info,
+                              noDataRequired,
                           },
                           {
                               // 21, column project sql function
@@ -4986,7 +5207,7 @@ SAggFunctionInfo aAggs[] = {{
                               no_next_step,
                               doFinalizer,
                               copy_function,
-                              no_data_info,
+                              noDataRequired,
                           },
                           {
                               // 23
@@ -5159,7 +5380,7 @@ SAggFunctionInfo aAggs[] = {{
                           },
                           {
                               // 35
-                              "tid_tag",   // return table id and the corresponding tags for join match and subscribe
+                              "tbid",   // return table id and the corresponding tags for join match and subscribe
                               TSDB_FUNC_TID_TAG,
                               TSDB_FUNC_TID_TAG,
                               TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE,
@@ -5170,4 +5391,18 @@ SAggFunctionInfo aAggs[] = {{
                               noop1,
                               noop1,
                               dataBlockRequired,
+                          },
+                          {
+                                // 35
+                                "_block_dist",   // return table id and the corresponding tags for join match and subscribe
+                                TSDB_FUNC_BLKINFO,
+                                TSDB_FUNC_BLKINFO,
+                                TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STABLE,
+                                function_setup,
+                                blockInfo_func,
+                                noop2,
+                                no_next_step,
+                                blockinfo_func_finalizer,
+                                block_func_merge,
+                                dataBlockRequired,
                           }};

src/query/src/qFill.c

@@ -23,18 +23,19 @@
 #include "qFill.h"
 #include "qExtbuffer.h"
 #include "queryLog.h"
+#include "qExecutor.h"
 
 #define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC)
 #define DO_INTERPOLATION(_v1, _v2, _k1, _k2, _k) ((_v1) + ((_v2) - (_v1)) * (((double)(_k)) - ((double)(_k1))) / (((double)(_k2)) - ((double)(_k1))))
 
-static void setTagsValue(SFillInfo* pFillInfo, tFilePage** data, int32_t genRows) {
+static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) {
   for(int32_t j = 0; j < pFillInfo->numOfCols; ++j) {
     SFillColInfo* pCol = &pFillInfo->pFillCol[j];
     if (TSDB_COL_IS_NORMAL_COL(pCol->flag)) {
       continue;
     }
 
-    char* val1 = elePtrAt(data[j]->data, pCol->col.bytes, genRows);
+    char* val1 = elePtrAt(data[j], pCol->col.bytes, genRows);
 
     assert(pCol->tagIndex >= 0 && pCol->tagIndex < pFillInfo->numOfTags);
     SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex];
@@ -44,17 +45,17 @@ static void setTagsValue(SFillInfo* pFillInfo, tFilePage** data, int32_t genRows
   }
 }
 
-static void setNullValueForRow(SFillInfo* pFillInfo, tFilePage** data, int32_t numOfCol, int32_t rowIndex) {
+static void setNullValueForRow(SFillInfo* pFillInfo, void** data, int32_t numOfCol, int32_t rowIndex) {
   // the first are always the timestamp column, so start from the second column.
   for (int32_t i = 1; i < numOfCol; ++i) {
     SFillColInfo* pCol = &pFillInfo->pFillCol[i];
 
-    char* output = elePtrAt(data[i]->data, pCol->col.bytes, rowIndex);
+    char* output = elePtrAt(data[i], pCol->col.bytes, rowIndex);
     setNull(output, pCol->col.type, pCol->col.bytes);
   }
 }
 
-static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** srcData, int64_t ts, bool outOfBound) {
+static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData, int64_t ts, bool outOfBound) {
   char* prev = pFillInfo->prevValues;
   char* next = pFillInfo->nextValues;
 
@@ -63,7 +64,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** sr
 
   // set the primary timestamp column value
   int32_t index = pFillInfo->numOfCurrent;
-  char* val = elePtrAt(data[0]->data, TSDB_KEYSIZE, index);
+  char* val = elePtrAt(data[0], TSDB_KEYSIZE, index);
   *(TSKEY*) val = pFillInfo->currentKey;
 
   // set the other values
@@ -77,7 +78,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** sr
           continue;
         }
 
-        char* output = elePtrAt(data[i]->data, pCol->col.bytes, index);
+        char* output = elePtrAt(data[i], pCol->col.bytes, index);
         assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type);
       }
     } else {  // no prev value yet, set the value for NULL
@@ -93,7 +94,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** sr
           continue;
         }
 
-        char* output = elePtrAt(data[i]->data, pCol->col.bytes, index);
+        char* output = elePtrAt(data[i], pCol->col.bytes, index);
         assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type);
       }
     } else { // no prev value yet, set the value for NULL
@@ -111,7 +112,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** sr
         int16_t type  = pCol->col.type;
         int16_t bytes = pCol->col.bytes;
 
-        char *val1 = elePtrAt(data[i]->data, pCol->col.bytes, index);
+        char *val1 = elePtrAt(data[i], pCol->col.bytes, index);
         if (type == TSDB_DATA_TYPE_BINARY|| type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) {
           setNull(val1, pCol->col.type, bytes);
           continue;
@@ -132,7 +133,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** sr
         continue;
       }
 
-      char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, index);
+      char* val1 = elePtrAt(data[i], pCol->col.bytes, index);
       assignVal(val1, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type);
     }
   }
@@ -162,7 +163,7 @@ static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, char** srcData, char* bu
   }
 }
 
-static int32_t fillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t outputRows) {
+static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputRows) {
   pFillInfo->numOfCurrent = 0;
 
   char** srcData = pFillInfo->pData;
@@ -213,7 +214,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t ou
           continue;
         }
 
-        char* output = elePtrAt(data[i]->data, pCol->col.bytes, pFillInfo->numOfCurrent);
+        char* output = elePtrAt(data[i], pCol->col.bytes, pFillInfo->numOfCurrent);
         char* src = elePtrAt(srcData[i], pCol->col.bytes, pFillInfo->index);
 
         if (i == 0 || (pCol->functionId != TSDB_FUNC_COUNT && !isNull(src, pCol->col.type)) ||
@@ -255,7 +256,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t ou
   return pFillInfo->numOfCurrent;
 }
 
-static int64_t appendFilledResult(SFillInfo* pFillInfo, tFilePage** output, int64_t resultCapacity) {
+static int64_t appendFilledResult(SFillInfo* pFillInfo, void** output, int64_t resultCapacity) {
   /*
    * These data are generated according to fill strategy, since the current timestamp is out of the time window of
    * real result set. Note that we need to keep the direct previous result rows, to generated the filled data.
@@ -278,7 +279,7 @@ static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t
   int32_t k = 0;
   for (int32_t i = 0; i < numOfCols; ++i) {
     SFillColInfo* pColInfo = &pFillInfo->pFillCol[i];
-    pFillInfo->pData[i] = calloc(1, pColInfo->col.bytes * capacity);
+    pFillInfo->pData[i] = NULL;
 
     if (TSDB_COL_IS_TAG(pColInfo->flag)) {
       bool exists = false;
@@ -356,6 +357,10 @@ SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int3
   pFillInfo->rowSize = setTagColumnInfo(pFillInfo, pFillInfo->numOfCols, pFillInfo->alloc);
   assert(pFillInfo->rowSize > 0);
 
+  for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
+    pFillInfo->pData[i] = malloc(pFillInfo->pFillCol[i].col.bytes * pFillInfo->alloc);
+  }
+
   return pFillInfo;
 }
 
@@ -375,12 +380,17 @@ void* taosDestroyFillInfo(SFillInfo* pFillInfo) {
 
   tfree(pFillInfo->prevValues);
   tfree(pFillInfo->nextValues);
-  tfree(pFillInfo->pTags);
+
+  for(int32_t i = 0; i < pFillInfo->numOfTags; ++i) {
+    tfree(pFillInfo->pTags[i].tagVal);
+  }
 
   for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
     tfree(pFillInfo->pData[i]);
   }
 
+  tfree(pFillInfo->pTags);
+  
   tfree(pFillInfo->pData);
   tfree(pFillInfo->pFillCol);
   
@@ -413,10 +423,19 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey)
   }
 }
 
-// copy the data into source data buffer
+void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput) {
-void taosFillSetDataBlockFromFilePage(SFillInfo* pFillInfo, const tFilePage** pInput) {
   for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
-    memcpy(pFillInfo->pData[i], pInput[i]->data, pFillInfo->numOfRows * pFillInfo->pFillCol[i].col.bytes);
+    SColumnInfoData* pColData = taosArrayGet(pInput->pDataBlock, i);
+//    pFillInfo->pData[i] = pColData->pData;
+    if (pInput->info.rows > pFillInfo->alloc) {
+      char* t = realloc(pFillInfo->pData[i], pColData->info.bytes * pInput->info.rows);
+      assert(t != NULL);
+
+      pFillInfo->pData[i] = t;
+      pFillInfo->alloc = pInput->info.rows;
+    }
+
+    memcpy(pFillInfo->pData[i], pColData->pData, pColData->info.bytes * pInput->info.rows);
   }
 }
 
@@ -427,12 +446,20 @@ void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, const tFilePage*
     SFillColInfo* pCol = &pFillInfo->pFillCol[i];
 
     const char* data = pInput->data + pCol->col.offset * pInput->num;
-    memcpy(pFillInfo->pData[i], data, (size_t)(pInput->num * pCol->col.bytes));
+    if (pInput->num > pFillInfo->alloc) {
+      char* t = realloc(pFillInfo->pData[i], (size_t)(pCol->col.bytes * pInput->num));
+      assert(t != NULL);
+
+      pFillInfo->pData[i] = t;
+      pFillInfo->alloc = (int32_t)pInput->num;
+    }
+
+    memcpy(pFillInfo->pData[i], data, (size_t)(pCol->col.bytes * pInput->num));
 
     if (TSDB_COL_IS_TAG(pCol->flag)) {  // copy the tag value to tag value buffer
       SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex];
       assert (pTag->col.colId == pCol->col.colId);
-      memcpy(pTag->tagVal, data, pCol->col.bytes);
+      memcpy(pTag->tagVal, data, pCol->col.bytes);  // TODO not memcpy??
     }
   }
 }
@@ -490,7 +517,7 @@ int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint*
   return TSDB_CODE_SUCCESS;
 }
 
-int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity) {
+int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity) {
   int32_t remain = taosNumOfRemainRows(pFillInfo);
 
   int64_t numOfRes = getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, capacity);

src/query/src/qParserImpl.c

@@ -135,28 +135,33 @@ tSQLExpr *tSqlExprIdValueCreate(SStrToken *pToken, int32_t optrType) {
   if (optrType == TK_INTEGER || optrType == TK_STRING || optrType == TK_FLOAT || optrType == TK_BOOL) {
     toTSDBType(pToken->type);
 
-    tVariantCreate(&pSqlExpr->val, pToken);
+    tVariantCreate(&pSqlExpr->value, pToken);
-    pSqlExpr->nSQLOptr = optrType;
+    pSqlExpr->tokenId = optrType;
+    pSqlExpr->type    = SQL_NODE_VALUE;
   } else if (optrType == TK_NOW) {
     // use microsecond by default
-    pSqlExpr->val.i64 = taosGetTimestamp(TSDB_TIME_PRECISION_MICRO);
+    pSqlExpr->value.i64 = taosGetTimestamp(TSDB_TIME_PRECISION_MICRO);
-    pSqlExpr->val.nType = TSDB_DATA_TYPE_BIGINT;
+    pSqlExpr->value.nType = TSDB_DATA_TYPE_BIGINT;
-    pSqlExpr->nSQLOptr = TK_TIMESTAMP;  // TK_TIMESTAMP used to denote the time value is in microsecond
+    pSqlExpr->tokenId = TK_TIMESTAMP;  // TK_TIMESTAMP used to denote the time value is in microsecond
+    pSqlExpr->type    = SQL_NODE_VALUE;
   } else if (optrType == TK_VARIABLE) {
-    int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSqlExpr->val.i64);
+    int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSqlExpr->value.i64);
     if (ret != TSDB_CODE_SUCCESS) {
       terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
     }
 
-    pSqlExpr->val.nType = TSDB_DATA_TYPE_BIGINT;
+    pSqlExpr->value.nType = TSDB_DATA_TYPE_BIGINT;
-    pSqlExpr->nSQLOptr = TK_TIMESTAMP;
+    pSqlExpr->tokenId = TK_TIMESTAMP;
-  } else {  // it must be the column name (tk_id) if it is not the number
+    pSqlExpr->type    = SQL_NODE_VALUE;
+  } else {
+    // Here it must be the column name (tk_id) if it is not a number or string.
     assert(optrType == TK_ID || optrType == TK_ALL);
     if (pToken != NULL) {
       pSqlExpr->colInfo = *pToken;
     }
 
-    pSqlExpr->nSQLOptr = optrType;
+    pSqlExpr->tokenId = optrType;
+    pSqlExpr->type    = SQL_NODE_TABLE_COLUMN;
   }
 
   return pSqlExpr;
@@ -167,19 +172,22 @@ tSQLExpr *tSqlExprIdValueCreate(SStrToken *pToken, int32_t optrType) {
  * function name is denoted by pFunctionToken
  */
 tSQLExpr *tSqlExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType) {
-  if (pFuncToken == NULL) return NULL;
+  if (pFuncToken == NULL) {
+    return NULL;
+  }
 
   tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr));
-  pExpr->nSQLOptr = optType;
+  pExpr->tokenId = optType;
+  pExpr->type    = SQL_NODE_SQLFUNCTION;
   pExpr->pParam  = pList;
 
   int32_t len = (int32_t)((endToken->z + endToken->n) - pFuncToken->z);
-  pExpr->operand.z = pFuncToken->z;
+  pExpr->operand = (*pFuncToken);
 
-  pExpr->operand.n = len;  // raw field name
+  pExpr->token.n = len;
-  pExpr->operand.type = pFuncToken->type;
+  pExpr->token.z = pFuncToken->z;
+  pExpr->token.type = pFuncToken->type;
 
-  pExpr->token = pExpr->operand;
   return pExpr;
 }
 
@@ -190,6 +198,7 @@ tSQLExpr *tSqlExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SSt
 tSQLExpr *tSqlExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) {
   tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr));
 
+  pExpr->type = SQL_NODE_EXPR;
   if (pLeft != NULL && pRight != NULL && (optrType != TK_IN)) {
     char* endPos = pRight->token.z + pRight->token.n;
     pExpr->token.z = pLeft->token.z;
@@ -203,32 +212,33 @@ tSQLExpr *tSqlExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) {
      * if a token is noted as the TK_TIMESTAMP, the time precision is microsecond
      * Otherwise, the time precision is adaptive, determined by the time precision from databases.
      */
-    if ((pLeft->nSQLOptr == TK_INTEGER && pRight->nSQLOptr == TK_INTEGER) ||
+    if ((pLeft->tokenId == TK_INTEGER && pRight->tokenId == TK_INTEGER) ||
-        (pLeft->nSQLOptr == TK_TIMESTAMP && pRight->nSQLOptr == TK_TIMESTAMP)) {
+        (pLeft->tokenId == TK_TIMESTAMP && pRight->tokenId == TK_TIMESTAMP)) {
-      pExpr->val.nType = TSDB_DATA_TYPE_BIGINT;
+      pExpr->value.nType = TSDB_DATA_TYPE_BIGINT;
-      pExpr->nSQLOptr = pLeft->nSQLOptr;
+      pExpr->tokenId = pLeft->tokenId;
+      pExpr->type    = SQL_NODE_VALUE;
 
       switch (optrType) {
         case TK_PLUS: {
-          pExpr->val.i64 = pLeft->val.i64 + pRight->val.i64;
+          pExpr->value.i64 = pLeft->value.i64 + pRight->value.i64;
           break;
         }
         case TK_MINUS: {
-          pExpr->val.i64 = pLeft->val.i64 - pRight->val.i64;
+          pExpr->value.i64 = pLeft->value.i64 - pRight->value.i64;
           break;
         }
         case TK_STAR: {
-          pExpr->val.i64 = pLeft->val.i64 * pRight->val.i64;
+          pExpr->value.i64 = pLeft->value.i64 * pRight->value.i64;
           break;
         }
         case TK_DIVIDE: {
-          pExpr->nSQLOptr = TK_FLOAT;
+          pExpr->tokenId = TK_FLOAT;
-          pExpr->val.nType = TSDB_DATA_TYPE_DOUBLE;
+          pExpr->value.nType = TSDB_DATA_TYPE_DOUBLE;
-          pExpr->val.dKey = (double)pLeft->val.i64 / pRight->val.i64;
+          pExpr->value.dKey = (double)pLeft->value.i64 / pRight->value.i64;
           break;
         }
         case TK_REM: {
-          pExpr->val.i64 = pLeft->val.i64 % pRight->val.i64;
+          pExpr->value.i64 = pLeft->value.i64 % pRight->value.i64;
           break;
         }
       }
@@ -236,33 +246,35 @@ tSQLExpr *tSqlExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) {
       tSqlExprDestroy(pLeft);
       tSqlExprDestroy(pRight);
 
-    } else if ((pLeft->nSQLOptr == TK_FLOAT && pRight->nSQLOptr == TK_INTEGER) || (pLeft->nSQLOptr == TK_INTEGER && pRight->nSQLOptr == TK_FLOAT) ||
+    } else if ((pLeft->tokenId == TK_FLOAT && pRight->tokenId == TK_INTEGER) ||
-        (pLeft->nSQLOptr == TK_FLOAT && pRight->nSQLOptr == TK_FLOAT)) {
+               (pLeft->tokenId == TK_INTEGER && pRight->tokenId == TK_FLOAT) ||
-      pExpr->val.nType = TSDB_DATA_TYPE_DOUBLE;
+               (pLeft->tokenId == TK_FLOAT && pRight->tokenId == TK_FLOAT)) {
-      pExpr->nSQLOptr  = TK_FLOAT;
+      pExpr->value.nType = TSDB_DATA_TYPE_DOUBLE;
+      pExpr->tokenId  = TK_FLOAT;
+      pExpr->type     = SQL_NODE_VALUE;
 
-      double left  = (pLeft->val.nType == TSDB_DATA_TYPE_DOUBLE) ? pLeft->val.dKey : pLeft->val.i64;
+      double left  = (pLeft->value.nType == TSDB_DATA_TYPE_DOUBLE) ? pLeft->value.dKey : pLeft->value.i64;
-      double right = (pRight->val.nType == TSDB_DATA_TYPE_DOUBLE) ? pRight->val.dKey : pRight->val.i64;
+      double right = (pRight->value.nType == TSDB_DATA_TYPE_DOUBLE) ? pRight->value.dKey : pRight->value.i64;
 
       switch (optrType) {
         case TK_PLUS: {
-          pExpr->val.dKey = left + right;
+          pExpr->value.dKey = left + right;
           break;
         }
         case TK_MINUS: {
-          pExpr->val.dKey = left - right;
+          pExpr->value.dKey = left - right;
           break;
         }
         case TK_STAR: {
-          pExpr->val.dKey = left * right;
+          pExpr->value.dKey = left * right;
           break;
         }
         case TK_DIVIDE: {
-          pExpr->val.dKey = left / right;
+          pExpr->value.dKey = left / right;
           break;
         }
         case TK_REM: {
-          pExpr->val.dKey = left - ((int64_t)(left / right)) * right;
+          pExpr->value.dKey = left - ((int64_t)(left / right)) * right;
           break;
         }
       }
@@ -271,21 +283,21 @@ tSQLExpr *tSqlExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) {
       tSqlExprDestroy(pRight);
 
     } else {
-      pExpr->nSQLOptr = optrType;
+      pExpr->tokenId = optrType;
       pExpr->pLeft = pLeft;
       pExpr->pRight = pRight;
     }
   } else if (optrType == TK_IN) {
-    pExpr->nSQLOptr = optrType;
+    pExpr->tokenId = optrType;
     pExpr->pLeft = pLeft;
 
     tSQLExpr *pRSub = calloc(1, sizeof(tSQLExpr));
-    pRSub->nSQLOptr = TK_SET;  // TODO refactor .....
+    pRSub->tokenId = TK_SET;  // TODO refactor .....
     pRSub->pParam = (tSQLExprList *)pRight;
 
     pExpr->pRight = pRSub;
   } else {
-    pExpr->nSQLOptr = optrType;
+    pExpr->tokenId = optrType;
     pExpr->pLeft = pLeft;
 
     if (pLeft != NULL && pRight == NULL) {
@@ -325,8 +337,8 @@ void tSqlExprNodeDestroy(tSQLExpr *pExpr) {
     return;
   }
 
-  if (pExpr->nSQLOptr == TK_STRING) {
+  if (pExpr->tokenId == TK_STRING) {
-    tVariantDestroy(&pExpr->val);
+    tVariantDestroy(&pExpr->value);
   }
 
   tSqlExprListDestroy(pExpr->pParam);
@@ -538,8 +550,8 @@ void tSqlSetColumnType(TAOS_FIELD *pField, SStrToken *type) {
 /*
  * extract the select info out of sql string
  */
-SQuerySQL *tSetQuerySqlElems(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere,
+SQuerySQL *tSetQuerySqlNode(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere,
-                             SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval,
+                             SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, SSessionWindowVal *pSession,
                              SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) {
   assert(pSelection != NULL);
 
@@ -562,14 +574,17 @@ SQuerySQL *tSetQuerySqlElems(SStrToken *pSelectToken, tSQLExprList *pSelection,
   }
 
   if (pInterval != NULL) {
-    pQuery->interval = pInterval->interval;
+    pQuery->interval = *pInterval;
-    pQuery->offset = pInterval->offset;
   }
 
   if (pSliding != NULL) {
     pQuery->sliding = *pSliding;
   }
 
+  if (pSession != NULL) {
+    pQuery->sessionVal = *pSession;
+  }
+
   pQuery->fillType = pFill;
   return pQuery;
 }
@@ -805,7 +820,7 @@ void setDCLSQLElems(SSqlInfo *pInfo, int32_t type, int32_t nParam, ...) {
   va_end(va);
 }
 
-void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck, int16_t tableType) {
+void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck, int16_t dbType, int16_t tableType) {
   pInfo->type = type;
 
   if (pInfo->pMiscInfo == NULL) {
@@ -816,6 +831,7 @@ void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrTo
   taosArrayPush(pInfo->pMiscInfo->a, pToken);
 
   pInfo->pMiscInfo->existsCheck = (existsCheck->n == 1);
+  pInfo->pMiscInfo->dbType = dbType;
   pInfo->pMiscInfo->tableType = tableType;
 }
 
@@ -935,6 +951,17 @@ void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo) {
   pDBInfo->keep = NULL;
 
   pDBInfo->update = -1;
-  pDBInfo->cachelast = 0;
+  pDBInfo->cachelast = -1;
+
+  pDBInfo->dbType = -1;
+  pDBInfo->partitions = -1;
+  
   memset(&pDBInfo->precision, 0, sizeof(SStrToken));
 }
+
+void setDefaultCreateTopicOption(SCreateDbInfo *pDBInfo) {
+  setDefaultCreateDbOption(pDBInfo);
+
+  pDBInfo->dbType = TSDB_DB_TYPE_TOPIC;
+  pDBInfo->partitions = TSDB_DEFAULT_DB_PARTITON_OPTION;
+}

src/query/src/qPercentile.c

@@ -254,7 +254,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval,
 
   resetSlotInfo(pBucket);
 
-  int32_t ret = createDiskbasedResultBuffer(&pBucket->pBuffer, pBucket->bytes, pBucket->bufPageSize, pBucket->bufPageSize * 512, NULL);
+  int32_t ret = createDiskbasedResultBuffer(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, NULL);
   if (ret != TSDB_CODE_SUCCESS) {
     tMemBucketDestroy(pBucket);
     return NULL;

src/query/src/qResultbuf.c

@@ -9,8 +9,7 @@
 #define GET_DATA_PAYLOAD(_p) ((char *)(_p)->pData + POINTER_BYTES)
 #define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages)
 
-int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t rowSize, int32_t pagesize,
+int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, const void* handle) {
-                                    int32_t inMemBufSize, const void* handle) {
   *pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf));
 
   SDiskbasedResultBuf* pResBuf = *pResultBuf;
@@ -31,7 +30,6 @@ int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t ro
   // at least more than 2 pages must be in memory
   assert(inMemBufSize >= pagesize * 2);
 
-  pResBuf->numOfRowsPerPage = (pagesize - sizeof(tFilePage)) / rowSize;
   pResBuf->lruList = tdListNew(POINTER_BYTES);
 
   // init id hash table
@@ -387,8 +385,6 @@ void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi) {
   pResultBuf->statis.releasePages += 1;
 }
 
-size_t getNumOfRowsPerPage(const SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; }
-
 size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->groupSet); }
 
 size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf) { return (size_t)pResultBuf->totalBufSize; }

src/query/src/qTokenizer.c

@@ -139,6 +139,7 @@ static SKeyword keywordTable[] = {
     {"FROM",         TK_FROM},
     {"VARIABLE",     TK_VARIABLE},
     {"INTERVAL",     TK_INTERVAL},
+    {"SESSION",      TK_SESSION},
     {"FILL",         TK_FILL},
     {"SLIDING",      TK_SLIDING},
     {"ORDER",        TK_ORDER},
@@ -200,25 +201,6 @@ static SKeyword keywordTable[] = {
     {"TRIGGER",      TK_TRIGGER},
     {"VIEW",         TK_VIEW},
     {"ALL",          TK_ALL},
-    {"COUNT",        TK_COUNT},
-    {"SUM",          TK_SUM},
-    {"AVG",          TK_AVG},
-    {"MIN",          TK_MIN},
-    {"MAX",          TK_MAX},
-    {"FIRST",        TK_FIRST},
-    {"LAST",         TK_LAST},
-    {"TOP",          TK_TOP},
-    {"BOTTOM",       TK_BOTTOM},
-    {"STDDEV",       TK_STDDEV},
-    {"PERCENTILE",   TK_PERCENTILE},
-    {"APERCENTILE",  TK_APERCENTILE},
-    {"LEASTSQUARES", TK_LEASTSQUARES},
-    {"HISTOGRAM",    TK_HISTOGRAM},
-    {"DIFF",         TK_DIFF},
-    {"SPREAD",       TK_SPREAD},
-    {"TWA",          TK_TWA},
-    {"INTERP",       TK_INTERP},
-    {"LAST_ROW",     TK_LAST_ROW},
     {"SEMI",         TK_SEMI},
     {"NONE",         TK_NONE},
     {"PREV",         TK_PREV},
@@ -228,19 +210,15 @@ static SKeyword keywordTable[] = {
     {"TBNAME",       TK_TBNAME},
     {"JOIN",         TK_JOIN},
     {"METRICS",      TK_METRICS},
-    {"TBID",         TK_TBID},
     {"STABLE",       TK_STABLE},
     {"FILE",         TK_FILE},
     {"VNODES",       TK_VNODES},
     {"UNION",        TK_UNION},
-    {"RATE",         TK_RATE},
-    {"IRATE",        TK_IRATE},
-    {"SUM_RATE",     TK_SUM_RATE},
-    {"SUM_IRATE",    TK_SUM_IRATE},
-    {"AVG_RATE",     TK_AVG_RATE},
-    {"AVG_IRATE",    TK_AVG_IRATE},
     {"CACHELAST",    TK_CACHELAST},
     {"DISTINCT",     TK_DISTINCT},
+    {"PARTITIONS",   TK_PARTITIONS},
+    {"TOPIC",        TK_TOPIC},
+    {"TOPICS",       TK_TOPICS}
 };
 
 static const char isIdChar[] = {
@@ -294,7 +272,7 @@ int tSQLKeywordCode(const char* z, int n) {
  * Return the length of the token that begins at z[0].
  * Store the token type in *type before returning.
  */
-uint32_t tSQLGetToken(char* z, uint32_t* tokenType) {
+uint32_t tSQLGetToken(char* z, uint32_t* tokenId) {
   uint32_t i;
   switch (*z) {
     case ' ':
@@ -304,121 +282,121 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) {
     case '\r': {
       for (i = 1; isspace(z[i]); i++) {
       }
-      *tokenType = TK_SPACE;
+      *tokenId = TK_SPACE;
       return i;
     }
     case ':': {
-      *tokenType = TK_COLON;
+      *tokenId = TK_COLON;
       return 1;
     }
     case '-': {
       if (z[1] == '-') {
         for (i = 2; z[i] && z[i] != '\n'; i++) {
         }
-        *tokenType = TK_COMMENT;
+        *tokenId = TK_COMMENT;
         return i;
       }
-      *tokenType = TK_MINUS;
+      *tokenId = TK_MINUS;
       return 1;
     }
     case '(': {
-      *tokenType = TK_LP;
+      *tokenId = TK_LP;
       return 1;
     }
     case ')': {
-      *tokenType = TK_RP;
+      *tokenId = TK_RP;
       return 1;
     }
     case ';': {
-      *tokenType = TK_SEMI;
+      *tokenId = TK_SEMI;
       return 1;
     }
     case '+': {
-      *tokenType = TK_PLUS;
+      *tokenId = TK_PLUS;
       return 1;
     }
     case '*': {
-      *tokenType = TK_STAR;
+      *tokenId = TK_STAR;
       return 1;
     }
     case '/': {
       if (z[1] != '*' || z[2] == 0) {
-        *tokenType = TK_SLASH;
+        *tokenId = TK_SLASH;
         return 1;
       }
       for (i = 3; z[i] && (z[i] != '/' || z[i - 1] != '*'); i++) {
       }
       if (z[i]) i++;
-      *tokenType = TK_COMMENT;
+      *tokenId = TK_COMMENT;
       return i;
     }
     case '%': {
-      *tokenType = TK_REM;
+      *tokenId = TK_REM;
       return 1;
     }
     case '=': {
-      *tokenType = TK_EQ;
+      *tokenId = TK_EQ;
       return 1 + (z[1] == '=');
     }
     case '<': {
       if (z[1] == '=') {
-        *tokenType = TK_LE;
+        *tokenId = TK_LE;
         return 2;
       } else if (z[1] == '>') {
-        *tokenType = TK_NE;
+        *tokenId = TK_NE;
         return 2;
       } else if (z[1] == '<') {
-        *tokenType = TK_LSHIFT;
+        *tokenId = TK_LSHIFT;
         return 2;
       } else {
-        *tokenType = TK_LT;
+        *tokenId = TK_LT;
         return 1;
       }
     }
     case '>': {
       if (z[1] == '=') {
-        *tokenType = TK_GE;
+        *tokenId = TK_GE;
         return 2;
       } else if (z[1] == '>') {
-        *tokenType = TK_RSHIFT;
+        *tokenId = TK_RSHIFT;
         return 2;
       } else {
-        *tokenType = TK_GT;
+        *tokenId = TK_GT;
         return 1;
       }
     }
     case '!': {
       if (z[1] != '=') {
-        *tokenType = TK_ILLEGAL;
+        *tokenId = TK_ILLEGAL;
         return 2;
       } else {
-        *tokenType = TK_NE;
+        *tokenId = TK_NE;
         return 2;
       }
     }
     case '|': {
       if (z[1] != '|') {
-        *tokenType = TK_BITOR;
+        *tokenId = TK_BITOR;
         return 1;
       } else {
-        *tokenType = TK_CONCAT;
+        *tokenId = TK_CONCAT;
         return 2;
       }
     }
     case ',': {
-      *tokenType = TK_COMMA;
+      *tokenId = TK_COMMA;
       return 1;
     }
     case '&': {
-      *tokenType = TK_BITAND;
+      *tokenId = TK_BITAND;
       return 1;
     }
     case '~': {
-      *tokenType = TK_BITNOT;
+      *tokenId = TK_BITNOT;
       return 1;
     }
     case '?': {
-      *tokenType = TK_QUESTION;
+      *tokenId = TK_QUESTION;
       return 1;
     }
     case '\'':
@@ -444,7 +422,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) {
       if (z[i]) i++;
 
       if (strEnd) {
-        *tokenType = TK_STRING;
+        *tokenId = TK_STRING;
         return i;
       }
 
@@ -468,10 +446,10 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) {
           }
         }
 
-        *tokenType = TK_FLOAT;
+        *tokenId = TK_FLOAT;
         return i;
       } else {
-        *tokenType = TK_DOT;
+        *tokenId = TK_DOT;
         return 1;
       }
     }
@@ -480,7 +458,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) {
       char next = z[1];
 
       if (next == 'b') { // bin number
-        *tokenType = TK_BIN;
+        *tokenId = TK_BIN;
         for (i = 2; (z[i] == '0' || z[i] == '1'); ++i) {
         }
 
@@ -490,7 +468,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) {
 
         return i;
       } else if (next == 'x') {  //hex number
-        *tokenType = TK_HEX;
+        *tokenId = TK_HEX;
         for (i = 2; isdigit(z[i]) || (z[i] >= 'a' && z[i] <= 'f') || (z[i] >= 'A' && z[i] <= 'F'); ++i) {
         }
 
@@ -510,7 +488,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) {
     case '7':
     case '8':
     case '9': {
-      *tokenType = TK_INTEGER;
+      *tokenId = TK_INTEGER;
       for (i = 1; isdigit(z[i]); i++) {
       }
 
@@ -520,7 +498,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) {
            z[i] == 'U' || z[i] == 'A' || z[i] == 'S' || z[i] == 'M' || z[i] == 'H' || z[i] == 'D' || z[i] == 'N' ||
            z[i] == 'Y' || z[i] == 'W') &&
           (isIdChar[(uint8_t)z[i + 1]] == 0)) {
-        *tokenType = TK_VARIABLE;
+        *tokenId = TK_VARIABLE;
         i += 1;
         return i;
       }
@@ -531,12 +509,12 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) {
         while (isdigit(z[i])) {
           i++;
         }
-        *tokenType = TK_FLOAT;
+        *tokenId = TK_FLOAT;
         seg++;
       }
 
       if (seg == 4) {  // ip address
-        *tokenType = TK_IPTOKEN;
+        *tokenId = TK_IPTOKEN;
         return i;
       }
 
@@ -546,14 +524,14 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) {
         while (isdigit(z[i])) {
           i++;
         }
-        *tokenType = TK_FLOAT;
+        *tokenId = TK_FLOAT;
       }
       return i;
     }
     case '[': {
       for (i = 1; z[i] && z[i - 1] != ']'; i++) {
       }
-      *tokenType = TK_ID;
+      *tokenId = TK_ID;
       return i;
     }
     case 'T':
@@ -564,7 +542,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) {
       }
 
       if ((i == 4 && strncasecmp(z, "true", 4) == 0) || (i == 5 && strncasecmp(z, "false", 5) == 0)) {
-        *tokenType = TK_BOOL;
+        *tokenId = TK_BOOL;
         return i;
       }
     }
@@ -574,12 +552,12 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) {
       }
       for (i = 1; ((z[i] & 0x80) == 0) && isIdChar[(uint8_t) z[i]]; i++) {
       }
-      *tokenType = tSQLKeywordCode(z, i);
+      *tokenId = tSQLKeywordCode(z, i);
       return i;
     }
   }
 
-  *tokenType = TK_ILLEGAL;
+  *tokenId = TK_ILLEGAL;
   return 0;
 }
 

src/query/src/qUtil.c

@@ -22,6 +22,7 @@
 #include "tbuffer.h"
 #include "tlosertree.h"
 #include "queryLog.h"
+#include "tscompression.h"
 
 typedef struct SCompSupporter {
   STableQueryInfo **pTableQueryInfo;
@@ -41,12 +42,11 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) {
 }
 
 int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size, int16_t type) {
-  pResultRowInfo->capacity = size;
-
   pResultRowInfo->type     = type;
-  pResultRowInfo->curIndex = -1;
   pResultRowInfo->size     = 0;
   pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL;
+  pResultRowInfo->curIndex = -1;
+  pResultRowInfo->capacity = size;
 
   pResultRowInfo->pResult = calloc(pResultRowInfo->capacity, POINTER_BYTES);
   if (pResultRowInfo->pResult == NULL) {
@@ -135,20 +135,22 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16
   if (pResultRow->pageId >= 0) {
     tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResultRow->pageId);
 
+    int16_t offset = 0;
     for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) {
       SResultRowCellInfo *pResultInfo = &pResultRow->pCellInfo[i];
 
-      char * s = getPosInResultPage(pRuntimeEnv, i, pResultRow, page);
+      int16_t size = pRuntimeEnv->pQuery->pExpr1[i].bytes;
-      size_t size = pRuntimeEnv->pQuery->pExpr1[i].bytes;
+      char * s = getPosInResultPage(pRuntimeEnv->pQuery, page, pResultRow->offset, offset);
       memset(s, 0, size);
 
+      offset += size;
       RESET_RESULT_INFO(pResultInfo);
     }
   }
 
   pResultRow->numOfRows = 0;
   pResultRow->pageId = -1;
-  pResultRow->rowId = -1;
+  pResultRow->offset = -1;
   pResultRow->closed = false;
 
   if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
@@ -158,13 +160,15 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16
   }
 }
 
-SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index) {
+// TODO refactor: use macro
-  assert(index >= 0 && index < pRuntimeEnv->pQuery->numOfOutput);
+SResultRowCellInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset) {
-  return (SResultRowCellInfo*)((char*) pRow->pCellInfo + pRuntimeEnv->rowCellInfoOffset[index]);
+  assert(index >= 0 && offset != NULL);
+  return (SResultRowCellInfo*)((char*) pRow->pCellInfo + offset[index]);
 }
 
 size_t getResultRowSize(SQueryRuntimeEnv* pRuntimeEnv) {
-  return (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultRowCellInfo)) + pRuntimeEnv->interBufSize + sizeof(SResultRow);
+  SQuery* pQuery = pRuntimeEnv->pQuery;
+  return (pQuery->numOfOutput * sizeof(SResultRowCellInfo)) + pQuery->interBufSize + sizeof(SResultRow);
 }
 
 SResultRowPool* initResultRowPool(size_t size) {
@@ -340,18 +344,18 @@ void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) {
   pGroupResInfo->index     = 0;
 }
 
-void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo, int32_t offset) {
+void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo) {
   if (pGroupResInfo->pRows != NULL) {
     taosArrayDestroy(pGroupResInfo->pRows);
   }
 
   pGroupResInfo->pRows = taosArrayFromList(pResultInfo->pResult, pResultInfo->size, POINTER_BYTES);
-  pGroupResInfo->index = offset;
+  pGroupResInfo->index = 0;
 
   assert(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo));
 }
 
-bool hasRemainData(SGroupResInfo* pGroupResInfo) {
+bool hasRemainDataInCurrentGroup(SGroupResInfo* pGroupResInfo) {
   if (pGroupResInfo->pRows == NULL) {
     return false;
   }
@@ -359,6 +363,14 @@ bool hasRemainData(SGroupResInfo* pGroupResInfo) {
   return pGroupResInfo->index < taosArrayGetSize(pGroupResInfo->pRows);
 }
 
+bool hasRemainData(SGroupResInfo* pGroupResInfo) {
+  if (hasRemainDataInCurrentGroup(pGroupResInfo)) {
+    return true;
+  }
+
+  return pGroupResInfo->currentGroup < pGroupResInfo->totalGroup;
+}
+
 bool incNextGroup(SGroupResInfo* pGroupResInfo) {
   return (++pGroupResInfo->currentGroup) < pGroupResInfo->totalGroup;
 }
@@ -372,7 +384,7 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) {
   return (int32_t) taosArrayGetSize(pGroupResInfo->pRows);
 }
 
-static int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow) {
+static int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow, int32_t* rowCellInfoOffset) {
   SQuery* pQuery = pRuntimeEnv->pQuery;
 
   for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
@@ -386,7 +398,7 @@ static int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow
       continue;
     }
 
-    SResultRowCellInfo *pResultInfo = getResultCell(pRuntimeEnv, pResultRow, j);
+    SResultRowCellInfo *pResultInfo = getResultCell(pResultRow, j, rowCellInfoOffset);
     assert(pResultInfo != NULL);
 
     if (pResultInfo->numOfRes > 0) {
@@ -437,7 +449,8 @@ static int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *
   }
 }
 
-static int32_t mergeIntoGroupResultImpl(SQueryRuntimeEnv *pRuntimeEnv, SGroupResInfo* pGroupResInfo, SArray *pTableList, void* qinfo) {
+static int32_t mergeIntoGroupResultImpl(SQueryRuntimeEnv *pRuntimeEnv, SGroupResInfo* pGroupResInfo, SArray *pTableList,
+    int32_t* rowCellInfoOffset) {
   bool ascQuery = QUERY_IS_ASC_QUERY(pRuntimeEnv->pQuery);
 
   int32_t code = TSDB_CODE_SUCCESS;
@@ -455,7 +468,7 @@ static int32_t mergeIntoGroupResultImpl(SQueryRuntimeEnv *pRuntimeEnv, SGroupRes
   pTableQueryInfoList = malloc(POINTER_BYTES * size);
 
   if (pTableQueryInfoList == NULL || posList == NULL || pGroupResInfo->pRows == NULL || pGroupResInfo->pRows == NULL) {
-    qError("QInfo:%p failed alloc memory", qinfo);
+    qError("QInfo:%p failed alloc memory", pRuntimeEnv->qinfo);
     code = TSDB_CODE_QRY_OUT_OF_MEMORY;
     goto _end;
   }
@@ -491,7 +504,7 @@ static int32_t mergeIntoGroupResultImpl(SQueryRuntimeEnv *pRuntimeEnv, SGroupRes
     SResultRowInfo *pWindowResInfo = &pTableQueryInfoList[tableIndex]->resInfo;
     SResultRow  *pWindowRes = getResultRow(pWindowResInfo, cs.rowIndex[tableIndex]);
 
-    int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes);
+    int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes, rowCellInfoOffset);
     if (num <= 0) {
       cs.rowIndex[tableIndex] += 1;
 
@@ -527,7 +540,7 @@ static int32_t mergeIntoGroupResultImpl(SQueryRuntimeEnv *pRuntimeEnv, SGroupRes
 
   int64_t endt = taosGetTimestampMs();
 
-  qDebug("QInfo:%p result merge completed for group:%d, elapsed time:%" PRId64 " ms", qinfo,
+  qDebug("QInfo:%p result merge completed for group:%d, elapsed time:%" PRId64 " ms", pRuntimeEnv->qinfo,
          pGroupResInfo->currentGroup, endt - startt);
 
   _end:
@@ -538,13 +551,13 @@ static int32_t mergeIntoGroupResultImpl(SQueryRuntimeEnv *pRuntimeEnv, SGroupRes
   return code;
 }
 
-int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQInfo *pQInfo) {
+int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQueryRuntimeEnv* pRuntimeEnv, int32_t* offset) {
   int64_t st = taosGetTimestampUs();
 
   while (pGroupResInfo->currentGroup < pGroupResInfo->totalGroup) {
-    SArray *group = GET_TABLEGROUP(pQInfo, pGroupResInfo->currentGroup);
+    SArray *group = GET_TABLEGROUP(pRuntimeEnv, pGroupResInfo->currentGroup);
 
-    int32_t ret = mergeIntoGroupResultImpl(&pQInfo->runtimeEnv, pGroupResInfo, group, pQInfo);
+    int32_t ret = mergeIntoGroupResultImpl(pRuntimeEnv, pGroupResInfo, group, offset);
     if (ret != TSDB_CODE_SUCCESS) {
       return ret;
     }
@@ -554,19 +567,83 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQInfo *pQInfo) {
       break;
     }
 
-    qDebug("QInfo:%p no result in group %d, continue", pQInfo, pGroupResInfo->currentGroup);
+    qDebug("QInfo:%p no result in group %d, continue", pRuntimeEnv->qinfo, pGroupResInfo->currentGroup);
     cleanupGroupResInfo(pGroupResInfo);
     incNextGroup(pGroupResInfo);
   }
 
-  if (pGroupResInfo->currentGroup >= pGroupResInfo->totalGroup && !hasRemainData(pGroupResInfo)) {
-    SET_STABLE_QUERY_OVER(pQInfo);
-  }
-
   int64_t elapsedTime = taosGetTimestampUs() - st;
-  qDebug("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", pQInfo,
+  qDebug("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", pRuntimeEnv->qinfo,
          pGroupResInfo->currentGroup, pGroupResInfo->totalGroup, elapsedTime);
 
-  pQInfo->runtimeEnv.summary.firstStageMergeTime += elapsedTime;
+//  pQInfo->summary.firstStageMergeTime += elapsedTime;
   return TSDB_CODE_SUCCESS;
 }
+
+void blockDistInfoToBinary(STableBlockDist* pDist, struct SBufferWriter* bw) {
+  tbufWriteUint32(bw, pDist->numOfTables);
+  tbufWriteUint16(bw, pDist->numOfFiles);
+  tbufWriteUint64(bw, pDist->totalSize);
+  tbufWriteUint32(bw, pDist->numOfRowsInMemTable);
+  tbufWriteUint64(bw, taosArrayGetSize(pDist->dataBlockInfos));
+
+  // compress the binary string
+  char* p = TARRAY_GET_START(pDist->dataBlockInfos);
+
+  // compress extra bytes
+  size_t x = taosArrayGetSize(pDist->dataBlockInfos) * pDist->dataBlockInfos->elemSize;
+  char* tmp = malloc(x + 2);
+
+  bool comp = false;
+  int32_t len = tsCompressString(p, (int32_t)x, 1, tmp, (int32_t)x, ONE_STAGE_COMP, NULL, 0);
+  if (len == -1 || len >= x) { // compress failed, do not compress this binary data
+    comp = false;
+    len = (int32_t)x;
+  } else {
+    comp = true;
+  }
+
+  tbufWriteUint8(bw, comp);
+  tbufWriteUint32(bw, len);
+  if (comp) {
+    tbufWriteBinary(bw, tmp, len);
+  } else {
+    tbufWriteBinary(bw, p, len);
+  }
+  tfree(tmp);
+}
+
+void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDist) {
+  SBufferReader br = tbufInitReader(data, len, false);
+
+  pDist->numOfTables = tbufReadUint32(&br);
+  pDist->numOfFiles  = tbufReadUint16(&br);
+  pDist->totalSize   = tbufReadUint64(&br);
+  pDist->numOfRowsInMemTable = tbufReadUint32(&br);
+  int64_t numOfBlocks = tbufReadUint64(&br);
+
+  bool comp = tbufReadUint8(&br);
+  uint32_t compLen = tbufReadUint32(&br);
+
+  size_t originalLen = (size_t) (numOfBlocks*sizeof(SFileBlockInfo));
+
+  char* outputBuf = NULL;
+  if (comp) {
+    outputBuf = malloc(originalLen);
+
+    size_t actualLen = compLen;
+    const char* compStr = tbufReadBinary(&br, &actualLen);
+
+    int32_t orignalLen = tsDecompressString(compStr, compLen, 1, outputBuf,
+                                            (int32_t)originalLen , ONE_STAGE_COMP, NULL, 0);
+    assert(orignalLen == numOfBlocks*sizeof(SFileBlockInfo));
+  } else {
+    outputBuf = (char*) tbufReadBinary(&br, &originalLen);
+  }
+
+  pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t) numOfBlocks, sizeof(SFileBlockInfo));
+  if (comp) {
+    tfree(outputBuf);
+  }
+}
+

src/query/src/queryMain.c

@@ -96,7 +96,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
   }
 
   if (param.pSecExprMsg != NULL) {
-    if ((code = createQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, &param.pSecExprs, param.pSecExprMsg, param.pTagColumnInfo)) != TSDB_CODE_SUCCESS) {
+    if ((code = createIndirectQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, &param.pSecExprs, param.pSecExprMsg, param.pExprs)) != TSDB_CODE_SUCCESS) {
       goto _over;
     }
   }
@@ -144,11 +144,11 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
         goto _over;
       }
 
-      qDebug("qmsg:%p query on %" PRIzu " tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables);
+      qDebug("qmsg:%p query on %u tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables);
     }
 
     int64_t el = taosGetTimestampUs() - st;
-    qDebug("qmsg:%p tag filter completed, numOfTables:%" PRIzu ", elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el);
+    qDebug("qmsg:%p tag filter completed, numOfTables:%u, elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el);
   } else {
     assert(0);
   }
@@ -209,6 +209,7 @@ bool qTableQuery(qinfo_t qinfo) {
     return false;
   }
 
+
   pQInfo->startExecTs = taosGetTimestampSec();
 
   if (isQueryKilled(pQInfo)) {
@@ -216,9 +217,10 @@ bool qTableQuery(qinfo_t qinfo) {
     return doBuildResCheck(pQInfo);
   }
 
-  if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) {
+  SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
+  if (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 0) {
     qDebug("QInfo:%p no table exists for query, abort", pQInfo);
-    setQueryStatus(pQInfo->runtimeEnv.pQuery, QUERY_COMPLETED);
+    setQueryStatus(pRuntimeEnv, QUERY_COMPLETED);
     return doBuildResCheck(pQInfo);
   }
 
@@ -232,26 +234,16 @@ bool qTableQuery(qinfo_t qinfo) {
 
   qDebug("QInfo:%p query task is launched", pQInfo);
 
-  SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
+  pRuntimeEnv->outputBuf = pRuntimeEnv->proot->exec(pRuntimeEnv->proot);
-  if (onlyQueryTags(pQInfo->runtimeEnv.pQuery)) {
-    assert(pQInfo->runtimeEnv.pQueryHandle == NULL);
-    buildTagQueryResult(pQInfo);
-  } else if (pQInfo->runtimeEnv.stableQuery) {
-    stableQueryImpl(pQInfo);
-  } else if (pQInfo->runtimeEnv.queryBlockDist){
-    buildTableBlockDistResult(pQInfo);
-  } else {
-    tableQueryImpl(pQInfo);
-  }
 
-  SQuery* pQuery = pRuntimeEnv->pQuery;
   if (isQueryKilled(pQInfo)) {
     qDebug("QInfo:%p query is killed", pQInfo);
-  } else if (pQuery->rec.rows == 0) {
+  } else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) {
-    qDebug("QInfo:%p over, %" PRIzu " tables queried, %"PRId64" rows are returned", pQInfo, pQInfo->tableqinfoGroupInfo.numOfTables, pQuery->rec.total);
+    qDebug("QInfo:%p over, %u tables queried, %"PRId64" rows are returned", pQInfo, pRuntimeEnv->tableqinfoGroupInfo.numOfTables,
+           pRuntimeEnv->resultInfo.total);
   } else {
-    qDebug("QInfo:%p query paused, %" PRId64 " rows returned, numOfTotal:%" PRId64 " rows",
+    qDebug("QInfo:%p query paused, %d rows returned, numOfTotal:%" PRId64 " rows",
-           pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows);
+           pQInfo, GET_NUM_OF_RESULTS(pRuntimeEnv), pRuntimeEnv->resultInfo.total + GET_NUM_OF_RESULTS(pRuntimeEnv));
   }
 
   return doBuildResCheck(pQInfo);
@@ -279,6 +271,7 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex
     *buildRes = true;
     code = pQInfo->code;
   } else {
+    SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
     SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
 
     pthread_mutex_lock(&pQInfo->lock);
@@ -286,8 +279,8 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex
     assert(pQInfo->rspContext == NULL);
     if (pQInfo->dataReady == QUERY_RESULT_READY) {
       *buildRes = true;
-      qDebug("QInfo:%p retrieve result info, rowsize:%d, rows:%" PRId64 ", code:%s", pQInfo, pQuery->resultRowSize,
+      qDebug("QInfo:%p retrieve result info, rowsize:%d, rows:%d, code:%s", pQInfo, pQuery->resultRowSize,
-             pQuery->rec.rows, tstrerror(pQInfo->code));
+             GET_NUM_OF_RESULTS(pRuntimeEnv), tstrerror(pQInfo->code));
     } else {
       *buildRes = false;
       qDebug("QInfo:%p retrieve req set query return result after paused", pQInfo);
@@ -309,12 +302,13 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
     return TSDB_CODE_QRY_INVALID_QHANDLE;
   }
 
-  SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
-  size_t  size = getResultSize(pQInfo, &pQuery->rec.rows);
+  SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
 
+  int32_t s = GET_NUM_OF_RESULTS(pRuntimeEnv);
+  size_t size = pQuery->resultRowSize * s;
   size += sizeof(int32_t);
-  size += sizeof(STableIdInfo) * taosHashGetSize(pQInfo->arrTableIdInfo);
+  size += sizeof(STableIdInfo) * taosHashGetSize(pRuntimeEnv->pTableRetrieveTsMap);
 
   *contLen = (int32_t)(size + sizeof(SRetrieveTableRsp));
 
@@ -324,27 +318,27 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
     return TSDB_CODE_QRY_OUT_OF_MEMORY;
   }
 
-  (*pRsp)->numOfRows = htonl((int32_t)pQuery->rec.rows);
+  (*pRsp)->numOfRows = htonl((int32_t)s);
 
   if (pQInfo->code == TSDB_CODE_SUCCESS) {
-    (*pRsp)->offset   = htobe64(pQuery->limit.offset);
+    (*pRsp)->offset   = htobe64(pQInfo->runtimeEnv.currentOffset);
-    (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime);
+    (*pRsp)->useconds = htobe64(pQInfo->summary.elapsedTime);
   } else {
     (*pRsp)->offset   = 0;
-    (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime);
+    (*pRsp)->useconds = htobe64(pQInfo->summary.elapsedTime);
   }
 
   (*pRsp)->precision = htons(pQuery->precision);
-  if (pQuery->rec.rows > 0 && pQInfo->code == TSDB_CODE_SUCCESS) {
+  if (GET_NUM_OF_RESULTS(&(pQInfo->runtimeEnv)) > 0 && pQInfo->code == TSDB_CODE_SUCCESS) {
     doDumpQueryResult(pQInfo, (*pRsp)->data);
   } else {
-    setQueryStatus(pQuery, QUERY_OVER);
+    setQueryStatus(pRuntimeEnv, QUERY_OVER);
   }
 
   pQInfo->rspContext = NULL;
   pQInfo->dataReady  = QUERY_RESULT_NOT_READY;
 
-  if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) {
+  if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_OVER)) {
     // here current thread hold the refcount, so it is safe to free tsdbQueryHandle.
     *continueExec = false;
     (*pRsp)->completed = 1;  // notify no more result to client
@@ -394,8 +388,7 @@ int32_t qQueryCompleted(qinfo_t qinfo) {
     return TSDB_CODE_QRY_INVALID_QHANDLE;
   }
 
-  SQuery* pQuery = pQInfo->runtimeEnv.pQuery;
+  return isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQInfo->runtimeEnv.status, QUERY_OVER);
-  return isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER);
 }
 
 void qDestroyQueryInfo(qinfo_t qHandle) {

src/query/tests/resultBufferTest.cpp

@@ -10,7 +10,7 @@ namespace {
 // simple test
 void simpleTest() {
   SDiskbasedResultBuf* pResultBuf = NULL;
-  int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4096, NULL);
+  int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4096, NULL);
   
   int32_t pageId = 0;
   int32_t groupId = 0;
@@ -52,7 +52,7 @@ void simpleTest() {
 
 void writeDownTest() {
   SDiskbasedResultBuf* pResultBuf = NULL;
-  int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4*1024, NULL);
+  int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4*1024, NULL);
 
   int32_t pageId = 0;
   int32_t writePageId = 0;
@@ -99,7 +99,7 @@ void writeDownTest() {
 
 void recyclePageTest() {
   SDiskbasedResultBuf* pResultBuf = NULL;
-  int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4*1024, NULL);
+  int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4*1024, NULL);
 
   int32_t pageId = 0;
   int32_t writePageId = 0;

src/tsdb/src/tsdbCommit.c

@@ -15,7 +15,13 @@
 #include "tsdbint.h"
 
 #define TSDB_MAX_SUBBLOCKS 8
-#define TSDB_KEY_FID(key, days, precision) ((key) / tsMsPerDay[(precision)] / (days))
+static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) {
+  if (key < 0) {
+    return (int)(-((-key) / tsMsPerDay[precision] / days + 1));
+  } else {
+    return (int)((key / tsMsPerDay[precision] / days));
+  }
+}
 
 typedef struct {
   SRtn         rtn;     // retention snapshot

src/tsdb/src/tsdbMeta.c

@@ -20,7 +20,7 @@
 static int     tsdbCompareSchemaVersion(const void *key1, const void *key2);
 static char *  getTagIndexKey(const void *pData);
 static STable *tsdbNewTable();
-static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper);
+static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper, STable *pSTable);
 static void    tsdbFreeTable(STable *pTable);
 static int     tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, bool lock);
 static void    tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFromIdx, bool lock);
@@ -43,6 +43,7 @@ static void *  tsdbInsertTableAct(STsdbRepo *pRepo, int8_t act, void *buf, STabl
 static int     tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable);
 static int     tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable);
 static int     tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid);
+static int     tsdbCheckTableTagVal(SKVRow *pKVRow, STSchema *pSchema);
 
 // ------------------ OUTER FUNCTIONS ------------------
 int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg) {
@@ -87,7 +88,7 @@ int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg) {
     super = tsdbGetTableByUid(pMeta, pCfg->superUid);
     if (super == NULL) {  // super table not exists, try to create it
       newSuper = true;
-      super = tsdbCreateTableFromCfg(pCfg, true);
+      super = tsdbCreateTableFromCfg(pCfg, true, NULL);
       if (super == NULL) goto _err;
     } else {
       if (TABLE_TYPE(super) != TSDB_SUPER_TABLE || TABLE_UID(super) != pCfg->superUid) {
@@ -108,7 +109,7 @@ int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg) {
     }
   }
 
-  table = tsdbCreateTableFromCfg(pCfg, false);
+  table = tsdbCreateTableFromCfg(pCfg, false, super);
   if (table == NULL) goto _err;
 
   // Register to meta
@@ -674,7 +675,7 @@ static STable *tsdbNewTable() {
   return pTable;
 }
 
-static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) {
+static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper, STable *pSTable) {
   STable *pTable = NULL;
   size_t  tsize = 0;
 
@@ -726,6 +727,9 @@ static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) {
 
     if (pCfg->type == TSDB_CHILD_TABLE) {
       TABLE_SUID(pTable) = pCfg->superUid;
+      if (tsdbCheckTableTagVal(pCfg->tagValues, pSTable->tagSchema) < 0) {
+        goto _err;
+      }
       pTable->tagVal = tdKVRowDup(pCfg->tagValues);
       if (pTable->tagVal == NULL) {
         terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
@@ -1302,3 +1306,20 @@ static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid) {
 
   return 0;
 }
+
+static int tsdbCheckTableTagVal(SKVRow *pKVRow, STSchema *pSchema) {
+  for (size_t i = 0; i < kvRowNCols(pKVRow); i++) {
+    SColIdx * pColIdx = kvRowColIdxAt(pKVRow, i);
+    STColumn *pCol = tdGetColOfID(pSchema, pColIdx->colId);
+
+    if ((pCol == NULL) || (!IS_VAR_DATA_TYPE(pCol->type))) continue;
+
+    void *pValue = tdGetKVRowValOfCol(pKVRow, pCol->colId);
+    if (varDataTLen(pValue) > pCol->bytes) {
+      terrno = TSDB_CODE_TDB_IVLD_TAG_VAL;
+      return -1;
+    }
+  }
+
+  return 0;
+}

src/tsdb/src/tsdbRead.c

@@ -34,11 +34,9 @@
                     .tid = (_checkInfo)->tableId.tid,                                  \
                     .uid = (_checkInfo)->tableId.uid})
 
-
 enum {
   TSDB_QUERY_TYPE_ALL      = 1,
   TSDB_QUERY_TYPE_LAST     = 2,
-  TSDB_QUERY_TYPE_EXTERNAL = 3,
 };
 
 typedef struct SQueryFilePos {
@@ -111,6 +109,8 @@ typedef struct STsdbQueryHandle {
   bool           checkFiles;       // check file stage
   bool           cachelastrow;     // check if last row cached
   bool           loadExternalRow;  // load time window external data rows
+  bool           currentLoadExternalRows; // current load external rows
+  int32_t        loadType;         // block load type
   void*          qinfo;            // query info handle, for debug purpose
   int32_t        type;             // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows
   SDFileSet*     pFileGroup;
@@ -147,6 +147,8 @@ static int32_t tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey,
 static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2);
 static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SMemRef* pMemRef);
 static void*   doFreeColumnInfoData(SArray* pColumnInfoData);
+static void*   destroyTableCheckInfo(SArray* pTableCheckInfo);
+static bool    tsdbGetExternalRow(TsdbQueryHandleT pHandle);
 
 static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) {
   pBlockLoadInfo->slot = -1;
@@ -297,37 +299,45 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa
 
   for (int32_t i = 0; i < gsize; ++i) {
     STableCheckInfo* pInfo = (STableCheckInfo*) taosArrayGet(pTableCheckInfo, i);
-    
     taosArrayPush(pTable, &pInfo->pTableObj);
   }
 
   *psTable = pTable;
-  
   return pTableCheckInfo;
 }
 
-static SArray* createCheckInfoFromCheckInfo(SArray* pTableCheckInfo, TSKEY skey, SArray** psTable) {
+static void resetCheckInfo(STsdbQueryHandle* pQueryHandle) {
-  size_t si = taosArrayGetSize(pTableCheckInfo);
+  size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
-  SArray* pNew = taosArrayInit(si, sizeof(STableCheckInfo));
+  assert(numOfTables >= 1);
-  if (pNew == NULL) {
+
-    return NULL;
+  // todo apply the lastkey of table check to avoid to load header file
+  for (int32_t i = 0; i < numOfTables; ++i) {
+    STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i);
+    pCheckInfo->lastKey = pQueryHandle->window.skey;
+    pCheckInfo->iter    = tSkipListDestroyIter(pCheckInfo->iter);
+    pCheckInfo->iiter   = tSkipListDestroyIter(pCheckInfo->iiter);
+    pCheckInfo->initBuf = false;
+
+    if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
+      assert(pCheckInfo->lastKey >= pQueryHandle->window.skey);
+    } else {
+      assert(pCheckInfo->lastKey <= pQueryHandle->window.skey);
+    }
+  }
 }
 
-  SArray* pTable = taosArrayInit(si, sizeof(STable*));
+// only one table, not need to sort again
+static SArray* createCheckInfoFromCheckInfo(STableCheckInfo* pCheckInfo, TSKEY skey, SArray** psTable) {
+  SArray* pNew = taosArrayInit(1, sizeof(STableCheckInfo));
+  SArray* pTable = taosArrayInit(1, sizeof(STable*));
 
-  for (int32_t j = 0; j < si; ++j) {
-    STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pTableCheckInfo, j);
   STableCheckInfo info = { .lastKey = skey, .pTableObj = pCheckInfo->pTableObj};
 
   info.tableId = pCheckInfo->tableId;
   taosArrayPush(pNew, &info);
   taosArrayPush(pTable, &pCheckInfo->pTableObj);
-  }
 
   *psTable = pTable;
-
-  // it is ordered already, no need to sort again.
-  taosArraySort(pNew, tsdbCheckInfoCompar);
   return pNew;
 }
 
@@ -351,14 +361,15 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pC
   pQueryHandle->locateStart = false;
   pQueryHandle->pMemRef     = pMemRef;
   pQueryHandle->loadExternalRow = pCond->loadExternalRows;
+  pQueryHandle->currentLoadExternalRows = pCond->loadExternalRows;
+
+  pQueryHandle->loadType    = pCond->type;
 
   if (tsdbInitReadH(&pQueryHandle->rhelper, (STsdbRepo*)tsdb) != 0) {
     goto out_of_memory;
   }
 
-  //tsdbMayTakeMemSnapshot(pQueryHandle);
   assert(pCond != NULL && pCond->numOfCols > 0 && pMemRef != NULL);
-
   if (ASCENDING_TRAVERSE(pCond->order)) {
     assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey);
   } else {
@@ -388,7 +399,9 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pC
     pQueryHandle->statis[i].colId = colInfo.info.colId;
   }
 
+  if (pCond->numOfCols > 0) {
     pQueryHandle->defaultLoadColumn = getDefaultLoadColumns(pQueryHandle, true);
+  }
 
   STsdbMeta* pMeta = tsdbGetMeta(tsdb);
   assert(pMeta != NULL);
@@ -433,6 +446,74 @@ TsdbQueryHandleT* tsdbQueryTables(STsdbRepo* tsdb, STsdbQueryCond* pCond, STable
   return (TsdbQueryHandleT) pQueryHandle;
 }
 
+void tsdbResetQueryHandle(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond) {
+  STsdbQueryHandle* pQueryHandle = queryHandle;
+
+  pQueryHandle->order       = pCond->order;
+  pQueryHandle->window      = pCond->twindow;
+  pQueryHandle->type        = TSDB_QUERY_TYPE_ALL;
+  pQueryHandle->cur.fid     = -1;
+  pQueryHandle->cur.win     = TSWINDOW_INITIALIZER;
+  pQueryHandle->checkFiles  = true;
+  pQueryHandle->activeIndex = 0;   // current active table index
+  pQueryHandle->locateStart = false;
+  pQueryHandle->loadExternalRow = pCond->loadExternalRows;
+
+  if (ASCENDING_TRAVERSE(pCond->order)) {
+    assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey);
+  } else {
+    assert(pQueryHandle->window.skey >= pQueryHandle->window.ekey);
+  }
+
+  // allocate buffer in order to load data blocks from file
+  memset(pQueryHandle->statis, 0, sizeof(SDataStatis));
+
+  tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo);
+  tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo);
+
+  resetCheckInfo(pQueryHandle);
+}
+
+void tsdbResetQueryHandleForNewTable(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList) {
+  STsdbQueryHandle* pQueryHandle = queryHandle;
+
+  pQueryHandle->order       = pCond->order;
+  pQueryHandle->window      = pCond->twindow;
+  pQueryHandle->type        = TSDB_QUERY_TYPE_ALL;
+  pQueryHandle->cur.fid     = -1;
+  pQueryHandle->cur.win     = TSWINDOW_INITIALIZER;
+  pQueryHandle->checkFiles  = true;
+  pQueryHandle->activeIndex = 0;   // current active table index
+  pQueryHandle->locateStart = false;
+  pQueryHandle->loadExternalRow = pCond->loadExternalRows;
+
+  if (ASCENDING_TRAVERSE(pCond->order)) {
+    assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey);
+  } else {
+    assert(pQueryHandle->window.skey >= pQueryHandle->window.ekey);
+  }
+
+  // allocate buffer in order to load data blocks from file
+  memset(pQueryHandle->statis, 0, sizeof(SDataStatis));
+
+  tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo);
+  tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo);
+
+  SArray* pTable = NULL;
+  STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb);
+
+  pQueryHandle->pTableCheckInfo = destroyTableCheckInfo(pQueryHandle->pTableCheckInfo);
+
+  pQueryHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pQueryHandle, groupList, pMeta, &pTable);
+  if (pQueryHandle->pTableCheckInfo == NULL) {
+    tsdbCleanupQueryHandle(pQueryHandle);
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+  }
+
+  pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev);
+  pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next);
+}
+
 TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pMemRef) {
   pCond->twindow = updateLastrowForEachGroup(groupList);
 
@@ -469,12 +550,45 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) {
   return res;
 }
 
-TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pRef) {
+// leave only one table for each group
-  STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo, pRef);
+static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGroupList) {
-  pQueryHandle->loadExternalRow = true;
+  assert(pGroupList);
-  if (pQueryHandle != NULL) {
+  size_t numOfGroup = taosArrayGetSize(pGroupList->pGroupList);
-    changeQueryHandleForInterpQuery(pQueryHandle);
+
+  STableGroupInfo* pNew = calloc(1, sizeof(STableGroupInfo));
+  pNew->pGroupList = taosArrayInit(numOfGroup, sizeof(SArray));
+
+  for(int32_t i = 0; i < numOfGroup; ++i) {
+    SArray* oneGroup = taosArrayGetP(pGroupList->pGroupList, i);
+    size_t numOfTables = taosArrayGetSize(oneGroup);
+
+    SArray* px = taosArrayInit(4, sizeof(STableKeyInfo));
+    for (int32_t j = 0; j < numOfTables; ++j) {
+      STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(oneGroup, j);
+      if (window->skey <= pInfo->lastKey && ((STable*)pInfo->pTable)->lastKey != TSKEY_INITIAL_VAL) {
+        taosArrayPush(px, pInfo);
+        pNew->numOfTables += 1;
+        break;
       }
+    }
+
+    // there are no data in this group
+    if (taosArrayGetSize(px) == 0) {
+      taosArrayDestroy(px);
+    } else {
+      taosArrayPush(pNew->pGroupList, &px);
+    }
+  }
+
+  return pNew;
+}
+
+TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pRef) {
+  STableGroupInfo* pNew = trimTableGroup(&pCond->twindow, groupList);
+
+  STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, pNew, qinfo, pRef);
+  pQueryHandle->loadExternalRow = true;
+  pQueryHandle->currentLoadExternalRows = true;
 
   return pQueryHandle;
 }
@@ -769,27 +883,22 @@ static int32_t binarySearchForBlock(SBlock* pBlock, int32_t numOfBlocks, TSKEY s
   return midSlot;
 }
 
-static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlocks) {
+static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int32_t* numOfBlocks) {
-  // load all the comp offset value for all tables in this file
+  int32_t code = 0;
-  int32_t code = TSDB_CODE_SUCCESS;
 
-  *numOfBlocks = 0;
+  STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, index);
-  size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
-
-  for (int32_t i = 0; i < numOfTables; ++i) {
-    STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
   pCheckInfo->numOfBlocks = 0;
 
   if (tsdbSetReadTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj) != TSDB_CODE_SUCCESS) {
     code = terrno;
-      break;
+    return code;
   }
 
   SBlockIdx* compIndex = pQueryHandle->rhelper.pBlkIdx;
 
   // no data block in this file, try next file
   if (compIndex == NULL || compIndex->uid != pCheckInfo->tableId.uid) {
-      continue;  // no data blocks in the file belongs to pCheckInfo->pTable
+    return 0;  // no data blocks in the file belongs to pCheckInfo->pTable
   }
 
   if (pCheckInfo->compSize < (int32_t)compIndex->len) {
@@ -799,7 +908,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
     if (t == NULL) {
       terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
       code = TSDB_CODE_TDB_OUT_OF_MEMORY;
-        break;
+      return code;
     }
 
     pCheckInfo->pCompInfo = (SBlockInfo*)t;
@@ -825,7 +934,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
   int32_t end = start;
 
   if (s > pCompInfo->blocks[start].keyLast) {
-      continue;
+    return 0;
   }
 
   // todo speedup the procedure of located end block
@@ -840,6 +949,28 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
   }
 
   (*numOfBlocks) += pCheckInfo->numOfBlocks;
+  return 0;
+}
+
+static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlocks) {
+  // load all the comp offset value for all tables in this file
+  int32_t code = TSDB_CODE_SUCCESS;
+  *numOfBlocks = 0;
+
+  size_t numOfTables = 0;
+  if (pQueryHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) {
+     code = loadBlockInfo(pQueryHandle, pQueryHandle->activeIndex, numOfBlocks);
+  } else if (pQueryHandle->loadType == BLOCK_LOAD_OFFSET_SEQ_ORDER) {
+    numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
+
+    for (int32_t i = 0; i < numOfTables; ++i) {
+      code = loadBlockInfo(pQueryHandle, i, numOfBlocks);
+      if (code != TSDB_CODE_SUCCESS) {
+        return code;
+      }
+    }
+  } else {
+    assert(0);
   }
 
   return code;
@@ -1730,6 +1861,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
   memset(pQueryHandle->pDataBlockInfo, 0, size);
   *numOfAllocBlocks = numOfBlocks;
 
+  // access data blocks according to the offset of each block in asc/desc order.
   int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo);
 
   SBlockOrderSupporter sup = {0};
@@ -1959,6 +2091,91 @@ static void moveToNextDataBlockInCurrentFile(STsdbQueryHandle* pQueryHandle) {
   cur->blockCompleted = false;
 }
 
+int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist* pTableBlockInfo) {
+  STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) queryHandle;
+
+  pTableBlockInfo->totalSize = 0;
+  STsdbFS* pFileHandle = REPO_FS(pQueryHandle->pTsdb);
+
+  // find the start data block in file
+  pQueryHandle->locateStart = true;
+  STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
+  int32_t   fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision);
+
+  tsdbRLockFS(pFileHandle);
+  tsdbFSIterInit(&pQueryHandle->fileIter, pFileHandle, pQueryHandle->order);
+  tsdbFSIterSeek(&pQueryHandle->fileIter, fid);
+  tsdbUnLockFS(pFileHandle);
+
+  pTableBlockInfo->numOfFiles += 1;
+
+  int32_t     code = TSDB_CODE_SUCCESS;
+  int32_t     numOfBlocks = 0;
+  int32_t     numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo);
+  STimeWindow win = TSWINDOW_INITIALIZER;
+
+  while (true) {
+    numOfBlocks = 0;
+    tsdbRLockFS(REPO_FS(pQueryHandle->pTsdb));
+
+    if ((pQueryHandle->pFileGroup = tsdbFSIterNext(&pQueryHandle->fileIter)) == NULL) {
+      tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb));
+      break;
+    }
+
+    tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fid, &win.skey, &win.ekey);
+
+    // current file are not overlapped with query time window, ignore remain files
+    if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) ||
+        (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) {
+      tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb));
+      tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %p", pQueryHandle,
+                pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qinfo);
+      pQueryHandle->pFileGroup = NULL;
+      break;
+    }
+
+    pTableBlockInfo->numOfFiles += 1;
+    if (tsdbSetAndOpenReadFSet(&pQueryHandle->rhelper, pQueryHandle->pFileGroup) < 0) {
+      tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb));
+      code = terrno;
+      break;
+    }
+
+    tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb));
+
+    if (tsdbLoadBlockIdx(&pQueryHandle->rhelper) < 0) {
+      code = terrno;
+      break;
+    }
+
+    if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) {
+      break;
+    }
+
+    tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %p", pQueryHandle, numOfBlocks, numOfTables,
+              pQueryHandle->pFileGroup->fid, pQueryHandle->qinfo);
+
+    if (numOfBlocks == 0) {
+      continue;
+    }
+
+    for (int32_t i = 0; i < numOfTables; ++i) {
+      STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
+
+      SBlock* pBlock = pCheckInfo->pCompInfo->blocks;
+      for (int32_t j = 0; j < pCheckInfo->numOfBlocks; ++j) {
+        pTableBlockInfo->totalSize += pBlock[j].len;
+
+        int32_t numOfRows = pBlock[j].numOfRows;
+        taosArrayPush(pTableBlockInfo->dataBlockInfos, &numOfRows);
+      }
+    }
+  }
+
+  return code;
+}
+
 static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists) {
   STsdbFS*       pFileHandle = REPO_FS(pQueryHandle->pTsdb);
   SQueryFilePos* cur = &pQueryHandle->cur;
@@ -2017,19 +2234,14 @@ static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) {
     pQueryHandle->activeIndex += 1;
   }
 
-  if (pQueryHandle->loadExternalRow && pQueryHandle->window.skey == pQueryHandle->window.ekey) {
-    SMemRef* pMemRef = pQueryHandle->pMemRef;
-    doGetExternalRow(pQueryHandle, TSDB_PREV_ROW, pMemRef);
-    doGetExternalRow(pQueryHandle, TSDB_NEXT_ROW, pMemRef);
-  }
-
   // no data in memtable or imemtable, decrease the memory reference.
-  tsdbMayUnTakeMemSnapshot(pQueryHandle);
+  // TODO !!
+//  tsdbMayUnTakeMemSnapshot(pQueryHandle);
   return false;
 }
 
 //todo not unref yet, since it is not support multi-group interpolation query
-static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) {
+static UNUSED_FUNC void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) {
   // filter the queried time stamp in the first place
   STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle;
 
@@ -2152,20 +2364,61 @@ static void destroyHelper(void* param) {
   free(param);
 }
 
-// handle data in cache situation
+static bool loadBlockOfActiveTable(STsdbQueryHandle* pQueryHandle) {
-bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
+  if (pQueryHandle->checkFiles) {
-  STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle;
+    // check if the query range overlaps with the file data block
+    bool exists = true;
 
-  int64_t stime = taosGetTimestampUs();
+    int32_t code = getDataBlocksInFiles(pQueryHandle, &exists);
-  int64_t elapsedTime = stime;
+    if (code != TSDB_CODE_SUCCESS) {
+      pQueryHandle->checkFiles = false;
+      return false;
+    }
 
-  size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
+    if (exists) {
-  assert(numOfTables > 0);
+      if (pQueryHandle->currentLoadExternalRows && pQueryHandle->window.skey == pQueryHandle->window.ekey) {
+        SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, 0);
+        assert(*(int64_t*)pColInfo->pData == pQueryHandle->window.skey);
+      }
 
-  if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST && pQueryHandle->cachelastrow) {
+      pQueryHandle->currentLoadExternalRows = false; // clear the flag, since the exact matched row is found.
+      return exists;
+    }
+
+    pQueryHandle->checkFiles = false;
+  }
+
+  if (hasMoreDataInCache(pQueryHandle)) {
+    pQueryHandle->currentLoadExternalRows = false;
+    return true;
+  }
+
+  // current result is empty
+  if (pQueryHandle->currentLoadExternalRows && pQueryHandle->window.skey == pQueryHandle->window.ekey && pQueryHandle->cur.rows == 0) {
+    SMemRef* pMemRef = pQueryHandle->pMemRef;
+
+    doGetExternalRow(pQueryHandle, TSDB_PREV_ROW, pMemRef);
+    doGetExternalRow(pQueryHandle, TSDB_NEXT_ROW, pMemRef);
+
+    bool result = tsdbGetExternalRow(pQueryHandle);
+
+    pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev);
+    pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next);
+    pQueryHandle->currentLoadExternalRows = false;
+
+    return result;
+  }
+
+  return false;
+}
+
+static bool loadCachedLastRow(STsdbQueryHandle* pQueryHandle) {
   // the last row is cached in buffer, return it directly.
   // here note that the pQueryHandle->window must be the TS_INITIALIZER
   int32_t numOfCols  = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle));
+  size_t  numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
+  assert(numOfTables > 0 && numOfCols > 0);
+
   SQueryFilePos* cur = &pQueryHandle->cur;
 
   SDataRow pRow = NULL;
@@ -2197,6 +2450,49 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
   return false;
 }
 
+static bool loadDataBlockFromTableSeq(STsdbQueryHandle* pQueryHandle) {
+  size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
+  assert(numOfTables > 0);
+
+  int64_t stime = taosGetTimestampUs();
+
+  while(pQueryHandle->activeIndex < numOfTables) {
+    if (loadBlockOfActiveTable(pQueryHandle)) {
+      return true;
+    }
+
+    STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex);
+    pCheckInfo->numOfBlocks = 0;
+
+    pQueryHandle->activeIndex += 1;
+    pQueryHandle->locateStart = false;
+    pQueryHandle->checkFiles  = true;
+    pQueryHandle->cur.rows    = 0;
+    pQueryHandle->currentLoadExternalRows = pQueryHandle->loadExternalRow;
+
+    terrno = TSDB_CODE_SUCCESS;
+
+    int64_t elapsedTime = taosGetTimestampUs() - stime;
+    pQueryHandle->cost.checkForNextTime += elapsedTime;
+  }
+
+  return false;
+}
+
+// handle data in cache situation
+bool tsdbNextDataBlock(TsdbQueryHandleT pHandle) {
+  STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle;
+
+  int64_t stime = taosGetTimestampUs();
+  int64_t elapsedTime = stime;
+
+  if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST && pQueryHandle->cachelastrow) {
+    return loadCachedLastRow(pQueryHandle);
+  }
+
+  if (pQueryHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) {
+    return loadDataBlockFromTableSeq(pQueryHandle);
+  } else { // loadType == RR and Offset Order
     if (pQueryHandle->checkFiles) {
       // check if the query range overlaps with the file data block
       bool exists = true;
@@ -2226,75 +2522,6 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
     pQueryHandle->cost.checkForNextTime += elapsedTime;
     return ret;
   }
-
-bool tsdbNextDataBlockWithoutMerge(TsdbQueryHandleT* pHandle) {
-  STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle;
-
-  int64_t stime = taosGetTimestampUs();
-  int64_t elapsedTime = stime;
-
-  size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
-  assert(numOfTables > 0);
-
-  if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST && pQueryHandle->cachelastrow) {
-    // the last row is cached in buffer, return it directly.
-    // here note that the pQueryHandle->window must be the TS_INITIALIZER
-    int32_t numOfCols  = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle));
-    SQueryFilePos* cur = &pQueryHandle->cur;
-
-    SDataRow pRow = NULL;
-    TSKEY    key  = TSKEY_INITIAL_VAL;
-    int32_t  step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1;
-
-    if (++pQueryHandle->activeIndex < numOfTables) {
-      STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex);
-      int32_t ret = tsdbGetCachedLastRow(pCheckInfo->pTableObj, &pRow, &key);
-      if (ret != TSDB_CODE_SUCCESS) {
-        return false;
-      }
-
-      copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, numOfCols, pCheckInfo->pTableObj, NULL);
-      tfree(pRow);
-
-      // update the last key value
-      pCheckInfo->lastKey = key + step;
-
-      cur->rows     = 1;  // only one row
-      cur->lastKey  = key + step;
-      cur->mixBlock = true;
-      cur->win.skey = key;
-      cur->win.ekey = key;
-
-      return true;
-    }
-
-    return false;
-  }
-
-  if (pQueryHandle->checkFiles) {
-    // check if the query range overlaps with the file data block
-    bool exists = true;
-
-    int32_t code = getDataBlocksInFiles(pQueryHandle, &exists);
-    if (code != TSDB_CODE_SUCCESS) {
-      pQueryHandle->activeIndex = 0;
-      pQueryHandle->checkFiles = false;
-
-      return false;
-    }
-
-    if (exists) {
-      pQueryHandle->cost.checkForNextTime += (taosGetTimestampUs() - stime);
-      return exists;
-    }
-
-    pQueryHandle->activeIndex = 0;
-    pQueryHandle->checkFiles = false;
-  }
-
-  elapsedTime = taosGetTimestampUs() - stime;
-  pQueryHandle->cost.checkForNextTime += elapsedTime;
-  return false;
 }
 
 static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SMemRef* pMemRef) {
@@ -2342,7 +2569,7 @@ static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SM
   }
 
   // load the previous row
-  STsdbQueryCond cond = {.numOfCols = numOfCols, .loadExternalRows = false,};
+  STsdbQueryCond cond = {.numOfCols = numOfCols, .loadExternalRows = false, .type = BLOCK_LOAD_OFFSET_SEQ_ORDER};
   if (type == TSDB_PREV_ROW) {
     cond.order = TSDB_ORDER_DESC;
     cond.twindow = (STimeWindow){pQueryHandle->window.skey, INT64_MIN};
@@ -2363,12 +2590,13 @@ static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SM
   }
 
   pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qinfo, pMemRef);
-
   tfree(cond.colList);
 
-  SArray* psTable = NULL;
+  // current table, only one table
+  STableCheckInfo* pCurrent = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex);
 
-  pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pQueryHandle->pTableCheckInfo, pSecQueryHandle->window.skey, &psTable);
+  SArray* psTable = NULL;
+  pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pCurrent, pSecQueryHandle->window.skey, &psTable);
   if (pSecQueryHandle->pTableCheckInfo == NULL) {
     terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
     goto out_of_memory;
@@ -2376,7 +2604,6 @@ static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SM
 
 
   tsdbMayTakeMemSnapshot(pSecQueryHandle, psTable);
-
   if (!tsdbNextDataBlock((void*)pSecQueryHandle)) {
     // no result in current query, free the corresponding result rows structure
     if (type == TSDB_PREV_ROW) {
@@ -2406,10 +2633,35 @@ out_of_memory:
   return terrno;
 }
 
-SArray* tsdbGetExternalRow(TsdbQueryHandleT *pHandle, SMemRef* pMemRef, int16_t type) {
+bool tsdbGetExternalRow(TsdbQueryHandleT pHandle) {
   STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle;
-  assert(type == TSDB_PREV_ROW || type == TSDB_NEXT_ROW);
+  SQueryFilePos* cur = &pQueryHandle->cur;
-  return (type == TSDB_PREV_ROW)? pQueryHandle->prev:pQueryHandle->next;
+
+  cur->fid = INT32_MIN;
+  cur->mixBlock = true;
+  if (pQueryHandle->prev == NULL || pQueryHandle->next == NULL) {
+    cur->rows = 0;
+    return false;
+  }
+
+  int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pQueryHandle);
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, i);
+    SColumnInfoData* first = taosArrayGet(pQueryHandle->prev, i);
+
+    memcpy(pColInfoData->pData, first->pData, pColInfoData->info.bytes);
+
+    SColumnInfoData* sec = taosArrayGet(pQueryHandle->next, i);
+    memcpy(((char*)pColInfoData->pData) + pColInfoData->info.bytes, sec->pData, pColInfoData->info.bytes);
+
+    if (i == 0 && pColInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP) {
+      cur->win.skey = *(TSKEY*)pColInfoData->pData;
+      cur->win.ekey = *(TSKEY*)(((char*)pColInfoData->pData) + TSDB_KEYSIZE);
+    }
+  }
+
+  cur->rows = 2;
+  return true;
 }
 
 /*
@@ -2949,10 +3201,10 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons
       goto _error;
     }
 
-    pGroupInfo->numOfTables = taosArrayGetSize(res);
+    pGroupInfo->numOfTables = (uint32_t) taosArrayGetSize(res);
     pGroupInfo->pGroupList  = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey);
 
-    tsdbDebug("%p no table name/tag condition, all tables belong to one group, numOfTables:%" PRIzu "", tsdb, pGroupInfo->numOfTables);
+    tsdbDebug("%p no table name/tag condition, all tables belong to one group, numOfTables:%u", tsdb, pGroupInfo->numOfTables);
     taosArrayDestroy(res);
 
     if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error;
@@ -2994,10 +3246,10 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons
   } END_TRY
 
   doQueryTableList(pTable, res, expr);
-  pGroupInfo->numOfTables = taosArrayGetSize(res);
+  pGroupInfo->numOfTables = (uint32_t)taosArrayGetSize(res);
   pGroupInfo->pGroupList  = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey);
 
-  tsdbDebug("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%" PRIzu ", belong to %" PRIzu " groups", tsdb, pTable->tableId.tid,
+  tsdbDebug("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%u, belong to %" PRIzu " groups", tsdb, pTable->tableId.tid,
       pTable->tableId.uid, pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList));
 
   taosArrayDestroy(res);
@@ -3074,7 +3326,7 @@ int32_t tsdbGetTableGroupFromIdList(STsdbRepo* tsdb, SArray* pTableIdList, STabl
     return terrno;
   }
 
-  pGroupInfo->numOfTables = taosArrayGetSize(group);
+  pGroupInfo->numOfTables = (uint32_t) taosArrayGetSize(group);
   if (pGroupInfo->numOfTables > 0) {
     taosArrayPush(pGroupInfo->pGroupList, &group);
   } else {
@@ -3099,23 +3351,26 @@ static void* doFreeColumnInfoData(SArray* pColumnInfoData) {
   return NULL;
 }
 
+static void* destroyTableCheckInfo(SArray* pTableCheckInfo) {
+  size_t size = taosArrayGetSize(pTableCheckInfo);
+  for (int32_t i = 0; i < size; ++i) {
+    STableCheckInfo* p = taosArrayGet(pTableCheckInfo, i);
+    destroyTableMemIterator(p);
+
+    tfree(p->pCompInfo);
+  }
+
+  taosArrayDestroy(pTableCheckInfo);
+  return NULL;
+}
+
 void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
   STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*)queryHandle;
   if (pQueryHandle == NULL) {
     return;
   }
 
-  if (pQueryHandle->pTableCheckInfo != NULL) {
+  pQueryHandle->pTableCheckInfo = destroyTableCheckInfo(pQueryHandle->pTableCheckInfo);
-    size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
-    for (int32_t i = 0; i < size; ++i) {
-      STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
-      destroyTableMemIterator(pTableCheckInfo);
-
-      tfree(pTableCheckInfo->pCompInfo);
-    }
-    taosArrayDestroy(pQueryHandle->pTableCheckInfo);
-  }
-
   pQueryHandle->pColumns = doFreeColumnInfoData(pQueryHandle->pColumns);
 
   taosArrayDestroy(pQueryHandle->defaultLoadColumn);

src/util/inc/tarray.h

@@ -26,6 +26,7 @@ extern "C" {
 #define TARRAY_MIN_SIZE 8
 #define TARRAY_GET_ELEM(array, index) ((void*)((char*)((array)->pData) + (index) * (array)->elemSize))
 #define TARRAY_ELEM_IDX(array, ele)   (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize)
+#define TARRAY_GET_START(array)       ((array)->pData)
 
 typedef struct SArray {
   size_t size;

src/util/inc/tbuffer.h

@@ -73,14 +73,14 @@ int main( int argc, char** argv ) {
 }
 */
 
-typedef struct {
+typedef struct SBufferReader {
   bool endian;
   const char* data;
   size_t pos;
   size_t size;
 } SBufferReader;
 
-typedef struct {
+typedef struct SBufferWriter {
   bool endian;
   char* data;
   size_t pos;

src/util/src/terror.c

@@ -197,6 +197,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_READY,         "Database unsynced")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_DAYS,   "Invalid database option: days out of range")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_KEEP,   "Invalid database option: keep >= keep1 >= keep0 >= days")
 
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC,            "Invalid topic name")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_OPTION,     "Invalid topic option")
+
 // dnode
 TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED,        "Message not processed")
 TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY,            "Dnode out of memory")

src/vnode/inc/vnodeInt.h

@@ -40,6 +40,7 @@ typedef struct {
   int32_t  queuedWMsg;
   int32_t  queuedRMsg;
   int32_t  flowctrlLevel;
+  int32_t  sequence;  // for topic
   int8_t   status;
   int8_t   role;
   int8_t   accessState;
@@ -47,7 +48,7 @@ typedef struct {
   int8_t   isCommiting;
   int8_t   dbReplica;
   int8_t   dropped;
-  int8_t   reserved;
+  int8_t   dbType;
   uint64_t version;   // current version
   uint64_t cversion;  // version while commit start
   uint64_t fversion;  // version on saved data file

src/vnode/src/vnodeCfg.c

@@ -42,6 +42,7 @@ static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) {
   pVnode->syncCfg.replica = vnodeMsg->cfg.vgReplica;
   pVnode->syncCfg.quorum = vnodeMsg->cfg.quorum;
   pVnode->dbReplica = vnodeMsg->cfg.dbReplica;
+  pVnode->dbType = vnodeMsg->cfg.dbType;
 
   for (int i = 0; i < pVnode->syncCfg.replica; ++i) {
     SVnodeDesc *node = &vnodeMsg->nodes[i];
@@ -214,7 +215,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) {
 
   cJSON *dbReplica = cJSON_GetObjectItem(root, "dbReplica");
   if (!dbReplica || dbReplica->type != cJSON_Number) {
-    vError("vgId:%d, failed to read %s, dbReplica not found", pVnode->vgId, file);
+    vWarn("vgId:%d, failed to read %s, dbReplica not found", pVnode->vgId, file);
     vnodeMsg.cfg.dbReplica = vnodeMsg.cfg.vgReplica;
     vnodeMsg.cfg.vgCfgVersion = 0;
   } else {
@@ -230,7 +231,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) {
 
   cJSON *update = cJSON_GetObjectItem(root, "update");
   if (!update || update->type != cJSON_Number) {
-    vError("vgId: %d, failed to read %s, update not found", pVnode->vgId, file);
+    vWarn("vgId: %d, failed to read %s, update not found", pVnode->vgId, file);
     vnodeMsg.cfg.update = 0;
     vnodeMsg.cfg.vgCfgVersion = 0;
   } else {
@@ -239,13 +240,21 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) {
 
   cJSON *cacheLastRow = cJSON_GetObjectItem(root, "cacheLastRow");
   if (!cacheLastRow || cacheLastRow->type != cJSON_Number) {
-    vError("vgId: %d, failed to read %s, cacheLastRow not found", pVnode->vgId, file);
+    vWarn("vgId: %d, failed to read %s, cacheLastRow not found", pVnode->vgId, file);
     vnodeMsg.cfg.cacheLastRow = 0;
     vnodeMsg.cfg.vgCfgVersion = 0;
   } else {
     vnodeMsg.cfg.cacheLastRow = (int8_t)cacheLastRow->valueint;
   }
 
+  cJSON *dbType = cJSON_GetObjectItem(root, "dbType");
+  if (!dbType || dbType->type != cJSON_Number) {
+    vWarn("vgId: %d, failed to read %s, dbType not found", pVnode->vgId, file);
+    vnodeMsg.cfg.dbType = 0;
+  } else {
+    vnodeMsg.cfg.dbType = (int8_t)dbType->valueint;
+  }
+
   cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
   if (!nodeInfos || nodeInfos->type != cJSON_Array) {
     vError("vgId:%d, failed to read %s, nodeInfos not found", pVnode->vgId, file);
@@ -337,6 +346,7 @@ int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) {
   len += snprintf(content + len, maxLen - len, "  \"quorum\": %d,\n", pMsg->cfg.quorum);
   len += snprintf(content + len, maxLen - len, "  \"update\": %d,\n", pMsg->cfg.update);
   len += snprintf(content + len, maxLen - len, "  \"cacheLastRow\": %d,\n", pMsg->cfg.cacheLastRow);
+  len += snprintf(content + len, maxLen - len, "  \"dbType\": %d,\n", pMsg->cfg.dbType);
   len += snprintf(content + len, maxLen - len, "  \"nodeInfos\": [{\n");
   for (int32_t i = 0; i < pMsg->cfg.vgReplica; i++) {
     SVnodeDesc *node = &pMsg->nodes[i];

src/vnode/src/vnodeRead.c

@@ -372,7 +372,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
 
   // kill current query and free corresponding resources.
   if (pRetrieve->free == 1) {
-    vWarn("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, *handle);
+    vDebug("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, *handle);
     qKillQuery(*handle);
     qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);