Merge branch 'develop' into test/testcase

CMakeLists.txt

@@ -16,6 +16,7 @@ SET(TD_GRANT FALSE)
 SET(TD_MQTT FALSE)
 SET(TD_TSDB_PLUGINS FALSE)
 SET(TD_STORAGE FALSE)
+SET(TD_TOPIC FALSE)
 
 SET(TD_COVER FALSE)
 SET(TD_MEM_CHECK FALSE)

cmake/define.inc

@@ -25,6 +25,10 @@ IF (TD_STORAGE)
   ADD_DEFINITIONS(-D_STORAGE)
 ENDIF ()
 
+IF (TD_TOPIC)
+  ADD_DEFINITIONS(-D_TOPIC)
+ENDIF ()
+
 IF (TD_GODLL)
   ADD_DEFINITIONS(-D_TD_GO_DLL_)
 ENDIF ()

cmake/input.inc

@@ -9,6 +9,14 @@ ELSEIF (${ACCOUNT} MATCHES "false")
   MESSAGE(STATUS "Build without account plugins")
 ENDIF ()
 
+IF (${TOPIC} MATCHES "true")
+  SET(TD_TOPIC TRUE)
+  MESSAGE(STATUS "Build with topic plugins")
+ELSEIF (${TOPIC} MATCHES "false")
+  SET(TD_TOPIC FALSE)
+  MESSAGE(STATUS "Build without topic plugins")
+ENDIF ()
+
 IF (${COVER} MATCHES "true")
   SET(TD_COVER TRUE)
   MESSAGE(STATUS "Build with test coverage")

cmake/version.inc

@@ -4,7 +4,7 @@ PROJECT(TDengine)
 IF (DEFINED VERNUMBER)
   SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "2.0.16.0")
+  SET(TD_VER_NUMBER "2.0.17.0")
 ENDIF ()
 
 IF (DEFINED VERCOMPATIBLE)

documentation20/cn/03.architecture/docs.md

@@ -166,7 +166,7 @@ TDengine 分布式架构的逻辑结构图如下：
 
 **虚拟节点(vnode)**: 为更好的支持数据分片、负载均衡，防止数据过热或倾斜，数据节点被虚拟化成多个虚拟节点(vnode，图中V2, V3, V4等)。每个 vnode 都是一个相对独立的工作单元，是时序数据存储的基本单元，具有独立的运行线程、内存空间与持久化存储的路径。一个 vnode 包含一定数量的表（数据采集点）。当创建一张新表时，系统会检查是否需要创建新的 vnode。一个数据节点上能创建的 vnode 的数量取决于该数据节点所在物理节点的硬件资源。一个 vnode 只属于一个DB，但一个DB可以有多个 vnode。一个 vnode 除存储的时序数据外，也保存有所包含的表的schema、标签值等。一个虚拟节点由所属的数据节点的EP，以及所属的VGroup ID在系统内唯一标识，由管理节点创建并管理。
 
-**管理节点(mnode):** 一个虚拟的逻辑单元，负责所有数据节点运行状态的监控和维护，以及节点之间的负载均衡(图中M)。同时，管理节点也负责元数据(包括用户、数据库、表、静态标签等)的存储和管理，因此也称为 Meta Node。TDengine 集群中可配置多个(最多不超过5个) mnode，它们自动构建成为一个虚拟管理节点组(图中M0, M1, M2)。mnode 间采用 master/slave 的机制进行管理，而且采取强一致方式进行数据同步, 任何数据更新操作只能在 Master 上进行。mnode 集群的创建由系统自动完成，无需人工干预。每个dnode上至多有一个mnode，由所属的数据节点的EP来唯一标识。每个dnode通过内部消息交互自动获取整个集群中所有 mnode 所在的 dnode 的EP。
+**管理节点(mnode):** 一个虚拟的逻辑单元，负责所有数据节点运行状态的监控和维护，以及节点之间的负载均衡(图中M)。同时，管理节点也负责元数据(包括用户、数据库、表、静态标签等)的存储和管理，因此也称为 Meta Node。TDengine 集群中可配置多个(开源版最多不超过3个) mnode，它们自动构建成为一个虚拟管理节点组(图中M0, M1, M2)。mnode 间采用 master/slave 的机制进行管理，而且采取强一致方式进行数据同步, 任何数据更新操作只能在 Master 上进行。mnode 集群的创建由系统自动完成，无需人工干预。每个dnode上至多有一个mnode，由所属的数据节点的EP来唯一标识。每个dnode通过内部消息交互自动获取整个集群中所有 mnode 所在的 dnode 的EP。
 
 **虚拟节点组(VGroup):** 不同数据节点上的 vnode 可以组成一个虚拟节点组(vnode group)来保证系统的高可靠。虚拟节点组内采取master/slave的方式进行管理。写操作只能在 master vnode 上进行，系统采用异步复制的方式将数据同步到 slave vnode，这样确保了一份数据在多个物理节点上有拷贝。一个 vgroup 里虚拟节点个数就是数据的副本数。如果一个DB的副本数为N，系统必须有至少N个数据节点。副本数在创建DB时通过参数 replica 可以指定，缺省为1。使用 TDengine 的多副本特性，可以不再需要昂贵的磁盘阵列等存储设备，就可以获得同样的数据高可靠性。虚拟节点组由管理节点创建、管理，并且由管理节点分配一个系统唯一的ID，VGroup ID。如果两个虚拟节点的vnode group ID相同，说明他们属于同一个组，数据互为备份。虚拟节点组里虚拟节点的个数是可以动态改变的，容许只有一个，也就是没有数据复制。VGroup ID是永远不变的，即使一个虚拟节点组被删除，它的ID也不会被收回重复利用。
 

documentation20/cn/10.cluster/docs.md

@@ -227,5 +227,11 @@ SHOW MNODES;
 
 如果副本数为偶数，当一个 vnode group 里一半 vnode 不工作时，是无法从中选出 master 的。同理，一半 mnode 不工作时，是无法选出 mnode 的 master 的，因为存在“split brain”问题。为解决这个问题，TDengine 引入了 Arbitrator 的概念。Arbitrator 模拟一个 vnode 或 mnode 在工作，但只简单的负责网络连接，不处理任何数据插入或访问。只要包含 Arbitrator 在内，超过半数的 vnode 或 mnode 工作，那么该 vnode group 或 mnode 组就可以正常的提供数据插入或查询服务。比如对于副本数为 2 的情形，如果一个节点 A 离线，但另外一个节点 B 正常，而且能连接到 Arbitrator，那么节点 B 就能正常工作。
 
-TDengine提供一个执行程序，名为 tarbitrator，找任何一台Linux服务器运行它即可。请点击[安装包下载](https://www.taosdata.com/cn/all-downloads/)，在TDengine Arbitrator Linux一节中，选择适合的版本下载并安装。该程序对系统资源几乎没有要求，只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号，缺省是6042。配置每个taosd实例时，可以在配置文件taos.cfg里将参数arbitrator设置为Arbitrator的End Point。如果该参数配置了，当副本数为偶数时，系统将自动连接配置的Arbitrator。如果副本数为奇数，即使配置了Arbitrator，系统也不会去建立连接。
+总之，在目前版本下，TDengine 建议在双副本环境要配置 Arbitrator，以提升系统的可用性。
+
+Arbitrator 的执行程序名为 tarbitrator。该程序对系统资源几乎没有要求，只需要保证有网络连接，找任何一台 Linux 服务器运行它即可。以下简要描述安装配置的步骤：
+1. 请点击 [安装包下载](https://www.taosdata.com/cn/all-downloads/)，在 TDengine Arbitrator Linux 一节中，选择合适的版本下载并安装。
+2. 该应用的命令行参数 `-p` 可以指定其对外服务的端口号，缺省是 6042。
+3. 修改每个 taosd 实例的配置文件，在 taos.cfg 里将参数 arbitrator 设置为 tarbitrator 程序所对应的 End Point。（如果该参数配置了，当副本数为偶数时，系统将自动连接配置的 Arbitrator。如果副本数为奇数，即使配置了 Arbitrator，系统也不会去建立连接。）
+4. 在配置文件中配置了的 Arbitrator，会出现在 `SHOW DNODES;` 指令的返回结果中，对应的 role 列的值会是“arb”。
 

documentation20/cn/11.administrator/docs.md

@@ -9,7 +9,7 @@
 每个 DB 可以创建固定数目的 vgroup，默认与 CPU 核数相同，可通过 maxVgroupsPerDb 配置；vgroup 中的每个副本会是一个 vnode；每个 vnode 会占用固定大小的内存（大小与数据库的配置参数 blocks 和 cache 有关)；每个 Table 会占用与标签总长度有关的内存；此外，系统会有一些固定的内存开销。因此，每个 DB 需要的系统内存可通过如下公式计算：
 
 ```
-Memory Size = maxVgroupsPerDb * (blocks * cache + 10Mb) + numOfTables * (tagSizePerTable + 0.5Kb)
+Memory Size = maxVgroupsPerDb * (blocks * cache + 10MB) + numOfTables * (tagSizePerTable + 0.5KB)
 ```
 
 示例：假设是 4 核机器，cache 是缺省大小 16M, blocks 是缺省值 6，假设有 10 万张表，标签总长度是 256 字节，则总的内存需求为：4 \* (16 \* 6 + 10) + 100000 \* (0.25 + 0.5) / 1000 = 499M。
@@ -437,7 +437,7 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下
 - 表的列名：不能包含特殊字符，不能超过 64 个字符
 - 数据库名、表名、列名，都不能以数字开头
 - 表的列数：不能超过 1024 列
-- 记录的最大长度：包括时间戳8 byte，不能超过16KB
+- 记录的最大长度：包括时间戳 8 byte，不能超过 16KB（每个 BINARY/NCHAR 类型的列还会额外占用 2 个 byte 的存储位置）
 - 单条 SQL 语句默认最大字符串长度：65480 byte
 - 数据库副本数：不能超过 3
 - 用户名：不能超过 23 个 byte
@@ -453,39 +453,41 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下
 
 | 关键字列表 |             |              |            |           |
 | ---------- | ----------- | ------------ | ---------- | --------- |
-| ABLOCKS    | CONNECTION  | GROUP        | MINUS      | SLASH     |
+| ABLOCKS    | CONNECTIONS | GT           | MNODES     | SLIDING   |
-| ABORT      | CONNECTIONS | GT           | MNODES     | SLIDING   |
+| ABORT      | COPY        | ID           | MODULES    | SLIMIT    |
-| ACCOUNT    | COPY        | ID           | MODULES    | SMALLINT  |
+| ACCOUNT    | COUNT       | IF           | NCHAR      | SMALLINT  |
-| ACCOUNTS   | COUNT       | IF           | NCHAR      | SPREAD    |
+| ACCOUNTS   | CREATE      | IGNORE       | NE         | SPREAD    |
-| ADD        | CREATE      | IGNORE       | NE         | STABLE    |
+| ADD        | CTIME       | IMMEDIATE    | NONE       | STABLE    |
-| AFTER      | CTIME       | IMMEDIATE    | NONE       | STABLES   |
+| AFTER      | DATABASE    | IMPORT       | NOT        | STABLES   |
-| ALL        | DATABASE    | IMPORT       | NOT        | STAR      |
+| ALL        | DATABASES   | IN           | NOTNULL    | STAR      |
-| ALTER      | DATABASES   | IN           | NOTNULL    | STATEMENT |
+| ALTER      | DAYS        | INITIALLY    | NOW        | STATEMENT |
-| AND        | DAYS        | INITIALLY    | NOW        | STDDEV    |
+| AND        | DEFERRED    | INSERT       | OF         | STDDEV    |
-| AS         | DEFERRED    | INSERT       | OF         | STREAM    |
+| AS         | DELIMITERS  | INSTEAD      | OFFSET     | STREAM    |
-| ASC        | DELIMITERS  | INSTEAD      | OFFSET     | STREAMS   |
+| ASC        | DESC        | INTEGER      | OR         | STREAMS   |
-| ATTACH     | DESC        | INTEGER      | OR         | STRING    |
+| ATTACH     | DESCRIBE    | INTERVAL     | ORDER      | STRING    |
-| AVG        | DESCRIBE    | INTERVAL     | ORDER      | SUM       |
+| AVG        | DETACH      | INTO         | PASS       | SUM       |
-| BEFORE     | DETACH      | INTO         | PASS       | TABLE     |
+| BEFORE     | DIFF        | IP           | PERCENTILE | TABLE     |
-| BEGIN      | DIFF        | IP           | PERCENTILE | TABLES    |
+| BEGIN      | DISTINCT    | IS           | PLUS       | TABLES    |
-| BETWEEN    | DISTINCT    | IS           | PLUS       | TAG       |
+| BETWEEN    | DIVIDE      | ISNULL       | PRAGMA     | TAG       |
-| BIGINT     | DIVIDE      | ISNULL       | PRAGMA     | TAGS      |
+| BIGINT     | DNODE       | JOIN         | PREV       | TAGS      |
-| BINARY     | DNODE       | JOIN         | PREV       | TBLOCKS   |
+| BINARY     | DNODES      | KEEP         | PRIVILEGE  | TBLOCKS   |
-| BITAND     | DNODES      | KEEP         | PRIVILEGE  | TBNAME    |
+| BITAND     | DOT         | KEY          | QUERIES    | TBNAME    |
-| BITNOT     | DOT         | KEY          | QUERIES    | TIMES     |
+| BITNOT     | DOUBLE      | KILL         | QUERY      | TIMES     |
-| BITOR      | DOUBLE      | KILL         | QUERY      | TIMESTAMP |
+| BITOR      | DROP        | LAST         | RAISE      | TIMESTAMP |
-| BOOL       | DROP        | LAST         | RAISE      | TINYINT   |
+| BOOL       | EACH        | LE           | REM        | TINYINT   |
-| BOTTOM     | EACH        | LE           | REM        | TOP       |
+| BOTTOM     | END         | LEASTSQUARES | REPLACE    | TOP       |
-| BY         | END         | LEASTSQUARES | REPLACE    | TRIGGER   |
+| BY         | EQ          | LIKE         | REPLICA    | TRIGGER   |
-| CACHE      | EQ          | LIKE         | REPLICA    | UMINUS    |
+| CACHE      | EXISTS      | LIMIT        | RESET      | UMINUS    |
-| CASCADE    | EXISTS      | LIMIT        | RESET      | UPLUS     |
+| CASCADE    | EXPLAIN     | LINEAR       | RESTRICT   | UPLUS     |
-| CHANGE     | EXPLAIN     | LINEAR       | RESTRICT   | USE       |
+| CHANGE     | FAIL        | LOCAL        | ROW        | USE       |
-| CLOG       | FAIL        | LOCAL        | ROW        | USER      |
+| CLOG       | FILL        | LP           | ROWS       | USER      |
-| CLUSTER    | FILL        | LP           | ROWS       | USERS     |
+| CLUSTER    | FIRST       | LSHIFT       | RP         | USERS     |
-| COLON      | FIRST       | LSHIFT       | RP         | USING     |
+| COLON      | FLOAT       | LT           | RSHIFT     | USING     |
-| COLUMN     | FLOAT       | LT           | RSHIFT     | VALUES    |
+| COLUMN     | FOR         | MATCH        | SCORES     | VALUES    |
-| COMMA      | FOR         | MATCH        | SCORES     | VARIABLE  |
+| COMMA      | FROM        | MAX          | SELECT     | VARIABLE  |
-| COMP       | FROM        | MAX          | SELECT     | VGROUPS   |
+| COMP       | GE          | METRIC       | SEMI       | VGROUPS   |
-| CONCAT     | GE          | METRIC       | SEMI       | VIEW      |
+| CONCAT     | GLOB        | METRICS      | SET        | VIEW      |
-| CONFIGS    | GLOB        | METRICS      | SET        | WAVG      |
+| CONFIGS    | GRANTS      | MIN          | SHOW       | WAVG      |
-| CONFLICT   | GRANTS      | MIN          | SHOW       | WHERE     |
+| CONFLICT   | GROUP       | MINUS        | SLASH      | WHERE     |
+| CONNECTION |             |              |            |           |
+

documentation20/cn/12.taos-sql/docs.md

@@ -4,6 +4,8 @@
 
 TAOS SQL 是用户对 TDengine 进行数据写入和查询的主要工具。TAOS SQL 为了便于用户快速上手，在一定程度上提供类似于标准 SQL 类似的风格和模式。严格意义上，TAOS SQL 并不是也不试图提供 SQL 标准的语法。此外，由于 TDengine 针对的时序性结构化数据不提供删除功能，因此在 TAO SQL 中不提供数据删除的相关功能。
 
+TAOS SQL 不支持关键字的缩写，例如 DESCRIBE 不能缩写为 DESC。 
+
 本章节 SQL 语法遵循如下约定：
 
 - < > 里的内容是用户需要输入的，但不要输入 <> 本身
@@ -113,7 +115,7 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
     ```mysql
     ALTER DATABASE db_name QUORUM 2;
     ```
-    QUORUM 参数是指数据写入成功所需要的确认数，取值范围 [1, 3]。对于异步复制，quorum 设为 1，具有 master 角色的虚拟节点自己确认即可。对于同步复制，需要至少大于等于 2。原则上，Quorum >= 1 并且 Quorum <= replica(副本数)，这个参数在启动一个同步模块实例时需要提供。
+    QUORUM 参数是指数据写入成功所需要的确认数，取值范围 [1, 2]。对于异步复制，quorum 设为 1，具有 master 角色的虚拟节点自己确认即可。对于同步复制，需要至少大于等于 2。原则上，Quorum >= 1 并且 Quorum <= replica(副本数)，这个参数在启动一个同步模块实例时需要提供。
 
     ```mysql
     ALTER DATABASE db_name BLOCKS 100;
@@ -146,7 +148,7 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
 
     2) 表名最大长度为 192；
 
-    3) 表的每行长度不能超过16k个字符;
+    3) 表的每行长度不能超过 16k 个字符;（注意：每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置）
 
     4) 子表名只能由字母、数字和下划线组成，且不能以数字开头
 
@@ -402,8 +404,8 @@ SELECT select_expr [, select_expr ...]
     FROM {tb_name_list}
     [WHERE where_condition]
     [INTERVAL (interval_val [, interval_offset])]
+    [SLIDING sliding_val]
     [FILL fill_val]
-    [SLIDING fill_val]
     [GROUP BY col_list]
     [ORDER BY col_list { DESC | ASC }]
     [SLIMIT limit_val [, SOFFSET offset_val]]
@@ -620,9 +622,10 @@ Query OK, 1 row(s) in set (0.001091s)
 ```
 
 - 可以使用 * 返回所有列，或指定列名。可以对数字列进行四则运算，可以给输出的列取列名
-- where语句可以使用各种逻辑判断来过滤数字值，或使用通配符来过滤字符串
+- WHERE 语句可以使用各种逻辑判断来过滤数字值，或使用通配符来过滤字符串
 - 输出结果缺省按首列时间戳升序排序，但可以指定按降序排序( _c0 指首列时间戳)。使用 ORDER BY 对其他字段进行排序为非法操作。
 - 参数 LIMIT 控制输出条数，OFFSET 指定从第几条开始输出。LIMIT/OFFSET 对结果集的执行顺序在 ORDER BY 之后。
+- 参数 SLIMIT 控制由 GROUP BY 指令划分的每个分组中的输出条数。
 - 通过”>>"输出结果可以导出到指定文件
 
 ### 支持的条件过滤操作
@@ -1162,17 +1165,20 @@ TDengine支持按时间段进行聚合，可以将表中数据按照时间段进
 SELECT function_list FROM tb_name
   [WHERE where_condition]
   INTERVAL (interval [, offset])
+  [SLIDING sliding]
   [FILL ({NONE | VALUE | PREV | NULL | LINEAR})]
 
 SELECT function_list FROM stb_name
   [WHERE where_condition]
   INTERVAL (interval [, offset])
+  [SLIDING sliding]
   [FILL ({ VALUE | PREV | NULL | LINEAR})]
   [GROUP BY tags]
 ```
 
 - 聚合时间段的长度由关键词INTERVAL指定，最短时间间隔10毫秒（10a），并且支持偏移（偏移必须小于间隔）。聚合查询中，能够同时执行的聚合和选择函数仅限于单个输出的函数：count、avg、sum 、stddev、leastsquares、percentile、min、max、first、last，不能使用具有多行输出结果的函数（例如：top、bottom、diff以及四则运算）。
 - WHERE语句可以指定查询的起止时间和其他过滤条件
+- SLIDING语句用于指定聚合时间段的前向增量
 - FILL语句指定某一时间区间数据缺失的情况下的填充模式。填充模式包括以下几种：
   * 不进行填充：NONE(默认填充模式)。
   * VALUE填充：固定值填充，此时需要指定填充的数值。例如：fill(value, 1.23)。
@@ -1184,6 +1190,8 @@ SELECT function_list FROM stb_name
   2. 在时间维度聚合中，返回的结果中时间序列严格单调递增。
   3. 如果查询对象是超级表，则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用group by语句，则返回的结果按照时间序列严格单调递增；如果查询中使用了group by语句分组，则返回结果中每个group内不按照时间序列严格单调递增。
 
+时间聚合也常被用于连续查询场景，可以参考文档 [连续查询(Continuous Query)](https://www.taosdata.com/cn/documentation/advanced-features#continuous-query)。
+
 **示例:** 智能电表的建表语句如下：
 
 ```mysql
@@ -1203,7 +1211,7 @@ SELECT AVG(current), MAX(current), LEASTSQUARES(current, start_val, step_val), P
 ## <a class="anchor" id="limitation"></a>TAOS SQL 边界限制
 
 - 数据库名最大长度为 32
-- 表名最大长度为192，每行数据最大长度16k个字符
+- 表名最大长度为 192，每行数据最大长度 16k 个字符（注意：数据行内每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置）
 - 列名最大长度为 64，最多允许 1024 列，最少需要 2 列，第一列必须是时间戳
 - 标签最多允许 128 个，可以 1 个，标签总长度不超过 16k 个字符
 - SQL 语句最大长度 65480 个字符，但可通过系统配置参数 maxSQLLength 修改，最长可配置为 1M
@@ -1222,3 +1230,4 @@ TAOS SQL支持表之间按主键时间戳来join两张表的列，暂不支持
 **is not null与不为空的表达式适用范围**
 
 is not null支持所有类型的列。不为空的表达式为 <>""，仅对非数值类型的列适用。
+

documentation20/cn/13.faq/docs.md

@@ -92,6 +92,8 @@ TDengine 目前尚不支持删除功能，未来根据用户需求可能会支
 
 从 2.0.8.0 开始，TDengine 支持更新已经写入数据的功能。使用更新功能需要在创建数据库时使用 UPDATE 1 参数，之后可以使用 INSERT INTO 命令更新已经写入的相同时间戳数据。UPDATE 参数不支持 ALTER DATABASE 命令修改。没有使用 UPDATE 1 参数创建的数据库，写入相同时间戳的数据不会修改之前的数据，也不会报错。
 
+另需注意，在 UPDATE 设置为 0 时，后发送的相同时间戳的数据会被直接丢弃，但并不会报错，而且仍然会被计入 affected rows （所以不能利用 INSERT 指令的返回信息进行时间戳查重）。这样设计的主要原因是，TDengine 把写入的数据看做一个数据流，无论时间戳是否出现冲突，TDengine 都认为产生数据的原始设备真实地产生了这样的数据。UPDATE 参数只是控制这样的流数据在进行持久化时要怎样处理——UPDATE 为 0 时，表示先写入的数据覆盖后写入的数据；而 UPDATE 为 1 时，表示后写入的数据覆盖先写入的数据。这种覆盖关系如何选择，取决于对数据的后续使用和统计中，希望以先还是后生成的数据为准。
+
 ## 10. 我怎么创建超过1024列的表？
 
 使用2.0及其以上版本，默认支持1024列；2.0之前的版本，TDengine最大允许创建250列的表。但是如果确实超过限值，建议按照数据特性，逻辑地将这个宽表分解成几个小表。

packaging/docker/Dockerfile

@@ -12,9 +12,13 @@ RUN tar -zxf ${pkgFile}
 WORKDIR /root/${dirName}/
 RUN /bin/bash install.sh -e no
 
+RUN apt-get clean && apt-get update && apt-get install -y locales
+RUN locale-gen en_US.UTF-8
 ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib"
-ENV LANG=C.UTF-8
+ENV LC_CTYPE=en_US.UTF-8
-ENV LC_ALL=C.UTF-8
+ENV LANG=en_US.UTF-8
+ENV LC_ALL=en_US.UTF-8
+
 EXPOSE 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042
 CMD ["taosd"]
 VOLUME [ "/var/lib/taos", "/var/log/taos","/etc/taos/" ]

packaging/docker/dockerManifest.sh

@@ -36,10 +36,10 @@ done
 echo "verNumber=${verNumber}"
 
 #docker manifest create -a tdengine/tdengine:${verNumber} tdengine/tdengine-amd64:${verNumber} tdengine/tdengine-aarch64:${verNumber} tdengine/tdengine-aarch32:${verNumber}
-docker manifest create -a tdengine/tdengine tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest
+docker manifest create -a tdengine/tdengine:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest
 
 docker login -u tdengine -p ${passWord}  #replace the docker registry username and password
 
-docker manifest push tdengine/tdengine
+docker manifest push tdengine/tdengine:latest
 
 # how set latest version ???

snap/snapcraft.yaml

@@ -1,6 +1,6 @@
 name: tdengine
 base: core18
-version: '2.0.16.0'
+version: '2.0.17.0'
 icon: snap/gui/t-dengine.svg
 summary: an open-source big data platform designed and optimized for IoT. 
 description: |
@@ -72,7 +72,7 @@ parts:
       - usr/bin/taosd
       - usr/bin/taos
       - usr/bin/taosdemo
-      - usr/lib/libtaos.so.2.0.16.0
+      - usr/lib/libtaos.so.2.0.17.0
       - usr/lib/libtaos.so.1
       - usr/lib/libtaos.so
 

src/client/src/tscSQLParser.c

@@ -1872,6 +1872,24 @@ void setResultColName(char* name, tSqlExprItem* pItem, int32_t functionId, SStrT
   }
 }
 
+
+void setLastOrderForGoupBy(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo) {  // todo refactor
+  SSqlGroupbyExpr* pGroupBy = &pQueryInfo->groupbyExpr;
+  if (pGroupBy->numOfGroupCols > 0) {
+    size_t idx = taosArrayGetSize(pQueryInfo->exprList);
+    for(int32_t k = 0; k < pGroupBy->numOfGroupCols; ++k) {
+      SColIndex* pIndex = taosArrayGet(pGroupBy->columnInfo, k);
+      if (!TSDB_COL_IS_TAG(pIndex->flag) && pIndex->colIndex < tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { // group by normal columns
+        SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, idx - 1);
+        pExpr->numOfParams = 1;
+        pExpr->param->i64 = TSDB_ORDER_ASC;
+
+        break;
+      }
+    }
+  }
+}
+
 int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSqlExprItem* pItem, bool finalResult) {
   STableMetaInfo* pTableMetaInfo = NULL;
   int32_t optr = pItem->pNode->nSQLOptr;
@@ -2152,6 +2170,10 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
               if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[j], cvtFunc, name, colIndex++, &index, finalResult) != 0) {
                 return TSDB_CODE_TSC_INVALID_SQL;
               }
+
+              if (optr == TK_LAST) {
+                setLastOrderForGoupBy(pQueryInfo, pTableMetaInfo);
+              }
             }
 
           } else {
@@ -2173,24 +2195,12 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
             bool multiColOutput = pItem->pNode->pParam->nExpr > 1;
             setResultColName(name, pItem, cvtFunc.originFuncId, &pParamElem->pNode->colInfo, multiColOutput);
 
-            if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, name, colIndex + i, &index, finalResult) != 0) {
+            if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, name, colIndex++, &index, finalResult) != 0) {
               return TSDB_CODE_TSC_INVALID_SQL;
             }
 
-            if (optr == TK_LAST) {  // todo refactor
+            if (optr == TK_LAST) {
-              SSqlGroupbyExpr* pGroupBy = &pQueryInfo->groupbyExpr;
+              setLastOrderForGoupBy(pQueryInfo, pTableMetaInfo);
-              if (pGroupBy->numOfGroupCols > 0) {
-                for(int32_t k = 0; k < pGroupBy->numOfGroupCols; ++k) {
-                  SColIndex* pIndex = taosArrayGet(pGroupBy->columnInfo, k);
-                  if (!TSDB_COL_IS_TAG(pIndex->flag) && pIndex->colIndex < tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { // group by normal columns
-                    SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, colIndex + i);
-                    pExpr->numOfParams = 1;
-                    pExpr->param->i64 = TSDB_ORDER_ASC;
-
-                    break;
-                  }
-                }
-              }
             }
           }
         }
@@ -2220,6 +2230,10 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
             }
 
             colIndex++;
+
+            if (optr == TK_LAST) {
+              setLastOrderForGoupBy(pQueryInfo, pTableMetaInfo);
+            }
           }
 
           numOfFields += tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
@@ -5614,6 +5628,8 @@ static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDbInfo* pCreateDb) {
   pMsg->ignoreExist = pCreateDb->ignoreExists;
   pMsg->update = pCreateDb->update;
   pMsg->cacheLastRow = pCreateDb->cachelast;
+  pMsg->dbType = pCreateDb->dbType;
+  pMsg->partitions = htons(pCreateDb->partitions);
 }
 
 int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDbInfo* pCreateDbSql) {
@@ -6244,6 +6260,15 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate) {
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
   }
 
+  val = (int16_t)htons(pCreate->partitions);
+  if (val != -1 &&
+      (val < TSDB_MIN_DB_PARTITON_OPTION || val > TSDB_MAX_DB_PARTITON_OPTION)) {
+    snprintf(msg, tListLen(msg), "invalid topic option partition: %d valid range: [%d, %d]", val,
+             TSDB_MIN_DB_PARTITON_OPTION, TSDB_MAX_DB_PARTITON_OPTION);
+    return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
+  }
+
+
   return TSDB_CODE_SUCCESS;
 }
 

src/client/src/tscServer.c

@@ -1055,7 +1055,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   SSqlCmd *pCmd = &pSql->cmd;
   pCmd->payloadLen = sizeof(SCreateDbMsg);
-  pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_DB;
+  
+  pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_CM_CREATE_DB : TSDB_MSG_TYPE_CM_CREATE_TP;
 
   SCreateDbMsg *pCreateDbMsg = (SCreateDbMsg *)pCmd->payload;
 
@@ -1187,7 +1188,7 @@ int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
   pDropDbMsg->ignoreNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0;
 
-  pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DB;
+  pCmd->msgType = (pInfo->pMiscInfo->dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_CM_DROP_DB : TSDB_MSG_TYPE_CM_DROP_TP;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1514,9 +1515,11 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) {
 int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   SSqlCmd *pCmd = &pSql->cmd;
   pCmd->payloadLen = sizeof(SAlterDbMsg);
-  pCmd->msgType = TSDB_MSG_TYPE_CM_ALTER_DB;
+  pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_CM_ALTER_DB : TSDB_MSG_TYPE_CM_ALTER_TP;
 
   SAlterDbMsg *pAlterDbMsg = (SAlterDbMsg* )pCmd->payload;
+  pAlterDbMsg->dbType = -1;
+  
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
   tNameExtractFullName(&pTableMetaInfo->name, pAlterDbMsg->db);
 

src/client/src/tscSubquery.c

@@ -1941,7 +1941,11 @@ void tscFirstRoundRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) {
 
           // tag or group by column
           if (TSDB_COL_IS_TAG(pExpr->colInfo.flag) || pExpr->functionId == TSDB_FUNC_PRJ) {
+            if (row[i] == NULL) {
+              setNull(p + offset, pExpr->resType, pExpr->resBytes);
+            } else {
               memcpy(p + offset, row[i], length[i]);
+            }
             offset += pExpr->resBytes;
           }
         }

src/common/inc/tglobal.h

@@ -95,6 +95,7 @@ extern int8_t  tsCompression;
 extern int8_t  tsWAL;
 extern int32_t tsFsyncPeriod;
 extern int32_t tsReplications;
+extern int16_t tsPartitons;
 extern int32_t tsQuorum;
 extern int8_t  tsUpdate;
 extern int8_t  tsCacheLastRow;

src/common/src/tglobal.c

@@ -126,8 +126,9 @@ int8_t  tsWAL           = TSDB_DEFAULT_WAL_LEVEL;
 int32_t tsFsyncPeriod   = TSDB_DEFAULT_FSYNC_PERIOD;
 int32_t tsReplications  = TSDB_DEFAULT_DB_REPLICA_OPTION;
 int32_t tsQuorum        = TSDB_DEFAULT_DB_QUORUM_OPTION;
+int16_t tsPartitons     = TSDB_DEFAULT_DB_PARTITON_OPTION;
 int8_t  tsUpdate        = TSDB_DEFAULT_DB_UPDATE_OPTION;
-int8_t  tsCacheLastRow  = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
+int8_t  tsCacheLastRow  = TSDB_DEFAULT_CACHE_LAST_ROW;
 int32_t tsMaxVgroupsPerDb  = 0;
 int32_t tsMinTablePerVnode = TSDB_TABLES_STEP;
 int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES;
@@ -853,6 +854,16 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "partitions";
+  cfg.ptr = &tsPartitons;
+  cfg.valType = TAOS_CFG_VTYPE_INT16;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = TSDB_MIN_DB_PARTITON_OPTION;
+  cfg.maxValue = TSDB_MAX_DB_PARTITON_OPTION;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   cfg.option = "quorum";
   cfg.ptr = &tsQuorum;
   cfg.valType = TAOS_CFG_VTYPE_INT32;

src/connector/jdbc/pom.xml

@@ -102,6 +102,7 @@
                         <include>**/*Test.java</include>
                     </includes>
                     <excludes>
+                        <exclude>**/DatetimeBefore1970Test.java</exclude>
                         <exclude>**/AppMemoryLeakTest.java</exclude>
                         <exclude>**/AuthenticationTest.java</exclude>
                         <exclude>**/TaosInfoMonitorTest.java</exclude>

src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDriver.java

@@ -1,76 +1,12 @@
 package com.taosdata.jdbc;
 
-import java.io.*;
 import java.sql.Driver;
 import java.sql.DriverPropertyInfo;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Properties;
 import java.util.StringTokenizer;
 
 public abstract class AbstractDriver implements Driver {
 
-    private static final String TAOS_CFG_FILENAME = "taos.cfg";
-
-    /**
-     * @param cfgDirPath
-     * @return return the config dir
-     **/
-    protected File loadConfigDir(String cfgDirPath) {
-        if (cfgDirPath == null)
-            return loadDefaultConfigDir();
-        File cfgDir = new File(cfgDirPath);
-        if (!cfgDir.exists())
-            return loadDefaultConfigDir();
-        return cfgDir;
-    }
-
-    /**
-     * @return search the default config dir, if the config dir is not exist will return null
-     */
-    protected File loadDefaultConfigDir() {
-        File cfgDir;
-        File cfgDir_linux = new File("/etc/taos");
-        cfgDir = cfgDir_linux.exists() ? cfgDir_linux : null;
-        File cfgDir_windows = new File("C:\\TDengine\\cfg");
-        cfgDir = (cfgDir == null && cfgDir_windows.exists()) ? cfgDir_windows : cfgDir;
-        return cfgDir;
-    }
-
-    protected List<String> loadConfigEndpoints(File cfgFile) {
-        List<String> endpoints = new ArrayList<>();
-        try (BufferedReader reader = new BufferedReader(new FileReader(cfgFile))) {
-            String line = null;
-            while ((line = reader.readLine()) != null) {
-                if (line.trim().startsWith("firstEp") || line.trim().startsWith("secondEp")) {
-                    endpoints.add(line.substring(line.indexOf('p') + 1).trim());
-                }
-                if (endpoints.size() > 1)
-                    break;
-            }
-        } catch (FileNotFoundException e) {
-            e.printStackTrace();
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-        return endpoints;
-    }
-
-    protected void loadTaosConfig(Properties info) {
-        if ((info.getProperty(TSDBDriver.PROPERTY_KEY_HOST) == null ||
-                info.getProperty(TSDBDriver.PROPERTY_KEY_HOST).isEmpty()) && (
-                info.getProperty(TSDBDriver.PROPERTY_KEY_PORT) == null ||
-                        info.getProperty(TSDBDriver.PROPERTY_KEY_PORT).isEmpty())) {
-            File cfgDir = loadConfigDir(info.getProperty(TSDBDriver.PROPERTY_KEY_CONFIG_DIR));
-            File cfgFile = cfgDir.listFiles((dir, name) -> TAOS_CFG_FILENAME.equalsIgnoreCase(name))[0];
-            List<String> endpoints = loadConfigEndpoints(cfgFile);
-            if (!endpoints.isEmpty()) {
-                info.setProperty(TSDBDriver.PROPERTY_KEY_HOST, endpoints.get(0).split(":")[0]);
-                info.setProperty(TSDBDriver.PROPERTY_KEY_PORT, endpoints.get(0).split(":")[1]);
-            }
-        }
-    }
-
     protected DriverPropertyInfo[] getPropertyInfo(Properties info) {
         DriverPropertyInfo hostProp = new DriverPropertyInfo(TSDBDriver.PROPERTY_KEY_HOST, info.getProperty(TSDBDriver.PROPERTY_KEY_HOST));
         hostProp.required = false;
@@ -156,6 +92,4 @@ public abstract class AbstractDriver implements Driver {
         return urlProps;
     }
 
-
-
 }

src/connector/jdbc/src/main/java/com/taosdata/jdbc/ColumnMetaData.java

@@ -52,4 +52,14 @@ public class ColumnMetaData {
     public void setColIndex(int colIndex) {
         this.colIndex = colIndex;
     }
+
+    @Override
+    public String toString() {
+        return "ColumnMetaData{" +
+                "colType=" + colType +
+                ", colName='" + colName + '\'' +
+                ", colSize=" + colSize +
+                ", colIndex=" + colIndex +
+                '}';
+    }
 }

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java

@@ -87,11 +87,10 @@ public class TSDBConnection extends AbstractConnection {
     }
 
     public void close() throws SQLException {
-        if (isClosed()) {
+        if (isClosed)
-            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+            return;
-        }
-        this.isClosed = true;
         this.connector.closeConnection();
+        this.isClosed = true;
     }
 
     public boolean isClosed() throws SQLException {

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java

@@ -112,8 +112,6 @@ public class TSDBDriver extends AbstractDriver {
         if ((props = parseURL(url, info)) == null) {
             return null;
         }
-        //load taos.cfg start
-        loadTaosConfig(info);
 
         try {
             TSDBJNIConnector.init((String) props.get(PROPERTY_KEY_CONFIG_DIR), (String) props.get(PROPERTY_KEY_LOCALE),

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java

@@ -1,4 +1,5 @@
-/***************************************************************************
+/**
+ * *************************************************************************
  * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
  *
  * This program is free software: you can use, redistribute, and/or modify
@@ -20,6 +21,9 @@ import java.sql.SQLException;
 import java.sql.SQLWarning;
 import java.util.List;
 
+/**
+ * JNI connector
+ * */
 public class TSDBJNIConnector {
     private static volatile Boolean isInitialized = false;
 

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java

@@ -20,18 +20,16 @@ import java.util.ArrayList;
 import java.util.List;
 
 public class TSDBResultSet extends AbstractResultSet implements ResultSet {
-    private TSDBJNIConnector jniConnector;
+    private final TSDBJNIConnector jniConnector;
-
     private final TSDBStatement statement;
-    private long resultSetPointer = 0L;
+    private final long resultSetPointer;
     private List<ColumnMetaData> columnMetaDataList = new ArrayList<>();
-
+    private final TSDBResultSetRowData rowData;
-    private TSDBResultSetRowData rowData;
+    private final TSDBResultSetBlockData blockData;
-    private TSDBResultSetBlockData blockData;
 
     private boolean batchFetch = false;
     private boolean lastWasNull = false;
-    private final int COLUMN_INDEX_START_VALUE = 1;
+    private boolean isClosed;
 
     public void setBatchFetch(boolean batchFetch) {
         this.batchFetch = batchFetch;
@@ -56,13 +54,13 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet {
 
         int code = this.jniConnector.getSchemaMetaData(this.resultSetPointer, this.columnMetaDataList);
         if (code == TSDBConstants.JNI_CONNECTION_NULL) {
-            throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL);
         }
         if (code == TSDBConstants.JNI_RESULT_SET_NULL) {
-            throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL));
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_RESULT_SET_NULL);
         }
         if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) {
-            throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0));
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_NUM_OF_FIELDS_0);
         }
         this.rowData = new TSDBResultSetRowData(this.columnMetaDataList.size());
         this.blockData = new TSDBResultSetBlockData(this.columnMetaDataList, this.columnMetaDataList.size());
@@ -78,16 +76,12 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet {
             this.blockData.reset();
 
             if (code == TSDBConstants.JNI_CONNECTION_NULL) {
-                throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL);
             } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) {
-                throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL));
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_RESULT_SET_NULL);
             } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) {
-                throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0));
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_NUM_OF_FIELDS_0);
-            } else if (code == TSDBConstants.JNI_FETCH_END) {
+            } else return code != TSDBConstants.JNI_FETCH_END;
-                return false;
-            }
-
-            return true;
         } else {
             if (rowData != null) {
                 this.rowData.clear();
@@ -95,11 +89,11 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet {
 
             int code = this.jniConnector.fetchRow(this.resultSetPointer, this.rowData);
             if (code == TSDBConstants.JNI_CONNECTION_NULL) {
-                throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL);
             } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) {
-                throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL));
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_RESULT_SET_NULL);
             } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) {
-                throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0));
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_NUM_OF_FIELDS_0);
             } else if (code == TSDBConstants.JNI_FETCH_END) {
                 return false;
             } else {
@@ -109,14 +103,17 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet {
     }
 
     public void close() throws SQLException {
+        if (isClosed)
+            return;
         if (this.jniConnector != null) {
             int code = this.jniConnector.freeResultSet(this.resultSetPointer);
             if (code == TSDBConstants.JNI_CONNECTION_NULL) {
-                throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL);
             } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) {
-                throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL));
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_RESULT_SET_NULL);
             }
         }
+        isClosed = true;
     }
 
     public boolean wasNull() throws SQLException {
@@ -415,8 +412,8 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet {
     }
 
     public boolean isClosed() throws SQLException {
-        //TODO: check if need release resources
+        if (isClosed)
-        boolean isClosed = true;
+            return true;
         if (jniConnector != null) {
             isClosed = jniConnector.isResultsetClosed();
         }
@@ -429,14 +426,12 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet {
     }
 
     private int getTrueColumnIndex(int columnIndex) throws SQLException {
-        if (columnIndex < this.COLUMN_INDEX_START_VALUE) {
+        if (columnIndex < 1)
-            throw new SQLException("Column Index out of range, " + columnIndex + " < " + this.COLUMN_INDEX_START_VALUE);
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE, "columnIndex(" + columnIndex + "): < 1");
-        }
 
         int numOfCols = this.columnMetaDataList.size();
-        if (columnIndex > numOfCols) {
+        if (columnIndex > numOfCols)
-            throw new SQLException("Column Index out of range, " + columnIndex + " > " + numOfCols);
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE, "columnIndex: " + columnIndex);
-        }
         return columnIndex - 1;
     }
 }

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java

@@ -73,12 +73,12 @@ public class TSDBStatement extends AbstractStatement {
     }
 
     public void close() throws SQLException {
-        if (!isClosed) {
+        if (isClosed)
-            if (this.resultSet != null)
+            return;
+        if (this.resultSet != null && !this.resultSet.isClosed())
             this.resultSet.close();
         isClosed = true;
     }
-    }
 
     public boolean execute(String sql) throws SQLException {
         // check if closed

src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBJNIConnectorTest.java

@@ -0,0 +1,89 @@
+package com.taosdata.jdbc;
+
+import org.junit.Test;
+
+import java.sql.SQLException;
+import java.sql.SQLWarning;
+import java.util.ArrayList;
+import java.util.List;
+
+public class TSDBJNIConnectorTest {
+
+    private static TSDBResultSetRowData rowData;
+
+    @Test
+    public void test() {
+        try {
+            // init
+            TSDBJNIConnector.init(null, null, null, null);
+            // connect
+            TSDBJNIConnector connector = new TSDBJNIConnector();
+            connector.connect("127.0.0.1", 6030, null, "root", "taosdata");
+            // executeQuery
+            long pSql = connector.executeQuery("show variables");
+            if (connector.isUpdateQuery(pSql)) {
+                connector.freeResultSet(pSql);
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_WITH_EXECUTEQUERY);
+            }
+            // get schema
+            List<ColumnMetaData> columnMetaDataList = new ArrayList<>();
+            int code = connector.getSchemaMetaData(pSql, columnMetaDataList);
+            if (code == TSDBConstants.JNI_CONNECTION_NULL) {
+                throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
+            }
+            if (code == TSDBConstants.JNI_RESULT_SET_NULL) {
+                throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL));
+            }
+            if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) {
+                throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0));
+            }
+            int columnSize = columnMetaDataList.size();
+            // print metadata
+            for (int i = 0; i < columnSize; i++) {
+                System.out.println(columnMetaDataList.get(i));
+            }
+            rowData = new TSDBResultSetRowData(columnSize);
+            // iterate resultSet
+            for (int i = 0; next(connector, pSql); i++) {
+                System.out.println("col[" + i + "] size: " + rowData.getColSize());
+                rowData.getData().stream().forEach(col -> System.out.print(col + "\t"));
+                System.out.println();
+            }
+            // close resultSet
+            code = connector.freeResultSet(pSql);
+            if (code == TSDBConstants.JNI_CONNECTION_NULL) {
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL);
+            } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) {
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_RESULT_SET_NULL);
+            }
+            // close statement
+
+            // close connection
+            connector.closeConnection();
+
+        } catch (SQLWarning throwables) {
+            throwables.printStackTrace();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+    private static boolean next(TSDBJNIConnector connector, long pSql) throws SQLException {
+        if (rowData != null)
+            rowData.clear();
+
+        int code = connector.fetchRow(pSql, rowData);
+        if (code == TSDBConstants.JNI_CONNECTION_NULL) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL);
+        } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_RESULT_SET_NULL);
+        } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_NUM_OF_FIELDS_0);
+        } else if (code == TSDBConstants.JNI_FETCH_END) {
+            return false;
+        } else {
+            return true;
+        }
+    }
+
+}

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DatetimeBefore1970Test.java

@@ -0,0 +1,68 @@
+package com.taosdata.jdbc.cases;
+
+import com.taosdata.jdbc.utils.TimestampUtil;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.sql.*;
+
+public class DatetimeBefore1970Test {
+
+    private static Connection conn;
+
+    @Test
+    public void test() {
+        try (Statement stmt = conn.createStatement()) {
+            stmt.executeUpdate("insert into weather(ts) values('1969-12-31 23:59:59.999')");
+            stmt.executeUpdate("insert into weather(ts) values('1970-01-01 00:00:00.000')");
+            stmt.executeUpdate("insert into weather(ts) values('1970-01-01 08:00:00.000')");
+            stmt.executeUpdate("insert into weather(ts) values('1970-01-01 07:59:59.999')");
+
+            ResultSet rs = stmt.executeQuery("select * from weather");
+            while (rs.next()) {
+                Timestamp ts = rs.getTimestamp("ts");
+                System.out.println("long: " + ts.getTime() + ", string: " + TimestampUtil.longToDatetime(ts.getTime()));
+            }
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+    public static void main(String[] args) {
+        System.out.println("timestamp: " + Long.MAX_VALUE + ", string: " + TimestampUtil.longToDatetime(Long.MAX_VALUE));
+        System.out.println("timestamp: " + Long.MIN_VALUE + ", string: " + TimestampUtil.longToDatetime(Long.MIN_VALUE));
+        System.out.println("timestamp: " + 0 + ", string: " + TimestampUtil.longToDatetime(0));
+        System.out.println("timestamp: " + -1 + ", string: " + TimestampUtil.longToDatetime(-1));
+        String datetime = "1970-01-01 00:00:00.000";
+        System.out.println("timestamp: " + TimestampUtil.datetimeToLong(datetime) + ", string: " + datetime);
+        datetime = "1969-12-31 23:59:59.999";
+        System.out.println("timestamp: " + TimestampUtil.datetimeToLong(datetime) + ", string: " + datetime);
+    }
+
+    @BeforeClass
+    public static void beforeClass() {
+        try {
+            Class.forName("com.taosdata.jdbc.TSDBDriver");
+            conn = DriverManager.getConnection("jdbc:TAOS://127.0.0.1:6030/?user=root&password=taosdata");
+            Statement stmt = conn.createStatement();
+            stmt.execute("drop database if exists test_timestamp");
+            stmt.execute("create database if not exists test_timestamp keep 36500");
+            stmt.execute("use test_timestamp");
+            stmt.execute("create table weather(ts timestamp,f1 float)");
+            stmt.close();
+        } catch (ClassNotFoundException | SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+    @AfterClass
+    public static void afterClass() {
+        try {
+            if (conn != null)
+                conn.close();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+}

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/FailOverTest.java

@@ -4,7 +4,6 @@ import org.junit.Test;
 
 import java.sql.*;
 import java.text.SimpleDateFormat;
-import java.util.Date;
 import java.util.concurrent.TimeUnit;
 
 public class FailOverTest {
@@ -18,13 +17,17 @@ public class FailOverTest {
 
         long end = System.currentTimeMillis() + 1000 * 60 * 5;
         while (System.currentTimeMillis() < end) {
-            try (Connection conn = DriverManager.getConnection(url)) {
+            try (Connection conn = DriverManager.getConnection(url); Statement stmt = conn.createStatement()) {
-                Statement stmt = conn.createStatement();
+                ResultSet rs = stmt.executeQuery("show dnodes");
-                ResultSet resultSet = stmt.executeQuery("select server_status()");
+                ResultSetMetaData meta = rs.getMetaData();
-                resultSet.next();
+                while (rs.next()) {
-                int status = resultSet.getInt("server_status()");
+                    for (int i = 1; i <= meta.getColumnCount(); i++) {
-                System.out.println(">>>>>>>>>" + sdf.format(new Date()) + " status : " + status);
+                        System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t");
-                stmt.close();
+                    }
+                    System.out.println();
+                }
+                System.out.println("=======================");
+                rs.close();
                 TimeUnit.SECONDS.sleep(5);
             } catch (SQLException | InterruptedException e) {
                 e.printStackTrace();

src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/TimestampUtil.java

@@ -4,7 +4,7 @@ import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 
-public class TimeStampUtil {
+public class TimestampUtil {
 
     private static final String datetimeFormat = "yyyy-MM-dd HH:mm:ss.SSS";
 

src/dnode/CMakeLists.txt

@@ -31,6 +31,10 @@ IF (TD_MQTT)
   TARGET_LINK_LIBRARIES(taosd mqtt)
 ENDIF ()
 
+IF (TD_TOPIC)
+  TARGET_LINK_LIBRARIES(taosd topic)
+ENDIF ()
+
 SET(PREPARE_ENV_CMD "prepare_env_cmd")
 SET(PREPARE_ENV_TARGET "prepare_env_target")
 ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD}

src/dnode/src/dnodeMain.c

@@ -189,6 +189,11 @@ static void dnodeCheckDataDirOpenned(char *dir) {
 }
 
 static int32_t dnodeInitStorage() {
+  if (tsDiskCfgNum == 1 && dnodeCreateDir(tsDataDir) < 0) {
+    dError("failed to create dir: %s, reason: %s", tsDataDir, strerror(errno));
+    return -1;
+  } 
+  
   if (tfsInit(tsDiskCfg, tsDiskCfgNum) < 0) {
     dError("failed to init TFS since %s", tstrerror(terrno));
     return -1;

src/dnode/src/dnodeShell.c

@@ -47,8 +47,11 @@ int32_t dnodeInitShell() {
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= dnodeDispatchToMWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE]  = dnodeDispatchToMWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB]   = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TP]   = dnodeDispatchToMWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB]     = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TP]     = dnodeDispatchToMWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB]    = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TP]    = dnodeDispatchToMWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= dnodeDispatchToMWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE]  = dnodeDispatchToMWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = dnodeDispatchToMWriteQueue;

src/dnode/src/dnodeVnodes.c

@@ -198,6 +198,14 @@ void dnodeCleanupVnodes() {
 static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
   if (pMsg->code != TSDB_CODE_SUCCESS) {
     dError("status rsp is received, error:%s", tstrerror(pMsg->code));
+    if (pMsg->code == TSDB_CODE_MND_DNODE_NOT_EXIST) {
+      char clusterId[TSDB_CLUSTER_ID_LEN];
+      dnodeGetClusterId(clusterId);
+      if (clusterId[0] != '\0') {
+	dError("exit zombie dropped dnode");
+	exit(EXIT_FAILURE);
+      }
+    }
     taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
     return;
   }

src/inc/taosdef.h

@@ -222,6 +222,9 @@ do { \
 #define TSDB_MQTT_TOPIC_LEN       64
 #define TSDB_MQTT_CLIENT_ID_LEN   32
 
+#define TSDB_DB_TYPE_DEFAULT      0
+#define TSDB_DB_TYPE_TOPIC        1
+
 #define TSDB_DEFAULT_PKT_SIZE     65480  //same as RPC_MAX_UDP_SIZE
 
 #define TSDB_PAYLOAD_SIZE         TSDB_DEFAULT_PKT_SIZE
@@ -306,6 +309,10 @@ do { \
 #define TSDB_MAX_DB_REPLICA_OPTION      3
 #define TSDB_DEFAULT_DB_REPLICA_OPTION  1
 
+#define TSDB_MIN_DB_PARTITON_OPTION     0
+#define TSDB_MAX_DB_PARTITON_OPTION     1000
+#define TSDB_DEFAULT_DB_PARTITON_OPTION 4
+
 #define TSDB_MIN_DB_QUORUM_OPTION       1
 #define TSDB_MAX_DB_QUORUM_OPTION       2
 #define TSDB_DEFAULT_DB_QUORUM_OPTION   1

src/inc/taoserror.h

@@ -185,6 +185,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_INVALID_DB_OPTION_DAYS    TAOS_DEF_ERROR_CODE(0, 0x0390)  //"Invalid database option: days out of range")
 #define TSDB_CODE_MND_INVALID_DB_OPTION_KEEP    TAOS_DEF_ERROR_CODE(0, 0x0391)  //"Invalid database option: keep >= keep1 >= keep0 >= days")
 
+#define TSDB_CODE_MND_INVALID_TOPIC             TAOS_DEF_ERROR_CODE(0, 0x0392)  //"Invalid topic name)
+#define TSDB_CODE_MND_INVALID_TOPIC_OPTION      TAOS_DEF_ERROR_CODE(0, 0x0393)  //"Invalid topic option)
+
 // dnode
 #define TSDB_CODE_DND_MSG_NOT_PROCESSED         TAOS_DEF_ERROR_CODE(0, 0x0400)  //"Message not processed")
 #define TSDB_CODE_DND_OUT_OF_MEMORY             TAOS_DEF_ERROR_CODE(0, 0x0401)  //"Dnode out of memory")

src/inc/taosmsg.h

@@ -107,6 +107,12 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY13, "dummy13" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY14, "dummy14" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "nettest" )
 
+// message for topic
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_TP, "create-tp" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_TP, "drop-tp" )	  
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_USE_TP, "use-tp" )	 
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_ALTER_TP, "alter-tp" )
+
 #ifndef TAOS_MESSAGE_C
   TSDB_MSG_TYPE_MAX  // 105
 #endif
@@ -141,6 +147,7 @@ enum _mgmt_table {
   TSDB_MGMT_TABLE_VNODES,
   TSDB_MGMT_TABLE_STREAMTABLES,
   TSDB_MGMT_TABLE_CLUSTER,
+  TSDB_MGMT_TABLE_TP,
   TSDB_MGMT_TABLE_MAX,
 };
 
@@ -555,7 +562,9 @@ typedef struct {
   int8_t   ignoreExist;
   int8_t   update;
   int8_t   cacheLastRow;
-  int8_t   reserve[8];
+  int8_t   dbType;
+  int16_t  partitions;
+  int8_t   reserve[5];
 } SCreateDbMsg, SAlterDbMsg;
 
 typedef struct {
@@ -674,7 +683,8 @@ typedef struct {
   int8_t   cacheLastRow;
   int32_t  vgCfgVersion;
   int8_t   dbReplica;
-  int8_t   reserved[9];
+  int8_t   dbType;
+  int8_t   reserved[8];
 } SVnodeCfg;
 
 typedef struct {

src/inc/tp.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TP
+#define TDENGINE_TP
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t tpInit();
+void    tpCleanUp();
+void    tpUpdateTs(int32_t *seq, void *pMsg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/inc/ttokendef.h

@@ -62,170 +62,176 @@
 #define TK_BITNOT                          43
 #define TK_SHOW                            44
 #define TK_DATABASES                       45
-#define TK_MNODES                          46
+#define TK_TOPICS                          46
-#define TK_DNODES                          47
+#define TK_MNODES                          47
-#define TK_ACCOUNTS                        48
+#define TK_DNODES                          48
-#define TK_USERS                           49
+#define TK_ACCOUNTS                        49
-#define TK_MODULES                         50
+#define TK_USERS                           50
-#define TK_QUERIES                         51
+#define TK_MODULES                         51
-#define TK_CONNECTIONS                     52
+#define TK_QUERIES                         52
-#define TK_STREAMS                         53
+#define TK_CONNECTIONS                     53
-#define TK_VARIABLES                       54
+#define TK_STREAMS                         54
-#define TK_SCORES                          55
+#define TK_VARIABLES                       55
-#define TK_GRANTS                          56
+#define TK_SCORES                          56
-#define TK_VNODES                          57
+#define TK_GRANTS                          57
-#define TK_IPTOKEN                         58
+#define TK_VNODES                          58
-#define TK_DOT                             59
+#define TK_IPTOKEN                         59
-#define TK_CREATE                          60
+#define TK_DOT                             60
-#define TK_TABLE                           61
+#define TK_CREATE                          61
-#define TK_DATABASE                        62
+#define TK_TABLE                           62
-#define TK_TABLES                          63
+#define TK_DATABASE                        63
-#define TK_STABLES                         64
+#define TK_TABLES                          64
-#define TK_VGROUPS                         65
+#define TK_STABLES                         65
-#define TK_DROP                            66
+#define TK_VGROUPS                         66
-#define TK_STABLE                          67
+#define TK_DROP                            67
-#define TK_DNODE                           68
+#define TK_STABLE                          68
-#define TK_USER                            69
+#define TK_TOPIC                           69
-#define TK_ACCOUNT                         70
+#define TK_DNODE                           70
-#define TK_USE                             71
+#define TK_USER                            71
-#define TK_DESCRIBE                        72
+#define TK_ACCOUNT                         72
-#define TK_ALTER                           73
+#define TK_USE                             73
-#define TK_PASS                            74
+#define TK_DESCRIBE                        74
-#define TK_PRIVILEGE                       75
+#define TK_ALTER                           75
-#define TK_LOCAL                           76
+#define TK_PASS                            76
-#define TK_IF                              77
+#define TK_PRIVILEGE                       77
-#define TK_EXISTS                          78
+#define TK_LOCAL                           78
-#define TK_PPS                             79
+#define TK_IF                              79
-#define TK_TSERIES                         80
+#define TK_EXISTS                          80
-#define TK_DBS                             81
+#define TK_PPS                             81
-#define TK_STORAGE                         82
+#define TK_TSERIES                         82
-#define TK_QTIME                           83
+#define TK_DBS                             83
-#define TK_CONNS                           84
+#define TK_STORAGE                         84
-#define TK_STATE                           85
+#define TK_QTIME                           85
-#define TK_KEEP                            86
+#define TK_CONNS                           86
-#define TK_CACHE                           87
+#define TK_STATE                           87
-#define TK_REPLICA                         88
+#define TK_KEEP                            88
-#define TK_QUORUM                          89
+#define TK_CACHE                           89
-#define TK_DAYS                            90
+#define TK_REPLICA                         90
-#define TK_MINROWS                         91
+#define TK_QUORUM                          91
-#define TK_MAXROWS                         92
+#define TK_DAYS                            92
-#define TK_BLOCKS                          93
+#define TK_MINROWS                         93
-#define TK_CTIME                           94
+#define TK_MAXROWS                         94
-#define TK_WAL                             95
+#define TK_BLOCKS                          95
-#define TK_FSYNC                           96
+#define TK_CTIME                           96
-#define TK_COMP                            97
+#define TK_WAL                             97
-#define TK_PRECISION                       98
+#define TK_FSYNC                           98
-#define TK_UPDATE                          99
+#define TK_COMP                            99
-#define TK_CACHELAST                      100
+#define TK_PRECISION                      100
-#define TK_LP                             101
+#define TK_UPDATE                         101
-#define TK_RP                             102
+#define TK_CACHELAST                      102
-#define TK_UNSIGNED                       103
+#define TK_PARTITIONS                     103
-#define TK_TAGS                           104
+#define TK_LP                             104
-#define TK_USING                          105
+#define TK_RP                             105
-#define TK_COMMA                          106
+#define TK_UNSIGNED                       106
-#define TK_AS                             107
+#define TK_TAGS                           107
-#define TK_NULL                           108
+#define TK_USING                          108
-#define TK_SELECT                         109
+#define TK_COMMA                          109
-#define TK_UNION                          110
+#define TK_AS                             110
-#define TK_ALL                            111
+#define TK_NULL                           111
-#define TK_DISTINCT                       112
+#define TK_SELECT                         112
-#define TK_FROM                           113
+#define TK_UNION                          113
-#define TK_VARIABLE                       114
+#define TK_ALL                            114
-#define TK_INTERVAL                       115
+#define TK_DISTINCT                       115
-#define TK_FILL                           116
+#define TK_FROM                           116
-#define TK_SLIDING                        117
+#define TK_VARIABLE                       117
-#define TK_ORDER                          118
+#define TK_INTERVAL                       118
-#define TK_BY                             119
+#define TK_FILL                           119
-#define TK_ASC                            120
+#define TK_SLIDING                        120
-#define TK_DESC                           121
+#define TK_ORDER                          121
-#define TK_GROUP                          122
+#define TK_BY                             122
-#define TK_HAVING                         123
+#define TK_ASC                            123
-#define TK_LIMIT                          124
+#define TK_DESC                           124
-#define TK_OFFSET                         125
+#define TK_GROUP                          125
-#define TK_SLIMIT                         126
+#define TK_HAVING                         126
-#define TK_SOFFSET                        127
+#define TK_LIMIT                          127
-#define TK_WHERE                          128
+#define TK_OFFSET                         128
-#define TK_NOW                            129
+#define TK_SLIMIT                         129
-#define TK_RESET                          130
+#define TK_SOFFSET                        130
-#define TK_QUERY                          131
+#define TK_WHERE                          131
-#define TK_ADD                            132
+#define TK_NOW                            132
-#define TK_COLUMN                         133
+#define TK_RESET                          133
-#define TK_TAG                            134
+#define TK_QUERY                          134
-#define TK_CHANGE                         135
+#define TK_ADD                            135
-#define TK_SET                            136
+#define TK_COLUMN                         136
-#define TK_KILL                           137
+#define TK_TAG                            137
-#define TK_CONNECTION                     138
+#define TK_CHANGE                         138
-#define TK_STREAM                         139
+#define TK_SET                            139
-#define TK_COLON                          140
+#define TK_KILL                           140
-#define TK_ABORT                          141
+#define TK_CONNECTION                     141
-#define TK_AFTER                          142
+#define TK_STREAM                         142
-#define TK_ATTACH                         143
+#define TK_COLON                          143
-#define TK_BEFORE                         144
+#define TK_ABORT                          144
-#define TK_BEGIN                          145
+#define TK_AFTER                          145
-#define TK_CASCADE                        146
+#define TK_ATTACH                         146
-#define TK_CLUSTER                        147
+#define TK_BEFORE                         147
-#define TK_CONFLICT                       148
+#define TK_BEGIN                          148
-#define TK_COPY                           149
+#define TK_CASCADE                        149
-#define TK_DEFERRED                       150
+#define TK_CLUSTER                        150
-#define TK_DELIMITERS                     151
+#define TK_CONFLICT                       151
-#define TK_DETACH                         152
+#define TK_COPY                           152
-#define TK_EACH                           153
+#define TK_DEFERRED                       153
-#define TK_END                            154
+#define TK_DELIMITERS                     154
-#define TK_EXPLAIN                        155
+#define TK_DETACH                         155
-#define TK_FAIL                           156
+#define TK_EACH                           156
-#define TK_FOR                            157
+#define TK_END                            157
-#define TK_IGNORE                         158
+#define TK_EXPLAIN                        158
-#define TK_IMMEDIATE                      159
+#define TK_FAIL                           159
-#define TK_INITIALLY                      160
+#define TK_FOR                            160
-#define TK_INSTEAD                        161
+#define TK_IGNORE                         161
-#define TK_MATCH                          162
+#define TK_IMMEDIATE                      162
-#define TK_KEY                            163
+#define TK_INITIALLY                      163
-#define TK_OF                             164
+#define TK_INSTEAD                        164
-#define TK_RAISE                          165
+#define TK_MATCH                          165
-#define TK_REPLACE                        166
+#define TK_KEY                            166
-#define TK_RESTRICT                       167
+#define TK_OF                             167
-#define TK_ROW                            168
+#define TK_RAISE                          168
-#define TK_STATEMENT                      169
+#define TK_REPLACE                        169
-#define TK_TRIGGER                        170
+#define TK_RESTRICT                       170
-#define TK_VIEW                           171
+#define TK_ROW                            171
-#define TK_COUNT                          172
+#define TK_STATEMENT                      172
-#define TK_SUM                            173
+#define TK_TRIGGER                        173
-#define TK_AVG                            174
+#define TK_VIEW                           174
-#define TK_MIN                            175
+#define TK_COUNT                          175
-#define TK_MAX                            176
+#define TK_SUM                            176
-#define TK_FIRST                          177
+#define TK_AVG                            177
-#define TK_LAST                           178
+#define TK_MIN                            178
-#define TK_TOP                            179
+#define TK_MAX                            179
-#define TK_BOTTOM                         180
+#define TK_FIRST                          180
-#define TK_STDDEV                         181
+#define TK_LAST                           181
-#define TK_PERCENTILE                     182
+#define TK_TOP                            182
-#define TK_APERCENTILE                    183
+#define TK_BOTTOM                         183
-#define TK_LEASTSQUARES                   184
+#define TK_STDDEV                         184
-#define TK_HISTOGRAM                      185
+#define TK_PERCENTILE                     185
-#define TK_DIFF                           186
+#define TK_APERCENTILE                    186
-#define TK_SPREAD                         187
+#define TK_LEASTSQUARES                   187
-#define TK_TWA                            188
+#define TK_HISTOGRAM                      188
-#define TK_INTERP                         189
+#define TK_DIFF                           189
-#define TK_LAST_ROW                       190
+#define TK_SPREAD                         190
-#define TK_RATE                           191
+#define TK_TWA                            191
-#define TK_IRATE                          192
+#define TK_INTERP                         192
-#define TK_SUM_RATE                       193
+#define TK_LAST_ROW                       193
-#define TK_SUM_IRATE                      194
+#define TK_RATE                           194
-#define TK_AVG_RATE                       195
+#define TK_IRATE                          195
-#define TK_AVG_IRATE                      196
+#define TK_SUM_RATE                       196
-#define TK_TBID                           197
+#define TK_SUM_IRATE                      197
-#define TK_SEMI                           198
+#define TK_AVG_RATE                       198
-#define TK_NONE                           199
+#define TK_AVG_IRATE                      199
-#define TK_PREV                           200
+#define TK_TBID                           200
-#define TK_LINEAR                         201
+#define TK_SEMI                           201
-#define TK_IMPORT                         202
+#define TK_NONE                           202
-#define TK_METRIC                         203
+#define TK_PREV                           203
-#define TK_TBNAME                         204
+#define TK_LINEAR                         204
-#define TK_JOIN                           205
+#define TK_IMPORT                         205
-#define TK_METRICS                        206
+#define TK_METRIC                         206
-#define TK_INSERT                         207
+#define TK_TBNAME                         207
-#define TK_INTO                           208
+#define TK_JOIN                           208
-#define TK_VALUES                         209
+#define TK_METRICS                        209
+#define TK_INSERT                         210
+#define TK_INTO                           211
+#define TK_VALUES                         212
+
+
+
 
 
 

src/kit/taosdump/taosdump.c

@@ -769,6 +769,7 @@ int32_t taosSaveTableOfMetricToTempFile(TAOS *taosCon, char* metric, struct argu
       }
       sprintf(tmpBuf, ".select-tbname.tmp");
       (void)remove(tmpBuf);      
+      free(tblBuf);
       close(fd);
       return -1;
     }
@@ -1523,6 +1524,7 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao
       }
       sprintf(tmpBuf, ".show-tables.tmp");
       (void)remove(tmpBuf);
+      free(tblBuf);
       close(fd);
       return -1;
     }

src/mnode/inc/mnodeDef.h

@@ -175,7 +175,9 @@ typedef struct {
   int8_t  quorum;
   int8_t  update;
   int8_t  cacheLastRow;
-  int8_t  reserved[10];
+  int8_t  dbType;
+  int16_t partitions;
+  int8_t  reserved[7];
 } SDbCfg;
 
 typedef struct SDbObj {

src/mnode/src/mnodeDb.c

@@ -22,6 +22,7 @@
 #include "tname.h"
 #include "tbn.h"
 #include "tdataformat.h"
+#include "tp.h"
 #include "mnode.h"
 #include "mnodeDef.h"
 #include "mnodeInt.h"
@@ -39,7 +40,7 @@
 
 #define VG_LIST_SIZE 8
 int64_t tsDbRid = -1;
-static void *  tsDbSdb = NULL;
+void *  tsDbSdb = NULL;
 static int32_t tsDbUpdateSize;
 
 static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg *pMsg);
@@ -48,8 +49,14 @@ static int32_t mnodeSetDbDropping(SDbObj *pDb);
 static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 static int32_t mnodeProcessCreateDbMsg(SMnodeMsg *pMsg);
-static int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg);
 static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg);
+int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg);
+
+#ifndef _TOPIC
+int32_t tpInit() { return 0; }
+void    tpCleanUp() {}
+void    tpUpdateTs(int32_t *seq, void *pMsg) {}
+#endif
 
 static void mnodeDestroyDb(SDbObj *pDb) {
   pthread_mutex_destroy(&pDb->mutex);
@@ -176,7 +183,7 @@ int32_t mnodeInitDbs() {
   mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_DB, mnodeCancelGetNextDb);
   
   mDebug("table:dbs table is created");
-  return 0;
+  return tpInit();
 }
 
 void *mnodeGetNextDb(void *pIter, SDbObj **pDb) {
@@ -332,6 +339,17 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) {
     return TSDB_CODE_MND_INVALID_DB_OPTION;
   }
 
+  if (pCfg->dbType < 0 || pCfg->dbType > 1) {
+    mError("invalid db option dbType:%d valid range: [%d, %d]", pCfg->dbType, 0, 1);
+    return TSDB_CODE_MND_INVALID_DB_OPTION;
+  }
+
+  if (pCfg->partitions < TSDB_MIN_DB_PARTITON_OPTION || pCfg->partitions > TSDB_MAX_DB_PARTITON_OPTION) {
+    mError("invalid db option partitions:%d valid range: [%d, %d]", pCfg->partitions, TSDB_MIN_DB_PARTITON_OPTION,
+           TSDB_MAX_DB_PARTITON_OPTION);
+    return TSDB_CODE_MND_INVALID_DB_OPTION;
+  }
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -354,6 +372,8 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) {
   if (pCfg->quorum < 0) pCfg->quorum = tsQuorum;
   if (pCfg->update < 0) pCfg->update = tsUpdate;
   if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = tsCacheLastRow;
+  if (pCfg->dbType < 0) pCfg->dbType = 0;
+  if (pCfg->partitions < 0) pCfg->partitions = tsPartitons;
 }
 
 static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) {
@@ -408,7 +428,9 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg *
     .replications        = pCreate->replications,
     .quorum              = pCreate->quorum,
     .update              = pCreate->update,
-    .cacheLastRow        = pCreate->cacheLastRow
+    .cacheLastRow        = pCreate->cacheLastRow,
+    .dbType              = pCreate->dbType,
+    .partitions          = pCreate->partitions
   };
 
   mnodeSetDefaultDbCfg(&pDb->cfg);
@@ -501,6 +523,7 @@ void mnodeRemoveVgroupFromDb(SVgObj *pVgroup) {
 }
 
 void mnodeCleanupDbs() {
+  tpCleanUp();
   sdbCloseTable(tsDbRid);
   tsDbSdb = NULL;
 }
@@ -660,7 +683,7 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
   return 0;
 }
 
-static char *mnodeGetDbStr(char *src) {
+char *mnodeGetDbStr(char *src) {
   char *pos = strstr(src, TS_PATH_DELIMITER);
   if (pos != NULL) ++pos;
 
@@ -852,6 +875,7 @@ static int32_t mnodeProcessCreateDbMsg(SMnodeMsg *pMsg) {
   pCreate->daysToKeep2     = htonl(pCreate->daysToKeep2);
   pCreate->commitTime      = htonl(pCreate->commitTime);
   pCreate->fsyncPeriod     = htonl(pCreate->fsyncPeriod);
+  pCreate->partitions      = htons(pCreate->partitions);
   pCreate->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock);
   pCreate->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock);
   
@@ -887,6 +911,8 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) {
   int8_t  precision      = pAlter->precision;
   int8_t  update         = pAlter->update;
   int8_t  cacheLastRow   = pAlter->cacheLastRow;
+  int8_t  dbType         = pAlter->dbType;
+  int16_t partitions     = htons(pAlter->partitions);
   
   terrno = TSDB_CODE_SUCCESS;
 
@@ -1004,6 +1030,16 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) {
     newCfg.cacheLastRow = cacheLastRow;
   }
 
+  if (dbType >= 0 && dbType != pDb->cfg.dbType) {
+    mDebug("db:%s, dbType:%d change to %d", pDb->name, pDb->cfg.dbType, dbType);
+    newCfg.dbType = dbType;
+  }
+
+  if (partitions >= 0 && partitions != pDb->cfg.partitions) {
+    mDebug("db:%s, partitions:%d change to %d", pDb->name, pDb->cfg.partitions, partitions);
+    newCfg.partitions = partitions;
+  }
+
   return newCfg;
 }
 
@@ -1031,6 +1067,8 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) {
 }
 
 static int32_t mnodeAlterDb(SDbObj *pDb, SAlterDbMsg *pAlter, void *pMsg) {
+  mDebug("db:%s, type:%d do alter operation", pDb->name, pDb->cfg.dbType);
+
   SDbCfg newCfg = mnodeGetAlterDbOption(pDb, pAlter);
   if (terrno != TSDB_CODE_SUCCESS) {
     return terrno;
@@ -1061,9 +1099,9 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SAlterDbMsg *pAlter, void *pMsg) {
   return code;
 }
 
-static int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg) {
+int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg) {
   SAlterDbMsg *pAlter = pMsg->rpcMsg.pCont;
-  mDebug("db:%s, alter db msg is received from thandle:%p", pAlter->db, pMsg->rpcMsg.handle);
+  mDebug("db:%s, alter db msg is received from thandle:%p, dbType:%d", pAlter->db, pMsg->rpcMsg.handle, pAlter->dbType);
 
   if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pAlter->db);
   if (pMsg->pDb == NULL) {

src/mnode/src/mnodeShow.c

@@ -109,6 +109,7 @@ static char *mnodeGetShowType(int32_t showType) {
     case TSDB_MGMT_TABLE_VNODES:  return "show vnodes";
     case TSDB_MGMT_TABLE_CLUSTER: return "show clusters";
     case TSDB_MGMT_TABLE_STREAMTABLES : return "show streamtables";
+    case TSDB_MGMT_TABLE_TP:      return "show topics";
     default:                      return "undefined";
   }
 }

src/mnode/src/mnodeVgroup.c

@@ -367,6 +367,11 @@ static int32_t mnodeAllocVgroupIdPool(SVgObj *pInputVgroup) {
     maxIdPoolSize = MAX(maxIdPoolSize, idPoolSize);
   }
 
+  // create one table each vnode
+  if (pDb->cfg.dbType == TSDB_DB_TYPE_TOPIC) {
+    maxIdPoolSize = 1;
+  }
+
   // new vgroup
   if (pInputVgroup->idPool == NULL) {
     pInputVgroup->idPool = taosInitIdPool(maxIdPoolSize);
@@ -379,6 +384,11 @@ static int32_t mnodeAllocVgroupIdPool(SVgObj *pInputVgroup) {
     }
   }
 
+  // create one table each vnode
+  if (pDb->cfg.dbType == TSDB_DB_TYPE_TOPIC) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   // realloc all vgroups in db
   int32_t newIdPoolSize;
   if (minIdPoolSize * 4 < tsTableIncStepPerVnode) {
@@ -449,6 +459,10 @@ int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSi
     maxVgroupsPerDb = MIN(maxVgroupsPerDb, TSDB_MAX_VNODES_PER_DB);
   }
 
+  if (pDb->cfg.dbType == TSDB_DB_TYPE_TOPIC) {
+    maxVgroupsPerDb = TSDB_MAX_DB_PARTITON_OPTION;
+  }
+
   int32_t code = TSDB_CODE_MND_NO_ENOUGH_DNODES;
   if (pDb->numOfVgroups < maxVgroupsPerDb) {
     mDebug("msg:%p, app:%p db:%s, try to create a new vgroup, numOfVgroups:%d maxVgroupsPerDb:%d", pMsg,
@@ -881,6 +895,7 @@ static SCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
   pCfg->update              = pDb->cfg.update;
   pCfg->cacheLastRow        = pDb->cfg.cacheLastRow;
   pCfg->dbReplica           = pDb->cfg.replications;
+  pCfg->dbType              = pDb->cfg.dbType;
   
   SVnodeDesc *pNodes = pVnode->nodes;
   for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) {

src/query/inc/qSqlparser.h

@@ -125,6 +125,8 @@ typedef struct SCreateDbInfo {
   int8_t    update;
   int8_t    cachelast; 
   SArray    *keep;
+  int8_t    dbType;
+  int16_t   partitions;
 } SCreateDbInfo;
 
 typedef struct SCreateAcctInfo {
@@ -155,6 +157,7 @@ typedef struct SUserInfo {
 typedef struct SMiscInfo {
   SArray    *a;         // SArray<SStrToken>
   bool       existsCheck;
+  int16_t    dbType;
   int16_t    tableType;
   SUserInfo  user;
   union {
@@ -265,7 +268,7 @@ void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pTableNameToken, SStrToken
 void SqlInfoDestroy(SSqlInfo *pInfo);
 
 void setDCLSQLElems(SSqlInfo *pInfo, int32_t type, int32_t nParams, ...);
-void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck,int16_t tableType);
+void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck,int16_t dbType,int16_t tableType);
 void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns);
 
 void setCreateDbInfo(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDbInfo *pDB, SStrToken *pIgExists);
@@ -276,6 +279,7 @@ void setKillSql(SSqlInfo *pInfo, int32_t type, SStrToken *ip);
 void setAlterUserSql(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege);
 
 void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo);
+void setDefaultCreateTopicOption(SCreateDbInfo *pDBInfo);
 
 // prefix show db.tables;
 void setDbName(SStrToken *pCpxName, SStrToken *pDb);

src/query/inc/sql.y

@@ -64,6 +64,7 @@ program ::= cmd.    {}
 
 //////////////////////////////////THE SHOW STATEMENT///////////////////////////////////////////
 cmd ::= SHOW DATABASES.  { setShowOptions(pInfo, TSDB_MGMT_TABLE_DB, 0, 0);}
+cmd ::= SHOW TOPICS.     { setShowOptions(pInfo, TSDB_MGMT_TABLE_TP, 0, 0);}
 cmd ::= SHOW MNODES.     { setShowOptions(pInfo, TSDB_MGMT_TABLE_MNODE, 0, 0);}
 cmd ::= SHOW DNODES.     { setShowOptions(pInfo, TSDB_MGMT_TABLE_DNODE, 0, 0);}
 cmd ::= SHOW ACCOUNTS.   { setShowOptions(pInfo, TSDB_MGMT_TABLE_ACCT, 0, 0);}
@@ -131,16 +132,18 @@ cmd ::= SHOW dbPrefix(X) VGROUPS ids(Y).    {
 //drop configure for tables
 cmd ::= DROP TABLE ifexists(Y) ids(X) cpxName(Z).   {
     X.n += Z.n;
-    setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y, -1);
+    setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y, -1, -1);
 }
 
 //drop stable
 cmd ::= DROP STABLE ifexists(Y) ids(X) cpxName(Z).   {
     X.n += Z.n;
-    setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y, TSDB_SUPER_TABLE);
+    setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y, -1, TSDB_SUPER_TABLE);
 }
 
-cmd ::= DROP DATABASE ifexists(Y) ids(X).    { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, -1); }
+cmd ::= DROP DATABASE ifexists(Y) ids(X).    { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, TSDB_DB_TYPE_DEFAULT, -1); }
+cmd ::= DROP TOPIC ifexists(Y) ids(X).    { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, TSDB_DB_TYPE_TOPIC, -1); }
+
 cmd ::= DROP DNODE ids(X).       { setDCLSQLElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &X);    }
 cmd ::= DROP USER ids(X).        { setDCLSQLElems(pInfo, TSDB_SQL_DROP_USER, 1, &X);     }
 cmd ::= DROP ACCOUNT ids(X).     { setDCLSQLElems(pInfo, TSDB_SQL_DROP_ACCT, 1, &X);  }
@@ -162,6 +165,7 @@ cmd ::= ALTER DNODE ids(X) ids(Y) ids(Z).       { setDCLSQLElems(pInfo, TSDB_SQL
 cmd ::= ALTER LOCAL ids(X).                     { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &X);              }
 cmd ::= ALTER LOCAL ids(X) ids(Y).              { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &X, &Y);          }
 cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SStrToken t = {0};  setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);}
+cmd ::= ALTER TOPIC ids(X) alter_topic_optr(Y). { SStrToken t = {0};  setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);}
 
 cmd ::= ALTER ACCOUNT ids(X) acct_optr(Z).      { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, NULL, &Z);}
 cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z).      { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, &Y, &Z);}
@@ -187,6 +191,7 @@ cmd ::= CREATE DNODE   ids(X).     { setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE
 cmd ::= CREATE ACCOUNT ids(X) PASS ids(Y) acct_optr(Z).
                                 { setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &X, &Y, &Z);}
 cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y).  { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);}
+cmd ::= CREATE TOPIC ifnotexists(Z) ids(X) topic_optr(Y).  { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);}
 cmd ::= CREATE USER ids(X) PASS ids(Y).     { setCreateUserSql(pInfo, &X, &Y);}
 
 pps(Y) ::= .                                { Y.n = 0;   }
@@ -247,9 +252,10 @@ comp(Y)    ::= COMP INTEGER(X).               { Y = X; }
 prec(Y)    ::= PRECISION STRING(X).           { Y = X; }
 update(Y)  ::= UPDATE INTEGER(X).             { Y = X; }     
 cachelast(Y) ::= CACHELAST INTEGER(X).        { Y = X; }
+partitions(Y) ::= PARTITIONS INTEGER(X).      { Y = X; }
 
 %type db_optr {SCreateDbInfo}
-db_optr(Y) ::= . {setDefaultCreateDbOption(&Y);}
+db_optr(Y) ::= . {setDefaultCreateDbOption(&Y); Y.dbType = TSDB_DB_TYPE_DEFAULT;}
 
 db_optr(Y) ::= db_optr(Z) cache(X).          { Y = Z; Y.cacheBlockSize = strtol(X.z, NULL, 10); }
 db_optr(Y) ::= db_optr(Z) replica(X).        { Y = Z; Y.replica = strtol(X.z, NULL, 10); }
@@ -267,8 +273,13 @@ db_optr(Y) ::= db_optr(Z) keep(X).           { Y = Z; Y.keep = X; }
 db_optr(Y) ::= db_optr(Z) update(X).         { Y = Z; Y.update = strtol(X.z, NULL, 10); }
 db_optr(Y) ::= db_optr(Z) cachelast(X).      { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); }
 
+%type topic_optr {SCreateDbInfo}
+
+topic_optr(Y) ::= db_optr(Z).                       { Y = Z; Y.dbType = TSDB_DB_TYPE_TOPIC; }
+topic_optr(Y) ::= topic_optr(Z) partitions(X).      { Y = Z; Y.partitions = strtol(X.z, NULL, 10); }
+
 %type alter_db_optr {SCreateDbInfo}
-alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y);}
+alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y); Y.dbType = TSDB_DB_TYPE_DEFAULT;}
 
 alter_db_optr(Y) ::= alter_db_optr(Z) replica(X).     { Y = Z; Y.replica = strtol(X.z, NULL, 10); }
 alter_db_optr(Y) ::= alter_db_optr(Z) quorum(X).      { Y = Z; Y.quorum = strtol(X.z, NULL, 10); }
@@ -280,6 +291,11 @@ alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X).       { Y = Z; Y.fsyncPeriod = s
 alter_db_optr(Y) ::= alter_db_optr(Z) update(X).      { Y = Z; Y.update = strtol(X.z, NULL, 10); }
 alter_db_optr(Y) ::= alter_db_optr(Z) cachelast(X).   { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); }
 
+%type alter_topic_optr {SCreateDbInfo}
+
+alter_topic_optr(Y) ::= alter_db_optr(Z).                       { Y = Z; Y.dbType = TSDB_DB_TYPE_TOPIC; }
+alter_topic_optr(Y) ::= alter_topic_optr(Z) partitions(X).      { Y = Z; Y.partitions = strtol(X.z, NULL, 10); }
+
 %type typename {TAOS_FIELD}
 typename(A) ::= ids(X). { 
   X.type = 0;
@@ -821,3 +837,4 @@ cmd ::= KILL QUERY INTEGER(X) COLON(Z) INTEGER(Y).        {X.n += (Z.n + Y.n); s
   COUNT SUM AVG MIN MAX FIRST LAST TOP BOTTOM STDDEV PERCENTILE APERCENTILE LEASTSQUARES HISTOGRAM DIFF
   SPREAD TWA INTERP LAST_ROW RATE IRATE SUM_RATE SUM_IRATE AVG_RATE AVG_IRATE TBID NOW IPTOKEN SEMI NONE PREV LINEAR IMPORT
   METRIC TBNAME JOIN METRICS STABLE NULL INSERT INTO VALUES.
+

src/query/src/qParserImpl.c

@@ -805,7 +805,7 @@ void setDCLSQLElems(SSqlInfo *pInfo, int32_t type, int32_t nParam, ...) {
   va_end(va);
 }
 
-void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck, int16_t tableType) {
+void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck, int16_t dbType, int16_t tableType) {
   pInfo->type = type;
 
   if (pInfo->pMiscInfo == NULL) {
@@ -816,6 +816,7 @@ void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrTo
   taosArrayPush(pInfo->pMiscInfo->a, pToken);
 
   pInfo->pMiscInfo->existsCheck = (existsCheck->n == 1);
+  pInfo->pMiscInfo->dbType = dbType;
   pInfo->pMiscInfo->tableType = tableType;
 }
 
@@ -935,6 +936,17 @@ void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo) {
   pDBInfo->keep = NULL;
 
   pDBInfo->update = -1;
-  pDBInfo->cachelast = 0;
+  pDBInfo->cachelast = -1;
+
+  pDBInfo->dbType = -1;
+  pDBInfo->partitions = -1;
+  
   memset(&pDBInfo->precision, 0, sizeof(SStrToken));
 }
+
+void setDefaultCreateTopicOption(SCreateDbInfo *pDBInfo) {
+  setDefaultCreateDbOption(pDBInfo);
+
+  pDBInfo->dbType = TSDB_DB_TYPE_TOPIC;
+  pDBInfo->partitions = TSDB_DEFAULT_DB_PARTITON_OPTION;
+}

src/query/src/qTokenizer.c

@@ -241,6 +241,9 @@ static SKeyword keywordTable[] = {
     {"AVG_IRATE",    TK_AVG_IRATE},
     {"CACHELAST",    TK_CACHELAST},
     {"DISTINCT",     TK_DISTINCT},
+    {"PARTITIONS",   TK_PARTITIONS},
+    {"TOPIC",        TK_TOPIC},
+    {"TOPICS",       TK_TOPICS}
 };
 
 static const char isIdChar[] = {

src/util/src/terror.c

@@ -197,6 +197,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_READY,         "Database unsynced")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_DAYS,   "Invalid database option: days out of range")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_KEEP,   "Invalid database option: keep >= keep1 >= keep0 >= days")
 
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC,            "Invalid topic name")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_OPTION,     "Invalid topic option")
+
 // dnode
 TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED,        "Message not processed")
 TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY,            "Dnode out of memory")

src/vnode/inc/vnodeInt.h

@@ -40,6 +40,7 @@ typedef struct {
   int32_t  queuedWMsg;
   int32_t  queuedRMsg;
   int32_t  flowctrlLevel;
+  int32_t  sequence;  // for topic
   int8_t   status;
   int8_t   role;
   int8_t   accessState;
@@ -47,7 +48,7 @@ typedef struct {
   int8_t   isCommiting;
   int8_t   dbReplica;
   int8_t   dropped;
-  int8_t   reserved;
+  int8_t   dbType;
   uint64_t version;   // current version
   uint64_t cversion;  // version while commit start
   uint64_t fversion;  // version on saved data file

src/vnode/src/vnodeCfg.c

@@ -42,6 +42,7 @@ static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) {
   pVnode->syncCfg.replica = vnodeMsg->cfg.vgReplica;
   pVnode->syncCfg.quorum = vnodeMsg->cfg.quorum;
   pVnode->dbReplica = vnodeMsg->cfg.dbReplica;
+  pVnode->dbType = vnodeMsg->cfg.dbType;
 
   for (int i = 0; i < pVnode->syncCfg.replica; ++i) {
     SVnodeDesc *node = &vnodeMsg->nodes[i];
@@ -214,7 +215,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) {
 
   cJSON *dbReplica = cJSON_GetObjectItem(root, "dbReplica");
   if (!dbReplica || dbReplica->type != cJSON_Number) {
-    vError("vgId:%d, failed to read %s, dbReplica not found", pVnode->vgId, file);
+    vWarn("vgId:%d, failed to read %s, dbReplica not found", pVnode->vgId, file);
     vnodeMsg.cfg.dbReplica = vnodeMsg.cfg.vgReplica;
     vnodeMsg.cfg.vgCfgVersion = 0;
   } else {
@@ -230,7 +231,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) {
 
   cJSON *update = cJSON_GetObjectItem(root, "update");
   if (!update || update->type != cJSON_Number) {
-    vError("vgId: %d, failed to read %s, update not found", pVnode->vgId, file);
+    vWarn("vgId: %d, failed to read %s, update not found", pVnode->vgId, file);
     vnodeMsg.cfg.update = 0;
     vnodeMsg.cfg.vgCfgVersion = 0;
   } else {
@@ -239,13 +240,21 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) {
 
   cJSON *cacheLastRow = cJSON_GetObjectItem(root, "cacheLastRow");
   if (!cacheLastRow || cacheLastRow->type != cJSON_Number) {
-    vError("vgId: %d, failed to read %s, cacheLastRow not found", pVnode->vgId, file);
+    vWarn("vgId: %d, failed to read %s, cacheLastRow not found", pVnode->vgId, file);
     vnodeMsg.cfg.cacheLastRow = 0;
     vnodeMsg.cfg.vgCfgVersion = 0;
   } else {
     vnodeMsg.cfg.cacheLastRow = (int8_t)cacheLastRow->valueint;
   }
 
+  cJSON *dbType = cJSON_GetObjectItem(root, "dbType");
+  if (!dbType || dbType->type != cJSON_Number) {
+    vWarn("vgId: %d, failed to read %s, dbType not found", pVnode->vgId, file);
+    vnodeMsg.cfg.dbType = 0;
+  } else {
+    vnodeMsg.cfg.dbType = (int8_t)dbType->valueint;
+  }
+
   cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
   if (!nodeInfos || nodeInfos->type != cJSON_Array) {
     vError("vgId:%d, failed to read %s, nodeInfos not found", pVnode->vgId, file);
@@ -337,6 +346,7 @@ int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) {
   len += snprintf(content + len, maxLen - len, "  \"quorum\": %d,\n", pMsg->cfg.quorum);
   len += snprintf(content + len, maxLen - len, "  \"update\": %d,\n", pMsg->cfg.update);
   len += snprintf(content + len, maxLen - len, "  \"cacheLastRow\": %d,\n", pMsg->cfg.cacheLastRow);
+  len += snprintf(content + len, maxLen - len, "  \"dbType\": %d,\n", pMsg->cfg.dbType);
   len += snprintf(content + len, maxLen - len, "  \"nodeInfos\": [{\n");
   for (int32_t i = 0; i < pMsg->cfg.vgReplica; i++) {
     SVnodeDesc *node = &pMsg->nodes[i];

src/vnode/src/vnodeWrite.c

@@ -15,6 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
+#include "tp.h"
 #include "taosmsg.h"
 #include "taoserror.h"
 #include "tglobal.h"
@@ -139,6 +140,10 @@ static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pR
 
   vTrace("vgId:%d, submit msg is processed", pVnode->vgId);
 
+  if (pVnode->dbType == TSDB_DB_TYPE_TOPIC && pVnode->role == TAOS_SYNC_ROLE_MASTER) {
+    tpUpdateTs(&pVnode->sequence, pCont);
+  }
+
   // save insert result into item
   SShellSubmitRspMsg *pRsp = NULL;
   if (pRet) {

tests/pytest/crash_gen/crash_gen_main.py

@@ -35,16 +35,19 @@ import os
 import signal
 import traceback
 import resource
-from guppy import hpy
+# from guppy import hpy
 import gc
 
 from crash_gen.service_manager import ServiceManager, TdeInstance
 from crash_gen.misc import Logging, Status, CrashGenError, Dice, Helper, Progress
 from crash_gen.db import DbConn, MyTDSql, DbConnNative, DbManager
+import crash_gen.settings 
 
 import taos
 import requests
 
+crash_gen.settings.init()
+
 # Require Python 3
 if sys.version_info[0] < 3:
     raise Exception("Must be using Python 3")
@@ -259,6 +262,7 @@ class ThreadCoordinator:
         self._execStats = ExecutionStats()
         self._runStatus = Status.STATUS_RUNNING
         self._initDbs()
+        self._stepStartTime = None  # Track how long it takes to execute each step
 
     def getTaskExecutor(self):
         return self._te
@@ -394,6 +398,10 @@ class ThreadCoordinator:
             try:
                 self._syncAtBarrier() # For now just cross the barrier
                 Progress.emit(Progress.END_THREAD_STEP)
+                if self._stepStartTime :
+                    stepExecTime = time.time() - self._stepStartTime
+                    Progress.emitStr('{:.3f}s/{}'.format(stepExecTime, DbConnNative.totalRequests))
+                    DbConnNative.resetTotalRequests() # reset to zero
             except threading.BrokenBarrierError as err:
                 self._execStats.registerFailure("Aborted due to worker thread timeout")
                 Logging.error("\n")
@@ -433,6 +441,7 @@ class ThreadCoordinator:
 
             # Then we move on to the next step
             Progress.emit(Progress.BEGIN_THREAD_STEP)
+            self._stepStartTime = time.time()
             self._releaseAllWorkerThreads(transitionFailed)                    
 
         if hasAbortedTask or transitionFailed : # abnormal ending, workers waiting at "gate"
@@ -691,7 +700,7 @@ class AnyState:
     def canDropDb(self):
         # If user requests to run up to a number of DBs,
         # we'd then not do drop_db operations any more
-        if gConfig.max_dbs > 0 : 
+        if gConfig.max_dbs > 0 or gConfig.use_shadow_db : 
             return False
         return self._info[self.CAN_DROP_DB]
 
@@ -699,6 +708,8 @@ class AnyState:
         return self._info[self.CAN_CREATE_FIXED_SUPER_TABLE]
 
     def canDropFixedSuperTable(self):
+        if gConfig.use_shadow_db: # duplicate writes to shaddow DB, in which case let's disable dropping s-table
+            return False
         return self._info[self.CAN_DROP_FIXED_SUPER_TABLE]
 
     def canAddData(self):
@@ -1037,7 +1048,7 @@ class Database:
     _clsLock = threading.Lock() # class wide lock
     _lastInt = 101  # next one is initial integer
     _lastTick = 0
-    _lastLaggingTick = 0 # lagging tick, for unsequenced insersions
+    _lastLaggingTick = 0 # lagging tick, for out-of-sequence (oos) data insertions
 
     def __init__(self, dbNum: int, dbc: DbConn): # TODO: remove dbc
         self._dbNum = dbNum # we assign a number to databases, for our testing purpose
@@ -1093,21 +1104,24 @@ class Database:
         t3 = datetime.datetime(2012, 1, 1)  # default "keep" is 10 years
         t4 = datetime.datetime.fromtimestamp(
             t3.timestamp() + elSec2)  # see explanation above
-        Logging.debug("Setting up TICKS to start from: {}".format(t4))
+        Logging.info("Setting up TICKS to start from: {}".format(t4))
         return t4
 
     @classmethod
     def getNextTick(cls):       
+        '''
+            Fetch a timestamp tick, with some random factor, may not be unique.
+        ''' 
         with cls._clsLock:  # prevent duplicate tick
             if cls._lastLaggingTick==0 or cls._lastTick==0 : # not initialized
                 # 10k at 1/20 chance, should be enough to avoid overlaps
                 tick = cls.setupLastTick()
                 cls._lastTick = tick
-                cls._lastLaggingTick = tick + datetime.timedelta(0, -10000)                 
+                cls._lastLaggingTick = tick + datetime.timedelta(0, -60*2)  # lagging behind 2 minutes, should catch up fast
                 # if : # should be quite a bit into the future
 
-            if Dice.throw(20) == 0:  # 1 in 20 chance, return lagging tick
+            if gConfig.mix_oos_data and Dice.throw(20) == 0:  # if asked to do so, and 1 in 20 chance, return lagging tick
-                cls._lastLaggingTick += datetime.timedelta(0, 1) # Go back in time 100 seconds
+                cls._lastLaggingTick += datetime.timedelta(0, 1) # pick the next sequence from the lagging tick sequence
                 return cls._lastLaggingTick 
             else:  # regular
                 # add one second to it
@@ -1334,7 +1348,8 @@ class Task():
             elif self._isErrAcceptable(errno2, err.__str__()):
                 self.logDebug("[=] Acceptable Taos library exception: errno=0x{:X}, msg: {}, SQL: {}".format(
                         errno2, err, wt.getDbConn().getLastSql()))
-                print("_", end="", flush=True)
+                # print("_", end="", flush=True)
+                Progress.emit(Progress.ACCEPTABLE_ERROR)
                 self._err = err
             else: # not an acceptable error
                 errMsg = "[=] Unexpected Taos library exception ({}): errno=0x{:X}, msg: {}, SQL: {}".format(
@@ -1563,8 +1578,11 @@ class TaskCreateDb(StateTransitionTask):
             # numReplica = Dice.throw(gConfig.max_replicas) + 1 # 1,2 ... N
             numReplica = gConfig.max_replicas # fixed, always
             repStr = "replica {}".format(numReplica)
-        self.execWtSql(wt, "create database {} {}"
+        updatePostfix = "update 1" if gConfig.verify_data else "" # allow update only when "verify data" is active
-            .format(self._db.getName(), repStr) )
+        dbName = self._db.getName()
+        self.execWtSql(wt, "create database {} {} {} ".format(dbName, repStr, updatePostfix ) )
+        if dbName == "db_0" and gConfig.use_shadow_db:
+            self.execWtSql(wt, "create database {} {} {} ".format("db_s", repStr, updatePostfix ) )
 
 class TaskDropDb(StateTransitionTask):
     @classmethod
@@ -1774,7 +1792,7 @@ class TdSuperTable:
                     ]) # TODO: add more from 'top'
 
             
-                if aggExpr not in ['stddev(speed)']: #TODO: STDDEV not valid for super tables?!
+                # if aggExpr not in ['stddev(speed)']: # STDDEV not valid for super tables?! (Done in TD-1049)
                 sql = "select {} from {}.{}".format(aggExpr, self._dbName, self.getName())
                 if Dice.throw(3) == 0: # 1 in X chance
                     sql = sql + ' GROUP BY color'
@@ -1988,7 +2006,7 @@ class TaskAddData(StateTransitionTask):
         numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS        
         fullTableName = db.getName() + '.' + regTableName
 
-        sql = "insert into {} values ".format(fullTableName)
+        sql = "INSERT INTO {} VALUES ".format(fullTableName)
         for j in range(numRecords):  # number of records per table
             nextInt = db.getNextInt()
             nextTick = db.getNextTick()
@@ -2016,12 +2034,24 @@ class TaskAddData(StateTransitionTask):
                 # print("_w" + str(nextInt % 100), end="", flush=True) # Trace what was written
 
             try:
-                sql = "insert into {} values ('{}', {}, '{}');".format( # removed: tags ('{}', {})
+                sql = "INSERT INTO {} VALUES ('{}', {}, '{}');".format( # removed: tags ('{}', {})
                     fullTableName,
                     # ds.getFixedSuperTableName(),
                     # ds.getNextBinary(), ds.getNextFloat(),
                     nextTick, nextInt, nextColor)
                 dbc.execute(sql)
+
+                # Quick hack, attach an update statement here. TODO: create an "update" task
+                if (not gConfig.use_shadow_db) and Dice.throw(5) == 0: # 1 in N chance, plus not using shaddow DB
+                    nextInt = db.getNextInt()
+                    nextColor = db.getNextColor()
+                    sql = "INSERt INTO {} VALUES ('{}', {}, '{}');".format( # "INSERt" means "update" here
+                    fullTableName,
+                    nextTick, nextInt, nextColor)
+                    # sql = "UPDATE {} set speed={}, color='{}' WHERE ts='{}'".format(
+                    #     fullTableName, db.getNextInt(), db.getNextColor(), nextTick)
+                    dbc.execute(sql)
+
             except: # Any exception at all
                 if gConfig.verify_data:
                     self.unlockTable(fullTableName)     
@@ -2070,7 +2100,8 @@ class TaskAddData(StateTransitionTask):
         random.shuffle(tblSeq) # now we have random sequence
         for i in tblSeq:
             if (i in self.activeTable):  # wow already active
-                print("x", end="", flush=True) # concurrent insertion
+                # print("x", end="", flush=True) # concurrent insertion
+                Progress.emit(Progress.CONCURRENT_INSERTION)
             else:
                 self.activeTable.add(i)  # marking it active
             
@@ -2373,6 +2404,11 @@ class MainExec:
             '--larger-data',
             action='store_true',
             help='Write larger amount of data during write operations (default: false)')
+        parser.add_argument(
+            '-m',
+            '--mix-oos-data',
+            action='store_false',
+            help='Mix out-of-sequence data into the test data stream (default: true)')
         parser.add_argument(
             '-n',
             '--dynamic-db-table-names',
@@ -2414,6 +2450,11 @@ class MainExec:
             '--verify-data',
             action='store_true',
             help='Verify data written in a number of places by reading back (default: false)')
+        parser.add_argument(
+            '-w',
+            '--use-shadow-db',
+            action='store_true',
+            help='Use a shaddow database to verify data integrity (default: false)')
         parser.add_argument(
             '-x',
             '--continue-on-exception',
@@ -2422,6 +2463,11 @@ class MainExec:
 
         global gConfig
         gConfig = parser.parse_args()
+        crash_gen.settings.gConfig = gConfig # TODO: fix this hack, consolidate this global var
+
+        # Sanity check for arguments
+        if gConfig.use_shadow_db and gConfig.max_dbs>1 :
+            raise CrashGenError("Cannot combine use-shadow-db with max-dbs of more than 1")
 
         Logging.clsInit(gConfig)
 

tests/pytest/crash_gen/db.py

@@ -18,6 +18,8 @@ import datetime
 import traceback
 # from .service_manager import TdeInstance
 
+import crash_gen.settings 
+
 class DbConn:
     TYPE_NATIVE = "native-c"
     TYPE_REST =   "rest-api"
@@ -257,6 +259,27 @@ class MyTDSql:
                 cls.longestQuery = sql
                 cls.longestQueryTime = queryTime
                 cls.lqStartTime = startTime
+
+        # Now write to the shadow database
+        if crash_gen.settings.gConfig.use_shadow_db:
+            if sql[:11] == "INSERT INTO":
+                if sql[:16] == "INSERT INTO db_0":
+                    sql2 = "INSERT INTO db_s" + sql[16:]
+                    self._cursor.execute(sql2)
+                else:
+                    raise CrashGenError("Did not find db_0 in INSERT statement: {}".format(sql))
+            else: # not an insert statement
+                pass
+
+            if sql[:12] == "CREATE TABLE":
+                if sql[:17] == "CREATE TABLE db_0":
+                    sql2 = sql.replace('db_0', 'db_s')
+                    self._cursor.execute(sql2)
+                else:
+                    raise CrashGenError("Did not find db_0 in CREATE TABLE statement: {}".format(sql))
+            else: # not an insert statement
+                pass
+        
         return ret
 
     def query(self, sql):
@@ -302,6 +325,7 @@ class DbConnNative(DbConn):
     _lock = threading.Lock()
     # _connInfoDisplayed = False # TODO: find another way to display this
     totalConnections = 0 # Not private
+    totalRequests = 0 
 
     def __init__(self, dbTarget):
         super().__init__(dbTarget)
@@ -309,6 +333,11 @@ class DbConnNative(DbConn):
         self._conn = None
         # self._cursor = None   
         
+    @classmethod
+    def resetTotalRequests(cls):        
+        with cls._lock: # force single threading for opening DB connections. # TODO: whaaat??!!!            
+            cls.totalRequests = 0
+
     def openByType(self):  # Open connection
         # global gContainer
         # tInst = tInst or gContainer.defTdeInstance # set up in ClientManager, type: TdeInstance
@@ -356,6 +385,8 @@ class DbConnNative(DbConn):
         Logging.debug("[SQL] Executing SQL: {}".format(sql))
         self._lastSql = sql
         nRows = self._tdSql.execute(sql)
+        cls = self.__class__
+        cls.totalRequests += 1
         Logging.debug(
             "[SQL] Execution Result, nRows = {}, SQL = {}".format(
                 nRows, sql))
@@ -369,6 +400,8 @@ class DbConnNative(DbConn):
         Logging.debug("[SQL] Executing SQL: {}".format(sql))
         self._lastSql = sql
         nRows = self._tdSql.query(sql)
+        cls = self.__class__
+        cls.totalRequests += 1
         Logging.debug(
             "[SQL] Query Result, nRows = {}, SQL = {}".format(
                 nRows, sql))

tests/pytest/crash_gen/misc.py

@@ -176,6 +176,8 @@ class Progress:
     SERVICE_START_NAP           = 7
     CREATE_TABLE_ATTEMPT        = 8
     QUERY_GROUP_BY              = 9
+    CONCURRENT_INSERTION        = 10
+    ACCEPTABLE_ERROR            = 11
 
     tokens = {
         STEP_BOUNDARY:      '.',
@@ -188,8 +190,14 @@ class Progress:
         SERVICE_START_NAP:           '_zz',
         CREATE_TABLE_ATTEMPT:       'c',
         QUERY_GROUP_BY:             'g',
+        CONCURRENT_INSERTION:       'x',
+        ACCEPTABLE_ERROR:           '_',
     }
 
     @classmethod
     def emit(cls, token):
         print(cls.tokens[token], end="", flush=True)
+
+    @classmethod
+    def emitStr(cls, str):
+        print('({})'.format(str), end="", flush=True)

tests/pytest/crash_gen/settings.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+import argparse
+
+gConfig:    argparse.Namespace 
+
+def init():
+    global gConfig
+    gConfig = []

tests/pytest/functions/function_operations.py

@@ -95,8 +95,7 @@ class TDTestCase:
         for i in col_list :
             for j in col_list :
                 for k in op_list :
-                    for l in [' order by ts' , ' order by ts desc' ]:
+                    sql = " select %s %s %s from test1 " % ( i , k , j )
-                        sql = " select %s %s %s from test1 %s" % ( i , k , j , l )
                     if i in err_list or j in err_list:
                         tdSql.error(sql)
                     else:

tests/script/general/cache/new_metrics.sim

@@ -2,7 +2,7 @@ system sh/stop_dnodes.sh
 
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 
 sleep 2000

tests/script/general/cache/restart_metrics.sim

@@ -2,7 +2,7 @@ system sh/stop_dnodes.sh
 
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 
 sleep 2000
@@ -51,7 +51,7 @@ print =============== step2
 system sh/exec.sh -n dnode1 -s stop
 sleep 3000
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 
 print =============== step3

tests/script/general/cache/restart_table.sim

@@ -2,7 +2,7 @@ system sh/stop_dnodes.sh
 
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 
 sleep 2000
@@ -35,7 +35,7 @@ print =============== step2
 system sh/exec.sh -n dnode1 -s stop
 sleep 3000
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 
 print =============== step3

tests/script/general/column/commit.sim

@@ -2,7 +2,7 @@ system sh/stop_dnodes.sh
 
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 
 sleep 2000

tests/script/general/column/metrics.sim

@@ -2,7 +2,7 @@ system sh/stop_dnodes.sh
 
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 
 sleep 2000

tests/script/general/column/table.sim

@@ -2,7 +2,7 @@ system sh/stop_dnodes.sh
 
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 
 sleep 2000

tests/script/general/compress/compress.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/cfg.sh -n dnode1 -c comp -v 1
 system sh/exec.sh -n dnode1 -s start
 

tests/script/general/compress/compress2.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/cfg.sh -n dnode1 -c comp -v 2
 system sh/exec.sh -n dnode1 -s start
 

tests/script/general/compress/uncompress.sim

@@ -1,6 +1,6 @@
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/cfg.sh -n dnode1 -c comp -v 1
 system sh/exec.sh -n dnode1 -s start
 

tests/script/general/compute/avg.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 sleep 2000
 sql connect

tests/script/general/compute/bottom.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 sleep 2000
 sql connect

tests/script/general/compute/count.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 sleep 2000
 sql connect

tests/script/general/compute/diff.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 sleep 2000
 sql connect

tests/script/general/compute/diff2.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 sleep 2000
 sql connect

tests/script/general/compute/first.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 sleep 2000
 sql connect

tests/script/general/compute/interval.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 sleep 2000
 sql connect

tests/script/general/compute/last.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 sleep 2000
 sql connect

tests/script/general/compute/last_row.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 sleep 2000
 sql connect

tests/script/general/compute/leastsquare.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 sleep 2000
 sql connect

tests/script/general/compute/max.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 sleep 2000
 sql connect

tests/script/general/compute/min.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 
 sleep 2000

tests/script/general/compute/null.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 sleep 2000
 sql connect

tests/script/general/compute/percentile.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 sleep 2000
 sql connect

tests/script/general/compute/stddev.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 sleep 2000
 sql connect

tests/script/general/compute/sum.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 sleep 2000
 sql connect

tests/script/general/compute/top.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 sleep 2000
 sql connect

tests/script/general/db/backup/keep.sim

@@ -6,9 +6,9 @@ system sh/deploy.sh -n dnode1 -i 1
 system sh/deploy.sh -n dnode2 -i 2
 system sh/deploy.sh -n dnode3 -i 3
 
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
-system sh/cfg.sh -n dnode2 -c walLevel -v 0
+system sh/cfg.sh -n dnode2 -c walLevel -v 1
-system sh/cfg.sh -n dnode3 -c walLevel -v 0
+system sh/cfg.sh -n dnode3 -c walLevel -v 1
 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1

tests/script/general/db/delete_reuse1.sim

@@ -5,10 +5,10 @@ system sh/deploy.sh -n dnode2 -i 2
 system sh/deploy.sh -n dnode3 -i 3
 system sh/deploy.sh -n dnode4 -i 4
 
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
-system sh/cfg.sh -n dnode2 -c walLevel -v 0
+system sh/cfg.sh -n dnode2 -c walLevel -v 1
-system sh/cfg.sh -n dnode3 -c walLevel -v 0
+system sh/cfg.sh -n dnode3 -c walLevel -v 1
-system sh/cfg.sh -n dnode4 -c walLevel -v 0
+system sh/cfg.sh -n dnode4 -c walLevel -v 1
 
 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1

tests/script/general/db/delete_reuse2.sim

@@ -5,10 +5,10 @@ system sh/deploy.sh -n dnode2 -i 2
 system sh/deploy.sh -n dnode3 -i 3
 system sh/deploy.sh -n dnode4 -i 4
 
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
-system sh/cfg.sh -n dnode2 -c walLevel -v 0
+system sh/cfg.sh -n dnode2 -c walLevel -v 1
-system sh/cfg.sh -n dnode3 -c walLevel -v 0
+system sh/cfg.sh -n dnode3 -c walLevel -v 1
-system sh/cfg.sh -n dnode4 -c walLevel -v 0
+system sh/cfg.sh -n dnode4 -c walLevel -v 1
 
 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1

tests/script/general/db/delete_writing1.sim

@@ -5,10 +5,10 @@ system sh/deploy.sh -n dnode2 -i 2
 system sh/deploy.sh -n dnode3 -i 3
 system sh/deploy.sh -n dnode4 -i 4
 
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
-system sh/cfg.sh -n dnode2 -c walLevel -v 0
+system sh/cfg.sh -n dnode2 -c walLevel -v 1
-system sh/cfg.sh -n dnode3 -c walLevel -v 0
+system sh/cfg.sh -n dnode3 -c walLevel -v 1
-system sh/cfg.sh -n dnode4 -c walLevel -v 0
+system sh/cfg.sh -n dnode4 -c walLevel -v 1
 
 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1

tests/script/general/db/len.sim

@@ -2,7 +2,7 @@ system sh/stop_dnodes.sh
 
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2000
 
 system sh/exec.sh -n dnode1 -s start

tests/script/general/db/show_create_db.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 
 system sh/exec.sh -n dnode1 -s start
 sleep 2000

tests/script/general/db/show_create_table.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 
 system sh/exec.sh -n dnode1 -s start
 sleep 2000

tests/script/general/db/tables.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 4
 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4
 

tests/script/general/db/topic1.sim

@@ -0,0 +1,856 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+
+system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 100
+system sh/cfg.sh -n dnode1 -c partitions -v 4
+system sh/exec.sh -n dnode1 -s start
+
+sleep 2000
+sql connect
+
+print ====step1 create with default para
+sql create topic t1;
+sql use t1;
+
+sql show topics;
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != t1 then 
+  return -1
+endi
+if $data02 != 4 then 
+  return -1
+endi
+
+sql show databases;
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != t1 then 
+  return -1
+endi
+#tables
+if $data02 < 1 then 
+  return -1
+endi
+#numofvgroups
+if $data03 < 1 then 
+  return -1
+endi
+
+sql show t1.vgroups;
+if $rows < 1 then 
+  return -1
+endi
+
+sql show t1.stables;
+if $rows != 1 then 
+  return -1
+endi
+if $data04 < 1 then 
+  return -1
+endi
+
+sql show t1.tables;
+if $rows < 1 then 
+  return -1
+endi
+
+sql drop topic t1
+sql show topics;
+if $rows != 0 then 
+  return -1
+endi
+sql show databases;
+if $rows != 0 then 
+  return -1
+endi
+
+sql_error use t1;
+sql_error show t1.vgroups;
+sql_error show t1.stables;
+sql_error show t1.tables;
+
+print ====step2 create with giving para
+sql create topic t1 partitions 6;
+
+sql show topics;
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != t1 then 
+  return -1
+endi
+if $data02 != 6 then 
+  return -1
+endi
+
+sql show databases;
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != t1 then 
+  return -1
+endi
+#tables
+if $data02 != 6 then 
+  return -1
+endi
+#numofvgroups
+if $data03 != 6 then 
+  return -1
+endi
+
+sql show t1.vgroups;
+if $rows != 6 then 
+  return -1
+endi
+
+sql show t1.stables;
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != ps then 
+  return -1
+endi
+if $data04 != 6 then 
+  return -1
+endi
+
+sql show t1.tables;
+if $rows != 6 then 
+  return -1
+endi
+
+sql describe t1.ps;
+if $data00 != off then 
+  return -1
+endi
+if $data10 != ts then 
+  return -1
+endi
+if $data20 != content then 
+  return -1
+endi
+if $data30 != pid then 
+  return -1
+endi
+
+sql describe t1.p1;
+if $data00 != off then 
+  return -1
+endi
+if $data10 != ts then 
+  return -1
+endi
+if $data20 != content then 
+  return -1
+endi
+if $data30 != pid then 
+  return -1
+endi
+
+sql drop topic t1
+sql show topics;
+if $rows != 0 then 
+  return -1
+endi
+
+sql show databases;
+if $rows != 0 then 
+  return -1
+endi
+
+sql_error show t1.vgroups;
+sql_error show t1.stables;
+sql_error show t1.tables;
+
+sql_error create topic t1 partitions -1;
+sql_error create topic t1 partitions 0;
+sql_error create topic t1 partitions 10001;
+
+print =============step3 create with db para
+sql create topic db cache 2 blocks 4 days 10 keep 20 minRows 300 maxRows 400 ctime 120 precision 'ms' comp 2 wal 1 replica 1
+sql show databases
+if $data00 != db then 
+  return -1
+endi
+if $data02 != 4 then 
+  return -1
+endi
+if $data03 != 4 then 
+  return -1
+endi
+if $data04 != 1 then 
+  return -1
+endi
+if $data06 != 10 then 
+  return -1
+endi
+if $data07 != 20,20,20 then 
+  return -1
+endi
+if $data08 != 2 then 
+  return -1
+endi
+if $data09 != 4 then 
+  return -1
+endi
+sql drop topic db;
+
+sql create topic db cache 2 blocks 4 days 10 keep 20 minRows 300 maxRows 400 ctime 120 precision 'ms' comp 2 wal 1 replica 1 partitions 7
+sql show databases
+if $data00 != db then 
+  return -1
+endi
+if $data02 != 7 then 
+  return -1
+endi
+if $data03 != 7 then 
+  return -1
+endi
+if $data04 != 1 then 
+  return -1
+endi
+if $data06 != 10 then 
+  return -1
+endi
+if $data07 != 20,20,20 then 
+  return -1
+endi
+if $data08 != 2 then 
+  return -1
+endi
+if $data09 != 4 then 
+  return -1
+endi
+
+sql show topics;
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != db then 
+  return -1
+endi
+
+sql show databases;
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != db then 
+  return -1
+endi
+#tables
+if $data02 != 7 then 
+  return -1
+endi
+#numofvgroups
+sql show db.vgroups;
+if $rows != 7 then 
+  return -1
+endi
+sql show db.stables;
+if $rows != 1 then 
+  return -1
+endi
+sql show db.tables;
+if $rows != 7 then 
+  return -1
+endi
+
+print ============== step name
+sql_error alter database db name d1
+sql_error alter database db name d2
+sql_error alter topic db name d1
+sql_error alter topic db name d2
+
+print ============== step ntables
+sql_error alter database db ntables -1
+sql_error alter database db ntables 0
+sql_error alter database db ntables 1
+sql_error alter database db ntables 10
+sql_error alter topic db ntables -1
+sql_error alter topic db ntables 0
+sql_error alter topic db ntables 1
+sql_error alter topic db ntables 10
+
+print ============== step vgroups
+sql_error alter database db vgroups -1
+sql_error alter database db vgroups 0
+sql_error alter database db vgroups 1
+sql_error alter database db vgroups 10
+sql_error alter topic db vgroups -1
+sql_error alter topic db vgroups 0
+sql_error alter topic db vgroups 1
+sql_error alter topic db vgroups 10
+
+print ============== step replica
+sql_error alter database db replica 2
+sql_error alter database db replica 3
+sql_error alter database db replica 0
+sql_error alter topic db replica 2
+sql_error alter topic db replica 3
+sql_error alter topic db replica 0
+
+sql alter database db replica 1
+sql show databases
+print replica $data4_db
+if $data4_db != 1 then 
+  return -1
+endi
+
+sql show topics
+if $rows != 1 then 
+  return -1
+endi
+
+print ============== step quorum
+sql show databases
+print quorum $data5_db
+if $data5_db != 1 then 
+  return -1
+endi
+
+sql_error alter topic db quorum 1
+sql alter database db quorum 1
+sql show databases
+print quorum $data5_db
+if $data5_db != 1 then 
+  return -1
+endi
+
+sql_error alter database db quorum 2
+sql_error alter database db quorum 3
+sql_error alter topic db quorum 2
+sql_error alter topic db quorum 3
+
+sql_error alter database db quorum 0
+sql_error alter database db quorum 4
+sql_error alter database db quorum 5
+sql_error alter database db quorum -1
+sql_error alter topic db quorum 0
+sql_error alter topic db quorum 4
+sql_error alter topic db quorum 5
+sql_error alter topic db quorum -1
+
+print ============== step days
+sql_error alter database db days 0
+sql_error alter database db days 1
+sql_error alter database db days 2
+sql_error alter database db days 10
+sql_error alter database db days 50
+sql_error alter database db days 100
+sql_error alter topic db days 0
+sql_error alter topic db days 1
+sql_error alter topic db days 2
+sql_error alter topic db days 10
+sql_error alter topic db days 50
+sql_error alter topic db days 100
+
+print ============== step keep
+sql show databases
+print keep $data7_db
+if $data7_db != 20,20,20 then 
+  return -1
+endi
+
+sql_error topic db keep 20
+sql alter database db keep 20
+sql show databases
+print keep $data7_db
+if $data7_db != 20,20,20 then
+  return -1
+endi
+
+sql_error topic db keep 30
+sql alter database db keep 30
+sql show databases
+print keep $data7_db
+if $data7_db != 20,20,30 then
+  return -1
+endi
+
+sql_error alter topic db keep 40
+sql alter database db keep 40
+sql show databases
+print keep $data7_db
+if $data7_db != 20,20,40 then
+  return -1
+endi
+
+sql alter database db keep 40
+sql alter database db keep 30
+sql alter database db keep 20
+sql_error alter database db keep 10
+sql_error alter database db keep 9
+sql_error alter database db keep 1
+sql alter database db keep 0
+sql alter database db keep -1
+sql_error alter database db keep 365001
+
+sql_error alter topic db keep 40
+sql_error alter topic db keep 30
+sql_error alter topic db keep 20
+sql_error alter topic db keep 10
+sql_error alter topic db keep 9
+sql_error alter topic db keep 1
+sql_error alter topic db keep 0
+sql_error alter topic db keep -1
+sql_error alter topic db keep 365001
+
+print ============== step cache
+sql_error alter database db cache 60
+sql_error alter database db cache 50
+sql_error alter database db cache 20
+sql_error alter database db cache 3
+sql_error alter database db cache 129
+sql_error alter database db cache 300
+sql_error alter database db cache 0
+sql_error alter database db cache -1
+
+sql_error alter topic db cache 60
+sql_error alter topic db cache 50
+sql_error alter topic db cache 20
+sql_error alter topic db cache 3
+sql_error alter topic db cache 129
+sql_error alter topic db cache 300
+sql_error alter topic db cache 0
+sql_error alter topic db cache -1
+
+print ============== step blocks
+sql show databases
+print blocks $data9_db
+if $data9_db != 4 then 
+  return -1
+endi
+
+sql_error alter topic db blocks 10
+sql alter database db blocks 10
+sql show databases
+print blocks $data9_db
+if $data9_db != 10 then 
+  return -1
+endi
+
+sql_error alter topic db blocks 20
+sql alter database db blocks 20
+sql show databases
+print blocks $data9_db
+if $data9_db != 20 then 
+  return -1
+endi
+
+sql_error alter topic db blocks 20
+sql alter database db blocks 30
+sql show databases
+print blocks $data9_db
+if $data9_db != 30 then 
+  return -1
+endi
+
+sql alter database db blocks 40
+sql alter database db blocks 30
+sql alter database db blocks 20
+sql alter database db blocks 10
+sql_error alter database db blocks 2
+sql_error alter database db blocks 1
+sql alter database db blocks 0
+sql_error alter database db blocks -1
+sql_error alter database db blocks 10001
+
+sql_error alter topic db blocks 40
+sql_error alter topic db blocks 30
+sql_error alter topic db blocks 20
+sql_error alter topic db blocks 10
+sql_error alter topic db blocks 2
+sql_error alter topic db blocks 1
+sql_error alter topic db blocks 0
+sql_error alter topic db blocks -1
+sql_error alter topic db blocks 10001
+
+print ============== step minrows
+sql_error alter database db minrows 1
+sql_error alter database db minrows 100
+sql_error alter database db minrows 1000
+
+sql_error alter topic db minrows 1
+sql_error alter topic db minrows 100
+sql_error alter topic db minrows 1000
+
+print ============== step maxrows
+sql_error alter database db maxrows 1
+sql_error alter database db maxrows 100
+sql_error alter database db maxrows 1000
+
+sql_error alter topic db maxrows 1
+sql_error alter topic db maxrows 100
+sql_error alter topic db maxrows 1000
+
+print ============== step wallevel
+sql show databases
+print wallevel $data12_db
+if $data12_db != 1 then 
+  return -1
+endi
+
+sql_error alter topic db wal 1
+sql alter database db wal 1
+sql show databases
+print wal $data12_db
+if $data12_db != 1 then 
+  return -1
+endi
+
+sql alter database db wal 1
+sql alter database db wal 2
+sql alter database db wal 1
+sql alter database db wal 2
+sql alter database db wal 0
+sql_error alter database db wal 3
+sql_error alter database db wal 4
+sql_error alter database db wal -1
+sql_error alter database db wal 1000
+
+sql_error alter topic db wal 1
+sql_error alter topic db wal 2
+sql_error alter topic db wal 1
+sql_error alter topic db wal 2
+sql_error alter topic db wal 0
+sql_error alter topic db wal 3
+sql_error alter topic db wal 4
+sql_error alter topic db wal -1
+sql_error alter topic db wal 1000
+
+print ============== step fsync
+sql alter database db fsync 0
+sql alter database db fsync 1
+sql alter database db fsync 3600
+sql alter database db fsync 18000
+sql alter database db fsync 180000
+sql_error alter database db fsync 180001
+sql_error alter database db fsync -1
+
+sql_error alter topic db fsync 0
+sql_error alter topic db fsync 1
+sql_error alter topic db fsync 3600
+sql_error alter topic db fsync 18000
+sql_error alter topic db fsync 180000
+sql_error alter topic db fsync 180001
+sql_error alter topic db fsync -1
+
+print ============== step comp
+sql show databases
+print comp $data14_db
+if $data14_db != 2 then 
+  return -1
+endi
+
+sql_error alter topic db comp 1
+sql alter database db comp 1
+sql show databases
+print comp $data14_db
+if $data14_db != 1 then 
+  return -1
+endi
+
+sql_error alter topic db comp 2
+sql alter database db comp 2
+sql show databases
+print comp $data14_db
+if $data14_db != 2 then 
+  return -1
+endi
+
+sql_error alter topic db comp 0
+sql alter database db comp 0
+sql show databases
+print comp $data14_db
+if $data14_db != 0 then 
+  return -1
+endi
+
+sql_error alter database db comp 3
+sql_error alter database db comp 4
+sql_error alter database db comp 5
+sql_error alter database db comp -1
+
+sql_error alter topic db comp 3
+sql_error alter topic db comp 4
+sql_error alter topic db comp 5
+sql_error alter topic db comp -1
+
+print ============== step precision
+sql_error alter database db prec 'us'
+sql_error alter topic db prec 'us'
+
+print ============== step status
+sql_error alter database db status 'delete'
+sql_error alter topic db status 'delete'
+
+print ============== step drop
+sql drop database db
+sql show topics;
+if $rows != 0 then 
+  return -1
+endi
+
+sql show databases;
+if $rows != 0 then 
+  return -1
+endi
+
+print ============== step db1
+sql create database d1
+sql_error alter database d1 partitions 2
+sql_error alter topic d1 partitions 2
+
+sql show topics;
+if $rows != 0 then 
+  return -1
+endi
+
+sql show databases;
+if $rows != 1 then 
+  return -1
+endi
+
+sql alter database d1 fsync 0
+sql show topics;
+if $rows != 0 then 
+  return -1
+endi
+
+sql show databases;
+if $rows != 1 then 
+  return -1
+endi
+
+sql drop database d1
+sql show topics;
+if $rows != 0 then 
+  return -1
+endi
+
+sql show databases;
+if $rows != 0 then 
+  return -1
+endi
+
+print ============== step db2
+sql create topic d1
+sql show topics;
+if $rows != 1 then 
+  return -1
+endi
+
+sql show databases;
+if $rows != 1 then 
+  return -1
+endi
+
+sql alter database d1 fsync 0
+sql show topics;
+if $rows != 1 then 
+  return -1
+endi
+
+sql show databases;
+if $rows != 1 then 
+  return -1
+endi
+
+sql drop database d1
+sql show topics;
+if $rows != 0 then 
+  return -1
+endi
+
+sql show databases;
+if $rows != 0 then 
+  return -1
+endi
+
+print ============== step db3
+sql create topic d1
+sql show topics;
+if $rows != 1 then 
+  return -1
+endi
+
+sql show databases;
+if $rows != 1 then 
+  return -1
+endi
+
+sql alter topic d1 partitions 2
+sql show topics;
+if $rows != 1 then 
+  return -1
+endi
+
+sql show databases;
+if $rows != 1 then 
+  return -1
+endi
+
+sql drop database d1
+sql show topics;
+if $rows != 0 then 
+  return -1
+endi
+
+sql show databases;
+if $rows != 0 then 
+  return -1
+endi
+
+print ============== step partitions
+sql create topic t1 partitions 5
+
+sql_error alter database t1 partitions -1
+sql_error alter database t1 partitions 0
+sql_error alter database t1 partitions 1
+sql_error alter database t1 partitions 2
+sql_error alter database t1 partitions 3
+sql_error alter database t1 partitions 100
+sql_error alter database t1 partitions 1000
+sql_error alter database t1 partitions 10000
+
+sql_error alter topic t1 partitions -1
+sql_error alter topic t1 partitions 0
+sql_error alter database t1 partitions 10000
+
+sql alter topic t1 partitions 1
+sql show topics;
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != t1 then 
+  return -1
+endi
+sql show databases;
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != t1 then 
+  return -1
+endi
+#tables
+if $data02 != 1 then 
+  return -1
+endi
+#numofvgroups
+sql show t1.vgroups;
+if $rows != 1 then 
+  return -1
+endi
+sql show t1.stables;
+if $rows != 1 then 
+  return -1
+endi
+sql show t1.tables;
+if $rows != 1 then 
+  return -1
+endi
+
+sql alter topic t1 partitions 2
+sql show topics;
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != t1 then 
+  return -1
+endi
+sql show databases;
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != t1 then 
+  return -1
+endi
+#tables
+if $data02 != 2 then 
+  return -1
+endi
+#numofvgroups
+sql show t1.vgroups;
+if $rows != 2 then 
+  return -1
+endi
+sql show t1.stables;
+if $rows != 1 then 
+  return -1
+endi
+sql show t1.tables;
+if $rows != 2 then 
+  return -1
+endi
+
+sql alter topic t1 partitions 3
+sql show topics;
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != t1 then 
+  return -1
+endi
+sql show databases;
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != t1 then 
+  return -1
+endi
+#tables
+if $data02 != 3 then 
+  return -1
+endi
+#numofvgroups
+sql show t1.vgroups;
+if $rows != 3 then 
+  return -1
+endi
+sql show t1.stables;
+if $rows != 1 then 
+  return -1
+endi
+sql show t1.tables;
+if $rows != 3 then 
+  return -1
+endi
+
+sql alter topic t1 partitions 10
+sql show topics;
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != t1 then 
+  return -1
+endi
+sql show databases;
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != t1 then 
+  return -1
+endi
+#tables
+if $data02 != 10 then 
+  return -1
+endi
+#numofvgroups
+sql show t1.vgroups;
+if $rows != 10 then 
+  return -1
+endi
+sql show t1.stables;
+if $rows != 1 then 
+  return -1
+endi
+sql show t1.tables;
+if $rows != 10 then 
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/general/db/topic2.sim

@@ -0,0 +1,321 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+
+system sh/exec.sh -n dnode1 -s start
+
+sleep 2000
+sql connect
+
+print ==== step1 
+sql create topic t1 partitions 2;
+sql show t1.tables
+if $rows != 2 then 
+  return -1
+endi
+sql show t1.vgroups
+if $rows != 2 then 
+  return -1
+endi
+
+sql insert into t1.p1 values(1, now, '1');
+sql insert into t1.p1 values(1, now, '2');
+sql insert into t1.p1 values(1, now, '3');
+sql insert into t1.p1 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9');
+sql insert into t1.p1 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15');
+sql insert into t1.p1 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33');
+
+sql insert into t1.p2 values(1, now, '1');
+sql insert into t1.p2 values(1, now, '2');
+sql insert into t1.p2 values(1, now, '3');
+sql insert into t1.p2 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9');
+sql insert into t1.p2 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15');
+sql insert into t1.p2 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33');
+
+sql_error insert into t1.p3 values(1, now, '1');
+sql_error insert into t1.p3 values(1, now, '2');
+sql_error insert into t1.p3 values(1, now, '3');
+sql_error insert into t1.p3 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9');
+sql_error insert into t1.p3 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15');
+sql_error insert into t1.p3 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33');
+
+sql select * from t1.p1 order by off asc
+if $rows != 33 then 
+  return -1
+endi
+if $data02 != 1 then 
+  return -1
+endi
+if $data12 != 2 then 
+  return -1
+endi
+if $data22 != 3 then 
+  return -1
+endi
+
+sql select * from t1.p2 order by off asc
+if $rows != 33 then 
+  return -1
+endi
+if $data02 != 1 then 
+  return -1
+endi
+if $data12 != 2 then 
+  return -1
+endi
+if $data22 != 3 then 
+  return -1
+endi
+
+print ==== step2
+sql alter topic t1 partitions 4;
+sql show t1.tables
+if $rows != 4 then 
+  return -1
+endi
+sql show t1.vgroups
+if $rows != 4 then 
+  return -1
+endi
+
+sql insert into t1.p1 values(1, now, '1');
+sql insert into t1.p1 values(1, now, '2');
+sql insert into t1.p1 values(1, now, '3');
+sql insert into t1.p1 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9');
+sql insert into t1.p1 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15');
+sql insert into t1.p1 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33');
+
+sql insert into t1.p2 values(1, now, '1');
+sql insert into t1.p2 values(1, now, '2');
+sql insert into t1.p2 values(1, now, '3');
+sql insert into t1.p2 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9');
+sql insert into t1.p2 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15');
+sql insert into t1.p2 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33');
+
+sql insert into t1.p3 values(1, now, '1');
+sql insert into t1.p3 values(1, now, '2');
+sql insert into t1.p3 values(1, now, '3');
+sql insert into t1.p3 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9');
+sql insert into t1.p3 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15');
+sql insert into t1.p3 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33');
+
+sql insert into t1.p4 values(1, now, '1');
+sql insert into t1.p4 values(1, now, '2');
+sql insert into t1.p4 values(1, now, '3');
+sql insert into t1.p4 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9');
+sql insert into t1.p4 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15');
+sql insert into t1.p4 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33');
+
+sql_error insert into t1.p5 values(1, now, '1');
+sql_error insert into t1.p5 values(1, now, '2');
+sql_error insert into t1.p5 values(1, now, '3');
+sql_error insert into t1.p5 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9');
+sql_error insert into t1.p5 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15');
+sql_error insert into t1.p5 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33');
+
+sql select * from t1.p1 order by off asc
+if $rows != 66 then 
+  return -1
+endi
+if $data02 != 1 then 
+  return -1
+endi
+if $data12 != 2 then 
+  return -1
+endi
+if $data22 != 3 then 
+  return -1
+endi
+
+sql select * from t1.p2 order by off asc
+if $rows != 66 then 
+  return -1
+endi
+if $data02 != 1 then 
+  return -1
+endi
+if $data12 != 2 then 
+  return -1
+endi
+if $data22 != 3 then 
+  return -1
+endi
+
+sql select * from t1.p3 order by off asc
+if $rows != 33 then 
+  return -1
+endi
+if $data02 != 1 then 
+  return -1
+endi
+if $data12 != 2 then 
+  return -1
+endi
+if $data22 != 3 then 
+  return -1
+endi
+
+sql select * from t1.p4 order by off asc
+if $rows != 33 then 
+  return -1
+endi
+if $data02 != 1 then 
+  return -1
+endi
+if $data12 != 2 then 
+  return -1
+endi
+if $data22 != 3 then 
+  return -1
+endi
+
+print ==== step3
+sql alter topic t1 partitions 1;
+sql show t1.tables
+if $rows != 1 then 
+  return -1
+endi
+sql show t1.vgroups
+if $rows != 1 then 
+  return -1
+endi
+
+sql insert into t1.p1 values(1, now, '1');
+sql insert into t1.p1 values(1, now, '2');
+sql insert into t1.p1 values(1, now, '3');
+sql insert into t1.p1 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9');
+sql insert into t1.p1 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15');
+sql insert into t1.p1 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33');
+
+sql_error insert into t1.p2 values(1, now, '1');
+sql_error insert into t1.p2 values(1, now, '2');
+sql_error insert into t1.p2 values(1, now, '3');
+sql_error insert into t1.p2 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9');
+sql_error insert into t1.p2 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15');
+sql_error insert into t1.p2 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33');
+
+sql_error insert into t1.p3 values(1, now, '1');
+sql_error insert into t1.p3 values(1, now, '2');
+sql_error insert into t1.p3 values(1, now, '3');
+sql_error insert into t1.p3 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9');
+sql_error insert into t1.p3 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15');
+sql_error insert into t1.p3 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33');
+
+sql_error insert into t1.p4 values(1, now, '1');
+sql_error insert into t1.p4 values(1, now, '2');
+sql_error insert into t1.p4 values(1, now, '3');
+sql_error insert into t1.p4 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9');
+sql_error insert into t1.p4 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15');
+sql_error insert into t1.p4 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33');
+
+sql_error insert into t1.p5 values(1, now, '1');
+sql_error insert into t1.p5 values(1, now, '2');
+sql_error insert into t1.p5 values(1, now, '3');
+sql_error insert into t1.p5 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9');
+sql_error insert into t1.p5 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15');
+sql_error insert into t1.p5 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33');
+
+sql select * from t1.p1 order by off asc
+if $rows != 99 then 
+  return -1
+endi
+if $data02 != 1 then 
+  return -1
+endi
+if $data12 != 2 then 
+  return -1
+endi
+if $data22 != 3 then 
+  return -1
+endi
+
+sql_error select * from t1.p2 order by off asc
+sql_error select * from t1.p3 order by off asc
+sql_error select * from t1.p4 order by off asc
+
+print ==== step4
+sql alter topic t1 partitions 3;
+sql show t1.tables
+if $rows != 3 then 
+  return -1
+endi
+sql show t1.vgroups
+if $rows != 3 then 
+  return -1
+endi
+
+sql insert into t1.p1 values(1, now, '1');
+sql insert into t1.p1 values(1, now, '2');
+sql insert into t1.p1 values(1, now, '3');
+sql insert into t1.p1 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9');
+sql insert into t1.p1 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15');
+sql insert into t1.p1 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33');
+
+sql insert into t1.p2 values(1, now, '1');
+sql insert into t1.p2 values(1, now, '2');
+sql insert into t1.p2 values(1, now, '3');
+sql insert into t1.p2 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9');
+sql insert into t1.p2 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15');
+sql insert into t1.p2 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33');
+
+sql insert into t1.p3 values(1, now, '1');
+sql insert into t1.p3 values(1, now, '2');
+sql insert into t1.p3 values(1, now, '3');
+sql insert into t1.p3 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9');
+sql insert into t1.p3 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15');
+sql insert into t1.p3 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33');
+
+sql_error insert into t1.p4 values(1, now, '1');
+sql_error insert into t1.p5 values(1, now, '1');
+sql_error insert into t1.p6 values(1, now, '1');
+sql_error select * from t1.p4 order by off asc
+sql_error select * from t1.p5 order by off asc
+sql_error select * from t1.p6 order by off asc
+
+sql select * from t1.p1 order by off asc
+if $rows != 132 then 
+  return -1
+endi
+if $data02 != 1 then 
+  return -1
+endi
+if $data12 != 2 then 
+  return -1
+endi
+if $data22 != 3 then 
+  return -1
+endi
+
+sql select * from t1.p2 order by off asc
+if $rows != 33 then 
+  return -1
+endi
+if $data02 != 1 then 
+  return -1
+endi
+if $data12 != 2 then 
+  return -1
+endi
+if $data22 != 3 then 
+  return -1
+endi
+
+sql select * from t1.p3 order by off asc
+if $rows != 33 then 
+  return -1
+endi
+if $data02 != 1 then 
+  return -1
+endi
+if $data12 != 2 then 
+  return -1
+endi
+if $data22 != 3 then 
+  return -1
+endi
+
+sql select * from t1.ps order by off asc
+if $rows != 198 then 
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/general/db/vnodes.sim

@@ -5,7 +5,7 @@ $maxTables = 4
 $totalRows = $totalVnodes * $maxTables
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel            -v 0
+system sh/cfg.sh -n dnode1 -c walLevel            -v 1
 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode   -v $maxTables
 system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb     -v $totalVnodes
 system sh/cfg.sh -n dnode1 -c maxVnodeConnections -v 100000

tests/script/general/field/2.sim

@@ -1,6 +1,6 @@
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 
 sleep 2000

tests/script/general/field/3.sim

@@ -1,6 +1,6 @@
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 
 sleep 2000

tests/script/general/field/4.sim

@@ -1,6 +1,6 @@
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 
 sleep 2000

tests/script/general/field/5.sim

@@ -1,6 +1,6 @@
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 
 sleep 2000

tests/script/general/field/6.sim

@@ -1,6 +1,6 @@
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/exec.sh -n dnode1 -s start
 
 sleep 2000