diff --git a/cmake/cmake.define b/cmake/cmake.define index 5639d212d7..989b69a89b 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -81,7 +81,7 @@ ENDIF () IF (TD_WINDOWS) MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}") - SET(COMMON_FLAGS "/w /D_WIN32 /DWIN32 /Zi") + SET(COMMON_FLAGS "/w /D_WIN32 /DWIN32 /Zi /MTd") SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO") # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) # SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") @@ -92,6 +92,12 @@ IF (TD_WINDOWS) IF (CMAKE_DEPFILE_FLAGS_CXX) SET(CMAKE_DEPFILE_FLAGS_CXX "") ENDIF () + IF (CMAKE_C_FLAGS_DEBUG) + SET(CMAKE_C_FLAGS_DEBUG "" CACHE STRING "" FORCE) + ENDIF () + IF (CMAKE_CXX_FLAGS_DEBUG) + SET(CMAKE_CXX_FLAGS_DEBUG "" CACHE STRING "" FORCE) + ENDIF () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMMON_FLAGS}") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON_FLAGS}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index b4e8825431..294b59fe95 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -273,7 +273,7 @@ endif(${BUILD_WITH_NURAFT}) # pthread if(${BUILD_PTHREAD}) - set(CMAKE_BUILD_TYPE release) + set(CMAKE_BUILD_TYPE debug) add_definitions(-DPTW32_STATIC_LIB) add_subdirectory(pthread EXCLUDE_FROM_ALL) set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread) diff --git a/docs/examples/python/tmq_example.py b/docs/examples/python/tmq_example.py index 1f6da3d1b6..cee036454e 100644 --- a/docs/examples/python/tmq_example.py +++ b/docs/examples/python/tmq_example.py @@ -1,59 +1,6 @@ import taos -from taos.tmq import * - -conn = taos.connect() - -# create database -conn.execute("drop database if exists py_tmq") -conn.execute("create database if not exists py_tmq vgroups 2") - -# create table and stables -conn.select_db("py_tmq") -conn.execute("create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)") -conn.execute("create table if not exists tb1 using stb1 tags(1)") -conn.execute("create table if not exists tb2 using stb1 tags(2)") -conn.execute("create table if not exists tb3 using stb1 tags(3)") - -# create topic -conn.execute("drop topic if exists topic_ctb_column") -conn.execute("create topic if not exists topic_ctb_column as select ts, c1, c2, c3 from stb1") - -# set consumer configure options -conf = TaosTmqConf() -conf.set("group.id", "tg2") -conf.set("td.connect.user", "root") -conf.set("td.connect.pass", "taosdata") -conf.set("enable.auto.commit", "true") -conf.set("msg.with.table.name", "true") - -def tmq_commit_cb_print(tmq, resp, offset, param=None): - print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}") - -conf.set_auto_commit_cb(tmq_commit_cb_print, None) - -# build consumer -tmq = conf.new_consumer() - -# build topic list -topic_list = TaosTmqList() -topic_list.append("topic_ctb_column") - -# subscribe consumer -tmq.subscribe(topic_list) - -# check subscriptions -sub_list = tmq.subscription() -print("subscribed topics: ",sub_list) - -# start subscribe -while 1: - res = tmq.poll(1000) - if res: - topic = res.get_topic_name() - vg = res.get_vgroup_id() - db = res.get_db_name() - print(f"topic: {topic}\nvgroup id: {vg}\ndb: {db}") - for row in res: - print(row) - tb = res.get_table_name() - print(f"from table: {tb}") +from taos.tmq import TaosConsumer +consumer = TaosConsumer('topic_ctb_column', group_id='vg2') +for msg in consumer: + for row in msg: + print(row) diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md index 846cd9e9cd..f1896e9c0a 100644 --- a/docs/zh/05-get-started/03-package.md +++ b/docs/zh/05-get-started/03-package.md @@ -5,12 +5,72 @@ title: 使用安装包立即开始 import Tabs from "@theme/Tabs"; import TabItem from "@theme/TabItem"; +import PkgListV3 from "/components/PkgListV3"; -在 Linux 系统上，TDengine 开源版本提供 deb 和 rpm 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 deb 支持 Debian/Ubuntu 及衍生系统，rpm 支持 CentOS/RHEL/SUSE 及衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包，也支持通过 `apt-get` 工具从线上进行安装。TDengine 也提供 Windows x64 平台的安装包。您也可以[用Docker立即体验](../../get-started/docker/)。如果您希望对 TDengine 贡献代码或对内部实现感兴趣，请参考我们的 [TDengine GitHub 主页](https://github.com/taosdata/TDengine) 下载源码构建和安装. +TDengine 完整的软件包包括服务端（taosd）、用于与第三方系统对接并提供 RESTful 接口的 taosAdapter、应用驱动（taosc）、命令行程序 (CLI，taos) 和一些工具软件，目前服务端 taosd 和 taosAdapter 仅在 Linux 系统上安装和运行，后续将支持 Windows、macOS 等系统。应用驱动 taosc 与 TDengine CLI 可以在 Windows 或 Linux 上安装和运行。TDengine 除了提供多种语言的连接器之外，还通过 [taosAdapter](../../reference/taosadapter/) 提供 [RESTful 接口](../../reference/rest-api/)。 + +在 Linux 系统上，TDengine 开源版本提供 deb 和 rpm 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 deb 支持 Debian/Ubuntu 及衍生系统，rpm 支持 CentOS/RHEL/SUSE 及衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包，也支持通过 `apt-get` 工具从线上进行安装。TDengine 也提供 Windows x64 平台的安装包。您也可以[用 Docker 立即体验](../../get-started/docker/)。如果您希望对 TDengine 贡献代码或对内部实现感兴趣，请参考我们的 [TDengine GitHub 主页](https://github.com/taosdata/TDengine) 下载源码构建和安装. + +为方便使用，标准的服务端安装包包含了 taos、taosd、taosAdapter、taosdump、taosBenchmark、TDinsight 安装脚本和示例代码；如果您只需要用到服务端程序和客户端连接的 C/C++ 语言支持，也可以仅下载 lite 版本的安装包。 + +在安装包格式上，我们提供 tar.gz, rpm 和 deb 格式，为企业客户提供 tar.gz 格式安装包，以方便在特定操作系统上使用。需要注意的是，rpm 和 deb 包不含 taosdump 和 TDinsight 安装脚本，这些工具需要通过安装 taosTool 包获得。 ## 安装 + + +1. 从列表中下载获得 deb 安装包，例如 TDengine-server-3.0.0.0-Linux-x64.deb； + +2. 进入到 TDengine-server-3.0.0.0-Linux-x64.deb 安装包所在目录，执行如下的安装命令： + +```bash +sudo dpkg -i TDengine-server-3.0.0.0-Linux-x64.deb +``` + + + + + +1. 从列表中下载获得 rpm 安装包，例如 TDengine-server-3.0.0.0-Linux-x64.rpm； + +2. 进入到 TDengine-server-3.0.0.0-Linux-x64.rpm 安装包所在目录，执行如下的安装命令： + +```bash +sudo rpm -ivh TDengine-server-3.0.0.0-Linux-x64.rpm +``` + + + + + +1. 从列表中下载获得 tar.gz 安装包，例如 TDengine-server-3.0.0.0-Linux-x64.tar.gz； + +2. 进入到 TDengine-server-3.0.0.0-Linux-x64.tar.gz 安装包所在目录，先解压文件后，进入子目录，执行其中的 install.sh 安装脚本： + +```bash +tar -zxvf TDengine-server-3.0.0.0-Linux-x64.tar.gz +``` + +解压后进入相应路径，执行 + +```bash +sudo ./install.sh +``` + +:::info +install.sh 安装脚本在执行过程中，会通过命令行交互界面询问一些配置信息。如果希望采取无交互安装方式，那么可以用 -e no 参数来执行 install.sh 脚本。运行 `./install.sh -h` 指令可以查看所有参数的详细说明信息。 +::: + + + + + +1. 从列表中下载获得 exe 安装程序，例如 TDengine-server-3.0.0.0-Windows-x64.exe； + +2. 运行 TDengine-server-3.0.0.0-Windows-x64.exe 来安装 TDengine。 + + 可以使用 apt-get 工具从官方仓库安装。 @@ -40,58 +100,12 @@ sudo apt-get install tdengine apt-get 方式只适用于 Debian 或 Ubuntu 系统 :::: - - -1. 从 [发布历史页面](../../releases) 下载获得 deb 安装包，例如 TDengine-server-3.0.0.0-Linux-x64.deb； -2. 进入到 TDengine-server-3.0.0.0-Linux-x64.deb 安装包所在目录，执行如下的安装命令： - -```bash -sudo dpkg -i TDengine-server-3.0.0.0-Linux-x64.deb -``` - - - - - -1. 从 [发布历史页面](../../releases) 下载获得 rpm 安装包，例如 TDengine-server-3.0.0.0-Linux-x64.rpm； -2. 进入到 TDengine-server-3.0.0.0-Linux-x64.rpm 安装包所在目录，执行如下的安装命令： - -```bash -sudo rpm -ivh TDengine-server-3.0.0.0-Linux-x64.rpm -``` - - - - - -1. 从 [发布历史页面](../../releases) 下载获得 tar.gz 安装包，例如 TDengine-server-3.0.0.0-Linux-x64.tar.gz； -2. 进入到 TDengine-server-3.0.0.0-Linux-x64.tar.gz 安装包所在目录，先解压文件后，进入子目录，执行其中的 install.sh 安装脚本： - -```bash -tar -zxvf TDengine-server-3.0.0.0-Linux-x64.tar.gz -``` - -解压后进入相应路径，执行 - -```bash -sudo ./install.sh -``` + :::info -install.sh 安装脚本在执行过程中，会通过命令行交互界面询问一些配置信息。如果希望采取无交互安装方式，那么可以用 -e no 参数来执行 install.sh 脚本。运行 `./install.sh -h` 指令可以查看所有参数的详细说明信息。 - +下载其他组件、最新 Beta 版及之前版本的安装包，请点击[发布历史页面](../../releases) ::: - - - - -1. 从 [发布历史页面](../../releases) 下载获得 exe 安装程序，例如 TDengine-server-3.0.0.0-Windows-x64.exe； -2. 运行 TDengine-server-3.0.0.0-Windows-x64.exe 来安装 TDengine。 - - - - :::note 当安装第一个节点时，出现 Enter FQDN：提示的时候，不需要输入任何内容。只有当安装第二个或以后更多的节点时，才需要输入已有集群中任何一个可用节点的 FQDN，支持该新节点加入集群。当然也可以不输入，而是在新节点启动前，配置到新节点的配置文件中。 diff --git a/docs/zh/07-develop/07-tmq.mdx b/docs/zh/07-develop/07-tmq.mdx index f36f76fd85..c9ac178081 100644 --- a/docs/zh/07-develop/07-tmq.mdx +++ b/docs/zh/07-develop/07-tmq.mdx @@ -88,6 +88,96 @@ void close() throws SQLException; ``` + + + +```python +class TaosConsumer(): + def __init__(self, *topics, **configs) + + def __iter__(self) + + def __next__(self) + + def sync_next(self) + + def subscription(self) + + def unsubscribe(self) + + def close(self) + + def __del__(self) +``` + + + + + +```go +func NewConsumer(conf *Config) (*Consumer, error) + +func (c *Consumer) Close() error + +func (c *Consumer) Commit(ctx context.Context, message unsafe.Pointer) error + +func (c *Consumer) FreeMessage(message unsafe.Pointer) + +func (c *Consumer) Poll(timeout time.Duration) (*Result, error) + +func (c *Consumer) Subscribe(topics []string) error + +func (c *Consumer) Unsubscribe() error +``` + + + + + +```csharp +ConsumerBuilder(IEnumerable> config) + +virtual IConsumer Build() + +Consumer(ConsumerBuilder builder) + +void Subscribe(IEnumerable topics) + +void Subscribe(string topic) + +ConsumeResult Consume(int millisecondsTimeout) + +List Subscription() + +void Unsubscribe() + +void Commit(ConsumeResult consumerResult) + +void Close() +``` + + + + + +```node +function TMQConsumer(config) + +function subscribe(topic) + +function consume(timeout) + +function subscription() + +function unsubscribe() + +function commit(msg) + +function close() +``` + + + ## 写入数据 @@ -230,6 +320,128 @@ public class MetersDeserializer extends ReferenceDeserializer { ``` + + + +Python 使用以下配置项创建一个 Consumer 实例。 + +| 参数名称 | 类型 | 参数说明 | 备注 | +| :----------------------------: | :-----: | -------------------------------------------------------- | ------------------------------------------- | +| `td_connect_ip` | string | 用于创建连接，同 `taos_connect` | | +| `td_connect_user` | string | 用于创建连接，同 `taos_connect` | | +| `td_connect_pass` | string | 用于创建连接，同 `taos_connect` | | +| `td_connect_port` | string | 用于创建连接，同 `taos_connect` | | +| `group_id` | string | 消费组 ID，同一消费组共享消费进度 | **必填项**。最大长度：192。 | +| `client_id` | string | 客户端 ID | 最大长度：192。 | +| `auto_offset_reset` | string | 消费组订阅的初始位置 | 可选：`earliest`, `latest`, `none`(default) | +| `enable_auto_commit` | string | 启用自动提交 | 合法值：`true`, `false`。 | +| `auto_commit_interval_ms` | string | 以毫秒为单位的自动提交时间间隔 | | +| `enable_heartbeat_background` | string | 启用后台心跳，启用后即使长时间不 poll 消息也不会造成离线 | 合法值：`true`, `false` | +| `experimental_snapshot_enable` | string | 从 WAL 开始消费，还是从 TSBS 开始消费 | 合法值：`true`, `false` | +| `msg_with_table_name` | string | 是否允许从消息中解析表名 | 合法值：`true`, `false` | +| `timeout` | int | 消费者拉去的超时时间 | | + + + + + +```go +config := tmq.NewConfig() +defer config.Destroy() +err = config.SetGroupID("test") +if err != nil { + panic(err) +} +err = config.SetAutoOffsetReset("earliest") +if err != nil { + panic(err) +} +err = config.SetConnectIP("127.0.0.1") +if err != nil { + panic(err) +} +err = config.SetConnectUser("root") +if err != nil { + panic(err) +} +err = config.SetConnectPass("taosdata") +if err != nil { + panic(err) +} +err = config.SetConnectPort("6030") +if err != nil { + panic(err) +} +err = config.SetMsgWithTableName(true) +if err != nil { + panic(err) +} +err = config.EnableHeartBeat() +if err != nil { + panic(err) +} +err = config.EnableAutoCommit(func(result *wrapper.TMQCommitCallbackResult) { + if result.ErrCode != 0 { + errStr := wrapper.TMQErr2Str(result.ErrCode) + err := errors.NewError(int(result.ErrCode), errStr) + panic(err) + } +}) +if err != nil { + panic(err) +} +``` + + + + + +```csharp +using TDengineTMQ; + +// 根据需要，设置消费组 (GourpId)、自动提交 (EnableAutoCommit)、 +// 自动提交时间间隔 (AutoCommitIntervalMs)、用户名 (TDConnectUser)、密码 (TDConnectPasswd) 等参数 +var cfg = new ConsumerConfig + { + EnableAutoCommit = "true" + AutoCommitIntervalMs = "1000" + GourpId = "TDengine-TMQ-C#", + TDConnectUser = "root", + TDConnectPasswd = "taosdata", + AutoOffsetReset = "earliest" + MsgWithTableName = "true", + TDConnectIp = "127.0.0.1", + TDConnectPort = "6030" + }; + +var consumer = new ConsumerBuilder(cfg).Build(); + +``` + + + + + +``` node +// 根据需要，设置消费组 (group.id)、自动提交 (enable.auto.commit)、 +// 自动提交时间间隔 (auto.commit.interval.ms)、用户名 (td.connect.user)、密码 (td.connect.pass) 等参数 + +let consumer = taos.consumer({ + 'enable.auto.commit': 'true', + 'auto.commit.interval.ms','1000', + 'group.id': 'tg2', + 'td.connect.user': 'root', + 'td.connect.pass': 'taosdata', + 'auto.offset.reset','earliest', + 'msg.with.table.name': 'true', + 'td.connect.ip','127.0.0.1', + 'td.connect.port','6030' + }); + +``` + + + 上述配置中包括 consumer group ID，如果多个 consumer 指定的 consumer group ID 一样，则自动形成一个 consumer group，共享消费进度。 @@ -260,6 +472,50 @@ topics.add("tmq_topic"); consumer.subscribe(topics); ``` + + + +```go +consumer, err := tmq.NewConsumer(config) +if err != nil { + panic(err) +} +err = consumer.Subscribe([]string{"example_tmq_topic"}) +if err != nil { + panic(err) +} +``` + + + + + +```csharp +// 创建订阅 topics 列表 +List topics = new List(); +topics.add("tmq_topic"); +// 启动订阅 +consumer.Subscribe(topics); +``` + + + + +```python +consumer = TaosConsumer('topic_ctb_column', group_id='vg2') +``` + + + + +```node +// 创建订阅 topics 列表 +let topics = ['topic_test'] + +// 启动订阅 +consumer.subscribe(topics); +``` + @@ -294,6 +550,60 @@ while(running){ ``` + + +```python +for msg in consumer: + for row in msg: + print(row) +``` + + + + +```go +for { + result, err := consumer.Poll(time.Second) + if err != nil { + panic(err) + } + fmt.Println(result) + consumer.Commit(context.Background(), result.Message) + consumer.FreeMessage(result.Message) +} +``` + + + + + +```csharp +// 消费数据 +while (true) +{ + var consumerRes = consumer.Consume(100); + // process ConsumeResult + ProcessMsg(consumerRes); + consumer.Commit(consumerRes); +} +``` + + + + + +```node +while(true){ + msg = consumer.consume(200); + // process message(consumeResult) + console.log(msg.topicPartition); + console.log(msg.block); + console.log(msg.fields) + } +``` + + + ## 结束消费 @@ -323,6 +633,45 @@ consumer.close(); ``` + + + +```python +/* 取消订阅 */ +consumer.unsubscribe(); + +/* 关闭消费 */ +consumer.close(); + + + + + +```go +consumer.Close() +``` + + + + +```csharp +// 取消订阅 +consumer.Unsubscribe(); + +// 关闭消费 +consumer.Close(); +``` + + + + +```node +consumer.unsubscribe(); +consumer.close(); +``` + + + ## 删除 *topic* @@ -670,65 +1019,15 @@ int main(int argc, char* argv[]) { ```python +import taos +from taos.tmq import TaosConsumer + import taos from taos.tmq import * - -conn = taos.connect() - -# create database -conn.execute("drop database if exists py_tmq") -conn.execute("create database if not exists py_tmq vgroups 2") - -# create table and stables -conn.select_db("py_tmq") -conn.execute("create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)") -conn.execute("create table if not exists tb1 using stb1 tags(1)") -conn.execute("create table if not exists tb2 using stb1 tags(2)") -conn.execute("create table if not exists tb3 using stb1 tags(3)") - -# create topic -conn.execute("drop topic if exists topic_ctb_column") -conn.execute("create topic if not exists topic_ctb_column as select ts, c1, c2, c3 from stb1") - -# set consumer configure options -conf = TaosTmqConf() -conf.set("group.id", "tg2") -conf.set("td.connect.user", "root") -conf.set("td.connect.pass", "taosdata") -conf.set("enable.auto.commit", "true") -conf.set("msg.with.table.name", "true") - -def tmq_commit_cb_print(tmq, resp, offset, param=None): - print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}") - -conf.set_auto_commit_cb(tmq_commit_cb_print, None) - -# build consumer -tmq = conf.new_consumer() - -# build topic list -topic_list = TaosTmqList() -topic_list.append("topic_ctb_column") - -# subscribe consumer -tmq.subscribe(topic_list) - -# check subscriptions -sub_list = tmq.subscription() -print("subscribed topics: ",sub_list) - -# start subscribe -while 1: - res = tmq.poll(1000) - if res: - topic = res.get_topic_name() - vg = res.get_vgroup_id() - db = res.get_db_name() - print(f"topic: {topic}\nvgroup id: {vg}\ndb: {db}") - for row in res: - print(row) - tb = res.get_table_name() - print(f"from table: {tb}") +consumer = TaosConsumer('topic_ctb_column', group_id='vg2') +for msg in consumer: + for row in msg: + print(row) ``` diff --git a/docs/zh/12-taos-sql/29-changes.md b/docs/zh/12-taos-sql/29-changes.md index 77544c214d..d653c59a5c 100644 --- a/docs/zh/12-taos-sql/29-changes.md +++ b/docs/zh/12-taos-sql/29-changes.md @@ -6,7 +6,7 @@ description: "TDengine 3.0 版本的语法变更说明" ## SQL 基本元素变更 -| # | **元素** |
**差异性**
| **说明** | +| # | **元素** | **
差异性
** | **说明** | | - | :------- | :-------- | :------- | | 1 | VARCHAR | 新增 | BINARY类型的别名。 | 2 | TIMESTAMP字面量 | 新增 | 新增支持 TIMESTAMP 'timestamp format' 语法。 @@ -22,7 +22,7 @@ description: "TDengine 3.0 版本的语法变更说明" 在 TDengine 中，普通表的数据模型中可使用以下数据类型。 -| # | **语句** | **
差异性
** | **说明** | +| # | **语句** | **
差异性
** | **说明** | | - | :------- | :-------- | :------- | | 1 | ALTER ACCOUNT | 废除 | 2.x中为企业版功能，3.0不再支持。语法暂时保留了，执行报“This statement is no longer supported”错误。 | 2 | ALTER ALL DNODES | 新增 | 修改所有DNODE的参数。 @@ -80,7 +80,7 @@ description: "TDengine 3.0 版本的语法变更说明" ## SQL 函数变更 -| # | **函数** | **
差异性
** | **说明** | +| # | **函数** | **
差异性
** | **说明** | | - | :------- | :-------- | :------- | | 1 | TWA | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 | 2 | IRATE | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 diff --git a/docs/zh/14-reference/03-connector/_linux_install.mdx b/docs/zh/14-reference/03-connector/_linux_install.mdx index eb7f683288..c3ddff53cd 100644 --- a/docs/zh/14-reference/03-connector/_linux_install.mdx +++ b/docs/zh/14-reference/03-connector/_linux_install.mdx @@ -1,10 +1,10 @@ -import PkgList from "/components/PkgList"; +import PkgListV3 from "/components/PkgListV3"; 1. 下载客户端安装包 - + - [所有下载](https://www.taosdata.com/cn/all-downloads/) + [所有下载](../../releases) 2. 解压缩软件包 diff --git a/docs/zh/14-reference/03-connector/_windows_install.mdx b/docs/zh/14-reference/03-connector/_windows_install.mdx index 755f96b2d7..9fdefa04c0 100644 --- a/docs/zh/14-reference/03-connector/_windows_install.mdx +++ b/docs/zh/14-reference/03-connector/_windows_install.mdx @@ -1,11 +1,10 @@ -import PkgList from "/components/PkgList"; +import PkgListV3 from "/components/PkgListV3"; 1. 下载客户端安装包 - - - [所有下载](https://www.taosdata.com/cn/all-downloads/) + + [所有下载](../../releases) 2. 执行安装程序，按提示选择默认值，完成安装 3. 安装路径 diff --git a/docs/zh/14-reference/03-connector/java.mdx b/docs/zh/14-reference/03-connector/java.mdx index c0b83e7d4a..6a78902b1e 100644 --- a/docs/zh/14-reference/03-connector/java.mdx +++ b/docs/zh/14-reference/03-connector/java.mdx @@ -93,12 +93,12 @@ Maven 项目中，在 pom.xml 中添加以下依赖： 可以通过下载 TDengine 的源码，自己编译最新版本的 Java connector ```shell -git clone https://github.com/taosdata/taos-connector-jdbc.git --branch 2.0 +git clone https://github.com/taosdata/taos-connector-jdbc.git cd taos-connector-jdbc mvn clean install -Dmaven.test.skip=true ``` -编译后，在 target 目录下会产生 taos-jdbcdriver-2.0.XX-dist.jar 的 jar 包，并自动将编译的 jar 文件放在本地的 Maven 仓库中。 +编译后，在 target 目录下会产生 taos-jdbcdriver-3.0.*-dist.jar 的 jar 包，并自动将编译的 jar 文件放在本地的 Maven 仓库中。 @@ -198,7 +198,7 @@ url 中的配置参数如下： - user：登录 TDengine 用户名，默认值 'root'。 - password：用户登录密码，默认值 'taosdata'。 -- batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。逐行拉取结果集使用 HTTP 方式进行数据传输。从 taos-jdbcdriver-2.0.38 开始，JDBC REST 连接增加批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTP，WebSocket 可以使 JDBC REST 连接支持大数据量查询，并提升查询性能。 +- batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。逐行拉取结果集使用 HTTP 方式进行数据传输。JDBC REST 连接支持批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTP，WebSocket 可以使 JDBC REST 连接支持大数据量查询，并提升查询性能。 - charset: 当开启批量拉取数据时，指定解析字符串数据的字符集。 - batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败，继续执行下面的 SQL 了。false：不再执行失败 SQL 后的任何语句。默认值为：false。 - httpConnectTimeout: 连接超时时间，单位 ms， 默认值为 5000。 @@ -216,7 +216,7 @@ url 中的配置参数如下： INSERT INTO test.t1 USING test.weather (ts, temperature) TAGS('California.SanFrancisco') VALUES(now, 24.6); ``` -- 从 taos-jdbcdriver-2.0.36 开始，如果在 url 中指定了 dbname，那么，JDBC REST 连接会默认使用/rest/sql/dbname 作为 restful 请求的 url，在 SQL 中不需要指定 dbname。例如：url 为 jdbc:TAOS-RS://127.0.0.1:6041/test，那么，可以执行 sql：insert into t1 using weather(ts, temperature) tags('California.SanFrancisco') values(now, 24.6); +- 如果在 url 中指定了 dbname，那么，JDBC REST 连接会默认使用/rest/sql/dbname 作为 restful 请求的 url，在 SQL 中不需要指定 dbname。例如：url 为 jdbc:TAOS-RS://127.0.0.1:6041/test，那么，可以执行 sql：insert into t1 using weather(ts, temperature) tags('California.SanFrancisco') values(now, 24.6); ::: @@ -230,7 +230,7 @@ INSERT INTO test.t1 USING test.weather (ts, temperature) TAGS('California.SanFra **注意**： - 应用中设置的 client parameter 为进程级别的，即如果要更新 client 的参数，需要重启应用。这是因为 client parameter 是全局参数，仅在应用程序的第一次设置生效。 -- 以下示例代码基于 taos-jdbcdriver-2.0.36。 +- 以下示例代码基于 taos-jdbcdriver-3.0.0。 ```java public Connection getConn() throws Exception{ @@ -367,7 +367,7 @@ TDengine 的 JDBC 原生连接实现大幅改进了参数绑定方式对数据 **注意**： - JDBC REST 连接目前不支持参数绑定 -- 以下示例代码基于 taos-jdbcdriver-2.0.36 +- 以下示例代码基于 taos-jdbcdriver-3.0.0 - binary 类型数据需要调用 setString 方法，nchar 类型数据需要调用 setNString 方法 - setString 和 setNString 都要求用户在 size 参数里声明表定义中对应列的列宽 @@ -635,7 +635,7 @@ TDengine 支持无模式写入功能。无模式写入兼容 InfluxDB 的 行协 **注意**： - JDBC REST 连接目前不支持无模式写入 -- 以下示例代码基于 taos-jdbcdriver-2.0.36 +- 以下示例代码基于 taos-jdbcdriver-3.0.0 ```java public class SchemalessInsertTest { @@ -666,7 +666,7 @@ public class SchemalessInsertTest { } ``` -### 订阅 +### 数据订阅 TDengine Java 连接器支持订阅功能，应用 API 如下： @@ -717,9 +717,14 @@ while(true) { #### 关闭订阅 ```java +// 取消订阅 +consumer.unsubscribe(); +// 关闭消费 consumer.close() ``` +详情请参考：[数据订阅](../../../develop/tmq) + ### 使用示例如下： ```java @@ -734,7 +739,7 @@ public abstract class ConsumerLoop { config.setProperty("msg.with.table.name", "true"); config.setProperty("enable.auto.commit", "true"); config.setProperty("group.id", "group1"); - config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ResultDeserializer"); + config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer"); this.consumer = new TaosConsumer<>(config); this.topics = Collections.singletonList("topic_speed"); @@ -754,8 +759,9 @@ public abstract class ConsumerLoop { process(record); } } + consumer.unsubscribe(); } finally { - consumer.close(); + consumer.close(); shutdownLatch.countDown(); } } @@ -875,6 +881,7 @@ public static void main(String[] args) throws Exception { | taos-jdbcdriver 版本 | 主要变化 | | :------------------: | :----------------------------: | +| 3.0.0 | 支持 TDengine 3.0 | | 2.0.39 - 2.0.40 | 增加 REST 连接/请求 超时设置 | | 2.0.38 | JDBC REST 连接增加批量拉取功能 | | 2.0.37 | 增加对 json tag 支持 | diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index aa563343f8..6d8895eb96 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -30,6 +30,7 @@ extern bool gRaftDetailLog; #define SYNC_SPEED_UP_HB_TIMER 400 #define SYNC_SPEED_UP_AFTER_MS (1000 * 20) #define SYNC_SLOW_DOWN_RANGE 100 +#define SYNC_MAX_READ_RANGE 10 #define SYNC_MAX_BATCH_SIZE 1 #define SYNC_INDEX_BEGIN 0 @@ -210,9 +211,12 @@ void syncStop(int64_t rid); int32_t syncSetStandby(int64_t rid); ESyncState syncGetMyRole(int64_t rid); bool syncIsReady(int64_t rid); +bool syncIsReadyForRead(int64_t rid); const char* syncGetMyRoleStr(int64_t rid); bool syncRestoreFinish(int64_t rid); SyncTerm syncGetMyTerm(int64_t rid); +SyncIndex syncGetLastIndex(int64_t rid); +SyncIndex syncGetCommitIndex(int64_t rid); SyncGroupId syncGetVgId(int64_t rid); void syncGetEpSet(int64_t rid, SEpSet* pEpSet); void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet); diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 7d77a0b23e..aae2e7c856 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -5,217 +5,97 @@ # # ######################################################## -# first fully qualified domain name (FQDN) for TDengine system +# The end point of the first dnode in the cluster to be connected to when `taosd` or `taos` is started # firstEp hostname:6030 -# local fully qualified domain name (FQDN) +# The end point of the second dnode to be connected to if the firstEp is not available when `taosd` or `taos` is started +# secondEp + +# The FQDN of the host where `taosd` will be started. It can be IP address # fqdn hostname -# first port number for the connection (12 continuous UDP/TCP port number are used) +# The port for external access after `taosd` is started # serverPort 6030 -# log file's directory +# The maximum number of connections a dnode can accept +# maxShellConns 5000 + +# The directory for writing log files # logDir /var/log/taos -# data file's directory +# All data files are stored in this directory # dataDir /var/lib/taos # temporary file's directory # tempDir /tmp/ -# the arbitrator's fully qualified domain name (FQDN) for TDengine system, for cluster only -# arbitrator arbitrator_hostname:6042 - -# number of threads per CPU core -# numOfThreadsPerCore 1.0 - -# number of threads to commit cache data -# numOfCommitThreads 4 - -# the proportion of total CPU cores available for query processing -# 2.0: the query threads will be set to double of the CPU cores. -# 1.0: all CPU cores are available for query processing [default]. -# 0.5: only half of the CPU cores are available for query. -# 0.0: only one core available. -# ratioOfQueryCores 1.0 - -# the last_row/first/last aggregator will not change the original column name in the result fields -keepColumnName 1 - -# number of management nodes in the system -# numOfMnodes 1 - -# enable/disable backuping vnode directory when removing vnode -# vnodeBak 1 - -# enable/disable installation / usage report +# Switch for allowing TDengine to collect and report service usage information # telemetryReporting 1 -# enable/disable load balancing -# balance 1 +# The maximum number of vnodes supported by dnode +# supportVnodes 0 -# role for dnode. 0 - any, 1 - mnode, 2 - dnode -# role 0 - -# max timer control blocks -# maxTmrCtrl 512 - -# time interval of system monitor, seconds -# monitorInterval 30 - -# number of seconds allowed for a dnode to be offline, for cluster only -# offlineThreshold 864000 - -# RPC re-try timer, millisecond -# rpcTimer 300 - -# RPC maximum time for ack, seconds. -# rpcMaxTime 600 - -# time interval of dnode status reporting to mnode, seconds, for cluster only +# The interval of dnode reporting status to mnode # statusInterval 1 -# time interval of heart beat from shell to dnode, seconds +# The interval for taos shell to send heartbeat to mnode # shellActivityTimer 3 -# minimum sliding window time, milli-second +# The minimum sliding window time, milli-second # minSlidingTime 10 -# minimum time window, milli-second +# The minimum time window, milli-second # minIntervalTime 10 -# maximum delay before launching a stream computation, milli-second -# maxStreamCompDelay 20000 +# The maximum allowed query buffer size in MB during query processing for each data node +# -1 no limit (default) +# 0 no query allowed, queries are disabled +# queryBufferSize -1 -# maximum delay before launching a stream computation for the first time, milli-second -# maxFirstStreamCompDelay 10000 - -# retry delay when a stream computation fails, milli-second -# retryStreamCompDelay 10 - -# the delayed time for launching a stream computation, from 0.1(default, 10% of whole computing time window) to 0.9 -# streamCompDelayRatio 0.1 - -# max number of vgroups per db, 0 means configured automatically -# maxVgroupsPerDb 0 - -# max number of tables per vnode -# maxTablesPerVnode 1000000 - -# cache block size (Mbyte) -# cache 16 - -# number of cache blocks per vnode -# blocks 6 - -# number of days per DB file -# days 10 - -# number of days to keep DB file -# keep 3650 - -# minimum rows of records in file block -# minRows 100 - -# maximum rows of records in file block -# maxRows 4096 - -# the number of acknowledgments required for successful data writing -# quorum 1 - -# enable/disable compression -# comp 2 - -# write ahead log (WAL) level, 0: no wal; 1: write wal, but no fysnc; 2: write wal, and call fsync -# walLevel 1 - -# if walLevel is set to 2, the cycle of fsync being executed, if set to 0, fsync is called right away -# fsync 3000 - -# number of replications, for cluster only -# replica 1 - -# the compressed rpc message, option: +# The compressed rpc message, option: # -1 (no compression) # 0 (all message compressed), # > 0 (rpc message body which larger than this value will be compressed) -# compressMsgSize -1 +# compressMsgSize -1 # query retrieved column data compression option: # -1 (no compression) # 0 (all retrieved column data compressed), # > 0 (any retrieved column size greater than this value all data will be compressed.) -# compressColData -1 - -# max length of an SQL -# maxSQLLength 65480 - -# max length of WildCards -# maxWildCardsLength 100 - -# the maximum number of records allowed for super table time sorting -# maxNumOfOrderedRes 100000 +# compressColData -1 # system time zone # timezone Asia/Shanghai (CST, +0800) # system time zone (for windows 10) -# timezone UTC-8 +# timezone UTC-8 # system locale -# locale en_US.UTF-8 +# locale en_US.UTF-8 # default system charset -# charset UTF-8 - -# max number of connections allowed in dnode -# maxShellConns 5000 - -# max number of connections allowed in client -# maxConnections 5000 +# charset UTF-8 # stop writing logs when the disk size of the log folder is less than this value -# minimalLogDirGB 1.0 +# minimalLogDirGB 1.0 # stop writing temporary files when the disk size of the tmp folder is less than this value -# minimalTmpDirGB 1.0 +# minimalTmpDirGB 1.0 # if disk free space is less than this value, taosd service exit directly within startup process -# minimalDataDirGB 2.0 - -# One mnode is equal to the number of vnode consumed -# mnodeEqualVnodeNum 4 - -# enbale/disable http service -# http 1 +# minimalDataDirGB 2.0 # enable/disable system monitor -# monitor 1 - -# enable/disable recording the SQL statements via restful interface -# httpEnableRecordSql 0 - -# number of threads used to process http requests -# httpMaxThreads 2 - -# maximum number of rows returned by the restful interface -# restfulRowLimit 10240 - -# database name must be specified in restful interface if the following parameter is set, off by default -# httpDbNameMandatory 1 - -# http keep alive, default is 30 seconds -# httpKeepAlive 30000 +# monitor 1 # The following parameter is used to limit the maximum number of lines in log files. # max number of lines per log filters -# numOfLogLines 10000000 +# numOfLogLines 10000000 # enable/disable async log -# asyncLog 1 +# asyncLog 1 # time of keeping log files, days -# logKeepDays 0 - +# logKeepDays 0 # The following parameters are used for debug purpose only. # debugFlag 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR @@ -226,87 +106,64 @@ keepColumnName 1 # 207: output trace, debug, warning and error to both screen and file # debug flag for all log type, take effect when non-zero value -# debugFlag 0 +# debugFlag 0 -# debug flag for meta management messages -# mDebugFlag 135 +# debug flag for timer +# tmrDebugFlag 131 -# debug flag for dnode messages -# dDebugFlag 135 +# debug flag for util +# uDebugFlag 131 -# debug flag for sync module -# sDebugFlag 135 +# debug flag for rpc +# rpcDebugFlag 131 -# debug flag for WAL -# wDebugFlag 135 - -# debug flag for SDB -# sdbDebugFlag 135 - -# debug flag for RPC -# rpcDebugFlag 131 - -# debug flag for TAOS TIMER -# tmrDebugFlag 131 - -# debug flag for TDengine client -# cDebugFlag 131 - -# debug flag for JNI -# jniDebugFlag 131 - -# debug flag for storage -# uDebugFlag 131 - -# debug flag for http server -# httpDebugFlag 131 - -# debug flag for monitor -# monDebugFlag 131 +# debug flag for jni +# jniDebugFlag 131 # debug flag for query -# qDebugFlag 131 +# qDebugFlag 131 + +# debug flag for taosc driver +# cDebugFlag 131 + +# debug flag for dnode messages +# dDebugFlag 135 # debug flag for vnode -# vDebugFlag 131 +# vDebugFlag 131 -# debug flag for TSDB -# tsdbDebugFlag 131 +# debug flag for meta management messages +# mDebugFlag 135 -# debug flag for continue query -# cqDebugFlag 131 +# debug flag for wal +# wDebugFlag 135 -# enable/disable recording the SQL in taos client -# enableRecordSql 0 +# debug flag for sync module +# sDebugFlag 135 + +# debug flag for tsdb +# tsdbDebugFlag 131 + +# debug flag for tq +# tqDebugFlag 131 + +# debug flag for fs +# fsDebugFlag 131 + +# debug flag for udf +# udfDebugFlag 131 + +# debug flag for sma +# smaDebugFlag 131 + +# debug flag for index +# idxDebugFlag 131 + +# debug flag for tdb +# tdbDebugFlag 131 + +# debug flag for meta +# metaDebugFlag 131 # generate core file when service crash -# enableCoreFile 1 - -# maximum display width of binary and nchar fields in the shell. The parts exceeding this limit will be hidden -# maxBinaryDisplayWidth 30 - -# enable/disable stream (continuous query) -# stream 1 - -# in retrieve blocking model, only in 50% query threads will be used in query processing in dnode -# retrieveBlockingModel 0 - -# the maximum allowed query buffer size in MB during query processing for each data node -# -1 no limit (default) -# 0 no query allowed, queries are disabled -# queryBufferSize -1 - -# percent of redundant data in tsdb meta will compact meta data,0 means donot compact -# tsdbMetaCompactRatio 0 - -# default string type used for storing JSON String, options can be binary/nchar, default is nchar -# defaultJSONStrType nchar - -# force TCP transmission -# rpcForceTcp 0 - -# unit MB. Flush vnode wal file if walSize > walFlushSize and walSize > cache*0.5*blocks -# walFlushSize 1024 - -# unit Hour. Latency of data migration -# keepTimeOffset 0 +# enableCoreFile 1 diff --git a/packaging/deb/DEBIAN/prerm b/packaging/deb/DEBIAN/prerm index 5676bf5c43..4953102842 100644 --- a/packaging/deb/DEBIAN/prerm +++ b/packaging/deb/DEBIAN/prerm @@ -29,6 +29,7 @@ else # Remove all links ${csudo}rm -f ${bin_link_dir}/taos || : ${csudo}rm -f ${bin_link_dir}/taosd || : + ${csudo}rm -f ${bin_link_dir}/udfd || : ${csudo}rm -f ${bin_link_dir}/taosadapter || : ${csudo}rm -f ${bin_link_dir}/taosdemo || : ${csudo}rm -f ${cfg_link_dir}/* || : diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 6de475a4c0..3db9005f95 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -60,6 +60,7 @@ cp ${compile_dir}/../packaging/tools/set_core.sh ${pkg_dir}${install_home_pat cp ${compile_dir}/../packaging/tools/taosd-dump-cfg.gdb ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin +cp ${compile_dir}/build/bin/udfd ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taosBenchmark ${pkg_dir}${install_home_path}/bin if [ -f "${compile_dir}/build/bin/taosadapter" ]; then diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 7a34f7a222..637d2d425a 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -69,6 +69,7 @@ cp %{_compiledir}/../packaging/tools/set_core.sh %{buildroot}%{homepath}/bin cp %{_compiledir}/../packaging/tools/taosd-dump-cfg.gdb %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taos %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin +cp %{_compiledir}/build/bin/udfd %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosBenchmark %{buildroot}%{homepath}/bin if [ -f %{_compiledir}/build/bin/taosadapter ]; then @@ -204,6 +205,7 @@ if [ $1 -eq 0 ];then # Remove all links ${csudo}rm -f ${bin_link_dir}/taos || : ${csudo}rm -f ${bin_link_dir}/taosd || : + ${csudo}rm -f ${bin_link_dir}/udfd || : ${csudo}rm -f ${bin_link_dir}/taosadapter || : ${csudo}rm -f ${cfg_link_dir}/* || : ${csudo}rm -f ${inc_link_dir}/taos.h || : diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index eda2b052d1..39606ead30 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -18,6 +18,7 @@ script_dir=$(dirname $(readlink -f "$0")) clientName="taos" serverName="taosd" +udfdName="udfd" configFile="taos.cfg" productName="TDengine" emailName="taosdata.com" @@ -192,6 +193,7 @@ function install_bin() { # Remove links ${csudo}rm -f ${bin_link_dir}/${clientName} || : ${csudo}rm -f ${bin_link_dir}/${serverName} || : + ${csudo}rm -f ${bin_link_dir}/${udfdName} || : ${csudo}rm -f ${bin_link_dir}/${adapterName} || : ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : ${csudo}rm -f ${bin_link_dir}/${demoName} || : @@ -205,6 +207,7 @@ function install_bin() { #Make link [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || : [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || : + [ -x ${install_main_dir}/bin/${udfdName} ] && ${csudo}ln -s ${install_main_dir}/bin/${udfdName} ${bin_link_dir}/${udfdName} || : [ -x ${install_main_dir}/bin/${adapterName} ] && ${csudo}ln -s ${install_main_dir}/bin/${adapterName} ${bin_link_dir}/${adapterName} || : [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || : [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || : @@ -742,7 +745,7 @@ function is_version_compatible() { fi exist_version=$(${installDir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 3) - vercomp $exist_version "2.0.16.0" + vercomp $exist_version "3.0.0.0" case $? in 2) prompt_force=1 diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 6103ce170c..f5e3bf1882 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -85,6 +85,7 @@ else ${build_dir}/bin/${clientName} \ ${taostools_bin_files} \ ${build_dir}/bin/taosadapter \ + ${build_dir}/bin/udfd \ ${script_dir}/remove.sh \ ${script_dir}/set_core.sh \ ${script_dir}/startPre.sh \ @@ -318,7 +319,7 @@ if [ "$verMode" == "cluster" ]; then fi # Copy release note -cp ${script_dir}/release_note ${install_dir} +# cp ${script_dir}/release_note ${install_dir} # exit 1 diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index aa80cfb86c..fcc8a2a942 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -118,6 +118,7 @@ function install_bin() { # Remove links ${csudo}rm -f ${bin_link_dir}/taos || : ${csudo}rm -f ${bin_link_dir}/taosd || : + ${csudo}rm -f ${bin_link_dir}/udfd || : ${csudo}rm -f ${bin_link_dir}/taosadapter || : ${csudo}rm -f ${bin_link_dir}/taosBenchmark || : ${csudo}rm -f ${bin_link_dir}/taosdemo || : @@ -130,6 +131,7 @@ function install_bin() { #Make link [ -x ${bin_dir}/taos ] && ${csudo}ln -s ${bin_dir}/taos ${bin_link_dir}/taos || : [ -x ${bin_dir}/taosd ] && ${csudo}ln -s ${bin_dir}/taosd ${bin_link_dir}/taosd || : + [ -x ${bin_dir}/udfd ] && ${csudo}ln -s ${bin_dir}/udfd ${bin_link_dir}/udfd || : [ -x ${bin_dir}/taosadapter ] && ${csudo}ln -s ${bin_dir}/taosadapter ${bin_link_dir}/taosadapter || : [ -x ${bin_dir}/taosBenchmark ] && ${csudo}ln -sf ${bin_dir}/taosBenchmark ${bin_link_dir}/taosdemo || : [ -x ${bin_dir}/taosBenchmark ] && ${csudo}ln -sf ${bin_dir}/taosBenchmark ${bin_link_dir}/taosBenchmark || : diff --git a/packaging/tools/tdengine.iss b/packaging/tools/tdengine.iss index 7310201815..272a0dfb5c 100644 --- a/packaging/tools/tdengine.iss +++ b/packaging/tools/tdengine.iss @@ -51,7 +51,7 @@ Source: taos.bat; DestDir: "{app}\include"; Flags: igNoreversion; ;Source: taosdemo.png; DestDir: "{app}\include"; Flags: igNoreversion; ;Source: taosShell.png; DestDir: "{app}\include"; Flags: igNoreversion; Source: favicon.ico; DestDir: "{app}\include"; Flags: igNoreversion; -Source: {#MyAppSourceDir}{#MyAppDLLName}; DestDir: "{win}\System32"; Flags: igNoreversion; +Source: {#MyAppSourceDir}{#MyAppDLLName}; DestDir: "{win}\System32"; Flags: 64bit;Check:IsWin64; Source: {#MyAppSourceDir}{#MyAppCfgName}; DestDir: "{app}\cfg"; Flags: igNoreversion recursesubdirs createallsubdirs onlyifdoesntexist uninsneveruninstall Source: {#MyAppSourceDir}{#MyAppDriverName}; DestDir: "{app}\driver"; Flags: igNoreversion recursesubdirs createallsubdirs ;Source: {#MyAppSourceDir}{#MyAppConnectorName}; DestDir: "{app}\connector"; Flags: igNoreversion recursesubdirs createallsubdirs diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index 1c11ee7085..9c6d941172 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -9,6 +9,11 @@ IF (TD_GRANT) ADD_DEFINITIONS(-D_GRANT) ENDIF () +IF (TD_STORAGE) + ADD_DEFINITIONS(-D_STORAGE) + TARGET_LINK_LIBRARIES(common PRIVATE storage) +ENDIF () + target_include_directories( common PUBLIC "${TD_SOURCE_DIR}/include/common" diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 8823e63db4..3956b99fdb 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -165,58 +165,11 @@ int32_t tsTtlUnit = 86400; int32_t tsTtlPushInterval = 86400; int32_t tsGrantHBInterval = 60; -void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) { - tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN); - tsDiskCfg[index].level = level; - tsDiskCfg[index].primary = primary; - uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary); -} - -static int32_t taosSetTfsCfg(SConfig *pCfg) { - SConfigItem *pItem = cfgGetItem(pCfg, "dataDir"); - memset(tsDataDir, 0, PATH_MAX); - - int32_t size = taosArrayGetSize(pItem->array); - if (size <= 0) { - tsDiskCfgNum = 1; - taosAddDataDir(0, pItem->str, 0, 1); - tstrncpy(tsDataDir, pItem->str, PATH_MAX); - if (taosMulMkDir(tsDataDir) != 0) { - uError("failed to create dataDir:%s since %s", tsDataDir, terrstr()); - return -1; - } - } else { - tsDiskCfgNum = size < TFS_MAX_DISKS ? size : TFS_MAX_DISKS; - for (int32_t index = 0; index < tsDiskCfgNum; ++index) { - SDiskCfg *pCfg = taosArrayGet(pItem->array, index); - memcpy(&tsDiskCfg[index], pCfg, sizeof(SDiskCfg)); - if (pCfg->level == 0 && pCfg->primary == 1) { - tstrncpy(tsDataDir, pCfg->dir, PATH_MAX); - } - if (taosMulMkDir(pCfg->dir) != 0) { - uError("failed to create tfsDir:%s since %s", tsDataDir, terrstr()); - return -1; - } - } - } - - if (tsDataDir[0] == 0) { - if (pItem->str != NULL) { - taosAddDataDir(tsDiskCfgNum, pItem->str, 0, 1); - tstrncpy(tsDataDir, pItem->str, PATH_MAX); - if (taosMulMkDir(tsDataDir) != 0) { - uError("failed to create tfsDir:%s since %s", tsDataDir, terrstr()); - return -1; - } - tsDiskCfgNum++; - } else { - uError("datadir not set"); - return -1; - } - } - - return 0; -} +#ifndef _STORAGE +int32_t taosSetTfsCfg(SConfig *pCfg) { return 0; } +#else +int32_t taosSetTfsCfg(SConfig *pCfg); +#endif struct SConfig *taosGetCfg() { return tsCfg; diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 77b18b8c02..5164e22474 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -98,6 +98,7 @@ void vnodeSyncStart(SVnode* pVnode); void vnodeSyncClose(SVnode* pVnode); void vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg); bool vnodeIsLeader(SVnode* pVnode); +bool vnodeIsReadyForRead(SVnode* pVnode); bool vnodeIsRoleLeader(SVnode* pVnode); #ifdef __cplusplus diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 8a249eb105..55630511bf 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -17,7 +17,7 @@ #include "tmsg.h" #include "tq.h" -int32_t tdBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBlock* pDataBlock, +int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq) { ASSERT(pDataBlock->info.type == STREAM_DELETE_RESULT); int32_t totRow = pDataBlock->info.rows; @@ -68,9 +68,10 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); if (pDataBlock->info.type == STREAM_DELETE_RESULT) { int32_t padding1 = 0; - void* padding2 = taosMemoryMalloc(1); + void* padding2 = NULL; taosArrayPush(schemaReqSz, &padding1); taosArrayPush(schemaReqs, &padding2); + continue; } STagVal tagVal = { @@ -138,8 +139,7 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem continue; } int32_t rows = pDataBlock->info.rows; - // TODO min - int32_t rowSize = pDataBlock->info.rowSize; + /*int32_t rowSize = pDataBlock->info.rowSize;*/ int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); int32_t schemaLen = 0; @@ -150,7 +150,6 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem } // assign data - // TODO ret = rpcMallocCont(cap); ret->header.vgId = pVnode->config.vgId; ret->length = sizeof(SSubmitReq); @@ -161,13 +160,12 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); if (pDataBlock->info.type == STREAM_DELETE_RESULT) { pDeleteReq->suid = suid; - tdBuildDeleteReq(pVnode, stbFullName, pDataBlock, pDeleteReq); + tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, pDeleteReq); continue; } blkHead->numOfRows = htonl(pDataBlock->info.rows); blkHead->sversion = htonl(pTSchema->version); - // TODO blkHead->suid = htobe64(suid); // uid is assigned by vnode blkHead->uid = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 8ae0e824cf..6fc6636623 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -196,9 +196,9 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid tsdbCacheDeleteLast(pTsdb->lruCache, pTbData->uid, eKey); } - tsdbError("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64 - " since %s", - TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code)); + tsdbInfo("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64 + " since %s", + TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code)); return code; _err: diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index ee75e39c5f..43c9b4c09f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -283,7 +283,7 @@ int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { vTrace("message in vnode query queue is processing"); - if ((pMsg->msgType == TDMT_SCH_QUERY) && !vnodeIsLeader(pVnode)) { + if ((pMsg->msgType == TDMT_SCH_QUERY) && !vnodeIsReadyForRead(pVnode)) { vnodeRedirectRpcMsg(pVnode, pMsg); return 0; } @@ -307,7 +307,7 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { vTrace("vgId:%d, msg:%p in fetch queue is processing", pVnode->config.vgId, pMsg); if ((pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_VND_TABLE_META || pMsg->msgType == TDMT_VND_TABLE_CFG || pMsg->msgType == TDMT_VND_BATCH_META) && - !vnodeIsLeader(pVnode)) { + !vnodeIsReadyForRead(pVnode)) { vnodeRedirectRpcMsg(pVnode, pMsg); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index b695352c3f..65d4e9aaf1 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -781,3 +781,17 @@ bool vnodeIsLeader(SVnode *pVnode) { return true; } + +bool vnodeIsReadyForRead(SVnode *pVnode) { + if (syncIsReady(pVnode->sync)) { + return true; + } + + if (syncIsReadyForRead(pVnode->sync)) { + return true; + } + + vDebug("vgId:%d, vnode not ready for read, state:%s, last:%ld, cmt:%ld", pVnode->config.vgId, + syncGetMyRoleStr(pVnode->sync), syncGetLastIndex(pVnode->sync), syncGetCommitIndex(pVnode->sync)); + return false; +} diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index de72c32fa1..cbf81f1d0d 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -4918,6 +4918,16 @@ int32_t mavgFunction(SqlFunctionCtx* pCtx) { return numOfElems; } +static SSampleInfo* getSampleOutputInfo(SqlFunctionCtx* pCtx) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SSampleInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + + pInfo->data = (char*)pInfo + sizeof(SSampleInfo); + pInfo->tuplePos = (STuplePos*)((char*)pInfo + sizeof(SSampleInfo) + pInfo->samples * pInfo->colBytes); + + return pInfo; +} + bool getSampleFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { SColumnNode* pCol = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0); SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); @@ -4972,7 +4982,7 @@ static void doReservoirSample(SqlFunctionCtx* pCtx, SSampleInfo* pInfo, char* da int32_t sampleFunction(SqlFunctionCtx* pCtx) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - SSampleInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + SSampleInfo* pInfo = getSampleOutputInfo(pCtx); SInputColumnInfoData* pInput = &pCtx->input; @@ -4998,7 +5008,7 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) { int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); - SSampleInfo* pInfo = GET_ROWCELL_INTERBUF(pEntryInfo); + SSampleInfo* pInfo = getSampleOutputInfo(pCtx); pEntryInfo->complete = true; int32_t slotId = pCtx->pExpr->base.resSchema.slotId; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index ef985a3894..1c7446ad6f 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1399,7 +1399,7 @@ static int32_t translateTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFu "%s function must be used in select statements", pFunc->functionName); } SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; - if (QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && + if (NULL != pSelect->pFromTable && QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && !isTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, "%s function requires valid time series input", pFunc->functionName); @@ -2037,16 +2037,13 @@ static int32_t setVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, code = getDBVgInfoImpl(pCxt, pName, &vgroupList); } - if (TSDB_CODE_SUCCESS == code && - 0 == strcmp(pRealTable->table.dbName, TSDB_INFORMATION_SCHEMA_DB) && - 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS) && - isSelectStmt(pCxt->pCurrStmt) && + if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.dbName, TSDB_INFORMATION_SCHEMA_DB) && + 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS) && isSelectStmt(pCxt->pCurrStmt) && 0 == taosArrayGetSize(vgroupList)) { ((SSelectStmt*)pCxt->pCurrStmt)->isEmptyResult = true; } - if (TSDB_CODE_SUCCESS == code && - 0 == strcmp(pRealTable->table.dbName, TSDB_INFORMATION_SCHEMA_DB) && + if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.dbName, TSDB_INFORMATION_SCHEMA_DB) && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES)) { code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &vgroupList); } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index c7784cd62e..1991560d42 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -392,6 +392,29 @@ bool syncIsReady(int64_t rid) { return b; } +bool syncIsReadyForRead(int64_t rid) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return false; + } + ASSERT(rid == pSyncNode->rid); + + // TODO: last not noop? + SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode); + bool b = (pSyncNode->state == TAOS_SYNC_STATE_LEADER) && (pSyncNode->commitIndex >= lastIndex - SYNC_MAX_READ_RANGE); + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + + // if false, set error code + if (false == b) { + if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { + terrno = TSDB_CODE_SYN_NOT_LEADER; + } else { + terrno = TSDB_CODE_APP_NOT_READY; + } + } + return b; +} + bool syncIsRestoreFinish(int64_t rid) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { @@ -519,6 +542,30 @@ SyncTerm syncGetMyTerm(int64_t rid) { return term; } +SyncIndex syncGetLastIndex(int64_t rid) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return SYNC_INDEX_INVALID; + } + ASSERT(rid == pSyncNode->rid); + SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode); + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return lastIndex; +} + +SyncIndex syncGetCommitIndex(int64_t rid) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return SYNC_INDEX_INVALID; + } + ASSERT(rid == pSyncNode->rid); + SyncIndex cmtIndex = pSyncNode->commitIndex; + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return cmtIndex; +} + SyncGroupId syncGetVgId(int64_t rid) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { @@ -828,6 +875,15 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { pSyncNode->changing = true; } + // not restored, vnode enable + if (!pSyncNode->restoreFinish && pSyncNode->vgId != 1) { + ret = -1; + terrno = TSDB_CODE_SYN_PROPOSE_NOT_READY; + sError("vgId:%d, failed to sync propose since not ready, type:%s, last:%ld, cmt:%ld", pSyncNode->vgId, + TMSG_INFO(pMsg->msgType), syncNodeGetLastIndex(pSyncNode), pSyncNode->commitIndex); + goto _END; + } + SRespStub stub; stub.createTime = taosGetTimestampMs(); stub.rpcMsg = *pMsg; diff --git a/source/os/src/osRand.c b/source/os/src/osRand.c index 461a72e962..bd2bfa486e 100644 --- a/source/os/src/osRand.c +++ b/source/os/src/osRand.c @@ -37,9 +37,13 @@ uint32_t taosRandR(uint32_t *pSeed) { uint32_t taosSafeRand(void) { #ifdef WINDOWS - uint32_t seed; + uint32_t seed = taosRand(); HCRYPTPROV hCryptProv; - if (!CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_RSA_FULL, 0)) return seed; + if (!CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_RSA_FULL, 0)) { + if (!CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_RSA_FULL, CRYPT_NEWKEYSET)) { + return seed; + } + } if (hCryptProv != NULL) { if (!CryptGenRandom(hCryptProv, 4, &seed)) return seed; } diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index fda5e5cb6e..97295d75e0 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -237,8 +237,8 @@ ./test.sh -f tsim/stream/distributeInterval0.sim ./test.sh -f tsim/stream/distributeIntervalRetrive0.sim ./test.sh -f tsim/stream/distributeSession0.sim -#./test.sh -f tsim/stream/session0.sim -#./test.sh -f tsim/stream/session1.sim +./test.sh -f tsim/stream/session0.sim +./test.sh -f tsim/stream/session1.sim ./test.sh -f tsim/stream/state0.sim ./test.sh -f tsim/stream/triggerInterval0.sim ./test.sh -f tsim/stream/triggerSession0.sim diff --git a/tests/script/tsim/stream/session0.sim b/tests/script/tsim/stream/session0.sim index fee8c98cce..3e0af354d8 100644 --- a/tests/script/tsim/stream/session0.sim +++ b/tests/script/tsim/stream/session0.sim @@ -83,22 +83,22 @@ if $data11 != 3 then goto loop0 endi -if $data12 != NULL then +if $data12 != 10 then print ======data12=$data12 goto loop0 endi -if $data13 != NULL then +if $data13 != 10 then print ======data13=$data13 goto loop0 endi -if $data14 != NULL then +if $data14 != 1.100000000 then print ======data14=$data14 return -1 endi -if $data15 != NULL then +if $data15 != 0.000000000 then print ======data15=$data15 return -1 endi @@ -141,38 +141,38 @@ if $data01 != 7 then goto loop1 endi -if $data02 != NULL then +if $data02 != 18 then print =====data02=$data02 goto loop1 endi -if $data03 != NULL then +if $data03 != 4 then print =====data03=$data03 goto loop1 endi -if $data04 != NULL then - print ======$data04 +if $data04 != 1.000000000 then + print ======data04=$data04 return -1 endi -if $data05 != NULL then - print ======$data05 +if $data05 != 1.154700538 then + print ======data05=$data05 return -1 endi if $data06 != 4 then - print ======$data06 + print ======data06=$data06 return -1 endi if $data07 != 1.000000000 then - print ======$data07 + print ======data07=$data07 return -1 endi if $data08 != 13 then - print ======$data08 + print ======data08=$data08 return -1 endi