Merge branch 'main' into fix/liaohj
This commit is contained in:
Haojun Liao
commit
2c9f28d97f
|
|
@ -35,11 +35,13 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0
|
|||
|
||||
:::note
|
||||
|
||||
- All the data in `tag_set` will be converted to NCHAR type automatically .
|
||||
- Each data in `field_set` must be self-descriptive for its data type. For example 1.2f32 means a value 1.2 of float type. Without the "f" type suffix, it will be treated as type double.
|
||||
- Multiple kinds of precision can be used for the `timestamp` field. Time precision can be from nanosecond (ns) to hour (h).
|
||||
- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be created automatically. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored.
|
||||
- All the data in `tag_set` will be converted to NCHAR type automatically
|
||||
- Each data in `field_set` must be self-descriptive for its data type. For example 1.2f32 means a value 1.2 of float type. Without the "f" type suffix, it will be treated as type double
|
||||
- Multiple kinds of precision can be used for the `timestamp` field. Time precision can be from nanosecond (ns) to hour (h)
|
||||
- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be created automatically. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored
|
||||
- It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur.(smlDataFormat in taos.cfg default to false after version of 3.0.1.3, smlDataFormat is discarded since 3.0.3.0)
|
||||
- Due to the fact that SQL table names do not support point(.), schemaless has also processed point(.) and automatically replaced them with underscores(_)
|
||||
|
||||
:::
|
||||
|
||||
For more details please refer to [InfluxDB Line Protocol](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) and [TDengine Schemaless](/reference/schemaless/#Schemaless-Line-Protocol)
|
||||
|
|
|
|||
|
|
@ -34,6 +34,8 @@ meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3
|
|||
```
|
||||
|
||||
- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored.
|
||||
- Due to the fact that SQL table names do not support point(.), schemaless has also processed point(.) and automatically replaced them with underscores(_)
|
||||
|
||||
Please refer to [OpenTSDB Telnet API](http://opentsdb.net/docs/build/html/api_telnet/put.html) for more details.
|
||||
|
||||
## Examples
|
||||
|
|
@ -68,11 +70,11 @@ Database changed.
|
|||
taos> show stables;
|
||||
name |
|
||||
=================================
|
||||
meters.current |
|
||||
meters.voltage |
|
||||
meters_current |
|
||||
meters_voltage |
|
||||
Query OK, 2 row(s) in set (0.002544s)
|
||||
|
||||
taos> select tbname, * from `meters.current`;
|
||||
taos> select tbname, * from `meters_current`;
|
||||
tbname | _ts | _value | groupid | location |
|
||||
==================================================================================================================================
|
||||
t_0e7bcfa21a02331c06764f275... | 2022-03-28 09:56:51.249 | 10.800000000 | 3 | California.LosAngeles |
|
||||
|
|
@ -87,5 +89,5 @@ Query OK, 4 row(s) in set (0.005399s)
|
|||
If you want query the data of `location=California.LosAngeles groupid=3`, here is the query SQL:
|
||||
|
||||
```sql
|
||||
SELECT * FROM `meters.current` WHERE location = "California.LosAngeles" AND groupid = 3;
|
||||
SELECT * FROM `meters_current` WHERE location = "California.LosAngeles" AND groupid = 3;
|
||||
```
|
||||
|
|
|
|||
|
|
@ -49,6 +49,8 @@ Please refer to [OpenTSDB HTTP API](http://opentsdb.net/docs/build/html/api_http
|
|||
|
||||
- In JSON protocol, strings will be converted to NCHAR type and numeric values will be converted to double type.
|
||||
- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored.
|
||||
- Due to the fact that SQL table names do not support point(.), schemaless has also processed point(.) and automatically replaced them with underscores(_)
|
||||
|
||||
:::
|
||||
|
||||
## Examples
|
||||
|
|
@ -83,11 +85,11 @@ Database changed.
|
|||
taos> show stables;
|
||||
name |
|
||||
=================================
|
||||
meters.current |
|
||||
meters.voltage |
|
||||
meters_current |
|
||||
meters_voltage |
|
||||
Query OK, 2 row(s) in set (0.001954s)
|
||||
|
||||
taos> select * from `meters.current`;
|
||||
taos> select * from `meters_current`;
|
||||
_ts | _value | groupid | location |
|
||||
===================================================================================================================
|
||||
2022-03-28 09:56:51.249 | 10.300000000 | 2.000000000 | California.SanFrancisco |
|
||||
|
|
@ -100,5 +102,5 @@ Query OK, 2 row(s) in set (0.004076s)
|
|||
If you want query the data of "tags": {"location": "California.LosAngeles", "groupid": 1}, here is the query SQL:
|
||||
|
||||
```sql
|
||||
SELECT * FROM `meters.current` WHERE location = "California.LosAngeles" AND groupid = 3;
|
||||
SELECT * FROM `meters_current` WHERE location = "California.LosAngeles" AND groupid = 3;
|
||||
```
|
||||
|
|
|
|||
|
|
@ -23,7 +23,25 @@ By subscribing to a topic, a consumer can obtain the latest data in that topic i
|
|||
|
||||
To implement these features, TDengine indexes its write-ahead log (WAL) file for fast random access and provides configurable methods for replacing and retaining this file. You can define a retention period and size for this file. For information, see the CREATE DATABASE statement. In this way, the WAL file is transformed into a persistent storage engine that remembers the order in which events occur. However, note that configuring an overly long retention period for your WAL files makes database compression inefficient. TDengine then uses the WAL file instead of the time-series database as its storage engine for queries in the form of topics. TDengine reads the data from the WAL file; uses a unified query engine instance to perform filtering, transformations, and other operations; and finally pushes the data to consumers.
|
||||
|
||||
Tips: Data subscription is to consume data from the wal. If some wal files are deleted according to WAL retention policy, the deleted data can't be consumed any more. So you need to set a reasonable value for parameter `WAL_RETENTION_PERIOD` or `WAL_RETENTION_SIZE` when creating the database and make sure your application consume the data in a timely way to make sure there is no data loss. This behavior is similar to Kafka and other widely used message queue products.
|
||||
Tips:(c interface for example)
|
||||
- A consumption group consumes all data under the same topic, and different consumption groups are independent of each other;
|
||||
- A consumption group consumes all vgroups of the same topic, which can be composed of multiple consumers, but a vgroup is only consumed by one consumer. If the number of consumers exceeds the number of vgroups, the excess consumers do not consume data;
|
||||
- On the server side, only one offset is saved for each vgroup, and the offsets for each vgroup are monotonically increasing, but not necessarily continuous. There is no correlation between the offsets of various vgroups;
|
||||
- Each poll server will return a result block, which belongs to a vgroup and may contain data from multiple versions of wal. This block can be accessed through tmq_get_vgroup_offset. The offset interface obtains the offset of the first record in the block;
|
||||
- If a consumer group has never committed an offset, when its member consumers restart and pull data again, they start consuming from the set value of the parameter auto.offset.reset; In a consumer lifecycle, the client locally records the offset of the most recent pull data and will not pull duplicate data;
|
||||
- If a consumer terminates abnormally (without calling tmq_close), they need to wait for about 12 seconds to trigger their consumer group rebalance. The consumer's status on the server will change to LOST, and after about 1 day, the consumer will be automatically deleted; Exit normally, and after exiting, the consumer will be deleted; Add a new consumer, wait for about 2 seconds to trigger Rebalance, and the consumer's status on the server will change to ready;
|
||||
- The consumer group Rebalance will reassign Vgroups to all consumer members in the ready state of the group, and consumers can only assign/see/commit/poll operations to the Vgroups they are responsible for;
|
||||
- Consumers can tmq_position to obtain the offset of the current consumption, seek to the specified offset, and consume again;
|
||||
- Seek points the position to the specified offset without executing the commit operation. Once the seek is successful, it can poll the specified offset and subsequent data;
|
||||
- Position is to obtain the current consumption position, which is the position to be taken next time, not the current consumption position
|
||||
- Commit is the submission of the consumption location. Without parameters, it is the submission of the current consumption location (the location to be taken next time, not the current consumption location). With parameters, it is the location in the submission parameters (i.e. the location to be taken after the next exit and restart)
|
||||
- Seek is to set the consumer's consumption position. Wherever the seek goes, the position will be returned, all of which are the positions to be taken next time
|
||||
- Seek does not affect commit, commit does not affect seek, independent of each other, the two are different concepts
|
||||
- The begin interface is the offset of the first data in wal, and the end interface is the offset+1 of the last data in wal10.
|
||||
- Before the seek operation, tmq must be call tmq_get_topic_assignment, The assignment interface obtains the vgroup ID and offset range of the consumer. The seek operation will detect whether the vgroup ID and offset are legal, and if they are illegal, an error will be reported;
|
||||
- Due to the existence of a WAL expiration deletion mechanism, even if the seek operation is successful, it is possible that the offset has expired when polling data. If the offset of poll is less than the WAL minimum version number, it will be consumed from the WAL minimum version number;
|
||||
- The tmq_get_vgroup_offset interface obtains the offset of the first data in the result block where the record is located. When seeking to this offset, it will consume all the data in this block. Refer to point four;
|
||||
- Data subscription is to consume data from the wal. If some wal files are deleted according to WAL retention policy, the deleted data can't be consumed any more. So you need to set a reasonable value for parameter `WAL_RETENTION_PERIOD` or `WAL_RETENTION_SIZE` when creating the database and make sure your application consume the data in a timely way to make sure there is no data loss. This behavior is similar to Kafka and other widely used message queue products.
|
||||
|
||||
## Data Schema and API
|
||||
|
||||
|
|
@ -33,36 +51,57 @@ The related schemas and APIs in various languages are described as follows:
|
|||
<TabItem value="c" label="C">
|
||||
|
||||
```c
|
||||
typedef struct tmq_t tmq_t;
|
||||
typedef struct tmq_conf_t tmq_conf_t;
|
||||
typedef struct tmq_list_t tmq_list_t;
|
||||
typedef struct tmq_t tmq_t;
|
||||
typedef struct tmq_conf_t tmq_conf_t;
|
||||
typedef struct tmq_list_t tmq_list_t;
|
||||
|
||||
typedef void(tmq_commit_cb(tmq_t *, int32_t code, void *param));
|
||||
typedef void(tmq_commit_cb(tmq_t *tmq, int32_t code, void *param));
|
||||
|
||||
DLL_EXPORT tmq_list_t *tmq_list_new();
|
||||
DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *);
|
||||
DLL_EXPORT void tmq_list_destroy(tmq_list_t *);
|
||||
DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
|
||||
DLL_EXPORT const char *tmq_err2str(int32_t code);
|
||||
typedef enum tmq_conf_res_t {
|
||||
TMQ_CONF_UNKNOWN = -2,
|
||||
TMQ_CONF_INVALID = -1,
|
||||
TMQ_CONF_OK = 0,
|
||||
} tmq_conf_res_t;
|
||||
|
||||
DLL_EXPORT int32_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list);
|
||||
DLL_EXPORT int32_t tmq_unsubscribe(tmq_t *tmq);
|
||||
DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout);
|
||||
DLL_EXPORT int32_t tmq_consumer_close(tmq_t *tmq);
|
||||
DLL_EXPORT int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg);
|
||||
DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const TAOS_RES *msg, tmq_commit_cb *cb, void *param);
|
||||
typedef struct tmq_topic_assignment {
|
||||
int32_t vgId;
|
||||
int64_t currentOffset;
|
||||
int64_t begin;
|
||||
int64_t end; // The last version of wal + 1
|
||||
} tmq_topic_assignment;
|
||||
|
||||
enum tmq_conf_res_t {
|
||||
TMQ_CONF_UNKNOWN = -2,
|
||||
TMQ_CONF_INVALID = -1,
|
||||
TMQ_CONF_OK = 0,
|
||||
};
|
||||
typedef enum tmq_conf_res_t tmq_conf_res_t;
|
||||
DLL_EXPORT tmq_conf_t *tmq_conf_new();
|
||||
DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value);
|
||||
DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf);
|
||||
DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param);
|
||||
|
||||
DLL_EXPORT tmq_conf_t *tmq_conf_new();
|
||||
DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value);
|
||||
DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf);
|
||||
DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param);
|
||||
DLL_EXPORT tmq_list_t *tmq_list_new();
|
||||
DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *);
|
||||
DLL_EXPORT void tmq_list_destroy(tmq_list_t *);
|
||||
DLL_EXPORT int32_t tmq_list_get_size(const tmq_list_t *);
|
||||
DLL_EXPORT char **tmq_list_to_c_array(const tmq_list_t *);
|
||||
|
||||
DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
|
||||
DLL_EXPORT int32_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list);
|
||||
DLL_EXPORT int32_t tmq_unsubscribe(tmq_t *tmq);
|
||||
DLL_EXPORT int32_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
|
||||
DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout);
|
||||
DLL_EXPORT int32_t tmq_consumer_close(tmq_t *tmq);
|
||||
DLL_EXPORT int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg); //Commit the msg’s offset + 1
|
||||
DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const TAOS_RES *msg, tmq_commit_cb *cb, void *param);
|
||||
DLL_EXPORT int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset);
|
||||
DLL_EXPORT void tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb *cb, void *param);
|
||||
DLL_EXPORT int32_t tmq_get_topic_assignment(tmq_t *tmq, const char *pTopicName, tmq_topic_assignment **assignment,int32_t *numOfAssignment);
|
||||
DLL_EXPORT void tmq_free_assignment(tmq_topic_assignment* pAssignment);
|
||||
DLL_EXPORT int32_t tmq_offset_seek(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset);
|
||||
DLL_EXPORT int64_t tmq_position(tmq_t *tmq, const char *pTopicName, int32_t vgId); // The current offset is the offset of the last consumed message + 1
|
||||
DLL_EXPORT int64_t tmq_committed(tmq_t *tmq, const char *pTopicName, int32_t vgId);
|
||||
|
||||
DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res);
|
||||
DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res);
|
||||
DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res);
|
||||
DLL_EXPORT int64_t tmq_get_vgroup_offset(TAOS_RES* res); // Get current offset of the result
|
||||
DLL_EXPORT const char *tmq_err2str(int32_t code);
|
||||
```
|
||||
|
||||
For more information, see [C/C++ Connector](/reference/connector/cpp).
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ sidebar_label: Access Control
|
|||
description: This document describes how to manage users and permissions in TDengine.
|
||||
---
|
||||
|
||||
This document describes how to manage permissions in TDengine.
|
||||
User and Access control is a distingguished feature of TDengine enterprise edition. In this section, only the most fundamental functionalities of user and access control are demonstrated. To get the full knowledge of user and access control, please contact the TDengine team.
|
||||
|
||||
## Create a User
|
||||
|
||||
|
|
|
|||
|
|
@ -378,7 +378,7 @@ In addition to writing data using the SQL method or the parameter binding API, w
|
|||
- `TAOS_RES* taos_schemaless_insert(TAOS* taos, const char* lines[], int numLines, int protocol, int precision)`
|
||||
|
||||
**Function description**
|
||||
This interface writes the text data of the line protocol to TDengine.
|
||||
- This interface writes the text data of the line protocol to TDengine.
|
||||
|
||||
**Parameter description**
|
||||
- taos: database connection, established by the `taos_connect()` function.
|
||||
|
|
@ -387,12 +387,13 @@ In addition to writing data using the SQL method or the parameter binding API, w
|
|||
- protocol: the protocol type of the lines, used to identify the text data format.
|
||||
- precision: precision string for the timestamp in the text data.
|
||||
|
||||
**return value**
|
||||
TAOS_RES structure, application can get error message by using `taos_errstr()` and also error code by using `taos_errno()`.
|
||||
**Return value**
|
||||
- TAOS_RES structure, application can get error message by using `taos_errstr()` and also error code by using `taos_errno()`.
|
||||
In some cases, the returned TAOS_RES is `NULL`, and it is still possible to call `taos_errno()` to safely get the error code information.
|
||||
The returned TAOS_RES needs to be freed by the caller in order to avoid memory leaks.
|
||||
|
||||
**Description**
|
||||
|
||||
The protocol type is enumerated and contains the following three formats.
|
||||
|
||||
- TSDB_SML_LINE_PROTOCOL: InfluxDB line protocol (Line Protocol)
|
||||
|
|
@ -427,3 +428,90 @@ In addition to writing data using the SQL method or the parameter binding API, w
|
|||
- Within _raw interfaces represent data through the passed parameters lines and len. In order to solve the problem that the original interface data contains '\0' and is truncated. The totalRows pointer returns the number of parsed data rows.
|
||||
- Within _ttl interfaces can pass the ttl parameter to control the ttl expiration time of the table.
|
||||
- Within _reqid interfaces can track the entire call chain by passing the reqid parameter.
|
||||
|
||||
### Subscription API
|
||||
|
||||
- `int32_t tmq_get_topic_assignment(tmq_t *tmq, const char *pTopicName, tmq_topic_assignment **assignment, int32_t *numOfAssignment)`
|
||||
- `void tmq_free_assignment(tmq_topic_assignment* pAssignment)`
|
||||
|
||||
tmq_topic_assignment defined as follows:
|
||||
```c
|
||||
typedef struct tmq_topic_assignment {
|
||||
int32_t vgId;
|
||||
int64_t currentOffset;
|
||||
int64_t begin;
|
||||
int64_t end;
|
||||
} tmq_topic_assignment;
|
||||
```
|
||||
**Function description**
|
||||
- tmq_get_topic_assignment get the current vgroup information of this consumer
|
||||
|
||||
**Parameter description**
|
||||
- numOfAssignment:the num of vgroups assigned to this consumer
|
||||
- assignment:the information of vgroups, needed to be freed by tmq_free_assignment
|
||||
|
||||
**Return value**
|
||||
- zero success,none zero failed, wrong message can be obtained through `char *tmq_err2str(int32_t code)`
|
||||
|
||||
- `int64_t tmq_committed(tmq_t *tmq, const char *pTopicName, int32_t vgId)`
|
||||
|
||||
**Function description**
|
||||
- get the committed offset
|
||||
|
||||
**Return value**
|
||||
- the value of committed offset, -2147467247 means no committed value, Other values less than 0 indicate failure
|
||||
|
||||
- `int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg)`
|
||||
- `void tmq_commit_async(tmq_t *tmq, const TAOS_RES *msg, tmq_commit_cb *cb, void *param)`
|
||||
- `int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset)`
|
||||
- `void tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb *cb, void *param)`
|
||||
|
||||
**Function description**
|
||||
|
||||
- The commit interface is divided into two types, each with synchronous and asynchronous interfaces:
|
||||
- The first type: based on message submission, submit the progress in the message. If the message passes NULL, submit the current progress of all vgroups consumed by the current consumer: tmq_commit_sync/tmq_commit_async
|
||||
- The second type: submit based on the offset of a Vgroup in a topic: tmq_commit_offset_sync/tmq_commit_offset_async
|
||||
|
||||
**Parameter description**
|
||||
- msg：Message consumed, If the message passes NULL, submit the current progress of all vgroups consumed by the current consumer
|
||||
|
||||
**Return value**
|
||||
- zero success, none zero failed, wrong message can be obtained through `char *tmq_err2str(int32_t code)`
|
||||
|
||||
- `int64_t tmq_position(tmq_t *tmq, const char *pTopicName, int32_t vgId)`
|
||||
|
||||
**Function description**
|
||||
- Obtain the current consumption location, which is the next location of the data consumed
|
||||
|
||||
**Return value**
|
||||
- the current consumption location, none zero failed, wrong message can be obtained through `char *tmq_err2str(int32_t code)`
|
||||
|
||||
- `int32_t tmq_offset_seek(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset)`
|
||||
|
||||
**Function description**
|
||||
- Set the offset position of the consumer in a Vgroup of a certain topic to start consumption
|
||||
|
||||
**Return value**
|
||||
- zero success, none zero failed, wrong message can be obtained through `char *tmq_err2str(int32_t code)`
|
||||
|
||||
- `int32_t int64_t tmq_get_vgroup_offset(TAOS_RES* res)`
|
||||
|
||||
**Function description**
|
||||
- Obtain the starting offset of the consumed data
|
||||
|
||||
**Parameter description**
|
||||
- msg：Message consumed
|
||||
|
||||
**Return value**
|
||||
- the starting offset of the consumed data, none zero failed, wrong message can be obtained through `char *tmq_err2str(int32_t code)`
|
||||
|
||||
- `int32_t int32_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics)`
|
||||
|
||||
**Function description**
|
||||
- Obtain a list of topics subscribed by consumers
|
||||
|
||||
**Parameter description**
|
||||
- topics: a list of topics subscribed by consumers，need to be freed by tmq_list_destroy
|
||||
|
||||
**Return value**
|
||||
- zero success，none zero failed, wrong message can be obtained through `char *tmq_err2str(int32_t code)`
|
||||
|
|
@ -1091,8 +1091,6 @@ public abstract class ConsumerLoop {
|
|||
config.setProperty("client.id", "1");
|
||||
config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer");
|
||||
config.setProperty("value.deserializer.encoding", "UTF-8");
|
||||
config.setProperty("experimental.snapshot.enable", "true");
|
||||
|
||||
|
||||
this.consumer = new TaosConsumer<>(config);
|
||||
this.topics = Collections.singletonList("topic_speed");
|
||||
|
|
|
|||
|
|
@ -32,8 +32,8 @@ All data in tag_set is automatically converted to the NCHAR data type and does n
|
|||
|
||||
In the schemaless writing data line protocol, each data item in the field_set needs to be described with its data type. Let's explain in detail:
|
||||
|
||||
- If there are English double quotes on both sides, it indicates the BINARY(32) type. For example, `"abc"`.
|
||||
- If there are double quotes on both sides and an L prefix, it means NCHAR(32) type. For example, `L"error message"`.
|
||||
- If there are English double quotes on both sides, it indicates the VARCHAR(N) type. For example, `"abc"`.
|
||||
- If there are double quotes on both sides and an L prefix, it means NCHAR(N) type. For example, `L"error message"`.
|
||||
- Spaces, equals sign (=), comma (,), double quote ("), and backslash (\\) need to be escaped with a backslash (\\) in front. (All refer to the ASCII character). The rules are as follows:
|
||||
|
||||
| **Serial number** | **Element** | **Escape characters** |
|
||||
|
|
|
|||
|
|
@ -227,12 +227,6 @@ tmq_t* build_consumer() {
|
|||
return NULL;
|
||||
}
|
||||
|
||||
code = tmq_conf_set(conf, "experimental.snapshot.enable", "false");
|
||||
if (TMQ_CONF_OK != code) {
|
||||
tmq_conf_destroy(conf);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
|
||||
|
||||
tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
|
||||
|
|
|
|||
|
|
@ -35,7 +35,6 @@ func main() {
|
|||
"td.connect.port": "6030",
|
||||
"client.id": "test_tmq_client",
|
||||
"enable.auto.commit": "false",
|
||||
"experimental.snapshot.enable": "true",
|
||||
"msg.with.table.name": "true",
|
||||
})
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -34,12 +34,13 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0
|
|||
|
||||
:::note
|
||||
|
||||
- tag_set 中的所有的数据自动转化为 NCHAR 数据类型;
|
||||
- field_set 中的每个数据项都需要对自身的数据类型进行描述, 比如 1.2f32 代表 FLOAT 类型的数值 1.2, 如果不带类型后缀会被当作 DOUBLE 处理;
|
||||
- timestamp 支持多种时间精度。写入数据的时候需要用参数指定时间精度，支持从小时到纳秒的 6 种时间精度。
|
||||
- tag_set 中的所有的数据自动转化为 NCHAR 数据类型
|
||||
- field_set 中的每个数据项都需要对自身的数据类型进行描述, 比如 1.2f32 代表 FLOAT 类型的数值 1.2, 如果不带类型后缀会被当作 DOUBLE 处理
|
||||
- timestamp 支持多种时间精度。写入数据的时候需要用参数指定时间精度，支持从小时到纳秒的 6 种时间精度
|
||||
- 为了提高写入的效率，默认假设同一个超级表中 field_set 的顺序是一样的（第一条数据包含所有的 field，后面的数据按照这个顺序），如果顺序不一样，需要配置参数 smlDataFormat 为 false，否则，数据写入按照相同顺序写入，库中数据会异常。（3.0.1.3 之后的版本 smlDataFormat 默认为 false，从3.0.3.0开始，该配置废弃） [TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议)
|
||||
- 默认产生的子表名是根据规则生成的唯一 ID 值。用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议)
|
||||
:::
|
||||
- 由于sql建表表名不支持点号（.），所以schemaless也对点号（.）做了处理，自动替换为下划线（_）
|
||||
::
|
||||
|
||||
要了解更多可参考：[InfluxDB Line 协议官方文档](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) 和 [TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议)
|
||||
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3
|
|||
```
|
||||
|
||||
- 默认生产的子表名是根据规则生成的唯一 ID 值。用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 meters.current 1648432611250 11.3 tname=cpu1 location=California.LosAngeles groupid=3 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。
|
||||
- 由于sql建表表名不支持点号（.），所以schemaless也对点号（.）做了处理，自动替换为下划线（_）。
|
||||
参考 [OpenTSDB Telnet API 文档](http://opentsdb.net/docs/build/html/api_telnet/put.html)。
|
||||
|
||||
## 示例代码
|
||||
|
|
@ -67,11 +68,11 @@ Database changed.
|
|||
taos> SHOW STABLES;
|
||||
name |
|
||||
=================================
|
||||
meters.current |
|
||||
meters.voltage |
|
||||
meters_current |
|
||||
meters_voltage |
|
||||
Query OK, 2 row(s) in set (0.002544s)
|
||||
|
||||
taos> SELECT TBNAME, * FROM `meters.current`;
|
||||
taos> SELECT TBNAME, * FROM `meters_current`;
|
||||
tbname | _ts | _value | groupid | location |
|
||||
==================================================================================================================================
|
||||
t_0e7bcfa21a02331c06764f275... | 2022-03-28 09:56:51.249 | 10.800000000 | 3 | California.LosAngeles |
|
||||
|
|
@ -83,10 +84,10 @@ Query OK, 4 row(s) in set (0.005399s)
|
|||
|
||||
## SQL 查询示例
|
||||
|
||||
`meters.current` 是插入数据的超级表名。
|
||||
`meters_current` 是插入数据的超级表名。
|
||||
|
||||
可以通过超级表的 TAG 来过滤数据，比如查询 `location=California.LosAngeles groupid=3` 可以通过如下 SQL：
|
||||
|
||||
```sql
|
||||
SELECT * FROM `meters.current` WHERE location = "California.LosAngeles" AND groupid = 3;
|
||||
SELECT * FROM `meters_current` WHERE location = "California.LosAngeles" AND groupid = 3;
|
||||
```
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ OpenTSDB JSON 格式协议采用一个 JSON 字符串表示一行或多行数据
|
|||
|
||||
- 对于 JSON 格式协议，TDengine 并不会自动把所有标签转成 NCHAR 类型， 字符串将将转为 NCHAR 类型， 数值将同样转换为 DOUBLE 类型。
|
||||
- 默认生成的子表名是根据规则生成的唯一 ID 值。用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 `"tags": { "host": "web02","dc": "lga","tname":"cpu1"}` 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。
|
||||
- 由于sql建表表名不支持点号（.），所以schemaless也对点号（.）做了处理，自动替换为下划线（_）。
|
||||
:::
|
||||
|
||||
## 示例代码
|
||||
|
|
@ -82,8 +83,8 @@ Database changed.
|
|||
taos> SHOW STABLES;
|
||||
name |
|
||||
=================================
|
||||
meters.current |
|
||||
meters.voltage |
|
||||
meters_current |
|
||||
meters_voltage |
|
||||
Query OK, 2 row(s) in set (0.001954s)
|
||||
|
||||
taos> SELECT * FROM `meters.current`;
|
||||
|
|
@ -96,10 +97,10 @@ Query OK, 2 row(s) in set (0.004076s)
|
|||
|
||||
## SQL 查询示例
|
||||
|
||||
`meters.voltage` 是插入数据的超级表名。
|
||||
`meters_voltage` 是插入数据的超级表名。
|
||||
|
||||
可以通过超级表的 TAG 来过滤数据，比如查询 `location=California.LosAngeles groupid=1` 可以通过如下 SQL：
|
||||
|
||||
```sql
|
||||
SELECT * FROM `meters.current` WHERE location = "California.LosAngeles" AND groupid = 3;
|
||||
SELECT * FROM `meters_current` WHERE location = "California.LosAngeles" AND groupid = 3;
|
||||
```
|
||||
|
|
|
|||
|
|
@ -25,7 +25,25 @@ import CDemo from "./_sub_c.mdx";
|
|||
|
||||
本文档不对消息队列本身的基础知识做介绍，如果需要了解，请自行搜索。
|
||||
|
||||
注意：数据订阅是从 WAL 消费数据，如果一些 WAL 文件被基于 WAL 保留策略删除，则已经删除的 WAL 文件中的数据就无法再消费到。需要根据业务需要在创建数据库时合理设置 `WAL_RETENTION_PERIOD` 或 `WAL_RETENTION_SIZE` ，并确保应用及时消费数据，这样才不会产生数据丢失的现象。数据订阅的行为与 Kafka 等广泛使用的消息队列类产品的行为相似。
|
||||
说明（以c接口为例）：
|
||||
- 一个消费组消费同一个topic下的所有数据，不同消费组之间相互独立；
|
||||
- 一个消费组消费同一个topic所有的vgroup，消费组可由多个消费者组成，但一个vgroup仅被一个消费者消费，如果消费者数量超过了vgroup数量，多余的消费者不消费数据；
|
||||
- 在服务端每个vgroup仅保存一个offset，每个vgroup的offset是单调递增的，但不一定连续。各个vgroup的offset之间没有关联；
|
||||
- 每次poll服务端会返回一个结果block，该block属于一个vgroup，可能包含多个wal版本的数据，可以通过 tmq_get_vgroup_offset 接口获得是该block第一条记录的offset；
|
||||
- 一个消费组如果从未commit过offset，当其成员消费者重启重新拉取数据时，均从参数auto.offset.reset设定值开始消费；在一个消费者生命周期中，客户端本地记录了最近一次拉取数据的offset，不会拉取重复数据；
|
||||
- 消费者如果异常终止（没有调用tmq_close），需等约12秒后触发其所属消费组rebalance，该消费者在服务端状态变为LOST，约1天后该消费者自动被删除；正常退出，退出后就会删除消费者；新增消费者，需等约2秒触发rebalance，该消费者在服务端状态变为ready；
|
||||
- 消费组rebalance会对该组所有ready状态的消费者成员重新进行vgroup分配，消费者仅能对自己负责的vgroup进行assignment/seek/commit/poll操作；
|
||||
- 消费者可利用 tmq_position 获得当前消费的offset，并seek到指定offset，重新消费；
|
||||
- seek将position指向指定offset，不执行commit操作，一旦seek成功，可poll拉取指定offset及以后的数据；
|
||||
- seek 操作之前须调用 tmq_get_topic_assignment 接口获取该consumer的vgroup ID和offset范围。seek 操作会检测vgroup ID 和 offset是否合法，如非法将报错；
|
||||
- position是获取当前的消费位置，是下次要取的位置，不是当前消费到的位置
|
||||
- commit是提交消费位置，不带参数的话，是提交当前消费位置（下次要取的位置，不是当前消费到的位置），带参数的话，是提交参数里的位置（也即下次退出重启后要取的位置）
|
||||
- seek是设置consumer消费位置，seek到哪，position就返回哪，都是下次要取的位置
|
||||
- seek不会影响commit，commit不影响seek，相互独立，两个是不同的概念
|
||||
- begin接口为wal 第一条数据的offset，end 接口为wal 最后一条数据的offset + 1
|
||||
- tmq_get_vgroup_offset接口获取的是记录所在结果block块里的第一条数据的offset，当seek至该offset时，将消费到这个block里的全部数据。参见第四点；
|
||||
- 由于存在 WAL 过期删除机制，即使seek 操作成功，poll数据时有可能offset已失效。如果poll 的offset 小于 WAL 最小版本号，将会从WAL最小版本号消费；
|
||||
- 数据订阅是从 WAL 消费数据，如果一些 WAL 文件被基于 WAL 保留策略删除，则已经删除的 WAL 文件中的数据就无法再消费到。需要根据业务需要在创建数据库时合理设置 `WAL_RETENTION_PERIOD` 或 `WAL_RETENTION_SIZE` ，并确保应用及时消费数据，这样才不会产生数据丢失的现象。数据订阅的行为与 Kafka 等广泛使用的消息队列类产品的行为相似；
|
||||
|
||||
## 主要数据结构和 API
|
||||
|
||||
|
|
@ -35,36 +53,57 @@ import CDemo from "./_sub_c.mdx";
|
|||
<TabItem value="c" label="C">
|
||||
|
||||
```c
|
||||
typedef struct tmq_t tmq_t;
|
||||
typedef struct tmq_conf_t tmq_conf_t;
|
||||
typedef struct tmq_list_t tmq_list_t;
|
||||
typedef struct tmq_t tmq_t;
|
||||
typedef struct tmq_conf_t tmq_conf_t;
|
||||
typedef struct tmq_list_t tmq_list_t;
|
||||
|
||||
typedef void(tmq_commit_cb(tmq_t *, int32_t code, void *param));
|
||||
typedef void(tmq_commit_cb(tmq_t *tmq, int32_t code, void *param));
|
||||
|
||||
DLL_EXPORT tmq_list_t *tmq_list_new();
|
||||
DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *);
|
||||
DLL_EXPORT void tmq_list_destroy(tmq_list_t *);
|
||||
DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
|
||||
DLL_EXPORT const char *tmq_err2str(int32_t code);
|
||||
typedef enum tmq_conf_res_t {
|
||||
TMQ_CONF_UNKNOWN = -2,
|
||||
TMQ_CONF_INVALID = -1,
|
||||
TMQ_CONF_OK = 0,
|
||||
} tmq_conf_res_t;
|
||||
|
||||
DLL_EXPORT int32_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list);
|
||||
DLL_EXPORT int32_t tmq_unsubscribe(tmq_t *tmq);
|
||||
DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout);
|
||||
DLL_EXPORT int32_t tmq_consumer_close(tmq_t *tmq);
|
||||
DLL_EXPORT int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg);
|
||||
DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const TAOS_RES *msg, tmq_commit_cb *cb, void *param);
|
||||
typedef struct tmq_topic_assignment {
|
||||
int32_t vgId;
|
||||
int64_t currentOffset;
|
||||
int64_t begin;
|
||||
int64_t end; // The last version of wal + 1
|
||||
} tmq_topic_assignment;
|
||||
|
||||
enum tmq_conf_res_t {
|
||||
TMQ_CONF_UNKNOWN = -2,
|
||||
TMQ_CONF_INVALID = -1,
|
||||
TMQ_CONF_OK = 0,
|
||||
};
|
||||
typedef enum tmq_conf_res_t tmq_conf_res_t;
|
||||
DLL_EXPORT tmq_conf_t *tmq_conf_new();
|
||||
DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value);
|
||||
DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf);
|
||||
DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param);
|
||||
|
||||
DLL_EXPORT tmq_conf_t *tmq_conf_new();
|
||||
DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value);
|
||||
DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf);
|
||||
DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param);
|
||||
DLL_EXPORT tmq_list_t *tmq_list_new();
|
||||
DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *);
|
||||
DLL_EXPORT void tmq_list_destroy(tmq_list_t *);
|
||||
DLL_EXPORT int32_t tmq_list_get_size(const tmq_list_t *);
|
||||
DLL_EXPORT char **tmq_list_to_c_array(const tmq_list_t *);
|
||||
|
||||
DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
|
||||
DLL_EXPORT int32_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list);
|
||||
DLL_EXPORT int32_t tmq_unsubscribe(tmq_t *tmq);
|
||||
DLL_EXPORT int32_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
|
||||
DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout);
|
||||
DLL_EXPORT int32_t tmq_consumer_close(tmq_t *tmq);
|
||||
DLL_EXPORT int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg); //Commit the msg’s offset + 1
|
||||
DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const TAOS_RES *msg, tmq_commit_cb *cb, void *param);
|
||||
DLL_EXPORT int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset);
|
||||
DLL_EXPORT void tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb *cb, void *param);
|
||||
DLL_EXPORT int32_t tmq_get_topic_assignment(tmq_t *tmq, const char *pTopicName, tmq_topic_assignment **assignment,int32_t *numOfAssignment);
|
||||
DLL_EXPORT void tmq_free_assignment(tmq_topic_assignment* pAssignment);
|
||||
DLL_EXPORT int32_t tmq_offset_seek(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset);
|
||||
DLL_EXPORT int64_t tmq_position(tmq_t *tmq, const char *pTopicName, int32_t vgId); // The current offset is the offset of the last consumed message + 1
|
||||
DLL_EXPORT int64_t tmq_committed(tmq_t *tmq, const char *pTopicName, int32_t vgId);
|
||||
|
||||
DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res);
|
||||
DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res);
|
||||
DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res);
|
||||
DLL_EXPORT int64_t tmq_get_vgroup_offset(TAOS_RES* res); // Get current offset of the result
|
||||
DLL_EXPORT const char *tmq_err2str(int32_t code);
|
||||
```
|
||||
|
||||
这些 API 的文档请见 [C/C++ Connector](../../connector/cpp)，下面介绍一下它们的具体用法（超级表和子表结构请参考“数据建模”一节），完整的示例代码请见下面 C 语言的示例代码。
|
||||
|
|
|
|||
|
|
@ -467,21 +467,22 @@ TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多
|
|||
- `TAOS_RES* taos_schemaless_insert(TAOS* taos, const char* lines[], int numLines, int protocol, int precision)`
|
||||
|
||||
**功能说明**
|
||||
该接口将行协议的文本数据写入到 TDengine 中。
|
||||
- 该接口将行协议的文本数据写入到 TDengine 中。
|
||||
|
||||
**参数说明**
|
||||
taos: 数据库连接，通过 `taos_connect()` 函数建立的数据库连接。
|
||||
lines：文本数据。满足解析格式要求的无模式文本字符串。
|
||||
numLines:文本数据的行数，不能为 0 。
|
||||
protocol: 行协议类型，用于标识文本数据格式。
|
||||
precision：文本数据中的时间戳精度字符串。
|
||||
- taos: 数据库连接，通过 `taos_connect()` 函数建立的数据库连接。
|
||||
- lines：文本数据。满足解析格式要求的无模式文本字符串。
|
||||
- numLines:文本数据的行数，不能为 0 。
|
||||
- protocol: 行协议类型，用于标识文本数据格式。
|
||||
- precision：文本数据中的时间戳精度字符串。
|
||||
|
||||
**返回值**
|
||||
TAOS_RES 结构体，应用可以通过使用 `taos_errstr()` 获得错误信息，也可以使用 `taos_errno()` 获得错误码。
|
||||
- TAOS_RES 结构体，应用可以通过使用 `taos_errstr()` 获得错误信息，也可以使用 `taos_errno()` 获得错误码。
|
||||
在某些情况下，返回的 TAOS_RES 为 `NULL`，此时仍然可以调用 `taos_errno()` 来安全地获得错误码信息。
|
||||
返回的 TAOS_RES 需要调用方来负责释放，否则会出现内存泄漏。
|
||||
|
||||
**说明**
|
||||
|
||||
协议类型是枚举类型，包含以下三种格式：
|
||||
|
||||
- TSDB_SML_LINE_PROTOCOL：InfluxDB 行协议（Line Protocol)
|
||||
|
|
@ -515,3 +516,90 @@ TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多
|
|||
- 带_raw的接口通过传递的参数lines指针和长度len来表示数据，为了解决原始接口数据包含'\0'而被截断的问题。totalRows指针返回解析出来的数据行数。
|
||||
- 带_ttl的接口可以传递ttl参数来控制建表的ttl到期时间。
|
||||
- 带_reqid的接口可以通过传递reqid参数来追踪整个的调用链。
|
||||
|
||||
### 数据订阅 API
|
||||
|
||||
- `int32_t tmq_get_topic_assignment(tmq_t *tmq, const char *pTopicName, tmq_topic_assignment **assignment, int32_t *numOfAssignment)`
|
||||
- `void tmq_free_assignment(tmq_topic_assignment* pAssignment)`
|
||||
|
||||
tmq_topic_assignment结构体定义如下：
|
||||
```c
|
||||
typedef struct tmq_topic_assignment {
|
||||
int32_t vgId;
|
||||
int64_t currentOffset;
|
||||
int64_t begin;
|
||||
int64_t end;
|
||||
} tmq_topic_assignment;
|
||||
```
|
||||
**功能说明**
|
||||
- tmq_get_topic_assignment 接口返回当前consumer分配的vgroup的信息，每个vgroup的信息包括vgId，wal的最大最小offset，以及当前消费到的offset。
|
||||
|
||||
**参数说明**
|
||||
- numOfAssignment ：分配给该consumer有效的vgroup个数。
|
||||
- assignment ：分配的信息，数据大小为numOfAssignment，需要通过 tmq_free_assignment 接口释放。
|
||||
|
||||
**返回值**
|
||||
- 错误码，0成功，非0失败，可通过 `char *tmq_err2str(int32_t code)` 函数获取错误信息。
|
||||
|
||||
- `int64_t tmq_committed(tmq_t *tmq, const char *pTopicName, int32_t vgId)`
|
||||
|
||||
**功能说明**
|
||||
- 获取当前 consumer 在某个 topic 和 vgroup上的 commit 位置。
|
||||
|
||||
**返回值**
|
||||
- 当前commit的位置，-2147467247表示没有消费进度，其他小于0的值表示失败，错误码就是返回值
|
||||
|
||||
- `int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg)`
|
||||
- `void tmq_commit_async(tmq_t *tmq, const TAOS_RES *msg, tmq_commit_cb *cb, void *param)`
|
||||
- `int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset)`
|
||||
- `void tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb *cb, void *param)`
|
||||
|
||||
**功能说明**
|
||||
|
||||
- commit接口分为两种类型，每种类型有同步和异步接口：
|
||||
- 第一种类型：根据消息提交，提交消息里的进度，如果消息传NULL，提交当前consumer所有消费的vgroup的当前进度 : tmq_commit_sync/tmq_commit_async
|
||||
- 第二种类型：根据某个topic的某个vgroup的offset提交 : tmq_commit_offset_sync/tmq_commit_offset_async
|
||||
|
||||
**参数说明**
|
||||
- msg：消费到的消息结构，如果msg传NULL，提交当前consumer所有消费的vgroup的当前进度
|
||||
|
||||
**返回值**
|
||||
- 错误码，0成功，非0失败，可通过 `char *tmq_err2str(int32_t code)` 函数获取错误信息
|
||||
|
||||
- `int64_t tmq_position(tmq_t *tmq, const char *pTopicName, int32_t vgId)`
|
||||
|
||||
**功能说明**
|
||||
- 获取当前消费位置，为消费到的数据位置的下一个位置
|
||||
|
||||
**返回值**
|
||||
- 消费位置，非0失败，可通过 `char *tmq_err2str(int32_t code)` 函数获取错误信息
|
||||
|
||||
- `int32_t tmq_offset_seek(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset)`
|
||||
|
||||
**功能说明**
|
||||
- 设置 consumer 在某个topic的某个vgroup的 offset位置，开始消费
|
||||
|
||||
**返回值**
|
||||
- 错误码，0成功，非0失败，可通过 `char *tmq_err2str(int32_t code)` 函数获取错误信息
|
||||
|
||||
- `int32_t int64_t tmq_get_vgroup_offset(TAOS_RES* res)`
|
||||
|
||||
**功能说明**
|
||||
- 获取 poll 消费到的数据的起始offset
|
||||
|
||||
**参数说明**
|
||||
- msg：消费到的消息结构
|
||||
|
||||
**返回值**
|
||||
- 消费到的offset，非0失败，可通过 `char *tmq_err2str(int32_t code)` 函数获取错误信息
|
||||
|
||||
- `int32_t int32_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics)`
|
||||
|
||||
**功能说明**
|
||||
- 获取消费者订阅的 topic 列表
|
||||
|
||||
**参数说明**
|
||||
- topics: 获取的 topic 列表存储在这个结构中，接口内分配内存，需调用tmq_list_destroy释放
|
||||
|
||||
**返回值**
|
||||
- 错误码，0成功，非0失败，可通过 `char *tmq_err2str(int32_t code)` 函数获取错误信息
|
||||
|
|
@ -4,7 +4,7 @@ title: 权限管理
|
|||
description: 企业版中才具有的权限管理功能
|
||||
---
|
||||
|
||||
本节讲述如何在 TDengine 中进行权限管理的相关操作。
|
||||
本节讲述如何在 TDengine 中进行权限管理的相关操作。权限管理是 TDengine 企业版的特有功能，本节只列举了一些基本的权限管理功能作为示例，更丰富的权限管理请联系 TDengine 销售或市场团队。
|
||||
|
||||
## 创建用户
|
||||
|
||||
|
|
|
|||
|
|
@ -33,8 +33,8 @@ tag_set 中的所有的数据自动转化为 nchar 数据类型，并不需要
|
|||
|
||||
在无模式写入数据行协议中，field_set 中的每个数据项都需要对自身的数据类型进行描述。具体来说：
|
||||
|
||||
- 如果两边有英文双引号，表示 BINARY(32) 类型。例如 `"abc"`。
|
||||
- 如果两边有英文双引号而且带有 L 前缀，表示 NCHAR(32) 类型。例如 `L"报错信息"`。
|
||||
- 如果两边有英文双引号，表示 VARCHAR(N) 类型。例如 `"abc"`。
|
||||
- 如果两边有英文双引号而且带有 L 前缀，表示 NCHAR(N) 类型。例如 `L"报错信息"`。
|
||||
- 对空格、等号（=）、逗号（,）、双引号（"）、反斜杠（\），前面需要使用反斜杠（\）进行转义。（都指的是英文半角符号）。具体转义规则如下：
|
||||
|
||||
| **序号** | **域** | **需转义字符** |
|
||||
|
|
|
|||
|
|
@ -228,11 +228,6 @@ tmq_t* build_consumer() {
|
|||
tmq_conf_destroy(conf);
|
||||
return NULL;
|
||||
}
|
||||
code = tmq_conf_set(conf, "experimental.snapshot.enable", "false");
|
||||
if (TMQ_CONF_OK != code) {
|
||||
tmq_conf_destroy(conf);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
|
||||
tmq = tmq_consumer_new(conf, NULL, 0);
|
||||
|
|
|
|||
|
|
@ -260,7 +260,25 @@ typedef struct tmq_t tmq_t;
|
|||
typedef struct tmq_conf_t tmq_conf_t;
|
||||
typedef struct tmq_list_t tmq_list_t;
|
||||
|
||||
typedef void(tmq_commit_cb(tmq_t *, int32_t code, void *param));
|
||||
typedef void(tmq_commit_cb(tmq_t *tmq, int32_t code, void *param));
|
||||
|
||||
typedef enum tmq_conf_res_t {
|
||||
TMQ_CONF_UNKNOWN = -2,
|
||||
TMQ_CONF_INVALID = -1,
|
||||
TMQ_CONF_OK = 0,
|
||||
} tmq_conf_res_t;
|
||||
|
||||
typedef struct tmq_topic_assignment {
|
||||
int32_t vgId;
|
||||
int64_t currentOffset;
|
||||
int64_t begin;
|
||||
int64_t end; // The last invalidate version of wal + 1
|
||||
} tmq_topic_assignment;
|
||||
|
||||
DLL_EXPORT tmq_conf_t *tmq_conf_new();
|
||||
DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value);
|
||||
DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf);
|
||||
DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param);
|
||||
|
||||
DLL_EXPORT tmq_list_t *tmq_list_new();
|
||||
DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *);
|
||||
|
|
@ -268,55 +286,27 @@ DLL_EXPORT void tmq_list_destroy(tmq_list_t *);
|
|||
DLL_EXPORT int32_t tmq_list_get_size(const tmq_list_t *);
|
||||
DLL_EXPORT char **tmq_list_to_c_array(const tmq_list_t *);
|
||||
|
||||
DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
|
||||
|
||||
DLL_EXPORT const char *tmq_err2str(int32_t code);
|
||||
|
||||
/* ------------------------TMQ CONSUMER INTERFACE------------------------ */
|
||||
typedef struct tmq_topic_assignment {
|
||||
int32_t vgId;
|
||||
int64_t currentOffset;
|
||||
int64_t begin;
|
||||
int64_t end;
|
||||
} tmq_topic_assignment;
|
||||
|
||||
DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
|
||||
DLL_EXPORT int32_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list);
|
||||
DLL_EXPORT int32_t tmq_unsubscribe(tmq_t *tmq);
|
||||
DLL_EXPORT int32_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
|
||||
DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout);
|
||||
DLL_EXPORT int32_t tmq_consumer_close(tmq_t *tmq);
|
||||
DLL_EXPORT int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg);
|
||||
DLL_EXPORT int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg); //Commit the msg’s offset + 1
|
||||
DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const TAOS_RES *msg, tmq_commit_cb *cb, void *param);
|
||||
DLL_EXPORT int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset);
|
||||
DLL_EXPORT void tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb *cb, void *param);
|
||||
DLL_EXPORT int32_t tmq_get_topic_assignment(tmq_t *tmq, const char *pTopicName, tmq_topic_assignment **assignment,
|
||||
int32_t *numOfAssignment);
|
||||
DLL_EXPORT int32_t tmq_get_topic_assignment(tmq_t *tmq, const char *pTopicName, tmq_topic_assignment **assignment,int32_t *numOfAssignment);
|
||||
DLL_EXPORT void tmq_free_assignment(tmq_topic_assignment* pAssignment);
|
||||
DLL_EXPORT int32_t tmq_offset_seek(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset);
|
||||
DLL_EXPORT int64_t tmq_position(tmq_t *tmq, const char *pTopicName, int32_t vgId); // The current offset is the offset of the last consumed message + 1
|
||||
DLL_EXPORT int64_t tmq_committed(tmq_t *tmq, const char *pTopicName, int32_t vgId);
|
||||
|
||||
DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res);
|
||||
DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res);
|
||||
DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res);
|
||||
DLL_EXPORT int64_t tmq_get_vgroup_offset(TAOS_RES* res);
|
||||
DLL_EXPORT int64_t tmq_position(tmq_t *tmq, const char *pTopicName, int32_t vgId);
|
||||
DLL_EXPORT int64_t tmq_committed(tmq_t *tmq, const char *pTopicName, int32_t vgId);
|
||||
|
||||
/* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */
|
||||
|
||||
enum tmq_conf_res_t {
|
||||
TMQ_CONF_UNKNOWN = -2,
|
||||
TMQ_CONF_INVALID = -1,
|
||||
TMQ_CONF_OK = 0,
|
||||
};
|
||||
|
||||
typedef enum tmq_conf_res_t tmq_conf_res_t;
|
||||
|
||||
DLL_EXPORT tmq_conf_t *tmq_conf_new();
|
||||
DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value);
|
||||
DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf);
|
||||
DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param);
|
||||
|
||||
/* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */
|
||||
DLL_EXPORT int64_t tmq_get_vgroup_offset(TAOS_RES* res); // Get current offset of the result
|
||||
DLL_EXPORT const char *tmq_err2str(int32_t code);
|
||||
|
||||
/* ------------------------------ TAOSX -----------------------------------*/
|
||||
// note: following apis are unstable
|
||||
|
|
|
|||
|
|
@ -199,6 +199,7 @@ extern bool tsFilterScalarMode;
|
|||
extern int32_t tsKeepTimeOffset;
|
||||
extern int32_t tsMaxStreamBackendCache;
|
||||
extern int32_t tsPQSortMemThreshold;
|
||||
extern int32_t tsResolveFQDNRetryTime;
|
||||
|
||||
// #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
|
||||
|
||||
|
|
|
|||
|
|
@ -153,7 +153,6 @@ struct SWalReader {
|
|||
int64_t capacity;
|
||||
TdThreadMutex mutex;
|
||||
SWalFilterCond cond;
|
||||
// TODO remove it
|
||||
SWalCkHead *pHead;
|
||||
};
|
||||
|
||||
|
|
@ -207,10 +206,9 @@ void walReaderValidVersionRange(SWalReader *pReader, int64_t *sver, int64
|
|||
void walReaderVerifyOffset(SWalReader *pWalReader, STqOffsetVal* pOffset);
|
||||
|
||||
// only for tq usage
|
||||
void walSetReaderCapacity(SWalReader *pRead, int32_t capacity);
|
||||
int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead);
|
||||
int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead);
|
||||
int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead);
|
||||
int32_t walFetchHead(SWalReader *pRead, int64_t ver);
|
||||
int32_t walFetchBody(SWalReader *pRead);
|
||||
int32_t walSkipFetchBody(SWalReader *pRead);
|
||||
|
||||
void walRefFirstVer(SWal *, SWalRef *);
|
||||
void walRefLastVer(SWal *, SWalRef *);
|
||||
|
|
|
|||
|
|
@ -236,6 +236,7 @@ int64_t tsStreamBufferSize = 128 * 1024 * 1024;
|
|||
int64_t tsCheckpointInterval = 3 * 60 * 60 * 1000;
|
||||
bool tsFilterScalarMode = false;
|
||||
int32_t tsKeepTimeOffset = 0; // latency of data migration
|
||||
int tsResolveFQDNRetryTime = 100; //seconds
|
||||
|
||||
#ifndef _STORAGE
|
||||
int32_t taosSetTfsCfg(SConfig *pCfg) {
|
||||
|
|
@ -580,6 +581,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
|
|||
if (cfgAddInt32(pCfg, "keepTimeOffset", tsKeepTimeOffset, 0, 23, CFG_SCOPE_SERVER) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "resolveFQDNRetryTime", tsResolveFQDNRetryTime, 1, 10240, 0) != 0) return -1;
|
||||
|
||||
GRANT_CFG_ADD;
|
||||
return 0;
|
||||
|
|
@ -979,6 +981,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
|
|||
tsKeepTimeOffset = cfgGetItem(pCfg, "keepTimeOffset")->i32;
|
||||
tsMaxStreamBackendCache = cfgGetItem(pCfg, "maxStreamBackendCache")->i32;
|
||||
tsPQSortMemThreshold = cfgGetItem(pCfg, "pqSortMemThreshold")->i32;
|
||||
tsResolveFQDNRetryTime = cfgGetItem(pCfg, "resolveFQDNRetryTime")->i32;
|
||||
|
||||
GRANT_CFG_GET;
|
||||
return 0;
|
||||
|
|
|
|||
|
|
@ -127,7 +127,7 @@ void tqDestroyTqHandle(void* data);
|
|||
// tqRead
|
||||
int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset);
|
||||
int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset);
|
||||
int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum, uint64_t reqId);
|
||||
int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t reqId);
|
||||
|
||||
// tqExec
|
||||
int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows);
|
||||
|
|
|
|||
|
|
@ -184,70 +184,63 @@ end:
|
|||
return tbSuid == realTbSuid;
|
||||
}
|
||||
|
||||
int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** ppCkHead, uint64_t reqId) {
|
||||
int32_t code = 0;
|
||||
int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t reqId) {
|
||||
int32_t code = -1;
|
||||
int32_t vgId = TD_VID(pTq->pVnode);
|
||||
|
||||
taosThreadMutexLock(&pHandle->pWalReader->mutex);
|
||||
int64_t offset = *fetchOffset;
|
||||
int64_t lastVer = walGetLastVer(pHandle->pWalReader->pWal);
|
||||
int64_t committedVer = walGetCommittedVer(pHandle->pWalReader->pWal);
|
||||
int64_t appliedVer = walGetAppliedVer(pHandle->pWalReader->pWal);
|
||||
|
||||
while (1) {
|
||||
if (walFetchHead(pHandle->pWalReader, offset, *ppCkHead) < 0) {
|
||||
wDebug("vgId:%d, wal start to fetch, index:%" PRId64 ", last index:%" PRId64 " commit index:%" PRId64 ", applied index:%" PRId64,
|
||||
vgId, offset, lastVer, committedVer, appliedVer);
|
||||
|
||||
while (offset <= appliedVer) {
|
||||
if (walFetchHead(pHandle->pWalReader, offset) < 0) {
|
||||
tqDebug("tmq poll: consumer:0x%" PRIx64 ", (epoch %d) vgId:%d offset %" PRId64
|
||||
", no more log to return, reqId:0x%" PRIx64,
|
||||
pHandle->consumerId, pHandle->epoch, vgId, offset, reqId);
|
||||
*fetchOffset = offset;
|
||||
code = -1;
|
||||
goto END;
|
||||
}
|
||||
|
||||
tqDebug("vgId:%d, consumer:0x%" PRIx64 " taosx get msg ver %" PRId64 ", type: %s, reqId:0x%" PRIx64, vgId,
|
||||
pHandle->consumerId, offset, TMSG_INFO((*ppCkHead)->head.msgType), reqId);
|
||||
pHandle->consumerId, offset, TMSG_INFO(pHandle->pWalReader->pHead->head.msgType), reqId);
|
||||
|
||||
if ((*ppCkHead)->head.msgType == TDMT_VND_SUBMIT) {
|
||||
code = walFetchBody(pHandle->pWalReader, ppCkHead);
|
||||
|
||||
if (code < 0) {
|
||||
*fetchOffset = offset;
|
||||
code = -1;
|
||||
goto END;
|
||||
}
|
||||
*fetchOffset = offset;
|
||||
code = 0;
|
||||
if (pHandle->pWalReader->pHead->head.msgType == TDMT_VND_SUBMIT) {
|
||||
code = walFetchBody(pHandle->pWalReader);
|
||||
goto END;
|
||||
} else {
|
||||
if (pHandle->fetchMeta != WITH_DATA) {
|
||||
SWalCont* pHead = &((*ppCkHead)->head);
|
||||
SWalCont* pHead = &(pHandle->pWalReader->pHead->head);
|
||||
if (IS_META_MSG(pHead->msgType) && !(pHead->msgType == TDMT_VND_DELETE && pHandle->fetchMeta == ONLY_META)) {
|
||||
code = walFetchBody(pHandle->pWalReader, ppCkHead);
|
||||
code = walFetchBody(pHandle->pWalReader);
|
||||
if (code < 0) {
|
||||
*fetchOffset = offset;
|
||||
code = -1;
|
||||
goto END;
|
||||
}
|
||||
|
||||
pHead = &(pHandle->pWalReader->pHead->head);
|
||||
if (isValValidForTable(pHandle, pHead)) {
|
||||
*fetchOffset = offset;
|
||||
code = 0;
|
||||
goto END;
|
||||
} else {
|
||||
offset++;
|
||||
code = -1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
code = walSkipFetchBody(pHandle->pWalReader, *ppCkHead);
|
||||
code = walSkipFetchBody(pHandle->pWalReader);
|
||||
if (code < 0) {
|
||||
*fetchOffset = offset;
|
||||
code = -1;
|
||||
goto END;
|
||||
}
|
||||
offset++;
|
||||
}
|
||||
code = -1;
|
||||
}
|
||||
|
||||
END:
|
||||
taosThreadMutexUnlock(&pHandle->pWalReader->mutex);
|
||||
*fetchOffset = offset;
|
||||
return code;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -179,7 +179,6 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
|
|||
SRpcMsg* pMsg, STqOffsetVal* offset) {
|
||||
int code = 0;
|
||||
int32_t vgId = TD_VID(pTq->pVnode);
|
||||
SWalCkHead* pCkHead = NULL;
|
||||
SMqMetaRsp metaRsp = {0};
|
||||
STaosxRsp taosxRsp = {0};
|
||||
tqInitTaosxRsp(&taosxRsp, *offset);
|
||||
|
|
@ -216,14 +215,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
|
|||
if (offset->type == TMQ_OFFSET__LOG) {
|
||||
walReaderVerifyOffset(pHandle->pWalReader, offset);
|
||||
int64_t fetchVer = offset->version;
|
||||
pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048);
|
||||
if (pCkHead == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
code = -1;
|
||||
goto end;
|
||||
}
|
||||
|
||||
walSetReaderCapacity(pHandle->pWalReader, 2048);
|
||||
int totalRows = 0;
|
||||
while (1) {
|
||||
int32_t savedEpoch = atomic_load_32(&pHandle->epoch);
|
||||
|
|
@ -234,14 +226,14 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
|
|||
break;
|
||||
}
|
||||
|
||||
if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead, pRequest->reqId) < 0) {
|
||||
if (tqFetchLog(pTq, pHandle, &fetchVer, pRequest->reqId) < 0) {
|
||||
tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
|
||||
// setRequestVersion(&taosxRsp.reqOffset, offset->version);
|
||||
code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
|
||||
goto end;
|
||||
}
|
||||
|
||||
SWalCont* pHead = &pCkHead->head;
|
||||
SWalCont* pHead = &pHandle->pWalReader->pHead->head;
|
||||
tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d",
|
||||
pRequest->consumerId, pRequest->epoch, vgId, fetchVer, pHead->msgType);
|
||||
|
||||
|
|
@ -291,7 +283,6 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
|
|||
end:
|
||||
|
||||
tDeleteSTaosxRsp(&taosxRsp);
|
||||
taosMemoryFreeClear(pCkHead);
|
||||
return code;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@
|
|||
#include "syncRaftCfg.h"
|
||||
#include "syncRaftStore.h"
|
||||
#include "syncSnapshot.h"
|
||||
#include "tglobal.h"
|
||||
|
||||
void syncCfg2SimpleStr(const SSyncCfg* pCfg, char* buf, int32_t bufLen) {
|
||||
int32_t len = snprintf(buf, bufLen, "{num:%d, as:%d, [", pCfg->replicaNum, pCfg->myIndex);
|
||||
|
|
@ -41,7 +42,22 @@ void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet) {
|
|||
}
|
||||
|
||||
bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId) {
|
||||
uint32_t ipv4 = taosGetIpv4FromFqdn(pInfo->nodeFqdn);
|
||||
uint32_t ipv4 = 0xFFFFFFFF;
|
||||
sDebug("vgId:%d, start to resolve sync addr fqdn in %d seconds, "
|
||||
"dnode:%d cluster:%" PRId64 " fqdn:%s port:%u ",
|
||||
vgId, tsResolveFQDNRetryTime,
|
||||
pInfo->nodeId, pInfo->clusterId, pInfo->nodeFqdn, pInfo->nodePort);
|
||||
for(int i = 0; i < tsResolveFQDNRetryTime; i++){
|
||||
ipv4 = taosGetIpv4FromFqdn(pInfo->nodeFqdn);
|
||||
if (ipv4 == 0xFFFFFFFF || ipv4 == 1) {
|
||||
sError("failed to resolve ipv4 addr, fqdn:%s, wait one second", pInfo->nodeFqdn);
|
||||
taosSsleep(1);
|
||||
}
|
||||
else{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (ipv4 == 0xFFFFFFFF || ipv4 == 1) {
|
||||
sError("failed to resolve ipv4 addr, fqdn:%s", pInfo->nodeFqdn);
|
||||
terrno = TSDB_CODE_TSC_INVALID_FQDN;
|
||||
|
|
|
|||
|
|
@ -16,10 +16,6 @@
|
|||
#include "taoserror.h"
|
||||
#include "walInt.h"
|
||||
|
||||
static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer);
|
||||
static int32_t walFetchBodyNew(SWalReader *pRead);
|
||||
static int32_t walSkipFetchBodyNew(SWalReader *pRead);
|
||||
|
||||
SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond) {
|
||||
SWalReader *pReader = taosMemoryCalloc(1, sizeof(SWalReader));
|
||||
if (pReader == NULL) {
|
||||
|
|
@ -80,19 +76,19 @@ int32_t walNextValidMsg(SWalReader *pReader) {
|
|||
return -1;
|
||||
}
|
||||
while (fetchVer <= appliedVer) {
|
||||
if (walFetchHeadNew(pReader, fetchVer) < 0) {
|
||||
if (walFetchHead(pReader, fetchVer) < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
int32_t type = pReader->pHead->head.msgType;
|
||||
if (type == TDMT_VND_SUBMIT || ((type == TDMT_VND_DELETE) && (pReader->cond.deleteMsg == 1)) ||
|
||||
(IS_META_MSG(type) && pReader->cond.scanMeta)) {
|
||||
if (walFetchBodyNew(pReader) < 0) {
|
||||
if (walFetchBody(pReader) < 0) {
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
} else {
|
||||
if (walSkipFetchBodyNew(pReader) < 0) {
|
||||
if (walSkipFetchBody(pReader) < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -254,104 +250,8 @@ int32_t walReaderSeekVer(SWalReader *pReader, int64_t ver) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
void walSetReaderCapacity(SWalReader *pRead, int32_t capacity) { pRead->capacity = capacity; }
|
||||
|
||||
static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) {
|
||||
int64_t contLen;
|
||||
bool seeked = false;
|
||||
|
||||
wDebug("vgId:%d, wal starts to fetch head, index:%" PRId64, pRead->pWal->cfg.vgId, fetchVer);
|
||||
|
||||
if (pRead->curVersion != fetchVer) {
|
||||
if (walReaderSeekVer(pRead, fetchVer) < 0) {
|
||||
return -1;
|
||||
}
|
||||
seeked = true;
|
||||
}
|
||||
|
||||
while (1) {
|
||||
contLen = taosReadFile(pRead->pLogFile, pRead->pHead, sizeof(SWalCkHead));
|
||||
if (contLen == sizeof(SWalCkHead)) {
|
||||
break;
|
||||
} else if (contLen == 0 && !seeked) {
|
||||
if(walReadSeekVerImpl(pRead, fetchVer) < 0){
|
||||
return -1;
|
||||
}
|
||||
seeked = true;
|
||||
continue;
|
||||
} else {
|
||||
if (contLen < 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
} else {
|
||||
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
// pRead->curInvalid = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t walFetchBodyNew(SWalReader *pReader) {
|
||||
SWalCont *pReadHead = &pReader->pHead->head;
|
||||
int64_t ver = pReadHead->version;
|
||||
|
||||
wDebug("vgId:%d, wal starts to fetch body, ver:%" PRId64 " ,len:%d, total", pReader->pWal->cfg.vgId, ver,
|
||||
pReadHead->bodyLen);
|
||||
|
||||
if (pReader->capacity < pReadHead->bodyLen) {
|
||||
SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen);
|
||||
if (ptr == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
|
||||
pReader->pHead = ptr;
|
||||
pReadHead = &pReader->pHead->head;
|
||||
pReader->capacity = pReadHead->bodyLen;
|
||||
}
|
||||
|
||||
if (pReadHead->bodyLen != taosReadFile(pReader->pLogFile, pReadHead->body, pReadHead->bodyLen)) {
|
||||
if (pReadHead->bodyLen < 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s",
|
||||
pReader->pWal->cfg.vgId, pReader->pHead->head.version, ver, tstrerror(terrno));
|
||||
} else {
|
||||
wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since file corrupted",
|
||||
pReader->pWal->cfg.vgId, pReader->pHead->head.version, ver);
|
||||
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (walValidBodyCksum(pReader->pHead) != 0) {
|
||||
wError("vgId:%d, wal fetch body error:%" PRId64 ", since body checksum not passed", pReader->pWal->cfg.vgId, ver);
|
||||
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
|
||||
return -1;
|
||||
}
|
||||
|
||||
wDebug("vgId:%d, index:%" PRId64 " is fetched, type:%d, cursor advance", pReader->pWal->cfg.vgId, ver, pReader->pHead->head.msgType);
|
||||
pReader->curVersion = ver + 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t walSkipFetchBodyNew(SWalReader *pRead) {
|
||||
int64_t code;
|
||||
|
||||
code = taosLSeekFile(pRead->pLogFile, pRead->pHead->head.bodyLen, SEEK_CUR);
|
||||
if (code < 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
// pRead->curInvalid = 1;
|
||||
return -1;
|
||||
}
|
||||
|
||||
pRead->curVersion++;
|
||||
wDebug("vgId:%d, version advance to %" PRId64 ", skip fetch", pRead->pWal->cfg.vgId, pRead->curVersion);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) {
|
||||
int32_t walFetchHead(SWalReader *pRead, int64_t ver) {
|
||||
int64_t code;
|
||||
int64_t contLen;
|
||||
bool seeked = false;
|
||||
|
|
@ -369,15 +269,13 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) {
|
|||
if (pRead->curVersion != ver) {
|
||||
code = walReaderSeekVer(pRead, ver);
|
||||
if (code < 0) {
|
||||
// pRead->curVersion = ver;
|
||||
// pRead->curInvalid = 1;
|
||||
return -1;
|
||||
}
|
||||
seeked = true;
|
||||
}
|
||||
|
||||
while (1) {
|
||||
contLen = taosReadFile(pRead->pLogFile, pHead, sizeof(SWalCkHead));
|
||||
contLen = taosReadFile(pRead->pLogFile, pRead->pHead, sizeof(SWalCkHead));
|
||||
if (contLen == sizeof(SWalCkHead)) {
|
||||
break;
|
||||
} else if (contLen == 0 && !seeked) {
|
||||
|
|
@ -392,12 +290,11 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) {
|
|||
} else {
|
||||
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
|
||||
}
|
||||
// pRead->curInvalid = 1;
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
code = walValidHeadCksum(pHead);
|
||||
code = walValidHeadCksum(pRead->pHead);
|
||||
|
||||
if (code != 0) {
|
||||
wError("vgId:%d, unexpected wal log index:%" PRId64 ", since head checksum not passed", pRead->pWal->cfg.vgId, ver);
|
||||
|
|
@ -405,32 +302,27 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
// pRead->curInvalid = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead) {
|
||||
int64_t code;
|
||||
|
||||
int32_t walSkipFetchBody(SWalReader *pRead) {
|
||||
wDebug("vgId:%d, skip fetch body %" PRId64 ", first ver:%" PRId64 ", commit ver:%" PRId64 ", last ver:%" PRId64
|
||||
", applied ver:%" PRId64,
|
||||
pRead->pWal->cfg.vgId, pHead->head.version, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer,
|
||||
pRead->pWal->cfg.vgId, pRead->pHead->head.version, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer,
|
||||
pRead->pWal->vers.lastVer, pRead->pWal->vers.appliedVer);
|
||||
|
||||
code = taosLSeekFile(pRead->pLogFile, pHead->head.bodyLen, SEEK_CUR);
|
||||
int64_t code = taosLSeekFile(pRead->pLogFile, pRead->pHead->head.bodyLen, SEEK_CUR);
|
||||
if (code < 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
// pRead->curInvalid = 1;
|
||||
return -1;
|
||||
}
|
||||
|
||||
pRead->curVersion++;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) {
|
||||
SWalCont *pReadHead = &((*ppHead)->head);
|
||||
int32_t walFetchBody(SWalReader *pRead) {
|
||||
SWalCont *pReadHead = &pRead->pHead->head;
|
||||
int64_t ver = pReadHead->version;
|
||||
|
||||
wDebug("vgId:%d, fetch body %" PRId64 ", first ver:%" PRId64 ", commit ver:%" PRId64 ", last ver:%" PRId64
|
||||
|
|
@ -439,13 +331,13 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) {
|
|||
pRead->pWal->vers.appliedVer);
|
||||
|
||||
if (pRead->capacity < pReadHead->bodyLen) {
|
||||
SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(*ppHead, sizeof(SWalCkHead) + pReadHead->bodyLen);
|
||||
SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen);
|
||||
if (ptr == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
*ppHead = ptr;
|
||||
pReadHead = &((*ppHead)->head);
|
||||
pRead->pHead = ptr;
|
||||
pReadHead = &pRead->pHead->head;
|
||||
pRead->capacity = pReadHead->bodyLen;
|
||||
}
|
||||
|
||||
|
|
@ -459,27 +351,24 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) {
|
|||
pRead->pWal->cfg.vgId, pReadHead->version, ver);
|
||||
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
|
||||
}
|
||||
// pRead->curInvalid = 1;
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (pReadHead->version != ver) {
|
||||
wError("vgId:%d, wal fetch body error, index:%" PRId64 ", read request index:%" PRId64, pRead->pWal->cfg.vgId,
|
||||
pReadHead->version, ver);
|
||||
// pRead->curInvalid = 1;
|
||||
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (walValidBodyCksum(*ppHead) != 0) {
|
||||
if (walValidBodyCksum(pRead->pHead) != 0) {
|
||||
wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId,
|
||||
ver);
|
||||
// pRead->curInvalid = 1;
|
||||
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
|
||||
return -1;
|
||||
}
|
||||
|
||||
pRead->curVersion = ver + 1;
|
||||
pRead->curVersion++;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -460,8 +460,7 @@ class TDTestCase:
|
|||
#self.test_db("db2", 5, 10*24*3600, 2*1024) # 2M size
|
||||
|
||||
# period + size
|
||||
self.test_db("db", checkTime = 5*60, wal_period = 60, wal_size_kb=10)
|
||||
#self.test_db("db", checkTime = 3*60, wal_period = 0, wal_size_kb=0)
|
||||
self.test_db("db", checkTime = 3*60, wal_period = 60, wal_size_kb=500)
|
||||
|
||||
|
||||
def stop(self):
|
||||
|
|
|
|||
|
|
@ -776,7 +776,7 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) {
|
|||
if (field->bytes > shell.args.displayWidth) {
|
||||
return TMAX(shell.args.displayWidth, width);
|
||||
} else {
|
||||
return TMAX(field->bytes, width);
|
||||
return TMAX(field->bytes + 2, width);
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_NCHAR:
|
||||
|
|
@ -785,7 +785,7 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) {
|
|||
if (bytes > shell.args.displayWidth) {
|
||||
return TMAX(shell.args.displayWidth, width);
|
||||
} else {
|
||||
return TMAX(bytes, width);
|
||||
return TMAX(bytes + 2, width);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue