xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into fix/TD-30837

This commit is contained in:
54liuyao 2024-09-02 13:53:32 +08:00
parent d7731b2a73 cfdcd5e60e
commit f1cbd426d6
73 changed files with 2297 additions and 774 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/en/08-develop/09-udf.md
View File

@ -585,7 +585,7 @@ def process(block):
return result
```
Crate and test the UDF:
Create and test the UDF:
```sql
create function nsum as '/root/udf/nsum.py' outputtype double language 'Python';

24
docs/en/14-reference/01-components/03-taosadapter.md
View File

@ -66,7 +66,8 @@ Usage of taosAdapter:
--cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials"
--cors.allowHeaders stringArray cors allow HEADERS. Env "TAOS_ADAPTER_ALLOW_HEADERS"
--cors.allowOrigins stringArray cors allow origins. Env "TAOS_ADAPTER_ALLOW_ORIGINS"
--cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets" --cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers"
--cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets"
--cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers"
--debug enable debug mode. Env "TAOS_ADAPTER_DEBUG" (default true)
--help Print this help message and exit
--httpCodeServerError Use a non-200 http status code when server returns an error. Env "TAOS_ADAPTER_HTTP_CODE_SERVER_ERROR"
@ -87,7 +88,8 @@ Usage of taosAdapter:
--monitor.password string TDengine password. Env "TAOS_ADAPTER_MONITOR_PASSWORD" (default "taosdata")
--monitor.pauseAllMemoryThreshold float Memory percentage threshold for pause all. Env "TAOS_ADAPTER_MONITOR_PAUSE_ALL_MEMORY_THRESHOLD" (default 80)
--monitor.pauseQueryMemoryThreshold float Memory percentage threshold for pause query. Env "TAOS_ADAPTER_MONITOR_PAUSE_QUERY_MEMORY_THRESHOLD" (default 70)
--monitor.user string TDengine user. Env "TAOS_ADAPTER_MONITOR_USER" (default "root") --monitor.writeInterval duration Set write to TDengine interval. Env "TAOS_ADAPTER_MONITOR_WRITE_INTERVAL" (default 30s)
--monitor.user string TDengine user. Env "TAOS_ADAPTER_MONITOR_USER" (default "root")
--monitor.writeInterval duration Set write to TDengine interval. Env "TAOS_ADAPTER_MONITOR_WRITE_INTERVAL" (default 30s)
--monitor.writeToTD Whether write metrics to TDengine. Env "TAOS_ADAPTER_MONITOR_WRITE_TO_TD"
--node_exporter.caCertFile string node_exporter ca cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CA_CERT_FILE"
--node_exporter.certFile string node_exporter cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CERT_FILE"
@ -123,7 +125,8 @@ Usage of taosAdapter:
--restfulRowLimit int restful returns the maximum number of rows (-1 means no limit). Env "TAOS_ADAPTER_RESTFUL_ROW_LIMIT" (default -1)
--smlAutoCreateDB Whether to automatically create db when writing with schemaless. Env "TAOS_ADAPTER_SML_AUTO_CREATE_DB"
--statsd.allowPendingMessages int statsd allow pending messages. Env "TAOS_ADAPTER_STATSD_ALLOW_PENDING_MESSAGES" (default 50000)
--statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd") --statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true)
--statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd")
--statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true)
--statsd.deleteGauges statsd delete gauge cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_GAUGES" (default true)
--statsd.deleteSets statsd delete set cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_SETS" (default true)
--statsd.deleteTimings statsd delete timing cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_TIMINGS" (default true)
@ -133,7 +136,8 @@ Usage of taosAdapter:
--statsd.password string statsd password. Env "TAOS_ADAPTER_STATSD_PASSWORD" (default "taosdata")
--statsd.port int statsd server port. Env "TAOS_ADAPTER_STATSD_PORT" (default 6044)
--statsd.protocol string statsd protocol [tcp or udp]. Env "TAOS_ADAPTER_STATSD_PROTOCOL" (default "udp")
--statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL"
--statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE"
--statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL"
--statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root")
--statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10)
--taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE"
@ -168,17 +172,17 @@ See [example/config/taosadapter.toml](https://github.com/taosdata/taosadapter/bl
- Compatible with OpenTSDB JSON and telnet format writes
- [http://opentsdb.net/docs/build/html/api_http/put.html](http://opentsdb.net/docs/build/html/api_http/put.html)
- [http://opentsdb.net/docs/build/html/api_telnet/put.html](http://opentsdb.net/docs/build/html/api_telnet/put.html)
- Seamless connection to collectd
- Seamless connection to collectd.
collectd is a system statistics collection daemon, please visit [https://collectd.org/](https://collectd.org/) for more information.
- Seamless connection with StatsD
- Seamless connection with StatsD.
StatsD is a simple yet powerful daemon for aggregating statistical information. Please visit [https://github.com/statsd/statsd](https://github.com/statsd/statsd) for more information.
- Seamless connection with icinga2
- Seamless connection with icinga2.
icinga2 is a software that collects inspection result metrics and performance data. Please visit [https://icinga.com/docs/icinga-2/latest/doc/14-features/#opentsdb-writer](https://icinga.com/docs/icinga-2/latest/doc/14-features/#opentsdb-writer) for more information.
- Seamless connection to TCollector
- Seamless connection to TCollector.
TCollector is a client process that collects data from a local collector and pushes the data to OpenTSDB. Please visit [http://opentsdb.net/docs/build/html/user_guide/utilities/tcollector.html](http://opentsdb.net/docs/build/html/user_guide/utilities/tcollector.html) for more information.
- Seamless connection to node_exporter
- Seamless connection to node_exporter.
node_export is an exporter for machine metrics. Please visit [https://github.com/prometheus/node_exporter](https://github.com/prometheus/node_exporter) for more information.
- Support for Prometheus remote_read and remote_write
- Support for Prometheus remote_read and remote_write.
remote_read and remote_write are interfaces for Prometheus data read and write from/to other data storage solution. Please visit [https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis) for more information.
- Get table's VGroup ID.

3
docs/en/20-third-party/11-kafka.md vendored
View File

@ -231,8 +231,9 @@ Input following content:
"config":{
"connector.class": "com.taosdata.kafka.connect.source.TDengineSourceConnector",
"tasks.max": 1,
"subscription.group.id": "source-demo",
"connection.url": "jdbc:TAOS://127.0.0.1:6030",
"connection.username": "root",
"connection.user": "root",
"connection.password": "taosdata",
"connection.database": "test",
"connection.attempts": 3,

2
docs/zh/14-reference/01-components/02-taosc.md
View File

@ -41,4 +41,4 @@ TDengine 客户端驱动提供了应用编程所需要的全部 API，并且在
## API
请参考[连接器]
请参考[连接器](../../connector)

39
docs/zh/14-reference/01-components/03-taosadapter.md
View File

@ -66,13 +66,15 @@ Usage of taosAdapter:
--cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials"
--cors.allowHeaders stringArray cors allow HEADERS. Env "TAOS_ADAPTER_ALLOW_HEADERS"
--cors.allowOrigins stringArray cors allow origins. Env "TAOS_ADAPTER_ALLOW_ORIGINS"
--cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets" --cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers"
--cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets"
--cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers"
--debug enable debug mode. Env "TAOS_ADAPTER_DEBUG" (default true)
--help Print this help message and exit
--httpCodeServerError Use a non-200 http status code when server returns an error. Env "TAOS_ADAPTER_HTTP_CODE_SERVER_ERROR"
--influxdb.enable enable influxdb. Env "TAOS_ADAPTER_INFLUXDB_ENABLE" (default true)
--log.enableRecordHttpSql whether to record http sql. Env "TAOS_ADAPTER_LOG_ENABLE_RECORD_HTTP_SQL"
--log.path string log path. Env "TAOS_ADAPTER_LOG_PATH" (default "/var/log/taos") --log.rotationCount uint log rotation count. Env "TAOS_ADAPTER_LOG_ROTATION_COUNT" (default 30)
--log.path string log path. Env "TAOS_ADAPTER_LOG_PATH" (default "/var/log/taos")
--log.rotationCount uint log rotation count. Env "TAOS_ADAPTER_LOG_ROTATION_COUNT" (default 30)
--log.rotationSize string log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_ROTATION_SIZE" (default "1GB")
--log.rotationTime duration log rotation time. Env "TAOS_ADAPTER_LOG_ROTATION_TIME" (default 24h0m0s)
--log.sqlRotationCount uint record sql log rotation count. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_COUNT" (default 2)
@ -87,7 +89,8 @@ Usage of taosAdapter:
--monitor.password string TDengine password. Env "TAOS_ADAPTER_MONITOR_PASSWORD" (default "taosdata")
--monitor.pauseAllMemoryThreshold float Memory percentage threshold for pause all. Env "TAOS_ADAPTER_MONITOR_PAUSE_ALL_MEMORY_THRESHOLD" (default 80)
--monitor.pauseQueryMemoryThreshold float Memory percentage threshold for pause query. Env "TAOS_ADAPTER_MONITOR_PAUSE_QUERY_MEMORY_THRESHOLD" (default 70)
--monitor.user string TDengine user. Env "TAOS_ADAPTER_MONITOR_USER" (default "root") --monitor.writeInterval duration Set write to TDengine interval. Env "TAOS_ADAPTER_MONITOR_WRITE_INTERVAL" (default 30s)
--monitor.user string TDengine user. Env "TAOS_ADAPTER_MONITOR_USER" (default "root")
--monitor.writeInterval duration Set write to TDengine interval. Env "TAOS_ADAPTER_MONITOR_WRITE_INTERVAL" (default 30s)
--monitor.writeToTD Whether write metrics to TDengine. Env "TAOS_ADAPTER_MONITOR_WRITE_TO_TD"
--node_exporter.caCertFile string node_exporter ca cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CA_CERT_FILE"
--node_exporter.certFile string node_exporter cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CERT_FILE"
@ -123,7 +126,8 @@ Usage of taosAdapter:
--restfulRowLimit int restful returns the maximum number of rows (-1 means no limit). Env "TAOS_ADAPTER_RESTFUL_ROW_LIMIT" (default -1)
--smlAutoCreateDB Whether to automatically create db when writing with schemaless. Env "TAOS_ADAPTER_SML_AUTO_CREATE_DB"
--statsd.allowPendingMessages int statsd allow pending messages. Env "TAOS_ADAPTER_STATSD_ALLOW_PENDING_MESSAGES" (default 50000)
--statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd") --statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true)
--statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd")
--statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true)
--statsd.deleteGauges statsd delete gauge cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_GAUGES" (default true)
--statsd.deleteSets statsd delete set cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_SETS" (default true)
--statsd.deleteTimings statsd delete timing cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_TIMINGS" (default true)
@ -133,7 +137,8 @@ Usage of taosAdapter:
--statsd.password string statsd password. Env "TAOS_ADAPTER_STATSD_PASSWORD" (default "taosdata")
--statsd.port int statsd server port. Env "TAOS_ADAPTER_STATSD_PORT" (default 6044)
--statsd.protocol string statsd protocol [tcp or udp]. Env "TAOS_ADAPTER_STATSD_PROTOCOL" (default "udp")
--statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL"
--statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE"
--statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL"
--statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root")
--statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10)
--taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE"
@ -166,19 +171,19 @@ AllowWebSockets
- 兼容 InfluxDB v1 写接口
[https://docs.influxdata.com/influxdb/v2.0/reference/api/influxdb-1x/write/](https://docs.influxdata.com/influxdb/v2.0/reference/api/influxdb-1x/write/)
- 兼容 OpenTSDB JSON 和 telnet 格式写入
- \<http://opentsdb.net/docs/build/html/api_http/put.html>
- \<http://opentsdb.net/docs/build/html/api_telnet/put.html>
- 与 collectd 无缝连接
- \<http://opentsdb.net/docs/build/html/api_http/put.html\>
- \<http://opentsdb.net/docs/build/html/api_telnet/put.html\>
- 与 collectd 无缝连接。
collectd 是一个系统统计收集守护程序，请访问 [https://collectd.org/](https://collectd.org/) 了解更多信息。
- Seamless connection with StatsD
- Seamless connection with StatsD。
StatsD 是一个简单而强大的统计信息汇总的守护程序。请访问 [https://github.com/statsd/statsd](https://github.com/statsd/statsd) 了解更多信息。
- 与 icinga2 的无缝连接
- 与 icinga2 的无缝连接。
icinga2 是一个收集检查结果指标和性能数据的软件。请访问 [https://icinga.com/docs/icinga-2/latest/doc/14-features/#opentsdb-writer](https://icinga.com/docs/icinga-2/latest/doc/14-features/#opentsdb-writer) 了解更多信息。
- 与 tcollector 无缝连接
- 与 tcollector 无缝连接。
TCollector是一个客户端进程，从本地收集器收集数据，并将数据推送到 OpenTSDB。请访问 [http://opentsdb.net/docs/build/html/user_guide/utilities/tcollector.html](http://opentsdb.net/docs/build/html/user_guide/utilities/tcollector.html) 了解更多信息。
- 无缝连接 node_exporter
- 无缝连接 node_exporter。
node_export 是一个机器指标的导出器。请访问 [https://github.com/prometheus/node_exporter](https://github.com/prometheus/node_exporter) 了解更多信息。
- 支持 Prometheus remote_read 和 remote_write
- 支持 Prometheus remote_read 和 remote_write。
remote_read 和 remote_write 是 Prometheus 数据读写分离的集群方案。请访问[https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis) 了解更多信息。
- 获取 table 所在的虚拟节点组（VGroup）的 VGroup ID。
@ -186,15 +191,11 @@ AllowWebSockets
### TDengine RESTful 接口
您可以使用任何支持 http 协议的客户端通过访问 RESTful 接口地址 `http://<fqdn>:6041/rest/sql` 来写入数据到 TDengine 或从 TDengine 中查询数据。细节请参考[官方文档](../../connector/rest-api/)。
您可以使用任何支持 http 协议的客户端通过访问 RESTful 接口地址 `http://<fqdn>:6041/rest/sql` 来写入数据到 TDengine 或从 TDengine 中查询数据。细节请参考[REST API 文档](../../connector/rest-api/)。
### InfluxDB
您可以使用任何支持 http 协议的客户端访问 Restful 接口地址 `http://<fqdn>:6041/<APIEndPoint>` 来写入 InfluxDB 兼容格式的数据到 TDengine。EndPoint 如下：
```text
/influxdb/v1/write
```
您可以使用任何支持 http 协议的客户端访问 Restful 接口地址 `http://<fqdn>:6041/influxdb/v1/write` 来写入 InfluxDB 兼容格式的数据到 TDengine。
支持 InfluxDB 参数如下：

9
docs/zh/20-third-party/01-collection/11-kafka.md vendored
View File

@ -229,8 +229,9 @@ vi source-demo.json
"config":{
"connector.class": "com.taosdata.kafka.connect.source.TDengineSourceConnector",
"tasks.max": 1,
"subscription.group.id": "source-demo",
"connection.url": "jdbc:TAOS://127.0.0.1:6030",
"connection.username": "root",
"connection.user": "root",
"connection.password": "taosdata",
"connection.database": "test",
"connection.attempts": 3,
@ -389,12 +390,12 @@ curl -X DELETE http://localhost:8083/connectors/TDengineSourceConnector
## 其他说明
1. 关于如何在独立安装的 Kafka 环境使用 Kafka Connect 插件， 请参考官方文档：\<https://kafka.apache.org/documentation/#connect>。
1. 关于如何在独立安装的 Kafka 环境使用 Kafka Connect 插件， 请参考官方文档：<https://kafka.apache.org/documentation/#connect>。
## 问题反馈
无论遇到任何问题，都欢迎在本项目的 Github 仓库反馈：\<https://github.com/taosdata/kafka-connect-tdengine/issues>。
无论遇到任何问题，都欢迎在本项目的 Github 仓库反馈：<https://github.com/taosdata/kafka-connect-tdengine/issues>。
## 参考
1. \<https://kafka.apache.org/documentation/>
1. <https://kafka.apache.org/documentation/>

4
include/libs/nodes/querynodes.h
View File

@ -61,6 +61,7 @@ typedef struct SExprNode {
bool asAlias;
bool asParam;
bool asPosition;
int32_t projIdx;
} SExprNode;
typedef enum EColumnType {
@ -91,6 +92,8 @@ typedef struct SColumnNode {
int16_t numOfPKs;
bool tableHasPk;
bool isPk;
int32_t projRefIdx;
int32_t resIdx;
} SColumnNode;
typedef struct SColumnRefNode {
@ -214,6 +217,7 @@ typedef struct SRealTableNode {
double ratio;
SArray* pSmaIndexes;
int8_t cacheLastMode;
int8_t stbRewrite;
SArray* pTsmas;
SArray* tsmaTargetTbVgInfo; // SArray<SVgroupsInfo*>, used for child table or normal table only
SArray* tsmaTargetTbInfo; // SArray<STsmaTargetTbInfo>, used for child table or normal table only

3
include/util/tlockfree.h
View File

@ -39,7 +39,6 @@ typedef void (*_ref_fn_t)(const void *pObj);
// set the initial reference count value
#define T_REF_INIT_VAL(x, _v) \
do { \
assert(_v >= 0); \
atomic_store_32(&((x)->_ref.val), (_v)); \
} while (0)
@ -64,8 +63,6 @@ typedef void (*_ref_fn_t)(const void *pObj);
} \
} while (0)
#define T_REF_VAL_CHECK(x) assert((x)->_ref.val >= 0);
#define T_REF_VAL_GET(x) (x)->_ref.val
// single writer multiple reader lock

4
source/client/src/clientEnv.c
View File

@ -933,7 +933,7 @@ void taos_init_imp(void) {
appInfo.pInstMapByClusterId =
taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
if (NULL == appInfo.pInstMap || NULL == appInfo.pInstMapByClusterId) {
tscError("failed to allocate memory when init appInfo");
(void)printf("failed to allocate memory when init appInfo\n");
tscInitRes = TSDB_CODE_OUT_OF_MEMORY;
return;
}
@ -961,7 +961,7 @@ void taos_init_imp(void) {
if (InitRegexCache() != 0) {
tscInitRes = -1;
tscError("failed to init regex cache");
(void)printf("failed to init regex cache\n");
return;
}

5
source/client/src/clientTmq.c
View File

@ -2183,6 +2183,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
taosWUnLockLatch(&tmq->lock);
}
setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId);
taosMemoryFreeClear(pollRspWrapper->pEpset);
tmqFreeRspWrapper(pRspWrapper);
taosFreeQitem(pRspWrapper);
} else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP) {
@ -2247,6 +2248,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
", vg total:%" PRId64 ", total:%" PRId64 ",QID:0x%" PRIx64,
tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows,
pollRspWrapper->reqId);
taosMemoryFreeClear(pollRspWrapper->pEpset);
taosFreeQitem(pRspWrapper);
taosWUnLockLatch(&tmq->lock);
return pRsp;
@ -2284,6 +2286,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
// build rsp
SMqMetaRspObj* pRsp = NULL;
(void)tmqBuildMetaRspFromWrapper(pollRspWrapper, &pRsp);
taosMemoryFreeClear(pollRspWrapper->pEpset);
taosFreeQitem(pRspWrapper);
taosWUnLockLatch(&tmq->lock);
return pRsp;
@ -2321,6 +2324,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
tmq->consumerId, true);
SMqBatchMetaRspObj* pRsp = NULL;
(void)tmqBuildBatchMetaRspFromWrapper(pollRspWrapper, &pRsp);
taosMemoryFreeClear(pollRspWrapper->pEpset);
taosFreeQitem(pRspWrapper);
taosWUnLockLatch(&tmq->lock);
return pRsp;
@ -2381,6 +2385,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows,
pollRspWrapper->reqId);
taosMemoryFreeClear(pollRspWrapper->pEpset);
taosFreeQitem(pRspWrapper);
taosWUnLockLatch(&tmq->lock);
return pRsp;

32
source/common/src/tglobal.c
View File

@ -485,27 +485,27 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input
}
if ((code = cfgLoad(pCfg, CFG_STYPE_APOLLO_URL, apolloUrl)) != 0) {
uError("failed to load from apollo url:%s since %s", apolloUrl, tstrerror(code));
(void)printf("failed to load from apollo url:%s since %s\n", apolloUrl, tstrerror(code));
TAOS_RETURN(code);
}
if ((code = cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgFile)) != 0) {
uError("failed to load from cfg file:%s since %s", cfgFile, tstrerror(code));
(void)printf("failed to load from cfg file:%s since %s\n", cfgFile, tstrerror(code));
TAOS_RETURN(code);
}
if ((code = cfgLoad(pCfg, CFG_STYPE_ENV_FILE, envFile)) != 0) {
uError("failed to load from env file:%s since %s", envFile, tstrerror(code));
(void)printf("failed to load from env file:%s since %s\n", envFile, tstrerror(code));
TAOS_RETURN(code);
}
if ((code = cfgLoad(pCfg, CFG_STYPE_ENV_VAR, NULL)) != 0) {
uError("failed to load from global env variables since %s", tstrerror(code));
(void)printf("failed to load from global env variables since %s\n", tstrerror(code));
TAOS_RETURN(code);
}
if ((code = cfgLoad(pCfg, CFG_STYPE_ENV_CMD, envCmd)) != 0) {
uError("failed to load from cmd env variables since %s", tstrerror(code));
(void)printf("failed to load from cmd env variables since %s\n", tstrerror(code));
TAOS_RETURN(code);
}
@ -1648,12 +1648,12 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
}
if ((code = taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl)) != TSDB_CODE_SUCCESS) {
printf("failed to load cfg since %s\n", tstrerror(code));
(void)printf("failed to load cfg since %s\n", tstrerror(code));
goto _exit;
}
if ((code = cfgLoadFromArray(pCfg, pArgs)) != TSDB_CODE_SUCCESS) {
printf("failed to load cfg from array since %s\n", tstrerror(code));
(void)printf("failed to load cfg from array since %s\n", tstrerror(code));
goto _exit;
}
@ -1669,18 +1669,18 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
TAOS_CHECK_GOTO(taosSetAllDebugFlag(pCfg, pDebugItem->i32), &lino, _exit);
if ((code = taosMulModeMkDir(tsLogDir, 0777, true)) != TSDB_CODE_SUCCESS) {
printf("failed to create dir:%s since %s\n", tsLogDir, tstrerror(code));
(void)printf("failed to create dir:%s since %s\n", tsLogDir, tstrerror(code));
goto _exit;
}
if ((code = taosInitLog(logname, logFileNum, tsc)) != 0) {
printf("failed to init log file since %s\n", tstrerror(code));
(void)printf("failed to init log file since %s\n", tstrerror(code));
goto _exit;
}
_exit:
if (TSDB_CODE_SUCCESS != code) {
printf("failed to create log at %d since %s:", lino, tstrerror(code));
(void)printf("failed to create log at %d since %s\n", lino, tstrerror(code));
}
cfgCleanup(pCfg);
@ -1700,12 +1700,12 @@ int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char *
TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) ,NULL, _exit);
if ((code = taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) {
printf("failed to load cfg since %s\n", tstrerror(code));
(void)printf("failed to load cfg since %s\n", tstrerror(code));
goto _exit;
}
if ((code = cfgLoadFromArray(pCfg, pArgs)) != 0) {
printf("failed to load cfg from array since %s\n", tstrerror(code));
(void)printf("failed to load cfg from array since %s\n", tstrerror(code));
goto _exit;
}
@ -1769,14 +1769,14 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
TAOS_CHECK_GOTO(taosAddSystemCfg(tsCfg), &lino, _exit);
if ((code = taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) {
uError("failed to load cfg since %s", tstrerror(code));
(void)printf("failed to load cfg since %s\n", tstrerror(code));
cfgCleanup(tsCfg);
tsCfg = NULL;
TAOS_RETURN(code);
}
if ((code = cfgLoadFromArray(tsCfg, pArgs)) != 0) {
uError("failed to load cfg from array since %s", tstrerror(code));
(void)printf("failed to load cfg from array since %s\n", tstrerror(code));
cfgCleanup(tsCfg);
tsCfg = NULL;
TAOS_RETURN(code);
@ -1798,7 +1798,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
SConfigItem *pItem = cfgGetItem(tsCfg, "debugFlag");
if (NULL == pItem) {
uError("debugFlag not found in cfg");
(void)printf("debugFlag not found in cfg\n");
TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND);
}
TAOS_CHECK_GOTO(taosSetAllDebugFlag(tsCfg, pItem->i32), &lino, _exit);
@ -1811,7 +1811,7 @@ _exit:
if (TSDB_CODE_SUCCESS != code) {
cfgCleanup(tsCfg);
tsCfg = NULL;
uError("failed to init cfg at %d since %s", lino, tstrerror(code));
(void)printf("failed to init cfg at %d since %s\n", lino, tstrerror(code));
}
TAOS_RETURN(code);

2
source/dnode/mnode/impl/src/mndDb.c
View File

@ -1892,7 +1892,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs,
mndReleaseDb(pMnode, pDb);
continue;
} else {
mInfo("db:%s, valid dbinfo, vgVersion:%d cfgVersion:%d stateTs:%" PRId64
mTrace("db:%s, valid dbinfo, vgVersion:%d cfgVersion:%d stateTs:%" PRId64
" numOfTables:%d, changed to vgVersion:%d cfgVersion:%d stateTs:%" PRId64 " numOfTables:%d",
pDbCacheInfo->dbFName, pDbCacheInfo->vgVersion, pDbCacheInfo->cfgVersion, pDbCacheInfo->stateTs,
pDbCacheInfo->numOfTable, pDb->vgVersion, pDb->cfgVersion, pDb->stateTs, numOfTable);

2
source/dnode/mnode/impl/src/mndStream.c
View File

@ -523,6 +523,7 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) {
int32_t code = tEncodeStreamTask(&encoder, pTask);
if (code == -1) {
tEncoderClear(&encoder);
return TSDB_CODE_INVALID_MSG;
}
@ -1009,6 +1010,7 @@ static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, int
tEncoderInit(&encoder, abuf, tlen);
int32_t pos = tEncodeStreamCheckpointSourceReq(&encoder, &req);
if (pos == -1) {
tEncoderClear(&encoder);
return TSDB_CODE_INVALID_MSG;
}

1
source/dnode/mnode/impl/test/stream/stream.cpp
View File

@ -91,6 +91,7 @@ SRpcMsg buildHbReq() {
tEncoderInit(&encoder, (uint8_t*)buf, tlen);
if ((code = tEncodeStreamHbMsg(&encoder, &msg)) < 0) {
rpcFreeCont(buf);
tEncoderClear(&encoder);
goto _end;
}
tEncoderClear(&encoder);

8
source/dnode/vnode/src/tq/tqSink.c
View File

@ -342,7 +342,10 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
// todo remove this
void* pGpIdData = colDataGetData(pGpIdColInfo, rowId);
ASSERT(gid == *(int64_t*)pGpIdData);
if (gid != *(int64_t*)pGpIdData) {
tqError("s-task:%s vgId:%d invalid groupId:%" PRId64 " actual:%" PRId64 " in sink task", id, vgId, gid,
*(int64_t*)pGpIdData);
}
}
code = setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, gid,
@ -747,7 +750,6 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
return code;
}
ASSERT(pRow);
void* p = taosArrayPush(pTableData->aRowP, &pRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -779,8 +781,6 @@ int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkI
bool isValid = isValidDstChildTable(&mr, vgId, dstTableName, suid);
if (isValid) { // not valid table, ignore it
tqDebug("s-task:%s set uid:%" PRIu64 " for dstTable:%s from meta", id, mr.me.uid, pTableSinkInfo->name.data);
ASSERT(terrno == 0);
// set the destination table uid
(*uid) = mr.me.uid;
pTableSinkInfo->uid = mr.me.uid;

6
source/dnode/vnode/src/tq/tqStreamTask.c
View File

@ -46,7 +46,10 @@ int32_t tqScanWal(STQ* pTq) {
streamMetaWLock(pMeta);
int32_t times = (--pMeta->scanInfo.scanCounter);
ASSERT(pMeta->scanInfo.scanCounter >= 0);
if (times < 0) {
tqError("vgId:%d invalid scan counter:%d, reset to 0", vgId, times);
times = 0;
}
numOfTasks = taosArrayGetSize(pMeta->pTaskList);
streamMetaWUnLock(pMeta);
@ -269,7 +272,6 @@ bool taskReadyForDataFromWal(SStreamTask* pTask) {
// fill-history task has entered into the last phase, no need to anything
if ((pTask->info.fillHistory == 1) && pTask->status.appendTranstateBlock) {
ASSERT(pState.state == TASK_STATUS__READY);
// the maximum version of data in the WAL has reached already, the step2 is done
tqDebug("s-task:%s fill-history reach the maximum ver:%" PRId64 ", not scan wal anymore", pTask->id.idStr,
pTask->dataRange.range.maxVer);

27
source/dnode/vnode/src/tqCommon/tqCommon.c
View File

@ -339,13 +339,15 @@ int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SMsgHead* pRspHead = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamDispatchRsp));
if (pRspHead == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
tqError("s-task:0x%x send dispatch error rsp, out of memory", req.taskId);
return -1;
return terrno;
}
pRspHead->vgId = htonl(req.upstreamNodeId);
ASSERT(pRspHead->vgId != 0);
if(pRspHead->vgId == 0) {
tqError("vgId:%d invalid dispatch msg from upstream to task:0x%x", pMeta->vgId, req.taskId);
return TSDB_CODE_INVALID_MSG;
}
SStreamDispatchRsp* pRsp = POINTER_SHIFT(pRspHead, sizeof(SMsgHead));
pRsp->streamId = htobe64(req.streamId);
@ -926,7 +928,6 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
streamTaskSetStatusReady(pTask);
} else if (pState.state == TASK_STATUS__UNINIT) {
// tqDebug("s-task:%s start task by checking downstream tasks", pTask->id.idStr);
// ASSERT(pTask->status.downstreamReady == 0);
// tqStreamTaskRestoreCheckpoint(pMeta, pTask->id.streamId, pTask->id.taskId);
tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name);
} else {
@ -1000,7 +1001,10 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
TSDB_CODE_ACTION_IN_PROGRESS);
}
} else { // upstream not recv the checkpoint-source/trigger till now
ASSERT(pState.state == TASK_STATUS__READY || pState.state == TASK_STATUS__HALT);
if (!(pState.state == TASK_STATUS__READY || pState.state == TASK_STATUS__HALT)) {
tqFatal("s-task:%s invalid task status:%s", pTask->id.idStr, pState.name);
}
tqWarn(
"s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all "
"upstream sending checkpoint-source/trigger",
@ -1109,9 +1113,7 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t
} else {
code = streamTrySchedExec(pTask);
}
} /*else {
ASSERT(status != TASK_STATUS__UNINIT);
}*/
}
streamMetaReleaseTask(pMeta, pTask);
return code;
@ -1234,7 +1236,14 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
pTask->id.idStr, vgId, pTask->chkInfo.checkpointId, req.checkpointId);
streamMutexLock(&pTask->lock);
ASSERT(pTask->chkInfo.checkpointId >= req.checkpointId);
if (pTask->chkInfo.checkpointId < req.checkpointId) {
tqFatal("s-task:%s vgId:%d invalid consensus-checkpointId:%" PRId64 ", greater than existed checkpointId:%"PRId64,
pTask->id.idStr, vgId, req.checkpointId, pTask->chkInfo.checkpointId);
streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_STREAM_INTERNAL_ERROR;
}
SConsenChkptInfo* pConsenInfo = &pTask->status.consenChkptInfo;
if (pConsenInfo->consenChkptTransId >= req.transId) {

7
source/dnode/vnode/src/tsdb/tsdbFS2.c
View File

@ -1020,7 +1020,10 @@ int32_t tsdbFSCreateRefSnapshotWithoutLock(STFileSystem *fs, TFileSetArray **fse
if (code) break;
code = TARRAY2_APPEND(fsetArr[0], fset1);
if (code) break;
if (code) {
tsdbTFileSetClear(&fset1);
break;
}
}
if (code) {
@ -1219,4 +1222,4 @@ int32_t tsdbFinishTaskOnFileSet(STsdb *tsdb, int32_t fid) {
}
return 0;
}
}

2
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -4817,7 +4817,7 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi
return code;
_err:
tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr);
tsdbError("failed to create data reader, error at:%d code:%s %s", lino, tstrerror(code), idstr);
tsdbReaderClose2(*ppReader);
*ppReader = NULL; // reset the pointer value.
return code;

3
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -759,8 +759,7 @@ int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
vTrace("message in vnode query queue is processing");
if ((pMsg->msgType == TDMT_SCH_QUERY || pMsg->msgType == TDMT_VND_TMQ_CONSUME ||
pMsg->msgType == TDMT_VND_TMQ_CONSUME_PUSH) &&
if ((pMsg->msgType == TDMT_SCH_QUERY || pMsg->msgType == TDMT_VND_TMQ_CONSUME) &&
!syncIsReadyForRead(pVnode->sync)) {
vnodeRedirectRpcMsg(pVnode, pMsg, terrno);
return 0;

10
source/libs/executor/src/scanoperator.c
View File

@ -744,12 +744,10 @@ _end:
insertRet = taosLRUCacheInsert(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(uint64_t), pVal,
sizeof(STableCachedVal), freeCachedMetaItem, NULL, TAOS_LRU_PRIORITY_LOW, NULL);
if (insertRet != TAOS_LRU_STATUS_OK) {
qError("failed to put meta into lru cache, code:%d, %s", insertRet, idStr);
taosMemoryFreeClear(pVal);
freeTableCachedValObj(&val);
qWarn("failed to put meta into lru cache, code:%d, %s", insertRet, idStr);
}
}
if (freeReader) {
pHandle->api.metaReaderFn.clearReader(&mr);
}
@ -5745,7 +5743,7 @@ static int32_t getBlockForTableMergeScan(void* param, SSDataBlock** ppBlock) {
pOperator->resultInfo.totalRows += pBlock->info.rows;
pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
*ppBlock = pBlock;
return code;
}
@ -5900,7 +5898,7 @@ void startGroupTableMergeScan(SOperatorInfo* pOperator) {
int32_t numOfTables = 0;
code = tableListGetSize(pInfo->base.pTableListInfo, &numOfTables);
QUERY_CHECK_CODE(code, lino, _end);
int32_t i = pInfo->tableStartIndex + 1;
for (; i < numOfTables; ++i) {
STableKeyInfo* tableKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, i);

3
source/libs/function/src/builtinsimpl.c
View File

@ -6007,6 +6007,7 @@ int32_t modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) {
pInfo->buf = taosMemoryMalloc(pInfo->colBytes);
if (NULL == pInfo->buf) {
taosHashCleanup(pInfo->pHash);
return TSDB_CODE_OUT_OF_MEMORY;
}
@ -6114,6 +6115,7 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
int32_t currentRow = pBlock->info.rows;
if (NULL == pCol) {
modeFunctionCleanup(pInfo);
return TSDB_CODE_OUT_OF_RANGE;
}
@ -6145,6 +6147,7 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
code = colDataSetVal(pCol, currentRow, pData, false);
if (TSDB_CODE_SUCCESS != code) {
modeFunctionCleanup(pInfo);
return code;
}
code = setSelectivityValue(pCtx, pBlock, &resTuplePos, currentRow);

2
source/libs/index/inc/indexFstNode.h
View File

@ -44,7 +44,7 @@ FstBuilderNode* fstBuilderNodeDefault();
FstBuilderNode* fstBuilderNodeClone(FstBuilderNode* src);
void fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src);
int32_t fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src);
// bool fstBuilderNodeCompileTo(FstBuilderNode *b, IdxFile' *wrt,
// CompiledAddr lastAddr, CompiledAddr startAddr);

6
source/libs/index/inc/indexTfile.h
View File

@ -18,7 +18,7 @@
#include "indexFst.h"
#include "indexFstFile.h"
#include "indexInt.h"
#include "indexTfile.h"
//#include "indexTfile.h"
#include "indexUtil.h"
#include "tlockfree.h"
@ -129,8 +129,8 @@ void tfileIteratorDestroy(Iterate* iterator);
TFileValue* tfileValueCreate(char* val);
int tfileValuePush(TFileValue* tf, uint64_t val);
void tfileValueDestroy(TFileValue* tf);
int32_t tfileValuePush(TFileValue* tf, uint64_t val);
void tfileValueDestroy(TFileValue* tf);
#ifdef __cplusplus
}

30
source/libs/index/inc/indexUtil.h
View File

@ -46,17 +46,17 @@ extern "C" {
buf += len; \
} while (0)
#define INDEX_MERGE_ADD_DEL(src, dst, tgt) \
{ \
bool f = false; \
for (int i = 0; i < taosArrayGetSize(src); i++) { \
if (*(uint64_t *)taosArrayGet(src, i) == tgt) { \
f = true; \
} \
} \
if (f == false) { \
(void)taosArrayPush(dst, &tgt); \
} \
#define INDEX_MERGE_ADD_DEL(src, dst, tgt) \
{ \
bool f = false; \
for (int i = 0; i < taosArrayGetSize(src); i++) { \
if (*(uint64_t *)taosArrayGet(src, i) == tgt) { \
f = true; \
} \
} \
if (f == false) { \
if (taosArrayPush(dst, &tgt) == NULL) code = TSDB_CODE_OUT_OF_MEMORY; \
} \
}
/* multi sorted result intersection
@ -65,7 +65,7 @@ extern "C" {
* [1, 4, 5]
* output:[4, 5]
*/
void iIntersection(SArray *in, SArray *out);
int32_t iIntersection(SArray *in, SArray *out);
/* multi sorted result union
* input: [1, 2, 4, 5]
@ -73,7 +73,7 @@ void iIntersection(SArray *in, SArray *out);
* [1, 4, 5]
* output:[1, 2, 3, 4, 5]
*/
void iUnion(SArray *in, SArray *out);
int32_t iUnion(SArray *in, SArray *out);
/* see example
* total: [1, 2, 4, 5, 7, 8]
@ -81,7 +81,7 @@ void iUnion(SArray *in, SArray *out);
* return: [1, 2, 7, 8] saved in total
*/
void iExcept(SArray *total, SArray *except);
int32_t iExcept(SArray *total, SArray *except);
int uidCompare(const void *a, const void *b);
@ -107,7 +107,7 @@ void idxTRsltClear(SIdxTRslt *tr);
void idxTRsltDestroy(SIdxTRslt *tr);
void idxTRsltMergeTo(SIdxTRslt *tr, SArray *out);
int32_t idxTRsltMergeTo(SIdxTRslt *tr, SArray *out);
#ifdef __cplusplus
}

135
source/libs/index/src/index.c
View File

@ -92,7 +92,7 @@ static int32_t idxMergeFinalResults(SArray* in, EIndexOperatorType oType, SArray
static int32_t idxGenTFile(SIndex* index, IndexCache* cache, SArray* batch);
// merge cache and tfile by opera type
static void idxMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv, SIdxTRslt* helper);
static int32_t idxMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv, SIdxTRslt* helper);
// static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf);
// int32_t indexSerialKey(ICacheKey* key, char* buf);
@ -212,6 +212,7 @@ void idxReleaseRef(int64_t ref) {
int32_t indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
// TODO(yihao): reduce the lock range
int32_t code = 0;
(void)taosThreadMutexLock(&index->mtx);
for (int i = 0; i < taosArrayGetSize(fVals); i++) {
SIndexTerm* p = taosArrayGetP(fVals, i);
@ -223,11 +224,19 @@ int32_t indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
IndexCache** cache = taosHashGet(index->colObj, buf, sz);
if (cache == NULL) {
IndexCache* pCache = idxCacheCreate(index, p->suid, p->colName, p->colType);
(void)taosHashPut(index->colObj, buf, sz, &pCache, sizeof(void*));
code = taosHashPut(index->colObj, buf, sz, &pCache, sizeof(void*));
if (code != 0) {
idxCacheDestroy(pCache);
break;
}
}
}
(void)taosThreadMutexUnlock(&index->mtx);
if (code != 0) {
return code;
}
for (int i = 0; i < taosArrayGetSize(fVals); i++) {
SIndexTerm* p = taosArrayGetP(fVals, i);
@ -247,15 +256,27 @@ int32_t indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
return 0;
}
int32_t indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result) {
int32_t code = 0;
EIndexOperatorType opera = multiQuerys->opera; // relation of querys
SArray* iRslts = taosArrayInit(4, POINTER_BYTES);
int nQuery = taosArrayGetSize(multiQuerys->query);
if (iRslts == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int nQuery = taosArrayGetSize(multiQuerys->query);
for (size_t i = 0; i < nQuery; i++) {
SIndexTermQuery* qterm = taosArrayGet(multiQuerys->query, i);
SArray* trslt = NULL;
(void)idxTermSearch(index, qterm, &trslt);
(void)taosArrayPush(iRslts, (void*)&trslt);
code = idxTermSearch(index, qterm, &trslt);
if (code != 0) {
idxInterRsltDestroy(iRslts);
return code;
}
if (taosArrayPush(iRslts, (void*)&trslt) == NULL) {
idxInterRsltDestroy(iRslts);
return TSDB_CODE_OUT_OF_MEMORY;
}
}
(void)idxMergeFinalResults(iRslts, opera, result);
idxInterRsltDestroy(iRslts);
@ -267,6 +288,9 @@ int indexDelete(SIndex* index, SIndexMultiTermQuery* query) { return 1; }
SIndexOpts* indexOptsCreate(int32_t cacheSize) {
SIndexOpts* opts = taosMemoryCalloc(1, sizeof(SIndexOpts));
if (opts == NULL) {
return NULL;
}
opts->cacheSize = cacheSize;
return opts;
}
@ -295,7 +319,7 @@ void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery) {
int32_t indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType qType) {
SIndexTermQuery q = {.qType = qType, .term = term};
if (taosArrayPush(pQuery->query, &q) == NULL) {
return terrno;
return TSDB_CODE_OUT_OF_MEMORY;
}
return 0;
}
@ -362,7 +386,9 @@ void indexTermDestroy(SIndexTerm* p) {
SIndexMultiTerm* indexMultiTermCreate() { return taosArrayInit(4, sizeof(SIndexTerm*)); }
int32_t indexMultiTermAdd(SIndexMultiTerm* terms, SIndexTerm* term) {
(void)taosArrayPush(terms, &term);
if (taosArrayPush(terms, &term) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
return 0;
}
void indexMultiTermDestroy(SIndexMultiTerm* terms) {
@ -422,6 +448,7 @@ bool indexJsonIsRebuild(SIndexJson* idx) {
}
static int32_t idxTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result) {
int32_t code = 0;
SIndexTerm* term = query->term;
const char* colName = term->colName;
int32_t nColName = term->nColName;
@ -452,6 +479,10 @@ static int32_t idxTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** resu
int64_t st = taosGetTimestampUs();
SIdxTRslt* tr = idxTRsltCreate();
if (tr == NULL) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, END);
}
if (0 == idxCacheSearch(cache, query, tr, &s)) {
if (s == kTypeDeletion) {
indexInfo("col: %s already drop by", term->colName);
@ -473,13 +504,14 @@ static int32_t idxTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** resu
int64_t cost = taosGetTimestampUs() - st;
indexInfo("search cost: %" PRIu64 "us", cost);
idxTRsltMergeTo(tr, *result);
code = idxTRsltMergeTo(tr, *result);
TAOS_CHECK_GOTO(code, NULL, END);
idxTRsltDestroy(tr);
return 0;
END:
idxTRsltDestroy(tr);
return 0;
return code;
}
static void idxInterRsltDestroy(SArray* results) {
if (results == NULL) {
@ -503,9 +535,9 @@ static int32_t idxMergeFinalResults(SArray* in, EIndexOperatorType oType, SArray
}
if (oType == MUST) {
iIntersection(in, out);
return iIntersection(in, out);
} else if (oType == SHOULD) {
iUnion(in, out);
return iUnion(in, out);
} else if (oType == NOT) {
// just one column index, enhance later
// taosArrayAddAll(fResults, interResults);
@ -514,30 +546,53 @@ static int32_t idxMergeFinalResults(SArray* in, EIndexOperatorType oType, SArray
return 0;
}
static void idxMayMergeTempToFinalRslt(SArray* result, TFileValue* tfv, SIdxTRslt* tr) {
static int32_t idxMayMergeTempToFinalRslt(SArray* result, TFileValue* tfv, SIdxTRslt* tr) {
int32_t code = 0;
int32_t sz = taosArrayGetSize(result);
if (sz > 0) {
TFileValue* lv = taosArrayGetP(result, sz - 1);
if (tfv != NULL && strcmp(lv->colVal, tfv->colVal) != 0) {
idxTRsltMergeTo(tr, lv->tableId);
code = idxTRsltMergeTo(tr, lv->tableId);
if (code != 0) {
indexFatal("failed to merge result since %s", tstrerror(code));
return code;
}
idxTRsltClear(tr);
(void)taosArrayPush(result, &tfv);
if (taosArrayPush(result, &tfv) == NULL) {
indexFatal("failed to merge result since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY));
}
} else if (tfv == NULL) {
// handle last iterator
idxTRsltMergeTo(tr, lv->tableId);
code = idxTRsltMergeTo(tr, lv->tableId);
if (code != 0) {
indexFatal("failed to merge result since %s", tstrerror(code));
}
} else {
tfileValueDestroy(tfv);
return 0;
}
} else {
(void)taosArrayPush(result, &tfv);
if (taosArrayPush(result, &tfv) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
return code;
}
static void idxMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue* tv, SIdxTRslt* tr) {
static int32_t idxMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue* tv, SIdxTRslt* tr) {
int32_t code = 0;
char* colVal = (cv != NULL) ? cv->colVal : tv->colVal;
TFileValue* tfv = tfileValueCreate(colVal);
if (tfv == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
idxMayMergeTempToFinalRslt(result, tfv, tr);
code = idxMayMergeTempToFinalRslt(result, tfv, tr);
if (code != 0) {
tfileValueDestroy(tfv);
return code;
}
tfv = NULL;
if (cv != NULL) {
uint64_t id = *(uint64_t*)taosArrayGet(cv->val, 0);
@ -549,8 +604,11 @@ static void idxMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue
}
}
if (tv != NULL) {
(void)taosArrayAddAll(tr->total, tv->val);
if (taosArrayAddAll(tr->total, tv->val) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
return 0;
}
static void idxDestroyFinalRslt(SArray* result) {
int32_t sz = result ? taosArrayGetSize(result) : 0;
@ -562,6 +620,7 @@ static void idxDestroyFinalRslt(SArray* result) {
}
int32_t idxFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) {
int32_t code = 0;
if (sIdx == NULL) {
return TSDB_CODE_INVALID_PTR;
}
@ -598,12 +657,16 @@ int32_t idxFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) {
}
SArray* result = taosArrayInit(1024, sizeof(void*));
if (result == NULL) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception);
}
bool cn = cacheIter ? cacheIter->next(cacheIter) : false;
bool tn = tfileIter ? tfileIter->next(tfileIter) : false;
SIdxTRslt* tr = idxTRsltCreate();
if (tr == NULL) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception);
}
while (cn == true || tn == true) {
IterateValue* cv = (cn == true) ? cacheIter->getValue(cacheIter) : NULL;
@ -618,27 +681,42 @@ int32_t idxFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) {
comp = 1;
}
if (comp == 0) {
idxMergeCacheAndTFile(result, cv, tv, tr);
code = idxMergeCacheAndTFile(result, cv, tv, tr);
if (code != 0) {
TAOS_CHECK_GOTO(code, NULL, _exception);
}
cn = cacheIter->next(cacheIter);
tn = tfileIter->next(tfileIter);
} else if (comp < 0) {
idxMergeCacheAndTFile(result, cv, NULL, tr);
code = idxMergeCacheAndTFile(result, cv, NULL, tr);
if (code != 0) {
TAOS_CHECK_GOTO(code, NULL, _exception);
}
cn = cacheIter->next(cacheIter);
} else {
idxMergeCacheAndTFile(result, NULL, tv, tr);
code = idxMergeCacheAndTFile(result, NULL, tv, tr);
if (code != 0) {
TAOS_CHECK_GOTO(code, NULL, _exception);
}
tn = tfileIter->next(tfileIter);
}
}
idxMayMergeTempToFinalRslt(result, NULL, tr);
if ((code = idxMayMergeTempToFinalRslt(result, NULL, tr)) != 0) {
idxTRsltDestroy(tr);
TAOS_CHECK_GOTO(code, NULL, _exception);
}
idxTRsltDestroy(tr);
int ret = idxGenTFile(sIdx, pCache, result);
if (ret != 0) {
indexError("failed to merge");
code = idxGenTFile(sIdx, pCache, result);
if (code != 0) {
indexError("failed to merge since %s", tstrerror(code));
} else {
int64_t cost = taosGetTimestampUs() - st;
indexInfo("success to merge , time cost: %" PRId64 "ms", cost / 1000);
}
_exception:
idxDestroyFinalRslt(result);
idxCacheDestroyImm(pCache);
@ -654,8 +732,11 @@ int32_t idxFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) {
idxPost(sIdx);
}
idxReleaseRef(sIdx->refId);
if (code != 0) {
indexError("failed to merge since %s", tstrerror(code));
}
return ret;
return code;
}
void iterateValueDestroy(IterateValue* value, bool destroy) {
if (destroy) {

37
source/libs/index/src/indexCache.c
View File

@ -75,6 +75,7 @@ static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTRslt* tr, STe
if (cache == NULL) {
return 0;
}
int32_t code = 0;
MemTable* mem = cache;
IndexCache* pCache = mem->pCache;
@ -98,6 +99,10 @@ static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTRslt* tr, STe
} else if (c->operaType == DEL_VALUE) {
INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid)
}
if (code != 0) {
break;
}
} else {
break;
}
@ -105,7 +110,7 @@ static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTRslt* tr, STe
taosMemoryFree(pCt);
(void)tSkipListDestroyIter(iter);
return 0;
return code;
}
static int32_t cacheSearchPrefix(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
// impl later
@ -123,6 +128,7 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTRslt*
if (cache == NULL) {
return 0;
}
int32_t code = TSDB_CODE_SUCCESS;
MemTable* mem = cache;
IndexCache* pCache = mem->pCache;
@ -148,7 +154,7 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTRslt*
}
CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node);
terrno = TSDB_CODE_SUCCESS;
TExeCond cond = cmpFn(c->colVal, pCt->colVal, pCt->colType);
TExeCond cond = cmpFn(c->colVal, pCt->colVal, pCt->colType);
if (terrno != TSDB_CODE_SUCCESS) {
code = terrno;
goto _return;
@ -156,11 +162,14 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTRslt*
if (cond == MATCH) {
if (c->operaType == ADD_VALUE) {
INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid)
// taosArrayPush(result, &c->uid);
*s = kTypeValue;
} else if (c->operaType == DEL_VALUE) {
INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid)
}
if (code != 0) {
break;
}
} else if (cond == CONTINUE) {
continue;
} else if (cond == BREAK) {
@ -190,6 +199,8 @@ static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr
if (cache == NULL) {
return 0;
}
int32_t code = 0;
MemTable* mem = cache;
IndexCache* pCache = mem->pCache;
@ -223,6 +234,10 @@ static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr
} else if (c->operaType == DEL_VALUE) {
INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid)
}
if (code != 0) {
break;
}
} else {
break;
}
@ -231,9 +246,7 @@ static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr
taosMemoryFree(pCt);
taosMemoryFree(exBuf);
(void)tSkipListDestroyIter(iter);
return 0;
return TSDB_CODE_SUCCESS;
return code;
}
static int32_t cacheSearchSuffix_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
return TSDB_CODE_SUCCESS;
@ -338,6 +351,10 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR
} else if (c->operaType == DEL_VALUE) {
INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid)
}
if (code != 0) {
break;
}
} else if (cond == CONTINUE) {
continue;
} else if (cond == BREAK) {
@ -800,7 +817,13 @@ static bool idxCacheIteratorNext(Iterate* itera) {
iv->type = ct->operaType;
iv->ver = ct->version;
iv->colVal = taosStrdup(ct->colVal);
(void)taosArrayPush(iv->val, &ct->uid);
if (iv->colVal == NULL) {
return false;
}
if (taosArrayPush(iv->val, &ct->uid) == NULL) {
taosMemoryFree(iv->colVal);
return false;
}
}
return next;
}

75
source/libs/index/src/indexFst.c
View File

@ -62,7 +62,9 @@ void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes* nodes, bool isFinal) {
node->trans = taosArrayInit(16, sizeof(FstTransition));
FstBuilderNodeUnfinished un = {.node = node, .last = NULL};
(void)taosArrayPush(nodes->stack, &un);
if (taosArrayPush(nodes->stack, &un) == NULL) {
fstBuilderNodeDestroy(node);
}
}
FstBuilderNode* fstUnFinishedNodesPopRoot(FstUnFinishedNodes* nodes) {
FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack);
@ -120,7 +122,10 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output
FstLastTransition* trn = fstLastTransitionCreate(data[i], 0);
FstBuilderNodeUnfinished un = {.node = n, .last = trn};
(void)taosArrayPush(nodes->stack, &un);
if (taosArrayPush(nodes->stack, &un) == NULL) {
fstBuilderNodeDestroy(n);
taosMemoryFree(trn);
}
}
fstUnFinishedNodesPushEmpty(nodes, true);
}
@ -892,7 +897,9 @@ void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, Comp
return;
}
FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr};
(void)taosArrayPush(unNode->node->trans, &t);
if (taosArrayPush(unNode->node->trans, &t) == NULL) {
return;
}
fstLastTransitionDestroy(trn);
unNode->last = NULL;
return;
@ -997,7 +1004,12 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) {
uint8_t* data = fstSliceData(b, &len);
SArray* nodes = (SArray*)taosArrayInit(len, sizeof(FstNode*));
(void)taosArrayPush(nodes, &root);
if (nodes == NULL) {
return false;
}
if (taosArrayPush(nodes, &root) == NULL) {
goto _return;
}
for (uint32_t i = 0; i < len; i++) {
uint8_t inp = data[i];
Output res = 0;
@ -1009,7 +1021,9 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) {
(void)fstNodeGetTransitionAt(root, res, &trn);
tOut += trn.out;
root = fstGetNode(fst, trn.addr);
(void)taosArrayPush(nodes, &root);
if (taosArrayPush(nodes, &root) == NULL) {
goto _return;
}
}
if (!FST_NODE_IS_FINAL(root)) {
goto _return;
@ -1156,7 +1170,9 @@ bool stmStSeekMin(FStmSt* sws, FstBoundWithData* min) {
.trans = 0,
.out = {.null = false, .out = 0},
.autState = automFuncs[aut->type].start(aut)}; // auto.start callback
(void)taosArrayPush(sws->stack, &s);
if (taosArrayPush(sws->stack, &s) == NULL) {
return false;
}
return true;
}
FstSlice* key = NULL;
@ -1185,12 +1201,16 @@ bool stmStSeekMin(FStmSt* sws, FstBoundWithData* min) {
(void)fstNodeGetTransitionAt(node, res, &trn);
void* preState = autState;
autState = automFuncs[aut->type].accept(aut, preState, b);
(void)taosArrayPush(sws->inp, &b);
if (taosArrayPush(sws->inp, &b) == NULL) {
return false;
}
FstStreamState s = {.node = node, .trans = res + 1, .out = {.null = false, .out = out}, .autState = preState};
node = NULL;
(void)taosArrayPush(sws->stack, &s);
if (taosArrayPush(sws->stack, &s) == NULL) {
return false;
}
out += trn.out;
node = fstGetNode(sws->fst, trn.addr);
} else {
@ -1209,7 +1229,9 @@ bool stmStSeekMin(FStmSt* sws, FstBoundWithData* min) {
}
FstStreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState};
(void)taosArrayPush(sws->stack, &s);
if (taosArrayPush(sws->stack, &s) == NULL) {
return false;
}
taosMemoryFree(trans);
return true;
}
@ -1230,7 +1252,9 @@ bool stmStSeekMin(FStmSt* sws, FstBoundWithData* min) {
(void)fstNodeGetTransitionAt(n, trans - 1, &trn);
FstStreamState s = {
.node = fstGetNode(sws->fst, trn.addr), .trans = 0, .out = {.null = false, .out = out}, .autState = autState};
(void)taosArrayPush(sws->stack, &s);
if (taosArrayPush(sws->stack, &s) == NULL) {
return false;
}
return true;
}
return false;
@ -1273,8 +1297,14 @@ FStmStRslt* stmStNextWith(FStmSt* sws, streamCallback__fn callback) {
bool isMatch = automFuncs[aut->type].isMatch(aut, nextState);
FstNode* nextNode = fstGetNode(sws->fst, trn.addr);
(void)taosArrayPush(nodes, &nextNode);
(void)taosArrayPush(sws->inp, &(trn.inp));
if (taosArrayPush(nodes, &nextNode) == NULL) {
taosArrayDestroy(nodes);
return NULL;
}
if (taosArrayPush(sws->inp, &(trn.inp)) == NULL) {
taosArrayDestroy(nodes);
return NULL;
}
if (FST_NODE_IS_FINAL(nextNode)) {
void* eofState = automFuncs[aut->type].acceptEof(aut, nextState);
@ -1283,10 +1313,16 @@ FStmStRslt* stmStNextWith(FStmSt* sws, streamCallback__fn callback) {
}
}
FstStreamState s1 = {.node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState};
(void)taosArrayPush(sws->stack, &s1);
if (taosArrayPush(sws->stack, &s1) == NULL) {
taosArrayDestroy(nodes);
return NULL;
}
FstStreamState s2 = {.node = nextNode, .trans = 0, .out = {.null = false, .out = out}, .autState = nextState};
(void)taosArrayPush(sws->stack, &s2);
if (taosArrayPush(sws->stack, &s2) == NULL) {
taosArrayDestroy(nodes);
return NULL;
}
int32_t isz = taosArrayGetSize(sws->inp);
uint8_t* buf = (uint8_t*)taosMemoryMalloc(isz * sizeof(uint8_t));
@ -1357,11 +1393,18 @@ FStmBuilder* stmBuilderCreate(Fst* fst, FAutoCtx* aut) {
b->aut = aut;
b->min = fstBoundStateCreate(Unbounded, NULL);
b->max = fstBoundStateCreate(Unbounded, NULL);
if (b->min == NULL || b->max == NULL) {
stmBuilderDestroy(b);
return NULL;
}
return b;
}
void stmBuilderDestroy(FStmBuilder* b) {
fstSliceDestroy(&b->min->data);
fstSliceDestroy(&b->max->data);
if (b->min) fstSliceDestroy(&b->min->data);
if (b->max) fstSliceDestroy(&b->max->data);
taosMemoryFreeClear(b->min);
taosMemoryFreeClear(b->max);
taosMemoryFree(b);

61
source/libs/index/src/indexFstDfa.c
View File

@ -14,6 +14,7 @@
*/
#include "indexFstDfa.h"
#include "indexInt.h"
#include "thash.h"
const static uint32_t STATE_LIMIT = 1000;
@ -68,23 +69,41 @@ FstDfa *dfaBuilderBuild(FstDfaBuilder *builder) {
uint32_t sz = taosArrayGetSize(builder->dfa->insts);
FstSparseSet *cur = sparSetCreate(sz);
FstSparseSet *nxt = sparSetCreate(sz);
if (cur == NULL || nxt == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
dfaAdd(builder->dfa, cur, 0);
uint32_t result;
SArray *states = taosArrayInit(0, sizeof(uint32_t));
if (states == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
if (dfaBuilderCacheState(builder, cur, &result)) {
(void)taosArrayPush(states, &result);
if (taosArrayPush(states, &result) == NULL) {
goto _exception;
}
}
SHashObj *seen = taosHashInit(12, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
if (seen == NULL) {
goto _exception;
}
while (taosArrayGetSize(states) != 0) {
result = *(uint32_t *)taosArrayPop(states);
for (int i = 0; i < 256; i++) {
uint32_t ns, dummpy = 0;
if (dfaBuilderRunState(builder, cur, nxt, result, i, &ns)) {
if (taosHashGet(seen, &ns, sizeof(ns)) == NULL) {
(void)taosHashPut(seen, &ns, sizeof(ns), &dummpy, sizeof(dummpy));
(void)taosArrayPush(states, &ns);
if (taosHashPut(seen, &ns, sizeof(ns), &dummpy, sizeof(dummpy)) != 0) {
goto _exception;
}
if (taosArrayPush(states, &ns) == NULL) {
goto _exception;
}
}
}
if (taosArrayGetSize(builder->dfa->states) > STATE_LIMIT) {
@ -96,6 +115,11 @@ FstDfa *dfaBuilderBuild(FstDfaBuilder *builder) {
taosArrayDestroy(states);
taosHashCleanup(seen);
return builder->dfa;
_exception:
taosArrayDestroy(states);
taosHashCleanup(seen);
indexError("failed to build dfa since %s", tstrerror(terrno));
return NULL;
}
bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet *next, uint32_t state, uint8_t byte,
@ -122,8 +146,13 @@ bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet
}
bool dfaBuilderCacheState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *result) {
int32_t code = 0;
SArray *tinsts = taosArrayInit(4, sizeof(uint32_t));
bool isMatch = false;
if (tinsts == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exception;
}
bool isMatch = false;
for (int i = 0; i < sparSetLen(set); i++) {
int32_t ip;
@ -133,10 +162,16 @@ bool dfaBuilderCacheState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *r
if (inst->ty == JUMP || inst->ty == SPLIT) {
continue;
} else if (inst->ty == RANGE) {
(void)taosArrayPush(tinsts, &ip);
if (taosArrayPush(tinsts, &ip) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exception;
}
} else if (inst->ty == MATCH) {
isMatch = true;
(void)taosArrayPush(tinsts, &ip);
if (taosArrayPush(tinsts, &ip) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exception;
}
}
}
if (taosArrayGetSize(tinsts) == 0) {
@ -149,13 +184,23 @@ bool dfaBuilderCacheState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *r
taosArrayDestroy(tinsts);
} else {
DfaState st = {.insts = tinsts, .isMatch = isMatch};
(void)taosArrayPush(builder->dfa->states, &st);
if (taosArrayPush(builder->dfa->states, &st) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exception;
}
int32_t sz = taosArrayGetSize(builder->dfa->states) - 1;
(void)taosHashPut(builder->cache, &tinsts, sizeof(POINTER_BYTES), &sz, sizeof(sz));
if ((code = taosHashPut(builder->cache, &tinsts, sizeof(POINTER_BYTES), &sz, sizeof(sz))) != 0) {
goto _exception;
}
*result = sz;
}
return true;
_exception:
indexError("failed to create dfa state, code:%d", code);
taosArrayDestroy(tinsts);
return false;
}
FstDfa *dfaCreate(SArray *insts, SArray *states) {

38
source/libs/index/src/indexFstNode.c
View File

@ -16,9 +16,14 @@
FstBuilderNode* fstBuilderNodeDefault() {
FstBuilderNode* bn = taosMemoryMalloc(sizeof(FstBuilderNode));
if (bn == NULL) return NULL;
bn->isFinal = false;
bn->finalOutput = 0;
bn->trans = taosArrayInit(16, sizeof(FstTransition));
if (bn->trans == NULL) {
taosMemoryFree(bn);
return NULL;
}
return bn;
}
void fstBuilderNodeDestroy(FstBuilderNode* node) {
@ -56,30 +61,11 @@ bool fstBuilderNodeEqual(FstBuilderNode* n1, FstBuilderNode* n2) {
return true;
}
FstBuilderNode* fstBuilderNodeClone(FstBuilderNode* src) {
FstBuilderNode* node = taosMemoryMalloc(sizeof(FstBuilderNode));
if (node == NULL) {
return NULL;
}
//
size_t sz = taosArrayGetSize(src->trans);
SArray* trans = taosArrayInit(sz, sizeof(FstTransition));
for (size_t i = 0; i < sz; i++) {
FstTransition* tran = taosArrayGet(src->trans, i);
(void)taosArrayPush(trans, tran);
}
node->trans = trans;
node->isFinal = src->isFinal;
node->finalOutput = src->finalOutput;
return node;
}
// not destroy src, User's bussiness
void fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src) {
int32_t fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src) {
if (dst == NULL || src == NULL) {
return;
return TSDB_CODE_INVALID_PARA;
}
dst->isFinal = src->isFinal;
@ -89,10 +75,18 @@ void fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src) {
taosArrayDestroy(dst->trans);
size_t sz = taosArrayGetSize(src->trans);
dst->trans = taosArrayInit(sz, sizeof(FstTransition));
if (dst->trans == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (size_t i = 0; i < sz; i++) {
FstTransition* trn = taosArrayGet(src->trans, i);
(void)taosArrayPush(dst->trans, trn);
if (taosArrayPush(dst->trans, trn) == NULL) {
taosArrayDestroy(dst->trans);
dst->trans = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
}
}
return 0;
}
// bool fstBuilderNodeCompileTo(FstBuilderNode *b, IdxFile *wrt, CompiledAddr lastAddr, CompiledAddr

7
source/libs/index/src/indexFstRegex.c
View File

@ -39,7 +39,12 @@ FstRegex *regexCreate(const char *str) {
for (int i = 0; i < strlen(str); i++) {
uint8_t v = str[i];
(void)taosArrayPush(insts, &v);
if (taosArrayPush(insts, &v) == NULL) {
taosArrayDestroy(insts);
taosMemoryFree(regex->orig);
taosMemoryFree(regex);
return NULL;
}
}
FstDfaBuilder *builder = dfaBuilderCreate(insts);
regex->dfa = dfaBuilderBuild(builder);

11
source/libs/index/src/indexFstRegister.c
View File

@ -84,7 +84,10 @@ FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) {
for (uint64_t i = 0; i < nCells; i++) {
FstRegistryCell cell = {.addr = NONE_ADDRESS, .node = fstBuilderNodeDefault()};
(void)taosArrayPush(tb, &cell);
if (taosArrayPush(tb, &cell) == NULL) {
fstRegistryDestroy(registry);
return NULL;
}
}
registry->table = tb;
@ -125,7 +128,7 @@ FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNo
entry->addr = cell->addr;
return entry;
} else {
fstBuilderNodeCloneFrom(cell->node, bNode);
(void)fstBuilderNodeCloneFrom(cell->node, bNode);
entry->state = NOTFOUND;
entry->cell = cell; // copy or not
}
@ -145,7 +148,7 @@ FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNo
return entry;
}
// clone from bNode, refactor later
fstBuilderNodeCloneFrom(cell2->node, bNode);
(void)fstBuilderNodeCloneFrom(cell2->node, bNode);
fstRegistryCellSwap(registry->table, start, start + 1);
FstRegistryCell* cCell = taosArrayGet(registry->table, start);
@ -166,7 +169,7 @@ FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNo
uint64_t last = end - 1;
FstRegistryCell* cell = (FstRegistryCell*)taosArrayGet(registry->table, last);
// clone from bNode, refactor later
fstBuilderNodeCloneFrom(cell->node, bNode);
(void)fstBuilderNodeCloneFrom(cell->node, bNode);
fstRegistryCellPromote(registry->table, last, start);
FstRegistryCell* cCell = taosArrayGet(registry->table, start);

129
source/libs/index/src/indexTfile.c
View File

@ -49,7 +49,7 @@ static int tfileReaderLoadFst(TFileReader* reader);
static int tfileReaderVerify(TFileReader* reader);
static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result);
static SArray* tfileGetFileList(const char* path);
static int32_t tfileGetFileList(const char* path, SArray** pResult);
static int tfileRmExpireFile(SArray* result);
static void tfileDestroyFileName(void* elem);
static int tfileCompare(const void* a, const void* b);
@ -97,9 +97,15 @@ TFileCache* tfileCacheCreate(SIndex* idx, const char* path) {
}
tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (tcache->tableCache == NULL) {
indexError("failed to open table cache since%s", tstrerror(terrno));
goto End;
}
tcache->capacity = 64;
SArray* files = tfileGetFileList(path);
SArray* files = NULL;
int32_t code = tfileGetFileList(path, &files);
for (size_t i = 0; i < taosArrayGetSize(files); i++) {
char* file = taosArrayGetP(files, i);
@ -125,7 +131,11 @@ TFileCache* tfileCacheCreate(SIndex* idx, const char* path) {
char buf[128] = {0};
int32_t sz = idxSerialCacheKey(&key, buf);
(void)taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*));
code = taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*));
if (code != 0) {
tfileReaderDestroy(reader);
goto End;
}
tfileReaderRef(reader);
}
taosArrayDestroyEx(files, tfileDestroyFileName);
@ -163,6 +173,7 @@ TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) {
return *reader;
}
int32_t tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) {
int32_t code = 0;
@ -172,16 +183,18 @@ int32_t tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) {
TFileReader** p = taosHashGet(tcache->tableCache, buf, sz);
if (p != NULL && *p != NULL) {
TFileReader* oldRdr = *p;
(void)taosHashRemove(tcache->tableCache, buf, sz);
indexInfo("found %s, should remove file %s", buf, oldRdr->ctx->file.buf);
oldRdr->remove = true;
tfileReaderUnRef(oldRdr);
if ((code = taosHashRemove(tcache->tableCache, buf, sz)) != 0) {
indexError("failed to remove old reader from cache since %s, suid:%" PRIu64 ", colName:%s", tstrerror(code),
oldRdr->header.suid, oldRdr->header.colName);
} else {
indexInfo("found %s, should remove file %s", buf, oldRdr->ctx->file.buf);
oldRdr->remove = true;
tfileReaderUnRef(oldRdr);
}
}
code = taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*));
if (code == 0) {
tfileReaderRef(reader);
}
tfileReaderRef(reader);
return code;
}
int32_t tfileReaderCreate(IFileCtx* ctx, TFileReader** pReader) {
@ -232,7 +245,7 @@ void tfileReaderDestroy(TFileReader* reader) {
}
static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
int ret = 0;
int32_t ret = 0;
char* p = tem->colVal;
uint64_t sz = tem->nColVal;
@ -246,6 +259,11 @@ static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
tem->suid, tem->colName, tem->colVal, cost);
ret = tfileReaderLoadTableIds((TFileReader*)reader, (int32_t)offset, tr->total);
if (ret != 0) {
fstSliceDestroy(&key);
indexError("faile to search since %s", tstrerror(ret));
return ret;
}
cost = taosGetTimestampUs() - et;
indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", tem->suid,
tem->colName, tem->colVal, cost);
@ -255,17 +273,29 @@ static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
}
static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
int32_t lino = 0;
int32_t code = 0;
char* p = tem->colVal;
uint64_t sz = tem->nColVal;
SArray* offsets = taosArrayInit(16, sizeof(uint64_t));
if (offsets == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
FAutoCtx* ctx = automCtxCreate((void*)p, AUTOMATION_PREFIX);
if (ctx == NULL) {
taosArrayDestroy(offsets);
return TSDB_CODE_OUT_OF_MEMORY;
}
FAutoCtx* ctx = automCtxCreate((void*)p, AUTOMATION_PREFIX);
FStmBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx);
FStmSt* st = stmBuilderIntoStm(sb);
FStmStRslt* rt = NULL;
while ((rt = stmStNextWith(st, NULL)) != NULL) {
(void)taosArrayPush(offsets, &(rt->out.out));
if (taosArrayPush(offsets, &(rt->out.out)) == NULL) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exception);
}
swsResultDestroy(rt);
}
stmStDestroy(st);
@ -275,14 +305,16 @@ static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
for (int i = 0; i < taosArrayGetSize(offsets); i++) {
uint64_t offset = *(uint64_t*)taosArrayGet(offsets, i);
ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total);
if (ret != 0) {
taosArrayDestroy(offsets);
indexError("failed to find target tablelist");
return TSDB_CODE_FILE_CORRUPTED;
}
TAOS_CHECK_GOTO(ret, &lino, _exception);
}
taosArrayDestroy(offsets);
return 0;
_exception:
stmStDestroy(st);
stmBuilderDestroy(sb);
taosArrayDestroy(offsets);
indexError("failed to searchPrefix since %s, lino:%d", tstrerror(code), lino);
return code;
}
static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
int ret = 0;
@ -393,6 +425,12 @@ static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
tem->suid, tem->colName, tem->colVal, cost);
ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total);
if (ret != 0) {
indexError("failed to search json since %s", tstrerror(ret));
taosMemoryFree(p);
fstSliceDestroy(&key);
return ret;
}
cost = taosGetTimestampUs() - et;
indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, offset: %" PRIu64
", size: %d, time cost: %" PRIu64 "us",
@ -863,14 +901,24 @@ TFileValue* tfileValueCreate(char* val) {
return NULL;
}
tf->colVal = taosStrdup(val);
if (tf->colVal == NULL) {
taosMemoryFree(tf);
}
tf->tableId = taosArrayInit(32, sizeof(uint64_t));
if (tf->tableId == NULL) {
taosMemoryFree(tf->colVal);
taosMemoryFree(tf);
return NULL;
}
return tf;
}
int tfileValuePush(TFileValue* tf, uint64_t val) {
int32_t tfileValuePush(TFileValue* tf, uint64_t val) {
if (tf == NULL) {
return -1;
return TSDB_CODE_INVALID_PARA;
}
if (taosArrayPush(tf->tableId, &val) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)taosArrayPush(tf->tableId, &val);
return 0;
}
void tfileValueDestroy(TFileValue* tf) {
@ -986,8 +1034,10 @@ static int tfileReaderLoadFst(TFileReader* reader) {
return reader->fst != NULL ? 0 : -1;
}
static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result) {
static int32_t tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result) {
// TODO(yihao): opt later
int32_t code = 0;
int32_t lino = 0;
IFileCtx* ctx = reader->ctx;
// add block cache
char block[4096] = {0};
@ -1003,7 +1053,9 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray*
while (nid > 0) {
int32_t left = block + sizeof(block) - p;
if (left >= sizeof(uint64_t)) {
(void)taosArrayPush(result, (uint64_t*)p);
if (taosArrayPush(result, (uint64_t*)p) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
p += sizeof(uint64_t);
} else {
char buf[sizeof(uint64_t)] = {0};
@ -1014,7 +1066,9 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray*
nread = ctx->readFrom(ctx, (uint8_t*)block, sizeof(block), offset);
memcpy(buf + left, block, sizeof(uint64_t) - left);
(void)taosArrayPush(result, (uint64_t*)buf);
if (taosArrayPush(result, (uint64_t*)buf) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
p = block + sizeof(uint64_t) - left;
}
nid -= 1;
@ -1059,16 +1113,19 @@ void tfileReaderUnRef(TFileReader* rd) {
}
}
static SArray* tfileGetFileList(const char* path) {
static int32_t tfileGetFileList(const char* path, SArray** ppResult) {
int32_t code = 0;
char buf[128] = {0};
uint64_t suid;
int64_t version;
SArray* files = taosArrayInit(4, sizeof(void*));
if (files == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
TdDirPtr pDir = taosOpenDir(path);
if (NULL == pDir) {
taosArrayDestroy(files);
return NULL;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _exception);
}
TdDirEntryPtr pDirEntry;
while ((pDirEntry = taosReadDir(pDir)) != NULL) {
@ -1079,15 +1136,29 @@ static SArray* tfileGetFileList(const char* path) {
size_t len = strlen(path) + 1 + strlen(file) + 1;
char* buf = taosMemoryCalloc(1, len);
if (buf == NULL) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception);
}
sprintf(buf, "%s/%s", path, file);
(void)taosArrayPush(files, &buf);
if (taosArrayPush(files, &buf) == NULL) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception);
}
}
(void)taosCloseDir(&pDir);
taosArraySort(files, tfileCompare);
(void)tfileRmExpireFile(files);
*ppResult = files;
return 0;
return files;
_exception:
(void)taosCloseDir(&pDir);
if (files != NULL) {
taosArrayDestroyEx(files, tfileDestroyFileName);
taosArrayDestroy(files);
}
return code;
}
static int tfileRmExpireFile(SArray* result) {
// TODO(yihao): remove expire tindex after restart

64
source/libs/index/src/indexUtil.c
View File

@ -36,12 +36,16 @@ static FORCE_INLINE int iBinarySearch(SArray *arr, int s, int e, uint64_t k) {
return s;
}
void iIntersection(SArray *in, SArray *out) {
int32_t iIntersection(SArray *in, SArray *out) {
int32_t code = 0;
int32_t sz = (int32_t)taosArrayGetSize(in);
if (sz <= 0) {
return;
return 0;
}
MergeIndex *mi = taosMemoryCalloc(sz, sizeof(MergeIndex));
if (mi == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int i = 0; i < sz; i++) {
SArray *t = taosArrayGetP(in, i);
mi[i].len = (int32_t)taosArrayGetSize(t);
@ -64,19 +68,25 @@ void iIntersection(SArray *in, SArray *out) {
}
}
if (has == true) {
(void)taosArrayPush(out, &tgt);
if (taosArrayPush(out, &tgt) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
break;
}
}
}
taosMemoryFreeClear(mi);
return code;
}
void iUnion(SArray *in, SArray *out) {
int32_t iUnion(SArray *in, SArray *out) {
int32_t code = 0;
int32_t sz = (int32_t)taosArrayGetSize(in);
if (sz <= 0) {
return;
return 0;
}
if (sz == 1) {
(void)taosArrayAddAll(out, taosArrayGetP(in, 0));
return;
if (taosArrayAddAll(out, taosArrayGetP(in, 0)) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
MergeIndex *mi = taosMemoryCalloc(sz, sizeof(MergeIndex));
@ -108,19 +118,23 @@ void iUnion(SArray *in, SArray *out) {
continue;
}
}
(void)taosArrayPush(out, &mVal);
if (taosArrayPush(out, &mVal) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
break;
}
} else {
break;
}
}
taosMemoryFreeClear(mi);
return 0;
}
void iExcept(SArray *total, SArray *except) {
int32_t iExcept(SArray *total, SArray *except) {
int32_t tsz = (int32_t)taosArrayGetSize(total);
int32_t esz = (int32_t)taosArrayGetSize(except);
if (esz == 0 || tsz == 0) {
return;
return 0;
}
int vIdx = 0;
@ -135,6 +149,7 @@ void iExcept(SArray *total, SArray *except) {
}
taosArrayPopTailBatch(total, tsz - vIdx);
return 0;
}
int uidCompare(const void *a, const void *b) {
@ -191,20 +206,37 @@ void idxTRsltDestroy(SIdxTRslt *tr) {
taosArrayDestroy(tr->del);
taosMemoryFree(tr);
}
void idxTRsltMergeTo(SIdxTRslt *tr, SArray *result) {
int32_t idxTRsltMergeTo(SIdxTRslt *tr, SArray *result) {
int32_t code = 0;
taosArraySort(tr->total, uidCompare);
taosArraySort(tr->add, uidCompare);
taosArraySort(tr->del, uidCompare);
if (taosArrayGetSize(tr->total) == 0 || taosArrayGetSize(tr->add) == 0) {
SArray *t = taosArrayGetSize(tr->total) == 0 ? tr->add : tr->total;
(void)taosArrayAddAll(result, t);
if (taosArrayAddAll(result, t) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} else {
SArray *arrs = taosArrayInit(2, sizeof(void *));
(void)taosArrayPush(arrs, &tr->total);
(void)taosArrayPush(arrs, &tr->add);
iUnion(arrs, result);
if (arrs == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (taosArrayPush(arrs, &tr->total) == NULL) {
taosArrayDestroy(arrs);
return TSDB_CODE_OUT_OF_MEMORY;
}
if (taosArrayPush(arrs, &tr->add) == NULL) {
taosArrayDestroy(arrs);
return TSDB_CODE_OUT_OF_MEMORY;
}
code = iUnion(arrs, result);
taosArrayDestroy(arrs);
}
iExcept(result, tr->del);
if (code == 0) {
code = iExcept(result, tr->del);
}
return code;
}

3
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -103,6 +103,7 @@ static int32_t exprNodeCopy(const SExprNode* pSrc, SExprNode* pDst) {
COPY_CHAR_ARRAY_FIELD(aliasName);
COPY_CHAR_ARRAY_FIELD(userAlias);
COPY_SCALAR_FIELD(orderAlias);
COPY_SCALAR_FIELD(projIdx);
return TSDB_CODE_SUCCESS;
}
@ -124,6 +125,8 @@ static int32_t columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) {
COPY_SCALAR_FIELD(tableHasPk);
COPY_SCALAR_FIELD(isPk);
COPY_SCALAR_FIELD(numOfPKs);
COPY_SCALAR_FIELD(projRefIdx);
COPY_SCALAR_FIELD(resIdx);
return TSDB_CODE_SUCCESS;
}

10
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -2263,8 +2263,13 @@ static EDealRes doCollect(SCollectColumnsCxt* pCxt, SColumnNode* pCol, SNode* pN
} else {
len = snprintf(name, sizeof(name), "%s.%s", pCol->tableAlias, pCol->colName);
}
if (NULL == taosHashGet(pCxt->pColHash, name, len)) {
pCxt->errCode = taosHashPut(pCxt->pColHash, name, len, NULL, 0);
if (pCol->projRefIdx > 0) {
len = taosHashBinary(name, strlen(name));
len += sprintf(name + len, "_%d", pCol->projRefIdx);
}
SNode** pNodeFound = taosHashGet(pCxt->pColHash, name, len);
if (pNodeFound == NULL) {
pCxt->errCode = taosHashPut(pCxt->pColHash, name, len, &pNode, POINTER_BYTES);
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
SNode* pNew = NULL;
pCxt->errCode = nodesCloneNode(pNode, &pNew);
@ -2307,7 +2312,6 @@ static EDealRes collectColumnsExt(SNode* pNode, void* pContext) {
return DEAL_RES_CONTINUE;
}
int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type,
SNodeList** pCols) {
if (NULL == pSelect || NULL == pCols) {

937
source/libs/parser/src/parAstCreater.c
View File

File diff suppressed because it is too large Load Diff

47
source/libs/parser/src/parAstParser.c
View File

@ -107,6 +107,7 @@ typedef struct SCollectMetaKeyCxt {
typedef struct SCollectMetaKeyFromExprCxt {
SCollectMetaKeyCxt* pComCxt;
bool hasLastRowOrLast;
bool tbnameCollect;
int32_t errCode;
} SCollectMetaKeyFromExprCxt;
@ -204,6 +205,45 @@ static EDealRes collectMetaKeyFromTempTable(SCollectMetaKeyFromExprCxt* pCxt, ST
return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR;
}
static int32_t isTbnameEqCondOperator(SOperatorNode* pOperator, char** ppTableName) {
if (pOperator->opType != OP_TYPE_EQUAL) {
return TSDB_CODE_SUCCESS;
}
SValueNode* pValueNode = NULL;
if (nodeType(pOperator->pLeft) == QUERY_NODE_FUNCTION &&
0 == strcasecmp(((SFunctionNode*)(pOperator->pLeft))->functionName, "tbname") &&
nodeType(pOperator->pRight) == QUERY_NODE_VALUE) {
pValueNode = (SValueNode*)pOperator->pRight;
} else if (nodeType(pOperator->pRight) == QUERY_NODE_FUNCTION &&
0 == strcasecmp(((SFunctionNode*)(pOperator->pRight))->functionName, "tbname") &&
nodeType(pOperator->pLeft) == QUERY_NODE_VALUE) {
pValueNode = (SValueNode*)pOperator->pLeft;
} else {
return TSDB_CODE_SUCCESS;
}
*ppTableName = pValueNode->literal;
return TSDB_CODE_SUCCESS;
}
static EDealRes collectMetaKeyFromOperator(SCollectMetaKeyFromExprCxt* pCxt, SOperatorNode* pOpNode) {
if (!pCxt->tbnameCollect) {
return DEAL_RES_CONTINUE;
}
char* pTableName = NULL;
int32_t code = isTbnameEqCondOperator((SOperatorNode*)pOpNode, &pTableName);
if (TSDB_CODE_SUCCESS != code) return DEAL_RES_CONTINUE;
if (pTableName) {
SSelectStmt* pSelect = (SSelectStmt*)pCxt->pComCxt->pStmt;
pCxt->errCode = collectMetaKeyFromRealTableImpl(pCxt->pComCxt, ((SRealTableNode*)pSelect->pFromTable)->table.dbName, pTableName, AUTH_TYPE_READ);
}
return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR;
}
static EDealRes collectMetaKeyFromExprImpl(SNode* pNode, void* pContext) {
SCollectMetaKeyFromExprCxt* pCxt = pContext;
switch (nodeType(pNode)) {
@ -213,6 +253,8 @@ static EDealRes collectMetaKeyFromExprImpl(SNode* pNode, void* pContext) {
return collectMetaKeyFromRealTable(pCxt, (SRealTableNode*)pNode);
case QUERY_NODE_TEMP_TABLE:
return collectMetaKeyFromTempTable(pCxt, (STempTableNode*)pNode);
case QUERY_NODE_OPERATOR:
return collectMetaKeyFromOperator(pCxt, (SOperatorNode*)pNode);
default:
break;
}
@ -220,7 +262,7 @@ static EDealRes collectMetaKeyFromExprImpl(SNode* pNode, void* pContext) {
}
static int32_t collectMetaKeyFromExprs(SCollectMetaKeyCxt* pCxt, SNodeList* pList) {
SCollectMetaKeyFromExprCxt cxt = {.pComCxt = pCxt, .errCode = TSDB_CODE_SUCCESS};
SCollectMetaKeyFromExprCxt cxt = {.pComCxt = pCxt, .errCode = TSDB_CODE_SUCCESS, .tbnameCollect = false};
nodesWalkExprs(pList, collectMetaKeyFromExprImpl, &cxt);
return cxt.errCode;
}
@ -245,6 +287,9 @@ static int32_t reserveDbCfgForLastRow(SCollectMetaKeyCxt* pCxt, SNode* pTable) {
static int32_t collectMetaKeyFromSelect(SCollectMetaKeyCxt* pCxt, SSelectStmt* pStmt) {
SCollectMetaKeyFromExprCxt cxt = {.pComCxt = pCxt, .hasLastRowOrLast = false, .errCode = TSDB_CODE_SUCCESS};
if (pStmt->pFromTable && QUERY_NODE_REAL_TABLE == nodeType(pStmt->pFromTable)) {
cxt.tbnameCollect = true;
}
nodesWalkSelectStmt(pStmt, SQL_CLAUSE_FROM, collectMetaKeyFromExprImpl, &cxt);
if (TSDB_CODE_SUCCESS == cxt.errCode && cxt.hasLastRowOrLast) {
cxt.errCode = reserveDbCfgForLastRow(pCxt, pStmt->pFromTable);

2
source/libs/parser/src/parInsertSql.c
View File

@ -2254,7 +2254,7 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpSt
static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pFilePath,
SRowsDataContext rowsDataCxt) {
char filePathStr[TSDB_FILENAME_LEN] = {0};
char filePathStr[PATH_MAX] = {0};
if (TK_NK_STRING == pFilePath->type) {
(void)trimString(pFilePath->z, pFilePath->n, filePathStr, sizeof(filePathStr));
} else {

98
source/libs/parser/src/parTranslater.c
View File

@ -1270,12 +1270,12 @@ static void setColumnPrimTs(STranslateContext* pCxt, SColumnNode* pCol, const ST
}
}
static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* pTable, bool igTags, SNodeList* pList) {
static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* pTable, bool igTags, SNodeList* pList, bool skipProjRef) {
int32_t code = 0;
if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) {
const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta;
int32_t nums = pMeta->tableInfo.numOfColumns +
(igTags ? 0 : ((TSDB_SUPER_TABLE == pMeta->tableType) ? pMeta->tableInfo.numOfTags : 0));
(igTags ? 0 : ((TSDB_SUPER_TABLE == pMeta->tableType || ((SRealTableNode*)pTable)->stbRewrite) ? pMeta->tableInfo.numOfTags : 0));
for (int32_t i = 0; i < nums; ++i) {
if (invisibleColumn(pCxt->pParseCxt->enableSysInfo, pMeta->tableType, pMeta->schema[i].flags)) {
pCxt->pParseCxt->hasInvisibleCol = true;
@ -1305,7 +1305,11 @@ static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* p
SListCell* pCell = nodesListGetCell(pList, LIST_LENGTH(pList) - 1);
code = setColumnInfoByExpr(pTempTable, (SExprNode*)pNode, (SColumnNode**)&pCell->pNode);
}
if (TSDB_CODE_SUCCESS != code) break;
if (TSDB_CODE_SUCCESS == code) {
if (!skipProjRef) pCol->projRefIdx = ((SExprNode*)pNode)->projIdx; // only set proj ref when select * from (select ...)
} else {
break;
}
}
}
return code;
@ -4770,7 +4774,7 @@ static int32_t createAllColumns(STranslateContext* pCxt, bool igTags, SNodeList*
size_t nums = taosArrayGetSize(pTables);
for (size_t i = 0; i < nums; ++i) {
STableNode* pTable = taosArrayGetP(pTables, i);
int32_t code = createColumnsByTable(pCxt, pTable, igTags, *pCols);
int32_t code = createColumnsByTable(pCxt, pTable, igTags, *pCols, nums > 1);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
@ -4833,7 +4837,7 @@ static int32_t createTableAllCols(STranslateContext* pCxt, SColumnNode* pCol, bo
}
}
if (TSDB_CODE_SUCCESS == code) {
code = createColumnsByTable(pCxt, pTable, igTags, *pOutput);
code = createColumnsByTable(pCxt, pTable, igTags, *pOutput, false);
}
return code;
}
@ -5175,6 +5179,11 @@ static int32_t translateProjectionList(STranslateContext* pCxt, SSelectStmt* pSe
if (!pSelect->isSubquery) {
return rewriteProjectAlias(pSelect->pProjectionList);
} else {
SNode* pNode;
int32_t projIdx = 1;
FOREACH(pNode, pSelect->pProjectionList) {
((SExprNode*)pNode)->projIdx = projIdx++;
}
return TSDB_CODE_SUCCESS;
}
}
@ -5976,6 +5985,7 @@ static int32_t isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pO
*pRet = false;
return TSDB_CODE_SUCCESS;
}
SFunctionNode* pTbnameFunc = NULL;
SValueNode* pValueNode = NULL;
if (nodeType(pOperator->pLeft) == QUERY_NODE_FUNCTION &&
@ -6053,7 +6063,6 @@ static int32_t isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pO
static int32_t findEqCondTbNameInOperatorNode(STranslateContext* pCxt, SNode* pWhere, SEqCondTbNameTableInfo* pInfo, bool* pRet) {
int32_t code = TSDB_CODE_SUCCESS;
char* pTableAlias = NULL;
char* pTbNameVal = NULL;
bool eqTbnameCond = false, tbnameInCond = false;
code = isOperatorEqTbnameCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, &pInfo->aTbnames, &eqTbnameCond);
if (TSDB_CODE_SUCCESS == code) {
@ -6234,12 +6243,73 @@ static void findVgroupsFromEqualTbname(STranslateContext* pCxt, SArray* aTbnames
}
}
static int32_t replaceToChildTableQuery(STranslateContext* pCxt, SEqCondTbNameTableInfo* pInfo) {
SName snameTb;
int32_t code = 0;
SRealTableNode* pRealTable = pInfo->pRealTable;
char* tbName = taosArrayGetP(pInfo->aTbnames, 0);
(void)toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, tbName, &snameTb);
STableMeta* pMeta = NULL;
TAOS_CHECK_RETURN(catalogGetCachedTableMeta(pCxt->pParseCxt->pCatalog, &snameTb, &pMeta));
if (NULL == pMeta || TSDB_CHILD_TABLE != pMeta->tableType || pMeta->suid != pRealTable->pMeta->suid) {
goto _return;
}
pRealTable->pMeta->uid = pMeta->uid;
pRealTable->pMeta->vgId = pMeta->vgId;
pRealTable->pMeta->tableType = pMeta->tableType;
tstrncpy(pRealTable->table.tableName, tbName, sizeof(pRealTable->table.tableName));
pRealTable->stbRewrite = true;
if (pRealTable->pTsmas) {
// if select from a child table, fetch it's corresponding tsma target child table infos
char buf[TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN + 1];
for (int32_t i = 0; i < pRealTable->pTsmas->size; ++i) {
STableTSMAInfo* pTsma = taosArrayGetP(pRealTable->pTsmas, i);
SName tsmaTargetTbName = {0};
(void)toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetTbName);
int32_t len = snprintf(buf, TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN, "%s.%s_%s", pTsma->dbFName, pTsma->name,
pRealTable->table.tableName);
len = taosCreateMD5Hash(buf, len);
strncpy(tsmaTargetTbName.tname, buf, MD5_OUTPUT_LEN);
STsmaTargetTbInfo ctbInfo = {0};
if (!pRealTable->tsmaTargetTbInfo) {
pRealTable->tsmaTargetTbInfo = taosArrayInit(pRealTable->pTsmas->size, sizeof(STsmaTargetTbInfo));
if (!pRealTable->tsmaTargetTbInfo) {
code = terrno;
break;
}
}
sprintf(ctbInfo.tableName, "%s", tsmaTargetTbName.tname);
ctbInfo.uid = pMeta->uid;
if (NULL == taosArrayPush(pRealTable->tsmaTargetTbInfo, &ctbInfo)) {
code = terrno;
goto _return;
}
}
}
_return:
taosMemoryFree(pMeta);
return code;
}
static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt* pSelect, SArray* aTables) {
int32_t code = TSDB_CODE_SUCCESS;
for (int i = 0; i < taosArrayGetSize(aTables); ++i) {
SEqCondTbNameTableInfo* pInfo = taosArrayGet(aTables, i);
int32_t nTbls = taosArrayGetSize(pInfo->aTbnames);
int32_t aTableNum = taosArrayGetSize(aTables);
int32_t nTbls = 0;
bool stableQuery = false;
SEqCondTbNameTableInfo* pInfo = NULL;
qDebug("start to update stable vg for tbname optimize, aTableNum:%d", aTableNum);
for (int i = 0; i < aTableNum; ++i) {
pInfo = taosArrayGet(aTables, i);
int32_t numOfVgs = pInfo->pRealTable->pVgroupList->numOfVgroups;
nTbls = taosArrayGetSize(pInfo->aTbnames);
SVgroupsInfo* vgsInfo = taosMemoryMalloc(sizeof(SVgroupsInfo) + nTbls * sizeof(SVgroupInfo));
findVgroupsFromEqualTbname(pCxt, pInfo->aTbnames, pInfo->pRealTable->table.dbName, numOfVgs, vgsInfo);
@ -6249,6 +6319,7 @@ static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt*
} else {
taosMemoryFree(vgsInfo);
}
stableQuery = pInfo->pRealTable->pMeta->tableType == TSDB_SUPER_TABLE;
vgsInfo = NULL;
if (pInfo->pRealTable->pTsmas) {
@ -6288,7 +6359,14 @@ static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt*
}
}
}
return TSDB_CODE_SUCCESS;
qDebug("before ctbname optimize, code:%d, aTableNum:%d, nTbls:%d, stableQuery:%d", code, aTableNum, nTbls, stableQuery);
if (TSDB_CODE_SUCCESS == code && 1 == aTableNum && 1 == nTbls && stableQuery && NULL == pInfo->pRealTable->pTsmas) {
code = replaceToChildTableQuery(pCxt, pInfo);
}
return code;
}
static int32_t setTableVgroupsFromEqualTbnameCond(STranslateContext* pCxt, SSelectStmt* pSelect) {

2
source/libs/planner/inc/planInt.h
View File

@ -68,6 +68,8 @@ int32_t tagScanSetExecutionMode(SScanLogicNode* pScan);
int32_t cloneLimit(SLogicNode* pParent, SLogicNode* pChild, uint8_t cloneWhat, bool* pCloned);
int32_t sortPriKeyOptGetSequencingNodesImpl(SLogicNode* pNode, bool groupSort, SSortLogicNode* pSort,
bool* pNotOptimize, SNodeList** pSequencingNodes, bool* keepSort);
bool isColRefExpr(const SColumnNode* pCol, const SExprNode* pExpr);
void rewriteTargetsWithResId(SNodeList* pTargets);
#ifdef __cplusplus

16
source/libs/planner/src/planLogicCreater.c
View File

@ -122,6 +122,7 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) {
strcpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName);
strcpy(pCol->node.userAlias, ((SExprNode*)pExpr)->userAlias);
strcpy(pCol->colName, ((SExprNode*)pExpr)->aliasName);
pCol->node.projIdx = ((SExprNode*)(*pNode))->projIdx;
if (QUERY_NODE_FUNCTION == nodeType(pExpr)) {
setColumnInfo((SFunctionNode*)pExpr, pCol, pCxt->isPartitionBy);
}
@ -637,6 +638,10 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
}
nodesDestroyList(pColList);
}
if (TSDB_CODE_SUCCESS == code) {
rewriteTargetsWithResId(pJoin->node.pTargets);
}
if (NULL == pJoin->node.pTargets && NULL != pLeft) {
code = nodesCloneList(pLeft->pTargets, &pJoin->node.pTargets);
@ -1357,6 +1362,9 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
? (pSort->groupSort ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_GLOBAL)
: DATA_ORDER_LEVEL_NONE;
code = nodesCollectColumns(pSelect, SQL_CLAUSE_ORDER_BY, NULL, COLLECT_COL_TYPE_ALL, &pSort->node.pTargets);
if (TSDB_CODE_SUCCESS == code) {
rewriteTargetsWithResId(pSort->node.pTargets);
}
if (TSDB_CODE_SUCCESS == code && NULL == pSort->node.pTargets) {
SNode* pNew = NULL;
code = nodesCloneNode(nodesListGetNode(pCxt->pCurrRoot->pTargets, 0), &pNew);
@ -1405,11 +1413,14 @@ static int32_t createColumnByProjections(SLogicPlanContext* pCxt, const char* pS
}
SNode* pNode;
int32_t projIdx = 1;
FOREACH(pNode, pExprs) {
if (TSDB_CODE_SUCCESS != (code = nodesListStrictAppend(pList, (SNode*)createColumnByExpr(pStmtName, (SExprNode*)pNode)))) {
SColumnNode* pCol = createColumnByExpr(pStmtName, (SExprNode*)pNode);
if (TSDB_CODE_SUCCESS != (code = nodesListStrictAppend(pList, (SNode*)pCol))) {
nodesDestroyList(pList);
return code;
}
pCol->resIdx = ((SExprNode*)pNode)->projIdx;
}
*pCols = pList;
@ -1474,6 +1485,9 @@ static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pS
code = nodesListMakeStrictAppend(&pPartition->node.pTargets, pNew);
}
}
if (TSDB_CODE_SUCCESS == code) {
rewriteTargetsWithResId(pPartition->node.pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
// code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, NULL, fmIsAggFunc, &pPartition->pAggFuncs);

42
source/libs/planner/src/planOptimizer.c
View File

@ -3475,6 +3475,20 @@ static EDealRes eliminateProjOptRewriteScanTableAlias(SNode* pNode, void* pConte
}
static void eliminateProjPushdownProjIdx(SNodeList* pParentProjects, SNodeList* pChildTargets) {
SNode* pChildTarget = NULL, *pParentProject = NULL;
FOREACH(pChildTarget, pChildTargets) {
SColumnNode* pTargetCol = (SColumnNode*)pChildTarget;
FOREACH(pParentProject, pParentProjects) {
SExprNode* pProject = (SExprNode*)pParentProject;
if (0 == strcmp(pTargetCol->colName, pProject->aliasName)) {
pTargetCol->resIdx = pProject->projIdx;
break;
}
}
}
}
static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan,
SProjectLogicNode* pProjectNode) {
SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pProjectNode->node.pChildren, 0);
@ -3546,6 +3560,7 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan*
nodesWalkExprs(pScan->pScanPseudoCols, eliminateProjOptRewriteScanTableAlias, &cxt);
nodesWalkExpr(pScan->node.pConditions, eliminateProjOptRewriteScanTableAlias, &cxt);
nodesWalkExprs(pChild->pTargets, eliminateProjOptRewriteScanTableAlias, &cxt);
eliminateProjPushdownProjIdx(pProjectNode->pProjections, pChild->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
@ -4883,6 +4898,31 @@ typedef struct SMergeProjectionsContext {
int32_t errCode;
} SMergeProjectionsContext;
static EDealRes mergeProjectionsExpr2(SNode** pNode, void* pContext) {
SMergeProjectionsContext* pCxt = pContext;
SProjectLogicNode* pChildProj = pCxt->pChildProj;
if (QUERY_NODE_COLUMN == nodeType(*pNode)) {
SColumnNode* pProjCol = (SColumnNode*)(*pNode);
SNode* pProjection;
int32_t projIdx = 1;
FOREACH(pProjection, pChildProj->pProjections) {
if (isColRefExpr(pProjCol, (SExprNode*)pProjection)) {
SNode* pExpr = NULL;
pCxt->errCode = nodesCloneNode(pProjection, &pExpr);
if (pExpr == NULL) {
return DEAL_RES_ERROR;
}
snprintf(((SExprNode*)pExpr)->aliasName, sizeof(((SExprNode*)pExpr)->aliasName), "%s",
((SExprNode*)*pNode)->aliasName);
nodesDestroyNode(*pNode);
*pNode = pExpr;
return DEAL_RES_IGNORE_CHILD;
}
}
}
return DEAL_RES_CONTINUE;
}
static EDealRes mergeProjectionsExpr(SNode** pNode, void* pContext) {
SMergeProjectionsContext* pCxt = pContext;
SProjectLogicNode* pChildProj = pCxt->pChildProj;
@ -4917,7 +4957,7 @@ static int32_t mergeProjectsOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan*
((SProjectLogicNode*)pSelfNode)->inputIgnoreGroup = true;
}
SMergeProjectionsContext cxt = {.pChildProj = (SProjectLogicNode*)pChild, .errCode = TSDB_CODE_SUCCESS};
nodesRewriteExprs(((SProjectLogicNode*)pSelfNode)->pProjections, mergeProjectionsExpr, &cxt);
nodesRewriteExprs(((SProjectLogicNode*)pSelfNode)->pProjections, mergeProjectionsExpr2, &cxt);
int32_t code = cxt.errCode;
if (TSDB_CODE_SUCCESS == code) {

91
source/libs/planner/src/planPhysiCreater.c
View File

@ -35,17 +35,18 @@ typedef struct SPhysiPlanContext {
int32_t errCode;
int16_t nextDataBlockId;
SArray* pLocationHelper;
SArray* pProjIdxLocHelper;
bool hasScan;
bool hasSysScan;
} SPhysiPlanContext;
static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int32_t *pLen) {
static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int32_t *pLen, uint16_t extraBufLen) {
int32_t code = 0;
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
SColumnNode* pCol = (SColumnNode*)pNode;
if (NULL != pStmtName) {
if ('\0' != pStmtName[0]) {
*ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1);
*ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen);
if (!*ppKey) {
return terrno;
}
@ -55,7 +56,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int
*pLen = taosHashBinary(*ppKey, strlen(*ppKey));
return code;
} else {
*ppKey = taosMemoryCalloc(1, TSDB_COL_NAME_LEN + 1);
*ppKey = taosMemoryCalloc(1, TSDB_COL_NAME_LEN + 1 + extraBufLen);
if (!*ppKey) {
return terrno;
}
@ -65,7 +66,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int
}
}
if ('\0' == pCol->tableAlias[0]) {
*ppKey = taosMemoryCalloc(1, TSDB_COL_NAME_LEN + 1);
*ppKey = taosMemoryCalloc(1, TSDB_COL_NAME_LEN + 1 + extraBufLen);
if (!*ppKey) {
return terrno;
}
@ -74,7 +75,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int
return code;
}
*ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1);
*ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen);
if (!*ppKey) {
return terrno;
}
@ -89,7 +90,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int
SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0);
if (pVal) {
if (NULL != pStmtName && '\0' != pStmtName[0]) {
*ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1);
*ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen);
if (!*ppKey) {
return terrno;
}
@ -99,7 +100,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int
*pLen = taosHashBinary(*ppKey, strlen(*ppKey));
return code;
}
*ppKey = taosMemoryCalloc(1, strlen(pVal->literal) + 1 + TSDB_COL_NAME_LEN + 1);
*ppKey = taosMemoryCalloc(1, strlen(pVal->literal) + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen);
if (!*ppKey) {
return terrno;
}
@ -113,7 +114,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int
}
if (NULL != pStmtName && '\0' != pStmtName[0]) {
*ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1);
*ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen);
if (!*ppKey) {
return terrno;
}
@ -124,7 +125,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int
return code;
}
*ppKey = taosMemoryCalloc(1, TSDB_COL_NAME_LEN + 1);
*ppKey = taosMemoryCalloc(1, TSDB_COL_NAME_LEN + 1 + extraBufLen);
if (!*ppKey) {
return terrno;
}
@ -191,22 +192,34 @@ static int32_t putSlotToHash(const char* pName, int32_t len, int16_t dataBlockId
}
static int32_t createDataBlockDescHash(SPhysiPlanContext* pCxt, int32_t capacity, int16_t dataBlockId,
SHashObj** pDescHash) {
SHashObj** pDescHash, SHashObj** ppProjIdxDescHash) {
SHashObj* pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
if (NULL == pHash) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SHashObj* pProjIdxHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
if (!pProjIdxHash) {
taosHashCleanup(pHash);
return TSDB_CODE_OUT_OF_MEMORY;
}
if (NULL == taosArrayInsert(pCxt->pLocationHelper, dataBlockId, &pHash)) {
taosHashCleanup(pHash);
taosHashCleanup(pProjIdxHash);
return TSDB_CODE_OUT_OF_MEMORY;
}
if (NULL == taosArrayInsert(pCxt->pProjIdxLocHelper, dataBlockId, &pProjIdxHash)) {
taosHashCleanup(pHash);
taosHashCleanup(pProjIdxHash);
return TSDB_CODE_OUT_OF_MEMORY;
}
*pDescHash = pHash;
*ppProjIdxDescHash = pProjIdxHash;
return TSDB_CODE_SUCCESS;
}
static int32_t buildDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc,
SHashObj* pHash) {
SHashObj* pHash, SHashObj* pProjIdxDescHash) {
pDataBlockDesc->pSlots = NULL;
int32_t code = nodesMakeList(&pDataBlockDesc->pSlots);
if (NULL == pDataBlockDesc->pSlots) {
@ -218,12 +231,16 @@ static int32_t buildDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SD
FOREACH(pNode, pList) {
char* name = NULL;
int32_t len = 0;
code = getSlotKey(pNode, NULL, &name, &len);
code = getSlotKey(pNode, NULL, &name, &len, 16);
if (TSDB_CODE_SUCCESS == code) {
code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, name, pNode, slotId, true, false));
}
code = putSlotToHash(name, len, pDataBlockDesc->dataBlockId, slotId, pNode, pHash);
if (TSDB_CODE_SUCCESS == code) {
code = putSlotToHash(name, len, pDataBlockDesc->dataBlockId, slotId, pNode, pHash);
if (nodeType(pNode) == QUERY_NODE_COLUMN && ((SColumnNode*)pNode)->resIdx > 0) {
sprintf(name + strlen(name), "_%d", ((SColumnNode*)pNode)->resIdx);
code = putSlotToHash(name, strlen(name), pDataBlockDesc->dataBlockId, slotId, pNode, pProjIdxDescHash);
}
}
taosMemoryFree(name);
if (TSDB_CODE_SUCCESS == code) {
@ -246,9 +263,10 @@ static int32_t createDataBlockDesc(SPhysiPlanContext* pCxt, SNodeList* pList, SD
pDesc->dataBlockId = pCxt->nextDataBlockId++;
SHashObj* pHash = NULL;
code = createDataBlockDescHash(pCxt, LIST_LENGTH(pList), pDesc->dataBlockId, &pHash);
SHashObj* pProjIdxHash = NULL;
code = createDataBlockDescHash(pCxt, LIST_LENGTH(pList), pDesc->dataBlockId, &pHash, &pProjIdxHash);
if (TSDB_CODE_SUCCESS == code) {
code = buildDataBlockSlots(pCxt, pList, pDesc, pHash);
code = buildDataBlockSlots(pCxt, pList, pDesc, pHash, pProjIdxHash);
}
if (TSDB_CODE_SUCCESS == code) {
@ -286,7 +304,7 @@ static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList,
SNode* pExpr = QUERY_NODE_ORDER_BY_EXPR == nodeType(pNode) ? ((SOrderByExprNode*)pNode)->pExpr : pNode;
char *name = NULL;
int32_t len = 0;
code = getSlotKey(pExpr, pStmtName, &name, &len);
code = getSlotKey(pExpr, pStmtName, &name, &len, 0);
if (TSDB_CODE_SUCCESS == code) {
SSlotIndex* pIndex = taosHashGet(pHash, name, len);
if (NULL == pIndex) {
@ -355,7 +373,9 @@ static int32_t pushdownDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList,
typedef struct SSetSlotIdCxt {
int32_t errCode;
SHashObj* pLeftHash;
SHashObj* pLeftProjIdxHash;
SHashObj* pRightHash;
SHashObj* pRightProdIdxHash;
} SSetSlotIdCxt;
static void dumpSlots(const char* pName, SHashObj* pHash) {
@ -379,13 +399,22 @@ static EDealRes doSetSlotId(SNode* pNode, void* pContext) {
SSetSlotIdCxt* pCxt = (SSetSlotIdCxt*)pContext;
char *name = NULL;
int32_t len = 0;
pCxt->errCode = getSlotKey(pNode, NULL, &name, &len);
pCxt->errCode = getSlotKey(pNode, NULL, &name, &len, 16);
if (TSDB_CODE_SUCCESS != pCxt->errCode) {
return DEAL_RES_ERROR;
}
SSlotIndex* pIndex = taosHashGet(pCxt->pLeftHash, name, len);
if (NULL == pIndex) {
pIndex = taosHashGet(pCxt->pRightHash, name, len);
SSlotIndex *pIndex = NULL;
if (((SColumnNode*)pNode)->projRefIdx > 0) {
sprintf(name + strlen(name), "_%d", ((SColumnNode*)pNode)->projRefIdx);
pIndex = taosHashGet(pCxt->pLeftProjIdxHash, name, strlen(name));
if (!pIndex) {
pIndex = taosHashGet(pCxt->pRightProdIdxHash, name, strlen(name));
}
} else {
pIndex = taosHashGet(pCxt->pLeftHash, name, len);
if (NULL == pIndex) {
pIndex = taosHashGet(pCxt->pRightHash, name, len);
}
}
// pIndex is definitely not NULL, otherwise it is a bug
if (NULL == pIndex) {
@ -396,9 +425,9 @@ static EDealRes doSetSlotId(SNode* pNode, void* pContext) {
taosMemoryFree(name);
return DEAL_RES_ERROR;
}
taosMemoryFree(name);
((SColumnNode*)pNode)->dataBlockId = pIndex->dataBlockId;
((SColumnNode*)pNode)->slotId = ((SSlotIdInfo*)taosArrayGet(pIndex->pSlotIdsInfo, 0))->slotId;
taosMemoryFree(name);
return DEAL_RES_IGNORE_CHILD;
}
return DEAL_RES_CONTINUE;
@ -419,7 +448,9 @@ static int32_t setNodeSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i
SSetSlotIdCxt cxt = {
.errCode = TSDB_CODE_SUCCESS,
.pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId),
.pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId))};
.pLeftProjIdxHash = taosArrayGetP(pCxt->pProjIdxLocHelper, leftDataBlockId),
.pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)),
.pRightProdIdxHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pProjIdxLocHelper, rightDataBlockId))};
nodesWalkExpr(pRes, doSetSlotId, &cxt);
if (TSDB_CODE_SUCCESS != cxt.errCode) {
nodesDestroyNode(pRes);
@ -445,7 +476,9 @@ static int32_t setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i
SSetSlotIdCxt cxt = {
.errCode = TSDB_CODE_SUCCESS,
.pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId),
.pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId))};
.pLeftProjIdxHash = taosArrayGetP(pCxt->pProjIdxLocHelper, leftDataBlockId),
.pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)),
.pRightProdIdxHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pProjIdxLocHelper, rightDataBlockId))};
nodesWalkExprs(pRes, doSetSlotId, &cxt);
if (TSDB_CODE_SUCCESS != cxt.errCode) {
nodesDestroyList(pRes);
@ -1254,7 +1287,7 @@ static int32_t sortHashJoinTargets(int16_t lBlkId, int16_t rBlkId, SHashJoinPhys
SColumnNode* pCol = (SColumnNode*)pNode;
char *pName = NULL;
int32_t len = 0;
code = getSlotKey(pNode, NULL, &pName, &len);
code = getSlotKey(pNode, NULL, &pName, &len, 0);
if (TSDB_CODE_SUCCESS == code) {
code = tSimpleHashPut(pHash, pName, len, &pCol, POINTER_BYTES);
}
@ -1272,7 +1305,7 @@ static int32_t sortHashJoinTargets(int16_t lBlkId, int16_t rBlkId, SHashJoinPhys
char* pName = NULL;
SColumnNode* pCol = (SColumnNode*)pNode;
int32_t len = 0;
code = getSlotKey(pNode, NULL, &pName, &len);
code = getSlotKey(pNode, NULL, &pName, &len, 0);
if (TSDB_CODE_SUCCESS == code) {
SNode** p = tSimpleHashGet(pHash, pName, len);
if (p) {
@ -1293,7 +1326,7 @@ static int32_t sortHashJoinTargets(int16_t lBlkId, int16_t rBlkId, SHashJoinPhys
char* pName = NULL;
SColumnNode* pCol = (SColumnNode*)pNode;
int32_t len = 0;
code = getSlotKey(pNode, NULL, &pName, &len);
code = getSlotKey(pNode, NULL, &pName, &len, 0);
if (TSDB_CODE_SUCCESS == code) {
SNode** p = tSimpleHashGet(pHash, pName, len);
if (p) {
@ -2999,6 +3032,7 @@ static void destoryLocationHash(void* p) {
static void destoryPhysiPlanContext(SPhysiPlanContext* pCxt) {
taosArrayDestroyEx(pCxt->pLocationHelper, destoryLocationHash);
taosArrayDestroyEx(pCxt->pProjIdxLocHelper, destoryLocationHash);
}
static void setExplainInfo(SPlanContext* pCxt, SQueryPlan* pPlan) {
@ -3030,9 +3064,12 @@ int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryP
.errCode = TSDB_CODE_SUCCESS,
.nextDataBlockId = 0,
.pLocationHelper = taosArrayInit(32, POINTER_BYTES),
.pProjIdxLocHelper = taosArrayInit(32, POINTER_BYTES),
.hasScan = false,
.hasSysScan = false};
if (NULL == cxt.pLocationHelper) {
if (NULL == cxt.pLocationHelper || !cxt.pProjIdxLocHelper) {
taosArrayDestroy(cxt.pLocationHelper);
taosArrayDestroy(cxt.pProjIdxLocHelper);
return TSDB_CODE_OUT_OF_MEMORY;
}

11
source/libs/planner/src/planUtil.c
View File

@ -716,5 +716,16 @@ int32_t tagScanSetExecutionMode(SScanLogicNode* pScan) {
return TSDB_CODE_SUCCESS;
}
bool isColRefExpr(const SColumnNode* pCol, const SExprNode* pExpr) {
if (pCol->projRefIdx > 0) return pCol->projRefIdx == pExpr->projIdx;
return 0 == strcmp(pCol->colName, pExpr->aliasName);
}
void rewriteTargetsWithResId(SNodeList* pTargets) {
SNode* pNode;
FOREACH(pNode, pTargets) {
SColumnNode* pCol = (SColumnNode*)pNode;
pCol->resIdx = pCol->projRefIdx;
}
}

2
source/libs/scheduler/inc/schInt.h
View File

@ -91,7 +91,7 @@ typedef struct SSchRuntimeStat {
#if defined(WINDOWS) || defined(_TD_DARWIN_64)
size_t avoidCompilationErrors;
#endif
int64_t hbConnNotFound;
} SSchRuntimeStat;
typedef struct SSchJobStat {

5
source/libs/scheduler/src/schUtil.c
View File

@ -251,9 +251,8 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) {
hb = taosHashGet(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId));
if (NULL == hb) {
SCH_UNLOCK(SCH_READ, &schMgmt.hbLock);
qDebug("taosHashGet hb connection not exists, nodeId:%d, fqdn:%s, port:%d", epId->nodeId, epId->ep.fqdn,
epId->ep.port);
SCH_ERR_RET(TSDB_CODE_APP_ERROR);
(void)atomic_add_fetch_64(&schMgmt.stat.runtime.hbConnNotFound, 1);
return TSDB_CODE_SUCCESS;
}
SCH_LOCK(SCH_WRITE, &hb->lock);

89
source/libs/stream/src/streamBackendRocksdb.c
View File

@ -1096,21 +1096,23 @@ int32_t delObsoleteCheckpoint(void* arg, const char* path) {
* replication is finished
*/
int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) {
int32_t code = 0;
STaskDbWrapper* pBackend = arg;
SArray * chkpDel = NULL, *chkpDup = NULL;
(void)taosThreadRwlockWrlock(&pBackend->chkpDirLock);
(void)taosArrayPush(pBackend->chkpSaved, &chkpId);
SArray* chkpDel = taosArrayInit(8, sizeof(int64_t));
if (chkpDel == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
if (taosArrayPush(pBackend->chkpSaved, &chkpId) == NULL) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception);
}
SArray* chkpDup = taosArrayInit(8, sizeof(int64_t));
chkpDel = taosArrayInit(8, sizeof(int64_t));
if (chkpDel == NULL) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception);
}
chkpDup = taosArrayInit(8, sizeof(int64_t));
if (chkpDup == NULL) {
taosArrayDestroy(chkpDel);
return TSDB_CODE_OUT_OF_MEMORY;
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception);
}
int64_t firsId = 0;
@ -1120,9 +1122,13 @@ int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) {
for (int i = 0; i < taosArrayGetSize(pBackend->chkpSaved); i++) {
int64_t id = *(int64_t*)taosArrayGet(pBackend->chkpSaved, i);
if (id >= firsId) {
(void)taosArrayPush(chkpDup, &id);
if (taosArrayPush(chkpDup, &id) == NULL) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception);
}
} else {
(void)taosArrayPush(chkpDel, &id);
if (taosArrayPush(chkpDel, &id) == NULL) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception);
}
}
}
} else {
@ -1131,13 +1137,18 @@ int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) {
for (int i = 0; i < dsz; i++) {
int64_t id = *(int64_t*)taosArrayGet(pBackend->chkpSaved, i);
(void)taosArrayPush(chkpDel, &id);
if (taosArrayPush(chkpDel, &id) == NULL) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception);
}
}
for (int i = dsz < 0 ? 0 : dsz; i < sz; i++) {
int64_t id = *(int64_t*)taosArrayGet(pBackend->chkpSaved, i);
(void)taosArrayPush(chkpDup, &id);
if (taosArrayPush(chkpDup, &id) == NULL) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception);
}
}
}
taosArrayDestroy(pBackend->chkpSaved);
pBackend->chkpSaved = chkpDup;
@ -1155,6 +1166,11 @@ int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) {
}
taosArrayDestroy(chkpDel);
return 0;
_exception:
taosArrayDestroy(chkpDup);
taosArrayDestroy(chkpDel);
(void)taosThreadRwlockUnlock(&pBackend->chkpDirLock);
return code;
}
#ifdef BUILD_NO_CALL
@ -1288,7 +1304,9 @@ int32_t taskDbLoadChkpInfo(STaskDbWrapper* pBackend) {
int ret = sscanf(taosGetDirEntryName(de), "checkpoint%" PRId64 "", &checkpointId);
if (ret == 1) {
(void)taosArrayPush(pBackend->chkpSaved, &checkpointId);
if (taosArrayPush(pBackend->chkpSaved, &checkpointId) == NULL) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception);
}
}
} else {
continue;
@ -1300,13 +1318,21 @@ int32_t taskDbLoadChkpInfo(STaskDbWrapper* pBackend) {
(void)taosCloseDir(&pDir);
return 0;
_exception:
taosMemoryFree(pChkpDir);
(void)taosCloseDir(&pDir);
return code;
}
int32_t chkpGetAllDbCfHandle2(STaskDbWrapper* pBackend, rocksdb_column_family_handle_t*** ppHandle) {
int32_t code = 0;
SArray* pHandle = taosArrayInit(8, POINTER_BYTES);
for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) {
if (pBackend->pCf[i]) {
rocksdb_column_family_handle_t* p = pBackend->pCf[i];
(void)taosArrayPush(pHandle, &p);
if (taosArrayPush(pHandle, &p) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exception;
}
}
}
int32_t nCf = taosArrayGetSize(pHandle);
@ -1316,13 +1342,20 @@ int32_t chkpGetAllDbCfHandle2(STaskDbWrapper* pBackend, rocksdb_column_family_ha
}
rocksdb_column_family_handle_t** ppCf = taosMemoryCalloc(nCf, sizeof(rocksdb_column_family_handle_t*));
if (ppCf == NULL) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception);
}
for (int i = 0; i < nCf; i++) {
ppCf[i] = taosArrayGetP(pHandle, i);
}
taosArrayDestroy(pHandle);
*ppHandle = ppCf;
return nCf;
_exception:
taosArrayDestroy(pHandle);
return code;
}
int32_t chkpDoDbCheckpoint(rocksdb_t* db, char* path) {
@ -2435,7 +2468,9 @@ void taskDbInitChkpOpt(STaskDbWrapper* pTaskDb) {
void taskDbRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp) {
(void)taosThreadRwlockWrlock(&pTaskDb->chkpDirLock);
(void)taosArrayPush(pTaskDb->chkpInUse, &chkp);
if (taosArrayPush(pTaskDb->chkpInUse, &chkp) == NULL) {
stError("failed to push chkp: %" PRIi64 " into inuse", chkp);
}
taosArraySort(pTaskDb->chkpInUse, chkpIdComp);
(void)taosThreadRwlockUnlock(&pTaskDb->chkpDirLock);
}
@ -4331,7 +4366,10 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co
if (strncmp(key, start, strlen(start)) == 0 && strlen(key) >= strlen(start) + 1) {
int64_t checkPoint = 0;
if (sscanf(key + strlen(key), ":%" PRId64 "", &checkPoint) == 1) {
(void)taosArrayPush(result, &checkPoint);
if (taosArrayPush(result, &checkPoint) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
break;
}
}
} else {
break;
@ -4547,7 +4585,10 @@ int32_t compareHashTableImpl(SHashObj* p1, SHashObj* p2, SArray* diff) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)strncpy(fname, name, len);
(void)taosArrayPush(diff, &fname);
if (taosArrayPush(diff, &fname) == NULL) {
taosMemoryFree(fname);
return TSDB_CODE_OUT_OF_MEMORY;
}
}
pIter = taosHashIterate(p2, pIter);
}
@ -4706,7 +4747,11 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) {
}
(void)strncpy(fname, name, len);
(void)taosArrayPush(p->pAdd, &fname);
if (taosArrayPush(p->pAdd, &fname) == NULL) {
taosMemoryFree(fname);
(void)taosThreadRwlockUnlock(&p->rwLock);
return TSDB_CODE_OUT_OF_MEMORY;
}
}
pIter = taosHashIterate(p->pSstTbl[1 - p->idx], pIter);
}
@ -4910,7 +4955,11 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _ERROR;
}
(void)taosArrayPush(list, &p);
if (taosArrayPush(list, &p) == NULL) {
taosMemoryFree(p);
code = TSDB_CODE_OUT_OF_MEMORY;
goto _ERROR;
}
}
// copy current file to dst dir

10
source/libs/stream/src/streamCheckStatus.c
View File

@ -585,12 +585,18 @@ void getCheckRspStatus(STaskCheckInfo* pInfo, int64_t el, int32_t* numOfReady, i
} else { // TASK_DOWNSTREAM_NOT_READY
if (p->rspTs == 0) { // not response yet
if (el >= CHECK_NOT_RSP_DURATION) { // not receive info for 10 sec.
(void)taosArrayPush(pTimeoutList, &p->taskId);
void* px = taosArrayPush(pTimeoutList, &p->taskId);
if (px == NULL) {
stError("s-task:%s failed to record time out task:0x%x", id, p->taskId);
}
} else { // el < CHECK_NOT_RSP_DURATION
(*numOfNotRsp) += 1; // do nothing and continue waiting for their rsp
}
} else {
(void)taosArrayPush(pNotReadyList, &p->taskId);
void* px = taosArrayPush(pNotReadyList, &p->taskId);
if (px == NULL) {
stError("s-task:%s failed to record not ready task:0x%x", id, p->taskId);
}
}
}
}

5
source/libs/stream/src/streamCheckpoint.c
View File

@ -973,7 +973,10 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
if (!recved) { // make sure the inputQ is opened for not recv upstream checkpoint-trigger message
streamTaskOpenUpstreamInput(pTask, pInfo->taskId);
(void)taosArrayPush(pNotSendList, pInfo);
void* px = taosArrayPush(pNotSendList, pInfo);
if (px == NULL) {
stError("s-task:%s failed to record not send info, code: out of memory", id);
}
}
}

115
source/libs/stream/src/streamDispatch.c
View File

@ -214,6 +214,7 @@ int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq,
tEncoderInit(&encoder, abuf, tlen);
if ((code = tEncodeStreamTaskCheckReq(&encoder, pReq)) < 0) {
rpcFreeCont(buf);
tEncoderClear(&encoder);
return code;
}
tEncoderClear(&encoder);
@ -306,7 +307,12 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD
int32_t code = 0;
int64_t now = taosGetTimestampMs();
int32_t numOfBlocks = taosArrayGetSize(pData->blocks);
ASSERT(numOfBlocks != 0 && pTask->msgInfo.pData == NULL);
if (!(numOfBlocks != 0 && pTask->msgInfo.pData == NULL)) {
stError("s-task:%s dispatch block number:%d, exist not rsp dispatch msg:%p, abort build new dispatch msg",
pTask->id.idStr, numOfBlocks, pTask->msgInfo.pData);
return TSDB_CODE_STREAM_INTERNAL_ERROR;
}
pTask->msgInfo.dispatchMsgType = pData->type;
@ -474,7 +480,10 @@ static void addDispatchEntry(SDispatchMsgInfo* pMsgInfo, int32_t nodeId, int64_t
streamMutexLock(&pMsgInfo->lock);
}
(void)taosArrayPush(pMsgInfo->pSendInfo, &entry);
void* p = taosArrayPush(pMsgInfo->pSendInfo, &entry);
if (p == NULL) {
stError("failed to add dispatch info");
}
if (lock) {
streamMutexUnlock(&pMsgInfo->lock);
@ -671,8 +680,8 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S
memcpy(bln.parTbName, pDataBlock->info.parTbName, strlen(pDataBlock->info.parTbName));
// failed to put into name buffer, no need to do anything
if (tSimpleHashGetSize(pTask->pNameMap) < MAX_BLOCK_NAME_NUM) {
(void)tSimpleHashPut(pTask->pNameMap, &groupId, sizeof(int64_t), &bln, sizeof(SBlockName));
if (tSimpleHashGetSize(pTask->pNameMap) < MAX_BLOCK_NAME_NUM) { // allow error, and do nothing
int32_t code = tSimpleHashPut(pTask->pNameMap, &groupId, sizeof(int64_t), &bln, sizeof(SBlockName));
}
}
@ -708,14 +717,15 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S
}
streamMutexUnlock(&pTask->msgInfo.lock);
ASSERT(found);
return 0;
if (!found) {
stError("s-task:%s not found req hash value:%u", pTask->id.idStr, hashValue);
return TSDB_CODE_STREAM_INTERNAL_ERROR;
} else {
return 0;
}
}
int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
ASSERT((pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH ||
pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH));
const char* id = pTask->id.idStr;
int32_t numOfElems = streamQueueGetNumOfItems(pTask->outputq.queue);
if (numOfElems > 0) {
@ -739,8 +749,11 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
return 0;
}
ASSERT(pTask->msgInfo.pData == NULL);
stDebug("s-task:%s start to dispatch msg, set output status:%d", id, pTask->outputq.status);
if (pTask->msgInfo.pData != NULL) {
stFatal("s-task:%s not rsp data:%p exist, should not dispatch msg now", id, pTask->msgInfo.pData);
} else {
stDebug("s-task:%s start to dispatch msg, set output status:%d", id, pTask->outputq.status);
}
SStreamDataBlock* pBlock = NULL;
streamQueueNextItem(pTask->outputq.queue, (SStreamQueueItem**)&pBlock);
@ -751,8 +764,11 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
}
int32_t type = pBlock->type;
ASSERT(type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__CHECKPOINT_TRIGGER ||
type == STREAM_INPUT__TRANS_STATE);
if (!(type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__CHECKPOINT_TRIGGER ||
type == STREAM_INPUT__TRANS_STATE)) {
stError("s-task:%s invalid dispatch block type:%d", id, type);
return TSDB_CODE_INTERNAL_ERROR;
}
pTask->execInfo.dispatch += 1;
@ -830,6 +846,7 @@ int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32
tEncoderInit(&encoder, abuf, tlen);
if ((code = tEncodeStreamCheckpointReadyMsg(&encoder, &req)) < 0) {
rpcFreeCont(buf);
tEncoderClear(&encoder);
return code;
}
tEncoderClear(&encoder);
@ -878,7 +895,6 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
SArray* pList = pActiveInfo->pReadyMsgList;
int32_t num = taosArrayGetSize(pList);
if (pTmrInfo->launchChkptId != pActiveInfo->activeId) {
streamMutexUnlock(&pActiveInfo->lock);
int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask);
@ -902,7 +918,15 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
SArray* pNotRspList = taosArrayInit(4, sizeof(int32_t));
ASSERT(taosArrayGetSize(pTask->upstreamInfo.pList) == num);
if (taosArrayGetSize(pTask->upstreamInfo.pList) != num) {
streamMutexUnlock(&pActiveInfo->lock);
int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask);
stWarn("s-task:%s vgId:%d upstream number:%d not equals sent readyMsg:%d, quit from readyMsg send tmr, ref:%d", id,
vgId, (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList), num, ref);
streamMetaReleaseTask(pTask->pMeta, pTask);
return;
}
for (int32_t i = 0; i < num; ++i) {
STaskCheckpointReadyInfo* pInfo = taosArrayGet(pList, i);
@ -914,9 +938,13 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
continue;
}
(void)taosArrayPush(pNotRspList, &pInfo->upstreamTaskId);
stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId,
pTask->info.taskLevel, pInfo->upstreamTaskId);
void* p = taosArrayPush(pNotRspList, &pInfo->upstreamTaskId);
if (p == NULL) {
stError("s-task:%s vgId:%d failed to record not rsp task, code: out of memory", id, vgId);
} else {
stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId,
pTask->info.taskLevel, pInfo->upstreamTaskId);
}
}
int32_t checkpointId = pActiveInfo->activeId;
@ -984,7 +1012,11 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) {
streamMutexLock(&pActiveInfo->lock);
int32_t num = taosArrayGetSize(pList);
ASSERT(taosArrayGetSize(pTask->upstreamInfo.pList) == num);
if (taosArrayGetSize(pTask->upstreamInfo.pList) != num) {
stError("s-task:%s invalid number of sent readyMsg:%d to upstream:%d", id, num,
(int32_t)taosArrayGetSize(pTask->upstreamInfo.pList));
return TSDB_CODE_STREAM_INTERNAL_ERROR;
}
for (int32_t i = 0; i < num; ++i) {
STaskCheckpointReadyInfo* pInfo = taosArrayGet(pList, i);
@ -1064,9 +1096,7 @@ int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) {
int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq) {
int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN;
ASSERT(dataStrLen > 0);
void* buf = taosMemoryCalloc(1, dataStrLen);
void* buf = taosMemoryCalloc(1, dataStrLen);
if (buf == NULL) {
return terrno;
}
@ -1100,8 +1130,17 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch
payloadLen += sizeof(SRetrieveTableRsp);
(void)taosArrayPush(pReq->dataLen, &payloadLen);
(void)taosArrayPush(pReq->data, &buf);
void* px = taosArrayPush(pReq->dataLen, &payloadLen);
if (px == NULL) {
taosMemoryFree(buf);
return terrno;
}
px = taosArrayPush(pReq->data, &buf);
if (px == NULL) {
taosMemoryFree(buf);
return terrno;
}
pReq->totalLen += dataStrLen;
return 0;
@ -1131,6 +1170,7 @@ int32_t doSendDispatchMsg(SStreamTask* pTask, const SStreamDispatchReq* pReq, in
SEncoder encoder;
tEncoderInit(&encoder, abuf, tlen);
if ((code = tEncodeStreamDispatchReq(&encoder, pReq)) < 0) {
tEncoderClear(&encoder);
goto FAIL;
}
tEncoderClear(&encoder);
@ -1205,8 +1245,6 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa
int32_t size = taosArrayGetSize(pActiveInfo->pReadyMsgList);
if (size > 0) {
ASSERT(size == 1);
STaskCheckpointReadyInfo* pReady = taosArrayGet(pActiveInfo->pReadyMsgList, 0);
if (pReady == NULL) {
return terrno;
@ -1221,8 +1259,12 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa
pTask->id.idStr, pReady->checkpointId, pReady->transId, pReq->transId, pReq->checkpointId);
}
} else {
(void)taosArrayPush(pActiveInfo->pReadyMsgList, &info);
stDebug("s-task:%s add checkpoint source rsp msg, total:%d", pTask->id.idStr, size + 1);
void* px = taosArrayPush(pActiveInfo->pReadyMsgList, &info);
if (px != NULL) {
stDebug("s-task:%s add checkpoint source rsp msg, total:%d", pTask->id.idStr, size + 1);
} else {
stError("s-task:%s failed to add readyMsg, code: out of memory", pTask->id.idStr);
}
}
streamMutexUnlock(&pActiveInfo->lock);
@ -1259,7 +1301,12 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId,
SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
streamMutexLock(&pActiveInfo->lock);
(void)taosArrayPush(pActiveInfo->pReadyMsgList, &info);
void* px = taosArrayPush(pActiveInfo->pReadyMsgList, &info);
if (px == NULL) {
streamMutexUnlock(&pActiveInfo->lock);
stError("s-task:%s failed to add readyMsg info, code: out of memory", pTask->id.idStr);
return terrno;
}
int32_t numOfRecv = taosArrayGetSize(pActiveInfo->pReadyMsgList);
int32_t total = streamTaskGetNumOfUpstream(pTask);
@ -1267,7 +1314,6 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId,
stDebug("s-task:%s recv checkpoint-trigger from all upstream, continue", pTask->id.idStr);
pActiveInfo->allUpstreamTriggerRecv = 1;
} else {
ASSERT(numOfRecv <= total);
stDebug("s-task:%s %d/%d checkpoint-trigger recv", pTask->id.idStr, numOfRecv, total);
}
@ -1516,7 +1562,9 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
if (pMsgInfo->dispatchMsgType == STREAM_INPUT__TRANS_STATE) {
stDebug("s-task:%s dispatch trans-state msgId:%d to downstream successfully, start to prepare transfer state", id,
msgId);
ASSERT(pTask->info.fillHistory == 1);
if (pTask->info.fillHistory != 1) {
stFatal("s-task:%s unexpected dispatch rsp, not scan-history task, not recv this dispatch rsp", id);
}
code = streamTransferStatePrepare(pTask);
if (code != TSDB_CODE_SUCCESS) { // todo: do nothing if error happens
@ -1542,7 +1590,10 @@ static int32_t buildDispatchRsp(const SStreamTask* pTask, const SStreamDispatchR
}
((SMsgHead*)(*pBuf))->vgId = htonl(pReq->upstreamNodeId);
ASSERT(((SMsgHead*)(*pBuf))->vgId != 0);
if (((SMsgHead*)(*pBuf))->vgId == 0) {
return TSDB_CODE_INVALID_MSG;
}
SStreamDispatchRsp* pDispatchRsp = POINTER_SHIFT((*pBuf), sizeof(SMsgHead));

81
source/libs/stream/src/streamExec.c
View File

@ -45,7 +45,11 @@ static int32_t doOutputResultBlockImpl(SStreamTask* pTask, SStreamDataBlock* pBl
pTask->outputInfo.smaSink.smaSink(pTask->outputInfo.smaSink.vnode, pTask->outputInfo.smaSink.smaId, pBlock->blocks);
destroyStreamDataBlock(pBlock);
} else {
ASSERT(type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH);
if (type != TASK_OUTPUT__FIXED_DISPATCH && type != TASK_OUTPUT__SHUFFLE_DISPATCH) {
stError("s-task:%s invalid stream output type:%d, internal error", pTask->id.idStr, type);
return TSDB_CODE_STREAM_INTERNAL_ERROR;
}
code = streamTaskPutDataIntoOutputQ(pTask, pBlock);
if (code != TSDB_CODE_SUCCESS) {
destroyStreamDataBlock(pBlock);
@ -127,7 +131,11 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to
if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) {
SSDataBlock block = {0};
const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)pItem;
ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1);
int32_t num = taosArrayGetSize(pRetrieveBlock->blocks);
if (num != 1) {
stError("s-task:%s invalid retrieve block number:%d, ignore", pTask->id.idStr, num);
continue;
}
(void)assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0));
block.info.type = STREAM_PULL_OVER;
@ -178,7 +186,6 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to
// current output should be dispatched to down stream nodes
if (numOfBlocks >= STREAM_RESULT_DUMP_THRESHOLD || size >= STREAM_RESULT_DUMP_SIZE_THRESHOLD) {
ASSERT(numOfBlocks == taosArrayGetSize(pRes));
code = doDumpResult(pTask, pItem, pRes, size, totalSize, totalBlocks);
// todo: here we need continue retry to put it into output buffer
if (code != TSDB_CODE_SUCCESS) {
@ -192,7 +199,6 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to
}
if (numOfBlocks > 0) {
ASSERT(numOfBlocks == taosArrayGetSize(pRes));
code = doDumpResult(pTask, pItem, pRes, size, totalSize, totalBlocks);
} else {
taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
@ -277,7 +283,10 @@ static SScanhistoryDataInfo buildScanhistoryExecRet(EScanHistoryCode code, int32
}
SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) {
ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE);
if(pTask->info.taskLevel != TASK_LEVEL__SOURCE) {
stError("s-task:%s not source scan-history task, not exec, quit", pTask->id.idStr);
return buildScanhistoryExecRet(TASK_SCANHISTORY_QUIT, 0);
}
void* exec = pTask->exec.pExecutor;
bool finished = false;
@ -374,10 +383,16 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) {
// It must be halted for a source stream task, since when the related scan-history-data task start scan the history
// for the step 2.
if (pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) {
ASSERT(status == TASK_STATUS__HALT || status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP);
if (!(status == TASK_STATUS__HALT || status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP)) {
stError("s-task:%s invalid task status:%d", id, status);
return TSDB_CODE_STREAM_INTERNAL_ERROR;
}
} else {
ASSERT(status == TASK_STATUS__READY || status == TASK_STATUS__PAUSE || status == TASK_STATUS__DROPPING ||
status == TASK_STATUS__STOP);
if (!(status == TASK_STATUS__READY || status == TASK_STATUS__PAUSE || status == TASK_STATUS__DROPPING ||
status == TASK_STATUS__STOP)) {
stError("s-task:%s invalid task status:%d", id, status);
return TSDB_CODE_STREAM_INTERNAL_ERROR;
}
code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT);
if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s halt stream task:%s failed, code:%s not transfer state to stream task", id,
@ -438,7 +453,10 @@ int32_t streamTransferStatePrepare(SStreamTask* pTask) {
int32_t code = TSDB_CODE_SUCCESS;
SStreamMeta* pMeta = pTask->pMeta;
ASSERT(pTask->status.appendTranstateBlock == 1);
if (pTask->status.appendTranstateBlock != 1) {
stError("s-task:%s not set appendTransBlock flag, internal error", pTask->id.idStr);
return TSDB_CODE_STREAM_INTERNAL_ERROR;
}
int32_t level = pTask->info.taskLevel;
if (level == TASK_LEVEL__AGG || level == TASK_LEVEL__SOURCE) { // do transfer task operator states.
@ -476,14 +494,16 @@ static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int
code = qSetMultiStreamInput(pExecutor, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK);
} else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) {
ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE);
const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)pInput;
code = qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT);
stDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, id, pSubmit,
pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver);
ASSERT((*pVer) <= pSubmit->submit.ver);
(*pVer) = pSubmit->submit.ver;
if ((*pVer) > pSubmit->submit.ver) {
stError("s-task:%s invalid recorded ver:%" PRId64 " greater than new block ver:%" PRId64 ", not update", id,
*pVer, pSubmit->submit.ver);
} else {
(*pVer) = pSubmit->submit.ver;
}
} else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) {
const SStreamDataBlock* pBlock = (const SStreamDataBlock*)pInput;
@ -500,8 +520,13 @@ static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int
stDebug("s-task:%s %p set (merged) submit blocks as a batch, numOfBlocks:%d, ver:%" PRId64, id, pTask, numOfBlocks,
pMerged->ver);
code = qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT);
ASSERT((*pVer) <= pMerged->ver);
(*pVer) = pMerged->ver;
if ((*pVer) > pMerged->ver) {
stError("s-task:%s invalid recorded ver:%" PRId64 " greater than new block ver:%" PRId64 ", not update", id,
*pVer, pMerged->ver);
} else {
(*pVer) = pMerged->ver;
}
} else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) {
const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)pInput;
@ -512,7 +537,8 @@ static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int
code = qSetMultiStreamInput(pExecutor, pCheckpoint->blocks, 1, pItem->type);
} else {
ASSERT(0);
stError("s-task:%s invalid input block type:%d, discard", id, pItem->type);
code = TSDB_CODE_STREAM_INTERNAL_ERROR;
}
return code;
@ -542,7 +568,6 @@ void streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock)
stDebug("s-task:%s add transfer-state block into outputQ", id);
} else {
stDebug("s-task:%s all upstream tasks send transfer-state block, add transfer-state block into outputQ", id);
ASSERT(pTask->streamTaskId.taskId != 0 && pTask->info.fillHistory == 1);
}
// agg task should dispatch trans-state msg to sink task, to flush all data to sink task.
@ -560,7 +585,6 @@ void streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock)
} else { // non-dispatch task, do task state transfer directly
streamFreeQitem((SStreamQueueItem*)pBlock);
stDebug("s-task:%s non-dispatch task, level:%d start to transfer state directly", id, level);
ASSERT(pTask->info.fillHistory == 1);
code = streamTransferStatePrepare(pTask);
if (code != TSDB_CODE_SUCCESS) {
@ -606,7 +630,11 @@ static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, i
}
// update the currentVer if processing the submit blocks.
ASSERT(pInfo->checkpointVer <= pInfo->nextProcessVer && ver >= pInfo->checkpointVer);
if (!(pInfo->checkpointVer <= pInfo->nextProcessVer && ver >= pInfo->checkpointVer)) {
stError("s-task:%s invalid info, checkpointVer:%" PRId64 ", nextProcessVer:%" PRId64 " currentVer:%" PRId64, id,
pInfo->checkpointVer, pInfo->nextProcessVer, ver);
return;
}
if (ver != pInfo->processedVer) {
stDebug("s-task:%s update processedVer(unsaved) from %" PRId64 " to %" PRId64 " nextProcessVer:%" PRId64
@ -622,8 +650,6 @@ void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointB
// 1. transfer the ownership of executor state
bool dropRelHTask = (streamTaskGetPrevStatus(pTask) == TASK_STATUS__HALT);
if (dropRelHTask) {
ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask));
STaskId* pHTaskId = &pTask->hTaskInfo.id;
SStreamTask* pHTask = NULL;
int32_t code = streamMetaAcquireTask(pTask->pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask);
@ -692,12 +718,10 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
EExtractDataCode ret = streamTaskGetDataFromInputQ(pTask, &pInput, &numOfBlocks, &blockSize);
if (ret == EXEC_AFTER_IDLE) {
ASSERT(pInput == NULL && numOfBlocks == 0);
streamTaskSetIdleInfo(pTask, MIN_INVOKE_INTERVAL);
return 0;
} else {
if (pInput == NULL) {
ASSERT(numOfBlocks == 0);
return 0;
}
}
@ -718,7 +742,10 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
}
if (pTask->info.taskLevel == TASK_LEVEL__SINK) {
ASSERT(type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__CHECKPOINT);
if (type != STREAM_INPUT__DATA_BLOCK && type != STREAM_INPUT__CHECKPOINT) {
stError("s-task:%s invalid block type:%d for sink task, discard", id, type);
continue;
}
int64_t st = taosGetTimestampMs();
@ -801,7 +828,11 @@ bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus) {
}
void streamResumeTask(SStreamTask* pTask) {
ASSERT(pTask->status.schedStatus == TASK_SCHED_STATUS__ACTIVE);
if (pTask->status.schedStatus != TASK_SCHED_STATUS__ACTIVE) {
stError("s-task:%s invalid sched status:%d, not resume task", pTask->id.idStr, pTask->status.schedStatus);
return;
}
const char* id = pTask->id.idStr;
while (1) {

1
source/libs/stream/src/streamHb.c
View File

@ -95,6 +95,7 @@ static int32_t doSendHbMsgInfo(SStreamHbMsg* pMsg, SStreamMeta* pMeta, SEpSet* p
tEncoderInit(&encoder, buf, tlen);
if ((code = tEncodeStreamHbMsg(&encoder, pMsg)) < 0) {
rpcFreeCont(buf);
tEncoderClear(&encoder);
stError("vgId:%d encode stream hb msg failed, code:%s", pMeta->vgId, tstrerror(code));
return TSDB_CODE_FAILED;
}

53
source/libs/stream/src/streamMeta.c
View File

@ -719,9 +719,13 @@ int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTa
}
int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta) {
size_t size = taosHashGetSize(pMeta->pTasksMap);
ASSERT(taosArrayGetSize(pMeta->pTaskList) == taosHashGetSize(pMeta->pTasksMap));
return (int32_t)size;
int32_t size = (int32_t)taosHashGetSize(pMeta->pTasksMap);
int32_t sizeInList = taosArrayGetSize(pMeta->pTaskList);
if (sizeInList != size) {
stError("vgId:%d tasks number not consistent in list:%d and map:%d, ", pMeta->vgId, sizeInList, size);
}
return size;
}
int32_t streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, SStreamTask** pTask) {
@ -775,7 +779,10 @@ static void doRemoveIdFromList(SArray* pTaskList, int32_t num, SStreamTaskId* id
break;
}
}
ASSERT(remove);
if (!remove) {
stError("s-task:0x%x not in meta task list, internal error", id->taskId);
}
}
static int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask, void* param) {
@ -787,6 +794,7 @@ static int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask, void* param) {
int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) {
SStreamTask* pTask = NULL;
int32_t vgId = pMeta->vgId;
// pre-delete operation
streamMetaWLock(pMeta);
@ -799,19 +807,19 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t
// desc the paused task counter
if (streamTaskShouldPause(pTask)) {
int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1);
stInfo("vgId:%d s-task:%s drop stream task. pause task num:%d", pMeta->vgId, pTask->id.idStr, num);
stInfo("vgId:%d s-task:%s drop stream task. pause task num:%d", vgId, pTask->id.idStr, num);
}
// handle the dropping event
(void)streamTaskHandleEventAsync(pTask->status.pSM, TASK_EVENT_DROPPING, streamTaskSendTransSuccessMsg, NULL);
} else {
stDebug("vgId:%d failed to find the task:0x%x, it may be dropped already", pMeta->vgId, taskId);
stDebug("vgId:%d failed to find the task:0x%x, it may be dropped already", vgId, taskId);
streamMetaWUnLock(pMeta);
return 0;
}
streamMetaWUnLock(pMeta);
stDebug("s-task:0x%x vgId:%d set task status:dropping and start to unregister it", taskId, pMeta->vgId);
stDebug("s-task:0x%x vgId:%d set task status:dropping and start to unregister it", taskId, vgId);
while (1) {
int32_t timerActive = 0;
@ -845,14 +853,22 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t
(void)atomic_sub_fetch_32(&pMeta->numOfStreamTasks, 1);
}
(void)taosHashRemove(pMeta->pTasksMap, &id, sizeof(id));
int32_t code = taosHashRemove(pMeta->pTasksMap, &id, sizeof(id));
doRemoveIdFromList(pMeta->pTaskList, (int32_t)taosArrayGetSize(pMeta->pTaskList), &pTask->id);
(void)streamMetaRemoveTask(pMeta, &id);
ASSERT(taosHashGetSize(pMeta->pTasksMap) == taosArrayGetSize(pMeta->pTaskList));
int32_t size = (int32_t) taosHashGetSize(pMeta->pTasksMap);
int32_t sizeInList = taosArrayGetSize(pMeta->pTaskList);
if (sizeInList != size) {
stError("vgId:%d tasks number not consistent in list:%d and map:%d, ", vgId, sizeInList, size);
}
streamMetaWUnLock(pMeta);
ASSERT(pTask->status.timerActive == 0);
int32_t numOfTmr = pTask->status.timerActive;
if (numOfTmr != 0) {
stError("s-task:%s vgId:%d invalid timer Active record:%d, internal error", pTask->id.idStr, vgId, numOfTmr);
}
if (pTask->info.delaySchedParam != 0 && pTask->info.fillHistory == 0) {
stDebug("s-task:%s stop schedTimer, and (before) desc ref:%d", pTask->id.idStr, pTask->refCnt);
(void)taosTmrStop(pTask->schedInfo.pDelayTimer);
@ -862,7 +878,7 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t
streamMetaReleaseTask(pMeta, pTask);
} else {
stDebug("vgId:%d failed to find the task:0x%x, it may have been dropped already", pMeta->vgId, taskId);
stDebug("vgId:%d failed to find the task:0x%x, it may have been dropped already", vgId, taskId);
streamMetaWUnLock(pMeta);
}
@ -1013,7 +1029,10 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) {
tFreeStreamTask(pTask);
STaskId id = streamTaskGetTaskId(pTask);
(void)taosArrayPush(pRecycleList, &id);
void* px = taosArrayPush(pRecycleList, &id);
if (px == NULL) {
stError("s-task:0x%x failed record the task into recycle list due to out of memory", taskId);
}
int32_t total = taosArrayGetSize(pRecycleList);
stDebug("s-task:0x%x is already dropped, add into recycle list, total:%d", taskId, total);
@ -1034,7 +1053,10 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) {
continue;
}
(void)taosArrayPush(pMeta->pTaskList, &pTask->id);
void* px = taosArrayPush(pMeta->pTaskList, &pTask->id);
if (px == NULL) {
stFatal("s-task:0x%x failed to add into task list due to out of memory", pTask->id.taskId);
}
} else {
// todo this should replace the existed object put by replay creating stream task msg from mnode
stError("s-task:0x%x already added into table meta by replaying WAL, need check", pTask->id.taskId);
@ -1044,7 +1066,7 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) {
if (taosHashPut(pMeta->pTasksMap, &id, sizeof(id), &pTask, POINTER_BYTES) != 0) {
stError("s-task:0x%x failed to put into hashTable, code:%s, continue", pTask->id.taskId, tstrerror(terrno));
(void)taosArrayPop(pMeta->pTaskList);
void* px = taosArrayPop(pMeta->pTaskList);
tFreeStreamTask(pTask);
continue;
}
@ -1056,8 +1078,6 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) {
if (streamTaskShouldPause(pTask)) {
(void)atomic_add_fetch_32(&pMeta->numOfPausedTasks, 1);
}
ASSERT(pTask->status.downstreamReady == 0);
}
tdbFree(pKey);
@ -1075,7 +1095,6 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) {
}
int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList);
ASSERT(pMeta->numOfStreamTasks <= numOfTasks && pMeta->numOfPausedTasks <= numOfTasks);
stDebug("vgId:%d load %d tasks into meta from disk completed, streamTask:%d, paused:%d", pMeta->vgId, numOfTasks,
pMeta->numOfStreamTasks, pMeta->numOfPausedTasks);

16
source/libs/stream/src/streamQueue.c
View File

@ -90,7 +90,6 @@ void streamQueueNextItem(SStreamQueue* pQueue, SStreamQueueItem** pItem) {
int8_t flag = atomic_exchange_8(&pQueue->status, STREAM_QUEUE__PROCESSING);
if (flag == STREAM_QUEUE__FAILED) {
ASSERT(pQueue->qItem != NULL);
*pItem = streamQueueCurItem(pQueue);
} else {
pQueue->qItem = NULL;
@ -105,13 +104,20 @@ void streamQueueNextItem(SStreamQueue* pQueue, SStreamQueueItem** pItem) {
}
void streamQueueProcessSuccess(SStreamQueue* queue) {
ASSERT(atomic_load_8(&queue->status) == STREAM_QUEUE__PROCESSING);
if (atomic_load_8(&queue->status) != STREAM_QUEUE__PROCESSING) {
stError("invalid queue status:%d, expect:%d", atomic_load_8(&queue->status), STREAM_QUEUE__PROCESSING);
return;
}
queue->qItem = NULL;
atomic_store_8(&queue->status, STREAM_QUEUE__SUCESS);
}
void streamQueueProcessFail(SStreamQueue* queue) {
ASSERT(atomic_load_8(&queue->status) == STREAM_QUEUE__PROCESSING);
if (atomic_load_8(&queue->status) != STREAM_QUEUE__PROCESSING) {
stError("invalid queue status:%d, expect:%d", atomic_load_8(&queue->status), STREAM_QUEUE__PROCESSING);
return;
}
atomic_store_8(&queue->status, STREAM_QUEUE__FAILED);
}
@ -229,7 +235,6 @@ EExtractDataCode streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueIte
}
} else {
if (*pInput == NULL) {
ASSERT((*numOfBlocks) == 0);
*pInput = qItem;
} else { // merge current block failed, let's handle the already merged blocks.
void* newRet = NULL;
@ -340,7 +345,8 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem)
double size = SIZE_IN_MiB(taosQueueMemorySize(pQueue));
stDebug("s-task:%s data res enqueue, current(blocks:%d, size:%.2fMiB)", pTask->id.idStr, total, size);
} else {
ASSERT(0);
stError("s-task:%s invalid type:%d to put in inputQ", pTask->id.idStr, type);
return TSDB_CODE_INVALID_PARA;
}
if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && type != STREAM_INPUT__CHECKPOINT_TRIGGER &&

47
source/libs/stream/src/streamStartHistory.c
View File

@ -111,11 +111,15 @@ void streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration) {
}
int32_t streamTaskStartScanHistory(SStreamTask* pTask) {
int32_t level = pTask->info.taskLevel;
ETaskStatus status = streamTaskGetStatus(pTask).state;
int32_t level = pTask->info.taskLevel;
SStreamTaskState state = streamTaskGetStatus(pTask);
ASSERT((pTask->status.downstreamReady == 1) && (status == TASK_STATUS__SCAN_HISTORY) &&
(pTask->info.fillHistory == 1));
if (((pTask->status.downstreamReady != 1) || (state.state != TASK_STATUS__SCAN_HISTORY) ||
(pTask->info.fillHistory != 1))) {
stFatal("s-task:%s invalid status:%s to start fill-history task, downReady:%d, is-fill-history task:%d",
pTask->id.idStr, state.name, pTask->status.downstreamReady, pTask->info.fillHistory);
return TSDB_CODE_STREAM_INTERNAL_ERROR;
}
if (level == TASK_LEVEL__SOURCE) {
return doStartScanHistoryTask(pTask);
@ -144,7 +148,6 @@ int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask) {
}
SStreamTaskState p = streamTaskGetStatus(pTask);
ASSERT(p.state == TASK_STATUS__READY);
int8_t schedStatus = pTask->status.schedStatus;
if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
@ -171,8 +174,6 @@ int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask) {
if (code == 0) {
SStreamTaskState p = streamTaskGetStatus(pTask);
ASSERT((p.state == TASK_STATUS__SCAN_HISTORY) && (pTask->info.fillHistory == 1));
stDebug("s-task:%s fill-history task enters into scan-history data stage, status:%s", pTask->id.idStr, p.name);
code = streamTaskStartScanHistory(pTask);
}
@ -348,8 +349,6 @@ void tryLaunchHistoryTask(void* param, void* tmrId) {
}
if (streamTaskShouldStop(*ppTask)) {
ASSERT((*ppTask)->status.timerActive >= 1);
char* p = streamTaskGetStatus(*ppTask).name;
int32_t ref = atomic_sub_fetch_32(&(*ppTask)->status.timerActive, 1);
stDebug("s-task:%s status:%s should stop, quit launch fill-history task timer, retry:%d, ref:%d",
@ -385,7 +384,10 @@ void tryLaunchHistoryTask(void* param, void* tmrId) {
notRetryLaunchFillHistoryTask(pTask, pInfo, now);
} else { // not reach the limitation yet, let's continue retrying launch related fill-history task.
streamTaskSetRetryInfoForLaunch(pHTaskInfo);
ASSERT(pTask->status.timerActive >= 1);
if (pTask->status.timerActive < 1) {
stError("s-task:%s invalid timerActive recorder:%d, abort timer", pTask->id.idStr, pTask->status.timerActive);
return;
}
// abort the timer if intend to stop task
SStreamTask* pHTask = NULL;
@ -451,8 +453,6 @@ int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) {
int32_t hTaskId = pTask->hTaskInfo.id.taskId;
SLaunchHTaskInfo* pInfo = NULL;
ASSERT(hTaskId != 0);
stWarn("s-task:%s vgId:%d failed to launch history task:0x%x, since not built yet", idStr, pMeta->vgId, hTaskId);
STaskId id = streamTaskGetTaskId(pTask);
@ -480,11 +480,18 @@ int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) {
return terrno;
}
ASSERT(ref >= 1);
if (ref < 1) {
stError("s-task:%s invalid timerActive recorder:%d, abort timer", pTask->id.idStr, pTask->status.timerActive);
return TSDB_CODE_STREAM_INTERNAL_ERROR;
}
stDebug("s-task:%s set timer active flag, ref:%d", idStr, ref);
} else { // timer exists
ASSERT(pTask->status.timerActive >= 1);
if (pTask->status.timerActive < 1) {
stError("s-task:%s invalid timerActive recorder:%d, abort timer", pTask->id.idStr, pTask->status.timerActive);
return TSDB_CODE_STREAM_INTERNAL_ERROR;
}
stDebug("s-task:%s set timer active flag, task timer not null", idStr);
streamTmrReset(tryLaunchHistoryTask, WAIT_FOR_MINIMAL_INTERVAL, pInfo, streamTimer, &pTask->hTaskInfo.pTimer,
pTask->pMeta->vgId, " start-history-task-tmr");
@ -500,7 +507,11 @@ int32_t streamTaskResetTimewindowFilter(SStreamTask* pTask) {
bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t nextProcessVer) {
SVersionRange* pRange = &pTask->dataRange.range;
ASSERT(nextProcessVer >= pRange->maxVer);
if (nextProcessVer < pRange->maxVer) {
stError("s-task:%s next processdVer:%"PRId64" is less than range max ver:%"PRId64, pTask->id.idStr, nextProcessVer,
pRange->maxVer);
return true;
}
// maxVer for fill-history task is the version, where the last timestamp is acquired.
// it's also the maximum version to scan data in tsdb.
@ -538,7 +549,11 @@ int32_t streamTaskSetRangeStreamCalc(SStreamTask* pTask) {
return TSDB_CODE_SUCCESS;
} else {
ASSERT(pTask->info.fillHistory == 0);
if (pTask->info.fillHistory != 0) {
stError("s-task:%s task should not be fill-history task, internal error", pTask->id.idStr);
return TSDB_CODE_STREAM_INTERNAL_ERROR;
}
if (pTask->info.taskLevel >= TASK_LEVEL__AGG) {
return TSDB_CODE_SUCCESS;
}

20
source/libs/stream/src/streamTask.c
View File

@ -134,8 +134,9 @@ int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool
return code;
}
if (fillHistory) {
ASSERT(hasFillhistory);
if (fillHistory && !hasFillhistory) {
stError("s-task:0x%x create task failed, due to inconsistent fill-history flag", pTask->id.taskId);
return TSDB_CODE_INVALID_PARA;
}
epsetAssign(&(pTask->info.mnodeEpset), pEpset);
@ -728,8 +729,11 @@ void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId) {
if ((pInfo != NULL) && pInfo->dataAllowed) {
pInfo->dataAllowed = false;
int32_t t = atomic_add_fetch_32(&pTask->upstreamInfo.numOfClosed, 1);
ASSERT(t <= streamTaskGetNumOfUpstream(pTask));
if (pTask->upstreamInfo.numOfClosed < streamTaskGetNumOfUpstream(pTask)) {
int32_t t = atomic_add_fetch_32(&pTask->upstreamInfo.numOfClosed, 1);
} else {
stError("s-task:%s not inc closed input, since they have been all closed already", pTask->id.idStr);
}
}
}
@ -739,7 +743,7 @@ void streamTaskOpenUpstreamInput(SStreamTask* pTask, int32_t taskId) {
if (pInfo != NULL && (!pInfo->dataAllowed)) {
int32_t t = atomic_sub_fetch_32(&pTask->upstreamInfo.numOfClosed, 1);
ASSERT(t >= 0);
stDebug("s-task:%s open inputQ for upstream:0x%x, remain closed:%d", pTask->id.idStr, taskId, t);
pInfo->dataAllowed = true;
}
}
@ -775,8 +779,6 @@ int8_t streamTaskSetSchedStatusActive(SStreamTask* pTask) {
int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask) {
streamMutexLock(&pTask->lock);
int8_t status = pTask->status.schedStatus;
ASSERT(status == TASK_SCHED_STATUS__WAITING || status == TASK_SCHED_STATUS__ACTIVE ||
status == TASK_SCHED_STATUS__INACTIVE);
pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE;
streamMutexUnlock(&pTask->lock);
@ -868,6 +870,7 @@ int32_t streamSendChkptReportMsg(SStreamTask* pTask, SCheckpointInfo* pCheckpoin
tEncoderInit(&encoder, buf, tlen);
if ((code = tEncodeStreamTaskChkptReport(&encoder, &req)) < 0) {
rpcFreeCont(buf);
tEncoderClear(&encoder);
stError("s-task:%s vgId:%d encode stream task checkpoint-report msg failed, code:%s", id, vgId, tstrerror(code));
return -1;
}
@ -892,8 +895,6 @@ void streamTaskInitForLaunchHTask(SHistoryTaskInfo* pInfo) {
}
void streamTaskSetRetryInfoForLaunch(SHistoryTaskInfo* pInfo) {
ASSERT(pInfo->tickCount == 0);
pInfo->waitInterval *= RETRY_LAUNCH_INTERVAL_INC_RATE;
pInfo->tickCount = ceil(pInfo->waitInterval / WAIT_FOR_MINIMAL_INTERVAL);
pInfo->retryTimes += 1;
@ -1021,6 +1022,7 @@ int32_t streamTaskSendCheckpointReq(SStreamTask* pTask) {
tEncoderInit(&encoder, buf, tlen);
if ((code = tEncodeStreamTaskCheckpointReq(&encoder, &req)) < 0) {
rpcFreeCont(buf);
tEncoderClear(&encoder);
stError("s-task:%s vgId:%d encode stream task req checkpoint msg failed, code:%s", id, vgId, tstrerror(code));
return -1;
}

3
source/libs/stream/src/streamTaskSm.c
View File

@ -191,10 +191,9 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName,
GET_EVT_NAME(pEvtInfo->event), pSM->current.name);
// remove it
(void) taosArrayPop(pSM->pWaitingEventList);
void* px = taosArrayPop(pSM->pWaitingEventList);
STaskStateTrans* pNextTrans = streamTaskFindTransform(pSM->current.state, pEvtInfo->event);
ASSERT(pSM->pActiveTrans == NULL && pNextTrans != NULL);
pSM->pActiveTrans = pNextTrans;
pSM->startTs = taosGetTimestampMs();

4
source/libs/stream/src/streamUpdate.c
View File

@ -455,7 +455,7 @@ int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo,
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) {
if (tStartEncode(&encoder) != 0) {
code = TSDB_CODE_FAILED;
QUERY_CHECK_CODE(code, lino, _end);
}
@ -551,10 +551,10 @@ int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo,
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
*pLen = tlen;
_end:
tEncoderClear(&encoder);
if (code != TSDB_CODE_SUCCESS) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}

5
source/libs/transport/src/thttp.c
View File

@ -507,6 +507,7 @@ static bool httpFailFastShoudIgnoreMsg(SHashObj* pTable, char* server, int16_t p
}
}
static void httpFailFastMayUpdate(SHashObj* pTable, char* server, int16_t port, int8_t succ) {
int32_t code = 0;
char buf[256] = {0};
sprintf(buf, "%s:%d", server, port);
@ -514,7 +515,9 @@ static void httpFailFastMayUpdate(SHashObj* pTable, char* server, int16_t port,
(void)taosHashRemove(pTable, buf, strlen(buf));
} else {
int32_t st = taosGetTimestampSec();
(void)taosHashPut(pTable, buf, strlen(buf), &st, sizeof(st));
if ((code = taosHashPut(pTable, buf, strlen(buf), &st, sizeof(st))) != 0) {
tError("http-report failed to update conn status, dst:%s, reason:%s", buf, tstrerror(code));
}
}
return;
}

97
source/libs/transport/src/transCli.c
View File

@ -332,6 +332,21 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) {
transQueueClear(&conn->cliMsgs);
memset(&conn->ctx, 0, sizeof(conn->ctx));
}
void cliResetTimer(SCliThrd* pThrd, SCliConn* conn) {
if (conn->timer) {
if (uv_is_active((uv_handle_t*)conn->timer)) {
tDebug("%s conn %p stop timer", CONN_GET_INST_LABEL(conn), conn);
(void)uv_timer_stop(conn->timer);
}
if (taosArrayPush(pThrd->timerList, &conn->timer) == NULL) {
tError("failed to push timer %p to list, reason:%s", conn->timer, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
conn->timer = NULL;
return;
}
conn->timer->data = NULL;
conn->timer = NULL;
}
}
bool cliMaySendCachedMsg(SCliConn* conn) {
if (!transQueueEmpty(&conn->cliMsgs)) {
SCliMsg* pCliMsg = NULL;
@ -376,15 +391,7 @@ void cliHandleResp(SCliConn* conn) {
SCliThrd* pThrd = conn->hostThrd;
STrans* pTransInst = pThrd->pTransInst;
if (conn->timer) {
if (uv_is_active((uv_handle_t*)conn->timer)) {
tDebug("%s conn %p stop timer", CONN_GET_INST_LABEL(conn), conn);
(void)uv_timer_stop(conn->timer);
}
(void)taosArrayPush(pThrd->timerList, &conn->timer);
conn->timer->data = NULL;
conn->timer = NULL;
}
cliResetTimer(pThrd, conn);
STransMsgHead* pHead = NULL;
@ -593,8 +600,8 @@ void cliConnTimeout(uv_timer_t* handle) {
(void)uv_timer_stop(handle);
handle->data = NULL;
(void)taosArrayPush(pThrd->timerList, &conn->timer);
conn->timer = NULL;
cliResetTimer(pThrd, conn);
cliMayUpdateFqdnCache(pThrd->fqdn2ipCache, conn->dstAddr);
cliHandleFastFail(conn, UV_ECANCELED);
@ -643,13 +650,16 @@ void* destroyConnPool(SCliThrd* pThrd) {
}
static SCliConn* getConnFromPool(SCliThrd* pThrd, char* key, bool* exceed) {
int32_t code = 0;
void* pool = pThrd->pool;
STrans* pTranInst = pThrd->pTransInst;
size_t klen = strlen(key);
SConnList* plist = taosHashGet((SHashObj*)pool, key, klen);
if (plist == NULL) {
SConnList list = {0};
(void)taosHashPut((SHashObj*)pool, key, klen, (void*)&list, sizeof(list));
if ((code = taosHashPut((SHashObj*)pool, key, klen, (void*)&list, sizeof(list))) != 0) {
return NULL;
}
plist = taosHashGet(pool, key, klen);
SMsgList* nList = taosMemoryCalloc(1, sizeof(SMsgList));
@ -686,13 +696,17 @@ static SCliConn* getConnFromPool(SCliThrd* pThrd, char* key, bool* exceed) {
}
static SCliConn* getConnFromPool2(SCliThrd* pThrd, char* key, SCliMsg** pMsg) {
int32_t code = 0;
void* pool = pThrd->pool;
STrans* pTransInst = pThrd->pTransInst;
size_t klen = strlen(key);
SConnList* plist = taosHashGet((SHashObj*)pool, key, klen);
if (plist == NULL) {
SConnList list = {0};
(void)taosHashPut((SHashObj*)pool, key, klen, (void*)&list, sizeof(list));
if ((code = taosHashPut((SHashObj*)pool, key, klen, (void*)&list, sizeof(list))) != 0) {
tError("failed to put key %s to pool, reason:%s", key, tstrerror(code));
return NULL;
}
plist = taosHashGet(pool, key, klen);
SMsgList* nList = taosMemoryCalloc(1, sizeof(SMsgList));
@ -805,12 +819,8 @@ static void addConnToPool(void* pool, SCliConn* conn) {
}
SCliThrd* thrd = conn->hostThrd;
if (conn->timer != NULL) {
(void)uv_timer_stop(conn->timer);
(void)taosArrayPush(thrd->timerList, &conn->timer);
conn->timer->data = NULL;
conn->timer = NULL;
}
cliResetTimer(thrd, conn);
if (T_REF_VAL_GET(conn) > 1) {
transUnrefCliHandle(conn);
}
@ -1053,12 +1063,7 @@ static void cliDestroyConn(SCliConn* conn, bool clear) {
transDQCancel(pThrd->timeoutQueue, conn->task);
conn->task = NULL;
}
if (conn->timer != NULL) {
(void)uv_timer_stop(conn->timer);
conn->timer->data = NULL;
(void)taosArrayPush(pThrd->timerList, &conn->timer);
conn->timer = NULL;
}
cliResetTimer(pThrd, conn);
if (clear) {
if (!uv_is_closing((uv_handle_t*)conn->stream)) {
@ -1073,12 +1078,7 @@ static void cliDestroy(uv_handle_t* handle) {
}
SCliConn* conn = handle->data;
SCliThrd* pThrd = conn->hostThrd;
if (conn->timer != NULL) {
(void)uv_timer_stop(conn->timer);
(void)taosArrayPush(pThrd->timerList, &conn->timer);
conn->timer->data = NULL;
conn->timer = NULL;
}
cliResetTimer(pThrd, conn);
(void)atomic_sub_fetch_32(&pThrd->connCount, 1);
@ -1385,10 +1385,7 @@ static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd) {
uint32_t ipaddr = 0;
if ((code = cliGetIpFromFqdnCache(pThrd->fqdn2ipCache, pList->ip, &ipaddr)) != 0) {
(void)uv_timer_stop(conn->timer);
conn->timer->data = NULL;
(void)taosArrayPush(pThrd->timerList, &conn->timer);
conn->timer = NULL;
cliResetTimer(pThrd, conn);
cliHandleFastFail(conn, code);
return;
}
@ -1421,10 +1418,7 @@ static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd) {
ret = uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, cliConnCb);
if (ret != 0) {
(void)uv_timer_stop(conn->timer);
conn->timer->data = NULL;
(void)taosArrayPush(pThrd->timerList, &conn->timer);
conn->timer = NULL;
cliResetTimer(pThrd, conn);
cliMayUpdateFqdnCache(pThrd->fqdn2ipCache, conn->dstAddr);
cliHandleFastFail(conn, -1);
@ -1502,7 +1496,10 @@ static void cliHandleFastFail(SCliConn* pConn, int status) {
}
} else {
SFailFastItem item = {.count = 1, .timestamp = cTimestamp};
(void)taosHashPut(pThrd->failFastCache, pConn->dstAddr, strlen(pConn->dstAddr), &item, sizeof(SFailFastItem));
int32_t code = taosHashPut(pThrd->failFastCache, pConn->dstAddr, strlen(pConn->dstAddr), &item, sizeof(SFailFastItem));
if (code != 0) {
tError("failed to put fail-fast item to cache, reason:%s", tstrerror(code));
}
}
}
} else {
@ -1522,10 +1519,7 @@ void cliConnCb(uv_connect_t* req, int status) {
if (pConn->timer == NULL) {
timeout = true;
} else {
(void)uv_timer_stop(pConn->timer);
pConn->timer->data = NULL;
(void)taosArrayPush(pThrd->timerList, &pConn->timer);
pConn->timer = NULL;
cliResetTimer(pThrd, pConn);
}
STUB_RAND_NETWORK_ERR(status);
@ -1870,11 +1864,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) {
uint32_t ipaddr;
int32_t code = cliGetIpFromFqdnCache(pThrd->fqdn2ipCache, fqdn, &ipaddr);
if (code != 0) {
(void)uv_timer_stop(conn->timer);
conn->timer->data = NULL;
(void)taosArrayPush(pThrd->timerList, &conn->timer);
conn->timer = NULL;
cliResetTimer(pThrd, conn);
cliHandleExcept(conn, code);
return;
}
@ -1910,10 +1900,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) {
ret = uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, cliConnCb);
if (ret != 0) {
(void)uv_timer_stop(conn->timer);
conn->timer->data = NULL;
(void)taosArrayPush(pThrd->timerList, &conn->timer);
conn->timer = NULL;
cliResetTimer(pThrd, conn);
cliMayUpdateFqdnCache(pThrd->fqdn2ipCache, conn->dstAddr);
cliHandleFastFail(conn, ret);
@ -2377,7 +2364,9 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end);
}
(void)uv_timer_init(pThrd->loop, timer);
(void)taosArrayPush(pThrd->timerList, &timer);
if (taosArrayPush(pThrd->timerList, &timer) == NULL) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end);
}
}
pThrd->pool = createConnPool(4);

13
source/libs/transport/src/transComm.c
View File

@ -375,11 +375,10 @@ void transCtxMerge(STransCtx* dst, STransCtx* src) {
STransCtxVal* sVal = (STransCtxVal*)iter;
key = taosHashGetKey(sVal, &klen);
// STransCtxVal* dVal = taosHashGet(dst->args, key, klen);
// if (dVal) {
// dst->freeFunc(dVal->val);
// }
(void)taosHashPut(dst->args, key, klen, sVal, sizeof(*sVal));
int32_t code = taosHashPut(dst->args, key, klen, sVal, sizeof(*sVal));
if (code != 0) {
tError("failed to put val to hash, reason:%s", tstrerror(code));
}
iter = taosHashIterate(src->args, iter);
}
taosHashCleanup(src->args);
@ -453,7 +452,9 @@ bool transQueuePush(STransQueue* queue, void* arg) {
if (queue->q == NULL) {
return true;
}
(void)taosArrayPush(queue->q, &arg);
if (taosArrayPush(queue->q, &arg) == NULL) {
return false;
}
if (taosArrayGetSize(queue->q) > 1) {
return false;
}

3
source/util/src/tcache.c
View File

@ -787,9 +787,6 @@ void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) {
STrashElem *pElem = pCacheObj->pTrash;
while (pElem) {
T_REF_VAL_CHECK(pElem->pData);
// A S S E R T(pElem->next != pElem && pElem->prev != pElem);
if (force || (T_REF_VAL_GET(pElem->pData) == 0)) {
uDebug("cache:%s, key:%p, %p removed from trashcan. numOfElem in trashcan:%d", pCacheObj->name, pElem->pData->key,
pElem->pData->data, pCacheObj->numOfElemsInTrash - 1);

34
source/util/src/tconfig.c
View File

@ -385,10 +385,6 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy
(void)taosThreadMutexUnlock(&pCfg->lock);
}
if (code != 0){
uError("cfg:%s, type:%s src:%s value:%s failed since %s", pItem->name, cfgDtypeStr(pItem->dtype),
cfgStypeStr(stype), value, tstrerror(code));
}
TAOS_RETURN(code);
}
@ -1000,7 +996,7 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) {
const char *filepath = ".env";
if (envFile != NULL && strlen(envFile) > 0) {
if (!taosCheckExistFile(envFile)) {
uError("failed to load env file:%s", envFile);
(void)printf("failed to load env file:%s\n", envFile);
TAOS_RETURN(TSDB_CODE_NOT_FOUND);
}
filepath = envFile;
@ -1075,7 +1071,7 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
uInfo("failed to load from cfg file %s since %s, use default parameters", filepath, tstrerror(code));
TAOS_RETURN(TSDB_CODE_SUCCESS);
} else {
uError("failed to load from cfg file %s since %s", filepath, tstrerror(code));
(void)printf("failed to load from cfg file %s since %s\n", filepath, tstrerror(code));
TAOS_RETURN(code);
}
}
@ -1117,7 +1113,10 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
}
code = cfgSetItem(pConfig, name, newValue, CFG_STYPE_CFG_FILE, true);
if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) break;
if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) {
(void)printf("cfg:%s, value:%s failed since %s\n", name,newValue, tstrerror(code));
break;
}
} else {
(void)paGetToken(value + vlen + 1, &value2, &vlen2);
if (vlen2 != 0) {
@ -1131,7 +1130,10 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
}
code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE, true);
if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) break;
if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) {
(void)printf("cfg:%s, value:%s failed since %s\n", name, value, tstrerror(code));
break;
}
}
if (strcasecmp(name, "dataDir") == 0) {
@ -1154,7 +1156,7 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
uInfo("load from cfg file %s success", filepath);
TAOS_RETURN(TSDB_CODE_SUCCESS);
} else {
uError("failed to load from cfg file %s since %s", filepath, tstrerror(code));
(void)printf("failed to load from cfg file %s since %s\n", filepath, tstrerror(code));
TAOS_RETURN(code);
}
}
@ -1236,7 +1238,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {
char *p = strchr(url, ':');
if (p == NULL) {
uError("fail to load apoll url: %s, unknown format", url);
(void)printf("fail to load apoll url: %s, unknown format\n", url);
TAOS_RETURN(TSDB_CODE_INVALID_PARA);
}
p++;
@ -1244,7 +1246,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {
if (strncmp(url, "jsonFile", 8) == 0) {
char *filepath = p;
if (!taosCheckExistFile(filepath)) {
uError("failed to load json file:%s", filepath);
(void)printf("failed to load json file:%s\n", filepath);
TAOS_RETURN(TSDB_CODE_NOT_FOUND);
}
@ -1256,7 +1258,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {
char *buf = taosMemoryMalloc(fileSize + 1);
if (!buf) {
(void)taosCloseFile(&pFile);
uError("load json file error: %s, failed to alloc memory", filepath);
(void)printf("load json file error: %s, failed to alloc memory\n", filepath);
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
@ -1264,7 +1266,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {
(void)taosLSeekFile(pFile, 0, SEEK_SET);
if (taosReadFile(pFile, buf, fileSize) <= 0) {
(void)taosCloseFile(&pFile);
uError("load json file error: %s", filepath);
(void)printf("load json file error: %s\n", filepath);
taosMemoryFreeClear(buf);
TAOS_RETURN(TSDB_CODE_INVALID_DATA_FMT);
}
@ -1273,7 +1275,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {
if (NULL == pJson) {
const char *jsonParseError = tjsonGetError();
if (jsonParseError != NULL) {
uError("load json file parse error: %s", jsonParseError);
(void)printf("load json file parse error: %s\n", jsonParseError);
}
taosMemoryFreeClear(buf);
TAOS_CHECK_EXIT(TSDB_CODE_INVALID_DATA_FMT);
@ -1341,7 +1343,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {
// } else if (strncmp(url, "jsonUrl", 7) == 0) {
// } else if (strncmp(url, "etcdUrl", 7) == 0) {
} else {
uError("Unsupported url: %s", url);
(void)printf("Unsupported url: %s\n", url);
TAOS_RETURN(TSDB_CODE_INVALID_PARA);
}
@ -1353,7 +1355,7 @@ _exit:
taosMemoryFree(cfgLineBuf);
tjsonDelete(pJson);
if (code != 0) {
uError("failed to load from apollo url:%s at line %d since %s", url, lino, tstrerror(code));
(void)printf("failed to load from apollo url:%s at line %d since %s\n", url, lino, tstrerror(code));
}
TAOS_RETURN(code);
}

14
source/util/src/tgeosctx.c
View File

@ -72,11 +72,23 @@ int32_t getThreadLocalGeosCtx(SGeosContext **ppCtx) {
}
int32_t code = 0, lino = 0;
if (atomic_val_compare_exchange_8(&tlGeosCtxKeyInited, 0, 1) == 0) {
int8_t old;
int32_t nLoops = 0;
while (1) {
old = atomic_val_compare_exchange_8(&tlGeosCtxKeyInited, 0, 2);
if (old != 2) break;
if (++nLoops > 1000) {
(void)sched_yield();
nLoops = 0;
}
}
if (old == 0) {
if ((taosThreadKeyCreate(&tlGeosCtxKey, destroyThreadLocalGeosCtx)) != 0) {
atomic_store_8(&tlGeosCtxKeyInited, 0);
TAOS_CHECK_EXIT(TAOS_SYSTEM_ERROR(errno));
}
atomic_store_8(&tlGeosCtxKeyInited, 1);
}
SGeosContext *tlGeosCtxObj = (SGeosContext *)taosMemoryCalloc(1, sizeof(SGeosContext));

14
source/util/src/tlog.c
View File

@ -206,7 +206,7 @@ int32_t taosInitSlowLog() {
(void)taosUmaskFile(0);
tsLogObj.slowHandle->pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (tsLogObj.slowHandle->pFile == NULL) {
printf("\nfailed to open slow log file:%s, reason:%s\n", name, strerror(errno));
(void)printf("\nfailed to open slow log file:%s, reason:%s\n", name, strerror(errno));
return TAOS_SYSTEM_ERROR(errno);
}
@ -529,7 +529,7 @@ static int32_t taosInitNormalLog(const char *logName, int32_t maxFileNum) {
processLogFileName(logName, maxFileNum);
char name[PATH_MAX + 50] = "\0";
sprintf(name, "%s.%d", tsLogObj.logName, tsLogObj.flag);
(void)sprintf(name, "%s.%d", tsLogObj.logName, tsLogObj.flag);
(void)taosThreadMutexInit(&tsLogObj.logMutex, NULL);
(void)taosUmaskFile(0);
@ -538,7 +538,7 @@ static int32_t taosInitNormalLog(const char *logName, int32_t maxFileNum) {
tsLogObj.logHandle->pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE);
if (tsLogObj.logHandle->pFile == NULL) {
printf("\nfailed to open log file:%s, reason:%s\n", name, strerror(errno));
(void)printf("\nfailed to open log file:%s, reason:%s\n", name, strerror(errno));
return TAOS_SYSTEM_ERROR(errno);
}
(void)taosLockLogFile(tsLogObj.logHandle->pFile);
@ -546,18 +546,18 @@ static int32_t taosInitNormalLog(const char *logName, int32_t maxFileNum) {
// only an estimate for number of lines
int64_t filesize = 0;
if (taosFStatFile(tsLogObj.logHandle->pFile, &filesize, NULL) < 0) {
printf("\nfailed to fstat log file:%s, reason:%s\n", name, strerror(errno));
(void)printf("\nfailed to fstat log file:%s, reason:%s\n", name, strerror(errno));
return TAOS_SYSTEM_ERROR(errno);
}
tsLogObj.lines = (int32_t)(filesize / 60);
(void)taosLSeekFile(tsLogObj.logHandle->pFile, 0, SEEK_END);
sprintf(name, "==================================================\n");
(void)sprintf(name, "==================================================\n");
(void)taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name));
sprintf(name, " new log file \n");
(void)sprintf(name, " new log file \n");
(void)taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name));
sprintf(name, "==================================================\n");
(void)sprintf(name, "==================================================\n");
(void)taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name));
return 0;

3
source/util/src/tlrucache.c
View File

@ -445,6 +445,9 @@ static LRUStatus taosLRUCacheShardInsert(SLRUCacheShard *shard, const void *key,
LRUPriority priority, void *ud) {
SLRUEntry *e = taosMemoryCalloc(1, sizeof(SLRUEntry) - 1 + keyLen);
if (!e) {
if (deleter) {
(*deleter)(key, keyLen, value, ud);
}
return TAOS_LRU_STATUS_FAIL;
}

4
source/util/src/tref.c
View File

@ -455,9 +455,7 @@ static void taosLockList(int64_t *lockedBy) {
static void taosUnlockList(int64_t *lockedBy) {
int64_t tid = taosGetSelfPthreadId();
if (atomic_val_compare_exchange_64(lockedBy, tid, 0) != tid) {
ASSERTS(false, "atomic_val_compare_exchange_64 tid failed");
}
(void)atomic_val_compare_exchange_64(lockedBy, tid, 0);
}
static void taosInitRefModule(void) { (void)taosThreadMutexInit(&tsRefMutex, NULL); }

53
tests/ci/count_assert.py Normal file
View File

@ -0,0 +1,53 @@
import os
import re
# List of source directories to search
source_dirs = [
"community/source",
"community/include",
"enterprise/src/plugins/"
]
# List of directories to exclude
exclude_dirs = [
"community/source/client/jni"
]
# List of files to exclude
exclude_source_files = [
"community/source/libs/parser/src/sql.c",
"community/source/util/src/tlog.c",
"community/include/util/tlog.h"
]
def grep_asserts_in_file(file_path, summary_list, detaild_list):
"""Search for assert, ASSERTS, or ASSERT function calls in a file and print them."""
match_count = 0
with open(file_path, 'r') as file:
for line_number, line in enumerate(file, start=1):
if re.search(r'\bassert\(.*\)|\bASSERT\(.*\)|\bASSERTS\(.*\)|\bASSERT_CORE\(.*\)', line):
detaild_list.append(f"{file_path}:{line_number}: {line.strip()}")
match_count += 1
if match_count > 0:
summary_list.append(f"Total matches in {file_path}: {match_count}")
def traverse_and_grep(source_dirs, exclude_dirs, exclude_source_files):
"""Traverse directories and grep for assert, ASSERTS, or ASSERT function calls in .h and .c files."""
summary_list = []
detaild_list = []
for source_dir in source_dirs:
for root, _, files in os.walk(source_dir):
# Skip directories named 'test' or 'tests' and directories in exclude_dirs
if 'test' in root.split(os.sep) or 'tests' in root.split(os.sep) or any(excluded in root for excluded in exclude_dirs):
continue
for file in files:
if file.endswith((".h", ".c")):
file_path = os.path.join(root, file)
if file_path not in exclude_source_files:
grep_asserts_in_file(file_path, summary_list, detaild_list)
return summary_list, detaild_list
if __name__ == "__main__":
summary_list, detaild_list = traverse_and_grep(source_dirs, exclude_dirs, exclude_source_files)
print("\n".join(summary_list))
# print("\n".join(detaild_list))

2
tests/parallel_test/cases.task
View File

@ -144,6 +144,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery2.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py
@ -227,6 +228,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/basic5.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb.py -N 3 -n 3
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/ts-4674.py -N 3 -n 3
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb1.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb2.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb3.py

167
tests/system-test/2-query/nestedQuery2.py Normal file
View File

@ -0,0 +1,167 @@
from random import randrange
import time
import threading
import secrets
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
from util.common import *
# from tmqCommon import *
class TDTestCase:
updatecfgDict = {'asynclog': 0, 'ttlUnit': 1, 'ttlPushInterval': 5, 'ratioOfVnodeStreamThrea': 4}
def __init__(self):
self.vgroups = 4
self.ctbNum = 10
self.rowsPerTbl = 10000
self.duraion = '1h'
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), False)
def create_database(self, tsql, dbName, dropFlag=1, vgroups=2, replica=1, duration: str = '1d'):
if dropFlag == 1:
tsql.execute("drop database if exists %s" % (dbName))
tsql.execute("create database if not exists %s vgroups %d replica %d duration %s" % (
dbName, vgroups, replica, duration))
tdLog.debug("complete to create database %s" % (dbName))
return
def create_stable(self, tsql, paraDict):
colString = tdCom.gen_column_type_str(
colname_prefix=paraDict["colPrefix"], column_elm_list=paraDict["colSchema"])
tagString = tdCom.gen_tag_type_str(
tagname_prefix=paraDict["tagPrefix"], tag_elm_list=paraDict["tagSchema"])
sqlString = f"create table if not exists %s.%s (%s) tags (%s)" % (
paraDict["dbName"], paraDict["stbName"], colString, tagString)
tdLog.debug("%s" % (sqlString))
tsql.execute(sqlString)
return
def create_ctable(self, tsql=None, dbName='dbx', stbName='stb', ctbPrefix='ctb', ctbNum=1, ctbStartIdx=0):
for i in range(ctbNum):
sqlString = "create table %s.%s%d using %s.%s tags(%d, 'tb%d', 'tb%d', %d, %d, %d)" % (dbName, ctbPrefix, i+ctbStartIdx, dbName, stbName, (i+ctbStartIdx) % 5, i+ctbStartIdx + random.randint(
1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100))
tsql.execute(sqlString)
tdLog.debug("complete to create %d child tables by %s.%s" %
(ctbNum, dbName, stbName))
return
def init_normal_tb(self, tsql, db_name: str, tb_name: str, rows: int, start_ts: int, ts_step: int):
sql = 'CREATE TABLE %s.%s (ts timestamp, c1 INT, c2 INT, c3 INT, c4 double, c5 VARCHAR(255))' % (
db_name, tb_name)
tsql.execute(sql)
sql = 'INSERT INTO %s.%s values' % (db_name, tb_name)
for j in range(rows):
sql += f'(%d, %d,%d,%d,{random.random()},"varchar_%d"),' % (start_ts + j * ts_step + randrange(500), j %
10 + randrange(200), j % 10, j % 10, j % 10 + randrange(100))
tsql.execute(sql)
def insert_data(self, tsql, dbName, ctbPrefix, ctbNum, rowsPerTbl, batchNum, startTs, tsStep):
tdLog.debug("start to insert data ............")
tsql.execute("use %s" % dbName)
pre_insert = "insert into "
sql = pre_insert
for i in range(ctbNum):
rowsBatched = 0
sql += " %s.%s%d values " % (dbName, ctbPrefix, i)
for j in range(rowsPerTbl):
if (i < ctbNum/2):
sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d') " % (startTs + j*tsStep + randrange(
500), j % 10 + randrange(100), j % 10 + randrange(200), j % 10, j % 10, j % 10, j % 10, j % 10, j % 10)
else:
sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d') " % (
startTs + j*tsStep + randrange(500), j % 10, j % 10, j % 10, j % 10, j % 10, j % 10)
rowsBatched += 1
if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)):
tsql.execute(sql)
rowsBatched = 0
if j < rowsPerTbl - 1:
sql = "insert into %s.%s%d values " % (dbName, ctbPrefix, i)
else:
sql = "insert into "
if sql != pre_insert:
tsql.execute(sql)
tdLog.debug("insert data ............ [OK]")
return
def init_data(self, db: str = 'test', ctb_num: int = 10, rows_per_ctb: int = 10000, start_ts: int = 1537146000000, ts_step: int = 500):
tdLog.printNoPrefix(
"======== prepare test env include database, stable, ctables, and insert data: ")
paraDict = {'dbName': db,
'dropFlag': 1,
'vgroups': 2,
'stbName': 'meters',
'colPrefix': 'c',
'tagPrefix': 't',
'colSchema': [{'type': 'INT', 'count': 1}, {'type': 'BIGINT', 'count': 1}, {'type': 'FLOAT', 'count': 1}, {'type': 'DOUBLE', 'count': 1}, {'type': 'smallint', 'count': 1}, {'type': 'tinyint', 'count': 1}, {'type': 'bool', 'count': 1}, {'type': 'binary', 'len': 10, 'count': 1}, {'type': 'nchar', 'len': 10, 'count': 1}],
'tagSchema': [{'type': 'INT', 'count': 1}, {'type': 'nchar', 'len': 20, 'count': 1}, {'type': 'binary', 'len': 20, 'count': 1}, {'type': 'BIGINT', 'count': 1}, {'type': 'smallint', 'count': 1}, {'type': 'DOUBLE', 'count': 1}],
'ctbPrefix': 't',
'ctbStartIdx': 0,
'ctbNum': ctb_num,
'rowsPerTbl': rows_per_ctb,
'batchNum': 3000,
'startTs': start_ts,
'tsStep': ts_step}
paraDict['vgroups'] = self.vgroups
paraDict['ctbNum'] = ctb_num
paraDict['rowsPerTbl'] = rows_per_ctb
tdLog.info("create database")
self.create_database(tsql=tdSql, dbName=paraDict["dbName"], dropFlag=paraDict["dropFlag"],
vgroups=paraDict["vgroups"], replica=self.replicaVar, duration=self.duraion)
tdLog.info("create stb")
self.create_stable(tsql=tdSql, paraDict=paraDict)
tdLog.info("create child tables")
self.create_ctable(tsql=tdSql, dbName=paraDict["dbName"],
stbName=paraDict["stbName"], ctbPrefix=paraDict["ctbPrefix"],
ctbNum=paraDict["ctbNum"], ctbStartIdx=paraDict["ctbStartIdx"])
self.insert_data(tsql=tdSql, dbName=paraDict["dbName"],
ctbPrefix=paraDict["ctbPrefix"], ctbNum=paraDict["ctbNum"],
rowsPerTbl=paraDict["rowsPerTbl"], batchNum=paraDict["batchNum"],
startTs=paraDict["startTs"], tsStep=paraDict["tsStep"])
self.init_normal_tb(tdSql, paraDict['dbName'], 'norm_tb',
paraDict['rowsPerTbl'], paraDict['startTs'], paraDict['tsStep'])
def test_select_asterisk_from_subquery_with_duplicate_aliasname(self):
sql = "select * from (select c8 as a, c9 as a from t1 order by ts desc limit 10)t;"
tdSql.query(sql, queryTimes=1)
tdSql.checkData(0, 0, "binary9")
tdSql.checkData(0, 1, "nchar9")
sql = "select * from (select c8 as a, c9 as a, ts from t1 order by ts desc limit 10)t order by ts desc;"
tdSql.query(sql, queryTimes=1)
tdSql.checkData(0, 0, "binary9")
tdSql.checkData(0, 1, "nchar9")
sql = "select * from (select c8 as a, c9 as a, ts, t1 from t1 order by ts desc limit 10)t partition by t1 order by ts desc;"
tdSql.query(sql, queryTimes=1)
tdSql.checkData(0, 0, "binary9")
tdSql.checkData(0, 1, "nchar9")
sql = " select * from (select a.c8, b.c8, a.ts, a.t1,b.t1 from t1 a, t3 b where a.ts = b.ts order by a.ts)ttt"
tdSql.query(sql, queryTimes=1)
tdSql.checkData(0, 3, 1)
tdSql.checkData(0, 4, 3)
def run(self):
self.init_data()
self.test_select_asterisk_from_subquery_with_duplicate_aliasname()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
event = threading.Event()
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

2
tests/system-test/2-query/tbname_vgroup.py
View File

@ -169,7 +169,7 @@ class TDTestCase:
tdSql.checkData(3, 2, 4)
tdSql.query("explain select * from st where tbname='ct1'")
tdSql.checkRows(2)
tdSql.checkRows(1)
tdSql.query("select table_name, vgroup_id from information_schema.ins_tables where db_name='dbvg' and type='CHILD_TABLE'");
print(tdSql.queryResult);

157
tests/system-test/7-tmq/ts-4674.py Normal file
View File

@ -0,0 +1,157 @@
import taos
import sys
import time
import socket
import os
import threading
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
from util.common import *
from taos.tmq import *
sys.path.append("./7-tmq")
from tmqCommon import *
class TDTestCase:
clientCfgDict = {'debugFlag': 135}
updatecfgDict = {'debugFlag': 135, 'clientCfg':clientCfgDict}
# updatecfgDict = {'debugFlag': 135, 'clientCfg':clientCfgDict, 'tmqRowSize':1}
def init(self, conn, logSql, replicaVar=1):
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor())
#tdSql.init(conn.cursor(), logSql) # output sql.txt file
# def consume_TS_4674_Test(self):
#
# os.system("nohup taosBenchmark -y -B 1 -t 4 -S 1000 -n 1000000 -i 1000 -v 1 -a 3 > /dev/null 2>&1 &")
# time.sleep()
# tdSql.execute(f'create topic topic_all with meta as database test')
# consumer_dict = {
# "group.id": "g1",
# "td.connect.user": "root",
# "td.connect.pass": "taosdata",
# "auto.offset.reset": "earliest",
# }
# consumer = Consumer(consumer_dict)
#
# try:
# consumer.subscribe(["topic_all"])
# except TmqError:
# tdLog.exit(f"subscribe error")
#
# try:
# while True:
# res = consumer.poll(5)
# if not res:
# print(f"null")
# continue
# val = res.value()
# if val is None:
# print(f"null")
# continue
# cnt = 0;
# for block in val:
# cnt += len(block.fetchall())
#
# print(f"block {cnt} rows")
#
# finally:
# consumer.close()
def get_leader(self):
tdLog.debug("get leader")
tdSql.query("show vnodes")
for result in tdSql.queryResult:
if result[3] == 'leader':
tdLog.debug("leader is %d"%(result[0]))
return result[0]
return -1
def balance_vnode(self):
leader_before = self.get_leader()
while True:
leader_after = -1
tdSql.query("balance vgroup leader")
while True:
leader_after = self.get_leader()
if leader_after != -1 :
break;
else:
time.sleep(1)
if leader_after != leader_before:
tdLog.debug("leader changed")
break;
else :
time.sleep(1)
def consume_TS_4674_Test(self):
tdSql.execute(f'create database if not exists d1 replica 3 vgroups 1')
tdSql.execute(f'use d1')
tdSql.execute(f'create table st(ts timestamp, i int) tags(t int)')
tdSql.execute(f'insert into t1 using st tags(1) values(now, 1) (now+1s, 2)')
tdSql.execute(f'insert into t2 using st tags(2) values(now, 1) (now+1s, 2)')
tdSql.execute(f'insert into t3 using st tags(3) values(now, 1) (now+1s, 2)')
tdSql.execute(f'create topic topic_all as select * from st')
consumer_dict = {
"group.id": "g1",
"td.connect.user": "root",
"td.connect.pass": "taosdata",
"auto.offset.reset": "earliest",
}
consumer = Consumer(consumer_dict)
try:
consumer.subscribe(["topic_all"])
except TmqError:
tdLog.exit(f"subscribe error")
cnt = 0;
balance = False
try:
while True:
res = consumer.poll(2)
if not res:
print(f"null res")
if balance == False and cnt != 6 :
tdLog.exit(f"subscribe num != 6")
if balance == True :
if cnt != 8 :
tdLog.exit(f"subscribe num != 8")
# tdLog.debug(f"subscribe num != 8")
# continue
else :
break
self.balance_vnode()
balance = True
tdSql.execute(f'insert into t1 using st tags(1) values(now+5s, 11) (now+10s, 12)')
continue
val = res.value()
if val is None:
print(f"null val")
continue
for block in val:
cnt += len(block.fetchall())
print(f"block {cnt} rows")
finally:
consumer.close()
def run(self):
self.consume_TS_4674_Test()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())