diff --git a/docs/en/08-develop/09-udf.md b/docs/en/08-develop/09-udf.md index 4105a89bcf..bf016e8765 100644 --- a/docs/en/08-develop/09-udf.md +++ b/docs/en/08-develop/09-udf.md @@ -585,7 +585,7 @@ def process(block): return result ``` -Crate and test the UDF: +Create and test the UDF: ```sql create function nsum as '/root/udf/nsum.py' outputtype double language 'Python'; diff --git a/docs/en/14-reference/01-components/03-taosadapter.md b/docs/en/14-reference/01-components/03-taosadapter.md index b257633d3b..50b4019fd7 100644 --- a/docs/en/14-reference/01-components/03-taosadapter.md +++ b/docs/en/14-reference/01-components/03-taosadapter.md @@ -66,7 +66,8 @@ Usage of taosAdapter: --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" --cors.allowHeaders stringArray cors allow HEADERS. Env "TAOS_ADAPTER_ALLOW_HEADERS" --cors.allowOrigins stringArray cors allow origins. Env "TAOS_ADAPTER_ALLOW_ORIGINS" - --cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets" --cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers" + --cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets" + --cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers" --debug enable debug mode. Env "TAOS_ADAPTER_DEBUG" (default true) --help Print this help message and exit --httpCodeServerError Use a non-200 http status code when server returns an error. Env "TAOS_ADAPTER_HTTP_CODE_SERVER_ERROR" @@ -87,7 +88,8 @@ Usage of taosAdapter: --monitor.password string TDengine password. Env "TAOS_ADAPTER_MONITOR_PASSWORD" (default "taosdata") --monitor.pauseAllMemoryThreshold float Memory percentage threshold for pause all. Env "TAOS_ADAPTER_MONITOR_PAUSE_ALL_MEMORY_THRESHOLD" (default 80) --monitor.pauseQueryMemoryThreshold float Memory percentage threshold for pause query. Env "TAOS_ADAPTER_MONITOR_PAUSE_QUERY_MEMORY_THRESHOLD" (default 70) - --monitor.user string TDengine user. Env "TAOS_ADAPTER_MONITOR_USER" (default "root") --monitor.writeInterval duration Set write to TDengine interval. Env "TAOS_ADAPTER_MONITOR_WRITE_INTERVAL" (default 30s) + --monitor.user string TDengine user. Env "TAOS_ADAPTER_MONITOR_USER" (default "root") + --monitor.writeInterval duration Set write to TDengine interval. Env "TAOS_ADAPTER_MONITOR_WRITE_INTERVAL" (default 30s) --monitor.writeToTD Whether write metrics to TDengine. Env "TAOS_ADAPTER_MONITOR_WRITE_TO_TD" --node_exporter.caCertFile string node_exporter ca cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CA_CERT_FILE" --node_exporter.certFile string node_exporter cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CERT_FILE" @@ -123,7 +125,8 @@ Usage of taosAdapter: --restfulRowLimit int restful returns the maximum number of rows (-1 means no limit). Env "TAOS_ADAPTER_RESTFUL_ROW_LIMIT" (default -1) --smlAutoCreateDB Whether to automatically create db when writing with schemaless. Env "TAOS_ADAPTER_SML_AUTO_CREATE_DB" --statsd.allowPendingMessages int statsd allow pending messages. Env "TAOS_ADAPTER_STATSD_ALLOW_PENDING_MESSAGES" (default 50000) - --statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd") --statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true) + --statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd") + --statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true) --statsd.deleteGauges statsd delete gauge cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_GAUGES" (default true) --statsd.deleteSets statsd delete set cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_SETS" (default true) --statsd.deleteTimings statsd delete timing cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_TIMINGS" (default true) @@ -133,7 +136,8 @@ Usage of taosAdapter: --statsd.password string statsd password. Env "TAOS_ADAPTER_STATSD_PASSWORD" (default "taosdata") --statsd.port int statsd server port. Env "TAOS_ADAPTER_STATSD_PORT" (default 6044) --statsd.protocol string statsd protocol [tcp or udp]. Env "TAOS_ADAPTER_STATSD_PROTOCOL" (default "udp") - --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" + --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" + --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" @@ -168,17 +172,17 @@ See [example/config/taosadapter.toml](https://github.com/taosdata/taosadapter/bl - Compatible with OpenTSDB JSON and telnet format writes - [http://opentsdb.net/docs/build/html/api_http/put.html](http://opentsdb.net/docs/build/html/api_http/put.html) - [http://opentsdb.net/docs/build/html/api_telnet/put.html](http://opentsdb.net/docs/build/html/api_telnet/put.html) -- Seamless connection to collectd +- Seamless connection to collectd. collectd is a system statistics collection daemon, please visit [https://collectd.org/](https://collectd.org/) for more information. -- Seamless connection with StatsD +- Seamless connection with StatsD. StatsD is a simple yet powerful daemon for aggregating statistical information. Please visit [https://github.com/statsd/statsd](https://github.com/statsd/statsd) for more information. -- Seamless connection with icinga2 +- Seamless connection with icinga2. icinga2 is a software that collects inspection result metrics and performance data. Please visit [https://icinga.com/docs/icinga-2/latest/doc/14-features/#opentsdb-writer](https://icinga.com/docs/icinga-2/latest/doc/14-features/#opentsdb-writer) for more information. -- Seamless connection to TCollector +- Seamless connection to TCollector. TCollector is a client process that collects data from a local collector and pushes the data to OpenTSDB. Please visit [http://opentsdb.net/docs/build/html/user_guide/utilities/tcollector.html](http://opentsdb.net/docs/build/html/user_guide/utilities/tcollector.html) for more information. -- Seamless connection to node_exporter +- Seamless connection to node_exporter. node_export is an exporter for machine metrics. Please visit [https://github.com/prometheus/node_exporter](https://github.com/prometheus/node_exporter) for more information. -- Support for Prometheus remote_read and remote_write +- Support for Prometheus remote_read and remote_write. remote_read and remote_write are interfaces for Prometheus data read and write from/to other data storage solution. Please visit [https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis) for more information. - Get table's VGroup ID. diff --git a/docs/en/20-third-party/11-kafka.md b/docs/en/20-third-party/11-kafka.md index b742b334d3..d8d8960392 100644 --- a/docs/en/20-third-party/11-kafka.md +++ b/docs/en/20-third-party/11-kafka.md @@ -231,8 +231,9 @@ Input following content: "config":{ "connector.class": "com.taosdata.kafka.connect.source.TDengineSourceConnector", "tasks.max": 1, + "subscription.group.id": "source-demo", "connection.url": "jdbc:TAOS://127.0.0.1:6030", - "connection.username": "root", + "connection.user": "root", "connection.password": "taosdata", "connection.database": "test", "connection.attempts": 3, diff --git a/docs/zh/14-reference/01-components/02-taosc.md b/docs/zh/14-reference/01-components/02-taosc.md index d198890bd9..dfcdf22aac 100644 --- a/docs/zh/14-reference/01-components/02-taosc.md +++ b/docs/zh/14-reference/01-components/02-taosc.md @@ -41,4 +41,4 @@ TDengine 客户端驱动提供了应用编程所需要的全部 API，并且在 ## API -请参考[连接器] \ No newline at end of file +请参考[连接器](../../connector) \ No newline at end of file diff --git a/docs/zh/14-reference/01-components/03-taosadapter.md b/docs/zh/14-reference/01-components/03-taosadapter.md index 97f2ab4074..ed90f54bca 100644 --- a/docs/zh/14-reference/01-components/03-taosadapter.md +++ b/docs/zh/14-reference/01-components/03-taosadapter.md @@ -66,13 +66,15 @@ Usage of taosAdapter: --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" --cors.allowHeaders stringArray cors allow HEADERS. Env "TAOS_ADAPTER_ALLOW_HEADERS" --cors.allowOrigins stringArray cors allow origins. Env "TAOS_ADAPTER_ALLOW_ORIGINS" - --cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets" --cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers" + --cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets" + --cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers" --debug enable debug mode. Env "TAOS_ADAPTER_DEBUG" (default true) --help Print this help message and exit --httpCodeServerError Use a non-200 http status code when server returns an error. Env "TAOS_ADAPTER_HTTP_CODE_SERVER_ERROR" --influxdb.enable enable influxdb. Env "TAOS_ADAPTER_INFLUXDB_ENABLE" (default true) --log.enableRecordHttpSql whether to record http sql. Env "TAOS_ADAPTER_LOG_ENABLE_RECORD_HTTP_SQL" - --log.path string log path. Env "TAOS_ADAPTER_LOG_PATH" (default "/var/log/taos") --log.rotationCount uint log rotation count. Env "TAOS_ADAPTER_LOG_ROTATION_COUNT" (default 30) + --log.path string log path. Env "TAOS_ADAPTER_LOG_PATH" (default "/var/log/taos") + --log.rotationCount uint log rotation count. Env "TAOS_ADAPTER_LOG_ROTATION_COUNT" (default 30) --log.rotationSize string log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_ROTATION_SIZE" (default "1GB") --log.rotationTime duration log rotation time. Env "TAOS_ADAPTER_LOG_ROTATION_TIME" (default 24h0m0s) --log.sqlRotationCount uint record sql log rotation count. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_COUNT" (default 2) @@ -87,7 +89,8 @@ Usage of taosAdapter: --monitor.password string TDengine password. Env "TAOS_ADAPTER_MONITOR_PASSWORD" (default "taosdata") --monitor.pauseAllMemoryThreshold float Memory percentage threshold for pause all. Env "TAOS_ADAPTER_MONITOR_PAUSE_ALL_MEMORY_THRESHOLD" (default 80) --monitor.pauseQueryMemoryThreshold float Memory percentage threshold for pause query. Env "TAOS_ADAPTER_MONITOR_PAUSE_QUERY_MEMORY_THRESHOLD" (default 70) - --monitor.user string TDengine user. Env "TAOS_ADAPTER_MONITOR_USER" (default "root") --monitor.writeInterval duration Set write to TDengine interval. Env "TAOS_ADAPTER_MONITOR_WRITE_INTERVAL" (default 30s) + --monitor.user string TDengine user. Env "TAOS_ADAPTER_MONITOR_USER" (default "root") + --monitor.writeInterval duration Set write to TDengine interval. Env "TAOS_ADAPTER_MONITOR_WRITE_INTERVAL" (default 30s) --monitor.writeToTD Whether write metrics to TDengine. Env "TAOS_ADAPTER_MONITOR_WRITE_TO_TD" --node_exporter.caCertFile string node_exporter ca cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CA_CERT_FILE" --node_exporter.certFile string node_exporter cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CERT_FILE" @@ -123,7 +126,8 @@ Usage of taosAdapter: --restfulRowLimit int restful returns the maximum number of rows (-1 means no limit). Env "TAOS_ADAPTER_RESTFUL_ROW_LIMIT" (default -1) --smlAutoCreateDB Whether to automatically create db when writing with schemaless. Env "TAOS_ADAPTER_SML_AUTO_CREATE_DB" --statsd.allowPendingMessages int statsd allow pending messages. Env "TAOS_ADAPTER_STATSD_ALLOW_PENDING_MESSAGES" (default 50000) - --statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd") --statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true) + --statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd") + --statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true) --statsd.deleteGauges statsd delete gauge cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_GAUGES" (default true) --statsd.deleteSets statsd delete set cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_SETS" (default true) --statsd.deleteTimings statsd delete timing cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_TIMINGS" (default true) @@ -133,7 +137,8 @@ Usage of taosAdapter: --statsd.password string statsd password. Env "TAOS_ADAPTER_STATSD_PASSWORD" (default "taosdata") --statsd.port int statsd server port. Env "TAOS_ADAPTER_STATSD_PORT" (default 6044) --statsd.protocol string statsd protocol [tcp or udp]. Env "TAOS_ADAPTER_STATSD_PROTOCOL" (default "udp") - --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" + --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" + --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" @@ -166,19 +171,19 @@ AllowWebSockets - 兼容 InfluxDB v1 写接口 [https://docs.influxdata.com/influxdb/v2.0/reference/api/influxdb-1x/write/](https://docs.influxdata.com/influxdb/v2.0/reference/api/influxdb-1x/write/) - 兼容 OpenTSDB JSON 和 telnet 格式写入 - - \ - - \ -- 与 collectd 无缝连接 + - \ + - \ +- 与 collectd 无缝连接。 collectd 是一个系统统计收集守护程序，请访问 [https://collectd.org/](https://collectd.org/) 了解更多信息。 -- Seamless connection with StatsD +- Seamless connection with StatsD。 StatsD 是一个简单而强大的统计信息汇总的守护程序。请访问 [https://github.com/statsd/statsd](https://github.com/statsd/statsd) 了解更多信息。 -- 与 icinga2 的无缝连接 +- 与 icinga2 的无缝连接。 icinga2 是一个收集检查结果指标和性能数据的软件。请访问 [https://icinga.com/docs/icinga-2/latest/doc/14-features/#opentsdb-writer](https://icinga.com/docs/icinga-2/latest/doc/14-features/#opentsdb-writer) 了解更多信息。 -- 与 tcollector 无缝连接 +- 与 tcollector 无缝连接。 TCollector是一个客户端进程，从本地收集器收集数据，并将数据推送到 OpenTSDB。请访问 [http://opentsdb.net/docs/build/html/user_guide/utilities/tcollector.html](http://opentsdb.net/docs/build/html/user_guide/utilities/tcollector.html) 了解更多信息。 -- 无缝连接 node_exporter +- 无缝连接 node_exporter。 node_export 是一个机器指标的导出器。请访问 [https://github.com/prometheus/node_exporter](https://github.com/prometheus/node_exporter) 了解更多信息。 -- 支持 Prometheus remote_read 和 remote_write +- 支持 Prometheus remote_read 和 remote_write。 remote_read 和 remote_write 是 Prometheus 数据读写分离的集群方案。请访问[https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis) 了解更多信息。 - 获取 table 所在的虚拟节点组（VGroup）的 VGroup ID。 @@ -186,15 +191,11 @@ AllowWebSockets ### TDengine RESTful 接口 -您可以使用任何支持 http 协议的客户端通过访问 RESTful 接口地址 `http://:6041/rest/sql` 来写入数据到 TDengine 或从 TDengine 中查询数据。细节请参考[官方文档](../../connector/rest-api/)。 +您可以使用任何支持 http 协议的客户端通过访问 RESTful 接口地址 `http://:6041/rest/sql` 来写入数据到 TDengine 或从 TDengine 中查询数据。细节请参考[REST API 文档](../../connector/rest-api/)。 ### InfluxDB -您可以使用任何支持 http 协议的客户端访问 Restful 接口地址 `http://:6041/` 来写入 InfluxDB 兼容格式的数据到 TDengine。EndPoint 如下： - -```text -/influxdb/v1/write -``` +您可以使用任何支持 http 协议的客户端访问 Restful 接口地址 `http://:6041/influxdb/v1/write` 来写入 InfluxDB 兼容格式的数据到 TDengine。 支持 InfluxDB 参数如下： diff --git a/docs/zh/20-third-party/01-collection/11-kafka.md b/docs/zh/20-third-party/01-collection/11-kafka.md index e9ac68251c..6234c28790 100644 --- a/docs/zh/20-third-party/01-collection/11-kafka.md +++ b/docs/zh/20-third-party/01-collection/11-kafka.md @@ -229,8 +229,9 @@ vi source-demo.json "config":{ "connector.class": "com.taosdata.kafka.connect.source.TDengineSourceConnector", "tasks.max": 1, + "subscription.group.id": "source-demo", "connection.url": "jdbc:TAOS://127.0.0.1:6030", - "connection.username": "root", + "connection.user": "root", "connection.password": "taosdata", "connection.database": "test", "connection.attempts": 3, @@ -389,12 +390,12 @@ curl -X DELETE http://localhost:8083/connectors/TDengineSourceConnector ## 其他说明 -1. 关于如何在独立安装的 Kafka 环境使用 Kafka Connect 插件， 请参考官方文档：\。 +1. 关于如何在独立安装的 Kafka 环境使用 Kafka Connect 插件， 请参考官方文档：。 ## 问题反馈 -无论遇到任何问题，都欢迎在本项目的 Github 仓库反馈：\。 +无论遇到任何问题，都欢迎在本项目的 Github 仓库反馈：。 ## 参考 -1. \ +1. diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 4bf0663def..b1d8be2f0e 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -61,6 +61,7 @@ typedef struct SExprNode { bool asAlias; bool asParam; bool asPosition; + int32_t projIdx; } SExprNode; typedef enum EColumnType { @@ -91,6 +92,8 @@ typedef struct SColumnNode { int16_t numOfPKs; bool tableHasPk; bool isPk; + int32_t projRefIdx; + int32_t resIdx; } SColumnNode; typedef struct SColumnRefNode { @@ -214,6 +217,7 @@ typedef struct SRealTableNode { double ratio; SArray* pSmaIndexes; int8_t cacheLastMode; + int8_t stbRewrite; SArray* pTsmas; SArray* tsmaTargetTbVgInfo; // SArray, used for child table or normal table only SArray* tsmaTargetTbInfo; // SArray, used for child table or normal table only diff --git a/include/util/tlockfree.h b/include/util/tlockfree.h index 1a575f8ea1..24e83f23ca 100644 --- a/include/util/tlockfree.h +++ b/include/util/tlockfree.h @@ -39,7 +39,6 @@ typedef void (*_ref_fn_t)(const void *pObj); // set the initial reference count value #define T_REF_INIT_VAL(x, _v) \ do { \ - assert(_v >= 0); \ atomic_store_32(&((x)->_ref.val), (_v)); \ } while (0) @@ -64,8 +63,6 @@ typedef void (*_ref_fn_t)(const void *pObj); } \ } while (0) -#define T_REF_VAL_CHECK(x) assert((x)->_ref.val >= 0); - #define T_REF_VAL_GET(x) (x)->_ref.val // single writer multiple reader lock diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index e3e5ece44c..3c2729c029 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -933,7 +933,7 @@ void taos_init_imp(void) { appInfo.pInstMapByClusterId = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (NULL == appInfo.pInstMap || NULL == appInfo.pInstMapByClusterId) { - tscError("failed to allocate memory when init appInfo"); + (void)printf("failed to allocate memory when init appInfo\n"); tscInitRes = TSDB_CODE_OUT_OF_MEMORY; return; } @@ -961,7 +961,7 @@ void taos_init_imp(void) { if (InitRegexCache() != 0) { tscInitRes = -1; - tscError("failed to init regex cache"); + (void)printf("failed to init regex cache\n"); return; } diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 3d0602ca3e..3927172b61 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -2183,6 +2183,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { taosWUnLockLatch(&tmq->lock); } setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + taosMemoryFreeClear(pollRspWrapper->pEpset); tmqFreeRspWrapper(pRspWrapper); taosFreeQitem(pRspWrapper); } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { @@ -2247,6 +2248,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { ", vg total:%" PRId64 ", total:%" PRId64 ",QID:0x%" PRIx64, tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId); + taosMemoryFreeClear(pollRspWrapper->pEpset); taosFreeQitem(pRspWrapper); taosWUnLockLatch(&tmq->lock); return pRsp; @@ -2284,6 +2286,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { // build rsp SMqMetaRspObj* pRsp = NULL; (void)tmqBuildMetaRspFromWrapper(pollRspWrapper, &pRsp); + taosMemoryFreeClear(pollRspWrapper->pEpset); taosFreeQitem(pRspWrapper); taosWUnLockLatch(&tmq->lock); return pRsp; @@ -2321,6 +2324,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { tmq->consumerId, true); SMqBatchMetaRspObj* pRsp = NULL; (void)tmqBuildBatchMetaRspFromWrapper(pollRspWrapper, &pRsp); + taosMemoryFreeClear(pollRspWrapper->pEpset); taosFreeQitem(pRspWrapper); taosWUnLockLatch(&tmq->lock); return pRsp; @@ -2381,6 +2385,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId); + taosMemoryFreeClear(pollRspWrapper->pEpset); taosFreeQitem(pRspWrapper); taosWUnLockLatch(&tmq->lock); return pRsp; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c5304aff08..100a24f44d 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -485,27 +485,27 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input } if ((code = cfgLoad(pCfg, CFG_STYPE_APOLLO_URL, apolloUrl)) != 0) { - uError("failed to load from apollo url:%s since %s", apolloUrl, tstrerror(code)); + (void)printf("failed to load from apollo url:%s since %s\n", apolloUrl, tstrerror(code)); TAOS_RETURN(code); } if ((code = cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgFile)) != 0) { - uError("failed to load from cfg file:%s since %s", cfgFile, tstrerror(code)); + (void)printf("failed to load from cfg file:%s since %s\n", cfgFile, tstrerror(code)); TAOS_RETURN(code); } if ((code = cfgLoad(pCfg, CFG_STYPE_ENV_FILE, envFile)) != 0) { - uError("failed to load from env file:%s since %s", envFile, tstrerror(code)); + (void)printf("failed to load from env file:%s since %s\n", envFile, tstrerror(code)); TAOS_RETURN(code); } if ((code = cfgLoad(pCfg, CFG_STYPE_ENV_VAR, NULL)) != 0) { - uError("failed to load from global env variables since %s", tstrerror(code)); + (void)printf("failed to load from global env variables since %s\n", tstrerror(code)); TAOS_RETURN(code); } if ((code = cfgLoad(pCfg, CFG_STYPE_ENV_CMD, envCmd)) != 0) { - uError("failed to load from cmd env variables since %s", tstrerror(code)); + (void)printf("failed to load from cmd env variables since %s\n", tstrerror(code)); TAOS_RETURN(code); } @@ -1648,12 +1648,12 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi } if ((code = taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl)) != TSDB_CODE_SUCCESS) { - printf("failed to load cfg since %s\n", tstrerror(code)); + (void)printf("failed to load cfg since %s\n", tstrerror(code)); goto _exit; } if ((code = cfgLoadFromArray(pCfg, pArgs)) != TSDB_CODE_SUCCESS) { - printf("failed to load cfg from array since %s\n", tstrerror(code)); + (void)printf("failed to load cfg from array since %s\n", tstrerror(code)); goto _exit; } @@ -1669,18 +1669,18 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi TAOS_CHECK_GOTO(taosSetAllDebugFlag(pCfg, pDebugItem->i32), &lino, _exit); if ((code = taosMulModeMkDir(tsLogDir, 0777, true)) != TSDB_CODE_SUCCESS) { - printf("failed to create dir:%s since %s\n", tsLogDir, tstrerror(code)); + (void)printf("failed to create dir:%s since %s\n", tsLogDir, tstrerror(code)); goto _exit; } if ((code = taosInitLog(logname, logFileNum, tsc)) != 0) { - printf("failed to init log file since %s\n", tstrerror(code)); + (void)printf("failed to init log file since %s\n", tstrerror(code)); goto _exit; } _exit: if (TSDB_CODE_SUCCESS != code) { - printf("failed to create log at %d since %s:", lino, tstrerror(code)); + (void)printf("failed to create log at %d since %s\n", lino, tstrerror(code)); } cfgCleanup(pCfg); @@ -1700,12 +1700,12 @@ int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char * TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) ,NULL, _exit); if ((code = taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) { - printf("failed to load cfg since %s\n", tstrerror(code)); + (void)printf("failed to load cfg since %s\n", tstrerror(code)); goto _exit; } if ((code = cfgLoadFromArray(pCfg, pArgs)) != 0) { - printf("failed to load cfg from array since %s\n", tstrerror(code)); + (void)printf("failed to load cfg from array since %s\n", tstrerror(code)); goto _exit; } @@ -1769,14 +1769,14 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile TAOS_CHECK_GOTO(taosAddSystemCfg(tsCfg), &lino, _exit); if ((code = taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) { - uError("failed to load cfg since %s", tstrerror(code)); + (void)printf("failed to load cfg since %s\n", tstrerror(code)); cfgCleanup(tsCfg); tsCfg = NULL; TAOS_RETURN(code); } if ((code = cfgLoadFromArray(tsCfg, pArgs)) != 0) { - uError("failed to load cfg from array since %s", tstrerror(code)); + (void)printf("failed to load cfg from array since %s\n", tstrerror(code)); cfgCleanup(tsCfg); tsCfg = NULL; TAOS_RETURN(code); @@ -1798,7 +1798,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile SConfigItem *pItem = cfgGetItem(tsCfg, "debugFlag"); if (NULL == pItem) { - uError("debugFlag not found in cfg"); + (void)printf("debugFlag not found in cfg\n"); TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); } TAOS_CHECK_GOTO(taosSetAllDebugFlag(tsCfg, pItem->i32), &lino, _exit); @@ -1811,7 +1811,7 @@ _exit: if (TSDB_CODE_SUCCESS != code) { cfgCleanup(tsCfg); tsCfg = NULL; - uError("failed to init cfg at %d since %s", lino, tstrerror(code)); + (void)printf("failed to init cfg at %d since %s\n", lino, tstrerror(code)); } TAOS_RETURN(code); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 2792ea79f7..20c5d09c9e 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1892,7 +1892,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs, mndReleaseDb(pMnode, pDb); continue; } else { - mInfo("db:%s, valid dbinfo, vgVersion:%d cfgVersion:%d stateTs:%" PRId64 + mTrace("db:%s, valid dbinfo, vgVersion:%d cfgVersion:%d stateTs:%" PRId64 " numOfTables:%d, changed to vgVersion:%d cfgVersion:%d stateTs:%" PRId64 " numOfTables:%d", pDbCacheInfo->dbFName, pDbCacheInfo->vgVersion, pDbCacheInfo->cfgVersion, pDbCacheInfo->stateTs, pDbCacheInfo->numOfTable, pDb->vgVersion, pDb->cfgVersion, pDb->stateTs, numOfTable); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 97eb7d609b..6fd90dc505 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -523,6 +523,7 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) { int32_t code = tEncodeStreamTask(&encoder, pTask); if (code == -1) { + tEncoderClear(&encoder); return TSDB_CODE_INVALID_MSG; } @@ -1009,6 +1010,7 @@ static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, int tEncoderInit(&encoder, abuf, tlen); int32_t pos = tEncodeStreamCheckpointSourceReq(&encoder, &req); if (pos == -1) { + tEncoderClear(&encoder); return TSDB_CODE_INVALID_MSG; } diff --git a/source/dnode/mnode/impl/test/stream/stream.cpp b/source/dnode/mnode/impl/test/stream/stream.cpp index 4c63c2ba2c..d508cf7390 100644 --- a/source/dnode/mnode/impl/test/stream/stream.cpp +++ b/source/dnode/mnode/impl/test/stream/stream.cpp @@ -91,6 +91,7 @@ SRpcMsg buildHbReq() { tEncoderInit(&encoder, (uint8_t*)buf, tlen); if ((code = tEncodeStreamHbMsg(&encoder, &msg)) < 0) { rpcFreeCont(buf); + tEncoderClear(&encoder); goto _end; } tEncoderClear(&encoder); diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 2e22b7524b..c42d5e7de5 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -342,7 +342,10 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S // todo remove this void* pGpIdData = colDataGetData(pGpIdColInfo, rowId); - ASSERT(gid == *(int64_t*)pGpIdData); + if (gid != *(int64_t*)pGpIdData) { + tqError("s-task:%s vgId:%d invalid groupId:%" PRId64 " actual:%" PRId64 " in sink task", id, vgId, gid, + *(int64_t*)pGpIdData); + } } code = setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, gid, @@ -747,7 +750,6 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat return code; } - ASSERT(pRow); void* p = taosArrayPush(pTableData->aRowP, &pRow); if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -779,8 +781,6 @@ int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkI bool isValid = isValidDstChildTable(&mr, vgId, dstTableName, suid); if (isValid) { // not valid table, ignore it tqDebug("s-task:%s set uid:%" PRIu64 " for dstTable:%s from meta", id, mr.me.uid, pTableSinkInfo->name.data); - ASSERT(terrno == 0); - // set the destination table uid (*uid) = mr.me.uid; pTableSinkInfo->uid = mr.me.uid; diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 4c44280311..6463a59dfb 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -46,7 +46,10 @@ int32_t tqScanWal(STQ* pTq) { streamMetaWLock(pMeta); int32_t times = (--pMeta->scanInfo.scanCounter); - ASSERT(pMeta->scanInfo.scanCounter >= 0); + if (times < 0) { + tqError("vgId:%d invalid scan counter:%d, reset to 0", vgId, times); + times = 0; + } numOfTasks = taosArrayGetSize(pMeta->pTaskList); streamMetaWUnLock(pMeta); @@ -269,7 +272,6 @@ bool taskReadyForDataFromWal(SStreamTask* pTask) { // fill-history task has entered into the last phase, no need to anything if ((pTask->info.fillHistory == 1) && pTask->status.appendTranstateBlock) { - ASSERT(pState.state == TASK_STATUS__READY); // the maximum version of data in the WAL has reached already, the step2 is done tqDebug("s-task:%s fill-history reach the maximum ver:%" PRId64 ", not scan wal anymore", pTask->id.idStr, pTask->dataRange.range.maxVer); diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 4471b93e80..44b3f75289 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -339,13 +339,15 @@ int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { SMsgHead* pRspHead = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamDispatchRsp)); if (pRspHead == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("s-task:0x%x send dispatch error rsp, out of memory", req.taskId); - return -1; + return terrno; } pRspHead->vgId = htonl(req.upstreamNodeId); - ASSERT(pRspHead->vgId != 0); + if(pRspHead->vgId == 0) { + tqError("vgId:%d invalid dispatch msg from upstream to task:0x%x", pMeta->vgId, req.taskId); + return TSDB_CODE_INVALID_MSG; + } SStreamDispatchRsp* pRsp = POINTER_SHIFT(pRspHead, sizeof(SMsgHead)); pRsp->streamId = htobe64(req.streamId); @@ -926,7 +928,6 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) { streamTaskSetStatusReady(pTask); } else if (pState.state == TASK_STATUS__UNINIT) { // tqDebug("s-task:%s start task by checking downstream tasks", pTask->id.idStr); - // ASSERT(pTask->status.downstreamReady == 0); // tqStreamTaskRestoreCheckpoint(pMeta, pTask->id.streamId, pTask->id.taskId); tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name); } else { @@ -1000,7 +1001,10 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) TSDB_CODE_ACTION_IN_PROGRESS); } } else { // upstream not recv the checkpoint-source/trigger till now - ASSERT(pState.state == TASK_STATUS__READY || pState.state == TASK_STATUS__HALT); + if (!(pState.state == TASK_STATUS__READY || pState.state == TASK_STATUS__HALT)) { + tqFatal("s-task:%s invalid task status:%s", pTask->id.idStr, pState.name); + } + tqWarn( "s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all " "upstream sending checkpoint-source/trigger", @@ -1109,9 +1113,7 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t } else { code = streamTrySchedExec(pTask); } - } /*else { - ASSERT(status != TASK_STATUS__UNINIT); - }*/ + } streamMetaReleaseTask(pMeta, pTask); return code; @@ -1234,7 +1236,14 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { pTask->id.idStr, vgId, pTask->chkInfo.checkpointId, req.checkpointId); streamMutexLock(&pTask->lock); - ASSERT(pTask->chkInfo.checkpointId >= req.checkpointId); + if (pTask->chkInfo.checkpointId < req.checkpointId) { + tqFatal("s-task:%s vgId:%d invalid consensus-checkpointId:%" PRId64 ", greater than existed checkpointId:%"PRId64, + pTask->id.idStr, vgId, req.checkpointId, pTask->chkInfo.checkpointId); + + streamMutexUnlock(&pTask->lock); + streamMetaReleaseTask(pMeta, pTask); + return TSDB_CODE_STREAM_INTERNAL_ERROR; + } SConsenChkptInfo* pConsenInfo = &pTask->status.consenChkptInfo; if (pConsenInfo->consenChkptTransId >= req.transId) { diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 7e79a618a8..24916bf47c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -1020,7 +1020,10 @@ int32_t tsdbFSCreateRefSnapshotWithoutLock(STFileSystem *fs, TFileSetArray **fse if (code) break; code = TARRAY2_APPEND(fsetArr[0], fset1); - if (code) break; + if (code) { + tsdbTFileSetClear(&fset1); + break; + } } if (code) { @@ -1219,4 +1222,4 @@ int32_t tsdbFinishTaskOnFileSet(STsdb *tsdb, int32_t fid) { } return 0; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 3833e3db79..b7bfd045d1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -4817,7 +4817,7 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi return code; _err: - tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr); + tsdbError("failed to create data reader, error at:%d code:%s %s", lino, tstrerror(code), idstr); tsdbReaderClose2(*ppReader); *ppReader = NULL; // reset the pointer value. return code; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 057ff0f7c1..c02ff465c6 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -759,8 +759,7 @@ int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { vTrace("message in vnode query queue is processing"); - if ((pMsg->msgType == TDMT_SCH_QUERY || pMsg->msgType == TDMT_VND_TMQ_CONSUME || - pMsg->msgType == TDMT_VND_TMQ_CONSUME_PUSH) && + if ((pMsg->msgType == TDMT_SCH_QUERY || pMsg->msgType == TDMT_VND_TMQ_CONSUME) && !syncIsReadyForRead(pVnode->sync)) { vnodeRedirectRpcMsg(pVnode, pMsg, terrno); return 0; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 9c49ce627d..28829b6043 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -744,12 +744,10 @@ _end: insertRet = taosLRUCacheInsert(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(uint64_t), pVal, sizeof(STableCachedVal), freeCachedMetaItem, NULL, TAOS_LRU_PRIORITY_LOW, NULL); if (insertRet != TAOS_LRU_STATUS_OK) { - qError("failed to put meta into lru cache, code:%d, %s", insertRet, idStr); - taosMemoryFreeClear(pVal); - freeTableCachedValObj(&val); + qWarn("failed to put meta into lru cache, code:%d, %s", insertRet, idStr); } } - + if (freeReader) { pHandle->api.metaReaderFn.clearReader(&mr); } @@ -5745,7 +5743,7 @@ static int32_t getBlockForTableMergeScan(void* param, SSDataBlock** ppBlock) { pOperator->resultInfo.totalRows += pBlock->info.rows; pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; *ppBlock = pBlock; - + return code; } @@ -5900,7 +5898,7 @@ void startGroupTableMergeScan(SOperatorInfo* pOperator) { int32_t numOfTables = 0; code = tableListGetSize(pInfo->base.pTableListInfo, &numOfTables); QUERY_CHECK_CODE(code, lino, _end); - + int32_t i = pInfo->tableStartIndex + 1; for (; i < numOfTables; ++i) { STableKeyInfo* tableKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, i); diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 08abf41973..6397b92191 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -6007,6 +6007,7 @@ int32_t modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { pInfo->buf = taosMemoryMalloc(pInfo->colBytes); if (NULL == pInfo->buf) { + taosHashCleanup(pInfo->pHash); return TSDB_CODE_OUT_OF_MEMORY; } @@ -6114,6 +6115,7 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); int32_t currentRow = pBlock->info.rows; if (NULL == pCol) { + modeFunctionCleanup(pInfo); return TSDB_CODE_OUT_OF_RANGE; } @@ -6145,6 +6147,7 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { code = colDataSetVal(pCol, currentRow, pData, false); if (TSDB_CODE_SUCCESS != code) { + modeFunctionCleanup(pInfo); return code; } code = setSelectivityValue(pCtx, pBlock, &resTuplePos, currentRow); diff --git a/source/libs/index/inc/indexFstNode.h b/source/libs/index/inc/indexFstNode.h index 5bdb2acb32..a6f1662cb3 100644 --- a/source/libs/index/inc/indexFstNode.h +++ b/source/libs/index/inc/indexFstNode.h @@ -44,7 +44,7 @@ FstBuilderNode* fstBuilderNodeDefault(); FstBuilderNode* fstBuilderNodeClone(FstBuilderNode* src); -void fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src); +int32_t fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src); // bool fstBuilderNodeCompileTo(FstBuilderNode *b, IdxFile' *wrt, // CompiledAddr lastAddr, CompiledAddr startAddr); diff --git a/source/libs/index/inc/indexTfile.h b/source/libs/index/inc/indexTfile.h index 115d5b3894..e769ea2393 100644 --- a/source/libs/index/inc/indexTfile.h +++ b/source/libs/index/inc/indexTfile.h @@ -18,7 +18,7 @@ #include "indexFst.h" #include "indexFstFile.h" #include "indexInt.h" -#include "indexTfile.h" +//#include "indexTfile.h" #include "indexUtil.h" #include "tlockfree.h" @@ -129,8 +129,8 @@ void tfileIteratorDestroy(Iterate* iterator); TFileValue* tfileValueCreate(char* val); -int tfileValuePush(TFileValue* tf, uint64_t val); -void tfileValueDestroy(TFileValue* tf); +int32_t tfileValuePush(TFileValue* tf, uint64_t val); +void tfileValueDestroy(TFileValue* tf); #ifdef __cplusplus } diff --git a/source/libs/index/inc/indexUtil.h b/source/libs/index/inc/indexUtil.h index 6b016900c2..9eb8001d17 100644 --- a/source/libs/index/inc/indexUtil.h +++ b/source/libs/index/inc/indexUtil.h @@ -46,17 +46,17 @@ extern "C" { buf += len; \ } while (0) -#define INDEX_MERGE_ADD_DEL(src, dst, tgt) \ - { \ - bool f = false; \ - for (int i = 0; i < taosArrayGetSize(src); i++) { \ - if (*(uint64_t *)taosArrayGet(src, i) == tgt) { \ - f = true; \ - } \ - } \ - if (f == false) { \ - (void)taosArrayPush(dst, &tgt); \ - } \ +#define INDEX_MERGE_ADD_DEL(src, dst, tgt) \ + { \ + bool f = false; \ + for (int i = 0; i < taosArrayGetSize(src); i++) { \ + if (*(uint64_t *)taosArrayGet(src, i) == tgt) { \ + f = true; \ + } \ + } \ + if (f == false) { \ + if (taosArrayPush(dst, &tgt) == NULL) code = TSDB_CODE_OUT_OF_MEMORY; \ + } \ } /* multi sorted result intersection @@ -65,7 +65,7 @@ extern "C" { * [1, 4, 5] * output:[4, 5] */ -void iIntersection(SArray *in, SArray *out); +int32_t iIntersection(SArray *in, SArray *out); /* multi sorted result union * input: [1, 2, 4, 5] @@ -73,7 +73,7 @@ void iIntersection(SArray *in, SArray *out); * [1, 4, 5] * output:[1, 2, 3, 4, 5] */ -void iUnion(SArray *in, SArray *out); +int32_t iUnion(SArray *in, SArray *out); /* see example * total: [1, 2, 4, 5, 7, 8] @@ -81,7 +81,7 @@ void iUnion(SArray *in, SArray *out); * return: [1, 2, 7, 8] saved in total */ -void iExcept(SArray *total, SArray *except); +int32_t iExcept(SArray *total, SArray *except); int uidCompare(const void *a, const void *b); @@ -107,7 +107,7 @@ void idxTRsltClear(SIdxTRslt *tr); void idxTRsltDestroy(SIdxTRslt *tr); -void idxTRsltMergeTo(SIdxTRslt *tr, SArray *out); +int32_t idxTRsltMergeTo(SIdxTRslt *tr, SArray *out); #ifdef __cplusplus } diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index e2c74f0dbb..0d834a68cb 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -92,7 +92,7 @@ static int32_t idxMergeFinalResults(SArray* in, EIndexOperatorType oType, SArray static int32_t idxGenTFile(SIndex* index, IndexCache* cache, SArray* batch); // merge cache and tfile by opera type -static void idxMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv, SIdxTRslt* helper); +static int32_t idxMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv, SIdxTRslt* helper); // static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); // int32_t indexSerialKey(ICacheKey* key, char* buf); @@ -212,6 +212,7 @@ void idxReleaseRef(int64_t ref) { int32_t indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { // TODO(yihao): reduce the lock range + int32_t code = 0; (void)taosThreadMutexLock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); @@ -223,11 +224,19 @@ int32_t indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { IndexCache** cache = taosHashGet(index->colObj, buf, sz); if (cache == NULL) { IndexCache* pCache = idxCacheCreate(index, p->suid, p->colName, p->colType); - (void)taosHashPut(index->colObj, buf, sz, &pCache, sizeof(void*)); + code = taosHashPut(index->colObj, buf, sz, &pCache, sizeof(void*)); + if (code != 0) { + idxCacheDestroy(pCache); + break; + } } } (void)taosThreadMutexUnlock(&index->mtx); + if (code != 0) { + return code; + } + for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); @@ -247,15 +256,27 @@ int32_t indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { return 0; } int32_t indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result) { + int32_t code = 0; EIndexOperatorType opera = multiQuerys->opera; // relation of querys SArray* iRslts = taosArrayInit(4, POINTER_BYTES); - int nQuery = taosArrayGetSize(multiQuerys->query); + if (iRslts == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int nQuery = taosArrayGetSize(multiQuerys->query); for (size_t i = 0; i < nQuery; i++) { SIndexTermQuery* qterm = taosArrayGet(multiQuerys->query, i); SArray* trslt = NULL; - (void)idxTermSearch(index, qterm, &trslt); - (void)taosArrayPush(iRslts, (void*)&trslt); + code = idxTermSearch(index, qterm, &trslt); + if (code != 0) { + idxInterRsltDestroy(iRslts); + return code; + } + if (taosArrayPush(iRslts, (void*)&trslt) == NULL) { + idxInterRsltDestroy(iRslts); + return TSDB_CODE_OUT_OF_MEMORY; + } } (void)idxMergeFinalResults(iRslts, opera, result); idxInterRsltDestroy(iRslts); @@ -267,6 +288,9 @@ int indexDelete(SIndex* index, SIndexMultiTermQuery* query) { return 1; } SIndexOpts* indexOptsCreate(int32_t cacheSize) { SIndexOpts* opts = taosMemoryCalloc(1, sizeof(SIndexOpts)); + if (opts == NULL) { + return NULL; + } opts->cacheSize = cacheSize; return opts; } @@ -295,7 +319,7 @@ void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery) { int32_t indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType qType) { SIndexTermQuery q = {.qType = qType, .term = term}; if (taosArrayPush(pQuery->query, &q) == NULL) { - return terrno; + return TSDB_CODE_OUT_OF_MEMORY; } return 0; } @@ -362,7 +386,9 @@ void indexTermDestroy(SIndexTerm* p) { SIndexMultiTerm* indexMultiTermCreate() { return taosArrayInit(4, sizeof(SIndexTerm*)); } int32_t indexMultiTermAdd(SIndexMultiTerm* terms, SIndexTerm* term) { - (void)taosArrayPush(terms, &term); + if (taosArrayPush(terms, &term) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } return 0; } void indexMultiTermDestroy(SIndexMultiTerm* terms) { @@ -422,6 +448,7 @@ bool indexJsonIsRebuild(SIndexJson* idx) { } static int32_t idxTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result) { + int32_t code = 0; SIndexTerm* term = query->term; const char* colName = term->colName; int32_t nColName = term->nColName; @@ -452,6 +479,10 @@ static int32_t idxTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** resu int64_t st = taosGetTimestampUs(); SIdxTRslt* tr = idxTRsltCreate(); + if (tr == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, END); + } + if (0 == idxCacheSearch(cache, query, tr, &s)) { if (s == kTypeDeletion) { indexInfo("col: %s already drop by", term->colName); @@ -473,13 +504,14 @@ static int32_t idxTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** resu int64_t cost = taosGetTimestampUs() - st; indexInfo("search cost: %" PRIu64 "us", cost); - idxTRsltMergeTo(tr, *result); + code = idxTRsltMergeTo(tr, *result); + TAOS_CHECK_GOTO(code, NULL, END); idxTRsltDestroy(tr); return 0; END: idxTRsltDestroy(tr); - return 0; + return code; } static void idxInterRsltDestroy(SArray* results) { if (results == NULL) { @@ -503,9 +535,9 @@ static int32_t idxMergeFinalResults(SArray* in, EIndexOperatorType oType, SArray } if (oType == MUST) { - iIntersection(in, out); + return iIntersection(in, out); } else if (oType == SHOULD) { - iUnion(in, out); + return iUnion(in, out); } else if (oType == NOT) { // just one column index, enhance later // taosArrayAddAll(fResults, interResults); @@ -514,30 +546,53 @@ static int32_t idxMergeFinalResults(SArray* in, EIndexOperatorType oType, SArray return 0; } -static void idxMayMergeTempToFinalRslt(SArray* result, TFileValue* tfv, SIdxTRslt* tr) { +static int32_t idxMayMergeTempToFinalRslt(SArray* result, TFileValue* tfv, SIdxTRslt* tr) { + int32_t code = 0; int32_t sz = taosArrayGetSize(result); if (sz > 0) { TFileValue* lv = taosArrayGetP(result, sz - 1); if (tfv != NULL && strcmp(lv->colVal, tfv->colVal) != 0) { - idxTRsltMergeTo(tr, lv->tableId); + code = idxTRsltMergeTo(tr, lv->tableId); + if (code != 0) { + indexFatal("failed to merge result since %s", tstrerror(code)); + return code; + } idxTRsltClear(tr); - (void)taosArrayPush(result, &tfv); + if (taosArrayPush(result, &tfv) == NULL) { + indexFatal("failed to merge result since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + } } else if (tfv == NULL) { // handle last iterator - idxTRsltMergeTo(tr, lv->tableId); + code = idxTRsltMergeTo(tr, lv->tableId); + if (code != 0) { + indexFatal("failed to merge result since %s", tstrerror(code)); + } } else { tfileValueDestroy(tfv); + return 0; } } else { - (void)taosArrayPush(result, &tfv); + if (taosArrayPush(result, &tfv) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } + return code; } -static void idxMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue* tv, SIdxTRslt* tr) { +static int32_t idxMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue* tv, SIdxTRslt* tr) { + int32_t code = 0; char* colVal = (cv != NULL) ? cv->colVal : tv->colVal; TFileValue* tfv = tfileValueCreate(colVal); + if (tfv == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } - idxMayMergeTempToFinalRslt(result, tfv, tr); + code = idxMayMergeTempToFinalRslt(result, tfv, tr); + if (code != 0) { + tfileValueDestroy(tfv); + return code; + } + tfv = NULL; if (cv != NULL) { uint64_t id = *(uint64_t*)taosArrayGet(cv->val, 0); @@ -549,8 +604,11 @@ static void idxMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue } } if (tv != NULL) { - (void)taosArrayAddAll(tr->total, tv->val); + if (taosArrayAddAll(tr->total, tv->val) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } + return 0; } static void idxDestroyFinalRslt(SArray* result) { int32_t sz = result ? taosArrayGetSize(result) : 0; @@ -562,6 +620,7 @@ static void idxDestroyFinalRslt(SArray* result) { } int32_t idxFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) { + int32_t code = 0; if (sIdx == NULL) { return TSDB_CODE_INVALID_PTR; } @@ -598,12 +657,16 @@ int32_t idxFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) { } SArray* result = taosArrayInit(1024, sizeof(void*)); + if (result == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception); + } bool cn = cacheIter ? cacheIter->next(cacheIter) : false; bool tn = tfileIter ? tfileIter->next(tfileIter) : false; SIdxTRslt* tr = idxTRsltCreate(); if (tr == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception); } while (cn == true || tn == true) { IterateValue* cv = (cn == true) ? cacheIter->getValue(cacheIter) : NULL; @@ -618,27 +681,42 @@ int32_t idxFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) { comp = 1; } if (comp == 0) { - idxMergeCacheAndTFile(result, cv, tv, tr); + code = idxMergeCacheAndTFile(result, cv, tv, tr); + if (code != 0) { + TAOS_CHECK_GOTO(code, NULL, _exception); + } + cn = cacheIter->next(cacheIter); tn = tfileIter->next(tfileIter); } else if (comp < 0) { - idxMergeCacheAndTFile(result, cv, NULL, tr); + code = idxMergeCacheAndTFile(result, cv, NULL, tr); + if (code != 0) { + TAOS_CHECK_GOTO(code, NULL, _exception); + } cn = cacheIter->next(cacheIter); } else { - idxMergeCacheAndTFile(result, NULL, tv, tr); + code = idxMergeCacheAndTFile(result, NULL, tv, tr); + if (code != 0) { + TAOS_CHECK_GOTO(code, NULL, _exception); + } tn = tfileIter->next(tfileIter); } } - idxMayMergeTempToFinalRslt(result, NULL, tr); + if ((code = idxMayMergeTempToFinalRslt(result, NULL, tr)) != 0) { + idxTRsltDestroy(tr); + TAOS_CHECK_GOTO(code, NULL, _exception); + } idxTRsltDestroy(tr); - int ret = idxGenTFile(sIdx, pCache, result); - if (ret != 0) { - indexError("failed to merge"); + code = idxGenTFile(sIdx, pCache, result); + if (code != 0) { + indexError("failed to merge since %s", tstrerror(code)); } else { int64_t cost = taosGetTimestampUs() - st; indexInfo("success to merge , time cost: %" PRId64 "ms", cost / 1000); } + +_exception: idxDestroyFinalRslt(result); idxCacheDestroyImm(pCache); @@ -654,8 +732,11 @@ int32_t idxFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) { idxPost(sIdx); } idxReleaseRef(sIdx->refId); + if (code != 0) { + indexError("failed to merge since %s", tstrerror(code)); + } - return ret; + return code; } void iterateValueDestroy(IterateValue* value, bool destroy) { if (destroy) { diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 734c4067c6..a89cc47925 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -75,6 +75,7 @@ static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTRslt* tr, STe if (cache == NULL) { return 0; } + int32_t code = 0; MemTable* mem = cache; IndexCache* pCache = mem->pCache; @@ -98,6 +99,10 @@ static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTRslt* tr, STe } else if (c->operaType == DEL_VALUE) { INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid) } + + if (code != 0) { + break; + } } else { break; } @@ -105,7 +110,7 @@ static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTRslt* tr, STe taosMemoryFree(pCt); (void)tSkipListDestroyIter(iter); - return 0; + return code; } static int32_t cacheSearchPrefix(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) { // impl later @@ -123,6 +128,7 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTRslt* if (cache == NULL) { return 0; } + int32_t code = TSDB_CODE_SUCCESS; MemTable* mem = cache; IndexCache* pCache = mem->pCache; @@ -148,7 +154,7 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTRslt* } CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); terrno = TSDB_CODE_SUCCESS; - TExeCond cond = cmpFn(c->colVal, pCt->colVal, pCt->colType); + TExeCond cond = cmpFn(c->colVal, pCt->colVal, pCt->colType); if (terrno != TSDB_CODE_SUCCESS) { code = terrno; goto _return; @@ -156,11 +162,14 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTRslt* if (cond == MATCH) { if (c->operaType == ADD_VALUE) { INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid) - // taosArrayPush(result, &c->uid); *s = kTypeValue; } else if (c->operaType == DEL_VALUE) { INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid) } + + if (code != 0) { + break; + } } else if (cond == CONTINUE) { continue; } else if (cond == BREAK) { @@ -190,6 +199,8 @@ static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr if (cache == NULL) { return 0; } + + int32_t code = 0; MemTable* mem = cache; IndexCache* pCache = mem->pCache; @@ -223,6 +234,10 @@ static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr } else if (c->operaType == DEL_VALUE) { INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid) } + + if (code != 0) { + break; + } } else { break; } @@ -231,9 +246,7 @@ static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr taosMemoryFree(pCt); taosMemoryFree(exBuf); (void)tSkipListDestroyIter(iter); - return 0; - - return TSDB_CODE_SUCCESS; + return code; } static int32_t cacheSearchSuffix_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) { return TSDB_CODE_SUCCESS; @@ -338,6 +351,10 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR } else if (c->operaType == DEL_VALUE) { INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid) } + + if (code != 0) { + break; + } } else if (cond == CONTINUE) { continue; } else if (cond == BREAK) { @@ -800,7 +817,13 @@ static bool idxCacheIteratorNext(Iterate* itera) { iv->type = ct->operaType; iv->ver = ct->version; iv->colVal = taosStrdup(ct->colVal); - (void)taosArrayPush(iv->val, &ct->uid); + if (iv->colVal == NULL) { + return false; + } + if (taosArrayPush(iv->val, &ct->uid) == NULL) { + taosMemoryFree(iv->colVal); + return false; + } } return next; } diff --git a/source/libs/index/src/indexFst.c b/source/libs/index/src/indexFst.c index 3edd61bd08..aa617c2de2 100644 --- a/source/libs/index/src/indexFst.c +++ b/source/libs/index/src/indexFst.c @@ -62,7 +62,9 @@ void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes* nodes, bool isFinal) { node->trans = taosArrayInit(16, sizeof(FstTransition)); FstBuilderNodeUnfinished un = {.node = node, .last = NULL}; - (void)taosArrayPush(nodes->stack, &un); + if (taosArrayPush(nodes->stack, &un) == NULL) { + fstBuilderNodeDestroy(node); + } } FstBuilderNode* fstUnFinishedNodesPopRoot(FstUnFinishedNodes* nodes) { FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); @@ -120,7 +122,10 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output FstLastTransition* trn = fstLastTransitionCreate(data[i], 0); FstBuilderNodeUnfinished un = {.node = n, .last = trn}; - (void)taosArrayPush(nodes->stack, &un); + if (taosArrayPush(nodes->stack, &un) == NULL) { + fstBuilderNodeDestroy(n); + taosMemoryFree(trn); + } } fstUnFinishedNodesPushEmpty(nodes, true); } @@ -892,7 +897,9 @@ void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, Comp return; } FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr}; - (void)taosArrayPush(unNode->node->trans, &t); + if (taosArrayPush(unNode->node->trans, &t) == NULL) { + return; + } fstLastTransitionDestroy(trn); unNode->last = NULL; return; @@ -997,7 +1004,12 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { uint8_t* data = fstSliceData(b, &len); SArray* nodes = (SArray*)taosArrayInit(len, sizeof(FstNode*)); - (void)taosArrayPush(nodes, &root); + if (nodes == NULL) { + return false; + } + if (taosArrayPush(nodes, &root) == NULL) { + goto _return; + } for (uint32_t i = 0; i < len; i++) { uint8_t inp = data[i]; Output res = 0; @@ -1009,7 +1021,9 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { (void)fstNodeGetTransitionAt(root, res, &trn); tOut += trn.out; root = fstGetNode(fst, trn.addr); - (void)taosArrayPush(nodes, &root); + if (taosArrayPush(nodes, &root) == NULL) { + goto _return; + } } if (!FST_NODE_IS_FINAL(root)) { goto _return; @@ -1156,7 +1170,9 @@ bool stmStSeekMin(FStmSt* sws, FstBoundWithData* min) { .trans = 0, .out = {.null = false, .out = 0}, .autState = automFuncs[aut->type].start(aut)}; // auto.start callback - (void)taosArrayPush(sws->stack, &s); + if (taosArrayPush(sws->stack, &s) == NULL) { + return false; + } return true; } FstSlice* key = NULL; @@ -1185,12 +1201,16 @@ bool stmStSeekMin(FStmSt* sws, FstBoundWithData* min) { (void)fstNodeGetTransitionAt(node, res, &trn); void* preState = autState; autState = automFuncs[aut->type].accept(aut, preState, b); - (void)taosArrayPush(sws->inp, &b); + if (taosArrayPush(sws->inp, &b) == NULL) { + return false; + } FstStreamState s = {.node = node, .trans = res + 1, .out = {.null = false, .out = out}, .autState = preState}; node = NULL; - (void)taosArrayPush(sws->stack, &s); + if (taosArrayPush(sws->stack, &s) == NULL) { + return false; + } out += trn.out; node = fstGetNode(sws->fst, trn.addr); } else { @@ -1209,7 +1229,9 @@ bool stmStSeekMin(FStmSt* sws, FstBoundWithData* min) { } FstStreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState}; - (void)taosArrayPush(sws->stack, &s); + if (taosArrayPush(sws->stack, &s) == NULL) { + return false; + } taosMemoryFree(trans); return true; } @@ -1230,7 +1252,9 @@ bool stmStSeekMin(FStmSt* sws, FstBoundWithData* min) { (void)fstNodeGetTransitionAt(n, trans - 1, &trn); FstStreamState s = { .node = fstGetNode(sws->fst, trn.addr), .trans = 0, .out = {.null = false, .out = out}, .autState = autState}; - (void)taosArrayPush(sws->stack, &s); + if (taosArrayPush(sws->stack, &s) == NULL) { + return false; + } return true; } return false; @@ -1273,8 +1297,14 @@ FStmStRslt* stmStNextWith(FStmSt* sws, streamCallback__fn callback) { bool isMatch = automFuncs[aut->type].isMatch(aut, nextState); FstNode* nextNode = fstGetNode(sws->fst, trn.addr); - (void)taosArrayPush(nodes, &nextNode); - (void)taosArrayPush(sws->inp, &(trn.inp)); + if (taosArrayPush(nodes, &nextNode) == NULL) { + taosArrayDestroy(nodes); + return NULL; + } + if (taosArrayPush(sws->inp, &(trn.inp)) == NULL) { + taosArrayDestroy(nodes); + return NULL; + } if (FST_NODE_IS_FINAL(nextNode)) { void* eofState = automFuncs[aut->type].acceptEof(aut, nextState); @@ -1283,10 +1313,16 @@ FStmStRslt* stmStNextWith(FStmSt* sws, streamCallback__fn callback) { } } FstStreamState s1 = {.node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState}; - (void)taosArrayPush(sws->stack, &s1); + if (taosArrayPush(sws->stack, &s1) == NULL) { + taosArrayDestroy(nodes); + return NULL; + } FstStreamState s2 = {.node = nextNode, .trans = 0, .out = {.null = false, .out = out}, .autState = nextState}; - (void)taosArrayPush(sws->stack, &s2); + if (taosArrayPush(sws->stack, &s2) == NULL) { + taosArrayDestroy(nodes); + return NULL; + } int32_t isz = taosArrayGetSize(sws->inp); uint8_t* buf = (uint8_t*)taosMemoryMalloc(isz * sizeof(uint8_t)); @@ -1357,11 +1393,18 @@ FStmBuilder* stmBuilderCreate(Fst* fst, FAutoCtx* aut) { b->aut = aut; b->min = fstBoundStateCreate(Unbounded, NULL); b->max = fstBoundStateCreate(Unbounded, NULL); + + if (b->min == NULL || b->max == NULL) { + stmBuilderDestroy(b); + return NULL; + } + return b; } void stmBuilderDestroy(FStmBuilder* b) { - fstSliceDestroy(&b->min->data); - fstSliceDestroy(&b->max->data); + if (b->min) fstSliceDestroy(&b->min->data); + if (b->max) fstSliceDestroy(&b->max->data); + taosMemoryFreeClear(b->min); taosMemoryFreeClear(b->max); taosMemoryFree(b); diff --git a/source/libs/index/src/indexFstDfa.c b/source/libs/index/src/indexFstDfa.c index f552d5feba..c95847525f 100644 --- a/source/libs/index/src/indexFstDfa.c +++ b/source/libs/index/src/indexFstDfa.c @@ -14,6 +14,7 @@ */ #include "indexFstDfa.h" +#include "indexInt.h" #include "thash.h" const static uint32_t STATE_LIMIT = 1000; @@ -68,23 +69,41 @@ FstDfa *dfaBuilderBuild(FstDfaBuilder *builder) { uint32_t sz = taosArrayGetSize(builder->dfa->insts); FstSparseSet *cur = sparSetCreate(sz); FstSparseSet *nxt = sparSetCreate(sz); + if (cur == NULL || nxt == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } dfaAdd(builder->dfa, cur, 0); uint32_t result; SArray *states = taosArrayInit(0, sizeof(uint32_t)); + if (states == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } if (dfaBuilderCacheState(builder, cur, &result)) { - (void)taosArrayPush(states, &result); + if (taosArrayPush(states, &result) == NULL) { + goto _exception; + } } SHashObj *seen = taosHashInit(12, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (seen == NULL) { + goto _exception; + } + while (taosArrayGetSize(states) != 0) { result = *(uint32_t *)taosArrayPop(states); for (int i = 0; i < 256; i++) { uint32_t ns, dummpy = 0; if (dfaBuilderRunState(builder, cur, nxt, result, i, &ns)) { if (taosHashGet(seen, &ns, sizeof(ns)) == NULL) { - (void)taosHashPut(seen, &ns, sizeof(ns), &dummpy, sizeof(dummpy)); - (void)taosArrayPush(states, &ns); + if (taosHashPut(seen, &ns, sizeof(ns), &dummpy, sizeof(dummpy)) != 0) { + goto _exception; + } + if (taosArrayPush(states, &ns) == NULL) { + goto _exception; + } } } if (taosArrayGetSize(builder->dfa->states) > STATE_LIMIT) { @@ -96,6 +115,11 @@ FstDfa *dfaBuilderBuild(FstDfaBuilder *builder) { taosArrayDestroy(states); taosHashCleanup(seen); return builder->dfa; +_exception: + taosArrayDestroy(states); + taosHashCleanup(seen); + indexError("failed to build dfa since %s", tstrerror(terrno)); + return NULL; } bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet *next, uint32_t state, uint8_t byte, @@ -122,8 +146,13 @@ bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet } bool dfaBuilderCacheState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *result) { + int32_t code = 0; SArray *tinsts = taosArrayInit(4, sizeof(uint32_t)); - bool isMatch = false; + if (tinsts == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exception; + } + bool isMatch = false; for (int i = 0; i < sparSetLen(set); i++) { int32_t ip; @@ -133,10 +162,16 @@ bool dfaBuilderCacheState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *r if (inst->ty == JUMP || inst->ty == SPLIT) { continue; } else if (inst->ty == RANGE) { - (void)taosArrayPush(tinsts, &ip); + if (taosArrayPush(tinsts, &ip) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exception; + } } else if (inst->ty == MATCH) { isMatch = true; - (void)taosArrayPush(tinsts, &ip); + if (taosArrayPush(tinsts, &ip) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exception; + } } } if (taosArrayGetSize(tinsts) == 0) { @@ -149,13 +184,23 @@ bool dfaBuilderCacheState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *r taosArrayDestroy(tinsts); } else { DfaState st = {.insts = tinsts, .isMatch = isMatch}; - (void)taosArrayPush(builder->dfa->states, &st); + if (taosArrayPush(builder->dfa->states, &st) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exception; + } int32_t sz = taosArrayGetSize(builder->dfa->states) - 1; - (void)taosHashPut(builder->cache, &tinsts, sizeof(POINTER_BYTES), &sz, sizeof(sz)); + if ((code = taosHashPut(builder->cache, &tinsts, sizeof(POINTER_BYTES), &sz, sizeof(sz))) != 0) { + goto _exception; + } + *result = sz; } return true; +_exception: + indexError("failed to create dfa state, code:%d", code); + taosArrayDestroy(tinsts); + return false; } FstDfa *dfaCreate(SArray *insts, SArray *states) { diff --git a/source/libs/index/src/indexFstNode.c b/source/libs/index/src/indexFstNode.c index 7f1a86bc55..245f5cf5fc 100644 --- a/source/libs/index/src/indexFstNode.c +++ b/source/libs/index/src/indexFstNode.c @@ -16,9 +16,14 @@ FstBuilderNode* fstBuilderNodeDefault() { FstBuilderNode* bn = taosMemoryMalloc(sizeof(FstBuilderNode)); + if (bn == NULL) return NULL; bn->isFinal = false; bn->finalOutput = 0; bn->trans = taosArrayInit(16, sizeof(FstTransition)); + if (bn->trans == NULL) { + taosMemoryFree(bn); + return NULL; + } return bn; } void fstBuilderNodeDestroy(FstBuilderNode* node) { @@ -56,30 +61,11 @@ bool fstBuilderNodeEqual(FstBuilderNode* n1, FstBuilderNode* n2) { return true; } -FstBuilderNode* fstBuilderNodeClone(FstBuilderNode* src) { - FstBuilderNode* node = taosMemoryMalloc(sizeof(FstBuilderNode)); - if (node == NULL) { - return NULL; - } - // - size_t sz = taosArrayGetSize(src->trans); - SArray* trans = taosArrayInit(sz, sizeof(FstTransition)); - - for (size_t i = 0; i < sz; i++) { - FstTransition* tran = taosArrayGet(src->trans, i); - (void)taosArrayPush(trans, tran); - } - - node->trans = trans; - node->isFinal = src->isFinal; - node->finalOutput = src->finalOutput; - return node; -} // not destroy src, User's bussiness -void fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src) { +int32_t fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src) { if (dst == NULL || src == NULL) { - return; + return TSDB_CODE_INVALID_PARA; } dst->isFinal = src->isFinal; @@ -89,10 +75,18 @@ void fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src) { taosArrayDestroy(dst->trans); size_t sz = taosArrayGetSize(src->trans); dst->trans = taosArrayInit(sz, sizeof(FstTransition)); + if (dst->trans == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } for (size_t i = 0; i < sz; i++) { FstTransition* trn = taosArrayGet(src->trans, i); - (void)taosArrayPush(dst->trans, trn); + if (taosArrayPush(dst->trans, trn) == NULL) { + taosArrayDestroy(dst->trans); + dst->trans = NULL; + return TSDB_CODE_OUT_OF_MEMORY; + } } + return 0; } // bool fstBuilderNodeCompileTo(FstBuilderNode *b, IdxFile *wrt, CompiledAddr lastAddr, CompiledAddr diff --git a/source/libs/index/src/indexFstRegex.c b/source/libs/index/src/indexFstRegex.c index c6ceb1d965..941cb82e90 100644 --- a/source/libs/index/src/indexFstRegex.c +++ b/source/libs/index/src/indexFstRegex.c @@ -39,7 +39,12 @@ FstRegex *regexCreate(const char *str) { for (int i = 0; i < strlen(str); i++) { uint8_t v = str[i]; - (void)taosArrayPush(insts, &v); + if (taosArrayPush(insts, &v) == NULL) { + taosArrayDestroy(insts); + taosMemoryFree(regex->orig); + taosMemoryFree(regex); + return NULL; + } } FstDfaBuilder *builder = dfaBuilderCreate(insts); regex->dfa = dfaBuilderBuild(builder); diff --git a/source/libs/index/src/indexFstRegister.c b/source/libs/index/src/indexFstRegister.c index a8c4365a44..cc069c5d0f 100644 --- a/source/libs/index/src/indexFstRegister.c +++ b/source/libs/index/src/indexFstRegister.c @@ -84,7 +84,10 @@ FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) { for (uint64_t i = 0; i < nCells; i++) { FstRegistryCell cell = {.addr = NONE_ADDRESS, .node = fstBuilderNodeDefault()}; - (void)taosArrayPush(tb, &cell); + if (taosArrayPush(tb, &cell) == NULL) { + fstRegistryDestroy(registry); + return NULL; + } } registry->table = tb; @@ -125,7 +128,7 @@ FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNo entry->addr = cell->addr; return entry; } else { - fstBuilderNodeCloneFrom(cell->node, bNode); + (void)fstBuilderNodeCloneFrom(cell->node, bNode); entry->state = NOTFOUND; entry->cell = cell; // copy or not } @@ -145,7 +148,7 @@ FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNo return entry; } // clone from bNode, refactor later - fstBuilderNodeCloneFrom(cell2->node, bNode); + (void)fstBuilderNodeCloneFrom(cell2->node, bNode); fstRegistryCellSwap(registry->table, start, start + 1); FstRegistryCell* cCell = taosArrayGet(registry->table, start); @@ -166,7 +169,7 @@ FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNo uint64_t last = end - 1; FstRegistryCell* cell = (FstRegistryCell*)taosArrayGet(registry->table, last); // clone from bNode, refactor later - fstBuilderNodeCloneFrom(cell->node, bNode); + (void)fstBuilderNodeCloneFrom(cell->node, bNode); fstRegistryCellPromote(registry->table, last, start); FstRegistryCell* cCell = taosArrayGet(registry->table, start); diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 8c06296732..55a9bb06d9 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -49,7 +49,7 @@ static int tfileReaderLoadFst(TFileReader* reader); static int tfileReaderVerify(TFileReader* reader); static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result); -static SArray* tfileGetFileList(const char* path); +static int32_t tfileGetFileList(const char* path, SArray** pResult); static int tfileRmExpireFile(SArray* result); static void tfileDestroyFileName(void* elem); static int tfileCompare(const void* a, const void* b); @@ -97,9 +97,15 @@ TFileCache* tfileCacheCreate(SIndex* idx, const char* path) { } tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (tcache->tableCache == NULL) { + indexError("failed to open table cache since%s", tstrerror(terrno)); + goto End; + } + tcache->capacity = 64; - SArray* files = tfileGetFileList(path); + SArray* files = NULL; + int32_t code = tfileGetFileList(path, &files); for (size_t i = 0; i < taosArrayGetSize(files); i++) { char* file = taosArrayGetP(files, i); @@ -125,7 +131,11 @@ TFileCache* tfileCacheCreate(SIndex* idx, const char* path) { char buf[128] = {0}; int32_t sz = idxSerialCacheKey(&key, buf); - (void)taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*)); + code = taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*)); + if (code != 0) { + tfileReaderDestroy(reader); + goto End; + } tfileReaderRef(reader); } taosArrayDestroyEx(files, tfileDestroyFileName); @@ -163,6 +173,7 @@ TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) { return *reader; } + int32_t tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) { int32_t code = 0; @@ -172,16 +183,18 @@ int32_t tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) { TFileReader** p = taosHashGet(tcache->tableCache, buf, sz); if (p != NULL && *p != NULL) { TFileReader* oldRdr = *p; - (void)taosHashRemove(tcache->tableCache, buf, sz); - indexInfo("found %s, should remove file %s", buf, oldRdr->ctx->file.buf); - oldRdr->remove = true; - tfileReaderUnRef(oldRdr); + if ((code = taosHashRemove(tcache->tableCache, buf, sz)) != 0) { + indexError("failed to remove old reader from cache since %s, suid:%" PRIu64 ", colName:%s", tstrerror(code), + oldRdr->header.suid, oldRdr->header.colName); + } else { + indexInfo("found %s, should remove file %s", buf, oldRdr->ctx->file.buf); + oldRdr->remove = true; + tfileReaderUnRef(oldRdr); + } } code = taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*)); - if (code == 0) { - tfileReaderRef(reader); - } + tfileReaderRef(reader); return code; } int32_t tfileReaderCreate(IFileCtx* ctx, TFileReader** pReader) { @@ -232,7 +245,7 @@ void tfileReaderDestroy(TFileReader* reader) { } static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { - int ret = 0; + int32_t ret = 0; char* p = tem->colVal; uint64_t sz = tem->nColVal; @@ -246,6 +259,11 @@ static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { tem->suid, tem->colName, tem->colVal, cost); ret = tfileReaderLoadTableIds((TFileReader*)reader, (int32_t)offset, tr->total); + if (ret != 0) { + fstSliceDestroy(&key); + indexError("faile to search since %s", tstrerror(ret)); + return ret; + } cost = taosGetTimestampUs() - et; indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", tem->suid, tem->colName, tem->colVal, cost); @@ -255,17 +273,29 @@ static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { } static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { + int32_t lino = 0; + int32_t code = 0; char* p = tem->colVal; uint64_t sz = tem->nColVal; SArray* offsets = taosArrayInit(16, sizeof(uint64_t)); + if (offsets == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + FAutoCtx* ctx = automCtxCreate((void*)p, AUTOMATION_PREFIX); + if (ctx == NULL) { + taosArrayDestroy(offsets); + return TSDB_CODE_OUT_OF_MEMORY; + } - FAutoCtx* ctx = automCtxCreate((void*)p, AUTOMATION_PREFIX); FStmBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx); FStmSt* st = stmBuilderIntoStm(sb); FStmStRslt* rt = NULL; while ((rt = stmStNextWith(st, NULL)) != NULL) { - (void)taosArrayPush(offsets, &(rt->out.out)); + if (taosArrayPush(offsets, &(rt->out.out)) == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exception); + } swsResultDestroy(rt); } stmStDestroy(st); @@ -275,14 +305,16 @@ static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { for (int i = 0; i < taosArrayGetSize(offsets); i++) { uint64_t offset = *(uint64_t*)taosArrayGet(offsets, i); ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total); - if (ret != 0) { - taosArrayDestroy(offsets); - indexError("failed to find target tablelist"); - return TSDB_CODE_FILE_CORRUPTED; - } + TAOS_CHECK_GOTO(ret, &lino, _exception); } taosArrayDestroy(offsets); return 0; +_exception: + stmStDestroy(st); + stmBuilderDestroy(sb); + taosArrayDestroy(offsets); + indexError("failed to searchPrefix since %s, lino:%d", tstrerror(code), lino); + return code; } static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { int ret = 0; @@ -393,6 +425,12 @@ static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { tem->suid, tem->colName, tem->colVal, cost); ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total); + if (ret != 0) { + indexError("failed to search json since %s", tstrerror(ret)); + taosMemoryFree(p); + fstSliceDestroy(&key); + return ret; + } cost = taosGetTimestampUs() - et; indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, offset: %" PRIu64 ", size: %d, time cost: %" PRIu64 "us", @@ -863,14 +901,24 @@ TFileValue* tfileValueCreate(char* val) { return NULL; } tf->colVal = taosStrdup(val); + if (tf->colVal == NULL) { + taosMemoryFree(tf); + } tf->tableId = taosArrayInit(32, sizeof(uint64_t)); + if (tf->tableId == NULL) { + taosMemoryFree(tf->colVal); + taosMemoryFree(tf); + return NULL; + } return tf; } -int tfileValuePush(TFileValue* tf, uint64_t val) { +int32_t tfileValuePush(TFileValue* tf, uint64_t val) { if (tf == NULL) { - return -1; + return TSDB_CODE_INVALID_PARA; + } + if (taosArrayPush(tf->tableId, &val) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; } - (void)taosArrayPush(tf->tableId, &val); return 0; } void tfileValueDestroy(TFileValue* tf) { @@ -986,8 +1034,10 @@ static int tfileReaderLoadFst(TFileReader* reader) { return reader->fst != NULL ? 0 : -1; } -static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result) { +static int32_t tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result) { // TODO(yihao): opt later + int32_t code = 0; + int32_t lino = 0; IFileCtx* ctx = reader->ctx; // add block cache char block[4096] = {0}; @@ -1003,7 +1053,9 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* while (nid > 0) { int32_t left = block + sizeof(block) - p; if (left >= sizeof(uint64_t)) { - (void)taosArrayPush(result, (uint64_t*)p); + if (taosArrayPush(result, (uint64_t*)p) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } p += sizeof(uint64_t); } else { char buf[sizeof(uint64_t)] = {0}; @@ -1014,7 +1066,9 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* nread = ctx->readFrom(ctx, (uint8_t*)block, sizeof(block), offset); memcpy(buf + left, block, sizeof(uint64_t) - left); - (void)taosArrayPush(result, (uint64_t*)buf); + if (taosArrayPush(result, (uint64_t*)buf) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } p = block + sizeof(uint64_t) - left; } nid -= 1; @@ -1059,16 +1113,19 @@ void tfileReaderUnRef(TFileReader* rd) { } } -static SArray* tfileGetFileList(const char* path) { +static int32_t tfileGetFileList(const char* path, SArray** ppResult) { + int32_t code = 0; char buf[128] = {0}; uint64_t suid; int64_t version; SArray* files = taosArrayInit(4, sizeof(void*)); + if (files == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } TdDirPtr pDir = taosOpenDir(path); if (NULL == pDir) { - taosArrayDestroy(files); - return NULL; + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _exception); } TdDirEntryPtr pDirEntry; while ((pDirEntry = taosReadDir(pDir)) != NULL) { @@ -1079,15 +1136,29 @@ static SArray* tfileGetFileList(const char* path) { size_t len = strlen(path) + 1 + strlen(file) + 1; char* buf = taosMemoryCalloc(1, len); + if (buf == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception); + } + sprintf(buf, "%s/%s", path, file); - (void)taosArrayPush(files, &buf); + if (taosArrayPush(files, &buf) == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception); + } } (void)taosCloseDir(&pDir); taosArraySort(files, tfileCompare); (void)tfileRmExpireFile(files); + *ppResult = files; + return 0; - return files; +_exception: + (void)taosCloseDir(&pDir); + if (files != NULL) { + taosArrayDestroyEx(files, tfileDestroyFileName); + taosArrayDestroy(files); + } + return code; } static int tfileRmExpireFile(SArray* result) { // TODO(yihao): remove expire tindex after restart diff --git a/source/libs/index/src/indexUtil.c b/source/libs/index/src/indexUtil.c index 0776e71180..6a52b7b5a4 100644 --- a/source/libs/index/src/indexUtil.c +++ b/source/libs/index/src/indexUtil.c @@ -36,12 +36,16 @@ static FORCE_INLINE int iBinarySearch(SArray *arr, int s, int e, uint64_t k) { return s; } -void iIntersection(SArray *in, SArray *out) { +int32_t iIntersection(SArray *in, SArray *out) { + int32_t code = 0; int32_t sz = (int32_t)taosArrayGetSize(in); if (sz <= 0) { - return; + return 0; } MergeIndex *mi = taosMemoryCalloc(sz, sizeof(MergeIndex)); + if (mi == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } for (int i = 0; i < sz; i++) { SArray *t = taosArrayGetP(in, i); mi[i].len = (int32_t)taosArrayGetSize(t); @@ -64,19 +68,25 @@ void iIntersection(SArray *in, SArray *out) { } } if (has == true) { - (void)taosArrayPush(out, &tgt); + if (taosArrayPush(out, &tgt) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + break; + } } } taosMemoryFreeClear(mi); + return code; } -void iUnion(SArray *in, SArray *out) { +int32_t iUnion(SArray *in, SArray *out) { + int32_t code = 0; int32_t sz = (int32_t)taosArrayGetSize(in); if (sz <= 0) { - return; + return 0; } if (sz == 1) { - (void)taosArrayAddAll(out, taosArrayGetP(in, 0)); - return; + if (taosArrayAddAll(out, taosArrayGetP(in, 0)) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } MergeIndex *mi = taosMemoryCalloc(sz, sizeof(MergeIndex)); @@ -108,19 +118,23 @@ void iUnion(SArray *in, SArray *out) { continue; } } - (void)taosArrayPush(out, &mVal); + if (taosArrayPush(out, &mVal) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + break; + } } else { break; } } taosMemoryFreeClear(mi); + return 0; } -void iExcept(SArray *total, SArray *except) { +int32_t iExcept(SArray *total, SArray *except) { int32_t tsz = (int32_t)taosArrayGetSize(total); int32_t esz = (int32_t)taosArrayGetSize(except); if (esz == 0 || tsz == 0) { - return; + return 0; } int vIdx = 0; @@ -135,6 +149,7 @@ void iExcept(SArray *total, SArray *except) { } taosArrayPopTailBatch(total, tsz - vIdx); + return 0; } int uidCompare(const void *a, const void *b) { @@ -191,20 +206,37 @@ void idxTRsltDestroy(SIdxTRslt *tr) { taosArrayDestroy(tr->del); taosMemoryFree(tr); } -void idxTRsltMergeTo(SIdxTRslt *tr, SArray *result) { +int32_t idxTRsltMergeTo(SIdxTRslt *tr, SArray *result) { + int32_t code = 0; taosArraySort(tr->total, uidCompare); taosArraySort(tr->add, uidCompare); taosArraySort(tr->del, uidCompare); if (taosArrayGetSize(tr->total) == 0 || taosArrayGetSize(tr->add) == 0) { SArray *t = taosArrayGetSize(tr->total) == 0 ? tr->add : tr->total; - (void)taosArrayAddAll(result, t); + if (taosArrayAddAll(result, t) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } else { SArray *arrs = taosArrayInit(2, sizeof(void *)); - (void)taosArrayPush(arrs, &tr->total); - (void)taosArrayPush(arrs, &tr->add); - iUnion(arrs, result); + if (arrs == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (taosArrayPush(arrs, &tr->total) == NULL) { + taosArrayDestroy(arrs); + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (taosArrayPush(arrs, &tr->add) == NULL) { + taosArrayDestroy(arrs); + return TSDB_CODE_OUT_OF_MEMORY; + } + code = iUnion(arrs, result); taosArrayDestroy(arrs); } - iExcept(result, tr->del); + if (code == 0) { + code = iExcept(result, tr->del); + } + return code; } diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 18d17cc702..52deeb1638 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -103,6 +103,7 @@ static int32_t exprNodeCopy(const SExprNode* pSrc, SExprNode* pDst) { COPY_CHAR_ARRAY_FIELD(aliasName); COPY_CHAR_ARRAY_FIELD(userAlias); COPY_SCALAR_FIELD(orderAlias); + COPY_SCALAR_FIELD(projIdx); return TSDB_CODE_SUCCESS; } @@ -124,6 +125,8 @@ static int32_t columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) { COPY_SCALAR_FIELD(tableHasPk); COPY_SCALAR_FIELD(isPk); COPY_SCALAR_FIELD(numOfPKs); + COPY_SCALAR_FIELD(projRefIdx); + COPY_SCALAR_FIELD(resIdx); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 153748d128..1547c985f6 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -2263,8 +2263,13 @@ static EDealRes doCollect(SCollectColumnsCxt* pCxt, SColumnNode* pCol, SNode* pN } else { len = snprintf(name, sizeof(name), "%s.%s", pCol->tableAlias, pCol->colName); } - if (NULL == taosHashGet(pCxt->pColHash, name, len)) { - pCxt->errCode = taosHashPut(pCxt->pColHash, name, len, NULL, 0); + if (pCol->projRefIdx > 0) { + len = taosHashBinary(name, strlen(name)); + len += sprintf(name + len, "_%d", pCol->projRefIdx); + } + SNode** pNodeFound = taosHashGet(pCxt->pColHash, name, len); + if (pNodeFound == NULL) { + pCxt->errCode = taosHashPut(pCxt->pColHash, name, len, &pNode, POINTER_BYTES); if (TSDB_CODE_SUCCESS == pCxt->errCode) { SNode* pNew = NULL; pCxt->errCode = nodesCloneNode(pNode, &pNew); @@ -2307,7 +2312,6 @@ static EDealRes collectColumnsExt(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } - int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type, SNodeList** pCols) { if (NULL == pSelect || NULL == pCols) { diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index f0748288e9..4f42827cc5 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -24,7 +24,7 @@ #define CHECK_MAKE_NODE(p) \ do { \ if (NULL == (p)) { \ - return NULL; \ + goto _err; \ } \ } while (0) @@ -32,14 +32,21 @@ do { \ if (NULL == (p)) { \ pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; \ - return NULL; \ + goto _err; \ } \ } while (0) #define CHECK_PARSER_STATUS(pCxt) \ do { \ if (TSDB_CODE_SUCCESS != pCxt->errCode) { \ - return NULL; \ + goto _err; \ + } \ + } while (0) + +#define CHECK_NAME(p) \ + do { \ + if (!p) { \ + goto _err; \ } \ } while (0) @@ -275,6 +282,9 @@ SNode* createRawExprNode(SAstCreateContext* pCxt, const SToken* pToken, SNode* p target->n = pToken->n; target->pNode = pNode; return (SNode*)target; +_err: + nodesDestroyNode(pNode); + return NULL; } SNode* createRawExprNodeExt(SAstCreateContext* pCxt, const SToken* pStart, const SToken* pEnd, SNode* pNode) { @@ -286,6 +296,9 @@ SNode* createRawExprNodeExt(SAstCreateContext* pCxt, const SToken* pStart, const target->n = (pEnd->z + pEnd->n) - pStart->z; target->pNode = pNode; return (SNode*)target; +_err: + nodesDestroyNode(pNode); + return NULL; } SNode* setRawExprNodeIsPseudoColumn(SAstCreateContext* pCxt, SNode* pNode, bool isPseudoColumn) { @@ -295,6 +308,9 @@ SNode* setRawExprNodeIsPseudoColumn(SAstCreateContext* pCxt, SNode* pNode, bool } ((SRawExprNode*)pNode)->isPseudoColumn = isPseudoColumn; return pNode; +_err: + nodesDestroyNode(pNode); + return NULL; } SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { @@ -325,6 +341,9 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { pRawExpr->pNode = NULL; nodesDestroyNode(pNode); return pRealizedExpr; +_err: + nodesDestroyNode(pNode); + return NULL; } SToken getTokenFromRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { @@ -348,12 +367,19 @@ SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode) { return NULL; } return list; +_err: + nodesDestroyNode(pNode); + return NULL; } SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode) { CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesListAppend(pList, pNode); return pList; +_err: + nodesDestroyNode(pNode); + nodesDestroyList(pList); + return NULL; } SNode* createColumnNode(SAstCreateContext* pCxt, SToken* pTableAlias, SToken* pColumnName) { @@ -369,6 +395,8 @@ SNode* createColumnNode(SAstCreateContext* pCxt, SToken* pTableAlias, SToken* pC } COPY_STRING_FORM_ID_TOKEN(col->colName, pColumnName); return (SNode*)col; +_err: + return NULL; } SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral) { @@ -393,6 +421,8 @@ SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* } val->translate = false; return (SNode*)val; +_err: + return NULL; } SNode* createRawValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode* pNode) { @@ -433,12 +463,15 @@ _exit: return NULL; } return (SNode*)val; +_err: + nodesDestroyNode(pNode); + return NULL; } SNode* createRawValueNodeExt(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode* pLeft, SNode* pRight) { - CHECK_PARSER_STATUS(pCxt); SValueNode* val = NULL; + CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&val); if (TSDB_CODE_SUCCESS != pCxt->errCode) { @@ -463,11 +496,13 @@ SNode* createRawValueNodeExt(SAstCreateContext* pCxt, int32_t dataType, const ST _exit: nodesDestroyNode(pLeft); nodesDestroyNode(pRight); - if (pCxt->errCode != 0) { - nodesDestroyNode((SNode*)val); - return NULL; - } + CHECK_PARSER_STATUS(pCxt); return (SNode*)val; +_err: + nodesDestroyNode((SNode*)val); + nodesDestroyNode(pLeft); + nodesDestroyNode(pRight); + return NULL; } static bool hasHint(SNodeList* pHintList, EHintOption hint) { @@ -516,7 +551,9 @@ bool addHintNodeToList(SAstCreateContext* pCxt, SNodeList** ppHintList, EHintOpt SHintNode* hint = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_HINT, (SNode**)&hint); - CHECK_MAKE_NODE(hint); + if (!hint) { + return true; + } hint->option = opt; hint->value = value; @@ -667,6 +704,8 @@ SNodeList* createHintNodeList(SAstCreateContext* pCxt, const SToken* pLiteral) { taosMemoryFree(hint); return pHintList; +_err: + return NULL; } SNode* createIdentifierValueNode(SAstCreateContext* pCxt, SToken* pLiteral) { @@ -724,6 +763,8 @@ SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral) val->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; val->node.resType.precision = TSDB_TIME_PRECISION_MILLI; return (SNode*)val; +_err: + return NULL; } SNode* createTimeOffsetValueNode(SAstCreateContext* pCxt, const SToken* pLiteral) { @@ -774,6 +815,8 @@ SNode* createTimeOffsetValueNode(SAstCreateContext* pCxt, const SToken* pLiteral val->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; val->node.resType.precision = TSDB_TIME_PRECISION_MILLI; return (SNode*)val; +_err: + return NULL; } SNode* createDefaultDatabaseCondValue(SAstCreateContext* pCxt) { @@ -796,6 +839,8 @@ SNode* createDefaultDatabaseCondValue(SAstCreateContext* pCxt) { val->node.resType.bytes = strlen(val->literal); val->node.resType.precision = TSDB_TIME_PRECISION_MILLI; return (SNode*)val; +_err: + return NULL; } SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLiteral) { @@ -823,9 +868,13 @@ SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLitera } if (NULL == taosArrayPush(pCxt->pPlaceholderValues, &val)) { pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + nodesDestroyNode((SNode*)val); + taosArrayDestroy(pCxt->pPlaceholderValues); + return NULL; } - CHECK_PARSER_STATUS(pCxt); return (SNode*)val; +_err: + return NULL; } static int32_t addParamToLogicConditionNode(SLogicConditionNode* pCond, SNode* pParam) { @@ -858,6 +907,10 @@ SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType typ return NULL; } return (SNode*)cond; +_err: + nodesDestroyNode(pParam1); + nodesDestroyNode(pParam2); + return NULL; } static uint8_t getMinusDataType(uint8_t orgType) { @@ -880,7 +933,7 @@ SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pL char* pNewLiteral = taosMemoryCalloc(1, strlen(pVal->literal) + 2); if (!pNewLiteral) { pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + goto _err; } if ('+' == pVal->literal[0]) { sprintf(pNewLiteral, "-%s", pVal->literal + 1); @@ -901,55 +954,54 @@ SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pL op->pLeft = pLeft; op->pRight = pRight; return (SNode*)op; +_err: + nodesDestroyNode(pLeft); + nodesDestroyNode(pRight); + return NULL; } SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight) { + SNode* pNew = NULL, *pGE = NULL, *pLE = NULL; CHECK_PARSER_STATUS(pCxt); - SNode* pNew = NULL; pCxt->errCode = nodesCloneNode(pExpr, &pNew); CHECK_PARSER_STATUS(pCxt); - SNode* pGE = createOperatorNode(pCxt, OP_TYPE_GREATER_EQUAL, pExpr, pLeft); - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - nodesDestroyNode(pNew); - return NULL; - } - SNode* pLE = createOperatorNode(pCxt, OP_TYPE_LOWER_EQUAL, pNew, pRight); - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - nodesDestroyNode(pNew); - nodesDestroyNode(pGE); - return NULL; - } + pGE = createOperatorNode(pCxt, OP_TYPE_GREATER_EQUAL, pExpr, pLeft); + CHECK_PARSER_STATUS(pCxt); + pLE = createOperatorNode(pCxt, OP_TYPE_LOWER_EQUAL, pNew, pRight); + CHECK_PARSER_STATUS(pCxt); SNode* pRet = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, pGE, pLE); - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - nodesDestroyNode(pNew); - nodesDestroyNode(pGE); - nodesDestroyNode(pLE); - } + CHECK_PARSER_STATUS(pCxt); return pRet; +_err: + nodesDestroyNode(pNew); + nodesDestroyNode(pGE); + nodesDestroyNode(pLE); + nodesDestroyNode(pExpr); + nodesDestroyNode(pLeft); + nodesDestroyNode(pRight); + return NULL; } SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight) { + SNode* pNew = NULL, *pLT = NULL, *pGT = NULL; CHECK_PARSER_STATUS(pCxt); - SNode* pLT = createOperatorNode(pCxt, OP_TYPE_LOWER_THAN, pExpr, pLeft); - CHECK_MAKE_NODE(pLT); - SNode* pNew = NULL; pCxt->errCode = nodesCloneNode(pExpr, &pNew); - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - nodesDestroyNode(pLT); - return NULL; - } - SNode* pGT = createOperatorNode(pCxt, OP_TYPE_GREATER_THAN, pNew, pRight); - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - nodesDestroyNode(pLT); - nodesDestroyNode(pNew); - } + CHECK_PARSER_STATUS(pCxt); + pLT = createOperatorNode(pCxt, OP_TYPE_LOWER_THAN, pExpr, pLeft); + CHECK_PARSER_STATUS(pCxt); + pGT = createOperatorNode(pCxt, OP_TYPE_GREATER_THAN, pNew, pRight); + CHECK_PARSER_STATUS(pCxt); SNode* pRet = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, pLT, pGT); - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - nodesDestroyNode(pLT); - nodesDestroyNode(pGT); - nodesDestroyNode(pNew); - } + CHECK_PARSER_STATUS(pCxt); return pRet; +_err: + nodesDestroyNode(pNew); + nodesDestroyNode(pGT); + nodesDestroyNode(pLT); + nodesDestroyNode(pExpr); + nodesDestroyNode(pLeft); + nodesDestroyNode(pRight); + return NULL; } static SNode* createPrimaryKeyCol(SAstCreateContext* pCxt, const SToken* pFuncName) { @@ -965,6 +1017,8 @@ static SNode* createPrimaryKeyCol(SAstCreateContext* pCxt, const SToken* pFuncNa } pCol->isPrimTs = true; return (SNode*)pCol; +_err: + return NULL; } SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList) { @@ -978,11 +1032,14 @@ SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNod COPY_STRING_FORM_ID_TOKEN(func->functionName, pFuncName); func->pParameterList = pParameterList; return (SNode*)func; +_err: + nodesDestroyList(pParameterList); + return NULL; } SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType dt) { - CHECK_PARSER_STATUS(pCxt); SFunctionNode* func = NULL; + CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&func); CHECK_MAKE_NODE(func); strcpy(func->functionName, "cast"); @@ -995,11 +1052,15 @@ SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType d pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr); CHECK_PARSER_STATUS(pCxt); return (SNode*)func; +_err: + nodesDestroyNode((SNode*)func); + nodesDestroyNode(pExpr); + return NULL; } SNode* createPositionFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SNode* pExpr2) { - CHECK_PARSER_STATUS(pCxt); SFunctionNode* func = NULL; + CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&func); CHECK_MAKE_NODE(func); strcpy(func->functionName, "position"); @@ -1008,11 +1069,16 @@ SNode* createPositionFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SNode* pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr2); CHECK_PARSER_STATUS(pCxt); return (SNode*)func; +_err: + nodesDestroyNode((SNode*)func); + nodesDestroyNode(pExpr); + nodesDestroyNode(pExpr2); + return NULL; } SNode* createTrimFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, ETrimType type) { - CHECK_PARSER_STATUS(pCxt); SFunctionNode* func = NULL; + CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&func); CHECK_MAKE_NODE(func); strcpy(func->functionName, "trim"); @@ -1020,11 +1086,15 @@ SNode* createTrimFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, ETrimType t pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr); CHECK_PARSER_STATUS(pCxt); return (SNode*)func; +_err: + nodesDestroyNode((SNode*)func); + nodesDestroyNode(pExpr); + return NULL; } SNode* createTrimFunctionNodeExt(SAstCreateContext* pCxt, SNode* pExpr, SNode* pExpr2, ETrimType type) { - CHECK_PARSER_STATUS(pCxt); SFunctionNode* func = NULL; + CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&func); CHECK_MAKE_NODE(func); strcpy(func->functionName, "trim"); @@ -1034,11 +1104,16 @@ SNode* createTrimFunctionNodeExt(SAstCreateContext* pCxt, SNode* pExpr, SNode* p pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr2); CHECK_PARSER_STATUS(pCxt); return (SNode*)func; +_err: + nodesDestroyNode((SNode*)func); + nodesDestroyNode(pExpr); + nodesDestroyNode(pExpr2); + return NULL; } SNode* createSubstrFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SNode* pExpr2) { - CHECK_PARSER_STATUS(pCxt); SFunctionNode* func = NULL; + CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&func); CHECK_MAKE_NODE(func); strcpy(func->functionName, "substr"); @@ -1047,11 +1122,16 @@ SNode* createSubstrFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SNode* pE pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr2); CHECK_PARSER_STATUS(pCxt); return (SNode*)func; +_err: + nodesDestroyNode((SNode*)func); + nodesDestroyNode(pExpr); + nodesDestroyNode(pExpr2); + return NULL; } SNode* createSubstrFunctionNodeExt(SAstCreateContext* pCxt, SNode* pExpr, SNode* pExpr2, SNode* pExpr3) { - CHECK_PARSER_STATUS(pCxt); SFunctionNode* func = NULL; + CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&func); CHECK_MAKE_NODE(func); strcpy(func->functionName, "substr"); @@ -1062,40 +1142,48 @@ SNode* createSubstrFunctionNodeExt(SAstCreateContext* pCxt, SNode* pExpr, SNode* pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr3); CHECK_PARSER_STATUS(pCxt); return (SNode*)func; +_err: + nodesDestroyNode((SNode*)func); + nodesDestroyNode(pExpr); + nodesDestroyNode(pExpr2); + nodesDestroyNode(pExpr3); + return NULL; } SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList) { - CHECK_PARSER_STATUS(pCxt); SNodeListNode* list = NULL; + CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_NODE_LIST, (SNode**)&list); CHECK_MAKE_NODE(list); list->pNodeList = pList; return (SNode*)list; +_err: + nodesDestroyList(pList); + return NULL; } SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2) { - CHECK_PARSER_STATUS(pCxt); SNodeListNode* list = NULL; + CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_NODE_LIST, (SNode**)&list); CHECK_MAKE_NODE(list); pCxt->errCode = nodesListMakeStrictAppend(&list->pNodeList, p1); - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - nodesDestroyNode((SNode*)list); - return NULL; - } + CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesListStrictAppend(list->pNodeList, p2); - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - nodesDestroyNode((SNode*)list); - return NULL; - } + CHECK_PARSER_STATUS(pCxt); return (SNode*)list; +_err: + nodesDestroyNode((SNode*)list); + nodesDestroyNode(p1); + nodesDestroyNode(p2); + return NULL; } SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias) { CHECK_PARSER_STATUS(pCxt); - if (!checkDbName(pCxt, pDbName, true) || !checkTableName(pCxt, pTableName) || !checkTableName(pCxt, pTableAlias)) { - return NULL; - } + CHECK_NAME(checkDbName(pCxt, pDbName, true)); + CHECK_NAME(checkTableName(pCxt, pTableName)); + CHECK_NAME(checkTableName(pCxt, pTableAlias)); SRealTableNode* realTable = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_REAL_TABLE, (SNode**)&realTable); CHECK_MAKE_NODE(realTable); @@ -1111,6 +1199,8 @@ SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTa } COPY_STRING_FORM_ID_TOKEN(realTable->table.tableName, pTableName); return (SNode*)realTable; +_err: + return NULL; } SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, SToken* pTableAlias) { @@ -1134,6 +1224,9 @@ SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, SToken* pT strcpy(((SSetOperator*)pSubquery)->stmtName, tempTable->table.tableAlias); } return (SNode*)tempTable; +_err: + nodesDestroyNode(pSubquery); + return NULL; } SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, EJoinSubType stype, SNode* pLeft, SNode* pRight, @@ -1148,13 +1241,17 @@ SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, EJoinSubType joinTable->pRight = pRight; joinTable->pOnCond = pJoinCond; return (SNode*)joinTable; +_err: + nodesDestroyNode(pLeft); + nodesDestroyNode(pRight); + nodesDestroyNode(pJoinCond); + return NULL; } SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewName) { CHECK_PARSER_STATUS(pCxt); - if (!checkDbName(pCxt, pDbName, true) || !checkViewName(pCxt, pViewName)) { - return NULL; - } + CHECK_NAME(checkDbName(pCxt, pDbName, true)); + CHECK_NAME(checkViewName(pCxt, pViewName)); SViewNode* pView = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_VIEW, (SNode**)&pView); CHECK_MAKE_NODE(pView); @@ -1165,6 +1262,8 @@ SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewNam } COPY_STRING_FORM_ID_TOKEN(pView->table.tableName, pViewName); return (SNode*)pView; +_err: + return NULL; } SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) { @@ -1177,6 +1276,8 @@ SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const STok limitNode->offset = taosStr2Int64(pOffset->z, NULL, 10); } return (SNode*)limitNode; +_err: + return NULL; } SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder) { @@ -1191,6 +1292,9 @@ SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order } orderByExpr->nullOrder = nullOrder; return (SNode*)orderByExpr; +_err: + nodesDestroyNode(pExpr); + return NULL; } SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap) { @@ -1201,121 +1305,145 @@ SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap session->pCol = (SColumnNode*)pCol; session->pGap = (SValueNode*)pGap; return (SNode*)session; +_err: + nodesDestroyNode(pCol); + nodesDestroyNode(pGap); + return NULL; } SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr) { - CHECK_PARSER_STATUS(pCxt); SStateWindowNode* state = NULL; + CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_STATE_WINDOW, (SNode**)&state); CHECK_MAKE_NODE(state); state->pCol = createPrimaryKeyCol(pCxt, NULL); - if (NULL == state->pCol) { - nodesDestroyNode((SNode*)state); - CHECK_MAKE_NODE(NULL); - } + CHECK_MAKE_NODE(state->pCol); state->pExpr = pExpr; return (SNode*)state; +_err: + nodesDestroyNode((SNode*)state); + nodesDestroyNode(pExpr); + return NULL; } SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond) { - CHECK_PARSER_STATUS(pCxt); SEventWindowNode* pEvent = NULL; + CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_EVENT_WINDOW, (SNode**)&pEvent); CHECK_MAKE_NODE(pEvent); pEvent->pCol = createPrimaryKeyCol(pCxt, NULL); - if (NULL == pEvent->pCol) { - nodesDestroyNode((SNode*)pEvent); - CHECK_MAKE_NODE(NULL); - } + CHECK_MAKE_NODE(pEvent->pCol); pEvent->pStartCond = pStartCond; pEvent->pEndCond = pEndCond; return (SNode*)pEvent; +_err: + nodesDestroyNode((SNode*)pEvent); + nodesDestroyNode(pStartCond); + nodesDestroyNode(pEndCond); + return NULL; } SNode* createCountWindowNode(SAstCreateContext* pCxt, const SToken* pCountToken, const SToken* pSlidingToken) { - CHECK_PARSER_STATUS(pCxt); SCountWindowNode* pCount = NULL; + CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_COUNT_WINDOW, (SNode**)&pCount); CHECK_MAKE_NODE(pCount); pCount->pCol = createPrimaryKeyCol(pCxt, NULL); - if (NULL == pCount->pCol) { - nodesDestroyNode((SNode*)pCount); - CHECK_MAKE_NODE(NULL); - } + CHECK_MAKE_NODE(pCount->pCol); pCount->windowCount = taosStr2Int64(pCountToken->z, NULL, 10); pCount->windowSliding = taosStr2Int64(pSlidingToken->z, NULL, 10); return (SNode*)pCount; +_err: + nodesDestroyNode((SNode*)pCount); + return NULL; } SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill) { - CHECK_PARSER_STATUS(pCxt); SIntervalWindowNode* interval = NULL; + CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_INTERVAL_WINDOW, (SNode**)&interval); CHECK_MAKE_NODE(interval); interval->pCol = createPrimaryKeyCol(pCxt, NULL); - if (NULL == interval->pCol) { - nodesDestroyNode((SNode*)interval); - CHECK_MAKE_NODE(NULL); - } + CHECK_MAKE_NODE(interval->pCol); interval->pInterval = pInterval; interval->pOffset = pOffset; interval->pSliding = pSliding; interval->pFill = pFill; return (SNode*)interval; +_err: + nodesDestroyNode((SNode*)interval); + nodesDestroyNode(pInterval); + nodesDestroyNode(pOffset); + nodesDestroyNode(pSliding); + nodesDestroyNode(pFill); + return NULL; } SNode* createWindowOffsetNode(SAstCreateContext* pCxt, SNode* pStartOffset, SNode* pEndOffset) { - CHECK_PARSER_STATUS(pCxt); SWindowOffsetNode* winOffset = NULL; + CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_WINDOW_OFFSET, (SNode**)&winOffset); CHECK_MAKE_NODE(winOffset); winOffset->pStartOffset = pStartOffset; winOffset->pEndOffset = pEndOffset; return (SNode*)winOffset; +_err: + nodesDestroyNode((SNode*)winOffset); + nodesDestroyNode(pStartOffset); + nodesDestroyNode(pEndOffset); + return NULL; } SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues) { - CHECK_PARSER_STATUS(pCxt); SFillNode* fill = NULL; + CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_FILL, (SNode**)&fill); CHECK_MAKE_NODE(fill); fill->mode = mode; fill->pValues = pValues; fill->pWStartTs = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&(fill->pWStartTs)); - if (NULL == fill->pWStartTs) { - nodesDestroyNode((SNode*)fill); - return NULL; - } + CHECK_MAKE_NODE(fill->pWStartTs); strcpy(((SFunctionNode*)fill->pWStartTs)->functionName, "_wstart"); return (SNode*)fill; +_err: + nodesDestroyNode((SNode*)fill); + nodesDestroyNode(pValues); + return NULL; } SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode) { - CHECK_PARSER_STATUS(pCxt); SGroupingSetNode* groupingSet = NULL; + CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_GROUPING_SET, (SNode**)&groupingSet); CHECK_MAKE_NODE(groupingSet); groupingSet->groupingSetType = GP_TYPE_NORMAL; groupingSet->pParameterList = NULL; pCxt->errCode = nodesListMakeAppend(&groupingSet->pParameterList, pNode); - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - nodesDestroyNode((SNode*)groupingSet); - return NULL; - } CHECK_PARSER_STATUS(pCxt); return (SNode*)groupingSet; +_err: + nodesDestroyNode((SNode*)groupingSet); + nodesDestroyNode(pNode); + return NULL; } SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd) { CHECK_PARSER_STATUS(pCxt); return createBetweenAnd(pCxt, createPrimaryKeyCol(pCxt, NULL), pStart, pEnd); +_err: + nodesDestroyNode(pStart); + nodesDestroyNode(pEnd); + return NULL; } SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint) { CHECK_PARSER_STATUS(pCxt); return createOperatorNode(pCxt, OP_TYPE_EQUAL, createPrimaryKeyCol(pCxt, NULL), pPoint); +_err: + nodesDestroyNode(pPoint); + return NULL; } SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen) { @@ -1326,6 +1454,10 @@ SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen) { pWhenThen->pWhen = pWhen; pWhenThen->pThen = pThen; return (SNode*)pWhenThen; +_err: + nodesDestroyNode(pWhen); + nodesDestroyNode(pThen); + return NULL; } SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse) { @@ -1337,6 +1469,11 @@ SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhe pCaseWhen->pWhenThenList = pWhenThenList; pCaseWhen->pElse = pElse; return (SNode*)pCaseWhen; +_err: + nodesDestroyNode(pCase); + nodesDestroyList(pWhenThenList); + nodesDestroyNode(pElse); + return NULL; } SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, SToken* pAlias) { @@ -1350,6 +1487,9 @@ SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, SToken* pAlias) pExpr->userAlias[len] = '\0'; pExpr->asAlias = true; return pNode; +_err: + nodesDestroyNode(pNode); + return NULL; } SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere) { @@ -1358,6 +1498,10 @@ SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere) { ((SSelectStmt*)pStmt)->pWhere = pWhere; } return pStmt; +_err: + nodesDestroyNode(pStmt); + nodesDestroyNode(pWhere); + return NULL; } SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList) { @@ -1366,6 +1510,10 @@ SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pP ((SSelectStmt*)pStmt)->pPartitionByList = pPartitionByList; } return pStmt; +_err: + nodesDestroyNode(pStmt); + nodesDestroyList(pPartitionByList); + return NULL; } SNode* addWindowClauseClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWindow) { @@ -1374,6 +1522,10 @@ SNode* addWindowClauseClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWind ((SSelectStmt*)pStmt)->pWindow = pWindow; } return pStmt; +_err: + nodesDestroyNode(pStmt); + nodesDestroyNode(pWindow); + return NULL; } SNode* addGroupByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pGroupByList) { @@ -1382,6 +1534,10 @@ SNode* addGroupByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pGroup ((SSelectStmt*)pStmt)->pGroupByList = pGroupByList; } return pStmt; +_err: + nodesDestroyNode(pStmt); + nodesDestroyList(pGroupByList); + return NULL; } SNode* addHavingClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pHaving) { @@ -1390,6 +1546,10 @@ SNode* addHavingClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pHaving) { ((SSelectStmt*)pStmt)->pHaving = pHaving; } return pStmt; +_err: + nodesDestroyNode(pStmt); + nodesDestroyNode(pHaving); + return NULL; } SNode* addOrderByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrderByList) { @@ -1403,6 +1563,10 @@ SNode* addOrderByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrder ((SSetOperator*)pStmt)->pOrderByList = pOrderByList; } return pStmt; +_err: + nodesDestroyNode(pStmt); + nodesDestroyList(pOrderByList); + return NULL; } SNode* addSlimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit) { @@ -1414,6 +1578,10 @@ SNode* addSlimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit) { ((SSelectStmt*)pStmt)->pSlimit = (SLimitNode*)pSlimit; } return pStmt; +_err: + nodesDestroyNode(pStmt); + nodesDestroyNode(pSlimit); + return NULL; } SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit) { @@ -1427,6 +1595,10 @@ SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit) { ((SSetOperator*)pStmt)->pLimit = pLimit; } return pStmt; +_err: + nodesDestroyNode(pStmt); + nodesDestroyNode(pLimit); + return NULL; } SNode* addRangeClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pRange) { @@ -1435,6 +1607,10 @@ SNode* addRangeClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pRange) { ((SSelectStmt*)pStmt)->pRange = pRange; } return pStmt; +_err: + nodesDestroyNode(pStmt); + nodesDestroyNode(pRange); + return NULL; } SNode* addEveryClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pEvery) { @@ -1443,6 +1619,10 @@ SNode* addEveryClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pEvery) { ((SSelectStmt*)pStmt)->pEvery = pEvery; } return pStmt; +_err: + nodesDestroyNode(pStmt); + nodesDestroyNode(pEvery); + return NULL; } SNode* addFillClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pFill) { @@ -1451,9 +1631,14 @@ SNode* addFillClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pFill) { SFillNode* pFillClause = (SFillNode*)pFill; nodesDestroyNode(pFillClause->pWStartTs); pFillClause->pWStartTs = createPrimaryKeyCol(pCxt, NULL); + CHECK_MAKE_NODE(pFillClause->pWStartTs); ((SSelectStmt*)pStmt)->pFill = (SNode*)pFillClause; } return pStmt; +_err: + nodesDestroyNode(pStmt); + nodesDestroyNode(pFill); + return NULL; } SNode* addJLimitClause(SAstCreateContext* pCxt, SNode* pJoin, SNode* pJLimit) { @@ -1465,6 +1650,10 @@ SNode* addJLimitClause(SAstCreateContext* pCxt, SNode* pJoin, SNode* pJLimit) { pJoinNode->pJLimit = pJLimit; return pJoin; +_err: + nodesDestroyNode(pJoin); + nodesDestroyNode(pJLimit); + return NULL; } SNode* addWindowOffsetClause(SAstCreateContext* pCxt, SNode* pJoin, SNode* pWinOffset) { @@ -1476,6 +1665,10 @@ SNode* addWindowOffsetClause(SAstCreateContext* pCxt, SNode* pJoin, SNode* pWinO pJoinNode->pWindowOffset = pWinOffset; return pJoin; +_err: + nodesDestroyNode(pJoin); + nodesDestroyNode(pWinOffset); + return NULL; } SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable, @@ -1485,6 +1678,11 @@ SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pPr pCxt->errCode = createSelectStmtImpl(isDistinct, pProjectionList, pTable, pHint, &select); CHECK_MAKE_NODE(select); return select; +_err: + nodesDestroyList(pProjectionList); + nodesDestroyNode(pTable); + nodesDestroyList(pHint); + return NULL; } SNode* setSelectStmtTagMode(SAstCreateContext* pCxt, SNode* pStmt, bool bSelectTags) { @@ -1516,6 +1714,10 @@ SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* setSubquery(setOp->pRight); sprintf(setOp->stmtName, "%p", setOp); return (SNode*)setOp; +_err: + nodesDestroyNode(pLeft); + nodesDestroyNode(pRight); + return NULL; } static void updateWalOptionsDefault(SDatabaseOptions* pOptions) { @@ -1569,6 +1771,8 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) { pOptions->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; pOptions->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO; return (SNode*)pOptions; +_err: + return NULL; } SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) { @@ -1610,6 +1814,8 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) { pOptions->withArbitrator = -1; pOptions->encryptAlgorithm = -1; return (SNode*)pOptions; +_err: + return NULL; } static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, void* pVal, @@ -1751,6 +1957,9 @@ static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, ED break; } return pOptions; +_err: + nodesDestroyNode(pOptions); + return NULL; } SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, void* pVal) { @@ -1767,13 +1976,15 @@ SNode* setAlterDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOp break; } return setDatabaseOptionImpl(pCxt, pOptions, pAlterOption->type, &pAlterOption->val, true); +_err: + nodesDestroyNode(pOptions); + nodesDestroyList(pAlterOption->pList); + return NULL; } SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pDbName, SNode* pOptions) { CHECK_PARSER_STATUS(pCxt); - if (!checkDbName(pCxt, pDbName, false)) { - return NULL; - } + CHECK_NAME(checkDbName(pCxt, pDbName, false)); SCreateDatabaseStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_CREATE_DATABASE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); @@ -1781,76 +1992,78 @@ SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, STok pStmt->ignoreExists = ignoreExists; pStmt->pOptions = (SDatabaseOptions*)pOptions; return (SNode*)pStmt; +_err: + nodesDestroyNode(pOptions); + return NULL; } SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pDbName) { CHECK_PARSER_STATUS(pCxt); - if (!checkDbName(pCxt, pDbName, false)) { - return NULL; - } + CHECK_NAME(checkDbName(pCxt, pDbName, false)); SDropDatabaseStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_DROP_DATABASE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName); pStmt->ignoreNotExists = ignoreNotExists; return (SNode*)pStmt; +_err: + return NULL; } SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pOptions) { CHECK_PARSER_STATUS(pCxt); - if (!checkDbName(pCxt, pDbName, false)) { - return NULL; - } + CHECK_NAME(checkDbName(pCxt, pDbName, false)); SAlterDatabaseStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_ALTER_DATABASE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName); pStmt->pOptions = (SDatabaseOptions*)pOptions; return (SNode*)pStmt; +_err: + nodesDestroyNode(pOptions); + return NULL; } SNode* createFlushDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName) { CHECK_PARSER_STATUS(pCxt); - if (!checkDbName(pCxt, pDbName, false)) { - return NULL; - } + CHECK_NAME(checkDbName(pCxt, pDbName, false)); SFlushDatabaseStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_FLUSH_DATABASE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName); return (SNode*)pStmt; +_err: + return NULL; } SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, int32_t maxSpeed) { CHECK_PARSER_STATUS(pCxt); - if (!checkDbName(pCxt, pDbName, false)) { - return NULL; - } + CHECK_NAME(checkDbName(pCxt, pDbName, false)); STrimDatabaseStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_TRIM_DATABASE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName); pStmt->maxSpeed = maxSpeed; return (SNode*)pStmt; +_err: + return NULL; } SNode* createS3MigrateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName) { CHECK_PARSER_STATUS(pCxt); - if (!checkDbName(pCxt, pDbName, false)) { - return NULL; - } + CHECK_NAME(checkDbName(pCxt, pDbName, false)); SS3MigrateDatabaseStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_S3MIGRATE_DATABASE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName); return (SNode*)pStmt; +_err: + return NULL; } SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pStart, SNode* pEnd) { CHECK_PARSER_STATUS(pCxt); - if (!checkDbName(pCxt, pDbName, false)) { - return NULL; - } + CHECK_NAME(checkDbName(pCxt, pDbName, false)); SCompactDatabaseStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_COMPACT_DATABASE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); @@ -1858,6 +2071,10 @@ SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pStart pStmt->pStart = pStart; pStmt->pEnd = pEnd; return (SNode*)pStmt; +_err: + nodesDestroyNode(pStart); + nodesDestroyNode(pEnd); + return NULL; } SNode* createDefaultTableOptions(SAstCreateContext* pCxt) { @@ -1872,6 +2089,8 @@ SNode* createDefaultTableOptions(SAstCreateContext* pCxt) { pOptions->ttl = TSDB_DEFAULT_TABLE_TTL; pOptions->commentNull = true; // mark null return (SNode*)pOptions; +_err: + return NULL; } SNode* createAlterTableOptions(SAstCreateContext* pCxt) { @@ -1882,6 +2101,8 @@ SNode* createAlterTableOptions(SAstCreateContext* pCxt) { pOptions->ttl = -1; pOptions->commentNull = true; // mark null return (SNode*)pOptions; +_err: + return NULL; } SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, void* pVal) { @@ -1922,6 +2143,9 @@ SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType break; } return pOptions; +_err: + nodesDestroyNode(pOptions); + return NULL; } SNode* createDefaultColumnOptions(SAstCreateContext* pCxt) { @@ -1932,6 +2156,8 @@ SNode* createDefaultColumnOptions(SAstCreateContext* pCxt) { pOptions->commentNull = true; pOptions->bPrimaryKey = false; return (SNode*)pOptions; +_err: + return NULL; } SNode* setColumnOptions(SAstCreateContext* pCxt, SNode* pOptions, EColumnOptionType type, void* pVal) { @@ -1965,16 +2191,17 @@ SNode* setColumnOptions(SAstCreateContext* pCxt, SNode* pOptions, EColumnOptionT break; } return pOptions; +_err: + nodesDestroyNode(pOptions); + return NULL; } SNode* createColumnDefNode(SAstCreateContext* pCxt, SToken* pColName, SDataType dataType, SNode* pNode) { CHECK_PARSER_STATUS(pCxt); - if (!checkColumnName(pCxt, pColName)) { - return NULL; - } + CHECK_NAME(checkColumnName(pCxt, pColName)); if (IS_VAR_DATA_TYPE(dataType.type) && dataType.bytes == 0) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); - return NULL; + CHECK_PARSER_STATUS(pCxt); } SColumnDefNode* pCol = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_COLUMN_DEF, (SNode**)&pCol); @@ -1984,6 +2211,9 @@ SNode* createColumnDefNode(SAstCreateContext* pCxt, SToken* pColName, SDataType pCol->pOptions = pNode; pCol->sma = true; return (SNode*)pCol; +_err: + nodesDestroyNode(pNode); + return NULL; } SDataType createDataType(uint8_t type) { @@ -2013,6 +2243,12 @@ SNode* createCreateTableStmt(SAstCreateContext* pCxt, bool ignoreExists, SNode* pStmt->pOptions = (STableOptions*)pOptions; nodesDestroyNode(pRealTable); return (SNode*)pStmt; +_err: + nodesDestroyNode(pRealTable); + nodesDestroyList(pCols); + nodesDestroyList(pTags); + nodesDestroyNode(pOptions); + return NULL; } SNode* createCreateSubTableClause(SAstCreateContext* pCxt, bool ignoreExists, SNode* pRealTable, SNode* pUseRealTable, @@ -2032,6 +2268,13 @@ SNode* createCreateSubTableClause(SAstCreateContext* pCxt, bool ignoreExists, SN nodesDestroyNode(pRealTable); nodesDestroyNode(pUseRealTable); return (SNode*)pStmt; +_err: + nodesDestroyNode(pRealTable); + nodesDestroyNode(pOptions); + nodesDestroyNode(pUseRealTable); + nodesDestroyList(pSpecificTags); + nodesDestroyList(pValsOfTags); + return NULL; } SNode* createCreateSubTableFromFileClause(SAstCreateContext* pCxt, bool ignoreExists, SNode* pUseRealTable, @@ -2052,6 +2295,10 @@ SNode* createCreateSubTableFromFileClause(SAstCreateContext* pCxt, bool ignoreEx nodesDestroyNode(pUseRealTable); return (SNode*)pStmt; +_err: + nodesDestroyNode(pUseRealTable); + nodesDestroyList(pSpecificTags); + return NULL; } SNode* createCreateMultiTableStmt(SAstCreateContext* pCxt, SNodeList* pSubTables) { @@ -2061,6 +2308,9 @@ SNode* createCreateMultiTableStmt(SAstCreateContext* pCxt, SNodeList* pSubTables CHECK_MAKE_NODE(pStmt); pStmt->pSubTables = pSubTables; return (SNode*)pStmt; +_err: + nodesDestroyList(pSubTables); + return NULL; } SNode* createDropTableClause(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable) { @@ -2073,6 +2323,9 @@ SNode* createDropTableClause(SAstCreateContext* pCxt, bool ignoreNotExists, SNod pStmt->ignoreNotExists = ignoreNotExists; nodesDestroyNode(pRealTable); return (SNode*)pStmt; +_err: + nodesDestroyNode(pRealTable); + return NULL; } SNode* createDropTableStmt(SAstCreateContext* pCxt, SNodeList* pTables) { @@ -2082,6 +2335,9 @@ SNode* createDropTableStmt(SAstCreateContext* pCxt, SNodeList* pTables) { CHECK_MAKE_NODE(pStmt); pStmt->pTables = pTables; return (SNode*)pStmt; +_err: + nodesDestroyList(pTables); + return NULL; } SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable) { @@ -2094,6 +2350,9 @@ SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool ignoreNotExists, S pStmt->ignoreNotExists = ignoreNotExists; nodesDestroyNode(pRealTable); return (SNode*)pStmt; +_err: + nodesDestroyNode(pRealTable); + return NULL; } static SNode* createAlterTableStmtFinalize(SNode* pRealTable, SAlterTableStmt* pStmt) { @@ -2111,14 +2370,16 @@ SNode* createAlterTableModifyOptions(SAstCreateContext* pCxt, SNode* pRealTable, pStmt->alterType = TSDB_ALTER_TABLE_UPDATE_OPTIONS; pStmt->pOptions = (STableOptions*)pOptions; return createAlterTableStmtFinalize(pRealTable, pStmt); +_err: + nodesDestroyNode(pRealTable); + nodesDestroyNode(pOptions); + return NULL; } SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, SToken* pColName, SDataType dataType) { CHECK_PARSER_STATUS(pCxt); - if (!checkColumnName(pCxt, pColName)) { - return NULL; - } + CHECK_NAME(checkColumnName(pCxt, pColName)); SAlterTableStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); @@ -2126,15 +2387,15 @@ SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable, COPY_STRING_FORM_ID_TOKEN(pStmt->colName, pColName); pStmt->dataType = dataType; return createAlterTableStmtFinalize(pRealTable, pStmt); +_err: + nodesDestroyNode(pRealTable); + return NULL; } SNode* createAlterTableAddModifyColOptions2(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, SToken* pColName, SDataType dataType, SNode* pOptions) { - CHECK_PARSER_STATUS(pCxt); - if (!checkColumnName(pCxt, pColName)) { - return NULL; - } - SAlterTableStmt* pStmt = NULL; + CHECK_PARSER_STATUS(pCxt); + CHECK_NAME(checkColumnName(pCxt, pColName)); pCxt->errCode = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); pStmt->alterType = alterType; @@ -2155,19 +2416,21 @@ SNode* createAlterTableAddModifyColOptions2(SAstCreateContext* pCxt, SNode* pRea } else { pCxt->errCode = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "not support alter column with option except compress"); - nodesDestroyNode((SNode*)pStmt); - return NULL; + CHECK_PARSER_STATUS(pCxt); } } return createAlterTableStmtFinalize(pRealTable, pStmt); +_err: + nodesDestroyNode(pOptions); + nodesDestroyNode((SNode*)pStmt); + nodesDestroyNode(pRealTable); + return NULL; } SNode* createAlterTableAddModifyColOptions(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, SToken* pColName, SNode* pOptions) { CHECK_PARSER_STATUS(pCxt); - if (!checkColumnName(pCxt, pColName)) { - return NULL; - } + CHECK_NAME(checkColumnName(pCxt, pColName)); SAlterTableStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); @@ -2175,27 +2438,31 @@ SNode* createAlterTableAddModifyColOptions(SAstCreateContext* pCxt, SNode* pReal COPY_STRING_FORM_ID_TOKEN(pStmt->colName, pColName); pStmt->pColOptions = (SColumnOptions*)pOptions; return createAlterTableStmtFinalize(pRealTable, pStmt); +_err: + nodesDestroyNode(pOptions); + nodesDestroyNode(pRealTable); + return NULL; } SNode* createAlterTableDropCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, SToken* pColName) { CHECK_PARSER_STATUS(pCxt); - if (!checkColumnName(pCxt, pColName)) { - return NULL; - } + CHECK_NAME(checkColumnName(pCxt, pColName)); SAlterTableStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); pStmt->alterType = alterType; COPY_STRING_FORM_ID_TOKEN(pStmt->colName, pColName); return createAlterTableStmtFinalize(pRealTable, pStmt); +_err: + nodesDestroyNode(pRealTable); + return NULL; } SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, SToken* pOldColName, SToken* pNewColName) { CHECK_PARSER_STATUS(pCxt); - if (!checkColumnName(pCxt, pOldColName) || !checkColumnName(pCxt, pNewColName)) { - return NULL; - } + CHECK_NAME(checkColumnName(pCxt, pOldColName)); + CHECK_NAME(checkColumnName(pCxt, pNewColName)); SAlterTableStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); @@ -2203,14 +2470,14 @@ SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int COPY_STRING_FORM_ID_TOKEN(pStmt->colName, pOldColName); COPY_STRING_FORM_ID_TOKEN(pStmt->newColName, pNewColName); return createAlterTableStmtFinalize(pRealTable, pStmt); +_err: + nodesDestroyNode(pRealTable); + return NULL; } SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, SToken* pTagName, SNode* pVal) { CHECK_PARSER_STATUS(pCxt); - if (!checkColumnName(pCxt, pTagName)) { - nodesDestroyNode(pVal); - return NULL; - } + CHECK_NAME(checkColumnName(pCxt, pTagName)); SAlterTableStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); @@ -2218,6 +2485,10 @@ SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, SToken COPY_STRING_FORM_ID_TOKEN(pStmt->colName, pTagName); pStmt->pVal = (SValueNode*)pVal; return createAlterTableStmtFinalize(pRealTable, pStmt); +_err: + nodesDestroyNode(pVal); + nodesDestroyNode(pRealTable); + return NULL; } SNode* setAlterSuperTableType(SNode* pStmt) { @@ -2228,14 +2499,14 @@ SNode* setAlterSuperTableType(SNode* pStmt) { SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName) { CHECK_PARSER_STATUS(pCxt); - if (!checkDbName(pCxt, pDbName, false)) { - return NULL; - } + CHECK_NAME(checkDbName(pCxt, pDbName, false)); SUseDatabaseStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_USE_DATABASE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName); return (SNode*)pStmt; +_err: + return NULL; } static bool needDbShowStmt(ENodeType type) { @@ -2252,6 +2523,8 @@ SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type) { CHECK_MAKE_NODE(pStmt); pStmt->withFull = false; return (SNode*)pStmt; +_err: + return NULL; } SNode* createShowStmtWithFull(SAstCreateContext* pCxt, ENodeType type) { @@ -2261,6 +2534,8 @@ SNode* createShowStmtWithFull(SAstCreateContext* pCxt, ENodeType type) { CHECK_MAKE_NODE(pStmt); pStmt->withFull = true; return (SNode*)pStmt; +_err: + return NULL; } SNode* createShowCompactsStmt(SAstCreateContext* pCxt, ENodeType type) { @@ -2269,6 +2544,8 @@ SNode* createShowCompactsStmt(SAstCreateContext* pCxt, ENodeType type) { pCxt->errCode = nodesMakeNode(type, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); return (SNode*)pStmt; +_err: + return NULL; } SNode* setShowKind(SAstCreateContext* pCxt, SNode* pStmt, EShowKind showKind) { @@ -2286,7 +2563,7 @@ SNode* createShowStmtWithCond(SAstCreateContext* pCxt, ENodeType type, SNode* pD if (needDbShowStmt(type) && NULL == pDbName) { snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "database not specified"); pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; - return NULL; + CHECK_PARSER_STATUS(pCxt); } SShowStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(type, (SNode**)&pStmt); @@ -2295,6 +2572,10 @@ SNode* createShowStmtWithCond(SAstCreateContext* pCxt, ENodeType type, SNode* pD pStmt->pTbName = pTbName; pStmt->tableCondType = tableCondType; return (SNode*)pStmt; +_err: + nodesDestroyNode(pDbName); + nodesDestroyNode(pTbName); + return NULL; } SNode* createShowTablesStmt(SAstCreateContext* pCxt, SShowTablesOption option, SNode* pTbName, @@ -2307,20 +2588,24 @@ SNode* createShowTablesStmt(SAstCreateContext* pCxt, SShowTablesOption option, S pDbName = createIdentifierValueNode(pCxt, &option.dbName); } SNode* pStmt = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, pDbName, pTbName, tableCondType); + CHECK_PARSER_STATUS(pCxt); (void)setShowKind(pCxt, pStmt, option.kind); return pStmt; +_err: + nodesDestroyNode(pTbName); + return NULL; } SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName) { CHECK_PARSER_STATUS(pCxt); - if (!checkDbName(pCxt, pDbName, true)) { - return NULL; - } + CHECK_NAME(checkDbName(pCxt, pDbName, true)); SShowCreateDatabaseStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_SHOW_CREATE_DATABASE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName); return (SNode*)pStmt; +_err: + return NULL; } SNode* createShowAliveStmt(SAstCreateContext* pCxt, SNode* pNode, ENodeType type) { @@ -2337,19 +2622,13 @@ SNode* createShowAliveStmt(SAstCreateContext* pCxt, SNode* pNode, ENodeType type } } - if (pDbToken && !checkDbName(pCxt, pDbToken, true)) { - nodesDestroyNode(pNode); - return NULL; + if (pDbToken) { + CHECK_NAME(checkDbName(pCxt, pDbToken, true)); } SShowAliveStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(type, (SNode**)&pStmt); - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - if (pNode) { - nodesDestroyNode(pNode); - } - return NULL; - } + CHECK_PARSER_STATUS(pCxt); if (pDbToken) { COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbToken); @@ -2359,6 +2638,9 @@ SNode* createShowAliveStmt(SAstCreateContext* pCxt, SNode* pNode, ENodeType type } return (SNode*)pStmt; +_err: + nodesDestroyNode(pNode); + return NULL; } SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable) { @@ -2370,6 +2652,9 @@ SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); nodesDestroyNode(pRealTable); return (SNode*)pStmt; +_err: + nodesDestroyNode(pRealTable); + return NULL; } SNode* createShowCreateViewStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable) { @@ -2381,6 +2666,9 @@ SNode* createShowCreateViewStmt(SAstCreateContext* pCxt, ENodeType type, SNode* strcpy(pStmt->viewName, ((SRealTableNode*)pRealTable)->table.tableName); nodesDestroyNode(pRealTable); return (SNode*)pStmt; +_err: + nodesDestroyNode(pRealTable); + return NULL; } SNode* createShowTableDistributedStmt(SAstCreateContext* pCxt, SNode* pRealTable) { @@ -2392,6 +2680,9 @@ SNode* createShowTableDistributedStmt(SAstCreateContext* pCxt, SNode* pRealTable strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); nodesDestroyNode(pRealTable); return (SNode*)pStmt; +_err: + nodesDestroyNode(pRealTable); + return NULL; } SNode* createShowDnodeVariablesStmt(SAstCreateContext* pCxt, SNode* pDnodeId, SNode* pLikePattern) { @@ -2402,6 +2693,10 @@ SNode* createShowDnodeVariablesStmt(SAstCreateContext* pCxt, SNode* pDnodeId, SN pStmt->pDnodeId = pDnodeId; pStmt->pLikePattern = pLikePattern; return (SNode*)pStmt; +_err: + nodesDestroyNode(pDnodeId); + nodesDestroyNode(pLikePattern); + return NULL; } SNode* createShowVnodesStmt(SAstCreateContext* pCxt, SNode* pDnodeId, SNode* pDnodeEndpoint) { @@ -2412,6 +2707,10 @@ SNode* createShowVnodesStmt(SAstCreateContext* pCxt, SNode* pDnodeId, SNode* pDn pStmt->pDnodeId = pDnodeId; pStmt->pDnodeEndpoint = pDnodeEndpoint; return (SNode*)pStmt; +_err: + nodesDestroyNode(pDnodeId); + nodesDestroyNode(pDnodeEndpoint); + return NULL; } SNode* createShowTableTagsStmt(SAstCreateContext* pCxt, SNode* pTbName, SNode* pDbName, SNodeList* pTags) { @@ -2419,7 +2718,7 @@ SNode* createShowTableTagsStmt(SAstCreateContext* pCxt, SNode* pTbName, SNode* p if (NULL == pDbName) { snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "database not specified"); pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; - return NULL; + CHECK_PARSER_STATUS(pCxt); } SShowTableTagsStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_SHOW_TABLE_TAGS_STMT, (SNode**)&pStmt); @@ -2428,6 +2727,11 @@ SNode* createShowTableTagsStmt(SAstCreateContext* pCxt, SNode* pTbName, SNode* p pStmt->pTbName = pTbName; pStmt->pTags = pTags; return (SNode*)pStmt; +_err: + nodesDestroyNode(pTbName); + nodesDestroyNode(pDbName); + nodesDestroyList(pTags); + return NULL; } SNode* createShowCompactDetailsStmt(SAstCreateContext* pCxt, SNode* pCompactId) { @@ -2437,6 +2741,9 @@ SNode* createShowCompactDetailsStmt(SAstCreateContext* pCxt, SNode* pCompactId) CHECK_MAKE_NODE(pStmt); pStmt->pCompactId = pCompactId; return (SNode*)pStmt; +_err: + nodesDestroyNode(pCompactId); + return NULL; } static int32_t getIpV4RangeFromWhitelistItem(char* ipRange, SIpV4Range* pIpRange) { @@ -2503,27 +2810,24 @@ SNode* addCreateUserStmtWhiteList(SAstCreateContext* pCxt, SNode* pCreateUserStm SCreateUserStmt* pCreateUser = (SCreateUserStmt*)pCreateUserStmt; pCreateUser->numIpRanges = LIST_LENGTH(pIpRangesNodeList); pCreateUser->pIpRanges = taosMemoryMalloc(pCreateUser->numIpRanges * sizeof(SIpV4Range)); - if (NULL == pCreateUser->pIpRanges) { - pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; - nodesDestroyNode(pCreateUserStmt); - return NULL; - } + CHECK_OUT_OF_MEM(pCreateUser->pIpRanges); + + pCxt->errCode = fillIpRangesFromWhiteList(pCxt, pIpRangesNodeList, pCreateUser->pIpRanges); + CHECK_PARSER_STATUS(pCxt); - int32_t code = fillIpRangesFromWhiteList(pCxt, pIpRangesNodeList, pCreateUser->pIpRanges); - if (TSDB_CODE_SUCCESS != code) { - nodesDestroyNode(pCreateUserStmt); - return NULL; - } return pCreateUserStmt; +_err: + nodesDestroyNode(pCreateUserStmt); + nodesDestroyList(pIpRangesNodeList); + return NULL; } SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword, int8_t sysinfo, int8_t createDb, int8_t is_import) { CHECK_PARSER_STATUS(pCxt); char password[TSDB_USET_PASSWORD_LEN + 3] = {0}; - if (!checkUserName(pCxt, pUserName) || !checkPassword(pCxt, pPassword, password)) { - return NULL; - } + CHECK_NAME(checkUserName(pCxt, pUserName)); + CHECK_NAME(checkPassword(pCxt, pPassword, password)); SCreateUserStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_CREATE_USER_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); @@ -2533,14 +2837,14 @@ SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const ST pStmt->createDb = createDb; pStmt->isImport = is_import; return (SNode*)pStmt; +_err: + return NULL; } SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t alterType, void* pAlterInfo) { - CHECK_PARSER_STATUS(pCxt); - if (!checkUserName(pCxt, pUserName)) { - return NULL; - } SAlterUserStmt* pStmt = NULL; + CHECK_PARSER_STATUS(pCxt); + CHECK_NAME(checkUserName(pCxt, pUserName)); pCxt->errCode = nodesMakeNode(QUERY_NODE_ALTER_USER_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName); @@ -2549,10 +2853,7 @@ SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t al case TSDB_ALTER_USER_PASSWD: { char password[TSDB_USET_PASSWORD_LEN] = {0}; SToken* pVal = pAlterInfo; - if (!checkPassword(pCxt, pVal, password)) { - nodesDestroyNode((SNode*)pStmt); - return NULL; - } + CHECK_NAME(checkPassword(pCxt, pVal, password)); strcpy(pStmt->password, password); break; } @@ -2577,35 +2878,31 @@ SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t al pStmt->pNodeListIpRanges = pIpRangesNodeList; pStmt->numIpRanges = LIST_LENGTH(pIpRangesNodeList); pStmt->pIpRanges = taosMemoryMalloc(pStmt->numIpRanges * sizeof(SIpV4Range)); - if (NULL == pStmt->pIpRanges) { - pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; - nodesDestroyNode((SNode*)pStmt); - return NULL; - } + CHECK_OUT_OF_MEM(pStmt->pIpRanges); - int32_t code = fillIpRangesFromWhiteList(pCxt, pIpRangesNodeList, pStmt->pIpRanges); - if (TSDB_CODE_SUCCESS != code) { - nodesDestroyNode((SNode*)pStmt); - return NULL; - } + pCxt->errCode = fillIpRangesFromWhiteList(pCxt, pIpRangesNodeList, pStmt->pIpRanges); + CHECK_PARSER_STATUS(pCxt); break; } default: break; } return (SNode*)pStmt; +_err: + nodesDestroyNode((SNode*)pStmt); + return NULL; } SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName) { CHECK_PARSER_STATUS(pCxt); - if (!checkUserName(pCxt, pUserName)) { - return NULL; - } + CHECK_NAME(checkUserName(pCxt, pUserName)); SDropUserStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_DROP_USER_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName); return (SNode*)pStmt; +_err: + return NULL; } SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort) { @@ -2618,6 +2915,8 @@ SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const return NULL; } return (SNode*)pStmt; +_err: + return NULL; } SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, bool force, bool unsafe) { @@ -2636,6 +2935,8 @@ SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, bool f pStmt->force = force; pStmt->unsafe = unsafe; return (SNode*)pStmt; +_err: + return NULL; } SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, @@ -2654,6 +2955,8 @@ SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const (void)trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value)); } return (SNode*)pStmt; +_err: + return NULL; } SNode* createEncryptKeyStmt(SAstCreateContext* pCxt, const SToken* pValue) { @@ -2685,10 +2988,7 @@ SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool igno // no db specified, if (pCxt->pQueryCxt->db == NULL) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DB_NOT_SPECIFIED); - nodesDestroyNode(pIndexName); - nodesDestroyNode(pRealTable); - nodesDestroyNode(pOptions); - return NULL; + CHECK_PARSER_STATUS(pCxt); } else { snprintf(pStmt->indexDbName, sizeof(pStmt->indexDbName), "%s", pCxt->pQueryCxt->db); } @@ -2703,6 +3003,12 @@ SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool igno pStmt->pCols = pCols; pStmt->pOptions = (SIndexOptions*)pOptions; return (SNode*)pStmt; +_err: + nodesDestroyNode(pIndexName); + nodesDestroyNode(pRealTable); + nodesDestroyNode(pOptions); + nodesDestroyList(pCols); + return NULL; } SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInterval, SNode* pOffset, SNode* pSliding, @@ -2717,6 +3023,12 @@ SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInt pOptions->pSliding = pSliding; pOptions->pStreamOptions = pStreamOptions; return (SNode*)pOptions; +_err: + nodesDestroyNode(pInterval); + nodesDestroyNode(pOffset); + nodesDestroyNode(pSliding); + nodesDestroyNode(pStreamOptions); + return NULL; } SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pIndexName) { @@ -2729,6 +3041,9 @@ SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* snprintf(pStmt->indexName, sizeof(pStmt->indexName), "%s", ((SRealTableNode*)pIndexName)->table.tableName); nodesDestroyNode(pIndexName); return (SNode*)pStmt; +_err: + nodesDestroyNode(pIndexName); + return NULL; } SNode* createCreateComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId) { @@ -2738,6 +3053,8 @@ SNode* createCreateComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, co CHECK_MAKE_NODE(pStmt); pStmt->dnodeId = taosStr2Int32(pDnodeId->z, NULL, 10); return (SNode*)pStmt; +_err: + return NULL; } SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId) { @@ -2747,6 +3064,8 @@ SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, cons CHECK_MAKE_NODE(pStmt); pStmt->dnodeId = taosStr2Int32(pDnodeId->z, NULL, 10); return (SNode*)pStmt; +_err: + return NULL; } SNode* createRestoreComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId) { @@ -2756,14 +3075,13 @@ SNode* createRestoreComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, c CHECK_MAKE_NODE(pStmt); pStmt->dnodeId = taosStr2Int32(pDnodeId->z, NULL, 10); return (SNode*)pStmt; +_err: + return NULL; } SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pQuery) { CHECK_PARSER_STATUS(pCxt); - if (!checkTopicName(pCxt, pTopicName)) { - nodesDestroyNode(pQuery); - return NULL; - } + CHECK_NAME(checkTopicName(pCxt, pTopicName)); SCreateTopicStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); @@ -2771,14 +3089,16 @@ SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, pStmt->ignoreExists = ignoreExists; pStmt->pQuery = pQuery; return (SNode*)pStmt; +_err: + nodesDestroyNode(pQuery); + return NULL; } SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SToken* pSubDbName, int8_t withMeta) { CHECK_PARSER_STATUS(pCxt); - if (!checkTopicName(pCxt, pTopicName) || !checkDbName(pCxt, pSubDbName, true)) { - return NULL; - } + CHECK_NAME(checkTopicName(pCxt, pTopicName)); + CHECK_NAME(checkDbName(pCxt, pSubDbName, true)); SCreateTopicStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); @@ -2787,14 +3107,14 @@ SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, ST COPY_STRING_FORM_ID_TOKEN(pStmt->subDbName, pSubDbName); pStmt->withMeta = withMeta; return (SNode*)pStmt; +_err: + return NULL; } SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pRealTable, int8_t withMeta, SNode* pWhere) { CHECK_PARSER_STATUS(pCxt); - if (!checkTopicName(pCxt, pTopicName)) { - return NULL; - } + CHECK_NAME(checkTopicName(pCxt, pTopicName)); SCreateTopicStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); @@ -2807,29 +3127,29 @@ SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, strcpy(pStmt->subSTbName, ((SRealTableNode*)pRealTable)->table.tableName); nodesDestroyNode(pRealTable); return (SNode*)pStmt; +_err: + nodesDestroyNode(pRealTable); + nodesDestroyNode(pWhere); + return NULL; } SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pTopicName) { CHECK_PARSER_STATUS(pCxt); - if (!checkTopicName(pCxt, pTopicName)) { - return NULL; - } + CHECK_NAME(checkTopicName(pCxt, pTopicName)); SDropTopicStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_DROP_TOPIC_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName); pStmt->ignoreNotExists = ignoreNotExists; return (SNode*)pStmt; +_err: + return NULL; } SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pCGroupId, SToken* pTopicName) { CHECK_PARSER_STATUS(pCxt); - if (!checkTopicName(pCxt, pTopicName)) { - return NULL; - } - if (!checkCGroupName(pCxt, pCGroupId)) { - return NULL; - } + CHECK_NAME(checkTopicName(pCxt, pTopicName)); + CHECK_NAME(checkCGroupName(pCxt, pCGroupId)); SDropCGroupStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_DROP_CGROUP_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); @@ -2837,6 +3157,8 @@ SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToke COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName); COPY_STRING_FORM_ID_TOKEN(pStmt->cgroup, pCGroupId); return (SNode*)pStmt; +_err: + return NULL; } SNode* createAlterClusterStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue) { @@ -2849,6 +3171,8 @@ SNode* createAlterClusterStmt(SAstCreateContext* pCxt, const SToken* pConfig, co (void)trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value)); } return (SNode*)pStmt; +_err: + return NULL; } SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue) { @@ -2861,6 +3185,8 @@ SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, cons (void)trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value)); } return (SNode*)pStmt; +_err: + return NULL; } SNode* createDefaultExplainOptions(SAstCreateContext* pCxt) { @@ -2871,18 +3197,24 @@ SNode* createDefaultExplainOptions(SAstCreateContext* pCxt) { pOptions->verbose = TSDB_DEFAULT_EXPLAIN_VERBOSE; pOptions->ratio = TSDB_DEFAULT_EXPLAIN_RATIO; return (SNode*)pOptions; +_err: + return NULL; } SNode* setExplainVerbose(SAstCreateContext* pCxt, SNode* pOptions, const SToken* pVal) { CHECK_PARSER_STATUS(pCxt); ((SExplainOptions*)pOptions)->verbose = (0 == strncasecmp(pVal->z, "true", pVal->n)); return pOptions; +_err: + return NULL; } SNode* setExplainRatio(SAstCreateContext* pCxt, SNode* pOptions, const SToken* pVal) { CHECK_PARSER_STATUS(pCxt); ((SExplainOptions*)pOptions)->ratio = taosStr2Double(pVal->z, NULL); return pOptions; +_err: + return NULL; } SNode* createExplainStmt(SAstCreateContext* pCxt, bool analyze, SNode* pOptions, SNode* pQuery) { @@ -2894,6 +3226,10 @@ SNode* createExplainStmt(SAstCreateContext* pCxt, bool analyze, SNode* pOptions, pStmt->pOptions = (SExplainOptions*)pOptions; pStmt->pQuery = pQuery; return (SNode*)pStmt; +_err: + nodesDestroyNode(pOptions); + nodesDestroyNode(pQuery); + return NULL; } SNode* createDescribeStmt(SAstCreateContext* pCxt, SNode* pRealTable) { @@ -2905,6 +3241,9 @@ SNode* createDescribeStmt(SAstCreateContext* pCxt, SNode* pRealTable) { strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); nodesDestroyNode(pRealTable); return (SNode*)pStmt; +_err: + nodesDestroyNode(pRealTable); + return NULL; } SNode* createResetQueryCacheStmt(SAstCreateContext* pCxt) { @@ -2913,6 +3252,8 @@ SNode* createResetQueryCacheStmt(SAstCreateContext* pCxt) { pCxt->errCode = nodesMakeNode(QUERY_NODE_RESET_QUERY_CACHE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); return pStmt; +_err: + return NULL; } static int32_t convertUdfLanguageType(SAstCreateContext* pCxt, const SToken* pLanguageToken, int8_t* pLanguage) { @@ -2933,12 +3274,11 @@ SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool CHECK_PARSER_STATUS(pCxt); if (pLibPath->n <= 2) { pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; - return NULL; + CHECK_PARSER_STATUS(pCxt); } int8_t language = 0; - if (TSDB_CODE_SUCCESS != convertUdfLanguageType(pCxt, pLanguage, &language)) { - return NULL; - } + pCxt->errCode = convertUdfLanguageType(pCxt, pLanguage, &language); + CHECK_PARSER_STATUS(pCxt); SCreateFunctionStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_CREATE_FUNCTION_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); @@ -2951,6 +3291,8 @@ SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool pStmt->bufSize = bufSize; pStmt->language = language; return (SNode*)pStmt; +_err: + return NULL; } SNode* createDropFunctionStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pFuncName) { @@ -2961,11 +3303,13 @@ SNode* createDropFunctionStmt(SAstCreateContext* pCxt, bool ignoreNotExists, con pStmt->ignoreNotExists = ignoreNotExists; COPY_STRING_FORM_ID_TOKEN(pStmt->funcName, pFuncName); return (SNode*)pStmt; +_err: + return NULL; } SNode* createCreateViewStmt(SAstCreateContext* pCxt, bool orReplace, SNode* pView, const SToken* pAs, SNode* pQuery) { - CHECK_PARSER_STATUS(pCxt); SCreateViewStmt* pStmt = NULL; + CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_CREATE_VIEW_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); int32_t i = pAs->n; @@ -2980,6 +3324,11 @@ SNode* createCreateViewStmt(SAstCreateContext* pCxt, bool orReplace, SNode* pVie pStmt->orReplace = orReplace; pStmt->pQuery = pQuery; return (SNode*)pStmt; +_err: + nodesDestroyNode(pView); + nodesDestroyNode(pQuery); + nodesDestroyNode((SNode*)pStmt); + return NULL; } SNode* createDropViewStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pView) { @@ -2992,6 +3341,9 @@ SNode* createDropViewStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* strcpy(pStmt->viewName, ((SViewNode*)pView)->table.tableName); nodesDestroyNode(pView); return (SNode*)pStmt; +_err: + nodesDestroyNode(pView); + return NULL; } SNode* createStreamOptions(SAstCreateContext* pCxt) { @@ -3004,6 +3356,8 @@ SNode* createStreamOptions(SAstCreateContext* pCxt) { pOptions->ignoreExpired = STREAM_DEFAULT_IGNORE_EXPIRED; pOptions->ignoreUpdate = STREAM_DEFAULT_IGNORE_UPDATE; return (SNode*)pOptions; +_err: + return NULL; } static int8_t getTriggerType(uint32_t tokenType) { @@ -3064,11 +3418,7 @@ SNode* setStreamOptions(SAstCreateContext* pCxt, SNode* pOptions, EStreamOptions SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pStreamName, SNode* pRealTable, SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery, SNodeList* pCols) { CHECK_PARSER_STATUS(pCxt); - if (!checkStreamName(pCxt, pStreamName)) { - nodesDestroyNode(pQuery); - nodesDestroyNode(pOptions); - return NULL; - } + CHECK_NAME(checkStreamName(pCxt, pStreamName)); SCreateStreamStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_CREATE_STREAM_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); @@ -3083,40 +3433,46 @@ SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken pStmt->pSubtable = pSubtable; pStmt->pCols = pCols; return (SNode*)pStmt; +_err: + nodesDestroyNode(pRealTable); + nodesDestroyNode(pQuery); + nodesDestroyNode(pOptions); + nodesDestroyList(pTags); + nodesDestroyNode(pSubtable); + nodesDestroyList(pCols); + return NULL; } SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pStreamName) { CHECK_PARSER_STATUS(pCxt); - if (!checkStreamName(pCxt, pStreamName)) { - return NULL; - } + CHECK_NAME(checkStreamName(pCxt, pStreamName)); SDropStreamStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_DROP_STREAM_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->streamName, pStreamName); pStmt->ignoreNotExists = ignoreNotExists; return (SNode*)pStmt; +_err: + return NULL; } SNode* createPauseStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pStreamName) { CHECK_PARSER_STATUS(pCxt); - if (!checkStreamName(pCxt, pStreamName)) { - return NULL; - } + CHECK_NAME(checkStreamName(pCxt, pStreamName)); SPauseStreamStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_PAUSE_STREAM_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->streamName, pStreamName); pStmt->ignoreNotExists = ignoreNotExists; return (SNode*)pStmt; +_err: + return NULL; } SNode* createResumeStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, bool ignoreUntreated, SToken* pStreamName) { CHECK_PARSER_STATUS(pCxt); - if (!checkStreamName(pCxt, pStreamName)) { - return NULL; - } + CHECK_NAME(checkStreamName(pCxt, pStreamName)); SResumeStreamStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_RESUME_STREAM_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); @@ -3124,6 +3480,8 @@ SNode* createResumeStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, boo pStmt->ignoreNotExists = ignoreNotExists; pStmt->ignoreUntreated = ignoreUntreated; return (SNode*)pStmt; +_err: + return NULL; } SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId) { @@ -3133,6 +3491,8 @@ SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId CHECK_MAKE_NODE(pStmt); pStmt->targetId = taosStr2Int32(pId->z, NULL, 10); return (SNode*)pStmt; +_err: + return NULL; } SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId) { @@ -3142,6 +3502,8 @@ SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId) { CHECK_MAKE_NODE(pStmt); (void)trimString(pQueryId->z, pQueryId->n, pStmt->queryId, sizeof(pStmt->queryId) - 1); return (SNode*)pStmt; +_err: + return NULL; } SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt) { @@ -3150,6 +3512,8 @@ SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt) { pCxt->errCode = nodesMakeNode(QUERY_NODE_BALANCE_VGROUP_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); return (SNode*)pStmt; +_err: + return NULL; } SNode* createBalanceVgroupLeaderStmt(SAstCreateContext* pCxt, const SToken* pVgId) { @@ -3161,6 +3525,8 @@ SNode* createBalanceVgroupLeaderStmt(SAstCreateContext* pCxt, const SToken* pVgI pStmt->vgId = taosStr2Int32(pVgId->z, NULL, 10); } return (SNode*)pStmt; +_err: + return NULL; } SNode* createBalanceVgroupLeaderDBNameStmt(SAstCreateContext* pCxt, const SToken* pDbName) { @@ -3172,6 +3538,8 @@ SNode* createBalanceVgroupLeaderDBNameStmt(SAstCreateContext* pCxt, const SToken COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName); } return (SNode*)pStmt; +_err: + return NULL; } SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, const SToken* pVgId2) { @@ -3182,6 +3550,8 @@ SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, cons pStmt->vgId1 = taosStr2Int32(pVgId1->z, NULL, 10); pStmt->vgId2 = taosStr2Int32(pVgId2->z, NULL, 10); return (SNode*)pStmt; +_err: + return NULL; } SNode* createRedistributeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId, SNodeList* pDnodes) { @@ -3192,6 +3562,9 @@ SNode* createRedistributeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId pStmt->vgId = taosStr2Int32(pVgId->z, NULL, 10); pStmt->pDnodes = pDnodes; return (SNode*)pStmt; +_err: + nodesDestroyList(pDnodes); + return NULL; } SNode* createSplitVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId) { @@ -3201,6 +3574,8 @@ SNode* createSplitVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId) { CHECK_MAKE_NODE(pStmt); pStmt->vgId = taosStr2Int32(pVgId->z, NULL, 10); return (SNode*)pStmt; +_err: + return NULL; } SNode* createSyncdbStmt(SAstCreateContext* pCxt, const SToken* pDbName) { @@ -3209,15 +3584,16 @@ SNode* createSyncdbStmt(SAstCreateContext* pCxt, const SToken* pDbName) { pCxt->errCode = nodesMakeNode(QUERY_NODE_SYNCDB_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); return pStmt; +_err: + return NULL; } SNode* createGrantStmt(SAstCreateContext* pCxt, int64_t privileges, STokenPair* pPrivLevel, SToken* pUserName, SNode* pTagCond) { CHECK_PARSER_STATUS(pCxt); - if (!checkDbName(pCxt, &pPrivLevel->first, false) || !checkUserName(pCxt, pUserName) || - !checkTableName(pCxt, &pPrivLevel->second)) { - return NULL; - } + CHECK_NAME(checkDbName(pCxt, &pPrivLevel->first, false)); + CHECK_NAME(checkUserName(pCxt, pUserName)); + CHECK_NAME(checkTableName(pCxt, &pPrivLevel->second)); SGrantStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_GRANT_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); @@ -3229,14 +3605,16 @@ SNode* createGrantStmt(SAstCreateContext* pCxt, int64_t privileges, STokenPair* COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName); pStmt->pTagCond = pTagCond; return (SNode*)pStmt; +_err: + nodesDestroyNode(pTagCond); + return NULL; } SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, STokenPair* pPrivLevel, SToken* pUserName, SNode* pTagCond) { CHECK_PARSER_STATUS(pCxt); - if (!checkDbName(pCxt, &pPrivLevel->first, false) || !checkUserName(pCxt, pUserName)) { - return NULL; - } + CHECK_NAME(checkDbName(pCxt, &pPrivLevel->first, false)); + CHECK_NAME(checkUserName(pCxt, pUserName)); SRevokeStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_REVOKE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); @@ -3248,29 +3626,30 @@ SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, STokenPair* COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName); pStmt->pTagCond = pTagCond; return (SNode*)pStmt; +_err: + nodesDestroyNode(pTagCond); + return NULL; } SNode* createFuncForDelete(SAstCreateContext* pCxt, const char* pFuncName) { SFunctionNode* pFunc = NULL; + CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&pFunc); CHECK_MAKE_NODE(pFunc); snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", pFuncName); SNode* pCol = createPrimaryKeyCol(pCxt, NULL); - if (!pCol) { - nodesDestroyNode((SNode*)pFunc); - return NULL; - } + CHECK_MAKE_NODE(pCol); pCxt->errCode = nodesListMakeStrictAppend(&pFunc->pParameterList, pCol); - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - nodesDestroyNode((SNode*)pFunc); - return NULL; - } + CHECK_PARSER_STATUS(pCxt); return (SNode*)pFunc; +_err: + nodesDestroyNode((SNode*)pFunc); + return NULL; } SNode* createDeleteStmt(SAstCreateContext* pCxt, SNode* pTable, SNode* pWhere) { - CHECK_PARSER_STATUS(pCxt); SDeleteStmt* pStmt = NULL; + CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_DELETE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); pStmt->pFromTable = pTable; @@ -3278,11 +3657,15 @@ SNode* createDeleteStmt(SAstCreateContext* pCxt, SNode* pTable, SNode* pWhere) { pStmt->pCountFunc = createFuncForDelete(pCxt, "count"); pStmt->pFirstFunc = createFuncForDelete(pCxt, "first"); pStmt->pLastFunc = createFuncForDelete(pCxt, "last"); - if (NULL == pStmt->pCountFunc || NULL == pStmt->pFirstFunc || NULL == pStmt->pLastFunc) { - nodesDestroyNode((SNode*)pStmt); - CHECK_MAKE_NODE(NULL); - } + CHECK_MAKE_NODE(pStmt->pCountFunc); + CHECK_MAKE_NODE(pStmt->pFirstFunc); + CHECK_MAKE_NODE(pStmt->pLastFunc); return (SNode*)pStmt; +_err: + nodesDestroyNode((SNode*)pStmt); + nodesDestroyNode(pTable); + nodesDestroyNode(pWhere); + return NULL; } SNode* createInsertStmt(SAstCreateContext* pCxt, SNode* pTable, SNodeList* pCols, SNode* pQuery) { @@ -3299,17 +3682,18 @@ SNode* createInsertStmt(SAstCreateContext* pCxt, SNode* pTable, SNodeList* pCols strcpy(((SSetOperator*)pQuery)->stmtName, ((STableNode*)pTable)->tableAlias); } return (SNode*)pStmt; +_err: + nodesDestroyNode(pTable); + nodesDestroyNode(pQuery); + nodesDestroyList(pCols); + return NULL; } SNode* createCreateTSMAStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* tsmaName, SNode* pOptions, SNode* pRealTable, SNode* pInterval) { - CHECK_PARSER_STATUS(pCxt); - if (!checkTsmaName(pCxt, tsmaName)) { - nodesDestroyNode(pInterval); - return NULL; - } - SCreateTSMAStmt* pStmt = NULL; + CHECK_PARSER_STATUS(pCxt); + CHECK_NAME(checkTsmaName(pCxt, tsmaName)); pCxt->errCode = nodesMakeNode(QUERY_NODE_CREATE_TSMA_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); @@ -3318,11 +3702,7 @@ SNode* createCreateTSMAStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* // recursive tsma pStmt->pOptions = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_TSMA_OPTIONS, (SNode**)&pStmt->pOptions); - if (!pStmt->pOptions) { - nodesDestroyNode(pInterval); - nodesDestroyNode((SNode*)pStmt); - return NULL; - } + CHECK_MAKE_NODE(pStmt->pOptions); pStmt->pOptions->recursiveTsma = true; } else { pStmt->pOptions = (STSMAOptions*)pOptions; @@ -3337,6 +3717,12 @@ SNode* createCreateTSMAStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* nodesDestroyNode(pRealTable); return (SNode*)pStmt; +_err: + nodesDestroyNode((SNode*)pStmt); + nodesDestroyNode(pOptions); + nodesDestroyNode(pRealTable); + nodesDestroyNode(pInterval); + return NULL; } SNode* createTSMAOptions(SAstCreateContext* pCxt, SNodeList* pFuncs) { @@ -3346,6 +3732,9 @@ SNode* createTSMAOptions(SAstCreateContext* pCxt, SNodeList* pFuncs) { CHECK_MAKE_NODE(pOptions); pOptions->pFuncs = pFuncs; return (SNode*)pOptions; +_err: + nodesDestroyList(pFuncs); + return NULL; } SNode* createDefaultTSMAOptions(SAstCreateContext* pCxt) { @@ -3354,6 +3743,8 @@ SNode* createDefaultTSMAOptions(SAstCreateContext* pCxt) { pCxt->errCode = nodesMakeNode(QUERY_NODE_TSMA_OPTIONS, (SNode**)&pOptions); CHECK_MAKE_NODE(pOptions); return (SNode*)pOptions; +_err: + return NULL; } SNode* createDropTSMAStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable) { @@ -3370,6 +3761,9 @@ SNode* createDropTSMAStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* nodesDestroyNode(pRealTable); return (SNode*)pStmt; +_err: + nodesDestroyNode(pRealTable); + return NULL; } SNode* createShowTSMASStmt(SAstCreateContext* pCxt, SNode* dbName) { @@ -3381,4 +3775,7 @@ SNode* createShowTSMASStmt(SAstCreateContext* pCxt, SNode* dbName) { pStmt->pDbName = dbName; return (SNode*)pStmt; +_err: + nodesDestroyNode(dbName); + return NULL; } diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 49be0b8d90..d323d45c81 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -107,6 +107,7 @@ typedef struct SCollectMetaKeyCxt { typedef struct SCollectMetaKeyFromExprCxt { SCollectMetaKeyCxt* pComCxt; bool hasLastRowOrLast; + bool tbnameCollect; int32_t errCode; } SCollectMetaKeyFromExprCxt; @@ -204,6 +205,45 @@ static EDealRes collectMetaKeyFromTempTable(SCollectMetaKeyFromExprCxt* pCxt, ST return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; } +static int32_t isTbnameEqCondOperator(SOperatorNode* pOperator, char** ppTableName) { + if (pOperator->opType != OP_TYPE_EQUAL) { + return TSDB_CODE_SUCCESS; + } + + SValueNode* pValueNode = NULL; + if (nodeType(pOperator->pLeft) == QUERY_NODE_FUNCTION && + 0 == strcasecmp(((SFunctionNode*)(pOperator->pLeft))->functionName, "tbname") && + nodeType(pOperator->pRight) == QUERY_NODE_VALUE) { + pValueNode = (SValueNode*)pOperator->pRight; + } else if (nodeType(pOperator->pRight) == QUERY_NODE_FUNCTION && + 0 == strcasecmp(((SFunctionNode*)(pOperator->pRight))->functionName, "tbname") && + nodeType(pOperator->pLeft) == QUERY_NODE_VALUE) { + pValueNode = (SValueNode*)pOperator->pLeft; + } else { + return TSDB_CODE_SUCCESS; + } + + *ppTableName = pValueNode->literal; + + return TSDB_CODE_SUCCESS; +} + +static EDealRes collectMetaKeyFromOperator(SCollectMetaKeyFromExprCxt* pCxt, SOperatorNode* pOpNode) { + if (!pCxt->tbnameCollect) { + return DEAL_RES_CONTINUE; + } + + char* pTableName = NULL; + int32_t code = isTbnameEqCondOperator((SOperatorNode*)pOpNode, &pTableName); + if (TSDB_CODE_SUCCESS != code) return DEAL_RES_CONTINUE; + if (pTableName) { + SSelectStmt* pSelect = (SSelectStmt*)pCxt->pComCxt->pStmt; + pCxt->errCode = collectMetaKeyFromRealTableImpl(pCxt->pComCxt, ((SRealTableNode*)pSelect->pFromTable)->table.dbName, pTableName, AUTH_TYPE_READ); + } + + return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; +} + static EDealRes collectMetaKeyFromExprImpl(SNode* pNode, void* pContext) { SCollectMetaKeyFromExprCxt* pCxt = pContext; switch (nodeType(pNode)) { @@ -213,6 +253,8 @@ static EDealRes collectMetaKeyFromExprImpl(SNode* pNode, void* pContext) { return collectMetaKeyFromRealTable(pCxt, (SRealTableNode*)pNode); case QUERY_NODE_TEMP_TABLE: return collectMetaKeyFromTempTable(pCxt, (STempTableNode*)pNode); + case QUERY_NODE_OPERATOR: + return collectMetaKeyFromOperator(pCxt, (SOperatorNode*)pNode); default: break; } @@ -220,7 +262,7 @@ static EDealRes collectMetaKeyFromExprImpl(SNode* pNode, void* pContext) { } static int32_t collectMetaKeyFromExprs(SCollectMetaKeyCxt* pCxt, SNodeList* pList) { - SCollectMetaKeyFromExprCxt cxt = {.pComCxt = pCxt, .errCode = TSDB_CODE_SUCCESS}; + SCollectMetaKeyFromExprCxt cxt = {.pComCxt = pCxt, .errCode = TSDB_CODE_SUCCESS, .tbnameCollect = false}; nodesWalkExprs(pList, collectMetaKeyFromExprImpl, &cxt); return cxt.errCode; } @@ -245,6 +287,9 @@ static int32_t reserveDbCfgForLastRow(SCollectMetaKeyCxt* pCxt, SNode* pTable) { static int32_t collectMetaKeyFromSelect(SCollectMetaKeyCxt* pCxt, SSelectStmt* pStmt) { SCollectMetaKeyFromExprCxt cxt = {.pComCxt = pCxt, .hasLastRowOrLast = false, .errCode = TSDB_CODE_SUCCESS}; + if (pStmt->pFromTable && QUERY_NODE_REAL_TABLE == nodeType(pStmt->pFromTable)) { + cxt.tbnameCollect = true; + } nodesWalkSelectStmt(pStmt, SQL_CLAUSE_FROM, collectMetaKeyFromExprImpl, &cxt); if (TSDB_CODE_SUCCESS == cxt.errCode && cxt.hasLastRowOrLast) { cxt.errCode = reserveDbCfgForLastRow(pCxt, pStmt->pFromTable); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index aa6116287e..a8db514ee3 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2254,7 +2254,7 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpSt static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pFilePath, SRowsDataContext rowsDataCxt) { - char filePathStr[TSDB_FILENAME_LEN] = {0}; + char filePathStr[PATH_MAX] = {0}; if (TK_NK_STRING == pFilePath->type) { (void)trimString(pFilePath->z, pFilePath->n, filePathStr, sizeof(filePathStr)); } else { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 4126603ea4..7634d209eb 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1270,12 +1270,12 @@ static void setColumnPrimTs(STranslateContext* pCxt, SColumnNode* pCol, const ST } } -static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* pTable, bool igTags, SNodeList* pList) { +static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* pTable, bool igTags, SNodeList* pList, bool skipProjRef) { int32_t code = 0; if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta; int32_t nums = pMeta->tableInfo.numOfColumns + - (igTags ? 0 : ((TSDB_SUPER_TABLE == pMeta->tableType) ? pMeta->tableInfo.numOfTags : 0)); + (igTags ? 0 : ((TSDB_SUPER_TABLE == pMeta->tableType || ((SRealTableNode*)pTable)->stbRewrite) ? pMeta->tableInfo.numOfTags : 0)); for (int32_t i = 0; i < nums; ++i) { if (invisibleColumn(pCxt->pParseCxt->enableSysInfo, pMeta->tableType, pMeta->schema[i].flags)) { pCxt->pParseCxt->hasInvisibleCol = true; @@ -1305,7 +1305,11 @@ static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* p SListCell* pCell = nodesListGetCell(pList, LIST_LENGTH(pList) - 1); code = setColumnInfoByExpr(pTempTable, (SExprNode*)pNode, (SColumnNode**)&pCell->pNode); } - if (TSDB_CODE_SUCCESS != code) break; + if (TSDB_CODE_SUCCESS == code) { + if (!skipProjRef) pCol->projRefIdx = ((SExprNode*)pNode)->projIdx; // only set proj ref when select * from (select ...) + } else { + break; + } } } return code; @@ -4770,7 +4774,7 @@ static int32_t createAllColumns(STranslateContext* pCxt, bool igTags, SNodeList* size_t nums = taosArrayGetSize(pTables); for (size_t i = 0; i < nums; ++i) { STableNode* pTable = taosArrayGetP(pTables, i); - int32_t code = createColumnsByTable(pCxt, pTable, igTags, *pCols); + int32_t code = createColumnsByTable(pCxt, pTable, igTags, *pCols, nums > 1); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -4833,7 +4837,7 @@ static int32_t createTableAllCols(STranslateContext* pCxt, SColumnNode* pCol, bo } } if (TSDB_CODE_SUCCESS == code) { - code = createColumnsByTable(pCxt, pTable, igTags, *pOutput); + code = createColumnsByTable(pCxt, pTable, igTags, *pOutput, false); } return code; } @@ -5175,6 +5179,11 @@ static int32_t translateProjectionList(STranslateContext* pCxt, SSelectStmt* pSe if (!pSelect->isSubquery) { return rewriteProjectAlias(pSelect->pProjectionList); } else { + SNode* pNode; + int32_t projIdx = 1; + FOREACH(pNode, pSelect->pProjectionList) { + ((SExprNode*)pNode)->projIdx = projIdx++; + } return TSDB_CODE_SUCCESS; } } @@ -5976,6 +5985,7 @@ static int32_t isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pO *pRet = false; return TSDB_CODE_SUCCESS; } + SFunctionNode* pTbnameFunc = NULL; SValueNode* pValueNode = NULL; if (nodeType(pOperator->pLeft) == QUERY_NODE_FUNCTION && @@ -6053,7 +6063,6 @@ static int32_t isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pO static int32_t findEqCondTbNameInOperatorNode(STranslateContext* pCxt, SNode* pWhere, SEqCondTbNameTableInfo* pInfo, bool* pRet) { int32_t code = TSDB_CODE_SUCCESS; char* pTableAlias = NULL; - char* pTbNameVal = NULL; bool eqTbnameCond = false, tbnameInCond = false; code = isOperatorEqTbnameCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, &pInfo->aTbnames, &eqTbnameCond); if (TSDB_CODE_SUCCESS == code) { @@ -6234,12 +6243,73 @@ static void findVgroupsFromEqualTbname(STranslateContext* pCxt, SArray* aTbnames } } +static int32_t replaceToChildTableQuery(STranslateContext* pCxt, SEqCondTbNameTableInfo* pInfo) { + SName snameTb; + int32_t code = 0; + SRealTableNode* pRealTable = pInfo->pRealTable; + char* tbName = taosArrayGetP(pInfo->aTbnames, 0); + (void)toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, tbName, &snameTb); + + STableMeta* pMeta = NULL; + TAOS_CHECK_RETURN(catalogGetCachedTableMeta(pCxt->pParseCxt->pCatalog, &snameTb, &pMeta)); + if (NULL == pMeta || TSDB_CHILD_TABLE != pMeta->tableType || pMeta->suid != pRealTable->pMeta->suid) { + goto _return; + } + + pRealTable->pMeta->uid = pMeta->uid; + pRealTable->pMeta->vgId = pMeta->vgId; + pRealTable->pMeta->tableType = pMeta->tableType; + tstrncpy(pRealTable->table.tableName, tbName, sizeof(pRealTable->table.tableName)); + + pRealTable->stbRewrite = true; + + if (pRealTable->pTsmas) { + // if select from a child table, fetch it's corresponding tsma target child table infos + char buf[TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN + 1]; + for (int32_t i = 0; i < pRealTable->pTsmas->size; ++i) { + STableTSMAInfo* pTsma = taosArrayGetP(pRealTable->pTsmas, i); + SName tsmaTargetTbName = {0}; + (void)toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetTbName); + int32_t len = snprintf(buf, TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN, "%s.%s_%s", pTsma->dbFName, pTsma->name, + pRealTable->table.tableName); + len = taosCreateMD5Hash(buf, len); + strncpy(tsmaTargetTbName.tname, buf, MD5_OUTPUT_LEN); + STsmaTargetTbInfo ctbInfo = {0}; + if (!pRealTable->tsmaTargetTbInfo) { + pRealTable->tsmaTargetTbInfo = taosArrayInit(pRealTable->pTsmas->size, sizeof(STsmaTargetTbInfo)); + if (!pRealTable->tsmaTargetTbInfo) { + code = terrno; + break; + } + } + sprintf(ctbInfo.tableName, "%s", tsmaTargetTbName.tname); + ctbInfo.uid = pMeta->uid; + + if (NULL == taosArrayPush(pRealTable->tsmaTargetTbInfo, &ctbInfo)) { + code = terrno; + goto _return; + } + } + } + +_return: + + taosMemoryFree(pMeta); + return code; +} + static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt* pSelect, SArray* aTables) { int32_t code = TSDB_CODE_SUCCESS; - for (int i = 0; i < taosArrayGetSize(aTables); ++i) { - SEqCondTbNameTableInfo* pInfo = taosArrayGet(aTables, i); - int32_t nTbls = taosArrayGetSize(pInfo->aTbnames); + int32_t aTableNum = taosArrayGetSize(aTables); + int32_t nTbls = 0; + bool stableQuery = false; + SEqCondTbNameTableInfo* pInfo = NULL; + + qDebug("start to update stable vg for tbname optimize, aTableNum:%d", aTableNum); + for (int i = 0; i < aTableNum; ++i) { + pInfo = taosArrayGet(aTables, i); int32_t numOfVgs = pInfo->pRealTable->pVgroupList->numOfVgroups; + nTbls = taosArrayGetSize(pInfo->aTbnames); SVgroupsInfo* vgsInfo = taosMemoryMalloc(sizeof(SVgroupsInfo) + nTbls * sizeof(SVgroupInfo)); findVgroupsFromEqualTbname(pCxt, pInfo->aTbnames, pInfo->pRealTable->table.dbName, numOfVgs, vgsInfo); @@ -6249,6 +6319,7 @@ static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt* } else { taosMemoryFree(vgsInfo); } + stableQuery = pInfo->pRealTable->pMeta->tableType == TSDB_SUPER_TABLE; vgsInfo = NULL; if (pInfo->pRealTable->pTsmas) { @@ -6288,7 +6359,14 @@ static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt* } } } - return TSDB_CODE_SUCCESS; + + qDebug("before ctbname optimize, code:%d, aTableNum:%d, nTbls:%d, stableQuery:%d", code, aTableNum, nTbls, stableQuery); + + if (TSDB_CODE_SUCCESS == code && 1 == aTableNum && 1 == nTbls && stableQuery && NULL == pInfo->pRealTable->pTsmas) { + code = replaceToChildTableQuery(pCxt, pInfo); + } + + return code; } static int32_t setTableVgroupsFromEqualTbnameCond(STranslateContext* pCxt, SSelectStmt* pSelect) { diff --git a/source/libs/planner/inc/planInt.h b/source/libs/planner/inc/planInt.h index 090071ee7f..beb277493c 100644 --- a/source/libs/planner/inc/planInt.h +++ b/source/libs/planner/inc/planInt.h @@ -68,6 +68,8 @@ int32_t tagScanSetExecutionMode(SScanLogicNode* pScan); int32_t cloneLimit(SLogicNode* pParent, SLogicNode* pChild, uint8_t cloneWhat, bool* pCloned); int32_t sortPriKeyOptGetSequencingNodesImpl(SLogicNode* pNode, bool groupSort, SSortLogicNode* pSort, bool* pNotOptimize, SNodeList** pSequencingNodes, bool* keepSort); +bool isColRefExpr(const SColumnNode* pCol, const SExprNode* pExpr); +void rewriteTargetsWithResId(SNodeList* pTargets); #ifdef __cplusplus diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 42609c9a13..e6c94ec5b1 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -122,6 +122,7 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { strcpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName); strcpy(pCol->node.userAlias, ((SExprNode*)pExpr)->userAlias); strcpy(pCol->colName, ((SExprNode*)pExpr)->aliasName); + pCol->node.projIdx = ((SExprNode*)(*pNode))->projIdx; if (QUERY_NODE_FUNCTION == nodeType(pExpr)) { setColumnInfo((SFunctionNode*)pExpr, pCol, pCxt->isPartitionBy); } @@ -637,6 +638,10 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect } nodesDestroyList(pColList); } + + if (TSDB_CODE_SUCCESS == code) { + rewriteTargetsWithResId(pJoin->node.pTargets); + } if (NULL == pJoin->node.pTargets && NULL != pLeft) { code = nodesCloneList(pLeft->pTargets, &pJoin->node.pTargets); @@ -1357,6 +1362,9 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect ? (pSort->groupSort ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_GLOBAL) : DATA_ORDER_LEVEL_NONE; code = nodesCollectColumns(pSelect, SQL_CLAUSE_ORDER_BY, NULL, COLLECT_COL_TYPE_ALL, &pSort->node.pTargets); + if (TSDB_CODE_SUCCESS == code) { + rewriteTargetsWithResId(pSort->node.pTargets); + } if (TSDB_CODE_SUCCESS == code && NULL == pSort->node.pTargets) { SNode* pNew = NULL; code = nodesCloneNode(nodesListGetNode(pCxt->pCurrRoot->pTargets, 0), &pNew); @@ -1405,11 +1413,14 @@ static int32_t createColumnByProjections(SLogicPlanContext* pCxt, const char* pS } SNode* pNode; + int32_t projIdx = 1; FOREACH(pNode, pExprs) { - if (TSDB_CODE_SUCCESS != (code = nodesListStrictAppend(pList, (SNode*)createColumnByExpr(pStmtName, (SExprNode*)pNode)))) { + SColumnNode* pCol = createColumnByExpr(pStmtName, (SExprNode*)pNode); + if (TSDB_CODE_SUCCESS != (code = nodesListStrictAppend(pList, (SNode*)pCol))) { nodesDestroyList(pList); return code; } + pCol->resIdx = ((SExprNode*)pNode)->projIdx; } *pCols = pList; @@ -1474,6 +1485,9 @@ static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pS code = nodesListMakeStrictAppend(&pPartition->node.pTargets, pNew); } } + if (TSDB_CODE_SUCCESS == code) { + rewriteTargetsWithResId(pPartition->node.pTargets); + } if (TSDB_CODE_SUCCESS == code) { // code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, NULL, fmIsAggFunc, &pPartition->pAggFuncs); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 3aab190eda..7da4f93dee 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -3475,6 +3475,20 @@ static EDealRes eliminateProjOptRewriteScanTableAlias(SNode* pNode, void* pConte } +static void eliminateProjPushdownProjIdx(SNodeList* pParentProjects, SNodeList* pChildTargets) { + SNode* pChildTarget = NULL, *pParentProject = NULL; + FOREACH(pChildTarget, pChildTargets) { + SColumnNode* pTargetCol = (SColumnNode*)pChildTarget; + FOREACH(pParentProject, pParentProjects) { + SExprNode* pProject = (SExprNode*)pParentProject; + if (0 == strcmp(pTargetCol->colName, pProject->aliasName)) { + pTargetCol->resIdx = pProject->projIdx; + break; + } + } + } +} + static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SProjectLogicNode* pProjectNode) { SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pProjectNode->node.pChildren, 0); @@ -3546,6 +3560,7 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* nodesWalkExprs(pScan->pScanPseudoCols, eliminateProjOptRewriteScanTableAlias, &cxt); nodesWalkExpr(pScan->node.pConditions, eliminateProjOptRewriteScanTableAlias, &cxt); nodesWalkExprs(pChild->pTargets, eliminateProjOptRewriteScanTableAlias, &cxt); + eliminateProjPushdownProjIdx(pProjectNode->pProjections, pChild->pTargets); } if (TSDB_CODE_SUCCESS == code) { @@ -4883,6 +4898,31 @@ typedef struct SMergeProjectionsContext { int32_t errCode; } SMergeProjectionsContext; +static EDealRes mergeProjectionsExpr2(SNode** pNode, void* pContext) { + SMergeProjectionsContext* pCxt = pContext; + SProjectLogicNode* pChildProj = pCxt->pChildProj; + if (QUERY_NODE_COLUMN == nodeType(*pNode)) { + SColumnNode* pProjCol = (SColumnNode*)(*pNode); + SNode* pProjection; + int32_t projIdx = 1; + FOREACH(pProjection, pChildProj->pProjections) { + if (isColRefExpr(pProjCol, (SExprNode*)pProjection)) { + SNode* pExpr = NULL; + pCxt->errCode = nodesCloneNode(pProjection, &pExpr); + if (pExpr == NULL) { + return DEAL_RES_ERROR; + } + snprintf(((SExprNode*)pExpr)->aliasName, sizeof(((SExprNode*)pExpr)->aliasName), "%s", + ((SExprNode*)*pNode)->aliasName); + nodesDestroyNode(*pNode); + *pNode = pExpr; + return DEAL_RES_IGNORE_CHILD; + } + } + } + return DEAL_RES_CONTINUE; +} + static EDealRes mergeProjectionsExpr(SNode** pNode, void* pContext) { SMergeProjectionsContext* pCxt = pContext; SProjectLogicNode* pChildProj = pCxt->pChildProj; @@ -4917,7 +4957,7 @@ static int32_t mergeProjectsOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* ((SProjectLogicNode*)pSelfNode)->inputIgnoreGroup = true; } SMergeProjectionsContext cxt = {.pChildProj = (SProjectLogicNode*)pChild, .errCode = TSDB_CODE_SUCCESS}; - nodesRewriteExprs(((SProjectLogicNode*)pSelfNode)->pProjections, mergeProjectionsExpr, &cxt); + nodesRewriteExprs(((SProjectLogicNode*)pSelfNode)->pProjections, mergeProjectionsExpr2, &cxt); int32_t code = cxt.errCode; if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 82cac6d6ef..29440603cf 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -35,17 +35,18 @@ typedef struct SPhysiPlanContext { int32_t errCode; int16_t nextDataBlockId; SArray* pLocationHelper; + SArray* pProjIdxLocHelper; bool hasScan; bool hasSysScan; } SPhysiPlanContext; -static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int32_t *pLen) { +static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int32_t *pLen, uint16_t extraBufLen) { int32_t code = 0; if (QUERY_NODE_COLUMN == nodeType(pNode)) { SColumnNode* pCol = (SColumnNode*)pNode; if (NULL != pStmtName) { if ('\0' != pStmtName[0]) { - *ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1); + *ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen); if (!*ppKey) { return terrno; } @@ -55,7 +56,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); return code; } else { - *ppKey = taosMemoryCalloc(1, TSDB_COL_NAME_LEN + 1); + *ppKey = taosMemoryCalloc(1, TSDB_COL_NAME_LEN + 1 + extraBufLen); if (!*ppKey) { return terrno; } @@ -65,7 +66,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int } } if ('\0' == pCol->tableAlias[0]) { - *ppKey = taosMemoryCalloc(1, TSDB_COL_NAME_LEN + 1); + *ppKey = taosMemoryCalloc(1, TSDB_COL_NAME_LEN + 1 + extraBufLen); if (!*ppKey) { return terrno; } @@ -74,7 +75,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int return code; } - *ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1); + *ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen); if (!*ppKey) { return terrno; } @@ -89,7 +90,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); if (pVal) { if (NULL != pStmtName && '\0' != pStmtName[0]) { - *ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1); + *ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen); if (!*ppKey) { return terrno; } @@ -99,7 +100,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); return code; } - *ppKey = taosMemoryCalloc(1, strlen(pVal->literal) + 1 + TSDB_COL_NAME_LEN + 1); + *ppKey = taosMemoryCalloc(1, strlen(pVal->literal) + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen); if (!*ppKey) { return terrno; } @@ -113,7 +114,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int } if (NULL != pStmtName && '\0' != pStmtName[0]) { - *ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1); + *ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen); if (!*ppKey) { return terrno; } @@ -124,7 +125,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int return code; } - *ppKey = taosMemoryCalloc(1, TSDB_COL_NAME_LEN + 1); + *ppKey = taosMemoryCalloc(1, TSDB_COL_NAME_LEN + 1 + extraBufLen); if (!*ppKey) { return terrno; } @@ -191,22 +192,34 @@ static int32_t putSlotToHash(const char* pName, int32_t len, int16_t dataBlockId } static int32_t createDataBlockDescHash(SPhysiPlanContext* pCxt, int32_t capacity, int16_t dataBlockId, - SHashObj** pDescHash) { + SHashObj** pDescHash, SHashObj** ppProjIdxDescHash) { SHashObj* pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (NULL == pHash) { return TSDB_CODE_OUT_OF_MEMORY; } + SHashObj* pProjIdxHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (!pProjIdxHash) { + taosHashCleanup(pHash); + return TSDB_CODE_OUT_OF_MEMORY; + } if (NULL == taosArrayInsert(pCxt->pLocationHelper, dataBlockId, &pHash)) { taosHashCleanup(pHash); + taosHashCleanup(pProjIdxHash); + return TSDB_CODE_OUT_OF_MEMORY; + } + if (NULL == taosArrayInsert(pCxt->pProjIdxLocHelper, dataBlockId, &pProjIdxHash)) { + taosHashCleanup(pHash); + taosHashCleanup(pProjIdxHash); return TSDB_CODE_OUT_OF_MEMORY; } *pDescHash = pHash; + *ppProjIdxDescHash = pProjIdxHash; return TSDB_CODE_SUCCESS; } static int32_t buildDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc, - SHashObj* pHash) { + SHashObj* pHash, SHashObj* pProjIdxDescHash) { pDataBlockDesc->pSlots = NULL; int32_t code = nodesMakeList(&pDataBlockDesc->pSlots); if (NULL == pDataBlockDesc->pSlots) { @@ -218,12 +231,16 @@ static int32_t buildDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SD FOREACH(pNode, pList) { char* name = NULL; int32_t len = 0; - code = getSlotKey(pNode, NULL, &name, &len); + code = getSlotKey(pNode, NULL, &name, &len, 16); if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, name, pNode, slotId, true, false)); } + code = putSlotToHash(name, len, pDataBlockDesc->dataBlockId, slotId, pNode, pHash); if (TSDB_CODE_SUCCESS == code) { - code = putSlotToHash(name, len, pDataBlockDesc->dataBlockId, slotId, pNode, pHash); + if (nodeType(pNode) == QUERY_NODE_COLUMN && ((SColumnNode*)pNode)->resIdx > 0) { + sprintf(name + strlen(name), "_%d", ((SColumnNode*)pNode)->resIdx); + code = putSlotToHash(name, strlen(name), pDataBlockDesc->dataBlockId, slotId, pNode, pProjIdxDescHash); + } } taosMemoryFree(name); if (TSDB_CODE_SUCCESS == code) { @@ -246,9 +263,10 @@ static int32_t createDataBlockDesc(SPhysiPlanContext* pCxt, SNodeList* pList, SD pDesc->dataBlockId = pCxt->nextDataBlockId++; SHashObj* pHash = NULL; - code = createDataBlockDescHash(pCxt, LIST_LENGTH(pList), pDesc->dataBlockId, &pHash); + SHashObj* pProjIdxHash = NULL; + code = createDataBlockDescHash(pCxt, LIST_LENGTH(pList), pDesc->dataBlockId, &pHash, &pProjIdxHash); if (TSDB_CODE_SUCCESS == code) { - code = buildDataBlockSlots(pCxt, pList, pDesc, pHash); + code = buildDataBlockSlots(pCxt, pList, pDesc, pHash, pProjIdxHash); } if (TSDB_CODE_SUCCESS == code) { @@ -286,7 +304,7 @@ static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList, SNode* pExpr = QUERY_NODE_ORDER_BY_EXPR == nodeType(pNode) ? ((SOrderByExprNode*)pNode)->pExpr : pNode; char *name = NULL; int32_t len = 0; - code = getSlotKey(pExpr, pStmtName, &name, &len); + code = getSlotKey(pExpr, pStmtName, &name, &len, 0); if (TSDB_CODE_SUCCESS == code) { SSlotIndex* pIndex = taosHashGet(pHash, name, len); if (NULL == pIndex) { @@ -355,7 +373,9 @@ static int32_t pushdownDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, typedef struct SSetSlotIdCxt { int32_t errCode; SHashObj* pLeftHash; + SHashObj* pLeftProjIdxHash; SHashObj* pRightHash; + SHashObj* pRightProdIdxHash; } SSetSlotIdCxt; static void dumpSlots(const char* pName, SHashObj* pHash) { @@ -379,13 +399,22 @@ static EDealRes doSetSlotId(SNode* pNode, void* pContext) { SSetSlotIdCxt* pCxt = (SSetSlotIdCxt*)pContext; char *name = NULL; int32_t len = 0; - pCxt->errCode = getSlotKey(pNode, NULL, &name, &len); + pCxt->errCode = getSlotKey(pNode, NULL, &name, &len, 16); if (TSDB_CODE_SUCCESS != pCxt->errCode) { return DEAL_RES_ERROR; } - SSlotIndex* pIndex = taosHashGet(pCxt->pLeftHash, name, len); - if (NULL == pIndex) { - pIndex = taosHashGet(pCxt->pRightHash, name, len); + SSlotIndex *pIndex = NULL; + if (((SColumnNode*)pNode)->projRefIdx > 0) { + sprintf(name + strlen(name), "_%d", ((SColumnNode*)pNode)->projRefIdx); + pIndex = taosHashGet(pCxt->pLeftProjIdxHash, name, strlen(name)); + if (!pIndex) { + pIndex = taosHashGet(pCxt->pRightProdIdxHash, name, strlen(name)); + } + } else { + pIndex = taosHashGet(pCxt->pLeftHash, name, len); + if (NULL == pIndex) { + pIndex = taosHashGet(pCxt->pRightHash, name, len); + } } // pIndex is definitely not NULL, otherwise it is a bug if (NULL == pIndex) { @@ -396,9 +425,9 @@ static EDealRes doSetSlotId(SNode* pNode, void* pContext) { taosMemoryFree(name); return DEAL_RES_ERROR; } - taosMemoryFree(name); ((SColumnNode*)pNode)->dataBlockId = pIndex->dataBlockId; ((SColumnNode*)pNode)->slotId = ((SSlotIdInfo*)taosArrayGet(pIndex->pSlotIdsInfo, 0))->slotId; + taosMemoryFree(name); return DEAL_RES_IGNORE_CHILD; } return DEAL_RES_CONTINUE; @@ -419,7 +448,9 @@ static int32_t setNodeSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i SSetSlotIdCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId), - .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId))}; + .pLeftProjIdxHash = taosArrayGetP(pCxt->pProjIdxLocHelper, leftDataBlockId), + .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)), + .pRightProdIdxHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pProjIdxLocHelper, rightDataBlockId))}; nodesWalkExpr(pRes, doSetSlotId, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyNode(pRes); @@ -445,7 +476,9 @@ static int32_t setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i SSetSlotIdCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId), - .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId))}; + .pLeftProjIdxHash = taosArrayGetP(pCxt->pProjIdxLocHelper, leftDataBlockId), + .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)), + .pRightProdIdxHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pProjIdxLocHelper, rightDataBlockId))}; nodesWalkExprs(pRes, doSetSlotId, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(pRes); @@ -1254,7 +1287,7 @@ static int32_t sortHashJoinTargets(int16_t lBlkId, int16_t rBlkId, SHashJoinPhys SColumnNode* pCol = (SColumnNode*)pNode; char *pName = NULL; int32_t len = 0; - code = getSlotKey(pNode, NULL, &pName, &len); + code = getSlotKey(pNode, NULL, &pName, &len, 0); if (TSDB_CODE_SUCCESS == code) { code = tSimpleHashPut(pHash, pName, len, &pCol, POINTER_BYTES); } @@ -1272,7 +1305,7 @@ static int32_t sortHashJoinTargets(int16_t lBlkId, int16_t rBlkId, SHashJoinPhys char* pName = NULL; SColumnNode* pCol = (SColumnNode*)pNode; int32_t len = 0; - code = getSlotKey(pNode, NULL, &pName, &len); + code = getSlotKey(pNode, NULL, &pName, &len, 0); if (TSDB_CODE_SUCCESS == code) { SNode** p = tSimpleHashGet(pHash, pName, len); if (p) { @@ -1293,7 +1326,7 @@ static int32_t sortHashJoinTargets(int16_t lBlkId, int16_t rBlkId, SHashJoinPhys char* pName = NULL; SColumnNode* pCol = (SColumnNode*)pNode; int32_t len = 0; - code = getSlotKey(pNode, NULL, &pName, &len); + code = getSlotKey(pNode, NULL, &pName, &len, 0); if (TSDB_CODE_SUCCESS == code) { SNode** p = tSimpleHashGet(pHash, pName, len); if (p) { @@ -2999,6 +3032,7 @@ static void destoryLocationHash(void* p) { static void destoryPhysiPlanContext(SPhysiPlanContext* pCxt) { taosArrayDestroyEx(pCxt->pLocationHelper, destoryLocationHash); + taosArrayDestroyEx(pCxt->pProjIdxLocHelper, destoryLocationHash); } static void setExplainInfo(SPlanContext* pCxt, SQueryPlan* pPlan) { @@ -3030,9 +3064,12 @@ int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryP .errCode = TSDB_CODE_SUCCESS, .nextDataBlockId = 0, .pLocationHelper = taosArrayInit(32, POINTER_BYTES), + .pProjIdxLocHelper = taosArrayInit(32, POINTER_BYTES), .hasScan = false, .hasSysScan = false}; - if (NULL == cxt.pLocationHelper) { + if (NULL == cxt.pLocationHelper || !cxt.pProjIdxLocHelper) { + taosArrayDestroy(cxt.pLocationHelper); + taosArrayDestroy(cxt.pProjIdxLocHelper); return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index e1e98f221f..b5f0bc50e8 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -716,5 +716,16 @@ int32_t tagScanSetExecutionMode(SScanLogicNode* pScan) { return TSDB_CODE_SUCCESS; } +bool isColRefExpr(const SColumnNode* pCol, const SExprNode* pExpr) { + if (pCol->projRefIdx > 0) return pCol->projRefIdx == pExpr->projIdx; + return 0 == strcmp(pCol->colName, pExpr->aliasName); +} +void rewriteTargetsWithResId(SNodeList* pTargets) { + SNode* pNode; + FOREACH(pNode, pTargets) { + SColumnNode* pCol = (SColumnNode*)pNode; + pCol->resIdx = pCol->projRefIdx; + } +} diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index c723360b36..f12d0fd246 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -91,7 +91,7 @@ typedef struct SSchRuntimeStat { #if defined(WINDOWS) || defined(_TD_DARWIN_64) size_t avoidCompilationErrors; #endif - + int64_t hbConnNotFound; } SSchRuntimeStat; typedef struct SSchJobStat { diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 01249dbb98..3f610ed387 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -251,9 +251,8 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) { hb = taosHashGet(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); if (NULL == hb) { SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); - qDebug("taosHashGet hb connection not exists, nodeId:%d, fqdn:%s, port:%d", epId->nodeId, epId->ep.fqdn, - epId->ep.port); - SCH_ERR_RET(TSDB_CODE_APP_ERROR); + (void)atomic_add_fetch_64(&schMgmt.stat.runtime.hbConnNotFound, 1); + return TSDB_CODE_SUCCESS; } SCH_LOCK(SCH_WRITE, &hb->lock); diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 6245c55b48..c00e56a4a3 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1096,21 +1096,23 @@ int32_t delObsoleteCheckpoint(void* arg, const char* path) { * replication is finished */ int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) { + int32_t code = 0; STaskDbWrapper* pBackend = arg; - + SArray * chkpDel = NULL, *chkpDup = NULL; (void)taosThreadRwlockWrlock(&pBackend->chkpDirLock); - (void)taosArrayPush(pBackend->chkpSaved, &chkpId); - - SArray* chkpDel = taosArrayInit(8, sizeof(int64_t)); - if (chkpDel == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + if (taosArrayPush(pBackend->chkpSaved, &chkpId) == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception); } - SArray* chkpDup = taosArrayInit(8, sizeof(int64_t)); + chkpDel = taosArrayInit(8, sizeof(int64_t)); + if (chkpDel == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception); + } + + chkpDup = taosArrayInit(8, sizeof(int64_t)); if (chkpDup == NULL) { - taosArrayDestroy(chkpDel); - return TSDB_CODE_OUT_OF_MEMORY; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception); } int64_t firsId = 0; @@ -1120,9 +1122,13 @@ int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) { for (int i = 0; i < taosArrayGetSize(pBackend->chkpSaved); i++) { int64_t id = *(int64_t*)taosArrayGet(pBackend->chkpSaved, i); if (id >= firsId) { - (void)taosArrayPush(chkpDup, &id); + if (taosArrayPush(chkpDup, &id) == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception); + } } else { - (void)taosArrayPush(chkpDel, &id); + if (taosArrayPush(chkpDel, &id) == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception); + } } } } else { @@ -1131,13 +1137,18 @@ int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) { for (int i = 0; i < dsz; i++) { int64_t id = *(int64_t*)taosArrayGet(pBackend->chkpSaved, i); - (void)taosArrayPush(chkpDel, &id); + if (taosArrayPush(chkpDel, &id) == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception); + } } for (int i = dsz < 0 ? 0 : dsz; i < sz; i++) { int64_t id = *(int64_t*)taosArrayGet(pBackend->chkpSaved, i); - (void)taosArrayPush(chkpDup, &id); + if (taosArrayPush(chkpDup, &id) == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception); + } } } + taosArrayDestroy(pBackend->chkpSaved); pBackend->chkpSaved = chkpDup; @@ -1155,6 +1166,11 @@ int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) { } taosArrayDestroy(chkpDel); return 0; +_exception: + taosArrayDestroy(chkpDup); + taosArrayDestroy(chkpDel); + (void)taosThreadRwlockUnlock(&pBackend->chkpDirLock); + return code; } #ifdef BUILD_NO_CALL @@ -1288,7 +1304,9 @@ int32_t taskDbLoadChkpInfo(STaskDbWrapper* pBackend) { int ret = sscanf(taosGetDirEntryName(de), "checkpoint%" PRId64 "", &checkpointId); if (ret == 1) { - (void)taosArrayPush(pBackend->chkpSaved, &checkpointId); + if (taosArrayPush(pBackend->chkpSaved, &checkpointId) == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception); + } } } else { continue; @@ -1300,13 +1318,21 @@ int32_t taskDbLoadChkpInfo(STaskDbWrapper* pBackend) { (void)taosCloseDir(&pDir); return 0; +_exception: + taosMemoryFree(pChkpDir); + (void)taosCloseDir(&pDir); + return code; } int32_t chkpGetAllDbCfHandle2(STaskDbWrapper* pBackend, rocksdb_column_family_handle_t*** ppHandle) { + int32_t code = 0; SArray* pHandle = taosArrayInit(8, POINTER_BYTES); for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) { if (pBackend->pCf[i]) { rocksdb_column_family_handle_t* p = pBackend->pCf[i]; - (void)taosArrayPush(pHandle, &p); + if (taosArrayPush(pHandle, &p) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exception; + } } } int32_t nCf = taosArrayGetSize(pHandle); @@ -1316,13 +1342,20 @@ int32_t chkpGetAllDbCfHandle2(STaskDbWrapper* pBackend, rocksdb_column_family_ha } rocksdb_column_family_handle_t** ppCf = taosMemoryCalloc(nCf, sizeof(rocksdb_column_family_handle_t*)); + if (ppCf == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception); + } for (int i = 0; i < nCf; i++) { ppCf[i] = taosArrayGetP(pHandle, i); } + taosArrayDestroy(pHandle); *ppHandle = ppCf; return nCf; +_exception: + taosArrayDestroy(pHandle); + return code; } int32_t chkpDoDbCheckpoint(rocksdb_t* db, char* path) { @@ -2435,7 +2468,9 @@ void taskDbInitChkpOpt(STaskDbWrapper* pTaskDb) { void taskDbRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp) { (void)taosThreadRwlockWrlock(&pTaskDb->chkpDirLock); - (void)taosArrayPush(pTaskDb->chkpInUse, &chkp); + if (taosArrayPush(pTaskDb->chkpInUse, &chkp) == NULL) { + stError("failed to push chkp: %" PRIi64 " into inuse", chkp); + } taosArraySort(pTaskDb->chkpInUse, chkpIdComp); (void)taosThreadRwlockUnlock(&pTaskDb->chkpDirLock); } @@ -4331,7 +4366,10 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co if (strncmp(key, start, strlen(start)) == 0 && strlen(key) >= strlen(start) + 1) { int64_t checkPoint = 0; if (sscanf(key + strlen(key), ":%" PRId64 "", &checkPoint) == 1) { - (void)taosArrayPush(result, &checkPoint); + if (taosArrayPush(result, &checkPoint) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + break; + } } } else { break; @@ -4547,7 +4585,10 @@ int32_t compareHashTableImpl(SHashObj* p1, SHashObj* p2, SArray* diff) { return TSDB_CODE_OUT_OF_MEMORY; } (void)strncpy(fname, name, len); - (void)taosArrayPush(diff, &fname); + if (taosArrayPush(diff, &fname) == NULL) { + taosMemoryFree(fname); + return TSDB_CODE_OUT_OF_MEMORY; + } } pIter = taosHashIterate(p2, pIter); } @@ -4706,7 +4747,11 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { } (void)strncpy(fname, name, len); - (void)taosArrayPush(p->pAdd, &fname); + if (taosArrayPush(p->pAdd, &fname) == NULL) { + taosMemoryFree(fname); + (void)taosThreadRwlockUnlock(&p->rwLock); + return TSDB_CODE_OUT_OF_MEMORY; + } } pIter = taosHashIterate(p->pSstTbl[1 - p->idx], pIter); } @@ -4910,7 +4955,11 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) { code = TSDB_CODE_OUT_OF_MEMORY; goto _ERROR; } - (void)taosArrayPush(list, &p); + if (taosArrayPush(list, &p) == NULL) { + taosMemoryFree(p); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _ERROR; + } } // copy current file to dst dir diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index da027d05a6..91196f31e0 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -585,12 +585,18 @@ void getCheckRspStatus(STaskCheckInfo* pInfo, int64_t el, int32_t* numOfReady, i } else { // TASK_DOWNSTREAM_NOT_READY if (p->rspTs == 0) { // not response yet if (el >= CHECK_NOT_RSP_DURATION) { // not receive info for 10 sec. - (void)taosArrayPush(pTimeoutList, &p->taskId); + void* px = taosArrayPush(pTimeoutList, &p->taskId); + if (px == NULL) { + stError("s-task:%s failed to record time out task:0x%x", id, p->taskId); + } } else { // el < CHECK_NOT_RSP_DURATION (*numOfNotRsp) += 1; // do nothing and continue waiting for their rsp } } else { - (void)taosArrayPush(pNotReadyList, &p->taskId); + void* px = taosArrayPush(pNotReadyList, &p->taskId); + if (px == NULL) { + stError("s-task:%s failed to record not ready task:0x%x", id, p->taskId); + } } } } diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index da882505f4..d0bf24bd03 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -973,7 +973,10 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) { if (!recved) { // make sure the inputQ is opened for not recv upstream checkpoint-trigger message streamTaskOpenUpstreamInput(pTask, pInfo->taskId); - (void)taosArrayPush(pNotSendList, pInfo); + void* px = taosArrayPush(pNotSendList, pInfo); + if (px == NULL) { + stError("s-task:%s failed to record not send info, code: out of memory", id); + } } } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 59210fe99f..0bc090cdfe 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -214,6 +214,7 @@ int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, tEncoderInit(&encoder, abuf, tlen); if ((code = tEncodeStreamTaskCheckReq(&encoder, pReq)) < 0) { rpcFreeCont(buf); + tEncoderClear(&encoder); return code; } tEncoderClear(&encoder); @@ -306,7 +307,12 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD int32_t code = 0; int64_t now = taosGetTimestampMs(); int32_t numOfBlocks = taosArrayGetSize(pData->blocks); - ASSERT(numOfBlocks != 0 && pTask->msgInfo.pData == NULL); + + if (!(numOfBlocks != 0 && pTask->msgInfo.pData == NULL)) { + stError("s-task:%s dispatch block number:%d, exist not rsp dispatch msg:%p, abort build new dispatch msg", + pTask->id.idStr, numOfBlocks, pTask->msgInfo.pData); + return TSDB_CODE_STREAM_INTERNAL_ERROR; + } pTask->msgInfo.dispatchMsgType = pData->type; @@ -474,7 +480,10 @@ static void addDispatchEntry(SDispatchMsgInfo* pMsgInfo, int32_t nodeId, int64_t streamMutexLock(&pMsgInfo->lock); } - (void)taosArrayPush(pMsgInfo->pSendInfo, &entry); + void* p = taosArrayPush(pMsgInfo->pSendInfo, &entry); + if (p == NULL) { + stError("failed to add dispatch info"); + } if (lock) { streamMutexUnlock(&pMsgInfo->lock); @@ -671,8 +680,8 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S memcpy(bln.parTbName, pDataBlock->info.parTbName, strlen(pDataBlock->info.parTbName)); // failed to put into name buffer, no need to do anything - if (tSimpleHashGetSize(pTask->pNameMap) < MAX_BLOCK_NAME_NUM) { - (void)tSimpleHashPut(pTask->pNameMap, &groupId, sizeof(int64_t), &bln, sizeof(SBlockName)); + if (tSimpleHashGetSize(pTask->pNameMap) < MAX_BLOCK_NAME_NUM) { // allow error, and do nothing + int32_t code = tSimpleHashPut(pTask->pNameMap, &groupId, sizeof(int64_t), &bln, sizeof(SBlockName)); } } @@ -708,14 +717,15 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S } streamMutexUnlock(&pTask->msgInfo.lock); - ASSERT(found); - return 0; + if (!found) { + stError("s-task:%s not found req hash value:%u", pTask->id.idStr, hashValue); + return TSDB_CODE_STREAM_INTERNAL_ERROR; + } else { + return 0; + } } int32_t streamDispatchStreamBlock(SStreamTask* pTask) { - ASSERT((pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH || - pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH)); - const char* id = pTask->id.idStr; int32_t numOfElems = streamQueueGetNumOfItems(pTask->outputq.queue); if (numOfElems > 0) { @@ -739,8 +749,11 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { return 0; } - ASSERT(pTask->msgInfo.pData == NULL); - stDebug("s-task:%s start to dispatch msg, set output status:%d", id, pTask->outputq.status); + if (pTask->msgInfo.pData != NULL) { + stFatal("s-task:%s not rsp data:%p exist, should not dispatch msg now", id, pTask->msgInfo.pData); + } else { + stDebug("s-task:%s start to dispatch msg, set output status:%d", id, pTask->outputq.status); + } SStreamDataBlock* pBlock = NULL; streamQueueNextItem(pTask->outputq.queue, (SStreamQueueItem**)&pBlock); @@ -751,8 +764,11 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { } int32_t type = pBlock->type; - ASSERT(type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__CHECKPOINT_TRIGGER || - type == STREAM_INPUT__TRANS_STATE); + if (!(type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__CHECKPOINT_TRIGGER || + type == STREAM_INPUT__TRANS_STATE)) { + stError("s-task:%s invalid dispatch block type:%d", id, type); + return TSDB_CODE_INTERNAL_ERROR; + } pTask->execInfo.dispatch += 1; @@ -830,6 +846,7 @@ int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32 tEncoderInit(&encoder, abuf, tlen); if ((code = tEncodeStreamCheckpointReadyMsg(&encoder, &req)) < 0) { rpcFreeCont(buf); + tEncoderClear(&encoder); return code; } tEncoderClear(&encoder); @@ -878,7 +895,6 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { SArray* pList = pActiveInfo->pReadyMsgList; int32_t num = taosArrayGetSize(pList); - if (pTmrInfo->launchChkptId != pActiveInfo->activeId) { streamMutexUnlock(&pActiveInfo->lock); int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); @@ -902,7 +918,15 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { SArray* pNotRspList = taosArrayInit(4, sizeof(int32_t)); - ASSERT(taosArrayGetSize(pTask->upstreamInfo.pList) == num); + if (taosArrayGetSize(pTask->upstreamInfo.pList) != num) { + streamMutexUnlock(&pActiveInfo->lock); + int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); + stWarn("s-task:%s vgId:%d upstream number:%d not equals sent readyMsg:%d, quit from readyMsg send tmr, ref:%d", id, + vgId, (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList), num, ref); + + streamMetaReleaseTask(pTask->pMeta, pTask); + return; + } for (int32_t i = 0; i < num; ++i) { STaskCheckpointReadyInfo* pInfo = taosArrayGet(pList, i); @@ -914,9 +938,13 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { continue; } - (void)taosArrayPush(pNotRspList, &pInfo->upstreamTaskId); - stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId, - pTask->info.taskLevel, pInfo->upstreamTaskId); + void* p = taosArrayPush(pNotRspList, &pInfo->upstreamTaskId); + if (p == NULL) { + stError("s-task:%s vgId:%d failed to record not rsp task, code: out of memory", id, vgId); + } else { + stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId, + pTask->info.taskLevel, pInfo->upstreamTaskId); + } } int32_t checkpointId = pActiveInfo->activeId; @@ -984,7 +1012,11 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) { streamMutexLock(&pActiveInfo->lock); int32_t num = taosArrayGetSize(pList); - ASSERT(taosArrayGetSize(pTask->upstreamInfo.pList) == num); + if (taosArrayGetSize(pTask->upstreamInfo.pList) != num) { + stError("s-task:%s invalid number of sent readyMsg:%d to upstream:%d", id, num, + (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList)); + return TSDB_CODE_STREAM_INTERNAL_ERROR; + } for (int32_t i = 0; i < num; ++i) { STaskCheckpointReadyInfo* pInfo = taosArrayGet(pList, i); @@ -1064,9 +1096,7 @@ int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) { int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq) { int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN; - ASSERT(dataStrLen > 0); - - void* buf = taosMemoryCalloc(1, dataStrLen); + void* buf = taosMemoryCalloc(1, dataStrLen); if (buf == NULL) { return terrno; } @@ -1100,8 +1130,17 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch payloadLen += sizeof(SRetrieveTableRsp); - (void)taosArrayPush(pReq->dataLen, &payloadLen); - (void)taosArrayPush(pReq->data, &buf); + void* px = taosArrayPush(pReq->dataLen, &payloadLen); + if (px == NULL) { + taosMemoryFree(buf); + return terrno; + } + + px = taosArrayPush(pReq->data, &buf); + if (px == NULL) { + taosMemoryFree(buf); + return terrno; + } pReq->totalLen += dataStrLen; return 0; @@ -1131,6 +1170,7 @@ int32_t doSendDispatchMsg(SStreamTask* pTask, const SStreamDispatchReq* pReq, in SEncoder encoder; tEncoderInit(&encoder, abuf, tlen); if ((code = tEncodeStreamDispatchReq(&encoder, pReq)) < 0) { + tEncoderClear(&encoder); goto FAIL; } tEncoderClear(&encoder); @@ -1205,8 +1245,6 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa int32_t size = taosArrayGetSize(pActiveInfo->pReadyMsgList); if (size > 0) { - ASSERT(size == 1); - STaskCheckpointReadyInfo* pReady = taosArrayGet(pActiveInfo->pReadyMsgList, 0); if (pReady == NULL) { return terrno; @@ -1221,8 +1259,12 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa pTask->id.idStr, pReady->checkpointId, pReady->transId, pReq->transId, pReq->checkpointId); } } else { - (void)taosArrayPush(pActiveInfo->pReadyMsgList, &info); - stDebug("s-task:%s add checkpoint source rsp msg, total:%d", pTask->id.idStr, size + 1); + void* px = taosArrayPush(pActiveInfo->pReadyMsgList, &info); + if (px != NULL) { + stDebug("s-task:%s add checkpoint source rsp msg, total:%d", pTask->id.idStr, size + 1); + } else { + stError("s-task:%s failed to add readyMsg, code: out of memory", pTask->id.idStr); + } } streamMutexUnlock(&pActiveInfo->lock); @@ -1259,7 +1301,12 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; streamMutexLock(&pActiveInfo->lock); - (void)taosArrayPush(pActiveInfo->pReadyMsgList, &info); + void* px = taosArrayPush(pActiveInfo->pReadyMsgList, &info); + if (px == NULL) { + streamMutexUnlock(&pActiveInfo->lock); + stError("s-task:%s failed to add readyMsg info, code: out of memory", pTask->id.idStr); + return terrno; + } int32_t numOfRecv = taosArrayGetSize(pActiveInfo->pReadyMsgList); int32_t total = streamTaskGetNumOfUpstream(pTask); @@ -1267,7 +1314,6 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, stDebug("s-task:%s recv checkpoint-trigger from all upstream, continue", pTask->id.idStr); pActiveInfo->allUpstreamTriggerRecv = 1; } else { - ASSERT(numOfRecv <= total); stDebug("s-task:%s %d/%d checkpoint-trigger recv", pTask->id.idStr, numOfRecv, total); } @@ -1516,7 +1562,9 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i if (pMsgInfo->dispatchMsgType == STREAM_INPUT__TRANS_STATE) { stDebug("s-task:%s dispatch trans-state msgId:%d to downstream successfully, start to prepare transfer state", id, msgId); - ASSERT(pTask->info.fillHistory == 1); + if (pTask->info.fillHistory != 1) { + stFatal("s-task:%s unexpected dispatch rsp, not scan-history task, not recv this dispatch rsp", id); + } code = streamTransferStatePrepare(pTask); if (code != TSDB_CODE_SUCCESS) { // todo: do nothing if error happens @@ -1542,7 +1590,10 @@ static int32_t buildDispatchRsp(const SStreamTask* pTask, const SStreamDispatchR } ((SMsgHead*)(*pBuf))->vgId = htonl(pReq->upstreamNodeId); - ASSERT(((SMsgHead*)(*pBuf))->vgId != 0); + + if (((SMsgHead*)(*pBuf))->vgId == 0) { + return TSDB_CODE_INVALID_MSG; + } SStreamDispatchRsp* pDispatchRsp = POINTER_SHIFT((*pBuf), sizeof(SMsgHead)); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 7d9a86538e..fca0bf403f 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -45,7 +45,11 @@ static int32_t doOutputResultBlockImpl(SStreamTask* pTask, SStreamDataBlock* pBl pTask->outputInfo.smaSink.smaSink(pTask->outputInfo.smaSink.vnode, pTask->outputInfo.smaSink.smaId, pBlock->blocks); destroyStreamDataBlock(pBlock); } else { - ASSERT(type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH); + if (type != TASK_OUTPUT__FIXED_DISPATCH && type != TASK_OUTPUT__SHUFFLE_DISPATCH) { + stError("s-task:%s invalid stream output type:%d, internal error", pTask->id.idStr, type); + return TSDB_CODE_STREAM_INTERNAL_ERROR; + } + code = streamTaskPutDataIntoOutputQ(pTask, pBlock); if (code != TSDB_CODE_SUCCESS) { destroyStreamDataBlock(pBlock); @@ -127,7 +131,11 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { SSDataBlock block = {0}; const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)pItem; - ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1); + int32_t num = taosArrayGetSize(pRetrieveBlock->blocks); + if (num != 1) { + stError("s-task:%s invalid retrieve block number:%d, ignore", pTask->id.idStr, num); + continue; + } (void)assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0)); block.info.type = STREAM_PULL_OVER; @@ -178,7 +186,6 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to // current output should be dispatched to down stream nodes if (numOfBlocks >= STREAM_RESULT_DUMP_THRESHOLD || size >= STREAM_RESULT_DUMP_SIZE_THRESHOLD) { - ASSERT(numOfBlocks == taosArrayGetSize(pRes)); code = doDumpResult(pTask, pItem, pRes, size, totalSize, totalBlocks); // todo: here we need continue retry to put it into output buffer if (code != TSDB_CODE_SUCCESS) { @@ -192,7 +199,6 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to } if (numOfBlocks > 0) { - ASSERT(numOfBlocks == taosArrayGetSize(pRes)); code = doDumpResult(pTask, pItem, pRes, size, totalSize, totalBlocks); } else { taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); @@ -277,7 +283,10 @@ static SScanhistoryDataInfo buildScanhistoryExecRet(EScanHistoryCode code, int32 } SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) { - ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE); + if(pTask->info.taskLevel != TASK_LEVEL__SOURCE) { + stError("s-task:%s not source scan-history task, not exec, quit", pTask->id.idStr); + return buildScanhistoryExecRet(TASK_SCANHISTORY_QUIT, 0); + } void* exec = pTask->exec.pExecutor; bool finished = false; @@ -374,10 +383,16 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) { // It must be halted for a source stream task, since when the related scan-history-data task start scan the history // for the step 2. if (pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) { - ASSERT(status == TASK_STATUS__HALT || status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP); + if (!(status == TASK_STATUS__HALT || status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP)) { + stError("s-task:%s invalid task status:%d", id, status); + return TSDB_CODE_STREAM_INTERNAL_ERROR; + } } else { - ASSERT(status == TASK_STATUS__READY || status == TASK_STATUS__PAUSE || status == TASK_STATUS__DROPPING || - status == TASK_STATUS__STOP); + if (!(status == TASK_STATUS__READY || status == TASK_STATUS__PAUSE || status == TASK_STATUS__DROPPING || + status == TASK_STATUS__STOP)) { + stError("s-task:%s invalid task status:%d", id, status); + return TSDB_CODE_STREAM_INTERNAL_ERROR; + } code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT); if (code != TSDB_CODE_SUCCESS) { stError("s-task:%s halt stream task:%s failed, code:%s not transfer state to stream task", id, @@ -438,7 +453,10 @@ int32_t streamTransferStatePrepare(SStreamTask* pTask) { int32_t code = TSDB_CODE_SUCCESS; SStreamMeta* pMeta = pTask->pMeta; - ASSERT(pTask->status.appendTranstateBlock == 1); + if (pTask->status.appendTranstateBlock != 1) { + stError("s-task:%s not set appendTransBlock flag, internal error", pTask->id.idStr); + return TSDB_CODE_STREAM_INTERNAL_ERROR; + } int32_t level = pTask->info.taskLevel; if (level == TASK_LEVEL__AGG || level == TASK_LEVEL__SOURCE) { // do transfer task operator states. @@ -476,14 +494,16 @@ static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int code = qSetMultiStreamInput(pExecutor, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { - ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE); const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)pInput; code = qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); stDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, id, pSubmit, pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver); - ASSERT((*pVer) <= pSubmit->submit.ver); - (*pVer) = pSubmit->submit.ver; - + if ((*pVer) > pSubmit->submit.ver) { + stError("s-task:%s invalid recorded ver:%" PRId64 " greater than new block ver:%" PRId64 ", not update", id, + *pVer, pSubmit->submit.ver); + } else { + (*pVer) = pSubmit->submit.ver; + } } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { const SStreamDataBlock* pBlock = (const SStreamDataBlock*)pInput; @@ -500,8 +520,13 @@ static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int stDebug("s-task:%s %p set (merged) submit blocks as a batch, numOfBlocks:%d, ver:%" PRId64, id, pTask, numOfBlocks, pMerged->ver); code = qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT); - ASSERT((*pVer) <= pMerged->ver); - (*pVer) = pMerged->ver; + + if ((*pVer) > pMerged->ver) { + stError("s-task:%s invalid recorded ver:%" PRId64 " greater than new block ver:%" PRId64 ", not update", id, + *pVer, pMerged->ver); + } else { + (*pVer) = pMerged->ver; + } } else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) { const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)pInput; @@ -512,7 +537,8 @@ static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int code = qSetMultiStreamInput(pExecutor, pCheckpoint->blocks, 1, pItem->type); } else { - ASSERT(0); + stError("s-task:%s invalid input block type:%d, discard", id, pItem->type); + code = TSDB_CODE_STREAM_INTERNAL_ERROR; } return code; @@ -542,7 +568,6 @@ void streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) stDebug("s-task:%s add transfer-state block into outputQ", id); } else { stDebug("s-task:%s all upstream tasks send transfer-state block, add transfer-state block into outputQ", id); - ASSERT(pTask->streamTaskId.taskId != 0 && pTask->info.fillHistory == 1); } // agg task should dispatch trans-state msg to sink task, to flush all data to sink task. @@ -560,7 +585,6 @@ void streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) } else { // non-dispatch task, do task state transfer directly streamFreeQitem((SStreamQueueItem*)pBlock); stDebug("s-task:%s non-dispatch task, level:%d start to transfer state directly", id, level); - ASSERT(pTask->info.fillHistory == 1); code = streamTransferStatePrepare(pTask); if (code != TSDB_CODE_SUCCESS) { @@ -606,7 +630,11 @@ static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, i } // update the currentVer if processing the submit blocks. - ASSERT(pInfo->checkpointVer <= pInfo->nextProcessVer && ver >= pInfo->checkpointVer); + if (!(pInfo->checkpointVer <= pInfo->nextProcessVer && ver >= pInfo->checkpointVer)) { + stError("s-task:%s invalid info, checkpointVer:%" PRId64 ", nextProcessVer:%" PRId64 " currentVer:%" PRId64, id, + pInfo->checkpointVer, pInfo->nextProcessVer, ver); + return; + } if (ver != pInfo->processedVer) { stDebug("s-task:%s update processedVer(unsaved) from %" PRId64 " to %" PRId64 " nextProcessVer:%" PRId64 @@ -622,8 +650,6 @@ void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointB // 1. transfer the ownership of executor state bool dropRelHTask = (streamTaskGetPrevStatus(pTask) == TASK_STATUS__HALT); if (dropRelHTask) { - ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask)); - STaskId* pHTaskId = &pTask->hTaskInfo.id; SStreamTask* pHTask = NULL; int32_t code = streamMetaAcquireTask(pTask->pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask); @@ -692,12 +718,10 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { EExtractDataCode ret = streamTaskGetDataFromInputQ(pTask, &pInput, &numOfBlocks, &blockSize); if (ret == EXEC_AFTER_IDLE) { - ASSERT(pInput == NULL && numOfBlocks == 0); streamTaskSetIdleInfo(pTask, MIN_INVOKE_INTERVAL); return 0; } else { if (pInput == NULL) { - ASSERT(numOfBlocks == 0); return 0; } } @@ -718,7 +742,10 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { } if (pTask->info.taskLevel == TASK_LEVEL__SINK) { - ASSERT(type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__CHECKPOINT); + if (type != STREAM_INPUT__DATA_BLOCK && type != STREAM_INPUT__CHECKPOINT) { + stError("s-task:%s invalid block type:%d for sink task, discard", id, type); + continue; + } int64_t st = taosGetTimestampMs(); @@ -801,7 +828,11 @@ bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus) { } void streamResumeTask(SStreamTask* pTask) { - ASSERT(pTask->status.schedStatus == TASK_SCHED_STATUS__ACTIVE); + if (pTask->status.schedStatus != TASK_SCHED_STATUS__ACTIVE) { + stError("s-task:%s invalid sched status:%d, not resume task", pTask->id.idStr, pTask->status.schedStatus); + return; + } + const char* id = pTask->id.idStr; while (1) { diff --git a/source/libs/stream/src/streamHb.c b/source/libs/stream/src/streamHb.c index 1ef938494e..1fd3106cff 100644 --- a/source/libs/stream/src/streamHb.c +++ b/source/libs/stream/src/streamHb.c @@ -95,6 +95,7 @@ static int32_t doSendHbMsgInfo(SStreamHbMsg* pMsg, SStreamMeta* pMeta, SEpSet* p tEncoderInit(&encoder, buf, tlen); if ((code = tEncodeStreamHbMsg(&encoder, pMsg)) < 0) { rpcFreeCont(buf); + tEncoderClear(&encoder); stError("vgId:%d encode stream hb msg failed, code:%s", pMeta->vgId, tstrerror(code)); return TSDB_CODE_FAILED; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index d6f1559578..084eb7b827 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -719,9 +719,13 @@ int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTa } int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta) { - size_t size = taosHashGetSize(pMeta->pTasksMap); - ASSERT(taosArrayGetSize(pMeta->pTaskList) == taosHashGetSize(pMeta->pTasksMap)); - return (int32_t)size; + int32_t size = (int32_t)taosHashGetSize(pMeta->pTasksMap); + int32_t sizeInList = taosArrayGetSize(pMeta->pTaskList); + if (sizeInList != size) { + stError("vgId:%d tasks number not consistent in list:%d and map:%d, ", pMeta->vgId, sizeInList, size); + } + + return size; } int32_t streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, SStreamTask** pTask) { @@ -775,7 +779,10 @@ static void doRemoveIdFromList(SArray* pTaskList, int32_t num, SStreamTaskId* id break; } } - ASSERT(remove); + + if (!remove) { + stError("s-task:0x%x not in meta task list, internal error", id->taskId); + } } static int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask, void* param) { @@ -787,6 +794,7 @@ static int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask, void* param) { int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { SStreamTask* pTask = NULL; + int32_t vgId = pMeta->vgId; // pre-delete operation streamMetaWLock(pMeta); @@ -799,19 +807,19 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t // desc the paused task counter if (streamTaskShouldPause(pTask)) { int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1); - stInfo("vgId:%d s-task:%s drop stream task. pause task num:%d", pMeta->vgId, pTask->id.idStr, num); + stInfo("vgId:%d s-task:%s drop stream task. pause task num:%d", vgId, pTask->id.idStr, num); } // handle the dropping event (void)streamTaskHandleEventAsync(pTask->status.pSM, TASK_EVENT_DROPPING, streamTaskSendTransSuccessMsg, NULL); } else { - stDebug("vgId:%d failed to find the task:0x%x, it may be dropped already", pMeta->vgId, taskId); + stDebug("vgId:%d failed to find the task:0x%x, it may be dropped already", vgId, taskId); streamMetaWUnLock(pMeta); return 0; } streamMetaWUnLock(pMeta); - stDebug("s-task:0x%x vgId:%d set task status:dropping and start to unregister it", taskId, pMeta->vgId); + stDebug("s-task:0x%x vgId:%d set task status:dropping and start to unregister it", taskId, vgId); while (1) { int32_t timerActive = 0; @@ -845,14 +853,22 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t (void)atomic_sub_fetch_32(&pMeta->numOfStreamTasks, 1); } - (void)taosHashRemove(pMeta->pTasksMap, &id, sizeof(id)); + int32_t code = taosHashRemove(pMeta->pTasksMap, &id, sizeof(id)); doRemoveIdFromList(pMeta->pTaskList, (int32_t)taosArrayGetSize(pMeta->pTaskList), &pTask->id); (void)streamMetaRemoveTask(pMeta, &id); - ASSERT(taosHashGetSize(pMeta->pTasksMap) == taosArrayGetSize(pMeta->pTaskList)); + int32_t size = (int32_t) taosHashGetSize(pMeta->pTasksMap); + int32_t sizeInList = taosArrayGetSize(pMeta->pTaskList); + if (sizeInList != size) { + stError("vgId:%d tasks number not consistent in list:%d and map:%d, ", vgId, sizeInList, size); + } streamMetaWUnLock(pMeta); - ASSERT(pTask->status.timerActive == 0); + int32_t numOfTmr = pTask->status.timerActive; + if (numOfTmr != 0) { + stError("s-task:%s vgId:%d invalid timer Active record:%d, internal error", pTask->id.idStr, vgId, numOfTmr); + } + if (pTask->info.delaySchedParam != 0 && pTask->info.fillHistory == 0) { stDebug("s-task:%s stop schedTimer, and (before) desc ref:%d", pTask->id.idStr, pTask->refCnt); (void)taosTmrStop(pTask->schedInfo.pDelayTimer); @@ -862,7 +878,7 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t streamMetaReleaseTask(pMeta, pTask); } else { - stDebug("vgId:%d failed to find the task:0x%x, it may have been dropped already", pMeta->vgId, taskId); + stDebug("vgId:%d failed to find the task:0x%x, it may have been dropped already", vgId, taskId); streamMetaWUnLock(pMeta); } @@ -1013,7 +1029,10 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { tFreeStreamTask(pTask); STaskId id = streamTaskGetTaskId(pTask); - (void)taosArrayPush(pRecycleList, &id); + void* px = taosArrayPush(pRecycleList, &id); + if (px == NULL) { + stError("s-task:0x%x failed record the task into recycle list due to out of memory", taskId); + } int32_t total = taosArrayGetSize(pRecycleList); stDebug("s-task:0x%x is already dropped, add into recycle list, total:%d", taskId, total); @@ -1034,7 +1053,10 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { continue; } - (void)taosArrayPush(pMeta->pTaskList, &pTask->id); + void* px = taosArrayPush(pMeta->pTaskList, &pTask->id); + if (px == NULL) { + stFatal("s-task:0x%x failed to add into task list due to out of memory", pTask->id.taskId); + } } else { // todo this should replace the existed object put by replay creating stream task msg from mnode stError("s-task:0x%x already added into table meta by replaying WAL, need check", pTask->id.taskId); @@ -1044,7 +1066,7 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { if (taosHashPut(pMeta->pTasksMap, &id, sizeof(id), &pTask, POINTER_BYTES) != 0) { stError("s-task:0x%x failed to put into hashTable, code:%s, continue", pTask->id.taskId, tstrerror(terrno)); - (void)taosArrayPop(pMeta->pTaskList); + void* px = taosArrayPop(pMeta->pTaskList); tFreeStreamTask(pTask); continue; } @@ -1056,8 +1078,6 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { if (streamTaskShouldPause(pTask)) { (void)atomic_add_fetch_32(&pMeta->numOfPausedTasks, 1); } - - ASSERT(pTask->status.downstreamReady == 0); } tdbFree(pKey); @@ -1075,7 +1095,6 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { } int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); - ASSERT(pMeta->numOfStreamTasks <= numOfTasks && pMeta->numOfPausedTasks <= numOfTasks); stDebug("vgId:%d load %d tasks into meta from disk completed, streamTask:%d, paused:%d", pMeta->vgId, numOfTasks, pMeta->numOfStreamTasks, pMeta->numOfPausedTasks); diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 3703ed07aa..eca728b2d5 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -90,7 +90,6 @@ void streamQueueNextItem(SStreamQueue* pQueue, SStreamQueueItem** pItem) { int8_t flag = atomic_exchange_8(&pQueue->status, STREAM_QUEUE__PROCESSING); if (flag == STREAM_QUEUE__FAILED) { - ASSERT(pQueue->qItem != NULL); *pItem = streamQueueCurItem(pQueue); } else { pQueue->qItem = NULL; @@ -105,13 +104,20 @@ void streamQueueNextItem(SStreamQueue* pQueue, SStreamQueueItem** pItem) { } void streamQueueProcessSuccess(SStreamQueue* queue) { - ASSERT(atomic_load_8(&queue->status) == STREAM_QUEUE__PROCESSING); + if (atomic_load_8(&queue->status) != STREAM_QUEUE__PROCESSING) { + stError("invalid queue status:%d, expect:%d", atomic_load_8(&queue->status), STREAM_QUEUE__PROCESSING); + return; + } + queue->qItem = NULL; atomic_store_8(&queue->status, STREAM_QUEUE__SUCESS); } void streamQueueProcessFail(SStreamQueue* queue) { - ASSERT(atomic_load_8(&queue->status) == STREAM_QUEUE__PROCESSING); + if (atomic_load_8(&queue->status) != STREAM_QUEUE__PROCESSING) { + stError("invalid queue status:%d, expect:%d", atomic_load_8(&queue->status), STREAM_QUEUE__PROCESSING); + return; + } atomic_store_8(&queue->status, STREAM_QUEUE__FAILED); } @@ -229,7 +235,6 @@ EExtractDataCode streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueIte } } else { if (*pInput == NULL) { - ASSERT((*numOfBlocks) == 0); *pInput = qItem; } else { // merge current block failed, let's handle the already merged blocks. void* newRet = NULL; @@ -340,7 +345,8 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) double size = SIZE_IN_MiB(taosQueueMemorySize(pQueue)); stDebug("s-task:%s data res enqueue, current(blocks:%d, size:%.2fMiB)", pTask->id.idStr, total, size); } else { - ASSERT(0); + stError("s-task:%s invalid type:%d to put in inputQ", pTask->id.idStr, type); + return TSDB_CODE_INVALID_PARA; } if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && type != STREAM_INPUT__CHECKPOINT_TRIGGER && diff --git a/source/libs/stream/src/streamStartHistory.c b/source/libs/stream/src/streamStartHistory.c index e68c088b9a..a8c76d3f52 100644 --- a/source/libs/stream/src/streamStartHistory.c +++ b/source/libs/stream/src/streamStartHistory.c @@ -111,11 +111,15 @@ void streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration) { } int32_t streamTaskStartScanHistory(SStreamTask* pTask) { - int32_t level = pTask->info.taskLevel; - ETaskStatus status = streamTaskGetStatus(pTask).state; + int32_t level = pTask->info.taskLevel; + SStreamTaskState state = streamTaskGetStatus(pTask); - ASSERT((pTask->status.downstreamReady == 1) && (status == TASK_STATUS__SCAN_HISTORY) && - (pTask->info.fillHistory == 1)); + if (((pTask->status.downstreamReady != 1) || (state.state != TASK_STATUS__SCAN_HISTORY) || + (pTask->info.fillHistory != 1))) { + stFatal("s-task:%s invalid status:%s to start fill-history task, downReady:%d, is-fill-history task:%d", + pTask->id.idStr, state.name, pTask->status.downstreamReady, pTask->info.fillHistory); + return TSDB_CODE_STREAM_INTERNAL_ERROR; + } if (level == TASK_LEVEL__SOURCE) { return doStartScanHistoryTask(pTask); @@ -144,7 +148,6 @@ int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask) { } SStreamTaskState p = streamTaskGetStatus(pTask); - ASSERT(p.state == TASK_STATUS__READY); int8_t schedStatus = pTask->status.schedStatus; if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { @@ -171,8 +174,6 @@ int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask) { if (code == 0) { SStreamTaskState p = streamTaskGetStatus(pTask); - ASSERT((p.state == TASK_STATUS__SCAN_HISTORY) && (pTask->info.fillHistory == 1)); - stDebug("s-task:%s fill-history task enters into scan-history data stage, status:%s", pTask->id.idStr, p.name); code = streamTaskStartScanHistory(pTask); } @@ -348,8 +349,6 @@ void tryLaunchHistoryTask(void* param, void* tmrId) { } if (streamTaskShouldStop(*ppTask)) { - ASSERT((*ppTask)->status.timerActive >= 1); - char* p = streamTaskGetStatus(*ppTask).name; int32_t ref = atomic_sub_fetch_32(&(*ppTask)->status.timerActive, 1); stDebug("s-task:%s status:%s should stop, quit launch fill-history task timer, retry:%d, ref:%d", @@ -385,7 +384,10 @@ void tryLaunchHistoryTask(void* param, void* tmrId) { notRetryLaunchFillHistoryTask(pTask, pInfo, now); } else { // not reach the limitation yet, let's continue retrying launch related fill-history task. streamTaskSetRetryInfoForLaunch(pHTaskInfo); - ASSERT(pTask->status.timerActive >= 1); + if (pTask->status.timerActive < 1) { + stError("s-task:%s invalid timerActive recorder:%d, abort timer", pTask->id.idStr, pTask->status.timerActive); + return; + } // abort the timer if intend to stop task SStreamTask* pHTask = NULL; @@ -451,8 +453,6 @@ int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) { int32_t hTaskId = pTask->hTaskInfo.id.taskId; SLaunchHTaskInfo* pInfo = NULL; - ASSERT(hTaskId != 0); - stWarn("s-task:%s vgId:%d failed to launch history task:0x%x, since not built yet", idStr, pMeta->vgId, hTaskId); STaskId id = streamTaskGetTaskId(pTask); @@ -480,11 +480,18 @@ int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) { return terrno; } - ASSERT(ref >= 1); + if (ref < 1) { + stError("s-task:%s invalid timerActive recorder:%d, abort timer", pTask->id.idStr, pTask->status.timerActive); + return TSDB_CODE_STREAM_INTERNAL_ERROR; + } stDebug("s-task:%s set timer active flag, ref:%d", idStr, ref); } else { // timer exists - ASSERT(pTask->status.timerActive >= 1); + if (pTask->status.timerActive < 1) { + stError("s-task:%s invalid timerActive recorder:%d, abort timer", pTask->id.idStr, pTask->status.timerActive); + return TSDB_CODE_STREAM_INTERNAL_ERROR; + } + stDebug("s-task:%s set timer active flag, task timer not null", idStr); streamTmrReset(tryLaunchHistoryTask, WAIT_FOR_MINIMAL_INTERVAL, pInfo, streamTimer, &pTask->hTaskInfo.pTimer, pTask->pMeta->vgId, " start-history-task-tmr"); @@ -500,7 +507,11 @@ int32_t streamTaskResetTimewindowFilter(SStreamTask* pTask) { bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t nextProcessVer) { SVersionRange* pRange = &pTask->dataRange.range; - ASSERT(nextProcessVer >= pRange->maxVer); + if (nextProcessVer < pRange->maxVer) { + stError("s-task:%s next processdVer:%"PRId64" is less than range max ver:%"PRId64, pTask->id.idStr, nextProcessVer, + pRange->maxVer); + return true; + } // maxVer for fill-history task is the version, where the last timestamp is acquired. // it's also the maximum version to scan data in tsdb. @@ -538,7 +549,11 @@ int32_t streamTaskSetRangeStreamCalc(SStreamTask* pTask) { return TSDB_CODE_SUCCESS; } else { - ASSERT(pTask->info.fillHistory == 0); + if (pTask->info.fillHistory != 0) { + stError("s-task:%s task should not be fill-history task, internal error", pTask->id.idStr); + return TSDB_CODE_STREAM_INTERNAL_ERROR; + } + if (pTask->info.taskLevel >= TASK_LEVEL__AGG) { return TSDB_CODE_SUCCESS; } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 3b7b999dcb..0791784656 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -134,8 +134,9 @@ int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool return code; } - if (fillHistory) { - ASSERT(hasFillhistory); + if (fillHistory && !hasFillhistory) { + stError("s-task:0x%x create task failed, due to inconsistent fill-history flag", pTask->id.taskId); + return TSDB_CODE_INVALID_PARA; } epsetAssign(&(pTask->info.mnodeEpset), pEpset); @@ -728,8 +729,11 @@ void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId) { if ((pInfo != NULL) && pInfo->dataAllowed) { pInfo->dataAllowed = false; - int32_t t = atomic_add_fetch_32(&pTask->upstreamInfo.numOfClosed, 1); - ASSERT(t <= streamTaskGetNumOfUpstream(pTask)); + if (pTask->upstreamInfo.numOfClosed < streamTaskGetNumOfUpstream(pTask)) { + int32_t t = atomic_add_fetch_32(&pTask->upstreamInfo.numOfClosed, 1); + } else { + stError("s-task:%s not inc closed input, since they have been all closed already", pTask->id.idStr); + } } } @@ -739,7 +743,7 @@ void streamTaskOpenUpstreamInput(SStreamTask* pTask, int32_t taskId) { if (pInfo != NULL && (!pInfo->dataAllowed)) { int32_t t = atomic_sub_fetch_32(&pTask->upstreamInfo.numOfClosed, 1); - ASSERT(t >= 0); + stDebug("s-task:%s open inputQ for upstream:0x%x, remain closed:%d", pTask->id.idStr, taskId, t); pInfo->dataAllowed = true; } } @@ -775,8 +779,6 @@ int8_t streamTaskSetSchedStatusActive(SStreamTask* pTask) { int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask) { streamMutexLock(&pTask->lock); int8_t status = pTask->status.schedStatus; - ASSERT(status == TASK_SCHED_STATUS__WAITING || status == TASK_SCHED_STATUS__ACTIVE || - status == TASK_SCHED_STATUS__INACTIVE); pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; streamMutexUnlock(&pTask->lock); @@ -868,6 +870,7 @@ int32_t streamSendChkptReportMsg(SStreamTask* pTask, SCheckpointInfo* pCheckpoin tEncoderInit(&encoder, buf, tlen); if ((code = tEncodeStreamTaskChkptReport(&encoder, &req)) < 0) { rpcFreeCont(buf); + tEncoderClear(&encoder); stError("s-task:%s vgId:%d encode stream task checkpoint-report msg failed, code:%s", id, vgId, tstrerror(code)); return -1; } @@ -892,8 +895,6 @@ void streamTaskInitForLaunchHTask(SHistoryTaskInfo* pInfo) { } void streamTaskSetRetryInfoForLaunch(SHistoryTaskInfo* pInfo) { - ASSERT(pInfo->tickCount == 0); - pInfo->waitInterval *= RETRY_LAUNCH_INTERVAL_INC_RATE; pInfo->tickCount = ceil(pInfo->waitInterval / WAIT_FOR_MINIMAL_INTERVAL); pInfo->retryTimes += 1; @@ -1021,6 +1022,7 @@ int32_t streamTaskSendCheckpointReq(SStreamTask* pTask) { tEncoderInit(&encoder, buf, tlen); if ((code = tEncodeStreamTaskCheckpointReq(&encoder, &req)) < 0) { rpcFreeCont(buf); + tEncoderClear(&encoder); stError("s-task:%s vgId:%d encode stream task req checkpoint msg failed, code:%s", id, vgId, tstrerror(code)); return -1; } diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index cbaccff01d..968a737ec7 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -191,10 +191,9 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName, GET_EVT_NAME(pEvtInfo->event), pSM->current.name); // remove it - (void) taosArrayPop(pSM->pWaitingEventList); + void* px = taosArrayPop(pSM->pWaitingEventList); STaskStateTrans* pNextTrans = streamTaskFindTransform(pSM->current.state, pEvtInfo->event); - ASSERT(pSM->pActiveTrans == NULL && pNextTrans != NULL); pSM->pActiveTrans = pNextTrans; pSM->startTs = taosGetTimestampMs(); diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 8e32822fb7..3f7210d3c9 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -455,7 +455,7 @@ int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) { + if (tStartEncode(&encoder) != 0) { code = TSDB_CODE_FAILED; QUERY_CHECK_CODE(code, lino, _end); } @@ -551,10 +551,10 @@ int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, tEndEncode(&encoder); int32_t tlen = encoder.pos; - tEncoderClear(&encoder); *pLen = tlen; _end: + tEncoderClear(&encoder); if (code != TSDB_CODE_SUCCESS) { uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 6096f69ca2..d19877dbf1 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -507,6 +507,7 @@ static bool httpFailFastShoudIgnoreMsg(SHashObj* pTable, char* server, int16_t p } } static void httpFailFastMayUpdate(SHashObj* pTable, char* server, int16_t port, int8_t succ) { + int32_t code = 0; char buf[256] = {0}; sprintf(buf, "%s:%d", server, port); @@ -514,7 +515,9 @@ static void httpFailFastMayUpdate(SHashObj* pTable, char* server, int16_t port, (void)taosHashRemove(pTable, buf, strlen(buf)); } else { int32_t st = taosGetTimestampSec(); - (void)taosHashPut(pTable, buf, strlen(buf), &st, sizeof(st)); + if ((code = taosHashPut(pTable, buf, strlen(buf), &st, sizeof(st))) != 0) { + tError("http-report failed to update conn status, dst:%s, reason:%s", buf, tstrerror(code)); + } } return; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 073d6c0f17..dfa9595eb0 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -332,6 +332,21 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { transQueueClear(&conn->cliMsgs); memset(&conn->ctx, 0, sizeof(conn->ctx)); } +void cliResetTimer(SCliThrd* pThrd, SCliConn* conn) { + if (conn->timer) { + if (uv_is_active((uv_handle_t*)conn->timer)) { + tDebug("%s conn %p stop timer", CONN_GET_INST_LABEL(conn), conn); + (void)uv_timer_stop(conn->timer); + } + if (taosArrayPush(pThrd->timerList, &conn->timer) == NULL) { + tError("failed to push timer %p to list, reason:%s", conn->timer, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + conn->timer = NULL; + return; + } + conn->timer->data = NULL; + conn->timer = NULL; + } +} bool cliMaySendCachedMsg(SCliConn* conn) { if (!transQueueEmpty(&conn->cliMsgs)) { SCliMsg* pCliMsg = NULL; @@ -376,15 +391,7 @@ void cliHandleResp(SCliConn* conn) { SCliThrd* pThrd = conn->hostThrd; STrans* pTransInst = pThrd->pTransInst; - if (conn->timer) { - if (uv_is_active((uv_handle_t*)conn->timer)) { - tDebug("%s conn %p stop timer", CONN_GET_INST_LABEL(conn), conn); - (void)uv_timer_stop(conn->timer); - } - (void)taosArrayPush(pThrd->timerList, &conn->timer); - conn->timer->data = NULL; - conn->timer = NULL; - } + cliResetTimer(pThrd, conn); STransMsgHead* pHead = NULL; @@ -593,8 +600,8 @@ void cliConnTimeout(uv_timer_t* handle) { (void)uv_timer_stop(handle); handle->data = NULL; - (void)taosArrayPush(pThrd->timerList, &conn->timer); - conn->timer = NULL; + + cliResetTimer(pThrd, conn); cliMayUpdateFqdnCache(pThrd->fqdn2ipCache, conn->dstAddr); cliHandleFastFail(conn, UV_ECANCELED); @@ -643,13 +650,16 @@ void* destroyConnPool(SCliThrd* pThrd) { } static SCliConn* getConnFromPool(SCliThrd* pThrd, char* key, bool* exceed) { + int32_t code = 0; void* pool = pThrd->pool; STrans* pTranInst = pThrd->pTransInst; size_t klen = strlen(key); SConnList* plist = taosHashGet((SHashObj*)pool, key, klen); if (plist == NULL) { SConnList list = {0}; - (void)taosHashPut((SHashObj*)pool, key, klen, (void*)&list, sizeof(list)); + if ((code = taosHashPut((SHashObj*)pool, key, klen, (void*)&list, sizeof(list))) != 0) { + return NULL; + } plist = taosHashGet(pool, key, klen); SMsgList* nList = taosMemoryCalloc(1, sizeof(SMsgList)); @@ -686,13 +696,17 @@ static SCliConn* getConnFromPool(SCliThrd* pThrd, char* key, bool* exceed) { } static SCliConn* getConnFromPool2(SCliThrd* pThrd, char* key, SCliMsg** pMsg) { + int32_t code = 0; void* pool = pThrd->pool; STrans* pTransInst = pThrd->pTransInst; size_t klen = strlen(key); SConnList* plist = taosHashGet((SHashObj*)pool, key, klen); if (plist == NULL) { SConnList list = {0}; - (void)taosHashPut((SHashObj*)pool, key, klen, (void*)&list, sizeof(list)); + if ((code = taosHashPut((SHashObj*)pool, key, klen, (void*)&list, sizeof(list))) != 0) { + tError("failed to put key %s to pool, reason:%s", key, tstrerror(code)); + return NULL; + } plist = taosHashGet(pool, key, klen); SMsgList* nList = taosMemoryCalloc(1, sizeof(SMsgList)); @@ -805,12 +819,8 @@ static void addConnToPool(void* pool, SCliConn* conn) { } SCliThrd* thrd = conn->hostThrd; - if (conn->timer != NULL) { - (void)uv_timer_stop(conn->timer); - (void)taosArrayPush(thrd->timerList, &conn->timer); - conn->timer->data = NULL; - conn->timer = NULL; - } + cliResetTimer(thrd, conn); + if (T_REF_VAL_GET(conn) > 1) { transUnrefCliHandle(conn); } @@ -1053,12 +1063,7 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { transDQCancel(pThrd->timeoutQueue, conn->task); conn->task = NULL; } - if (conn->timer != NULL) { - (void)uv_timer_stop(conn->timer); - conn->timer->data = NULL; - (void)taosArrayPush(pThrd->timerList, &conn->timer); - conn->timer = NULL; - } + cliResetTimer(pThrd, conn); if (clear) { if (!uv_is_closing((uv_handle_t*)conn->stream)) { @@ -1073,12 +1078,7 @@ static void cliDestroy(uv_handle_t* handle) { } SCliConn* conn = handle->data; SCliThrd* pThrd = conn->hostThrd; - if (conn->timer != NULL) { - (void)uv_timer_stop(conn->timer); - (void)taosArrayPush(pThrd->timerList, &conn->timer); - conn->timer->data = NULL; - conn->timer = NULL; - } + cliResetTimer(pThrd, conn); (void)atomic_sub_fetch_32(&pThrd->connCount, 1); @@ -1385,10 +1385,7 @@ static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd) { uint32_t ipaddr = 0; if ((code = cliGetIpFromFqdnCache(pThrd->fqdn2ipCache, pList->ip, &ipaddr)) != 0) { - (void)uv_timer_stop(conn->timer); - conn->timer->data = NULL; - (void)taosArrayPush(pThrd->timerList, &conn->timer); - conn->timer = NULL; + cliResetTimer(pThrd, conn); cliHandleFastFail(conn, code); return; } @@ -1421,10 +1418,7 @@ static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd) { ret = uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, cliConnCb); if (ret != 0) { - (void)uv_timer_stop(conn->timer); - conn->timer->data = NULL; - (void)taosArrayPush(pThrd->timerList, &conn->timer); - conn->timer = NULL; + cliResetTimer(pThrd, conn); cliMayUpdateFqdnCache(pThrd->fqdn2ipCache, conn->dstAddr); cliHandleFastFail(conn, -1); @@ -1502,7 +1496,10 @@ static void cliHandleFastFail(SCliConn* pConn, int status) { } } else { SFailFastItem item = {.count = 1, .timestamp = cTimestamp}; - (void)taosHashPut(pThrd->failFastCache, pConn->dstAddr, strlen(pConn->dstAddr), &item, sizeof(SFailFastItem)); + int32_t code = taosHashPut(pThrd->failFastCache, pConn->dstAddr, strlen(pConn->dstAddr), &item, sizeof(SFailFastItem)); + if (code != 0) { + tError("failed to put fail-fast item to cache, reason:%s", tstrerror(code)); + } } } } else { @@ -1522,10 +1519,7 @@ void cliConnCb(uv_connect_t* req, int status) { if (pConn->timer == NULL) { timeout = true; } else { - (void)uv_timer_stop(pConn->timer); - pConn->timer->data = NULL; - (void)taosArrayPush(pThrd->timerList, &pConn->timer); - pConn->timer = NULL; + cliResetTimer(pThrd, pConn); } STUB_RAND_NETWORK_ERR(status); @@ -1870,11 +1864,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { uint32_t ipaddr; int32_t code = cliGetIpFromFqdnCache(pThrd->fqdn2ipCache, fqdn, &ipaddr); if (code != 0) { - (void)uv_timer_stop(conn->timer); - conn->timer->data = NULL; - (void)taosArrayPush(pThrd->timerList, &conn->timer); - conn->timer = NULL; - + cliResetTimer(pThrd, conn); cliHandleExcept(conn, code); return; } @@ -1910,10 +1900,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { ret = uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, cliConnCb); if (ret != 0) { - (void)uv_timer_stop(conn->timer); - conn->timer->data = NULL; - (void)taosArrayPush(pThrd->timerList, &conn->timer); - conn->timer = NULL; + cliResetTimer(pThrd, conn); cliMayUpdateFqdnCache(pThrd->fqdn2ipCache, conn->dstAddr); cliHandleFastFail(conn, ret); @@ -2377,7 +2364,9 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); } (void)uv_timer_init(pThrd->loop, timer); - (void)taosArrayPush(pThrd->timerList, &timer); + if (taosArrayPush(pThrd->timerList, &timer) == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + } } pThrd->pool = createConnPool(4); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index a0836eae3d..1758ca65cc 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -375,11 +375,10 @@ void transCtxMerge(STransCtx* dst, STransCtx* src) { STransCtxVal* sVal = (STransCtxVal*)iter; key = taosHashGetKey(sVal, &klen); - // STransCtxVal* dVal = taosHashGet(dst->args, key, klen); - // if (dVal) { - // dst->freeFunc(dVal->val); - // } - (void)taosHashPut(dst->args, key, klen, sVal, sizeof(*sVal)); + int32_t code = taosHashPut(dst->args, key, klen, sVal, sizeof(*sVal)); + if (code != 0) { + tError("failed to put val to hash, reason:%s", tstrerror(code)); + } iter = taosHashIterate(src->args, iter); } taosHashCleanup(src->args); @@ -453,7 +452,9 @@ bool transQueuePush(STransQueue* queue, void* arg) { if (queue->q == NULL) { return true; } - (void)taosArrayPush(queue->q, &arg); + if (taosArrayPush(queue->q, &arg) == NULL) { + return false; + } if (taosArrayGetSize(queue->q) > 1) { return false; } diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index c33e0ee501..3b5c46c2b3 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -787,9 +787,6 @@ void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { STrashElem *pElem = pCacheObj->pTrash; while (pElem) { - T_REF_VAL_CHECK(pElem->pData); - // A S S E R T(pElem->next != pElem && pElem->prev != pElem); - if (force || (T_REF_VAL_GET(pElem->pData) == 0)) { uDebug("cache:%s, key:%p, %p removed from trashcan. numOfElem in trashcan:%d", pCacheObj->name, pElem->pData->key, pElem->pData->data, pCacheObj->numOfElemsInTrash - 1); diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 6e8ad547e1..c6da1537f5 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -385,10 +385,6 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy (void)taosThreadMutexUnlock(&pCfg->lock); } - if (code != 0){ - uError("cfg:%s, type:%s src:%s value:%s failed since %s", pItem->name, cfgDtypeStr(pItem->dtype), - cfgStypeStr(stype), value, tstrerror(code)); - } TAOS_RETURN(code); } @@ -1000,7 +996,7 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) { const char *filepath = ".env"; if (envFile != NULL && strlen(envFile) > 0) { if (!taosCheckExistFile(envFile)) { - uError("failed to load env file:%s", envFile); + (void)printf("failed to load env file:%s\n", envFile); TAOS_RETURN(TSDB_CODE_NOT_FOUND); } filepath = envFile; @@ -1075,7 +1071,7 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { uInfo("failed to load from cfg file %s since %s, use default parameters", filepath, tstrerror(code)); TAOS_RETURN(TSDB_CODE_SUCCESS); } else { - uError("failed to load from cfg file %s since %s", filepath, tstrerror(code)); + (void)printf("failed to load from cfg file %s since %s\n", filepath, tstrerror(code)); TAOS_RETURN(code); } } @@ -1117,7 +1113,10 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { } code = cfgSetItem(pConfig, name, newValue, CFG_STYPE_CFG_FILE, true); - if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) break; + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) { + (void)printf("cfg:%s, value:%s failed since %s\n", name,newValue, tstrerror(code)); + break; + } } else { (void)paGetToken(value + vlen + 1, &value2, &vlen2); if (vlen2 != 0) { @@ -1131,7 +1130,10 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { } code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE, true); - if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) break; + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) { + (void)printf("cfg:%s, value:%s failed since %s\n", name, value, tstrerror(code)); + break; + } } if (strcasecmp(name, "dataDir") == 0) { @@ -1154,7 +1156,7 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { uInfo("load from cfg file %s success", filepath); TAOS_RETURN(TSDB_CODE_SUCCESS); } else { - uError("failed to load from cfg file %s since %s", filepath, tstrerror(code)); + (void)printf("failed to load from cfg file %s since %s\n", filepath, tstrerror(code)); TAOS_RETURN(code); } } @@ -1236,7 +1238,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { char *p = strchr(url, ':'); if (p == NULL) { - uError("fail to load apoll url: %s, unknown format", url); + (void)printf("fail to load apoll url: %s, unknown format\n", url); TAOS_RETURN(TSDB_CODE_INVALID_PARA); } p++; @@ -1244,7 +1246,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { if (strncmp(url, "jsonFile", 8) == 0) { char *filepath = p; if (!taosCheckExistFile(filepath)) { - uError("failed to load json file:%s", filepath); + (void)printf("failed to load json file:%s\n", filepath); TAOS_RETURN(TSDB_CODE_NOT_FOUND); } @@ -1256,7 +1258,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { char *buf = taosMemoryMalloc(fileSize + 1); if (!buf) { (void)taosCloseFile(&pFile); - uError("load json file error: %s, failed to alloc memory", filepath); + (void)printf("load json file error: %s, failed to alloc memory\n", filepath); TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } @@ -1264,7 +1266,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { (void)taosLSeekFile(pFile, 0, SEEK_SET); if (taosReadFile(pFile, buf, fileSize) <= 0) { (void)taosCloseFile(&pFile); - uError("load json file error: %s", filepath); + (void)printf("load json file error: %s\n", filepath); taosMemoryFreeClear(buf); TAOS_RETURN(TSDB_CODE_INVALID_DATA_FMT); } @@ -1273,7 +1275,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { if (NULL == pJson) { const char *jsonParseError = tjsonGetError(); if (jsonParseError != NULL) { - uError("load json file parse error: %s", jsonParseError); + (void)printf("load json file parse error: %s\n", jsonParseError); } taosMemoryFreeClear(buf); TAOS_CHECK_EXIT(TSDB_CODE_INVALID_DATA_FMT); @@ -1341,7 +1343,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { // } else if (strncmp(url, "jsonUrl", 7) == 0) { // } else if (strncmp(url, "etcdUrl", 7) == 0) { } else { - uError("Unsupported url: %s", url); + (void)printf("Unsupported url: %s\n", url); TAOS_RETURN(TSDB_CODE_INVALID_PARA); } @@ -1353,7 +1355,7 @@ _exit: taosMemoryFree(cfgLineBuf); tjsonDelete(pJson); if (code != 0) { - uError("failed to load from apollo url:%s at line %d since %s", url, lino, tstrerror(code)); + (void)printf("failed to load from apollo url:%s at line %d since %s\n", url, lino, tstrerror(code)); } TAOS_RETURN(code); } diff --git a/source/util/src/tgeosctx.c b/source/util/src/tgeosctx.c index 805d398275..ddd04b1e88 100644 --- a/source/util/src/tgeosctx.c +++ b/source/util/src/tgeosctx.c @@ -72,11 +72,23 @@ int32_t getThreadLocalGeosCtx(SGeosContext **ppCtx) { } int32_t code = 0, lino = 0; - if (atomic_val_compare_exchange_8(&tlGeosCtxKeyInited, 0, 1) == 0) { + + int8_t old; + int32_t nLoops = 0; + while (1) { + old = atomic_val_compare_exchange_8(&tlGeosCtxKeyInited, 0, 2); + if (old != 2) break; + if (++nLoops > 1000) { + (void)sched_yield(); + nLoops = 0; + } + } + if (old == 0) { if ((taosThreadKeyCreate(&tlGeosCtxKey, destroyThreadLocalGeosCtx)) != 0) { atomic_store_8(&tlGeosCtxKeyInited, 0); TAOS_CHECK_EXIT(TAOS_SYSTEM_ERROR(errno)); } + atomic_store_8(&tlGeosCtxKeyInited, 1); } SGeosContext *tlGeosCtxObj = (SGeosContext *)taosMemoryCalloc(1, sizeof(SGeosContext)); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 3c41e00b7a..49e1d87d80 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -206,7 +206,7 @@ int32_t taosInitSlowLog() { (void)taosUmaskFile(0); tsLogObj.slowHandle->pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); if (tsLogObj.slowHandle->pFile == NULL) { - printf("\nfailed to open slow log file:%s, reason:%s\n", name, strerror(errno)); + (void)printf("\nfailed to open slow log file:%s, reason:%s\n", name, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } @@ -529,7 +529,7 @@ static int32_t taosInitNormalLog(const char *logName, int32_t maxFileNum) { processLogFileName(logName, maxFileNum); char name[PATH_MAX + 50] = "\0"; - sprintf(name, "%s.%d", tsLogObj.logName, tsLogObj.flag); + (void)sprintf(name, "%s.%d", tsLogObj.logName, tsLogObj.flag); (void)taosThreadMutexInit(&tsLogObj.logMutex, NULL); (void)taosUmaskFile(0); @@ -538,7 +538,7 @@ static int32_t taosInitNormalLog(const char *logName, int32_t maxFileNum) { tsLogObj.logHandle->pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE); if (tsLogObj.logHandle->pFile == NULL) { - printf("\nfailed to open log file:%s, reason:%s\n", name, strerror(errno)); + (void)printf("\nfailed to open log file:%s, reason:%s\n", name, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } (void)taosLockLogFile(tsLogObj.logHandle->pFile); @@ -546,18 +546,18 @@ static int32_t taosInitNormalLog(const char *logName, int32_t maxFileNum) { // only an estimate for number of lines int64_t filesize = 0; if (taosFStatFile(tsLogObj.logHandle->pFile, &filesize, NULL) < 0) { - printf("\nfailed to fstat log file:%s, reason:%s\n", name, strerror(errno)); + (void)printf("\nfailed to fstat log file:%s, reason:%s\n", name, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } tsLogObj.lines = (int32_t)(filesize / 60); (void)taosLSeekFile(tsLogObj.logHandle->pFile, 0, SEEK_END); - sprintf(name, "==================================================\n"); + (void)sprintf(name, "==================================================\n"); (void)taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)); - sprintf(name, " new log file \n"); + (void)sprintf(name, " new log file \n"); (void)taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)); - sprintf(name, "==================================================\n"); + (void)sprintf(name, "==================================================\n"); (void)taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)); return 0; diff --git a/source/util/src/tlrucache.c b/source/util/src/tlrucache.c index 24b60e8d13..7faff88155 100644 --- a/source/util/src/tlrucache.c +++ b/source/util/src/tlrucache.c @@ -445,6 +445,9 @@ static LRUStatus taosLRUCacheShardInsert(SLRUCacheShard *shard, const void *key, LRUPriority priority, void *ud) { SLRUEntry *e = taosMemoryCalloc(1, sizeof(SLRUEntry) - 1 + keyLen); if (!e) { + if (deleter) { + (*deleter)(key, keyLen, value, ud); + } return TAOS_LRU_STATUS_FAIL; } diff --git a/source/util/src/tref.c b/source/util/src/tref.c index 4b1b477e06..0eac7b4427 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -455,9 +455,7 @@ static void taosLockList(int64_t *lockedBy) { static void taosUnlockList(int64_t *lockedBy) { int64_t tid = taosGetSelfPthreadId(); - if (atomic_val_compare_exchange_64(lockedBy, tid, 0) != tid) { - ASSERTS(false, "atomic_val_compare_exchange_64 tid failed"); - } + (void)atomic_val_compare_exchange_64(lockedBy, tid, 0); } static void taosInitRefModule(void) { (void)taosThreadMutexInit(&tsRefMutex, NULL); } diff --git a/tests/ci/count_assert.py b/tests/ci/count_assert.py new file mode 100644 index 0000000000..415c197b3f --- /dev/null +++ b/tests/ci/count_assert.py @@ -0,0 +1,53 @@ +import os +import re + +# List of source directories to search +source_dirs = [ + "community/source", + "community/include", + "enterprise/src/plugins/" +] + +# List of directories to exclude +exclude_dirs = [ + "community/source/client/jni" +] + +# List of files to exclude +exclude_source_files = [ + "community/source/libs/parser/src/sql.c", + "community/source/util/src/tlog.c", + "community/include/util/tlog.h" +] + +def grep_asserts_in_file(file_path, summary_list, detaild_list): + """Search for assert, ASSERTS, or ASSERT function calls in a file and print them.""" + match_count = 0 + with open(file_path, 'r') as file: + for line_number, line in enumerate(file, start=1): + if re.search(r'\bassert\(.*\)|\bASSERT\(.*\)|\bASSERTS\(.*\)|\bASSERT_CORE\(.*\)', line): + detaild_list.append(f"{file_path}:{line_number}: {line.strip()}") + match_count += 1 + if match_count > 0: + summary_list.append(f"Total matches in {file_path}: {match_count}") + +def traverse_and_grep(source_dirs, exclude_dirs, exclude_source_files): + """Traverse directories and grep for assert, ASSERTS, or ASSERT function calls in .h and .c files.""" + summary_list = [] + detaild_list = [] + for source_dir in source_dirs: + for root, _, files in os.walk(source_dir): + # Skip directories named 'test' or 'tests' and directories in exclude_dirs + if 'test' in root.split(os.sep) or 'tests' in root.split(os.sep) or any(excluded in root for excluded in exclude_dirs): + continue + for file in files: + if file.endswith((".h", ".c")): + file_path = os.path.join(root, file) + if file_path not in exclude_source_files: + grep_asserts_in_file(file_path, summary_list, detaild_list) + return summary_list, detaild_list + +if __name__ == "__main__": + summary_list, detaild_list = traverse_and_grep(source_dirs, exclude_dirs, exclude_source_files) + print("\n".join(summary_list)) + # print("\n".join(detaild_list)) \ No newline at end of file diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e172e471e7..8c6f9fda15 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -144,6 +144,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery2.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py @@ -227,6 +228,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/basic5.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/ts-4674.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb2.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb3.py diff --git a/tests/system-test/2-query/nestedQuery2.py b/tests/system-test/2-query/nestedQuery2.py new file mode 100644 index 0000000000..dce119b1f2 --- /dev/null +++ b/tests/system-test/2-query/nestedQuery2.py @@ -0,0 +1,167 @@ +from random import randrange +import time +import threading +import secrets +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +# from tmqCommon import * + +class TDTestCase: + updatecfgDict = {'asynclog': 0, 'ttlUnit': 1, 'ttlPushInterval': 5, 'ratioOfVnodeStreamThrea': 4} + + def __init__(self): + self.vgroups = 4 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + self.duraion = '1h' + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def create_database(self, tsql, dbName, dropFlag=1, vgroups=2, replica=1, duration: str = '1d'): + if dropFlag == 1: + tsql.execute("drop database if exists %s" % (dbName)) + + tsql.execute("create database if not exists %s vgroups %d replica %d duration %s" % ( + dbName, vgroups, replica, duration)) + tdLog.debug("complete to create database %s" % (dbName)) + return + + def create_stable(self, tsql, paraDict): + colString = tdCom.gen_column_type_str( + colname_prefix=paraDict["colPrefix"], column_elm_list=paraDict["colSchema"]) + tagString = tdCom.gen_tag_type_str( + tagname_prefix=paraDict["tagPrefix"], tag_elm_list=paraDict["tagSchema"]) + sqlString = f"create table if not exists %s.%s (%s) tags (%s)" % ( + paraDict["dbName"], paraDict["stbName"], colString, tagString) + tdLog.debug("%s" % (sqlString)) + tsql.execute(sqlString) + return + + def create_ctable(self, tsql=None, dbName='dbx', stbName='stb', ctbPrefix='ctb', ctbNum=1, ctbStartIdx=0): + for i in range(ctbNum): + sqlString = "create table %s.%s%d using %s.%s tags(%d, 'tb%d', 'tb%d', %d, %d, %d)" % (dbName, ctbPrefix, i+ctbStartIdx, dbName, stbName, (i+ctbStartIdx) % 5, i+ctbStartIdx + random.randint( + 1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100)) + tsql.execute(sqlString) + + tdLog.debug("complete to create %d child tables by %s.%s" % + (ctbNum, dbName, stbName)) + return + + def init_normal_tb(self, tsql, db_name: str, tb_name: str, rows: int, start_ts: int, ts_step: int): + sql = 'CREATE TABLE %s.%s (ts timestamp, c1 INT, c2 INT, c3 INT, c4 double, c5 VARCHAR(255))' % ( + db_name, tb_name) + tsql.execute(sql) + sql = 'INSERT INTO %s.%s values' % (db_name, tb_name) + for j in range(rows): + sql += f'(%d, %d,%d,%d,{random.random()},"varchar_%d"),' % (start_ts + j * ts_step + randrange(500), j % + 10 + randrange(200), j % 10, j % 10, j % 10 + randrange(100)) + tsql.execute(sql) + + def insert_data(self, tsql, dbName, ctbPrefix, ctbNum, rowsPerTbl, batchNum, startTs, tsStep): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" % dbName) + pre_insert = "insert into " + sql = pre_insert + + for i in range(ctbNum): + rowsBatched = 0 + sql += " %s.%s%d values " % (dbName, ctbPrefix, i) + for j in range(rowsPerTbl): + if (i < ctbNum/2): + sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d') " % (startTs + j*tsStep + randrange( + 500), j % 10 + randrange(100), j % 10 + randrange(200), j % 10, j % 10, j % 10, j % 10, j % 10, j % 10) + else: + sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d') " % ( + startTs + j*tsStep + randrange(500), j % 10, j % 10, j % 10, j % 10, j % 10, j % 10) + rowsBatched += 1 + if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + rowsBatched = 0 + if j < rowsPerTbl - 1: + sql = "insert into %s.%s%d values " % (dbName, ctbPrefix, i) + else: + sql = "insert into " + if sql != pre_insert: + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def init_data(self, db: str = 'test', ctb_num: int = 10, rows_per_ctb: int = 10000, start_ts: int = 1537146000000, ts_step: int = 500): + tdLog.printNoPrefix( + "======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': db, + 'dropFlag': 1, + 'vgroups': 2, + 'stbName': 'meters', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count': 1}, {'type': 'BIGINT', 'count': 1}, {'type': 'FLOAT', 'count': 1}, {'type': 'DOUBLE', 'count': 1}, {'type': 'smallint', 'count': 1}, {'type': 'tinyint', 'count': 1}, {'type': 'bool', 'count': 1}, {'type': 'binary', 'len': 10, 'count': 1}, {'type': 'nchar', 'len': 10, 'count': 1}], + 'tagSchema': [{'type': 'INT', 'count': 1}, {'type': 'nchar', 'len': 20, 'count': 1}, {'type': 'binary', 'len': 20, 'count': 1}, {'type': 'BIGINT', 'count': 1}, {'type': 'smallint', 'count': 1}, {'type': 'DOUBLE', 'count': 1}], + 'ctbPrefix': 't', + 'ctbStartIdx': 0, + 'ctbNum': ctb_num, + 'rowsPerTbl': rows_per_ctb, + 'batchNum': 3000, + 'startTs': start_ts, + 'tsStep': ts_step} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = ctb_num + paraDict['rowsPerTbl'] = rows_per_ctb + + tdLog.info("create database") + self.create_database(tsql=tdSql, dbName=paraDict["dbName"], dropFlag=paraDict["dropFlag"], + vgroups=paraDict["vgroups"], replica=self.replicaVar, duration=self.duraion) + + tdLog.info("create stb") + self.create_stable(tsql=tdSql, paraDict=paraDict) + + tdLog.info("create child tables") + self.create_ctable(tsql=tdSql, dbName=paraDict["dbName"], + stbName=paraDict["stbName"], ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"], ctbStartIdx=paraDict["ctbStartIdx"]) + self.insert_data(tsql=tdSql, dbName=paraDict["dbName"], + ctbPrefix=paraDict["ctbPrefix"], ctbNum=paraDict["ctbNum"], + rowsPerTbl=paraDict["rowsPerTbl"], batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"], tsStep=paraDict["tsStep"]) + self.init_normal_tb(tdSql, paraDict['dbName'], 'norm_tb', + paraDict['rowsPerTbl'], paraDict['startTs'], paraDict['tsStep']) + + def test_select_asterisk_from_subquery_with_duplicate_aliasname(self): + sql = "select * from (select c8 as a, c9 as a from t1 order by ts desc limit 10)t;" + tdSql.query(sql, queryTimes=1) + tdSql.checkData(0, 0, "binary9") + tdSql.checkData(0, 1, "nchar9") + sql = "select * from (select c8 as a, c9 as a, ts from t1 order by ts desc limit 10)t order by ts desc;" + tdSql.query(sql, queryTimes=1) + tdSql.checkData(0, 0, "binary9") + tdSql.checkData(0, 1, "nchar9") + sql = "select * from (select c8 as a, c9 as a, ts, t1 from t1 order by ts desc limit 10)t partition by t1 order by ts desc;" + tdSql.query(sql, queryTimes=1) + tdSql.checkData(0, 0, "binary9") + tdSql.checkData(0, 1, "nchar9") + sql = " select * from (select a.c8, b.c8, a.ts, a.t1,b.t1 from t1 a, t3 b where a.ts = b.ts order by a.ts)ttt" + tdSql.query(sql, queryTimes=1) + + tdSql.checkData(0, 3, 1) + tdSql.checkData(0, 4, 3) + + def run(self): + self.init_data() + self.test_select_asterisk_from_subquery_with_duplicate_aliasname() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/tbname_vgroup.py b/tests/system-test/2-query/tbname_vgroup.py index 564c78b47a..7a76a07e7c 100644 --- a/tests/system-test/2-query/tbname_vgroup.py +++ b/tests/system-test/2-query/tbname_vgroup.py @@ -169,7 +169,7 @@ class TDTestCase: tdSql.checkData(3, 2, 4) tdSql.query("explain select * from st where tbname='ct1'") - tdSql.checkRows(2) + tdSql.checkRows(1) tdSql.query("select table_name, vgroup_id from information_schema.ins_tables where db_name='dbvg' and type='CHILD_TABLE'"); print(tdSql.queryResult); diff --git a/tests/system-test/7-tmq/ts-4674.py b/tests/system-test/7-tmq/ts-4674.py new file mode 100644 index 0000000000..709debaef1 --- /dev/null +++ b/tests/system-test/7-tmq/ts-4674.py @@ -0,0 +1,157 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from taos.tmq import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + clientCfgDict = {'debugFlag': 135} + updatecfgDict = {'debugFlag': 135, 'clientCfg':clientCfgDict} + # updatecfgDict = {'debugFlag': 135, 'clientCfg':clientCfgDict, 'tmqRowSize':1} + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + # def consume_TS_4674_Test(self): + # + # os.system("nohup taosBenchmark -y -B 1 -t 4 -S 1000 -n 1000000 -i 1000 -v 1 -a 3 > /dev/null 2>&1 &") + # time.sleep() + # tdSql.execute(f'create topic topic_all with meta as database test') + # consumer_dict = { + # "group.id": "g1", + # "td.connect.user": "root", + # "td.connect.pass": "taosdata", + # "auto.offset.reset": "earliest", + # } + # consumer = Consumer(consumer_dict) + # + # try: + # consumer.subscribe(["topic_all"]) + # except TmqError: + # tdLog.exit(f"subscribe error") + # + # try: + # while True: + # res = consumer.poll(5) + # if not res: + # print(f"null") + # continue + # val = res.value() + # if val is None: + # print(f"null") + # continue + # cnt = 0; + # for block in val: + # cnt += len(block.fetchall()) + # + # print(f"block {cnt} rows") + # + # finally: + # consumer.close() + + def get_leader(self): + tdLog.debug("get leader") + tdSql.query("show vnodes") + for result in tdSql.queryResult: + if result[3] == 'leader': + tdLog.debug("leader is %d"%(result[0])) + return result[0] + return -1 + + def balance_vnode(self): + leader_before = self.get_leader() + + while True: + leader_after = -1 + tdSql.query("balance vgroup leader") + while True: + leader_after = self.get_leader() + if leader_after != -1 : + break; + else: + time.sleep(1) + if leader_after != leader_before: + tdLog.debug("leader changed") + break; + else : + time.sleep(1) + + + def consume_TS_4674_Test(self): + + tdSql.execute(f'create database if not exists d1 replica 3 vgroups 1') + tdSql.execute(f'use d1') + tdSql.execute(f'create table st(ts timestamp, i int) tags(t int)') + tdSql.execute(f'insert into t1 using st tags(1) values(now, 1) (now+1s, 2)') + tdSql.execute(f'insert into t2 using st tags(2) values(now, 1) (now+1s, 2)') + tdSql.execute(f'insert into t3 using st tags(3) values(now, 1) (now+1s, 2)') + + + tdSql.execute(f'create topic topic_all as select * from st') + consumer_dict = { + "group.id": "g1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.offset.reset": "earliest", + } + consumer = Consumer(consumer_dict) + + try: + consumer.subscribe(["topic_all"]) + except TmqError: + tdLog.exit(f"subscribe error") + + cnt = 0; + balance = False + try: + while True: + res = consumer.poll(2) + if not res: + print(f"null res") + if balance == False and cnt != 6 : + tdLog.exit(f"subscribe num != 6") + if balance == True : + if cnt != 8 : + tdLog.exit(f"subscribe num != 8") + # tdLog.debug(f"subscribe num != 8") + # continue + else : + break + self.balance_vnode() + balance = True + tdSql.execute(f'insert into t1 using st tags(1) values(now+5s, 11) (now+10s, 12)') + continue + val = res.value() + if val is None: + print(f"null val") + continue + for block in val: + cnt += len(block.fetchall()) + + print(f"block {cnt} rows") + + finally: + consumer.close() + def run(self): + self.consume_TS_4674_Test() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase())