diff --git a/config/agent/agent_config.tpl b/config/agent/agent_config.tpl index 16770879..7456f277 100644 --- a/config/agent/agent_config.tpl +++ b/config/agent/agent_config.tpl @@ -16,17 +16,72 @@ POST .infini_configs/_doc/system_ingest_config_yml "location": "system_ingest_config.yml", "content": """ -configs.template: - - name: "default_ingest_config" - path: ./config/ingest_config.tpl - variable: - INGEST_CLUSTER_ID: infini_default_ingest_cluster - INGEST_CLUSTER_ENDPOINT: [ "http://192.168.3.185:8000" ] - INGEST_CLUSTER_USERNAME: "ingest" - INGEST_CLUSTER_PASSWORD: "password" - CLUSTER_VER: "1.6.0" - CLUSTER_DISTRIBUTION: "easysearch" - INDEX_PREFIX: ".infini_" +metrics: + enabled: true + queue: metrics + network: + enabled: true + summary: true + sockets: true + #throughput: true + details: true + memory: + metrics: + - swap + - memory + disk: + metrics: + - iops + - usage + cpu: + metrics: + - idle + - system + - user + - iowait + - load + instance: + enabled: true + +elastic: + availability_check: + enabled: false + +pipeline: + - name: replicate_message_to_gateway + enabled: true + auto_start: true + keep_running: true + processor: + - consumer: + max_worker_size: 1 + queue_selector: + keys: + - metrics + - logs + consumer: + group: replication + processor: + - http: + max_sending_qps: 100 + method: POST + path: /$[[queue_name]]/_doc/ + headers: + Content-Type: application/json + body: $[[message]] + basic_auth: + username: ingest + password: password +# tls: #for mTLS connection with config servers +# enabled: true +# ca_file: /Users/medcl/Desktop/ca.crt +# cert_file: /Users/medcl/Desktop/client.crt +# key_file: /Users/medcl/Desktop/client.key +# skip_insecure_verify: false + schema: "http" + hosts: # receiver endpoint, fallback in order + - "192.168.3.185:8888" + valid_status_code: [200,201] #panic on other status code """, "version": 3 diff --git a/config/agent/init_agent_config.tpl b/config/agent/init_agent_config.tpl index 4693292a..4b0150dd 100644 --- a/config/agent/init_agent_config.tpl +++ b/config/agent/init_agent_config.tpl @@ -71,130 +71,3 @@ pipeline: "version": 1 } } - -#system ingest template -POST .infini_configs/_doc/ingest_config_tpl -{ - "id": "ingest_config_tpl", - "updated": "2023-10-19T14:49:56.768754+08:00", - "metadata": { - "category": "app_settings", - "name": "agent", - "labels": { - "instance": "_all" - } - }, - "payload": { - "name": "ingest_config.tpl", - "location": "ingest_config.tpl", - "content": """ - -elasticsearch: - - name: $[[INGEST_CLUSTER_ID]] - enabled: true - endpoints: $[[INGEST_CLUSTER_ENDPOINT]] - discovery: - enabled: false - basic_auth: - username: $[[INGEST_CLUSTER_USERNAME]] - password: $[[INGEST_CLUSTER_PASSWORD]] - traffic_control: - enabled: true - max_qps_per_node: 1000 - max_bytes_per_node: 10485760 - max_connection_per_node: 10 - -metrics: - enabled: true - queue: metrics - network: - enabled: true - summary: true - sockets: true - #throughput: true - details: true - memory: - metrics: - - swap - - memory - disk: - metrics: - - iops - - usage - cpu: - metrics: - - idle - - system - - user - - iowait - - load - instance: - enabled: true - -elastic: - availability_check: - enabled: false - -pipeline: - - name: merge_logs - auto_start: true - keep_running: true - retry_delay_in_ms: 10000 - processor: - - indexing_merge: - elasticsearch: "$[[INGEST_CLUSTER_ID]]" - index_name: ".infini_logs" - type_name: "_doc" - input_queue: "logs" - idle_timeout_in_seconds: 10 - output_queue: - name: "merged_requests" - worker_size: 1 - bulk_size_in_mb: 5 - - name: merge_metrics - auto_start: true - keep_running: true - retry_delay_in_ms: 10000 - processor: - - indexing_merge: - elasticsearch: "$[[INGEST_CLUSTER_ID]]" - index_name: ".infini_metrics" - type_name: "_doc" - input_queue: "metrics" - output_queue: - name: "merged_requests" - worker_size: 1 - bulk_size_in_mb: 5 - - name: ingest_merged_requests - enabled: true - auto_start: true - keep_running: true - retry_delay_in_ms: 10000 - processor: - - bulk_indexing: - max_worker_size: 1 - verbose_bulk_result: false - bulk: - batch_size_in_mb: 5 - batch_size_in_docs: 5000 - max_retry_times: 0 - invalid_queue: "" - response_handle: - include_index_stats: false - include_action_stats: false - output_bulk_stats: false - include_error_details: true - save_error_results: true - save_success_results: false - save_busy_results: false - consumer: - fetch_max_messages: 5 - queues: - type: indexing_merge - when: - cluster_available: ["$[[INGEST_CLUSTER_ID]]"] -""", - "version": 1 - } -} - diff --git a/config/agent/init_gateway_config.tpl b/config/agent/init_gateway_config.tpl index 915ed2fb..6c275559 100644 --- a/config/agent/init_gateway_config.tpl +++ b/config/agent/init_gateway_config.tpl @@ -13,80 +13,77 @@ POST .infini_configs/_doc/agent_relay_gateway_config_yml "name": "agent_relay_gateway_config.yml", "location": "agent_relay_gateway_config.yml", "content": """ -env: - CLUSTER_ENDPOINTS: ["http://10.0.0.3:7102"] - CLUSTER_USERNAME: admin - CLUSTER_PASSWORD: admin path.data: data path.logs: log +allow_multi_instance: true +configs.auto_reload: false + entry: - name: my_es_entry enabled: true router: my_router max_concurrency: 200000 network: - binding: 0.0.0.0:8000 + binding: 0.0.0.0:8888 +# tls: #for mTLS connection with config servers +# enabled: true +# ca_file: /Users/medcl/Desktop/ca.crt +# cert_file: /Users/medcl/Desktop/server.crt +# key_file: /Users/medcl/Desktop/server.key +# skip_insecure_verify: false flow: - - name: async_bulk + - name: deny_flow + filter: + - set_response: + body: "request not allowed" + status: 500 + - name: ingest_flow filter: - basic_auth: valid_users: ingest: password + - rewrite_to_bulk: + type_removed: false + - bulk_request_mutate: + fix_null_id: true + generate_enhanced_id: true +# fix_null_type: true +# default_type: m-type +# default_index: m-index + index_rename: + metrics: ".infini_metrics" + logs: ".infini_logs" - bulk_reshuffle: when: contains: _ctx.request.path: /_bulk elasticsearch: prod - level: cluster - partition_size: 3 + level: node + partition_size: 1 fix_null_id: true - - elasticsearch: - elasticsearch: prod #elasticsearch configure reference name - max_connection_per_node: 1000 #max tcp connection to upstream, default for all nodes - max_response_size: -1 #default for all nodes - balancer: weight - refresh: # refresh upstream nodes list, need to enable this feature to use elasticsearch nodes auto discovery - enabled: true - interval: 60s - filter: - roles: - exclude: - - master router: - name: my_router - default_flow: async_bulk - + default_flow: deny_flow + rules: + - method: + - "POST" + enabled: true + pattern: + - "/{any_index}/_doc/" + flow: + - ingest_flow elasticsearch: - name: prod enabled: true - endpoints: $[[env.CLUSTER_ENDPOINTS]] - discovery: - enabled: false basic_auth: - username: $[[env.CLUSTER_USERNAME]] - password: $[[env.CLUSTER_PASSWORD]] - traffic_control: - enabled: true - max_qps_per_node: 100 - max_bytes_per_node: 10485760 - max_connection_per_node: 5 - -elastic: - enabled: true - remote_configs: false - elasticsearch: prod - metadata_refresh: - enabled: true - interval: 30s - discovery: - enabled: true - refresh: - enabled: true - interval: 30s + username: admin + password: admin + endpoints: + - https://10.0.0.3:9200 pipeline: - name: bulk_request_ingest