diff --git a/config/initialization.tpl b/config/initialization.tpl index 27953449..1f710465 100644 --- a/config/initialization.tpl +++ b/config/initialization.tpl @@ -5709,6 +5709,258 @@ POST $[[INDEX_PREFIX]]layout/_doc/cicmhbt3q95ich72lrvg "is_fixed": true } -GET / +#agent + + +POST .infini_configs/_doc/task_config_tpl +{ + "id": "task_config_tpl", + "updated": "2023-10-19T14:49:56.768754+08:00", + "metadata": { + "category": "app_settings", + "name": "agent", + "labels": { + "instance": "_all" + } + }, + "payload": { + "name": "task_config.tpl", + "location": "task_config.tpl", + "content": """env: + CLUSTER_PASSWORD: $[[keystore.$[[CLUSTER_ID]]_password]] + +elasticsearch: + - id: $[[CLUSTER_ID]] + name: $[[CLUSTER_ID]] + enabled: true + endpoints: $[[CLUSTER_ENDPOINT]] + discovery: + enabled: false + basic_auth: + username: $[[CLUSTER_USERNAME]] + password: $[[CLUSTER_PASSWORD]] + +pipeline: +#clsuter level metrics +- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + keep_running: true + singleton: true + name: collect_$[[CLUSTER_ID]]_es_cluster_stats + retry_delay_in_ms: 10000 + processor: + - es_cluster_stats: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + when: + cluster_available: ["$[[CLUSTER_ID]]"] + +- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + keep_running: true + singleton: true + name: collect_$[[CLUSTER_ID]]_es_index_stats + retry_delay_in_ms: 10000 + processor: + - es_index_stats: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + when: + cluster_available: ["$[[CLUSTER_ID]]"] + +- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]] + keep_running: true + singleton: true + name: collect_$[[CLUSTER_ID]]_es_cluster_health + retry_delay_in_ms: 10000 + processor: + - es_cluster_health: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + when: + cluster_available: ["$[[CLUSTER_ID]]"] + +#node level metrics +- auto_start: $[[NODE_LEVEL_TASKS_ENABLED]] + enabled: $[[NODE_LEVEL_TASKS_ENABLED]] + keep_running: true + name: collect_$[[CLUSTER_ID]]_es_node_stats + retry_delay_in_ms: 10000 + processor: + - es_node_stats: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + when: + cluster_available: ["$[[CLUSTER_ID]]"] + +#node logs +- auto_start: $[[NODE_LEVEL_TASKS_ENABLED]] + enabled: $[[NODE_LEVEL_TASKS_ENABLED]] + keep_running: true + name: collect_$[[CLUSTER_ID]]_es_logs + retry_delay_in_ms: 10000 + processor: + - es_logs_processor: + elasticsearch: $[[CLUSTER_ID]] + labels: + cluster_id: $[[CLUSTER_ID]] + logs_path: $[[NODE_LOGS_PATH]] + queue_name: logs + when: + cluster_available: ["$[[CLUSTER_ID]]"] +""", + "version": 1 + } +} + +POST .infini_configs/_doc/ingest_config_tpl +{ + "id": "ingest_config_tpl", + "updated": "2023-10-19T14:49:56.768754+08:00", + "metadata": { + "category": "app_settings", + "name": "agent", + "labels": { + "instance": "_all" + } + }, + "payload": { + "name": "ingest_config.tpl", + "location": "ingest_config.tpl", + "content": """elasticsearch: + - name: $[[INGEST_CLUSTER_ID]] + enabled: true + endpoints: $[[INGEST_CLUSTER_ENDPOINT]] + discovery: + enabled: false + basic_auth: + username: $[[INGEST_CLUSTER_USERNAME]] + password: $[[keystore.ingest_cluster_password]] + +metrics: + enabled: true + queue: metrics + network: + enabled: true + summary: true + details: true + memory: + metrics: + - swap + - memory + disk: + metrics: + - iops + - usage + cpu: + metrics: + - idle + - system + - user + - iowait + - load + instance: + enabled: true + +elastic: + availability_check: + enabled: false + +pipeline: + - name: merge_logs + auto_start: true + keep_running: true + processor: + - indexing_merge: + elasticsearch: "$[[INGEST_CLUSTER_ID]]" + index_name: ".infini_logs" + type_name: "_doc" + input_queue: "logs" + idle_timeout_in_seconds: 10 + output_queue: + name: "merged_requests" + worker_size: 1 + bulk_size_in_mb: 5 + - name: merge_metrics + auto_start: true + keep_running: true + processor: + - indexing_merge: + elasticsearch: "$[[INGEST_CLUSTER_ID]]" + index_name: ".infini_metrics" + type_name: "_doc" + input_queue: "metrics" + output_queue: + name: "merged_requests" + worker_size: 1 + bulk_size_in_mb: 5 + - name: ingest_merged_requests + enabled: true + auto_start: true + keep_running: true + processor: + - bulk_indexing: + max_worker_size: 1 + verbose_bulk_result: false + bulk: + batch_size_in_mb: 5 + batch_size_in_docs: 5000 + max_retry_times: 0 + invalid_queue: "" + response_handle: + include_index_stats: false + include_action_stats: false + output_bulk_stats: false + include_error_details: true + save_error_results: true + save_success_results: false + save_busy_results: false + consumer: + fetch_max_messages: 5 + queues: + type: indexing_merge + when: + cluster_available: ["$[[INGEST_CLUSTER_ID]]"] +""", + "version": 1 + } +} + + +POST .infini_configs/_doc/system_ingest_config_yml +{ + "id": "system_ingest_config_yml", + "updated": "2023-10-19T14:49:56.768754+08:00", + "metadata": { + "category": "app_settings", + "name": "agent", + "labels": { + "instance": "_all" + } + }, + "payload": { + "name": "system_ingest_config.yml", + "location": "system_ingest_config.yml", + "content": """configs.template: + - name: "default_ingest_config" + path: ./config/ingest_config.tpl + variable: + INGEST_CLUSTER_ID: infini_default_ingest_cluster + INGEST_CLUSTER_ENDPOINT: [ "https://localhost:9200" ] + INGEST_CLUSTER_USERNAME: "admin" + CLUSTER_VER: "1.6.0" + CLUSTER_DISTRIBUTION: "easysearch" + INDEX_PREFIX: ".infini_" + +""", + "version": 1 + } +} +