update config for agent

config/agent/agent_config.tpl

@@ -16,17 +16,72 @@ POST .infini_configs/_doc/system_ingest_config_yml
     "location": "system_ingest_config.yml",
     "content": """
 
-configs.template:
+metrics:
-  - name: "default_ingest_config"
+  enabled: true
-    path: ./config/ingest_config.tpl
+  queue: metrics
-    variable:
+  network:
-      INGEST_CLUSTER_ID: infini_default_ingest_cluster
+    enabled: true
-      INGEST_CLUSTER_ENDPOINT: [ "http://192.168.3.185:8000" ]
+    summary: true
-      INGEST_CLUSTER_USERNAME: "ingest"
+    sockets: true
-      INGEST_CLUSTER_PASSWORD: "password"
+    #throughput: true
-      CLUSTER_VER: "1.6.0"
+    details: true
-      CLUSTER_DISTRIBUTION: "easysearch"
+  memory:
-      INDEX_PREFIX: ".infini_"
+    metrics:
+      - swap
+      - memory
+  disk:
+    metrics:
+      - iops
+      - usage
+  cpu:
+    metrics:
+      - idle
+      - system
+      - user
+      - iowait
+      - load
+  instance:
+    enabled: true
+
+elastic:
+  availability_check:
+    enabled: false
+
+pipeline:
+  - name: replicate_message_to_gateway
+    enabled: true
+    auto_start: true
+    keep_running: true
+    processor:
+      - consumer:
+          max_worker_size: 1
+          queue_selector:
+            keys:
+              - metrics
+              - logs
+          consumer:
+            group: replication
+          processor:
+            - http:
+                max_sending_qps: 100
+                method: POST
+                path: /$[[queue_name]]/_doc/
+                headers:
+                    Content-Type: application/json
+                body: $[[message]]
+                basic_auth:
+                  username: ingest
+                  password: password
+#                tls: #for mTLS connection with config servers
+#                  enabled: true
+#                  ca_file: /Users/medcl/Desktop/ca.crt
+#                  cert_file: /Users/medcl/Desktop/client.crt
+#                  key_file: /Users/medcl/Desktop/client.key
+#                  skip_insecure_verify: false
+                schema: "http"
+                hosts: # receiver endpoint, fallback in order
+                  - "192.168.3.185:8888"
+                valid_status_code: [200,201] #panic on other status code
 
 """,
     "version": 3

config/agent/init_agent_config.tpl

@@ -71,130 +71,3 @@ pipeline:
     "version": 1
   }
 }
-
-#system ingest template
-POST .infini_configs/_doc/ingest_config_tpl
-{
-  "id": "ingest_config_tpl",
-  "updated": "2023-10-19T14:49:56.768754+08:00",
-  "metadata": {
-    "category": "app_settings",
-    "name": "agent",
-    "labels": {
-      "instance": "_all"
-    }
-  },
-  "payload": {
-    "name": "ingest_config.tpl",
-    "location": "ingest_config.tpl",
-    "content": """
-
-elasticsearch:
-  - name: $[[INGEST_CLUSTER_ID]]
-    enabled: true
-    endpoints: $[[INGEST_CLUSTER_ENDPOINT]]
-    discovery:
-      enabled: false
-    basic_auth:
-      username: $[[INGEST_CLUSTER_USERNAME]]
-      password: $[[INGEST_CLUSTER_PASSWORD]]
-    traffic_control:
-      enabled: true
-      max_qps_per_node: 1000
-      max_bytes_per_node: 10485760
-      max_connection_per_node: 10
-
-metrics:
-  enabled: true
-  queue: metrics
-  network:
-    enabled: true
-    summary: true
-    sockets: true
-    #throughput: true
-    details: true
-  memory:
-    metrics:
-      - swap
-      - memory
-  disk:
-    metrics:
-      - iops
-      - usage
-  cpu:
-    metrics:
-      - idle
-      - system
-      - user
-      - iowait
-      - load
-  instance:
-    enabled: true
-
-elastic:
-  availability_check:
-    enabled: false
-
-pipeline:
-  - name: merge_logs
-    auto_start: true
-    keep_running: true
-    retry_delay_in_ms: 10000
-    processor:
-      - indexing_merge:
-          elasticsearch: "$[[INGEST_CLUSTER_ID]]"
-          index_name: ".infini_logs"
-          type_name: "_doc"
-          input_queue: "logs"
-          idle_timeout_in_seconds: 10
-          output_queue:
-            name: "merged_requests"
-          worker_size: 1
-          bulk_size_in_mb: 5
-  - name: merge_metrics
-    auto_start: true
-    keep_running: true
-    retry_delay_in_ms: 10000
-    processor:
-      - indexing_merge:
-          elasticsearch: "$[[INGEST_CLUSTER_ID]]"
-          index_name: ".infini_metrics"
-          type_name: "_doc"
-          input_queue: "metrics"
-          output_queue:
-            name: "merged_requests"
-          worker_size: 1
-          bulk_size_in_mb: 5
-  - name: ingest_merged_requests
-    enabled: true
-    auto_start: true
-    keep_running: true
-    retry_delay_in_ms: 10000
-    processor:
-      - bulk_indexing:
-          max_worker_size: 1
-          verbose_bulk_result: false
-          bulk:
-            batch_size_in_mb: 5
-            batch_size_in_docs: 5000
-            max_retry_times: 0
-            invalid_queue: ""
-            response_handle:
-              include_index_stats: false
-              include_action_stats: false
-              output_bulk_stats: false
-              include_error_details: true
-              save_error_results: true
-              save_success_results: false
-              save_busy_results: false
-          consumer:
-            fetch_max_messages: 5
-          queues:
-            type: indexing_merge
-          when:
-            cluster_available: ["$[[INGEST_CLUSTER_ID]]"]
-""",
-    "version": 1
-  }
-}
-

config/agent/init_gateway_config.tpl

@@ -13,80 +13,77 @@ POST .infini_configs/_doc/agent_relay_gateway_config_yml
     "name": "agent_relay_gateway_config.yml",
     "location": "agent_relay_gateway_config.yml",
     "content": """
-env:
-  CLUSTER_ENDPOINTS: ["http://10.0.0.3:7102"]
-  CLUSTER_USERNAME: admin
-  CLUSTER_PASSWORD: admin
 
 path.data: data
 path.logs: log
 
+allow_multi_instance: true
+configs.auto_reload: false
+
 entry:
   - name: my_es_entry
     enabled: true
     router: my_router
     max_concurrency: 200000
     network:
-      binding: 0.0.0.0:8000
+      binding: 0.0.0.0:8888
+#    tls: #for mTLS connection with config servers
+#      enabled: true
+#      ca_file: /Users/medcl/Desktop/ca.crt
+#      cert_file: /Users/medcl/Desktop/server.crt
+#      key_file: /Users/medcl/Desktop/server.key
+#      skip_insecure_verify: false
 
 flow:
-  - name: async_bulk
+  - name: deny_flow
+    filter:
+      - set_response:
+          body: "request not allowed"
+          status: 500
+  - name: ingest_flow
     filter:
       - basic_auth:
           valid_users:
             ingest: password
+      - rewrite_to_bulk:
+          type_removed: false
+      - bulk_request_mutate:
+          fix_null_id: true
+          generate_enhanced_id: true
+#          fix_null_type: true
+#          default_type: m-type
+#          default_index: m-index
+          index_rename:
+            metrics: ".infini_metrics"
+            logs: ".infini_logs"
       - bulk_reshuffle:
           when:
             contains:
               _ctx.request.path: /_bulk
           elasticsearch: prod
-          level: cluster
+          level: node
-          partition_size: 3
+          partition_size: 1
           fix_null_id: true
-      - elasticsearch:
-          elasticsearch: prod  #elasticsearch configure reference name
-          max_connection_per_node: 1000 #max tcp connection to upstream, default for all nodes
-          max_response_size: -1 #default for all nodes
-          balancer: weight
-          refresh: # refresh upstream nodes list, need to enable this feature to use elasticsearch nodes auto discovery
-            enabled: true
-            interval: 60s
-          filter:
-            roles:
-              exclude:
-                - master
 
 router:
   - name: my_router
-    default_flow: async_bulk
+    default_flow: deny_flow
-
+    rules:
+      - method:
+          - "POST"
+        enabled: true
+        pattern:
+          - "/{any_index}/_doc/"
+        flow:
+          - ingest_flow
 elasticsearch:
   - name: prod
     enabled: true
-    endpoints: $[[env.CLUSTER_ENDPOINTS]]
-    discovery:
-      enabled: false
     basic_auth:
-      username: $[[env.CLUSTER_USERNAME]]
+      username: admin
-      password: $[[env.CLUSTER_PASSWORD]]
+      password: admin
-    traffic_control:
+    endpoints:
-      enabled: true
+     - https://10.0.0.3:9200
-      max_qps_per_node: 100
-      max_bytes_per_node: 10485760
-      max_connection_per_node: 5
-
-elastic:
-  enabled: true
-  remote_configs: false
-  elasticsearch: prod
-  metadata_refresh:
-    enabled: true
-    interval: 30s
-  discovery:
-    enabled: true
-    refresh:
-      enabled: true
-      interval: 30s
 
 pipeline:
   - name: bulk_request_ingest