update configs

config/agent/agent_config.tpl

@@ -0,0 +1,34 @@
+
+#TODO, need to replace cleartext password to keystore, and ingest endpoint
+POST .infini_configs/_doc/system_ingest_config_yml
+{
+  "id": "system_ingest_config_yml",
+  "updated": "2023-10-18T14:49:56.768754+08:00",
+  "metadata": {
+    "category": "app_settings",
+    "name": "agent",
+    "labels": {
+      "instance": "_all"
+    }
+  },
+  "payload": {
+    "name": "system_ingest_config.yml",
+    "location": "system_ingest_config.yml",
+    "content": """
+
+configs.template:
+  - name: "default_ingest_config"
+    path: ./config/ingest_config.tpl
+    variable:
+      INGEST_CLUSTER_ID: infini_default_ingest_cluster
+      INGEST_CLUSTER_ENDPOINT: [ "http://192.168.3.185:8000" ]
+      INGEST_CLUSTER_USERNAME: "ingest"
+      INGEST_CLUSTER_PASSWORD: "password"
+      CLUSTER_VER: "1.6.0"
+      CLUSTER_DISTRIBUTION: "easysearch"
+      INDEX_PREFIX: ".infini_"
+
+""",
+    "version": 3
+  }
+}

config/agent/init_agent_config.tpl

@@ -14,7 +14,9 @@ POST .infini_configs/_doc/task_config_tpl
   "payload": {
     "name": "task_config.tpl",
     "location": "task_config.tpl",
-    "content": """env:
+    "content": """
+
+env:
    CLUSTER_PASSWORD: $[[keystore.$[[CLUSTER_ID]]_password]]
 
 elasticsearch:
@@ -85,7 +87,9 @@ POST .infini_configs/_doc/ingest_config_tpl
   "payload": {
     "name": "ingest_config.tpl",
     "location": "ingest_config.tpl",
-    "content": """elasticsearch:
+    "content": """
+
+elasticsearch:
   - name: $[[INGEST_CLUSTER_ID]]
     enabled: true
     endpoints: $[[INGEST_CLUSTER_ENDPOINT]]
@@ -191,36 +195,3 @@ pipeline:
   }
 }
 
-
-#TODO, need to replace cleartext password to keystore, and ingest endpoint
-POST .infini_configs/_doc/system_ingest_config_yml
-{
-  "id": "system_ingest_config_yml",
-  "updated": "2023-10-18T14:49:56.768754+08:00",
-  "metadata": {
-    "category": "app_settings",
-    "name": "agent",
-    "labels": {
-      "instance": "_all"
-    }
-  },
-  "payload": {
-    "name": "system_ingest_config.yml",
-    "location": "system_ingest_config.yml",
-    "content": """configs.template:
-  - name: "default_ingest_config"
-    path: ./config/ingest_config.tpl
-    variable:
-      INGEST_CLUSTER_ID: infini_default_ingest_cluster
-      INGEST_CLUSTER_ENDPOINT: [ "http://10.0.0.3:7102" ]
-      INGEST_CLUSTER_USERNAME: "admin"
-      INGEST_CLUSTER_PASSWORD: "admin"
-      CLUSTER_VER: "1.6.0"
-      CLUSTER_DISTRIBUTION: "easysearch"
-      INDEX_PREFIX: ".infini_"
-
-""",
-    "version": 3
-  }
-}
-

config/agent/init_gateway_config.tpl

@@ -0,0 +1,114 @@
+POST .infini_configs/_doc/agent_relay_gateway_config_yml
+{
+  "id": "agent_relay_gateway_config_yml",
+  "updated": "2023-10-19T14:49:56.768754+08:00",
+  "metadata": {
+    "category": "app_settings",
+    "name": "gateway",
+    "labels": {
+      "instance": "_all"
+    }
+  },
+  "payload": {
+    "name": "agent_relay_gateway_config.yml",
+    "location": "agent_relay_gateway_config.yml",
+    "content": """
+env:
+  CLUSTER_ENDPOINTS: ["http://10.0.0.3:7102"]
+  CLUSTER_USERNAME: admin
+  CLUSTER_PASSWORD: admin
+
+path.data: data
+path.logs: log
+
+entry:
+  - name: my_es_entry
+    enabled: true
+    router: my_router
+    max_concurrency: 200000
+    network:
+      binding: 0.0.0.0:8000
+
+flow:
+  - name: async_bulk
+    filter:
+      - basic_auth:
+          valid_users:
+            ingest: password
+      - bulk_reshuffle:
+          when:
+            contains:
+              _ctx.request.path: /_bulk
+          elasticsearch: prod
+          level: cluster
+          partition_size: 3
+          fix_null_id: true
+      - elasticsearch:
+          elasticsearch: prod  #elasticsearch configure reference name
+          max_connection_per_node: 1000 #max tcp connection to upstream, default for all nodes
+          max_response_size: -1 #default for all nodes
+          balancer: weight
+          refresh: # refresh upstream nodes list, need to enable this feature to use elasticsearch nodes auto discovery
+            enabled: true
+            interval: 60s
+          filter:
+            roles:
+              exclude:
+                - master
+
+router:
+  - name: my_router
+    default_flow: async_bulk
+
+elasticsearch:
+  - name: prod
+    enabled: true
+    endpoints: $[[env.CLUSTER_ENDPOINTS]]
+    discovery:
+      enabled: false
+    basic_auth:
+      username: $[[env.CLUSTER_USERNAME]]
+      password: $[[env.CLUSTER_PASSWORD]]
+    traffic_control:
+      enabled: true
+      max_qps_per_node: 100
+      max_bytes_per_node: 10485760
+      max_connection_per_node: 5
+
+elastic:
+  enabled: true
+  remote_configs: false
+  elasticsearch: prod
+  metadata_refresh:
+    enabled: true
+    interval: 30s
+  discovery:
+    enabled: true
+    refresh:
+      enabled: true
+      interval: 30s
+
+pipeline:
+  - name: bulk_request_ingest
+    auto_start: true
+    keep_running: true
+    retry_delay_in_ms: 1000
+    processor:
+      - bulk_indexing:
+          max_connection_per_node: 100
+          num_of_slices: 3
+          max_worker_size: 30
+          idle_timeout_in_seconds: 10
+          bulk:
+            compress: false
+            batch_size_in_mb: 10
+            batch_size_in_docs: 10000
+          consumer:
+            fetch_max_messages: 100
+          queue_selector:
+            labels:
+              type: bulk_reshuffle
+""",
+    "version": 1
+  }
+}