Logstash update

deploy/ansible/roles/data/defaults/main.yml

@@ -83,39 +83,5 @@ logstash_image_tag: "{{ image_tag }}"
 logstash_image_pull: "{{ image_pull }}"
 logstash_image_repository: logstash
 # Settings for the stats server
-logstash_stats_server: ophidialab.cmcc.it
-logstash_stats_port: 5045
-logstash_stats_certificate: |
-  -----BEGIN CERTIFICATE-----
-  MIIFpDCCA4wCCQDwfiQImQrAHjANBgkqhkiG9w0BAQsFADCBkzELMAkGA1UEBhMC
-  SVQxDjAMBgNVBAgMBUl0YWx5MQ4wDAYDVQQHDAVMZWNjZTENMAsGA1UECgwERVNH
-  RjENMAsGA1UECwwERVNHRjEbMBkGA1UEAwwSb3BoaWRpYWxhYi5jbWNjLml0MSkw
-  JwYJKoZIhvcNAQkBFhpzdGF0aXN0aWNzLXN1cHBvcnRAY21jYy5pdDAeFw0yMzA5
-  MjUxNDAwMzRaFw0yNDA5MjQxNDAwMzRaMIGTMQswCQYDVQQGEwJJVDEOMAwGA1UE
-  CAwFSXRhbHkxDjAMBgNVBAcMBUxlY2NlMQ0wCwYDVQQKDARFU0dGMQ0wCwYDVQQL
-  DARFU0dGMRswGQYDVQQDDBJvcGhpZGlhbGFiLmNtY2MuaXQxKTAnBgkqhkiG9w0B
-  CQEWGnN0YXRpc3RpY3Mtc3VwcG9ydEBjbWNjLml0MIICIjANBgkqhkiG9w0BAQEF
-  AAOCAg8AMIICCgKCAgEA0ink9ncyjBGU9207PiK/26Ue1lx+fCVjKUVDl569t7Kg
-  n5XELdAdgyWQRuTGG1w7H03OFVKPsUGFb80agZBb5T6eP8yRXLz5Ejp5DQHlo0nh
-  9ia4bMNJcx0jE1WdYfCP7GCKohTKlFpuThvuu3IhXGHLzgR06ygvDI7Hf554P59D
-  OxQ6JO3fTUutO9z78xvyxtY4MBZyMSDRmiJJLc4hzJ1AaPvFjNgU7AZBZ/X8Rl3Y
-  69k/sWQtGZlJGZ4ZR66+OMJZEk5NiqrnShBRsczJuUwehpLv8kWVfQ/0nfRmI1aZ
-  BGjq1pUH6611G2A42PqseaCMXGEbidPVUZIRwMbMPXud0bdf4+wVAJftehnCHg5G
-  LizNgK2/jQWTo69BPbJyAMgI4qjK+NAfNwknRgpamvikQ66GuV4YCOKUV8/GUtLO
-  uYMXfYlg2wNhnXJ2YDiEKJAY1/NnAkPxvYpS+r9pi6kd+6uTH8H11G+kjDzK+29y
-  /6O1V2LSZL902KF7gdTA104r3HCYOf9BfRWA3C/C5P2vf376fLwskcUy5iQSX0Tc
-  IORRkjUKsoKN/FZ2IsPS9I05o5uUW1075Lf6aS/kyg0cEVfTSPyiWug0ozS1oIP4
-  dvfbvJv/R7Sz6nz04rcEq6CYaqlw+65WMDlBoUIVu65Gsg05gbnLO3fcNDPlbBsC
-  AwEAATANBgkqhkiG9w0BAQsFAAOCAgEAm6iITRmACwLZqSRMC0X9mb0/dwo75n3q
-  H6bXHHSU9tozJkrq+OywTNCnWQV9oqUZvh9Iect2tQHmp00LxKrRKgPLEJ3PCq1n
-  adQifN10CeKt9b/657X8DgfitaaPYdDu4bVbITRLyeiMJRXfGAzBbEO1hIGapyuI
-  5FmuwicCIxxwHU3Ej9fc7ZJr7zQFbmLuLLr6ntsIddNwoMlOrzgj9U3VpZeKoyXo
-  rdn2Nnl0WkVowic+Q7mSkQ2dYU4pflVkDHKuFHRZxc8OBEr2QOSzEgx/57BDDsd+
-  KTJ6n951f4w0IdviRV7U23H+YojKPIrXsAyaT+axd3C71eE5ZidggAgk0u4dE6Hq
-  W/sEIlzlY+JMPQj8ynLpJUeoSO3osHoKn7S4SxCwQQLdaWeUWcLq8t1tCQOYf6CX
-  B+6Hvjq5tAH7DVmDcqdXgFQYb/1rdOWiZsuChD2jNEENw7YoJAq0yQPH4BcpkmPU
-  qxT26GCw9q8dHas7LkL4oXovUCozKyS02FEud/fan+5jSaiHvbQ1Ge7pHQeclhqr
-  3k9JO5ZfOvZad3FRBEVnsEYb0VX+JuEqZarnZhqrzxc/764StngT7zd+YT3O1P+f
-  ZOSy8kG7ZaRItC+KjVwOma4rEF6/kKCBUu/LeOgbHnXmwVPBoFiNUQA1FHMRwGPH
-  HWB6z/apQUM=
-  -----END CERTIFICATE-----
+logstash_stats_url: https://esgf-logstash.cmcc.it/api/exchanges/%2f/log_exchange/publish
+logstash_stats_authorization_key: ""

deploy/ansible/roles/data/tasks/logstash_install.yml

@@ -12,18 +12,10 @@
   docker_network:
     name: esgf
 
-- name: Ensure logstash config directories exist
+- name: Ensure logstash config directory exist
   file:
-    path: /esg/config/logstash/{{ item }}
+    path: /esg/config/logstash/pipelines
     state: directory
-  loop:
-    - certs
-    - pipelines
-
-- name: Install certificate for stats server
-  copy:
-    content: "{{ logstash_stats_certificate }}"
-    dest: /esg/config/logstash/certs/lumberjack.cert
 
 - name: Install stats logstash configuration
   template:
@@ -44,8 +36,6 @@
     user: "{{ data_security_context_user }}"
     groups: "{{ data_security_context_groups }}"
     volumes:
-      # Logstash certificates
-      - "/esg/config/logstash/certs:/etc/logstash/certs:ro"
       # Logstash configuration
       - "/esg/config/logstash/pipelines:/etc/logstash/conf.d:ro"
       # Log directory

deploy/ansible/roles/data/templates/stats.conf.j2

@@ -63,16 +63,34 @@ filter {
         replace => { "host" => "{{ hostname }}" }
         # Move the country code to the top-level
         add_field => { "country_code" => "%{[geoip][country_code2]}" }
+        # Construct a new message string
+        add_field => {
+            "rabbitmq_message" => '%{host} %{country_code} %{request_time} %{method} %{request} HTTP/%{http_version} %{status_code} %{bytes_sent} %{request_duration} \"%{user_agent}\"'
+        }
+    }
+    mutate {
+        # For Rabbit queue compatibility
+        add_field => {
+            "rabbitmq_payload" => '{ "properties": { "delivery_mode": 2 }, "routing_key": "log_queue", "payload": "%{rabbitmq_message}", "payload_encoding": "string" }'
+        }
         # Remove any unrequired fields
-        remove_field => ["client_ip", "command", "geoip", "tags", "user"]
+        remove_field => ["rabbitmq_message", "command", "geoip", "tags", "user", "host", "country_code", "request_time", "method", "request", "http_version", "status_code", "bytes_sent", "referrer", "user_agent", "client_ip", "request_duration"]
     }
 }
 output {
-    lumberjack {
-        codec => json
-        hosts => ["{{ logstash_stats_server }}"]
-        port => {{ logstash_stats_port }}
-        ssl_certificate => "/etc/logstash/certs/lumberjack.cert"
+    http {
+        url => "{{ logstash_stats_url }}"
+        http_method => "post"
+        headers => {
+            "Authorization" => "Basic {{ logstash_stats_authorization_key }}"
+            "Content-Type" => "application/json"
+        }
+
+        format => "message" 
+
+        message => "%{rabbitmq_payload}"
+
+        http_compression => false
     }
     stdout {
         codec => rubydebug

deploy/kubernetes/chart/files/logstash/10-stats.conf

@@ -57,12 +57,18 @@ filter {
         replace => { "host" => "${EXTERNAL_HOSTNAME}" }
         # Move the country code to the top-level
         add_field => { "country_code" => "%{[geoip][country_code2]}" }
-        # Remove any unrequired fields
-        remove_field => ["client_ip", "command", "geoip", "tags", "user"]
+        # Construct a new message string
+        add_field => {
+            "rabbitmq_message" => '%{host} %{country_code} %{request_time} %{method} %{request} HTTP/%{http_version} %{status_code} %{bytes_sent} %{request_duration} \"%{user_agent}\"'
+        }
+    }
+    mutate {
         # For Rabbit queue compatibility
         add_field => {
-            "rabbitmq_payload" => '{ "properties": { "delivery_mode": 2 }, "routing_key": "log_queue", "payload": "%{message}", "payload_encoding": "string" }'
+            "rabbitmq_payload" => '{ "properties": { "delivery_mode": 2 }, "routing_key": "log_queue", "payload": "%{rabbitmq_message}", "payload_encoding": "string" }'
         }
+        # Remove any unrequired fields
+        remove_field => ["rabbitmq_message", "command", "geoip", "tags", "user", "host", "country_code", "request_time", "method", "request", "http_version", "status_code", "bytes_sent", "referrer", "user_agent", "client_ip", "request_duration"]
     }
 }
 output {

deploy/kubernetes/chart/templates/fileServer/deployment.yaml

@@ -25,7 +25,6 @@ spec:
         {{- if $accessLogSidecar.enabled }}
         # When the access log sidecar is enabled, roll the deployment if the logstash pipelines or certificates change
         checksum/logstash-pipelines: {{ include (print $.Template.BasePath "/logstash/pipelines.yaml") . | sha256sum }}
-        checksum/logstash-certs: {{ include (print $.Template.BasePath "/logstash/certs.yaml") . | sha256sum }}
         {{- end }}
         {{- with $fileServer.annotations }}
         {{- toYaml . | nindent 8 }}
@@ -127,8 +126,6 @@ spec:
           volumeMounts:
             - name: logstash-conf
               mountPath: /etc/logstash/conf.d
-            - name: logstash-certs
-              mountPath: /etc/logstash/certs
             - name: nginx-logs
               mountPath: /var/log/nginx
             - name: logstash-data
@@ -200,10 +197,6 @@ spec:
         - name: logstash-conf
           secret:
             secretName: {{ include "esgf.component.fullname" (list . "logstash-pipelines") }}
-        # As are any certificates that are required
-        - name: logstash-certs
-          secret:
-            secretName: {{ include "esgf.component.fullname" (list . "logstash-certs") }}
         # Logstash also needs a tmp dir and a data dir
         - name: logstash-data
           emptyDir: {}

deploy/kubernetes/chart/templates/thredds/deployment.yaml

@@ -26,7 +26,6 @@ spec:
         {{- if $accessLogSidecar.enabled }}
         # When the access log sidecar is enabled, roll the deployment if the logstash pipelines or certificates change
         checksum/logstash-pipelines: {{ include (print $.Template.BasePath "/logstash/pipelines.yaml") . | sha256sum }}
-        checksum/logstash-certs: {{ include (print $.Template.BasePath "/logstash/certs.yaml") . | sha256sum }}
         {{- end }}
         checksum/config: {{ include (print $.Template.BasePath "/thredds/configmap-config.yaml") . | sha256sum }}
         {{- with $thredds.annotations }}
@@ -183,8 +182,6 @@ spec:
           volumeMounts:
             - name: logstash-conf
               mountPath: /etc/logstash/conf.d
-            - name: logstash-certs
-              mountPath: /etc/logstash/certs
             - name: tomcat-logs
               mountPath: /thredds/logs
             - name: logstash-data
@@ -239,10 +236,6 @@ spec:
         - name: logstash-conf
           secret:
             secretName: {{ include "esgf.component.fullname" (list . "logstash-pipelines") }}
-        # As are any certificates that are required
-        - name: logstash-certs
-          secret:
-            secretName: {{ include "esgf.component.fullname" (list . "logstash-certs") }}
         # Logstash also needs a tmp dir and a data dir
         - name: logstash-data
           emptyDir: {}

deploy/kubernetes/chart/values.yaml

@@ -264,7 +264,7 @@ data:
       # The default server is the CMCC server
       # Your IP must be whitelisted in order to send data to the CMCC service
       url: https://esgf-logstash.cmcc.it/api/exchanges/%2f/log_exchange/publish
-      authorizationKey: 
+      authorizationKey: ""
 
   # Configuration for the THREDDS pod
   thredds: