From edec20f05b184cf1c193fea0895f9356373a0915 Mon Sep 17 00:00:00 2001 From: William Tucker Date: Thu, 27 Nov 2025 14:01:37 +0000 Subject: [PATCH 1/5] Updated logstash configuration to fix payload problem --- deploy/kubernetes/chart/files/logstash/10-stats.conf | 12 +++++++++--- deploy/kubernetes/chart/values.yaml | 3 ++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/deploy/kubernetes/chart/files/logstash/10-stats.conf b/deploy/kubernetes/chart/files/logstash/10-stats.conf index ede5022..e374ed9 100644 --- a/deploy/kubernetes/chart/files/logstash/10-stats.conf +++ b/deploy/kubernetes/chart/files/logstash/10-stats.conf @@ -57,12 +57,18 @@ filter { replace => { "host" => "${EXTERNAL_HOSTNAME}" } # Move the country code to the top-level add_field => { "country_code" => "%{[geoip][country_code2]}" } - # Remove any unrequired fields - remove_field => ["client_ip", "command", "geoip", "tags", "user"] + # Construct a new message string + add_field => { + "rabbitmq_message" => '%{country_code} - - [%{request_time}] "%{method} %{request} HTTP/%{http_version}" %{status_code} %{bytes_sent} "%{referrer}" "%{user_agent}" "%{client_ip}" [%{request_duration}]' + } + } + mutate { # For Rabbit queue compatibility add_field => { - "rabbitmq_payload" => '{ "properties": { "delivery_mode": 2 }, "routing_key": "log_queue", "payload": "%{message}", "payload_encoding": "string" }' + "rabbitmq_payload" => '{ "properties": { "delivery_mode": 2 }, "routing_key": "log_queue", "payload": "%{rabbitmq_message}", "payload_encoding": "string" }' } + # Remove any unrequired fields + remove_field => ["rabbitmq_message", "command", "geoip", "tags", "user", "country_code", "request_time", "method", "request", "http_version", "status_code", "bytes_sent", "referrer", "user_agent", "client_ip", "request_duration"] } } output { diff --git a/deploy/kubernetes/chart/values.yaml b/deploy/kubernetes/chart/values.yaml index 9b4ebdf..2b4e1c8 100644 --- a/deploy/kubernetes/chart/values.yaml +++ b/deploy/kubernetes/chart/values.yaml @@ -264,7 +264,8 @@ data: # The default server is the CMCC server # Your IP must be whitelisted in order to send data to the CMCC service url: https://esgf-logstash.cmcc.it/api/exchanges/%2f/log_exchange/publish - authorizationKey: + authorizationKey: "" + certificate: "" # Configuration for the THREDDS pod thredds: From b2668d53436738170a0978861201648eb338d562 Mon Sep 17 00:00:00 2001 From: William Tucker Date: Fri, 28 Nov 2025 09:47:47 +0000 Subject: [PATCH 2/5] Correcting log string for stats --- deploy/kubernetes/chart/files/logstash/10-stats.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/kubernetes/chart/files/logstash/10-stats.conf b/deploy/kubernetes/chart/files/logstash/10-stats.conf index e374ed9..8e1db01 100644 --- a/deploy/kubernetes/chart/files/logstash/10-stats.conf +++ b/deploy/kubernetes/chart/files/logstash/10-stats.conf @@ -59,7 +59,7 @@ filter { add_field => { "country_code" => "%{[geoip][country_code2]}" } # Construct a new message string add_field => { - "rabbitmq_message" => '%{country_code} - - [%{request_time}] "%{method} %{request} HTTP/%{http_version}" %{status_code} %{bytes_sent} "%{referrer}" "%{user_agent}" "%{client_ip}" [%{request_duration}]' + "rabbitmq_message" => '%{host} %{country_code} %{request_time} %{method} %{request} HTTP/%{http_version} %{status_code} %{bytes_sent} %{request_duration} %{user_agent}' } } mutate { @@ -68,7 +68,7 @@ filter { "rabbitmq_payload" => '{ "properties": { "delivery_mode": 2 }, "routing_key": "log_queue", "payload": "%{rabbitmq_message}", "payload_encoding": "string" }' } # Remove any unrequired fields - remove_field => ["rabbitmq_message", "command", "geoip", "tags", "user", "country_code", "request_time", "method", "request", "http_version", "status_code", "bytes_sent", "referrer", "user_agent", "client_ip", "request_duration"] + remove_field => ["rabbitmq_message", "command", "geoip", "tags", "user", "host", "country_code", "request_time", "method", "request", "http_version", "status_code", "bytes_sent", "referrer", "user_agent", "client_ip", "request_duration"] } } output { From 061f006b2f553d436f8e1b3ad3b242c9d7417e8e Mon Sep 17 00:00:00 2001 From: William Tucker Date: Fri, 28 Nov 2025 15:41:13 +0000 Subject: [PATCH 3/5] Modifying log message for stats pipeline --- deploy/kubernetes/chart/files/logstash/10-stats.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/kubernetes/chart/files/logstash/10-stats.conf b/deploy/kubernetes/chart/files/logstash/10-stats.conf index 8e1db01..344f01d 100644 --- a/deploy/kubernetes/chart/files/logstash/10-stats.conf +++ b/deploy/kubernetes/chart/files/logstash/10-stats.conf @@ -59,7 +59,7 @@ filter { add_field => { "country_code" => "%{[geoip][country_code2]}" } # Construct a new message string add_field => { - "rabbitmq_message" => '%{host} %{country_code} %{request_time} %{method} %{request} HTTP/%{http_version} %{status_code} %{bytes_sent} %{request_duration} %{user_agent}' + "rabbitmq_message" => '%{host} %{country_code} %{request_time} %{method} %{request} HTTP/%{http_version} %{status_code} %{bytes_sent} %{request_duration} \"%{user_agent}\"' } } mutate { From 5f3d459898bc0627f2417bc8d2d0c8068a6d2567 Mon Sep 17 00:00:00 2001 From: William Tucker Date: Thu, 15 Jan 2026 17:11:55 +0000 Subject: [PATCH 4/5] Implement logstash changes for Ansible deployment --- deploy/ansible/roles/data/defaults/main.yml | 38 +------------------ .../roles/data/tasks/logstash_install.yml | 14 +------ .../roles/data/templates/stats.conf.j2 | 32 ++++++++++++---- .../templates/fileServer/deployment.yaml | 7 ---- .../chart/templates/logstash/certs.yaml | 13 ------- .../chart/templates/thredds/deployment.yaml | 7 ---- deploy/kubernetes/chart/values.yaml | 1 - 7 files changed, 29 insertions(+), 83 deletions(-) delete mode 100644 deploy/kubernetes/chart/templates/logstash/certs.yaml diff --git a/deploy/ansible/roles/data/defaults/main.yml b/deploy/ansible/roles/data/defaults/main.yml index 12d271d..14338f3 100644 --- a/deploy/ansible/roles/data/defaults/main.yml +++ b/deploy/ansible/roles/data/defaults/main.yml @@ -83,39 +83,5 @@ logstash_image_tag: "{{ image_tag }}" logstash_image_pull: "{{ image_pull }}" logstash_image_repository: logstash # Settings for the stats server -logstash_stats_server: ophidialab.cmcc.it -logstash_stats_port: 5045 -logstash_stats_certificate: | - -----BEGIN CERTIFICATE----- - MIIFpDCCA4wCCQDwfiQImQrAHjANBgkqhkiG9w0BAQsFADCBkzELMAkGA1UEBhMC - SVQxDjAMBgNVBAgMBUl0YWx5MQ4wDAYDVQQHDAVMZWNjZTENMAsGA1UECgwERVNH - RjENMAsGA1UECwwERVNHRjEbMBkGA1UEAwwSb3BoaWRpYWxhYi5jbWNjLml0MSkw - JwYJKoZIhvcNAQkBFhpzdGF0aXN0aWNzLXN1cHBvcnRAY21jYy5pdDAeFw0yMzA5 - MjUxNDAwMzRaFw0yNDA5MjQxNDAwMzRaMIGTMQswCQYDVQQGEwJJVDEOMAwGA1UE - CAwFSXRhbHkxDjAMBgNVBAcMBUxlY2NlMQ0wCwYDVQQKDARFU0dGMQ0wCwYDVQQL - DARFU0dGMRswGQYDVQQDDBJvcGhpZGlhbGFiLmNtY2MuaXQxKTAnBgkqhkiG9w0B - CQEWGnN0YXRpc3RpY3Mtc3VwcG9ydEBjbWNjLml0MIICIjANBgkqhkiG9w0BAQEF - AAOCAg8AMIICCgKCAgEA0ink9ncyjBGU9207PiK/26Ue1lx+fCVjKUVDl569t7Kg - n5XELdAdgyWQRuTGG1w7H03OFVKPsUGFb80agZBb5T6eP8yRXLz5Ejp5DQHlo0nh - 9ia4bMNJcx0jE1WdYfCP7GCKohTKlFpuThvuu3IhXGHLzgR06ygvDI7Hf554P59D - OxQ6JO3fTUutO9z78xvyxtY4MBZyMSDRmiJJLc4hzJ1AaPvFjNgU7AZBZ/X8Rl3Y - 69k/sWQtGZlJGZ4ZR66+OMJZEk5NiqrnShBRsczJuUwehpLv8kWVfQ/0nfRmI1aZ - BGjq1pUH6611G2A42PqseaCMXGEbidPVUZIRwMbMPXud0bdf4+wVAJftehnCHg5G - LizNgK2/jQWTo69BPbJyAMgI4qjK+NAfNwknRgpamvikQ66GuV4YCOKUV8/GUtLO - uYMXfYlg2wNhnXJ2YDiEKJAY1/NnAkPxvYpS+r9pi6kd+6uTH8H11G+kjDzK+29y - /6O1V2LSZL902KF7gdTA104r3HCYOf9BfRWA3C/C5P2vf376fLwskcUy5iQSX0Tc - IORRkjUKsoKN/FZ2IsPS9I05o5uUW1075Lf6aS/kyg0cEVfTSPyiWug0ozS1oIP4 - dvfbvJv/R7Sz6nz04rcEq6CYaqlw+65WMDlBoUIVu65Gsg05gbnLO3fcNDPlbBsC - AwEAATANBgkqhkiG9w0BAQsFAAOCAgEAm6iITRmACwLZqSRMC0X9mb0/dwo75n3q - H6bXHHSU9tozJkrq+OywTNCnWQV9oqUZvh9Iect2tQHmp00LxKrRKgPLEJ3PCq1n - adQifN10CeKt9b/657X8DgfitaaPYdDu4bVbITRLyeiMJRXfGAzBbEO1hIGapyuI - 5FmuwicCIxxwHU3Ej9fc7ZJr7zQFbmLuLLr6ntsIddNwoMlOrzgj9U3VpZeKoyXo - rdn2Nnl0WkVowic+Q7mSkQ2dYU4pflVkDHKuFHRZxc8OBEr2QOSzEgx/57BDDsd+ - KTJ6n951f4w0IdviRV7U23H+YojKPIrXsAyaT+axd3C71eE5ZidggAgk0u4dE6Hq - W/sEIlzlY+JMPQj8ynLpJUeoSO3osHoKn7S4SxCwQQLdaWeUWcLq8t1tCQOYf6CX - B+6Hvjq5tAH7DVmDcqdXgFQYb/1rdOWiZsuChD2jNEENw7YoJAq0yQPH4BcpkmPU - qxT26GCw9q8dHas7LkL4oXovUCozKyS02FEud/fan+5jSaiHvbQ1Ge7pHQeclhqr - 3k9JO5ZfOvZad3FRBEVnsEYb0VX+JuEqZarnZhqrzxc/764StngT7zd+YT3O1P+f - ZOSy8kG7ZaRItC+KjVwOma4rEF6/kKCBUu/LeOgbHnXmwVPBoFiNUQA1FHMRwGPH - HWB6z/apQUM= - -----END CERTIFICATE----- +logstash_stats_url: https://esgf-logstash.cmcc.it/api/exchanges/%2f/log_exchange/publish +logstash_stats_authorization_key: "" diff --git a/deploy/ansible/roles/data/tasks/logstash_install.yml b/deploy/ansible/roles/data/tasks/logstash_install.yml index 784cef1..8ff1c3c 100644 --- a/deploy/ansible/roles/data/tasks/logstash_install.yml +++ b/deploy/ansible/roles/data/tasks/logstash_install.yml @@ -12,18 +12,10 @@ docker_network: name: esgf -- name: Ensure logstash config directories exist +- name: Ensure logstash config directory exist file: - path: /esg/config/logstash/{{ item }} + path: /esg/config/logstash/pipelines state: directory - loop: - - certs - - pipelines - -- name: Install certificate for stats server - copy: - content: "{{ logstash_stats_certificate }}" - dest: /esg/config/logstash/certs/lumberjack.cert - name: Install stats logstash configuration template: @@ -44,8 +36,6 @@ user: "{{ data_security_context_user }}" groups: "{{ data_security_context_groups }}" volumes: - # Logstash certificates - - "/esg/config/logstash/certs:/etc/logstash/certs:ro" # Logstash configuration - "/esg/config/logstash/pipelines:/etc/logstash/conf.d:ro" # Log directory diff --git a/deploy/ansible/roles/data/templates/stats.conf.j2 b/deploy/ansible/roles/data/templates/stats.conf.j2 index 7092620..2a42608 100644 --- a/deploy/ansible/roles/data/templates/stats.conf.j2 +++ b/deploy/ansible/roles/data/templates/stats.conf.j2 @@ -60,19 +60,37 @@ filter { } mutate { # Replace the host with the specified external hostname - replace => { "host" => "{{ hostname }}" } + replace => { "host" => "${EXTERNAL_HOSTNAME}" } # Move the country code to the top-level add_field => { "country_code" => "%{[geoip][country_code2]}" } + # Construct a new message string + add_field => { + "rabbitmq_message" => '%{host} %{country_code} %{request_time} %{method} %{request} HTTP/%{http_version} %{status_code} %{bytes_sent} %{request_duration} \"%{user_agent}\"' + } + } + mutate { + # For Rabbit queue compatibility + add_field => { + "rabbitmq_payload" => '{ "properties": { "delivery_mode": 2 }, "routing_key": "log_queue", "payload": "%{rabbitmq_message}", "payload_encoding": "string" }' + } # Remove any unrequired fields - remove_field => ["client_ip", "command", "geoip", "tags", "user"] + remove_field => ["rabbitmq_message", "command", "geoip", "tags", "user", "host", "country_code", "request_time", "method", "request", "http_version", "status_code", "bytes_sent", "referrer", "user_agent", "client_ip", "request_duration"] } } output { - lumberjack { - codec => json - hosts => ["{{ logstash_stats_server }}"] - port => {{ logstash_stats_port }} - ssl_certificate => "/etc/logstash/certs/lumberjack.cert" + http { + url => "{{ logstash_stats_url }}" + http_method => "post" + headers => { + "Authorization" => "Basic {{ logstash_stats_authorization_key }}" + "Content-Type" => "application/json" + } + + format => "message" + + message => "%{rabbitmq_payload}" + + http_compression => false } stdout { codec => rubydebug diff --git a/deploy/kubernetes/chart/templates/fileServer/deployment.yaml b/deploy/kubernetes/chart/templates/fileServer/deployment.yaml index 4ff3602..db6d4dd 100644 --- a/deploy/kubernetes/chart/templates/fileServer/deployment.yaml +++ b/deploy/kubernetes/chart/templates/fileServer/deployment.yaml @@ -25,7 +25,6 @@ spec: {{- if $accessLogSidecar.enabled }} # When the access log sidecar is enabled, roll the deployment if the logstash pipelines or certificates change checksum/logstash-pipelines: {{ include (print $.Template.BasePath "/logstash/pipelines.yaml") . | sha256sum }} - checksum/logstash-certs: {{ include (print $.Template.BasePath "/logstash/certs.yaml") . | sha256sum }} {{- end }} {{- with $fileServer.annotations }} {{- toYaml . | nindent 8 }} @@ -127,8 +126,6 @@ spec: volumeMounts: - name: logstash-conf mountPath: /etc/logstash/conf.d - - name: logstash-certs - mountPath: /etc/logstash/certs - name: nginx-logs mountPath: /var/log/nginx - name: logstash-data @@ -200,10 +197,6 @@ spec: - name: logstash-conf secret: secretName: {{ include "esgf.component.fullname" (list . "logstash-pipelines") }} - # As are any certificates that are required - - name: logstash-certs - secret: - secretName: {{ include "esgf.component.fullname" (list . "logstash-certs") }} # Logstash also needs a tmp dir and a data dir - name: logstash-data emptyDir: {} diff --git a/deploy/kubernetes/chart/templates/logstash/certs.yaml b/deploy/kubernetes/chart/templates/logstash/certs.yaml deleted file mode 100644 index c666bd4..0000000 --- a/deploy/kubernetes/chart/templates/logstash/certs.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- $accessLogSidecar := .Values.data.accessLogSidecar -}} -{{- $statsPipeline := $accessLogSidecar.statsPipeline -}} -{{- if (and .Values.data.enabled $accessLogSidecar.enabled) -}} -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "esgf.component.fullname" (list . "logstash-certs") }} - labels: {{ include "esgf.component.labels" (list . "logstash" dict) | nindent 4 }} -data: -{{- if $statsPipeline.enabled }} - lumberjack.cert: {{ $statsPipeline.certificate | b64enc | quote }} -{{- end }} -{{- end }} diff --git a/deploy/kubernetes/chart/templates/thredds/deployment.yaml b/deploy/kubernetes/chart/templates/thredds/deployment.yaml index d45de9a..15252ef 100644 --- a/deploy/kubernetes/chart/templates/thredds/deployment.yaml +++ b/deploy/kubernetes/chart/templates/thredds/deployment.yaml @@ -26,7 +26,6 @@ spec: {{- if $accessLogSidecar.enabled }} # When the access log sidecar is enabled, roll the deployment if the logstash pipelines or certificates change checksum/logstash-pipelines: {{ include (print $.Template.BasePath "/logstash/pipelines.yaml") . | sha256sum }} - checksum/logstash-certs: {{ include (print $.Template.BasePath "/logstash/certs.yaml") . | sha256sum }} {{- end }} checksum/config: {{ include (print $.Template.BasePath "/thredds/configmap-config.yaml") . | sha256sum }} {{- with $thredds.annotations }} @@ -183,8 +182,6 @@ spec: volumeMounts: - name: logstash-conf mountPath: /etc/logstash/conf.d - - name: logstash-certs - mountPath: /etc/logstash/certs - name: tomcat-logs mountPath: /thredds/logs - name: logstash-data @@ -239,10 +236,6 @@ spec: - name: logstash-conf secret: secretName: {{ include "esgf.component.fullname" (list . "logstash-pipelines") }} - # As are any certificates that are required - - name: logstash-certs - secret: - secretName: {{ include "esgf.component.fullname" (list . "logstash-certs") }} # Logstash also needs a tmp dir and a data dir - name: logstash-data emptyDir: {} diff --git a/deploy/kubernetes/chart/values.yaml b/deploy/kubernetes/chart/values.yaml index 2b4e1c8..fee1db9 100644 --- a/deploy/kubernetes/chart/values.yaml +++ b/deploy/kubernetes/chart/values.yaml @@ -265,7 +265,6 @@ data: # Your IP must be whitelisted in order to send data to the CMCC service url: https://esgf-logstash.cmcc.it/api/exchanges/%2f/log_exchange/publish authorizationKey: "" - certificate: "" # Configuration for the THREDDS pod thredds: From ac3218b64bd2fdc327cad5558438fa0160fde9a5 Mon Sep 17 00:00:00 2001 From: William Tucker Date: Thu, 15 Jan 2026 17:15:14 +0000 Subject: [PATCH 5/5] Corrected bug in Ansible logstash conf --- deploy/ansible/roles/data/templates/stats.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/ansible/roles/data/templates/stats.conf.j2 b/deploy/ansible/roles/data/templates/stats.conf.j2 index 2a42608..6c0f015 100644 --- a/deploy/ansible/roles/data/templates/stats.conf.j2 +++ b/deploy/ansible/roles/data/templates/stats.conf.j2 @@ -60,7 +60,7 @@ filter { } mutate { # Replace the host with the specified external hostname - replace => { "host" => "${EXTERNAL_HOSTNAME}" } + replace => { "host" => "{{ hostname }}" } # Move the country code to the top-level add_field => { "country_code" => "%{[geoip][country_code2]}" } # Construct a new message string