diff --git a/deploy/ansible/roles/data/defaults/main.yml b/deploy/ansible/roles/data/defaults/main.yml index 12d271dd..14338f31 100644 --- a/deploy/ansible/roles/data/defaults/main.yml +++ b/deploy/ansible/roles/data/defaults/main.yml @@ -83,39 +83,5 @@ logstash_image_tag: "{{ image_tag }}" logstash_image_pull: "{{ image_pull }}" logstash_image_repository: logstash # Settings for the stats server -logstash_stats_server: ophidialab.cmcc.it -logstash_stats_port: 5045 -logstash_stats_certificate: | - -----BEGIN CERTIFICATE----- - MIIFpDCCA4wCCQDwfiQImQrAHjANBgkqhkiG9w0BAQsFADCBkzELMAkGA1UEBhMC - SVQxDjAMBgNVBAgMBUl0YWx5MQ4wDAYDVQQHDAVMZWNjZTENMAsGA1UECgwERVNH - RjENMAsGA1UECwwERVNHRjEbMBkGA1UEAwwSb3BoaWRpYWxhYi5jbWNjLml0MSkw - JwYJKoZIhvcNAQkBFhpzdGF0aXN0aWNzLXN1cHBvcnRAY21jYy5pdDAeFw0yMzA5 - MjUxNDAwMzRaFw0yNDA5MjQxNDAwMzRaMIGTMQswCQYDVQQGEwJJVDEOMAwGA1UE - CAwFSXRhbHkxDjAMBgNVBAcMBUxlY2NlMQ0wCwYDVQQKDARFU0dGMQ0wCwYDVQQL - DARFU0dGMRswGQYDVQQDDBJvcGhpZGlhbGFiLmNtY2MuaXQxKTAnBgkqhkiG9w0B - CQEWGnN0YXRpc3RpY3Mtc3VwcG9ydEBjbWNjLml0MIICIjANBgkqhkiG9w0BAQEF - AAOCAg8AMIICCgKCAgEA0ink9ncyjBGU9207PiK/26Ue1lx+fCVjKUVDl569t7Kg - n5XELdAdgyWQRuTGG1w7H03OFVKPsUGFb80agZBb5T6eP8yRXLz5Ejp5DQHlo0nh - 9ia4bMNJcx0jE1WdYfCP7GCKohTKlFpuThvuu3IhXGHLzgR06ygvDI7Hf554P59D - OxQ6JO3fTUutO9z78xvyxtY4MBZyMSDRmiJJLc4hzJ1AaPvFjNgU7AZBZ/X8Rl3Y - 69k/sWQtGZlJGZ4ZR66+OMJZEk5NiqrnShBRsczJuUwehpLv8kWVfQ/0nfRmI1aZ - BGjq1pUH6611G2A42PqseaCMXGEbidPVUZIRwMbMPXud0bdf4+wVAJftehnCHg5G - LizNgK2/jQWTo69BPbJyAMgI4qjK+NAfNwknRgpamvikQ66GuV4YCOKUV8/GUtLO - uYMXfYlg2wNhnXJ2YDiEKJAY1/NnAkPxvYpS+r9pi6kd+6uTH8H11G+kjDzK+29y - /6O1V2LSZL902KF7gdTA104r3HCYOf9BfRWA3C/C5P2vf376fLwskcUy5iQSX0Tc - IORRkjUKsoKN/FZ2IsPS9I05o5uUW1075Lf6aS/kyg0cEVfTSPyiWug0ozS1oIP4 - dvfbvJv/R7Sz6nz04rcEq6CYaqlw+65WMDlBoUIVu65Gsg05gbnLO3fcNDPlbBsC - AwEAATANBgkqhkiG9w0BAQsFAAOCAgEAm6iITRmACwLZqSRMC0X9mb0/dwo75n3q - H6bXHHSU9tozJkrq+OywTNCnWQV9oqUZvh9Iect2tQHmp00LxKrRKgPLEJ3PCq1n - adQifN10CeKt9b/657X8DgfitaaPYdDu4bVbITRLyeiMJRXfGAzBbEO1hIGapyuI - 5FmuwicCIxxwHU3Ej9fc7ZJr7zQFbmLuLLr6ntsIddNwoMlOrzgj9U3VpZeKoyXo - rdn2Nnl0WkVowic+Q7mSkQ2dYU4pflVkDHKuFHRZxc8OBEr2QOSzEgx/57BDDsd+ - KTJ6n951f4w0IdviRV7U23H+YojKPIrXsAyaT+axd3C71eE5ZidggAgk0u4dE6Hq - W/sEIlzlY+JMPQj8ynLpJUeoSO3osHoKn7S4SxCwQQLdaWeUWcLq8t1tCQOYf6CX - B+6Hvjq5tAH7DVmDcqdXgFQYb/1rdOWiZsuChD2jNEENw7YoJAq0yQPH4BcpkmPU - qxT26GCw9q8dHas7LkL4oXovUCozKyS02FEud/fan+5jSaiHvbQ1Ge7pHQeclhqr - 3k9JO5ZfOvZad3FRBEVnsEYb0VX+JuEqZarnZhqrzxc/764StngT7zd+YT3O1P+f - ZOSy8kG7ZaRItC+KjVwOma4rEF6/kKCBUu/LeOgbHnXmwVPBoFiNUQA1FHMRwGPH - HWB6z/apQUM= - -----END CERTIFICATE----- +logstash_stats_url: https://esgf-logstash.cmcc.it/api/exchanges/%2f/log_exchange/publish +logstash_stats_authorization_key: "" diff --git a/deploy/ansible/roles/data/tasks/logstash_install.yml b/deploy/ansible/roles/data/tasks/logstash_install.yml index 784cef19..8ff1c3c3 100644 --- a/deploy/ansible/roles/data/tasks/logstash_install.yml +++ b/deploy/ansible/roles/data/tasks/logstash_install.yml @@ -12,18 +12,10 @@ docker_network: name: esgf -- name: Ensure logstash config directories exist +- name: Ensure logstash config directory exist file: - path: /esg/config/logstash/{{ item }} + path: /esg/config/logstash/pipelines state: directory - loop: - - certs - - pipelines - -- name: Install certificate for stats server - copy: - content: "{{ logstash_stats_certificate }}" - dest: /esg/config/logstash/certs/lumberjack.cert - name: Install stats logstash configuration template: @@ -44,8 +36,6 @@ user: "{{ data_security_context_user }}" groups: "{{ data_security_context_groups }}" volumes: - # Logstash certificates - - "/esg/config/logstash/certs:/etc/logstash/certs:ro" # Logstash configuration - "/esg/config/logstash/pipelines:/etc/logstash/conf.d:ro" # Log directory diff --git a/deploy/ansible/roles/data/templates/stats.conf.j2 b/deploy/ansible/roles/data/templates/stats.conf.j2 index 70926200..6c0f0156 100644 --- a/deploy/ansible/roles/data/templates/stats.conf.j2 +++ b/deploy/ansible/roles/data/templates/stats.conf.j2 @@ -63,16 +63,34 @@ filter { replace => { "host" => "{{ hostname }}" } # Move the country code to the top-level add_field => { "country_code" => "%{[geoip][country_code2]}" } + # Construct a new message string + add_field => { + "rabbitmq_message" => '%{host} %{country_code} %{request_time} %{method} %{request} HTTP/%{http_version} %{status_code} %{bytes_sent} %{request_duration} \"%{user_agent}\"' + } + } + mutate { + # For Rabbit queue compatibility + add_field => { + "rabbitmq_payload" => '{ "properties": { "delivery_mode": 2 }, "routing_key": "log_queue", "payload": "%{rabbitmq_message}", "payload_encoding": "string" }' + } # Remove any unrequired fields - remove_field => ["client_ip", "command", "geoip", "tags", "user"] + remove_field => ["rabbitmq_message", "command", "geoip", "tags", "user", "host", "country_code", "request_time", "method", "request", "http_version", "status_code", "bytes_sent", "referrer", "user_agent", "client_ip", "request_duration"] } } output { - lumberjack { - codec => json - hosts => ["{{ logstash_stats_server }}"] - port => {{ logstash_stats_port }} - ssl_certificate => "/etc/logstash/certs/lumberjack.cert" + http { + url => "{{ logstash_stats_url }}" + http_method => "post" + headers => { + "Authorization" => "Basic {{ logstash_stats_authorization_key }}" + "Content-Type" => "application/json" + } + + format => "message" + + message => "%{rabbitmq_payload}" + + http_compression => false } stdout { codec => rubydebug diff --git a/deploy/kubernetes/chart/files/logstash/10-stats.conf b/deploy/kubernetes/chart/files/logstash/10-stats.conf index ede50228..344f01da 100644 --- a/deploy/kubernetes/chart/files/logstash/10-stats.conf +++ b/deploy/kubernetes/chart/files/logstash/10-stats.conf @@ -57,12 +57,18 @@ filter { replace => { "host" => "${EXTERNAL_HOSTNAME}" } # Move the country code to the top-level add_field => { "country_code" => "%{[geoip][country_code2]}" } - # Remove any unrequired fields - remove_field => ["client_ip", "command", "geoip", "tags", "user"] + # Construct a new message string + add_field => { + "rabbitmq_message" => '%{host} %{country_code} %{request_time} %{method} %{request} HTTP/%{http_version} %{status_code} %{bytes_sent} %{request_duration} \"%{user_agent}\"' + } + } + mutate { # For Rabbit queue compatibility add_field => { - "rabbitmq_payload" => '{ "properties": { "delivery_mode": 2 }, "routing_key": "log_queue", "payload": "%{message}", "payload_encoding": "string" }' + "rabbitmq_payload" => '{ "properties": { "delivery_mode": 2 }, "routing_key": "log_queue", "payload": "%{rabbitmq_message}", "payload_encoding": "string" }' } + # Remove any unrequired fields + remove_field => ["rabbitmq_message", "command", "geoip", "tags", "user", "host", "country_code", "request_time", "method", "request", "http_version", "status_code", "bytes_sent", "referrer", "user_agent", "client_ip", "request_duration"] } } output { diff --git a/deploy/kubernetes/chart/templates/fileServer/deployment.yaml b/deploy/kubernetes/chart/templates/fileServer/deployment.yaml index 4ff36028..db6d4dd0 100644 --- a/deploy/kubernetes/chart/templates/fileServer/deployment.yaml +++ b/deploy/kubernetes/chart/templates/fileServer/deployment.yaml @@ -25,7 +25,6 @@ spec: {{- if $accessLogSidecar.enabled }} # When the access log sidecar is enabled, roll the deployment if the logstash pipelines or certificates change checksum/logstash-pipelines: {{ include (print $.Template.BasePath "/logstash/pipelines.yaml") . | sha256sum }} - checksum/logstash-certs: {{ include (print $.Template.BasePath "/logstash/certs.yaml") . | sha256sum }} {{- end }} {{- with $fileServer.annotations }} {{- toYaml . | nindent 8 }} @@ -127,8 +126,6 @@ spec: volumeMounts: - name: logstash-conf mountPath: /etc/logstash/conf.d - - name: logstash-certs - mountPath: /etc/logstash/certs - name: nginx-logs mountPath: /var/log/nginx - name: logstash-data @@ -200,10 +197,6 @@ spec: - name: logstash-conf secret: secretName: {{ include "esgf.component.fullname" (list . "logstash-pipelines") }} - # As are any certificates that are required - - name: logstash-certs - secret: - secretName: {{ include "esgf.component.fullname" (list . "logstash-certs") }} # Logstash also needs a tmp dir and a data dir - name: logstash-data emptyDir: {} diff --git a/deploy/kubernetes/chart/templates/logstash/certs.yaml b/deploy/kubernetes/chart/templates/logstash/certs.yaml deleted file mode 100644 index c666bd4e..00000000 --- a/deploy/kubernetes/chart/templates/logstash/certs.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- $accessLogSidecar := .Values.data.accessLogSidecar -}} -{{- $statsPipeline := $accessLogSidecar.statsPipeline -}} -{{- if (and .Values.data.enabled $accessLogSidecar.enabled) -}} -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "esgf.component.fullname" (list . "logstash-certs") }} - labels: {{ include "esgf.component.labels" (list . "logstash" dict) | nindent 4 }} -data: -{{- if $statsPipeline.enabled }} - lumberjack.cert: {{ $statsPipeline.certificate | b64enc | quote }} -{{- end }} -{{- end }} diff --git a/deploy/kubernetes/chart/templates/thredds/deployment.yaml b/deploy/kubernetes/chart/templates/thredds/deployment.yaml index d45de9ab..15252efb 100644 --- a/deploy/kubernetes/chart/templates/thredds/deployment.yaml +++ b/deploy/kubernetes/chart/templates/thredds/deployment.yaml @@ -26,7 +26,6 @@ spec: {{- if $accessLogSidecar.enabled }} # When the access log sidecar is enabled, roll the deployment if the logstash pipelines or certificates change checksum/logstash-pipelines: {{ include (print $.Template.BasePath "/logstash/pipelines.yaml") . | sha256sum }} - checksum/logstash-certs: {{ include (print $.Template.BasePath "/logstash/certs.yaml") . | sha256sum }} {{- end }} checksum/config: {{ include (print $.Template.BasePath "/thredds/configmap-config.yaml") . | sha256sum }} {{- with $thredds.annotations }} @@ -183,8 +182,6 @@ spec: volumeMounts: - name: logstash-conf mountPath: /etc/logstash/conf.d - - name: logstash-certs - mountPath: /etc/logstash/certs - name: tomcat-logs mountPath: /thredds/logs - name: logstash-data @@ -239,10 +236,6 @@ spec: - name: logstash-conf secret: secretName: {{ include "esgf.component.fullname" (list . "logstash-pipelines") }} - # As are any certificates that are required - - name: logstash-certs - secret: - secretName: {{ include "esgf.component.fullname" (list . "logstash-certs") }} # Logstash also needs a tmp dir and a data dir - name: logstash-data emptyDir: {} diff --git a/deploy/kubernetes/chart/values.yaml b/deploy/kubernetes/chart/values.yaml index 9b4ebdfe..fee1db9b 100644 --- a/deploy/kubernetes/chart/values.yaml +++ b/deploy/kubernetes/chart/values.yaml @@ -264,7 +264,7 @@ data: # The default server is the CMCC server # Your IP must be whitelisted in order to send data to the CMCC service url: https://esgf-logstash.cmcc.it/api/exchanges/%2f/log_exchange/publish - authorizationKey: + authorizationKey: "" # Configuration for the THREDDS pod thredds: