From 3be20aeee938e6656603c58fb8abb574e7a593ea Mon Sep 17 00:00:00 2001 From: wahyu anggana Date: Wed, 7 May 2025 16:00:09 +0700 Subject: [PATCH 1/9] feat(metrics): Adding support metrics using promhttp & opentelemetry --- go.mod | 34 ++++++- go.sum | 62 +++++++++++++ main.go | 38 ++++++++ metrics.go | 232 ++++++++++++++++++++++++++++++++++++++++++++++++ metrics_test.go | 95 ++++++++++++++++++++ proxy.go | 60 ++++++++++--- proxy_test.go | 6 +- 7 files changed, 509 insertions(+), 18 deletions(-) create mode 100644 metrics.go create mode 100644 metrics_test.go diff --git a/go.mod b/go.mod index 3977fde..0de8734 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,35 @@ module github.com/miklosn/clamdproxy -go 1.22 +go 1.22.0 -require github.com/alecthomas/kong v1.9.0 +toolchain go1.22.4 + +require ( + github.com/alecthomas/kong v1.9.0 + github.com/prometheus/client_golang v1.22.0 + github.com/stretchr/testify v1.10.0 + go.opentelemetry.io/otel v1.35.0 + go.opentelemetry.io/otel/exporters/prometheus v0.57.0 + go.opentelemetry.io/otel/metric v1.35.0 + go.opentelemetry.io/otel/sdk/metric v1.35.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel/sdk v1.35.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect + golang.org/x/sys v0.30.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum index 17c1dfb..d779a6b 100644 --- a/go.sum +++ b/go.sum @@ -4,5 +4,67 @@ github.com/alecthomas/kong v1.9.0 h1:Wgg0ll5Ys7xDnpgYBuBn/wPeLGAuK0NvYmEcisJgrIs github.com/alecthomas/kong v1.9.0/go.mod h1:p2vqieVMeTAnaC83txKtXe8FLke2X07aruPWXyMPQrU= github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/exporters/prometheus v0.57.0 h1:AHh/lAP1BHrY5gBwk8ncc25FXWm/gmmY3BX258z5nuk= +go.opentelemetry.io/otel/exporters/prometheus v0.57.0/go.mod h1:QpFWz1QxqevfjwzYdbMb4Y1NnlJvqSGwyuU0B4iuc9c= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index 9d01f42..7cf7c2a 100644 --- a/main.go +++ b/main.go @@ -20,6 +20,7 @@ var cli struct { Backend string `name:"backend" help:"Address of the backend clamd server" default:"127.0.0.1:3311"` LogLevel string `name:"log-level" help:"Log level (debug, info, warn, error)" default:"warn" enum:"debug,info,warn,error"` PprofAddr string `name:"pprof" help:"Address for pprof HTTP server (disabled if empty)" default:""` + MetricsAddr string `name:"metrics" help:"Address for OpenTelemetry metrics HTTP server (disabled if empty)" default:""` AllowedCommands []string `name:"allow-command" help:"ClamAV commands to allow (can be specified multiple times)"` } @@ -49,6 +50,14 @@ func getLogger(logLevel string) *slog.Logger { } func main() { + // Add panic recovery + defer func() { + if r := recover(); r != nil { + logger.Error("Panic recovered", "error", r) + os.Exit(1) + } + }() + // Parse command line arguments with Kong ctx := kong.Parse(&cli) _ = ctx // You can use ctx for subcommands if needed in the future @@ -60,6 +69,22 @@ func main() { // Initialize allowed commands initAllowedCommands() + // Initialize metrics if enabled + if cli.MetricsAddr != "" { + logger.Debug("Initializing metrics", "addr", cli.MetricsAddr) // Add debug log + metrics, err := InitMetrics(cli.MetricsAddr) + if err != nil { + logger.Error("Failed to initialize metrics", "error", err) + os.Exit(1) + } + defer func() { + if err := metrics.Close(); err != nil { + logger.Error("Failed to close metrics server", "error", err) + } + }() + logger.Info("Metrics enabled", "addr", cli.MetricsAddr) + } + logger.Warn("Starting clamdproxy", "listen", cli.Listen, "backend", cli.Backend) @@ -104,17 +129,30 @@ func handleConnection(clientConn net.Conn) { if err := clientConn.Close(); err != nil { logger.Error("Failed to close client connection", "error", err) } + // Record connection closed in metrics + if proxyMetrics != nil { + proxyMetrics.RecordConnectionClosed() + } }() clientAddr := clientConn.RemoteAddr() logger.Info("Connection established", "client", clientAddr) + // Record connection in metrics + if proxyMetrics != nil { + proxyMetrics.RecordConnection(clientAddr.String()) + } + backendConn, err := net.Dial("tcp", cli.Backend) if err != nil { logger.Error("Failed to connect to backend", "backend", &cli.Backend, "client", &clientAddr, "error", err) + // Record backend error in metrics + if proxyMetrics != nil { + proxyMetrics.RecordBackendError() + } return } defer func() { diff --git a/metrics.go b/metrics.go new file mode 100644 index 0000000..91d40c7 --- /dev/null +++ b/metrics.go @@ -0,0 +1,232 @@ +// Package main implements metrics collection for clamdproxy using OpenTelemetry +package main + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/prometheus" + "go.opentelemetry.io/otel/metric" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" +) + +// Metrics holds all the metrics instruments used by the proxy +type Metrics struct { + ConnectionsTotal metric.Int64Counter + CommandsTotal metric.Int64Counter + CommandDuration metric.Float64Histogram + ActiveConnections metric.Int64UpDownCounter + BackendErrors metric.Int64Counter + CommandErrors metric.Int64Counter + FilesScanned metric.Int64Counter + FilesSizeBytes metric.Int64Counter + FilesWithVirus metric.Int64Counter + metricServer *http.Server +} + +// Global metrics instance +var proxyMetrics *Metrics + +// InitMetrics initializes the OpenTelemetry metrics system and creates all metric instruments +func InitMetrics(metricsAddr string) (*Metrics, error) { + // Create a new Prometheus exporter + exporter, err := prometheus.New() + if err != nil { + return nil, fmt.Errorf("failed to create Prometheus exporter: %w", err) + } + + // Create a new MeterProvider with the Prometheus exporter + provider := sdkmetric.NewMeterProvider( + sdkmetric.WithReader(exporter), + ) + + // Set the global MeterProvider + otel.SetMeterProvider(provider) + + // Create a meter for our application + meter := provider.Meter("clamdproxy") + + // Create metrics + m := &Metrics{} + + // Initialize counters + m.ConnectionsTotal, err = meter.Int64Counter( + "clamdproxy_connections_total", + metric.WithDescription("Total number of client connections"), + metric.WithUnit("{connections}"), + ) + if err != nil { + return nil, fmt.Errorf("failed to create connections counter: %w", err) + } + + m.CommandsTotal, err = meter.Int64Counter( + "clamdproxy_commands_total", + metric.WithDescription("Total number of commands processed"), + metric.WithUnit("{commands}"), + ) + if err != nil { + return nil, fmt.Errorf("failed to create commands counter: %w", err) + } + + m.CommandDuration, err = meter.Float64Histogram( + "clamdproxy_command_duration_seconds", + metric.WithDescription("Duration of command processing in seconds"), + metric.WithUnit("s"), + ) + if err != nil { + return nil, fmt.Errorf("failed to create command duration histogram: %w", err) + } + + m.ActiveConnections, err = meter.Int64UpDownCounter( + "clamdproxy_active_connections", + metric.WithDescription("Current number of active connections"), + metric.WithUnit("{connections}"), + ) + if err != nil { + return nil, fmt.Errorf("failed to create active connections counter: %w", err) + } + + m.BackendErrors, err = meter.Int64Counter( + "clamdproxy_backend_errors_total", + metric.WithDescription("Total number of backend connection errors"), + metric.WithUnit("{errors}"), + ) + if err != nil { + return nil, fmt.Errorf("failed to create backend errors counter: %w", err) + } + + m.CommandErrors, err = meter.Int64Counter( + "clamdproxy_command_errors_total", + metric.WithDescription("Total number of command processing errors"), + metric.WithUnit("{errors}"), + ) + if err != nil { + return nil, fmt.Errorf("failed to create command errors counter: %w", err) + } + + // Remove the initialization of BytesTransferred and file scanning metrics + // The following section should be removed: + // m.BytesTransferred = ... + // m.FilesScanned = ... + // m.FilesSizeBytes = ... + // m.FilesWithVirus = ... + + // Start metrics HTTP server if address is provided + if metricsAddr != "" { + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + + m.metricServer = &http.Server{ + Addr: metricsAddr, + Handler: mux, + } + + go func() { + logger.Info("Starting metrics server", "addr", metricsAddr) + if err := m.metricServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { + logger.Error("Metrics server failed", "error", err) + } + }() + } + + proxyMetrics = m + return m, nil +} + +// RecordConnection records a new client connection +func (m *Metrics) RecordConnection(clientAddr string) { + if m == nil { + return + } + m.ConnectionsTotal.Add(context.Background(), 1, metric.WithAttributes( + attribute.String("client", clientAddr), + )) + m.ActiveConnections.Add(context.Background(), 1) +} + +// RecordConnectionClosed records a closed client connection +func (m *Metrics) RecordConnectionClosed() { + if m == nil { + return + } + m.ActiveConnections.Add(context.Background(), -1) +} + +// RecordCommand records a command being processed +func (m *Metrics) RecordCommand(cmd string, allowed bool) { + if m == nil { + return + } + m.CommandsTotal.Add(context.Background(), 1, metric.WithAttributes( + attribute.String("command", cmd), + attribute.Bool("allowed", allowed), + )) +} + +// RecordCommandDuration records the duration of a command +func (m *Metrics) RecordCommandDuration(cmd string, duration time.Duration) { + if m == nil { + return + } + m.CommandDuration.Record(context.Background(), duration.Seconds(), metric.WithAttributes( + attribute.String("command", cmd), + )) +} + +// RecordBackendError records a backend connection error +func (m *Metrics) RecordBackendError() { + if m == nil { + return + } + m.BackendErrors.Add(context.Background(), 1) +} + +// RecordCommandError records a command processing error +func (m *Metrics) RecordCommandError(cmd string, errMsg string) { + if m == nil { + return + } + m.CommandErrors.Add(context.Background(), 1, metric.WithAttributes( + attribute.String("command", cmd), + attribute.String("error", errMsg), + )) +} + +// RecordFileScan records metrics for a scanned file +func (m *Metrics) RecordFileScan(filename string, sizeBytes int64, virusFound bool, virusName string) { + if m == nil { + return + } + + attrs := []attribute.KeyValue{ + attribute.String("filename", filename), + } + + if virusFound { + attrs = append(attrs, attribute.String("virus_name", virusName)) + } + + m.FilesScanned.Add(context.Background(), 1, metric.WithAttributes(attrs...)) + m.FilesSizeBytes.Add(context.Background(), sizeBytes, metric.WithAttributes(attrs...)) + + if virusFound { + m.FilesWithVirus.Add(context.Background(), 1, metric.WithAttributes(attrs...)) + } +} + +// Close shuts down the metrics server if it's running +func (m *Metrics) Close() error { + if m == nil || m.metricServer == nil { + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + return m.metricServer.Shutdown(ctx) +} diff --git a/metrics_test.go b/metrics_test.go new file mode 100644 index 0000000..59cdb51 --- /dev/null +++ b/metrics_test.go @@ -0,0 +1,95 @@ +package main + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMetrics(t *testing.T) { + // Initialize metrics with a test server + metrics, err := InitMetrics("localhost:0") + require.NoError(t, err) + require.NotNil(t, metrics) + defer func() { + err := metrics.Close() + require.NoError(t, err) + }() + + t.Run("RecordConnection", func(t *testing.T) { + metrics.RecordConnection("127.0.0.1:1234") + // Verify metrics through HTTP endpoint + resp := getMetrics(t, metrics) + assert.Contains(t, resp, `clamdproxy_connections_total{client="127.0.0.1:1234"`) + assert.Contains(t, resp, `clamdproxy_active_connections`) + }) + + t.Run("RecordConnectionClosed", func(t *testing.T) { + metrics.RecordConnection("127.0.0.1:1234") + metrics.RecordConnectionClosed() + resp := getMetrics(t, metrics) + assert.Contains(t, resp, `clamdproxy_active_connections`) + }) + + t.Run("RecordCommand", func(t *testing.T) { + metrics.RecordCommand("PING", true) + resp := getMetrics(t, metrics) + assert.Contains(t, resp, `clamdproxy_commands_total{allowed="true",command="PING",otel_scope_name="clamdproxy"`) + }) + + t.Run("RecordCommandError", func(t *testing.T) { + metrics.RecordCommandError("SCAN", "connection refused") + resp := getMetrics(t, metrics) + assert.Contains(t, resp, `clamdproxy_command_errors_total{command="SCAN",error="connection refused",otel_scope_name="clamdproxy"`) + }) +} + +// Helper function to get metrics from the HTTP endpoint +func getMetrics(t *testing.T, m *Metrics) string { + req := httptest.NewRequest("GET", "/metrics", nil) + w := httptest.NewRecorder() + + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + m.metricServer.Handler.ServeHTTP(w, r) + }) + + handler.ServeHTTP(w, req) + resp := w.Result() + defer func() { + err := resp.Body.Close() + require.NoError(t, err) + }() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("Expected status code 200, got %d", resp.StatusCode) + } + + body := new(strings.Builder) + _, err := io.Copy(body, resp.Body) + require.NoError(t, err) + + return body.String() +} + +func TestNilMetrics(t *testing.T) { + var m *Metrics + + // Test that nil metrics don't panic + t.Run("NilMetricsOperations", func(t *testing.T) { + assert.NotPanics(t, func() { + m.RecordConnection("test") + m.RecordConnectionClosed() + m.RecordCommand("test", true) + m.RecordCommandDuration("test", time.Second) + m.RecordBackendError() + m.RecordCommandError("test", "error") + _ = m.Close() + }) + }) +} diff --git a/proxy.go b/proxy.go index bd71e72..6cf61ca 100644 --- a/proxy.go +++ b/proxy.go @@ -10,6 +10,7 @@ import ( "strings" "sync" "syscall" + "time" ) // Buffer pools to reduce GC pressure @@ -107,16 +108,19 @@ func (p *ClamdProxy) Start() { bytesWritten += int64(nw) } if ew != nil { + logger.Error("Error writing to client buffer", "error", ew) err = ew break } if nr != nw { + logger.Error("Short write to client buffer", "expected", nr, "written", nw) err = io.ErrShortWrite break } } if er != nil { if er != io.EOF { + logger.Error("Error reading from backend", "error", er) err = er } break @@ -125,30 +129,24 @@ func (p *ClamdProxy) Start() { // Flush the buffer periodically to avoid delays if p.clientBuf.Buffered() > 32*1024 { if err := p.clientBuf.Flush(); err != nil { - logger.Debug("Error flushing buffer to client", "error", err) + logger.Error("Error flushing buffer to client", "error", err) } } } // Final flush if err := p.clientBuf.Flush(); err != nil { - logger.Debug("Error flushing final buffer to client", "error", err) + logger.Error("Error flushing final buffer to client", "error", err) } if err != nil { if isConnectionClosed(err) { - logger.Debug("Backend connection closed", - "client", clientAddr, - "error", err) + logger.Debug("Backend connection closed", "client", clientAddr, "error", err) } else { - logger.Debug("Error copying from backend to client", - "client", clientAddr, - "error", err) + logger.Error("Error copying from backend to client", "client", clientAddr, "error", err) } } else { - logger.Info("Proxy completed", - "client", clientAddr, - "bytesTransferred", bytesWritten) + logger.Info("Proxy completed", "client", clientAddr, "bytesTransferred", bytesWritten) } } @@ -171,6 +169,10 @@ func (p *ClamdProxy) handleClientToBackend() { logger.Debug("Client connection closed", "client", clientAddr, "error", err) } else { logger.Debug("Error reading command", "client", clientAddr, "error", err) + // Record command error in metrics + if proxyMetrics != nil { + proxyMetrics.RecordCommandError("unknown", err.Error()) + } } } // Close the backend connection to signal we're done @@ -183,11 +185,23 @@ func (p *ClamdProxy) handleClientToBackend() { // Only log commands at appropriate levels logger.Debug("Command received", "client", clientAddr, "command", cmd) - // Check if command is allowed - if isCommandAllowed(cmd) { + // Check if command is allowed and record start time + allowed := isCommandAllowed(cmd) + commandStart := time.Now() + + // Record command metrics + if proxyMetrics != nil { + proxyMetrics.RecordCommand(cmd, allowed) + } + + if allowed { // Forward the command to backend using buffered writer if _, err := p.backendBuf.Write(append([]byte(cmd), delim)); err != nil { logger.Debug("Error forwarding command", "error", err) + // Record command error in metrics + if proxyMetrics != nil { + proxyMetrics.RecordCommandError(cmd, err.Error()) + } break } // Flush after each command to ensure it's sent immediately @@ -196,16 +210,31 @@ func (p *ClamdProxy) handleClientToBackend() { break } + // Record command duration + if proxyMetrics != nil && !isInstreamCommand(cmd) { + proxyMetrics.RecordCommandDuration(cmd, time.Since(commandStart)) + } + // Handle special case for INSTREAM command (file streaming) if isInstreamCommand(cmd) { logger.Debug("Processing INSTREAM data", "client", clientAddr) + instreamStart := time.Now() if err := p.handleInstream(reader); err != nil { logger.Debug("Error handling INSTREAM data", "client", clientAddr, "error", err) + // Record command error in metrics + if proxyMetrics != nil { + proxyMetrics.RecordCommandError(cmd, err.Error()) + } break } + + // Record INSTREAM command duration after completion + if proxyMetrics != nil { + proxyMetrics.RecordCommandDuration(cmd, time.Since(instreamStart)) + } } } else { logger.Info("Blocked command", "client", clientAddr, "command", cmd) @@ -215,6 +244,11 @@ func (p *ClamdProxy) handleClientToBackend() { logger.Debug("Error sending error response", "error", err) break } + + // Record command duration for blocked commands too + if proxyMetrics != nil { + proxyMetrics.RecordCommandDuration(cmd, time.Since(commandStart)) + } if err := p.clientBuf.Flush(); err != nil { logger.Debug("Error flushing error response", "error", err) break diff --git a/proxy_test.go b/proxy_test.go index 3ebf165..bd4addb 100644 --- a/proxy_test.go +++ b/proxy_test.go @@ -90,12 +90,12 @@ func TestIsCommandAllowed(t *testing.T) { for k, v := range allowedCommands { originalAllowed[k] = v } - + // Restore after test defer func() { allowedCommands = originalAllowed }() - + // Set test allowed commands allowedCommands = map[string]bool{ "PING": true, @@ -103,7 +103,7 @@ func TestIsCommandAllowed(t *testing.T) { "VERSIONCOMMANDS": true, "INSTREAM": true, } - + allowedCmds := []string{ "PING", "VERSION", "VERSIONCOMMANDS", "INSTREAM", "zPING", "zVERSION", "zVERSIONCOMMANDS", "zINSTREAM", From fe4f51053cf0302132b34711073ab30fac35ed70 Mon Sep 17 00:00:00 2001 From: wahyu anggana Date: Wed, 7 May 2025 16:00:26 +0700 Subject: [PATCH 2/9] feat(monitoring): Adding docker compose, Dockerfile, Grafana Dashboard template & prometheus scrape config --- Dockerfile | 20 + docker-compose.yml | 71 ++ grafana.json | 1702 ++++++++++++++++++++++++++++++++++++++++++++ prometheus.yml | 9 + 4 files changed, 1802 insertions(+) create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 grafana.json create mode 100644 prometheus.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c51c155 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +FROM golang:1.22-alpine AS builder + +WORKDIR /app +COPY go.mod go.sum ./ + +RUN go mod download +COPY *.go ./ +RUN CGO_ENABLED=0 GOOS=linux go build -o clamdproxy . + +FROM alpine:latest + +RUN apk --no-cache add ca-certificates +WORKDIR /app +COPY --from=builder /app/clamdproxy /app/ + +EXPOSE 3315 2112 +ENTRYPOINT ["/app/clamdproxy"] + +# clamav:3310 get from docker-compose url +CMD ["--listen", "0.0.0.0:3315", "--backend", "clamav:3310", "--log-level", "debug", "--metrics", "0.0.0.0:2112"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8039ca5 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,71 @@ +version: '3.8' + +services: + clamav: + image: mkodockx/docker-clamav:alpine + container_name: clamav-docker + ports: + - "3310:3310" + volumes: + - ./testfiles:/scan + - clamav_data:/var/lib/clamav + restart: unless-stopped + healthcheck: + test: ["CMD", "./check.sh"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 120s + networks: + - clam-net + clamdproxy: + build: + context: . + dockerfile: Dockerfile + container_name: clamdproxy + ports: + - "3315:3315" + - "2112:2112" + depends_on: + clamav: + condition: service_healthy + restart: unless-stopped + networks: + - clam-net + + prometheus: + image: prom/prometheus:latest + ports: + - "9090:9090" + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--storage.tsdb.retention.time=15d' + networks: + - clam-net + + grafana: + image: grafana/grafana:latest + ports: + - "3001:3000" + environment: + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_SECURITY_ADMIN_USER=admin + volumes: + - grafana_data:/var/lib/grafana + depends_on: + - prometheus + networks: + - clam-net + +volumes: + clamav_data: + prometheus_data: + grafana_data: + +networks: + clam-net: + driver: bridge \ No newline at end of file diff --git a/grafana.json b/grafana.json new file mode 100644 index 0000000..a3ba7b9 --- /dev/null +++ b/grafana.json @@ -0,0 +1,1702 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Process status published by Go Prometheus client library, e.g. memory used, fds open, GC details", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 12, + "panels": [], + "title": "ClamD Dashboard", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 16, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "uid": "Prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(job) (clamdproxy_connections_total{job=\"clamdproxy\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "status {{code}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Connections", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 17, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "uid": "Prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(job) (clamdproxy_active_connections{job=\"clamdproxy\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "status {{code}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Active Connections", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 13, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "sum by(code) (\n promhttp_metric_handler_requests_total{code=\"200\"}\n)\n", + "legendFormat": "status {{code}}", + "range": true, + "refId": "A" + } + ], + "title": "http status 200", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 14, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "sum by(code) (\n promhttp_metric_handler_requests_total{code=\"500\"}\n)\n", + "legendFormat": "status {{code}}", + "range": true, + "refId": "A" + } + ], + "title": "http status 500", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 15, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "uid": "Prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(job) (clamdproxy_backend_errors_total{job=\"clamdproxy\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "status {{code}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Backend Error Total", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(command) (increase(clamdproxy_commands_total{allowed=\"true\"}[1m]))", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "{{command}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Command Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 11, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "titleSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "uid": "Prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(command) (clamdproxy_commands_total{allowed=\"true\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{command}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Commands Total", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 10, + "panels": [], + "title": "Go Process", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "resident" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "resident" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "resident" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "expr": "process_resident_memory_bytes{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}} - resident", + "metric": "process_resident_memory_bytes", + "refId": "A", + "step": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "expr": "process_virtual_memory_bytes{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}} - virtual", + "metric": "process_virtual_memory_bytes", + "refId": "B", + "step": 4 + } + ], + "title": "process memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "expr": "rate(process_resident_memory_bytes{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}} - resident", + "metric": "process_resident_memory_bytes", + "refId": "A", + "step": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "expr": "deriv(process_virtual_memory_bytes{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}} - virtual", + "metric": "process_virtual_memory_bytes", + "refId": "B", + "step": 4 + } + ], + "title": "process memory deriv", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "alloc rate" + }, + "properties": [ + { + "id": "unit", + "value": "Bps" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "alloc rate" + }, + "properties": [ + { + "id": "unit", + "value": "Bps" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "alloc rate" + }, + "properties": [ + { + "id": "unit", + "value": "Bps" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "expr": "go_memstats_alloc_bytes{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}} - bytes allocated", + "metric": "go_memstats_alloc_bytes", + "refId": "A", + "step": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "expr": "rate(go_memstats_alloc_bytes_total{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}[30s])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}} - alloc rate", + "metric": "go_memstats_alloc_bytes_total", + "refId": "B", + "step": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "expr": "go_memstats_stack_inuse_bytes{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}} - stack inuse", + "metric": "go_memstats_stack_inuse_bytes", + "refId": "C", + "step": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "expr": "go_memstats_heap_inuse_bytes{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{pod}} - heap inuse", + "metric": "go_memstats_heap_inuse_bytes", + "refId": "D", + "step": 4 + } + ], + "title": "go memstats", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "alloc rate" + }, + "properties": [ + { + "id": "unit", + "value": "Bps" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "alloc rate" + }, + "properties": [ + { + "id": "unit", + "value": "Bps" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "alloc rate" + }, + "properties": [ + { + "id": "unit", + "value": "Bps" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "expr": "deriv(go_memstats_alloc_bytes{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}} - bytes allocated", + "metric": "go_memstats_alloc_bytes", + "refId": "A", + "step": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "expr": "rate(go_memstats_alloc_bytes_total{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}} - alloc rate", + "metric": "go_memstats_alloc_bytes_total", + "refId": "B", + "step": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "expr": "deriv(go_memstats_stack_inuse_bytes{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}} - stack inuse", + "metric": "go_memstats_stack_inuse_bytes", + "refId": "C", + "step": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "expr": "deriv(go_memstats_heap_inuse_bytes{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}[$interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{pod}} - heap inuse", + "metric": "go_memstats_heap_inuse_bytes", + "refId": "D", + "step": 4 + } + ], + "title": "go memstats deriv", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "expr": "process_open_fds{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "metric": "process_open_fds", + "refId": "A", + "step": 4 + } + ], + "title": "open fds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "expr": "deriv(process_open_fds{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "metric": "process_open_fds", + "refId": "A", + "step": 4 + } + ], + "title": "open fds deriv", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "expr": "go_goroutines{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "metric": "go_goroutines", + "refId": "A", + "step": 4 + } + ], + "title": "Goroutines", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fel1813ec2cjkd" + }, + "expr": "go_gc_duration_seconds{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}: {{quantile}}", + "metric": "go_gc_duration_seconds", + "refId": "A", + "step": 4 + } + ], + "title": "GC duration quantiles", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "fel1813ec2cjkd", + "includeAll": true, + "multi": true, + "name": "namespace", + "options": [], + "query": "label_values(go_memstats_alloc_bytes, namespace)", + "refresh": 2, + "regex": "", + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "fel1813ec2cjkd", + "includeAll": true, + "multi": true, + "name": "pod", + "options": [], + "query": "label_values(process_resident_memory_bytes, pod)", + "refresh": 2, + "regex": "", + "type": "query" + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "1m,5m,10m,30m,1h", + "refresh": 2, + "type": "interval" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Go Processes", + "uid": "ypFZFgvmz", + "version": 2 +} \ No newline at end of file diff --git a/prometheus.yml b/prometheus.yml new file mode 100644 index 0000000..770405e --- /dev/null +++ b/prometheus.yml @@ -0,0 +1,9 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: 'clamdproxy' + static_configs: + # docker.for.mac.localhost is the IP address of the Docker host on macOS. + - targets: ['docker.for.mac.localhost:2112'] \ No newline at end of file From 0a4d8288830f9752648e4be620db14b2c9c4d216 Mon Sep 17 00:00:00 2001 From: wahyu anggana Date: Wed, 7 May 2025 23:54:48 +0700 Subject: [PATCH 3/9] chore(metrics): change Error Logger to Debug Logger & Enable function RecordFileScan --- metrics.go | 33 +++++++++-- proxy.go | 161 +++++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 159 insertions(+), 35 deletions(-) diff --git a/metrics.go b/metrics.go index 91d40c7..98cfc08 100644 --- a/metrics.go +++ b/metrics.go @@ -109,12 +109,33 @@ func InitMetrics(metricsAddr string) (*Metrics, error) { return nil, fmt.Errorf("failed to create command errors counter: %w", err) } - // Remove the initialization of BytesTransferred and file scanning metrics - // The following section should be removed: - // m.BytesTransferred = ... - // m.FilesScanned = ... - // m.FilesSizeBytes = ... - // m.FilesWithVirus = ... + // Initialize file scanning metrics + m.FilesScanned, err = meter.Int64Counter( + "clamdproxy_files_scanned_total", + metric.WithDescription("Total number of files scanned"), + metric.WithUnit("{files}"), + ) + if err != nil { + return nil, fmt.Errorf("failed to create files scanned counter: %w", err) + } + + m.FilesSizeBytes, err = meter.Int64Counter( + "clamdproxy_files_size_bytes_total", + metric.WithDescription("Total size of files scanned in bytes"), + metric.WithUnit("By"), + ) + if err != nil { + return nil, fmt.Errorf("failed to create file size counter: %w", err) + } + + m.FilesWithVirus, err = meter.Int64Counter( + "clamdproxy_files_with_virus_total", + metric.WithDescription("Total number of files with viruses detected"), + metric.WithUnit("{files}"), + ) + if err != nil { + return nil, fmt.Errorf("failed to create files with virus counter: %w", err) + } // Start metrics HTTP server if address is provided if metricsAddr != "" { diff --git a/proxy.go b/proxy.go index 6cf61ca..c343848 100644 --- a/proxy.go +++ b/proxy.go @@ -63,15 +63,17 @@ var allowedCommands = map[string]bool{ "INSTREAM": true, "VERSION": true, "VERSIONCOMMANDS": true, + "": true, // Allow empty commands } // ClamdProxy handles bidirectional proxying between client and backend clamd server. // It filters commands to prevent unsafe operations from reaching the backend. type ClamdProxy struct { - client net.Conn // Connection to the client - backend net.Conn // Connection to the backend clamd server - backendBuf *bufio.Writer // Buffered writer for backend - clientBuf *bufio.Writer // Buffered writer for client + client net.Conn // Connection to the client + backend net.Conn // Connection to the backend clamd server + backendBuf *bufio.Writer // Buffered writer for backend + clientBuf *bufio.Writer // Buffered writer for client + lastStreamSize int64 // Size of the last streamed file } // NewClamdProxy creates a new proxy instance with the given client and backend connections @@ -100,27 +102,46 @@ func (p *ClamdProxy) Start() { bytesWritten := int64(0) var err error + // Buffer to accumulate response for parsing + responseBuf := make([]byte, 0, 1024) + for { nr, er := p.backend.Read(buf) if nr > 0 { + // Add to response buffer for parsing + responseBuf = append(responseBuf, buf[:nr]...) + + // Log the raw response for debugging + logger.Debug("Received response from backend", + "bytes", nr, + "preview", truncateString(string(buf[:nr]), 100)) + + // Check for virus detection in the response + p.parseResponseForVirus(responseBuf) + + // Clear the response buffer if it gets too large or contains a complete response + if len(responseBuf) > 4096 || strings.Contains(string(responseBuf), "\n") { + responseBuf = make([]byte, 0, 1024) + } + nw, ew := p.clientBuf.Write(buf[0:nr]) if nw > 0 { bytesWritten += int64(nw) } if ew != nil { - logger.Error("Error writing to client buffer", "error", ew) + logger.Debug("Error writing to client buffer", "error", ew) err = ew break } if nr != nw { - logger.Error("Short write to client buffer", "expected", nr, "written", nw) + logger.Debug("Short write to client buffer", "expected", nr, "written", nw) err = io.ErrShortWrite break } } if er != nil { if er != io.EOF { - logger.Error("Error reading from backend", "error", er) + logger.Debug("Error reading from backend", "error", er) err = er } break @@ -129,21 +150,21 @@ func (p *ClamdProxy) Start() { // Flush the buffer periodically to avoid delays if p.clientBuf.Buffered() > 32*1024 { if err := p.clientBuf.Flush(); err != nil { - logger.Error("Error flushing buffer to client", "error", err) + logger.Debug("Error flushing buffer to client", "error", err) } } } // Final flush if err := p.clientBuf.Flush(); err != nil { - logger.Error("Error flushing final buffer to client", "error", err) + logger.Debug("Error flushing final buffer to client", "error", err) } if err != nil { if isConnectionClosed(err) { logger.Debug("Backend connection closed", "client", clientAddr, "error", err) } else { - logger.Error("Error copying from backend to client", "client", clientAddr, "error", err) + logger.Debug("Error copying from backend to client", "client", clientAddr, "error", err) } } else { logger.Info("Proxy completed", "client", clientAddr, "bytesTransferred", bytesWritten) @@ -182,6 +203,11 @@ func (p *ClamdProxy) handleClientToBackend() { break } + // Skip empty commands + if cmd == "" { + continue + } + // Only log commands at appropriate levels logger.Debug("Command received", "client", clientAddr, "command", cmd) @@ -237,7 +263,7 @@ func (p *ClamdProxy) handleClientToBackend() { } } } else { - logger.Info("Blocked command", "client", clientAddr, "command", cmd) + logger.Debug("Blocked command", "client", clientAddr, "command", cmd) // Changed from Info to Debug // Send error response to client using buffered writer response := "ERROR: Command not allowed\n" if _, err := p.clientBuf.WriteString(response); err != nil { @@ -407,27 +433,104 @@ func (p *ClamdProxy) handleInstream(reader *bufio.Reader) error { totalBytes += size chunks++ - - // Only log chunk details at the most verbose level and only occasionally - if chunks%100 == 0 { - logger.Debug("INSTREAM progress", - "client", clientAddr, - "chunks", chunks, - "totalBytes", totalBytes) - } - - // Flush periodically to balance between batching and responsiveness - if chunks%10 == 0 { - if err := p.backendBuf.Flush(); err != nil { - return fmt.Errorf("failed to flush data: %w", err) - } - } } - // Final flush to ensure all data is sent + // After the INSTREAM command completes successfully, store the final file size + p.lastStreamSize = int64(totalBytes) + + // Flush the backend buffer to ensure all data is sent if err := p.backendBuf.Flush(); err != nil { - return fmt.Errorf("failed to flush final data: %w", err) + return fmt.Errorf("failed to flush backend buffer: %w", err) } - + return nil } + +// parseResponseForVirus checks if the response contains virus detection information +// and records metrics if a virus is found +func (p *ClamdProxy) parseResponseForVirus(response []byte) { + // Convert to string for easier parsing + respStr := string(response) + + // Add debug logging to see what's in the response + logger.Debug("Parsing response", "length", len(respStr), "preview", truncateString(respStr, 100)) + + // Check if this is an INSTREAM response (either clean or with virus) + if strings.Contains(respStr, "stream") { + // Default to clean file (no virus) + virusFound := false + virusName := "" + filename := "stream" + + // Check for virus detection + if strings.Contains(respStr, " FOUND") { + logger.Debug("Found virus detection pattern in response") + lines := strings.Split(respStr, "\n") + for _, line := range lines { + if strings.Contains(line, " FOUND") { + logger.Debug("Processing virus line", "line", line) + parts := strings.Split(line, ": ") + if len(parts) >= 2 { + fileInfo := parts[0] + detectionInfo := strings.TrimSuffix(parts[1], " FOUND") + + // Extract filename from fileInfo + // Format: "instream(172.18.0.5@35478)" + filename = fileInfo + if strings.Contains(fileInfo, "(") && strings.Contains(fileInfo, ")") { + // Extract the part between parentheses + start := strings.Index(fileInfo, "(") + 1 + end := strings.Index(fileInfo, ")") + if start > 0 && end > start { + clientInfo := fileInfo[start:end] + // Use client info as part of the filename + filename = fileInfo[:start-1] + "_" + clientInfo + } + } + + virusFound = true + virusName = detectionInfo + + logger.Info("Virus detected", + "file", filename, + "virus", detectionInfo, + "size", p.lastStreamSize) + } + } + } + } else if strings.Contains(respStr, "OK") { + // This is a clean file + logger.Debug("Clean file detected", "size", p.lastStreamSize) + } + + // Record metrics for all scanned files, whether clean or infected + if proxyMetrics != nil { + // Use the actual file size if available + fileSize := p.lastStreamSize + if fileSize == 0 { + fileSize = int64(1024) // Default 1KB if size unknown + } + + proxyMetrics.RecordFileScan(filename, fileSize, virusFound, virusName) + + if virusFound { + logger.Debug("Recorded virus metrics", + "file", filename, + "size", fileSize, + "virus", virusName) + } else { + logger.Debug("Recorded clean file metrics", + "file", filename, + "size", fileSize) + } + } + } +} + +// Helper function to truncate long strings for logging +func truncateString(s string, maxLen int) string { + if len(s) <= maxLen { + return s + } + return s[:maxLen] + "..." +} From 8b7b588f28665be743c29051517cefbbfcec07e7 Mon Sep 17 00:00:00 2001 From: wahyu anggana Date: Wed, 7 May 2025 23:58:40 +0700 Subject: [PATCH 4/9] chore(docker): move docker compose and configs into docker folder --- Dockerfile => docker/Dockerfile | 4 +- .../docker-compose.yml | 4 +- grafana.json => docker/grafana.json | 365 +++++++++++++----- docker/prometheus.yml | 8 + prometheus.yml | 9 - proxy.go | 156 ++++---- 6 files changed, 356 insertions(+), 190 deletions(-) rename Dockerfile => docker/Dockerfile (96%) rename docker-compose.yml => docker/docker-compose.yml (96%) rename grafana.json => docker/grafana.json (90%) create mode 100644 docker/prometheus.yml delete mode 100644 prometheus.yml diff --git a/Dockerfile b/docker/Dockerfile similarity index 96% rename from Dockerfile rename to docker/Dockerfile index c51c155..aee68c8 100644 --- a/Dockerfile +++ b/docker/Dockerfile @@ -1,10 +1,10 @@ FROM golang:1.22-alpine AS builder WORKDIR /app -COPY go.mod go.sum ./ +COPY go.mod go.sum ./ RUN go mod download -COPY *.go ./ +COPY ./*.go ./ RUN CGO_ENABLED=0 GOOS=linux go build -o clamdproxy . FROM alpine:latest diff --git a/docker-compose.yml b/docker/docker-compose.yml similarity index 96% rename from docker-compose.yml rename to docker/docker-compose.yml index 8039ca5..46e0e35 100644 --- a/docker-compose.yml +++ b/docker/docker-compose.yml @@ -20,8 +20,8 @@ services: - clam-net clamdproxy: build: - context: . - dockerfile: Dockerfile + context: ../ + dockerfile: docker/Dockerfile container_name: clamdproxy ports: - "3315:3315" diff --git a/grafana.json b/docker/grafana.json similarity index 90% rename from grafana.json rename to docker/grafana.json index a3ba7b9..8a3a31a 100644 --- a/grafana.json +++ b/docker/grafana.json @@ -19,7 +19,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 3, + "id": 1, "links": [], "panels": [ { @@ -38,7 +38,168 @@ { "datasource": { "type": "prometheus", - "uid": "fel1813ec2cjkd" + "uid": "ael6f69gclslca" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ael6f69gclslca" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max by(virus_name) (clamdproxy_files_with_virus_total{virus_name!=\"\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Virus Founded", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ael6f69gclslca" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 11, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "titleSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "uid": "Prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(command) (clamdproxy_commands_total{allowed=\"true\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{command}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Commands Total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ael6f69gclslca" }, "fieldConfig": { "defaults": { @@ -61,7 +222,7 @@ "h": 6, "w": 3, "x": 0, - "y": 1 + "y": 9 }, "id": 16, "options": { @@ -102,7 +263,7 @@ { "datasource": { "type": "prometheus", - "uid": "fel1813ec2cjkd" + "uid": "ael6f69gclslca" }, "fieldConfig": { "defaults": { @@ -125,7 +286,7 @@ "h": 6, "w": 3, "x": 3, - "y": 1 + "y": 9 }, "id": 17, "options": { @@ -166,7 +327,7 @@ { "datasource": { "type": "prometheus", - "uid": "fel1813ec2cjkd" + "uid": "ael6f69gclslca" }, "fieldConfig": { "defaults": { @@ -189,7 +350,7 @@ "h": 6, "w": 3, "x": 6, - "y": 1 + "y": 9 }, "id": 13, "options": { @@ -226,7 +387,72 @@ { "datasource": { "type": "prometheus", - "uid": "fel1813ec2cjkd" + "uid": "ael6f69gclslca" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange" + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 9, + "y": 9 + }, + "id": 21, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "uid": "Prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(clamdproxy_files_size_bytes_total)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "status {{code}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total File Scanned", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ael6f69gclslca" }, "fieldConfig": { "defaults": { @@ -248,8 +474,8 @@ "gridPos": { "h": 6, "w": 3, - "x": 9, - "y": 1 + "x": 12, + "y": 9 }, "id": 14, "options": { @@ -286,7 +512,7 @@ { "datasource": { "type": "prometheus", - "uid": "fel1813ec2cjkd" + "uid": "ael6f69gclslca" }, "fieldConfig": { "defaults": { @@ -308,8 +534,8 @@ "gridPos": { "h": 6, "w": 3, - "x": 12, - "y": 1 + "x": 15, + "y": 9 }, "id": 15, "options": { @@ -350,7 +576,7 @@ { "datasource": { "type": "prometheus", - "uid": "fel1813ec2cjkd" + "uid": "ael6f69gclslca" }, "fieldConfig": { "defaults": { @@ -412,7 +638,7 @@ "h": 8, "w": 12, "x": 0, - "y": 7 + "y": 15 }, "id": 9, "options": { @@ -452,80 +678,13 @@ "title": "Command Rate", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "fel1813ec2cjkd" - }, - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 7 - }, - "id": 11, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": { - "titleSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "uid": "Prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum by(command) (clamdproxy_commands_total{allowed=\"true\"})", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "{{command}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Commands Total", - "type": "stat" - }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 15 + "y": 23 }, "id": 10, "panels": [], @@ -535,7 +694,7 @@ { "datasource": { "type": "prometheus", - "uid": "fel1813ec2cjkd" + "uid": "ael6f69gclslca" }, "fieldConfig": { "defaults": { @@ -645,7 +804,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 24 }, "id": 1, "options": { @@ -700,7 +859,7 @@ { "datasource": { "type": "prometheus", - "uid": "fel1813ec2cjkd" + "uid": "ael6f69gclslca" }, "fieldConfig": { "defaults": { @@ -761,7 +920,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 24 }, "id": 4, "options": { @@ -816,7 +975,7 @@ { "datasource": { "type": "prometheus", - "uid": "fel1813ec2cjkd" + "uid": "ael6f69gclslca" }, "fieldConfig": { "defaults": { @@ -926,7 +1085,7 @@ "h": 7, "w": 12, "x": 0, - "y": 24 + "y": 32 }, "id": 2, "options": { @@ -1118,7 +1277,7 @@ "h": 7, "w": 12, "x": 12, - "y": 24 + "y": 32 }, "id": 5, "options": { @@ -1261,7 +1420,7 @@ "h": 7, "w": 12, "x": 0, - "y": 31 + "y": 39 }, "id": 3, "options": { @@ -1364,7 +1523,7 @@ "h": 7, "w": 12, "x": 12, - "y": 31 + "y": 39 }, "id": 6, "options": { @@ -1467,7 +1626,7 @@ "h": 7, "w": 12, "x": 0, - "y": 38 + "y": 46 }, "id": 7, "options": { @@ -1570,7 +1729,7 @@ "h": 7, "w": 12, "x": 12, - "y": 38 + "y": 46 }, "id": 8, "options": { @@ -1619,8 +1778,12 @@ { "allValue": ".*", "current": { - "text": "All", - "value": "$__all" + "text": [ + "All" + ], + "value": [ + "$__all" + ] }, "datasource": "fel1813ec2cjkd", "includeAll": true, @@ -1635,8 +1798,12 @@ { "allValue": ".*", "current": { - "text": "All", - "value": "$__all" + "text": [ + "All" + ], + "value": [ + "$__all" + ] }, "datasource": "fel1813ec2cjkd", "includeAll": true, @@ -1698,5 +1865,5 @@ "timezone": "browser", "title": "Go Processes", "uid": "ypFZFgvmz", - "version": 2 + "version": 16 } \ No newline at end of file diff --git a/docker/prometheus.yml b/docker/prometheus.yml new file mode 100644 index 0000000..5654dc7 --- /dev/null +++ b/docker/prometheus.yml @@ -0,0 +1,8 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: 'clamdproxy' + static_configs: + - targets: ['clamdproxy:2112'] \ No newline at end of file diff --git a/prometheus.yml b/prometheus.yml deleted file mode 100644 index 770405e..0000000 --- a/prometheus.yml +++ /dev/null @@ -1,9 +0,0 @@ -global: - scrape_interval: 15s - evaluation_interval: 15s - -scrape_configs: - - job_name: 'clamdproxy' - static_configs: - # docker.for.mac.localhost is the IP address of the Docker host on macOS. - - targets: ['docker.for.mac.localhost:2112'] \ No newline at end of file diff --git a/proxy.go b/proxy.go index c343848..5cd49cf 100644 --- a/proxy.go +++ b/proxy.go @@ -437,94 +437,94 @@ func (p *ClamdProxy) handleInstream(reader *bufio.Reader) error { // After the INSTREAM command completes successfully, store the final file size p.lastStreamSize = int64(totalBytes) - + // Flush the backend buffer to ensure all data is sent if err := p.backendBuf.Flush(); err != nil { return fmt.Errorf("failed to flush backend buffer: %w", err) } - + return nil } // parseResponseForVirus checks if the response contains virus detection information // and records metrics if a virus is found func (p *ClamdProxy) parseResponseForVirus(response []byte) { - // Convert to string for easier parsing - respStr := string(response) - - // Add debug logging to see what's in the response - logger.Debug("Parsing response", "length", len(respStr), "preview", truncateString(respStr, 100)) - - // Check if this is an INSTREAM response (either clean or with virus) - if strings.Contains(respStr, "stream") { - // Default to clean file (no virus) - virusFound := false - virusName := "" - filename := "stream" - - // Check for virus detection - if strings.Contains(respStr, " FOUND") { - logger.Debug("Found virus detection pattern in response") - lines := strings.Split(respStr, "\n") - for _, line := range lines { - if strings.Contains(line, " FOUND") { - logger.Debug("Processing virus line", "line", line) - parts := strings.Split(line, ": ") - if len(parts) >= 2 { - fileInfo := parts[0] - detectionInfo := strings.TrimSuffix(parts[1], " FOUND") - - // Extract filename from fileInfo - // Format: "instream(172.18.0.5@35478)" - filename = fileInfo - if strings.Contains(fileInfo, "(") && strings.Contains(fileInfo, ")") { - // Extract the part between parentheses - start := strings.Index(fileInfo, "(") + 1 - end := strings.Index(fileInfo, ")") - if start > 0 && end > start { - clientInfo := fileInfo[start:end] - // Use client info as part of the filename - filename = fileInfo[:start-1] + "_" + clientInfo - } - } - - virusFound = true - virusName = detectionInfo - - logger.Info("Virus detected", - "file", filename, - "virus", detectionInfo, - "size", p.lastStreamSize) - } - } - } - } else if strings.Contains(respStr, "OK") { - // This is a clean file - logger.Debug("Clean file detected", "size", p.lastStreamSize) - } - - // Record metrics for all scanned files, whether clean or infected - if proxyMetrics != nil { - // Use the actual file size if available - fileSize := p.lastStreamSize - if fileSize == 0 { - fileSize = int64(1024) // Default 1KB if size unknown - } - - proxyMetrics.RecordFileScan(filename, fileSize, virusFound, virusName) - - if virusFound { - logger.Debug("Recorded virus metrics", - "file", filename, - "size", fileSize, - "virus", virusName) - } else { - logger.Debug("Recorded clean file metrics", - "file", filename, - "size", fileSize) - } - } - } + // Convert to string for easier parsing + respStr := string(response) + + // Add debug logging to see what's in the response + logger.Debug("Parsing response", "length", len(respStr), "preview", truncateString(respStr, 100)) + + // Check if this is an INSTREAM response (either clean or with virus) + if strings.Contains(respStr, "stream") { + // Default to clean file (no virus) + virusFound := false + virusName := "" + filename := "stream" + + // Check for virus detection + if strings.Contains(respStr, " FOUND") { + logger.Debug("Found virus detection pattern in response") + lines := strings.Split(respStr, "\n") + for _, line := range lines { + if strings.Contains(line, " FOUND") { + logger.Debug("Processing virus line", "line", line) + parts := strings.Split(line, ": ") + if len(parts) >= 2 { + fileInfo := parts[0] + detectionInfo := strings.TrimSuffix(parts[1], " FOUND") + + // Extract filename from fileInfo + // Format: "instream(172.18.0.5@35478)" + filename = fileInfo + if strings.Contains(fileInfo, "(") && strings.Contains(fileInfo, ")") { + // Extract the part between parentheses + start := strings.Index(fileInfo, "(") + 1 + end := strings.Index(fileInfo, ")") + if start > 0 && end > start { + clientInfo := fileInfo[start:end] + // Use client info as part of the filename + filename = fileInfo[:start-1] + "_" + clientInfo + } + } + + virusFound = true + virusName = detectionInfo + + logger.Info("Virus detected", + "file", filename, + "virus", detectionInfo, + "size", p.lastStreamSize) + } + } + } + } else if strings.Contains(respStr, "OK") { + // This is a clean file + logger.Debug("Clean file detected", "size", p.lastStreamSize) + } + + // Record metrics for all scanned files, whether clean or infected + if proxyMetrics != nil { + // Use the actual file size if available + fileSize := p.lastStreamSize + if fileSize == 0 { + fileSize = int64(1024) // Default 1KB if size unknown + } + + proxyMetrics.RecordFileScan(filename, fileSize, virusFound, virusName) + + if virusFound { + logger.Debug("Recorded virus metrics", + "file", filename, + "size", fileSize, + "virus", virusName) + } else { + logger.Debug("Recorded clean file metrics", + "file", filename, + "size", fileSize) + } + } + } } // Helper function to truncate long strings for logging From 734fe177c8a3bb7d86800d6404b5b5f5789e6292 Mon Sep 17 00:00:00 2001 From: wahyu anggana Date: Thu, 8 May 2025 17:44:18 +0700 Subject: [PATCH 5/9] chore(metrics): change attribute virus_name to virus_detected to avoid high cardinality --- metrics.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/metrics.go b/metrics.go index 98cfc08..8d1640d 100644 --- a/metrics.go +++ b/metrics.go @@ -225,11 +225,7 @@ func (m *Metrics) RecordFileScan(filename string, sizeBytes int64, virusFound bo } attrs := []attribute.KeyValue{ - attribute.String("filename", filename), - } - - if virusFound { - attrs = append(attrs, attribute.String("virus_name", virusName)) + attribute.Bool("virus_detected", virusFound), } m.FilesScanned.Add(context.Background(), 1, metric.WithAttributes(attrs...)) @@ -237,6 +233,12 @@ func (m *Metrics) RecordFileScan(filename string, sizeBytes int64, virusFound bo if virusFound { m.FilesWithVirus.Add(context.Background(), 1, metric.WithAttributes(attrs...)) + + // Log the detailed information instead of adding it as a label + logger.Info("Virus detected", + "filename", filename, + "virus_name", virusName, + "size_bytes", sizeBytes) } } From 2d696f58d613c7f07649c055d120b6fc9b03b6b8 Mon Sep 17 00:00:00 2001 From: wahyu anggana Date: Thu, 8 May 2025 20:43:12 +0700 Subject: [PATCH 6/9] chore(docker): add parameter for dockerfile --- docker/Dockerfile | 22 ++++++++++++++--- docker/docker.md | 62 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 4 deletions(-) create mode 100644 docker/docker.md diff --git a/docker/Dockerfile b/docker/Dockerfile index aee68c8..f8ec9dc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,12 +9,26 @@ RUN CGO_ENABLED=0 GOOS=linux go build -o clamdproxy . FROM alpine:latest +# Define build arguments with default values +ARG LISTEN_ADDR="0.0.0.0:3315" +ARG BACKEND_ADDR="clamav:3310" +ARG LOG_LEVEL="debug" +ARG METRICS_ADDR="0.0.0.0:2112" + +# Create environment variables from the ARGs +ENV LISTEN_ADDR=${LISTEN_ADDR} +ENV BACKEND_ADDR=${BACKEND_ADDR} +ENV LOG_LEVEL=${LOG_LEVEL} +ENV METRICS_ADDR=${METRICS_ADDR} + RUN apk --no-cache add ca-certificates WORKDIR /app COPY --from=builder /app/clamdproxy /app/ -EXPOSE 3315 2112 -ENTRYPOINT ["/app/clamdproxy"] +# Create a startup script +RUN echo '#!/bin/sh' > /app/start.sh && \ + echo 'exec /app/clamdproxy --listen "$LISTEN_ADDR" --backend "$BACKEND_ADDR" --log-level "$LOG_LEVEL" --metrics "$METRICS_ADDR"' >> /app/start.sh && \ + chmod +x /app/start.sh -# clamav:3310 get from docker-compose url -CMD ["--listen", "0.0.0.0:3315", "--backend", "clamav:3310", "--log-level", "debug", "--metrics", "0.0.0.0:2112"] \ No newline at end of file +EXPOSE 3315 2112 +ENTRYPOINT ["/app/start.sh"] \ No newline at end of file diff --git a/docker/docker.md b/docker/docker.md new file mode 100644 index 0000000..6ddde69 --- /dev/null +++ b/docker/docker.md @@ -0,0 +1,62 @@ +# Docker Build and Run Guide + +This guide explains how to build and run the ClamdProxy Docker container with custom configuration. + +## Building the Docker Image + +You can build the ClamdProxy Docker image using the following command: + +```bash +docker build -t clamdproxy \ + --build-arg LISTEN_ADDR=0.0.0.0:3315 \ + --build-arg BACKEND_ADDR=docker.for.mac.localhost:3310 \ + --build-arg LOG_LEVEL=debug \ + --build-arg METRICS_ADDR=0.0.0.0:2112 \ + -f docker/Dockerfile . +``` + +### Build Arguments + +The Dockerfile supports the following build arguments: + +| Argument | Description | Default Value | +|----------|-------------|---------------| +| `LISTEN_ADDR` | Address and port for ClamdProxy to listen on | `0.0.0.0:3315` | +| `BACKEND_ADDR` | Address and port of the clamd backend | `clamav:3310` | +| `LOG_LEVEL` | Logging level (debug, info, warn, error) | `debug` | +| `METRICS_ADDR` | Address and port for exposing Prometheus metrics | `0.0.0.0:2112` | + +## Running the Container + +After building the image, you can run the container with: + +```bash +docker run -p 3315:3315 -p 2112:2112 \ + -e LISTEN_ADDR=0.0.0.0:3315 \ + -e BACKEND_ADDR=docker.for.mac.localhost:3310 \ + -e LOG_LEVEL=debug \ + -e METRICS_ADDR=0.0.0.0:2112 \ + clamdproxy +``` + +### Environment Variables + +The container uses the following environment variables: + +| Variable | Description | Default Value | +|----------|-------------|---------------| +| `LISTEN_ADDR` | Address and port for ClamdProxy to listen on | Value from build arg | +| `BACKEND_ADDR` | Address and port of the clamd backend | Value from build arg | +| `LOG_LEVEL` | Logging level (debug, info, warn, error) | Value from build arg | +| `METRICS_ADDR` | Address and port for exposing Prometheus metrics | Value from build arg | + +## Notes for macOS Users + +When running Docker on macOS, use `docker.for.mac.localhost` to connect to clamD services running on your host machine. This is why the example uses `docker.for.mac.localhost:3310` as the backend address. + +## Ports + +- The proxy service listens on port 3315 (or as configured) +- Metrics are exposed on port 2112 (or as configured) + +Make sure to map these ports correctly when running the container. \ No newline at end of file From b7dc04f03c17de6cad42cbe911bc5996120855ab Mon Sep 17 00:00:00 2001 From: wahyu anggana Date: Thu, 8 May 2025 20:46:48 +0700 Subject: [PATCH 7/9] chore(dashboard): change command rate, virus detected counter and files scanned size on grafana dashboard panel --- docker/grafana.json | 1386 ++++++------------------------------------- 1 file changed, 165 insertions(+), 1221 deletions(-) diff --git a/docker/grafana.json b/docker/grafana.json index 8a3a31a..03e387c 100644 --- a/docker/grafana.json +++ b/docker/grafana.json @@ -15,7 +15,7 @@ } ] }, - "description": "Process status published by Go Prometheus client library, e.g. memory used, fds open, GC details", + "description": "ClamAV Dashboard", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, @@ -38,52 +38,24 @@ { "datasource": { "type": "prometheus", - "uid": "ael6f69gclslca" + "uid": "del6mcwp74xz4e" }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, "mappings": [], "thresholds": { - "mode": "absolute", + "mode": "percentage", "steps": [ { "color": "green" + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 85 } ] } @@ -91,24 +63,26 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, + "h": 6, + "w": 3, "x": 0, "y": 1 }, "id": 20, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } + "showThresholdLabels": false, + "showThresholdMarkers": false, + "sizing": "auto" }, "pluginVersion": "12.0.0", "targets": [ @@ -119,49 +93,46 @@ }, "editorMode": "code", "exemplar": false, - "expr": "max by(virus_name) (clamdproxy_files_with_virus_total{virus_name!=\"\"})", + "expr": "max by(virus_detected) (clamdproxy_files_scanned_total{virus_detected=\"true\"})", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Virus Founded", - "type": "timeseries" + "title": "Virus Detected", + "type": "gauge" }, { "datasource": { "type": "prometheus", - "uid": "ael6f69gclslca" + "uid": "del6mcwp74xz4e" }, "fieldConfig": { "defaults": { "mappings": [], "thresholds": { - "mode": "absolute", + "mode": "percentage", "steps": [ { "color": "green" } ] - }, - "unit": "short" + } }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, + "h": 6, + "w": 3, + "x": 3, "y": 1 }, - "id": 11, + "id": 22, "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", + "minVizHeight": 75, + "minVizWidth": 75, "orientation": "auto", - "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -169,37 +140,33 @@ "fields": "", "values": false }, - "showPercentChange": false, - "text": { - "titleSize": 15 - }, - "textMode": "auto", - "wideLayout": true + "showThresholdLabels": false, + "showThresholdMarkers": false, + "sizing": "auto" }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { - "uid": "Prometheus" + "type": "prometheus", + "uid": "ael6f69gclslca" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum by(command) (clamdproxy_commands_total{allowed=\"true\"})", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "{{command}}", + "editorMode": "code", + "exemplar": false, + "expr": "max by(virus_detected) (clamdproxy_files_scanned_total{virus_detected=\"false\"})", + "instant": false, + "legendFormat": "__auto", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" } ], - "title": "Commands Total", - "type": "stat" + "title": "Files Scanned", + "type": "gauge" }, { "datasource": { "type": "prometheus", - "uid": "ael6f69gclslca" + "uid": "del6mcwp74xz4e" }, "fieldConfig": { "defaults": { @@ -211,7 +178,7 @@ "mode": "absolute", "steps": [ { - "color": "blue" + "color": "green" } ] } @@ -221,10 +188,10 @@ "gridPos": { "h": 6, "w": 3, - "x": 0, - "y": 9 + "x": 6, + "y": 1 }, - "id": 16, + "id": 13, "options": { "minVizHeight": 75, "minVizWidth": 75, @@ -246,24 +213,20 @@ "datasource": { "uid": "Prometheus" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum by(job) (clamdproxy_connections_total{job=\"clamdproxy\"})", - "fullMetaSearch": false, - "includeNullMetadata": true, + "editorMode": "code", + "expr": "sum by(code) (\n promhttp_metric_handler_requests_total{code=\"200\"}\n)\n", "legendFormat": "status {{code}}", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" } ], - "title": "Total Connections", + "title": "http status 200", "type": "gauge" }, { "datasource": { "type": "prometheus", - "uid": "ael6f69gclslca" + "uid": "del6mcwp74xz4e" }, "fieldConfig": { "defaults": { @@ -275,20 +238,21 @@ "mode": "absolute", "steps": [ { - "color": "purple" + "color": "orange" } ] - } + }, + "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 6, "w": 3, - "x": 3, - "y": 9 + "x": 9, + "y": 1 }, - "id": 17, + "id": 21, "options": { "minVizHeight": 75, "minVizWidth": 75, @@ -311,8 +275,8 @@ "uid": "Prometheus" }, "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum by(job) (clamdproxy_active_connections{job=\"clamdproxy\"})", + "editorMode": "code", + "expr": "sum(clamdproxy_files_size_bytes_total)", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "status {{code}}", @@ -321,13 +285,13 @@ "useBackend": false } ], - "title": "Active Connections", + "title": "Total File Scanned in size", "type": "gauge" }, { "datasource": { "type": "prometheus", - "uid": "ael6f69gclslca" + "uid": "del6mcwp74xz4e" }, "fieldConfig": { "defaults": { @@ -339,7 +303,7 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "dark-red" } ] } @@ -349,10 +313,10 @@ "gridPos": { "h": 6, "w": 3, - "x": 6, - "y": 9 + "x": 12, + "y": 1 }, - "id": 13, + "id": 14, "options": { "minVizHeight": 75, "minVizWidth": 75, @@ -375,19 +339,19 @@ "uid": "Prometheus" }, "editorMode": "code", - "expr": "sum by(code) (\n promhttp_metric_handler_requests_total{code=\"200\"}\n)\n", + "expr": "sum by(code) (\n promhttp_metric_handler_requests_total{code=\"500\"}\n)\n", "legendFormat": "status {{code}}", "range": true, "refId": "A" } ], - "title": "http status 200", + "title": "http status 500", "type": "gauge" }, { "datasource": { "type": "prometheus", - "uid": "ael6f69gclslca" + "uid": "del6mcwp74xz4e" }, "fieldConfig": { "defaults": { @@ -399,21 +363,20 @@ "mode": "absolute", "steps": [ { - "color": "orange" + "color": "dark-red" } ] - }, - "unit": "decbytes" + } }, "overrides": [] }, "gridPos": { "h": 6, "w": 3, - "x": 9, - "y": 9 + "x": 15, + "y": 1 }, - "id": 21, + "id": 15, "options": { "minVizHeight": 75, "minVizWidth": 75, @@ -436,8 +399,8 @@ "uid": "Prometheus" }, "disableTextWrap": false, - "editorMode": "code", - "expr": "sum(clamdproxy_files_size_bytes_total)", + "editorMode": "builder", + "expr": "sum by(job) (clamdproxy_backend_errors_total{job=\"clamdproxy\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "status {{code}}", @@ -446,13 +409,13 @@ "useBackend": false } ], - "title": "Total File Scanned", + "title": "Backend Error Total", "type": "gauge" }, { "datasource": { "type": "prometheus", - "uid": "ael6f69gclslca" + "uid": "del6mcwp74xz4e" }, "fieldConfig": { "defaults": { @@ -464,7 +427,7 @@ "mode": "absolute", "steps": [ { - "color": "dark-red" + "color": "blue" } ] } @@ -474,10 +437,10 @@ "gridPos": { "h": 6, "w": 3, - "x": 12, - "y": 9 + "x": 0, + "y": 7 }, - "id": 14, + "id": 16, "options": { "minVizHeight": 75, "minVizWidth": 75, @@ -499,20 +462,24 @@ "datasource": { "uid": "Prometheus" }, - "editorMode": "code", - "expr": "sum by(code) (\n promhttp_metric_handler_requests_total{code=\"500\"}\n)\n", + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(job) (clamdproxy_connections_total{job=\"clamdproxy\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, "legendFormat": "status {{code}}", "range": true, - "refId": "A" + "refId": "A", + "useBackend": false } ], - "title": "http status 500", + "title": "Total Connections", "type": "gauge" }, { "datasource": { "type": "prometheus", - "uid": "ael6f69gclslca" + "uid": "del6mcwp74xz4e" }, "fieldConfig": { "defaults": { @@ -524,7 +491,7 @@ "mode": "absolute", "steps": [ { - "color": "dark-red" + "color": "purple" } ] } @@ -534,10 +501,10 @@ "gridPos": { "h": 6, "w": 3, - "x": 15, - "y": 9 + "x": 3, + "y": 7 }, - "id": 15, + "id": 17, "options": { "minVizHeight": 75, "minVizWidth": 75, @@ -561,7 +528,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "sum by(job) (clamdproxy_backend_errors_total{job=\"clamdproxy\"})", + "expr": "sum by(job) (clamdproxy_active_connections{job=\"clamdproxy\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "status {{code}}", @@ -570,296 +537,80 @@ "useBackend": false } ], - "title": "Backend Error Total", + "title": "Active Connections", "type": "gauge" }, { "datasource": { "type": "prometheus", - "uid": "ael6f69gclslca" + "uid": "del6mcwp74xz4e" }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "none" + "unit": "short" }, "overrides": [] }, "gridPos": { - "h": 8, + "h": 6, "w": 12, - "x": 0, - "y": 15 + "x": 6, + "y": 7 }, - "id": 9, + "id": 11, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "titleSize": 15 + }, + "textMode": "auto", + "wideLayout": true }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { - "type": "prometheus", - "uid": "fel1813ec2cjkd" + "uid": "Prometheus" }, "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "sum by(command) (increase(clamdproxy_commands_total{allowed=\"true\"}[1m]))", - "format": "time_series", + "editorMode": "builder", + "expr": "sum by(command) (clamdproxy_commands_total{allowed=\"true\"})", "fullMetaSearch": false, "includeNullMetadata": true, - "instant": true, "legendFormat": "{{command}}", "range": true, "refId": "A", "useBackend": false } ], - "title": "Command Rate", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 10, - "panels": [], - "title": "Go Process", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ael6f69gclslca" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "resident" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "custom.axisPlacement", - "value": "right" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "resident" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "custom.axisPlacement", - "value": "right" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "resident" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "custom.axisPlacement", - "value": "right" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 1, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "fel1813ec2cjkd" - }, - "expr": "process_resident_memory_bytes{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod}} - resident", - "metric": "process_resident_memory_bytes", - "refId": "A", - "step": 4 - }, - { - "datasource": { - "type": "prometheus", - "uid": "fel1813ec2cjkd" - }, - "expr": "process_virtual_memory_bytes{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod}} - virtual", - "metric": "process_virtual_memory_bytes", - "refId": "B", - "step": 4 - } - ], - "title": "process memory", - "type": "timeseries" + "title": "Commands Total", + "type": "stat" }, { "datasource": { "type": "prometheus", - "uid": "ael6f69gclslca" + "uid": "del6mcwp74xz4e" }, "fieldConfig": { "defaults": { @@ -875,7 +626,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -884,13 +635,13 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 2, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "never", - "spanNulls": true, + "showPoints": "auto", + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -899,6 +650,7 @@ "mode": "off" } }, + "fieldMinMax": false, "mappings": [], "thresholds": { "mode": "absolute", @@ -908,35 +660,31 @@ }, { "color": "red", - "value": 80 + "value": 80.0001 } ] }, - "unit": "bytes" + "unit": "none" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 + "h": 13, + "w": 24, + "x": 0, + "y": 13 }, - "id": 4, + "id": 9, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", + "calcs": [], + "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -947,830 +695,26 @@ "type": "prometheus", "uid": "fel1813ec2cjkd" }, - "expr": "rate(process_resident_memory_bytes{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}[$interval])", + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(clamdproxy_files_scanned_total{job=\"clamdproxy\"}[1m]))", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod}} - resident", - "metric": "process_resident_memory_bytes", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{command}}", + "range": true, "refId": "A", - "step": 4 - }, - { - "datasource": { - "type": "prometheus", - "uid": "fel1813ec2cjkd" - }, - "expr": "deriv(process_virtual_memory_bytes{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}[$interval])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod}} - virtual", - "metric": "process_virtual_memory_bytes", - "refId": "B", - "step": 4 + "useBackend": false } ], - "title": "process memory deriv", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ael6f69gclslca" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "alloc rate" - }, - "properties": [ - { - "id": "unit", - "value": "Bps" - }, - { - "id": "custom.axisPlacement", - "value": "right" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "alloc rate" - }, - "properties": [ - { - "id": "unit", - "value": "Bps" - }, - { - "id": "custom.axisPlacement", - "value": "right" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "alloc rate" - }, - "properties": [ - { - "id": "unit", - "value": "Bps" - }, - { - "id": "custom.axisPlacement", - "value": "right" - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 2, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "fel1813ec2cjkd" - }, - "expr": "go_memstats_alloc_bytes{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod}} - bytes allocated", - "metric": "go_memstats_alloc_bytes", - "refId": "A", - "step": 4 - }, - { - "datasource": { - "type": "prometheus", - "uid": "fel1813ec2cjkd" - }, - "expr": "rate(go_memstats_alloc_bytes_total{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}[30s])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod}} - alloc rate", - "metric": "go_memstats_alloc_bytes_total", - "refId": "B", - "step": 4 - }, - { - "datasource": { - "type": "prometheus", - "uid": "fel1813ec2cjkd" - }, - "expr": "go_memstats_stack_inuse_bytes{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod}} - stack inuse", - "metric": "go_memstats_stack_inuse_bytes", - "refId": "C", - "step": 4 - }, - { - "datasource": { - "type": "prometheus", - "uid": "fel1813ec2cjkd" - }, - "expr": "go_memstats_heap_inuse_bytes{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{pod}} - heap inuse", - "metric": "go_memstats_heap_inuse_bytes", - "refId": "D", - "step": 4 - } - ], - "title": "go memstats", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "fel1813ec2cjkd" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "alloc rate" - }, - "properties": [ - { - "id": "unit", - "value": "Bps" - }, - { - "id": "custom.axisPlacement", - "value": "right" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "alloc rate" - }, - "properties": [ - { - "id": "unit", - "value": "Bps" - }, - { - "id": "custom.axisPlacement", - "value": "right" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "alloc rate" - }, - "properties": [ - { - "id": "unit", - "value": "Bps" - }, - { - "id": "custom.axisPlacement", - "value": "right" - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 5, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "fel1813ec2cjkd" - }, - "expr": "deriv(go_memstats_alloc_bytes{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}[$interval])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod}} - bytes allocated", - "metric": "go_memstats_alloc_bytes", - "refId": "A", - "step": 4 - }, - { - "datasource": { - "type": "prometheus", - "uid": "fel1813ec2cjkd" - }, - "expr": "rate(go_memstats_alloc_bytes_total{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}[$interval])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod}} - alloc rate", - "metric": "go_memstats_alloc_bytes_total", - "refId": "B", - "step": 4 - }, - { - "datasource": { - "type": "prometheus", - "uid": "fel1813ec2cjkd" - }, - "expr": "deriv(go_memstats_stack_inuse_bytes{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}[$interval])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod}} - stack inuse", - "metric": "go_memstats_stack_inuse_bytes", - "refId": "C", - "step": 4 - }, - { - "datasource": { - "type": "prometheus", - "uid": "fel1813ec2cjkd" - }, - "expr": "deriv(go_memstats_heap_inuse_bytes{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}[$interval])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{pod}} - heap inuse", - "metric": "go_memstats_heap_inuse_bytes", - "refId": "D", - "step": 4 - } - ], - "title": "go memstats deriv", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "fel1813ec2cjkd" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 39 - }, - "id": 3, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "fel1813ec2cjkd" - }, - "expr": "process_open_fds{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "metric": "process_open_fds", - "refId": "A", - "step": 4 - } - ], - "title": "open fds", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "fel1813ec2cjkd" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 39 - }, - "id": 6, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "fel1813ec2cjkd" - }, - "expr": "deriv(process_open_fds{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}[$interval])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "metric": "process_open_fds", - "refId": "A", - "step": 4 - } - ], - "title": "open fds deriv", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "fel1813ec2cjkd" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 46 - }, - "id": 7, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "fel1813ec2cjkd" - }, - "expr": "go_goroutines{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "metric": "go_goroutines", - "refId": "A", - "step": 4 - } - ], - "title": "Goroutines", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "fel1813ec2cjkd" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 46 - }, - "id": 8, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "fel1813ec2cjkd" - }, - "expr": "go_gc_duration_seconds{namespace=~\"^($namespace)$\",pod=~\"^($pod)$\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod}}: {{quantile}}", - "metric": "go_gc_duration_seconds", - "refId": "A", - "step": 4 - } - ], - "title": "GC duration quantiles", + "title": "Command Rate", "type": "timeseries" } ], "preload": false, - "refresh": "30s", + "refresh": "auto", "schemaVersion": 41, "tags": [], "templating": { @@ -1858,12 +802,12 @@ ] }, "time": { - "from": "now-30m", + "from": "now-15m", "to": "now" }, "timepicker": {}, "timezone": "browser", - "title": "Go Processes", + "title": "ClamAV Dashboard", "uid": "ypFZFgvmz", - "version": 16 + "version": 20 } \ No newline at end of file From a48b8084553c497ea22b9f7f62a3705ee758c3e2 Mon Sep 17 00:00:00 2001 From: wahyu anggana Date: Fri, 9 May 2025 17:33:58 +0700 Subject: [PATCH 8/9] fix(metrics): fix some high cardinality for connection that collect client address --- metrics.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/metrics.go b/metrics.go index 8d1640d..58ee701 100644 --- a/metrics.go +++ b/metrics.go @@ -164,9 +164,7 @@ func (m *Metrics) RecordConnection(clientAddr string) { if m == nil { return } - m.ConnectionsTotal.Add(context.Background(), 1, metric.WithAttributes( - attribute.String("client", clientAddr), - )) + m.ConnectionsTotal.Add(context.Background(), 1) m.ActiveConnections.Add(context.Background(), 1) } @@ -233,10 +231,10 @@ func (m *Metrics) RecordFileScan(filename string, sizeBytes int64, virusFound bo if virusFound { m.FilesWithVirus.Add(context.Background(), 1, metric.WithAttributes(attrs...)) - + // Log the detailed information instead of adding it as a label - logger.Info("Virus detected", - "filename", filename, + logger.Info("Virus detected", + "filename", filename, "virus_name", virusName, "size_bytes", sizeBytes) } From 05bb8f19fa91a9467d75d4ce0ab64b913a89ccdc Mon Sep 17 00:00:00 2001 From: wahyu anggana Date: Fri, 9 May 2025 18:05:17 +0700 Subject: [PATCH 9/9] fix(test): fix FAIL test for TestMetrics/RecordConnection --- metrics_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/metrics_test.go b/metrics_test.go index 59cdb51..c6aeec6 100644 --- a/metrics_test.go +++ b/metrics_test.go @@ -24,10 +24,13 @@ func TestMetrics(t *testing.T) { t.Run("RecordConnection", func(t *testing.T) { metrics.RecordConnection("127.0.0.1:1234") - // Verify metrics through HTTP endpoint resp := getMetrics(t, metrics) - assert.Contains(t, resp, `clamdproxy_connections_total{client="127.0.0.1:1234"`) - assert.Contains(t, resp, `clamdproxy_active_connections`) + + // Check for the counter without the client label + assert.Contains(t, resp, "clamdproxy_connections_total") + + // Check for active connections + assert.Contains(t, resp, "clamdproxy_active_connections") }) t.Run("RecordConnectionClosed", func(t *testing.T) {