From 363e864aa17092928795894fc3699d3d7618f88f Mon Sep 17 00:00:00 2001 From: Kevin McConnell Date: Fri, 17 Jul 2026 12:38:13 +0100 Subject: [PATCH] Allow publishing per-service healthcheck endpoints When running Kamal Proxy behind downstream load balancers, it can be difficult to route those load balancers' health checks to the correct service if those checks don't carry the correct `Host` header. (Unfortunately, many cloud load balancers don't allow setting that header in their healthcheck configuration). To make this easier, we add the ability to publish a service's configured healthcheck at a well-known service-specific path. For example, a service `app` can be health-checked at `/.kamal-proxy/app/health`. To enable the published health check endpoint for a service, set the `--publish-health-check` flag: kamal-proxy deploy app --target web-1:3000 \ --host app1.example.com --publish-health-check The published health check endpoints are not subject to host checking, TLS requirements, or canonical redirects. --- README.md | 19 ++ internal/cmd/deploy.go | 1 + internal/server/router.go | 20 ++ internal/server/router_test.go | 287 ++++++++++++++++++++++++++++ internal/server/service.go | 15 +- internal/server/service_map.go | 32 +++- internal/server/service_map_test.go | 22 +++ internal/server/target.go | 32 +++- 8 files changed, 418 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 2bdbb09d..460e892d 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,25 @@ To configure health checks to run on a different port than your main service kamal-proxy deploy service1 --target web-1:3000 --health-check-port 8080 +### Published health check endpoints + +When running Kamal Proxy behind downstream load balancers, it can be difficult +to route those load balancers' health checks to the correct service if those +checks don't carry the correct `Host` header. (Unfortunately, many cloud load +balancers don't allow setting that header in their healthcheck configuration). + +To make this easier, we add the ability to publish a service's configured +healthcheck at a well-known service-specific path. For example, a service `app` +can be health-checked at `/.kamal-proxy/app/health`. + +To enable the published health check endpoint for a service, set the +`--publish-health-check` flag: + + kamal-proxy deploy app --target web-1:3000 --host app1.example.com --publish-health-check + +The published health check endpoints are not subject to host checking, TLS +requirements, or canonical redirects. + ### Host-based routing Host-based routing allows you to run multiple applications on the same server, diff --git a/internal/cmd/deploy.go b/internal/cmd/deploy.go index 82af0e18..92ee4f49 100644 --- a/internal/cmd/deploy.go +++ b/internal/cmd/deploy.go @@ -49,6 +49,7 @@ func newDeployCommand() *deployCommand { deployCommand.cmd.Flags().StringVar(&deployCommand.args.TargetOptions.HealthCheckConfig.Path, "health-check-path", server.DefaultHealthCheckPath, "Path to check for health") deployCommand.cmd.Flags().IntVar(&deployCommand.args.TargetOptions.HealthCheckConfig.Port, "health-check-port", server.DefaultHealthCheckPort, "Port to check for health (default matches target port)") deployCommand.cmd.Flags().StringVar(&deployCommand.args.TargetOptions.HealthCheckConfig.Host, "health-check-host", "", "Host header to send with health check requests") + deployCommand.cmd.Flags().BoolVar(&deployCommand.args.ServiceOptions.PublishHealthCheck, "publish-health-check", false, "Publish this service's health check at /.kamal-proxy//health") deployCommand.cmd.Flags().DurationVar(&deployCommand.args.ServiceOptions.WriterAffinityTimeout, "writer-affinity-timeout", server.DefaultWriterAffinityTimeout, "Time after a write before read requests will be routed to readers") deployCommand.cmd.Flags().BoolVar(&deployCommand.args.ServiceOptions.ReadTargetsAcceptWebsockets, "read-target-websockets", false, "Route WebSocket traffic to read targets, when available") diff --git a/internal/server/router.go b/internal/server/router.go index 3b891b6a..d2897978 100644 --- a/internal/server/router.go +++ b/internal/server/router.go @@ -102,6 +102,11 @@ func (r *Router) RestoreLastSavedState() error { } func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) { + if service := r.serviceForPublishedHealthCheck(req); service != nil { + service.ServeHTTP(w, req.WithContext(markHealthCheckProbe(markInternalRequest(req.Context())))) + return + } + service, prefix := r.serviceForRequest(req) if service == nil { SetErrorResponse(w, req, http.StatusNotFound, nil) @@ -380,6 +385,21 @@ func (r *Router) saveStateSnapshot() error { return nil } +func (r *Router) serviceForPublishedHealthCheck(req *http.Request) *Service { + if !strings.HasPrefix(req.URL.Path, publishedHealthCheckPrefix) { + return nil + } + + if req.Method != http.MethodGet && req.Method != http.MethodHead { + return nil + } + + r.serviceLock.RLock() + defer r.serviceLock.RUnlock() + + return r.services.PublishedHealthCheck(req.URL.Path) +} + func (r *Router) serviceForRequest(req *http.Request) (*Service, string) { r.serviceLock.RLock() defer r.serviceLock.RUnlock() diff --git a/internal/server/router_test.go b/internal/server/router_test.go index 28184868..5eb640ea 100644 --- a/internal/server/router_test.go +++ b/internal/server/router_test.go @@ -4,10 +4,12 @@ import ( "context" "crypto/tls" "encoding/json" + "net" "net/http" "net/http/httptest" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -760,6 +762,291 @@ func TestRouter_EnablingRollout(t *testing.T) { checkResponse("first") } +func TestRouter_PublishedHealthCheckClaimsOnlyItsOwnPath(t *testing.T) { + router := testRouter(t) + + statusCode, _ := sendGETRequest(router, "http://192.168.1.1/.kamal-proxy/service1/health") + assert.Equal(t, http.StatusNotFound, statusCode) + + _, target := testBackendWithHandler(t, func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(r.URL.String())) + }) + require.NoError(t, router.DeployService("service1", []string{target}, defaultEmptyReaders, defaultServiceOptions, defaultTargetOptions, defaultDeploymentOptions)) + + // While no service publishes its health check, its path is routed to + // services like any other request + statusCode, body := sendGETRequest(router, "http://example.com/.kamal-proxy/service1/health") + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, "/.kamal-proxy/service1/health", body) + + // Publishing a health check claims exactly its own path... + serviceOptions := defaultServiceOptions + serviceOptions.PublishHealthCheck = true + require.NoError(t, router.DeployService("service1", []string{target}, defaultEmptyReaders, serviceOptions, defaultTargetOptions, defaultDeploymentOptions)) + + statusCode, body = sendGETRequest(router, "http://example.com/.kamal-proxy/service1/health") + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, "/up", body) + + // ...while all other traffic is routed as normal + for _, path := range []string{ + "/.kamal-proxy/other/health", + "/.kamal-proxy/service1/health/extra", + "/.kamal-proxy/service1", + "/.kamal-proxy", + } { + statusCode, body = sendGETRequest(router, "http://example.com"+path) + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, path, body) + } + + // Unpublishing the health check releases its path again + require.NoError(t, router.DeployService("service1", []string{target}, defaultEmptyReaders, defaultServiceOptions, defaultTargetOptions, defaultDeploymentOptions)) + + statusCode, body = sendGETRequest(router, "http://example.com/.kamal-proxy/service1/health") + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, "/.kamal-proxy/service1/health", body) +} + +func TestRouter_PublishedHealthCheck(t *testing.T) { + router := testRouter(t) + _, first := testBackend(t, "first", http.StatusOK) + _, second := testBackend(t, "second", http.StatusOK) + + serviceOptions := defaultServiceOptions + serviceOptions.Hosts = []string{"one.example.com"} + serviceOptions.PublishHealthCheck = true + require.NoError(t, router.DeployService("service1", []string{first}, defaultEmptyReaders, serviceOptions, defaultTargetOptions, defaultDeploymentOptions)) + + serviceOptions = defaultServiceOptions + serviceOptions.Hosts = []string{"two.example.com"} + require.NoError(t, router.DeployService("service2", []string{second}, defaultEmptyReaders, serviceOptions, defaultTargetOptions, defaultDeploymentOptions)) + + // The published path works with any Host header: an unmatched one, the + // service's own, and one belonging to a different service. + for _, host := range []string{"192.168.1.1", "one.example.com", "two.example.com"} { + statusCode, body := sendGETRequest(router, "http://"+host+"/.kamal-proxy/service1/health") + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, "first", body) + } +} + +func TestRouter_PublishedHealthCheckUsesHealthCheckPortAndHost(t *testing.T) { + router := testRouter(t) + _, target := testBackend(t, "main", http.StatusOK) + _, healthTarget := testBackendWithHandler(t, func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(r.Host + " " + r.URL.String())) + }) + + _, portString, err := net.SplitHostPort(healthTarget) + require.NoError(t, err) + healthPort, err := strconv.Atoi(portString) + require.NoError(t, err) + + serviceOptions := defaultServiceOptions + serviceOptions.PublishHealthCheck = true + + // Probes are sent to the health check port, with the configured host, + // just like the proxy's own health checks + targetOptions := defaultTargetOptions + targetOptions.HealthCheckConfig.Port = healthPort + targetOptions.HealthCheckConfig.Host = "app.internal" + require.NoError(t, router.DeployService("service1", []string{target}, defaultEmptyReaders, serviceOptions, targetOptions, defaultDeploymentOptions)) + + statusCode, body := sendGETRequest(router, "http://10.0.0.1/.kamal-proxy/service1/health") + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, "app.internal /up", body) + + // Without a configured health check host, probes carry the health check + // address as their host, rather than the client's host + targetOptions.HealthCheckConfig.Host = "" + require.NoError(t, router.DeployService("service1", []string{target}, defaultEmptyReaders, serviceOptions, targetOptions, defaultDeploymentOptions)) + + statusCode, body = sendGETRequest(router, "http://10.0.0.1/.kamal-proxy/service1/health") + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, healthTarget+" /up", body) +} + +func TestRouter_PublishedHealthCheckUsesHealthCheckPath(t *testing.T) { + router := testRouter(t) + _, target := testBackendWithHandler(t, func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(r.URL.String())) + }) + + serviceOptions := defaultServiceOptions + serviceOptions.PublishHealthCheck = true + + targetOptions := defaultTargetOptions + targetOptions.HealthCheckConfig.Path = "/healthz" + require.NoError(t, router.DeployService("service1", []string{target}, defaultEmptyReaders, serviceOptions, targetOptions, defaultDeploymentOptions)) + + statusCode, body := sendGETRequest(router, "http://10.0.0.1/.kamal-proxy/service1/health") + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, "/healthz", body) + + // A path without a leading slash is normalized, matching how the proxy's + // own health checks address it + targetOptions.HealthCheckConfig.Path = "healthz" + require.NoError(t, router.DeployService("service1", []string{target}, defaultEmptyReaders, serviceOptions, targetOptions, defaultDeploymentOptions)) + + statusCode, body = sendGETRequest(router, "http://10.0.0.1/.kamal-proxy/service1/health") + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, "/healthz", body) +} + +func TestRouter_PublishedHealthCheckDropsQueryString(t *testing.T) { + router := testRouter(t) + _, target := testBackendWithHandler(t, func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(r.URL.String())) + }) + + serviceOptions := defaultServiceOptions + serviceOptions.PublishHealthCheck = true + require.NoError(t, router.DeployService("service1", []string{target}, defaultEmptyReaders, serviceOptions, defaultTargetOptions, defaultDeploymentOptions)) + + statusCode, body := sendGETRequest(router, "http://10.0.0.1/.kamal-proxy/service1/health?foo=bar") + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, "/up", body) + + // Including when the query string is empty + statusCode, body = sendGETRequest(router, "http://10.0.0.1/.kamal-proxy/service1/health?") + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, "/up", body) +} + +func TestRouter_PublishedHealthCheckDoesNotRedirectToTLS(t *testing.T) { + router := testRouter(t) + _, target := testBackend(t, "first", http.StatusOK) + + serviceOptions := defaultServiceOptions + serviceOptions.Hosts = []string{"example.com"} + serviceOptions.TLSEnabled = true + serviceOptions.TLSRedirect = true + serviceOptions.PublishHealthCheck = true + require.NoError(t, router.DeployService("service1", []string{target}, defaultEmptyReaders, serviceOptions, defaultTargetOptions, defaultDeploymentOptions)) + + // Regular plain-HTTP traffic is redirected to HTTPS + statusCode, _ := sendGETRequest(router, "http://example.com/") + assert.Equal(t, http.StatusMovedPermanently, statusCode) + + // Plain-HTTP probes are not + statusCode, body := sendGETRequest(router, "http://10.0.0.1/.kamal-proxy/service1/health") + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, "first", body) +} + +func TestRouter_PublishedHealthCheckIsNotSubjectToTLSRequirements(t *testing.T) { + router := testRouter(t) + _, target := testBackend(t, "first", http.StatusOK) + + serviceOptions := defaultServiceOptions + serviceOptions.Hosts = []string{"example.com"} + serviceOptions.PublishHealthCheck = true + require.NoError(t, router.DeployService("service1", []string{target}, defaultEmptyReaders, serviceOptions, defaultTargetOptions, defaultDeploymentOptions)) + + // Regular HTTPS traffic to a non-TLS service is rejected + req := httptest.NewRequest(http.MethodGet, "https://example.com/", nil) + req.TLS = &tls.ConnectionState{} + statusCode, _ := sendRequest(router, req) + assert.Equal(t, http.StatusServiceUnavailable, statusCode) + + // HTTPS probes are served + req = httptest.NewRequest(http.MethodGet, "https://10.0.0.1/.kamal-proxy/service1/health", nil) + req.TLS = &tls.ConnectionState{} + statusCode, body := sendRequest(router, req) + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, "first", body) +} + +func TestRouter_PublishedHealthCheckOnlyMatchesGETAndHEAD(t *testing.T) { + router := testRouter(t) + _, target := testBackendWithHandler(t, func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(r.Method + " " + r.URL.String())) + }) + + serviceOptions := defaultServiceOptions + serviceOptions.PublishHealthCheck = true + require.NoError(t, router.DeployService("service1", []string{target}, defaultEmptyReaders, serviceOptions, defaultTargetOptions, defaultDeploymentOptions)) + + statusCode, body := sendGETRequest(router, "http://example.com/.kamal-proxy/service1/health") + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, "GET /up", body) + + statusCode, _ = sendRequest(router, httptest.NewRequest(http.MethodHead, "http://example.com/.kamal-proxy/service1/health", nil)) + assert.Equal(t, http.StatusOK, statusCode) + + // Other methods are routed as normal requests + statusCode, body = sendRequest(router, httptest.NewRequest(http.MethodPost, "http://example.com/.kamal-proxy/service1/health", nil)) + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, "POST /.kamal-proxy/service1/health", body) +} + +func TestRouter_PublishedHealthCheckMetricsExclusion(t *testing.T) { + router := testRouter(t) + _, target := testBackend(t, "first", http.StatusOK) + + sendProbe := func() *loggingRequestContext { + lrc := &loggingRequestContext{} + req := httptest.NewRequest(http.MethodGet, "http://10.0.0.1/.kamal-proxy/service1/health", nil) + req = req.WithContext(context.WithValue(req.Context(), contextKeyRequestContext, lrc)) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + require.Equal(t, http.StatusOK, w.Result().StatusCode) + return lrc + } + + serviceOptions := defaultServiceOptions + serviceOptions.PublishHealthCheck = true + serviceOptions.ExcludeMetricsPaths = []string{"/.kamal-proxy/service1/health"} + require.NoError(t, router.DeployService("service1", []string{target}, defaultEmptyReaders, serviceOptions, defaultTargetOptions, defaultDeploymentOptions)) + + // Probes are recorded as the published path, so they are excluded from + // metrics when that path is + assert.True(t, sendProbe().ExcludeMetrics) + + serviceOptions.ExcludeMetricsPaths = nil + require.NoError(t, router.DeployService("service1", []string{target}, defaultEmptyReaders, serviceOptions, defaultTargetOptions, defaultDeploymentOptions)) + + assert.False(t, sendProbe().ExcludeMetrics) +} + +func TestRouter_PublishedHealthCheckWhilePaused(t *testing.T) { + router := testRouter(t) + _, target := testBackend(t, "first", http.StatusOK) + + serviceOptions := defaultServiceOptions + serviceOptions.PublishHealthCheck = true + require.NoError(t, router.DeployService("service1", []string{target}, defaultEmptyReaders, serviceOptions, defaultTargetOptions, defaultDeploymentOptions)) + require.NoError(t, router.PauseService("service1", time.Second, time.Millisecond*10)) + + // Paused services still report themselves healthy + statusCode, _ := sendGETRequest(router, "http://10.0.0.1/.kamal-proxy/service1/health") + assert.Equal(t, http.StatusOK, statusCode) + + // While other requests wait for the pause to end + statusCode, _ = sendGETRequest(router, "http://10.0.0.1/other") + assert.Equal(t, http.StatusGatewayTimeout, statusCode) +} + +func TestRouter_PublishedHealthCheckWithPathPrefix(t *testing.T) { + router := testRouter(t) + _, target := testBackendWithHandler(t, func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(r.URL.String())) + }) + + serviceOptions := defaultServiceOptions + serviceOptions.PathPrefixes = []string{"/api"} + serviceOptions.StripPrefix = true + serviceOptions.PublishHealthCheck = true + require.NoError(t, router.DeployService("service1", []string{target}, defaultEmptyReaders, serviceOptions, defaultTargetOptions, defaultDeploymentOptions)) + + // The probe reaches the target at the bare health check path, matching how + // the proxy's own health checks address it + statusCode, body := sendGETRequest(router, "http://example.com/.kamal-proxy/service1/health") + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, "/up", body) +} + func TestRouter_RestoreLastSavedState(t *testing.T) { statePath := filepath.Join(t.TempDir(), "state.json") diff --git a/internal/server/service.go b/internal/server/service.go index d3c88d6d..474ee95e 100644 --- a/internal/server/service.go +++ b/internal/server/service.go @@ -58,7 +58,8 @@ var ( ErrorAutomaticTLSDoesNotSupportWildcards = errors.New("automatic TLS does not support wildcards") ErrServiceOptionsInvalid = errors.New("service options invalid") - contextKeyInternalRequest = contextKey("internal-request") + contextKeyInternalRequest = contextKey("internal-request") + contextKeyHealthCheckProbe = contextKey("health-check-probe") ) // markInternalRequest marks the context as belonging to an internal request: @@ -73,6 +74,15 @@ func isInternalRequest(r *http.Request) bool { return internal } +func markHealthCheckProbe(ctx context.Context) context.Context { + return context.WithValue(ctx, contextKeyHealthCheckProbe, true) +} + +func isHealthCheckProbe(r *http.Request) bool { + probe, _ := r.Context().Value(contextKeyHealthCheckProbe).(bool) + return probe +} + type TargetSlot int const ( @@ -111,6 +121,7 @@ type ServiceOptions struct { ReadTargetsAcceptWebsockets bool `json:"read_targets_accept_websockets"` ExcludeMetricsPaths []string `json:"exclude_metrics_paths"` ClientIPHeader string `json:"client_ip_header"` + PublishHealthCheck bool `json:"publish_health_check"` } func (so *ServiceOptions) ShouldExcludeMetrics(r *http.Request) bool { @@ -522,7 +533,7 @@ func (s *Service) createMiddleware(options ServiceOptions, certManager CertManag func (s *Service) serviceRequestWithTarget(w http.ResponseWriter, r *http.Request) { LoggingRequestContext(r).Service = s.name - if !s.options.TLSEnabled && r.TLS != nil { + if !s.options.TLSEnabled && r.TLS != nil && !isHealthCheckProbe(r) { SetErrorResponse(w, r, http.StatusServiceUnavailable, nil) return } diff --git a/internal/server/service_map.go b/internal/server/service_map.go index c2df79c1..2530409b 100644 --- a/internal/server/service_map.go +++ b/internal/server/service_map.go @@ -10,6 +10,8 @@ import ( const ( rootPath = "/" + + publishedHealthCheckPrefix = "/.kamal-proxy/" ) type pathBinding struct { @@ -20,15 +22,17 @@ type pathBinding struct { type requestServiceMap map[string][]*pathBinding type ServiceMap struct { - services map[string]*Service - requestServiceMap requestServiceMap - defaultTLSHostname string + services map[string]*Service + requestServiceMap requestServiceMap + publishedHealthChecks map[string]*Service + defaultTLSHostname string } func NewServiceMap() *ServiceMap { return &ServiceMap{ - services: map[string]*Service{}, - requestServiceMap: requestServiceMap{}, + services: map[string]*Service{}, + requestServiceMap: requestServiceMap{}, + publishedHealthChecks: map[string]*Service{}, } } @@ -39,15 +43,21 @@ func (m *ServiceMap) Get(name string) *Service { func (m *ServiceMap) Set(service *Service) { m.services[service.name] = service m.updateRequestServiceMap() + m.updatePublishedHealthChecks() m.updateDefaultTLSHostname() } func (m *ServiceMap) Remove(name string) { delete(m.services, name) m.updateRequestServiceMap() + m.updatePublishedHealthChecks() m.updateDefaultTLSHostname() } +func (m *ServiceMap) PublishedHealthCheck(path string) *Service { + return m.publishedHealthChecks[path] +} + func (m *ServiceMap) All() iter.Seq2[string, *Service] { return func(yield func(string, *Service) bool) { for name, service := range m.services { @@ -153,6 +163,18 @@ func (m *ServiceMap) updateRequestServiceMap() { m.syncTLSOptionsFromRootDomain() } +func (m *ServiceMap) updatePublishedHealthChecks() { + published := map[string]*Service{} + + for name, service := range m.services { + if service.options.PublishHealthCheck { + published[publishedHealthCheckPrefix+name+"/health"] = service + } + } + + m.publishedHealthChecks = published +} + func (m *ServiceMap) updateDefaultTLSHostname() { for _, service := range m.services { if service.options.TLSEnabled && len(service.options.Hosts) > 0 && service.options.Hosts[0] != "" { diff --git a/internal/server/service_map_test.go b/internal/server/service_map_test.go index 6b55f195..c05c0726 100644 --- a/internal/server/service_map_test.go +++ b/internal/server/service_map_test.go @@ -54,6 +54,28 @@ func TestServiceMap_ServiceForRequest(t *testing.T) { checkService("6", "http://second.example.com/non-api/test") } +func TestServiceMap_PublishedHealthCheck(t *testing.T) { + sm := NewServiceMap() + + assert.Nil(t, sm.PublishedHealthCheck("/.kamal-proxy/1/health")) + + sm.Set(&Service{name: "1", options: normalizedServiceOptions(ServiceOptions{Hosts: []string{"example.com"}})}) + + assert.Nil(t, sm.PublishedHealthCheck("/.kamal-proxy/1/health")) + + sm.Set(&Service{name: "2", options: normalizedServiceOptions(ServiceOptions{Hosts: []string{"app.example.com"}, PublishHealthCheck: true})}) + + assert.Equal(t, "2", sm.PublishedHealthCheck("/.kamal-proxy/2/health").name) + + for _, path := range []string{"/.kamal-proxy/1/health", "/.kamal-proxy/nosuch/health", "/.kamal-proxy/2/health/extra", "/.kamal-proxy/health", "/.kamal-proxy"} { + assert.Nil(t, sm.PublishedHealthCheck(path)) + } + + sm.Remove("2") + + assert.Nil(t, sm.PublishedHealthCheck("/.kamal-proxy/2/health")) +} + func TestServiceMap_CheckAvailability(t *testing.T) { sm := NewServiceMap() sm.Set(&Service{name: "1", options: normalizedServiceOptions(ServiceOptions{Hosts: []string{"example.com"}})}) diff --git a/internal/server/target.go b/internal/server/target.go index b1bd627b..6b8381c3 100644 --- a/internal/server/target.go +++ b/internal/server/target.go @@ -76,7 +76,8 @@ type TargetOptions struct { } func (to *TargetOptions) IsHealthCheckRequest(r *http.Request) bool { - return (r.Method == http.MethodGet || r.Method == http.MethodHead) && RoutedTargetPath(r) == to.HealthCheckConfig.Path + return isHealthCheckProbe(r) || + ((r.Method == http.MethodGet || r.Method == http.MethodHead) && RoutedTargetPath(r) == to.HealthCheckConfig.Path) } func (to *TargetOptions) canonicalizeLogHeaders() { @@ -278,16 +279,25 @@ func (t *Target) HealthCheckCompleted(success bool) { func (t *Target) buildHealthCheckURL() *url.URL { healthCheckURL := *t.targetURL + healthCheckURL.Host = t.healthCheckAddress() + // Join from the root so the result is always an absolute path, even when + // the configured path is missing its leading slash. + healthCheckURL.Path = "/" + + return healthCheckURL.JoinPath(t.options.HealthCheckConfig.Path) +} + +func (t *Target) healthCheckAddress() string { if t.options.HealthCheckConfig.Port > 0 { host, _, err := net.SplitHostPort(t.targetURL.Host) if err != nil { host = t.targetURL.Host } - healthCheckURL.Host = fmt.Sprintf("%s:%d", host, t.options.HealthCheckConfig.Port) + return fmt.Sprintf("%s:%d", host, t.options.HealthCheckConfig.Port) } - return healthCheckURL.JoinPath(t.options.HealthCheckConfig.Path) + return t.targetURL.Host } func (t *Target) createProxyHandler() http.Handler { @@ -310,6 +320,11 @@ func (t *Target) rewrite(req *httputil.ProxyRequest) { req.SetURL(t.targetURL) req.Out.Host = req.In.Host + if isHealthCheckProbe(req.In) { + t.rewriteHealthCheckProbe(req) + return + } + req.Out.URL.Path = RoutedTargetPath(req.In) // Ensure query params are preserved exactly, including those we could not @@ -338,6 +353,17 @@ func (t *Target) rewrite(req *httputil.ProxyRequest) { req.Out.URL.RawQuery = req.In.URL.RawQuery } +func (t *Target) rewriteHealthCheckProbe(req *httputil.ProxyRequest) { + healthCheckURL := t.buildHealthCheckURL() + + req.Out.URL.Host = healthCheckURL.Host + req.Out.URL.Path = healthCheckURL.Path + req.Out.URL.RawPath = healthCheckURL.RawPath + req.Out.URL.RawQuery = "" + req.Out.URL.ForceQuery = false + req.Out.Host = t.options.HealthCheckConfig.Host +} + func (t *Target) forwardHeaders(req *httputil.ProxyRequest) { if t.options.ForwardHeaders { req.Out.Header["X-Forwarded-For"] = req.In.Header["X-Forwarded-For"]