diff --git a/src/Aspire.Hosting/ApplicationModel/EndpointAnnotation.cs b/src/Aspire.Hosting/ApplicationModel/EndpointAnnotation.cs index 354499973c1..4fe6a30685f 100644 --- a/src/Aspire.Hosting/ApplicationModel/EndpointAnnotation.cs +++ b/src/Aspire.Hosting/ApplicationModel/EndpointAnnotation.cs @@ -1,10 +1,11 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Diagnostics; +using System.Collections; using System.Collections.Concurrent; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Net.Sockets; -using System.Collections; namespace Aspire.Hosting.ApplicationModel; @@ -454,6 +455,24 @@ public Task GetAllocatedEndpointAsync(NetworkIdentifier netwo return nes.Snapshot.GetValueAsync(cancellationToken); } + internal bool TryGetAllocatedEndpoint(NetworkIdentifier networkId, [NotNullWhen(true)] out AllocatedEndpoint? endpoint) + { + endpoint = null; + + foreach (var endpointSnapshot in _snapshots) + { + if (!endpointSnapshot.NetworkID.Equals(networkId) || !endpointSnapshot.Snapshot.IsValueSet) + { + continue; + } + + endpoint = endpointSnapshot.Snapshot.GetValueAsync().GetAwaiter().GetResult(); + return true; + } + + return false; + } + private NetworkEndpointSnapshot GetSnapshotFor(NetworkIdentifier networkId) { lock (_snapshots) diff --git a/src/Aspire.Hosting/ApplicationModel/EndpointReference.cs b/src/Aspire.Hosting/ApplicationModel/EndpointReference.cs index 38bf5581644..f90c97c1e10 100644 --- a/src/Aspire.Hosting/ApplicationModel/EndpointReference.cs +++ b/src/Aspire.Hosting/ApplicationModel/EndpointReference.cs @@ -221,6 +221,26 @@ public ReferenceExpression GetTlsValue(ReferenceExpression enabledValue, Referen GetAllocatedEndpoint() ?? throw new InvalidOperationException($"The endpoint `{EndpointName}` is not allocated for the resource `{Resource.Name}`."); + internal Task GetAllocatedEndpointAsync(NetworkIdentifier networkId, CancellationToken cancellationToken = default) + { + var endpointAnnotation = EndpointAnnotation; + if (endpointAnnotation.AllAllocatedEndpoints.TryGetAllocatedEndpoint(networkId, out var endpoint)) + { + return Task.FromResult(endpoint); + } + + foreach (var allocationAnnotation in Resource.Annotations.OfType()) + { + endpoint = allocationAnnotation.TryAllocate(endpointAnnotation, networkId); + if (endpoint is not null) + { + return Task.FromResult(endpoint); + } + } + + return endpointAnnotation.AllAllocatedEndpoints.GetAllocatedEndpointAsync(networkId, cancellationToken); + } + private EndpointAnnotation? GetEndpointAnnotation() { if (_endpointAnnotation is not null) @@ -242,20 +262,9 @@ public ReferenceExpression GetTlsValue(ReferenceExpression enabledValue, Referen return null; } - foreach (var nes in endpointAnnotation.AllAllocatedEndpoints) - { - if (string.Equals(nes.NetworkID.Value, (_contextNetworkId ?? KnownNetworkIdentifiers.LocalhostNetwork).Value, StringComparisons.NetworkId)) - { - if (!nes.Snapshot.IsValueSet) - { - continue; - } - - return nes.Snapshot.GetValueAsync().GetAwaiter().GetResult(); - } - } - - return null; + return endpointAnnotation.AllAllocatedEndpoints.TryGetAllocatedEndpoint(_contextNetworkId ?? KnownNetworkIdentifiers.LocalhostNetwork, out var allocatedEndpoint) + ? allocatedEndpoint + : null; } /// @@ -382,8 +391,7 @@ public class EndpointReferenceExpression(EndpointReference endpointReference, En async ValueTask ResolveValueWithAllocatedAddress() { - var endpointSnapshots = Endpoint.EndpointAnnotation.AllAllocatedEndpoints; - var allocatedEndpoint = await endpointSnapshots.GetAllocatedEndpointAsync(networkContext, cancellationToken).ConfigureAwait(false); + var allocatedEndpoint = await Endpoint.GetAllocatedEndpointAsync(networkContext, cancellationToken).ConfigureAwait(false); return Property switch { diff --git a/src/Aspire.Hosting/ApplicationModel/OnDemandEndpointAllocationAnnotation.cs b/src/Aspire.Hosting/ApplicationModel/OnDemandEndpointAllocationAnnotation.cs new file mode 100644 index 00000000000..ac1b266c32d --- /dev/null +++ b/src/Aspire.Hosting/ApplicationModel/OnDemandEndpointAllocationAnnotation.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Stores a resource-owned endpoint allocator that can run before normal allocation completes. +/// +internal sealed class OnDemandEndpointAllocationAnnotation(Func allocator) : IResourceAnnotation +{ + private Func? _allocator = allocator; + + public AllocatedEndpoint? TryAllocate(EndpointAnnotation endpoint, NetworkIdentifier networkId) + { + var allocator = _allocator; + + return allocator?.Invoke(endpoint, networkId); + } + + public void StopAllocating() + { + Interlocked.Exchange(ref _allocator, null); + } +} diff --git a/src/Aspire.Hosting/Dcp/ContainerCreator.cs b/src/Aspire.Hosting/Dcp/ContainerCreator.cs index 0bc3532dc56..a9eabcf36ff 100644 --- a/src/Aspire.Hosting/Dcp/ContainerCreator.cs +++ b/src/Aspire.Hosting/Dcp/ContainerCreator.cs @@ -199,7 +199,7 @@ public IEnumerable> PrepareObjects() } var containerAppResource = new RenderedModelResource(container, ctr); - DcpModelUtilities.AddServicesProducedInfo(containerAppResource, _appResources.Get()); + DcpModelUtilities.AddServicesProducedInfo(containerAppResource, _appResources.Get(), _logger); _appResources.Add(containerAppResource); result.Add(containerAppResource); } @@ -306,11 +306,6 @@ private async Task BuildAndCreateContainerAsync(RenderedModelResource var spec = dcpContainer.Spec; - if (cr.ServicesProduced.Count > 0) - { - spec.Ports = BuildContainerPorts(cr); - } - spec.VolumeMounts = BuildContainerMounts(cr.ModelResource); var (runArgs, failedToApplyRunArgs) = await BuildRunArgsAsync(logger, cr.ModelResource, cToken).ConfigureAwait(false); @@ -321,11 +316,24 @@ private async Task BuildAndCreateContainerAsync(RenderedModelResource spec.RunArgs = runArgs; var (configuration, pemCertificates, createFiles) = await BuildContainerConfiguration(cr, logger, cToken).ConfigureAwait(false); + // Configuration callbacks are the last pre-creation point where on-demand allocation can run. + cr.ModelResource.Annotations + .OfType() + .SingleOrDefault() + ?.StopAllocating(); + if (configuration.Exception is not null) { throw new FailedToApplyEnvironmentException($"Failed to apply configuration to container {cr.ModelResource.Name}", configuration.Exception); } + // Environment callbacks can resolve proxyless endpoint ports and commit a fallback host port, + // so build ports afterward. + if (cr.ServicesProduced.Count > 0) + { + spec.Ports = BuildContainerPorts(cr); + } + var args = configuration.Arguments.ToList(); if (modelContainer is ContainerResource { ShellExecution: true }) { @@ -987,10 +995,11 @@ private static List BuildContainerPorts(RenderedModelResource if (!ea.IsProxied && ea.SpecifiedPort is int hostPort) { + sp.Service.Spec.Port ??= hostPort; portSpec.HostPort = hostPort; } - switch (sp.EndpointAnnotation.Protocol) + switch (ea.Protocol) { case ProtocolType.Tcp: portSpec.Protocol = PortProtocol.TCP; @@ -1000,9 +1009,9 @@ private static List BuildContainerPorts(RenderedModelResource break; } - if (sp.EndpointAnnotation.TargetHost != KnownHostNames.Localhost) + if (ea.TargetHost != KnownHostNames.Localhost) { - portSpec.HostIP = sp.EndpointAnnotation.TargetHost; + portSpec.HostIP = ea.TargetHost; } ports.Add(portSpec); diff --git a/src/Aspire.Hosting/Dcp/DcpModelUtilities.cs b/src/Aspire.Hosting/Dcp/DcpModelUtilities.cs index bc3fdd0000a..05f58864dcd 100644 --- a/src/Aspire.Hosting/Dcp/DcpModelUtilities.cs +++ b/src/Aspire.Hosting/Dcp/DcpModelUtilities.cs @@ -7,6 +7,7 @@ using System.Net; using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Dcp.Model; +using Microsoft.Extensions.Logging; namespace Aspire.Hosting.Dcp; @@ -33,7 +34,8 @@ internal static bool ShouldDeferCreateForExplicitStart(IResource modelResource, /// internal static void AddServicesProducedInfo( RenderedModelResource appResource, - IEnumerable appResources) + IEnumerable appResources, + ILogger? logger = null) where TDcpResource : CustomResource, IKubernetesStaticMetadata { var modelResource = appResource.ModelResource; @@ -87,6 +89,16 @@ internal static void AddServicesProducedInfo( appResource.ServicesProduced.Add(sp); } + if (appResource.ServicesProduced.Any(sp => IsDynamicProxylessContainerEndpoint(appResource, sp)) && + !modelResource.Annotations.OfType().Any()) + { + // These endpoints normally get their host port during container creation. If a + // reference needs the allocated endpoint while building the container configuration, + // commit the fallback port before waiting would deadlock resource creation. + modelResource.Annotations.Add(new OnDemandEndpointAllocationAnnotation( + (endpoint, networkId) => TryAllocateDynamicProxylessContainerEndpoint(appResource, endpoint, networkId, logger))); + } + static bool HasMultipleReplicas(CustomResource resource) { if (resource is Executable exe && exe.Metadata.Annotations.TryGetValue(CustomResource.ResourceReplicaCount, out var value) && int.TryParse(value, CultureInfo.InvariantCulture, out var replicas) && replicas > 1) @@ -161,9 +173,10 @@ internal static bool TryApplyServiceAddressToEndpoint(Service observedService, I return isDynamicProxylessContainerEndpoint && AreResourceEndpointsAllocated(modelResource); } - private static bool TryAddLocalhostAllocatedEndpoint(ServiceWithModelResource sp, bool allowPending) + private static bool TryAddLocalhostAllocatedEndpoint(ServiceWithModelResource sp, bool allowPending, int? fallbackPort = null) { var svc = sp.DcpResource; + var allocatedPort = svc.AllocatedPort ?? fallbackPort; if (sp.EndpointAnnotation.AllocatedEndpoint is not null) { @@ -182,7 +195,7 @@ private static bool TryAddLocalhostAllocatedEndpoint(ServiceWithModelResource sp throw new InvalidDataException($"Service {svc.Metadata.Name} should have valid address at this point"); } - if (!sp.EndpointAnnotation.IsProxied && svc.AllocatedPort is null) + if (!sp.EndpointAnnotation.IsProxied && allocatedPort is null) { if (allowPending) { @@ -192,7 +205,7 @@ private static bool TryAddLocalhostAllocatedEndpoint(ServiceWithModelResource sp throw new InvalidOperationException($"Service '{svc.Metadata.Name}' needs to specify a port for endpoint '{sp.EndpointAnnotation.Name}' since it isn't using a proxy."); } - if (!svc.HasCompleteAddress) + if (allocatedPort is null || string.IsNullOrEmpty(svc.AllocatedAddress)) { if (allowPending) { @@ -207,7 +220,7 @@ private static bool TryAddLocalhostAllocatedEndpoint(ServiceWithModelResource sp sp.EndpointAnnotation.AllocatedEndpoint = new AllocatedEndpoint( sp.EndpointAnnotation, targetHost, - (int)svc.AllocatedPort!, + allocatedPort.Value, bindingMode, targetPortExpression: $$$"""{{- portForServing "{{{svc.Metadata.Name}}}" -}}""", KnownNetworkIdentifiers.LocalhostNetwork); @@ -278,6 +291,42 @@ private static bool IsDynamicProxylessContainerEndpoint(RenderedMo sp.EndpointAnnotation.SpecifiedPort is null; } + private static AllocatedEndpoint? TryAllocateDynamicProxylessContainerEndpoint( + RenderedModelResource resource, + EndpointAnnotation endpoint, + NetworkIdentifier networkId, + ILogger? logger) + where TDcpResource : CustomResource, IKubernetesStaticMetadata + { + var sp = resource.ServicesProduced.SingleOrDefault(sp => + ReferenceEquals(sp.EndpointAnnotation, endpoint) && + IsDynamicProxylessContainerEndpoint(resource, sp)); + if (sp is null) + { + return null; + } + + Debug.Assert(endpoint.TargetPort is not null); + + var targetPort = endpoint.TargetPort.Value; + endpoint.Port = targetPort; + logger?.LogInformation( + "Endpoint '{EndpointName}' on container resource '{ResourceName}' was resolved before the container was created, so Aspire is assigning public port {PublicPort} to match target port {TargetPort} for proxyless access.", + endpoint.Name, + sp.ModelResource.Name, + targetPort, + targetPort); + + if (TryAddLocalhostAllocatedEndpoint(sp, allowPending: false, fallbackPort: targetPort)) + { + AddContainerNetworkAllocatedEndpoint(resource, sp); + } + + return endpoint.AllAllocatedEndpoints.TryGetAllocatedEndpoint(networkId, out var allocatedEndpoint) + ? allocatedEndpoint + : null; + } + internal static void AddContainerTunnelAllocatedEndpoints( IEnumerable affectedResources, DcpAppResourceStore allAppResources, diff --git a/tests/Aspire.Hosting.Tests/Dcp/DcpExecutorTests.cs b/tests/Aspire.Hosting.Tests/Dcp/DcpExecutorTests.cs index 06c3ac465f6..711b42f19a7 100644 --- a/tests/Aspire.Hosting.Tests/Dcp/DcpExecutorTests.cs +++ b/tests/Aspire.Hosting.Tests/Dcp/DcpExecutorTests.cs @@ -25,7 +25,9 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.FileProviders; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Logging.Testing; using Microsoft.Extensions.Options; using Polly; using Polly.Retry; @@ -1665,7 +1667,7 @@ public async Task EndpointPortsContainerProxylessNoPortTargetPortSetPublishesAll var builder = DistributedApplication.CreateBuilder(); const int desiredTargetPort = TestKubernetesService.StartOfAutoPortRange - 999; - builder.AddContainer("database", "image") + var database = builder.AddContainer("database", "image") .WithEndpoint(name: "NoPortTargetPortSet", targetPort: desiredTargetPort, isProxied: false); var allocatedPortChannel = Channel.CreateUnbounded(); @@ -1691,8 +1693,80 @@ public async Task EndpointPortsContainerProxylessNoPortTargetPortSetPublishesAll await appExecutor.RunApplicationAsync(); var allocatedPort = await allocatedPortChannel.Reader.ReadAsync().AsTask().DefaultTimeout(); + var dcpCtr = Assert.Single(kubernetesService.CreatedResources.OfType()); + var svc = kubernetesService.CreatedResources.OfType().Single(s => s.Name() == "database"); + Assert.NotNull(dcpCtr.Spec.Ports); + Assert.Contains(dcpCtr.Spec.Ports!, p => p.HostPort is null && p.ContainerPort == desiredTargetPort); + Assert.Equal(allocatedPort, svc.Status?.EffectivePort); + Assert.NotEqual(desiredTargetPort, allocatedPort); Assert.True(allocatedPort >= TestKubernetesService.StartOfAutoPortRange); + Assert.Equal(allocatedPort.ToString(CultureInfo.InvariantCulture), await database.GetEndpoint("NoPortTargetPortSet").Property(EndpointProperty.Port).GetValueAsync()); + } + + [Fact] + public async Task EndpointPortsContainerProxylessNoPortTargetPortSetUsesTargetPortFallbackWhenResolvedBeforeContainerCreation() + { + var builder = DistributedApplication.CreateBuilder(); + + const int desiredTargetPort = TestKubernetesService.StartOfAutoPortRange - 999; + var database = builder.AddContainer("database", "image") + .WithEndpoint(name: "NoPortTargetPortSet", targetPort: desiredTargetPort, isProxied: false); + database.WithEnvironment("PUBLIC_PORT", database.GetEndpoint("NoPortTargetPortSet").Property(EndpointProperty.Port)); + database.WithEnvironment("PUBLIC_PORT_AGAIN", database.GetEndpoint("NoPortTargetPortSet").Property(EndpointProperty.Port)); + + var kubernetesService = new TestKubernetesService(); + var testSink = new TestSink(); + var containerCreatorLogger = new TestLogger(new TestLoggerFactory(testSink, enabled: true)); + using var app = builder.Build(); + var distributedAppModel = app.Services.GetRequiredService(); + var appExecutor = CreateAppExecutor(distributedAppModel, kubernetesService: kubernetesService, containerCreatorLogger: containerCreatorLogger); + await appExecutor.RunApplicationAsync(); + + var dcpCtr = Assert.Single(kubernetesService.CreatedResources.OfType()); + var svc = kubernetesService.CreatedResources.OfType().Single(s => s.Name() == "database"); + + Assert.Equal(AddressAllocationModes.Proxyless, svc.Spec.AddressAllocationMode); + Assert.Equal(desiredTargetPort, svc.Status?.EffectivePort); + Assert.NotNull(dcpCtr.Spec.Ports); + Assert.Contains(dcpCtr.Spec.Ports!, p => p.HostPort == desiredTargetPort && p.ContainerPort == desiredTargetPort); + var envVarVal = dcpCtr.Spec.Env?.Single(v => v.Name == "PUBLIC_PORT").Value; + Assert.False(string.IsNullOrWhiteSpace(envVarVal)); + Assert.Equal(desiredTargetPort, int.Parse(envVarVal, CultureInfo.InvariantCulture)); + var secondEnvVarVal = dcpCtr.Spec.Env?.Single(v => v.Name == "PUBLIC_PORT_AGAIN").Value; + Assert.False(string.IsNullOrWhiteSpace(secondEnvVarVal)); + Assert.Equal(desiredTargetPort, int.Parse(secondEnvVarVal, CultureInfo.InvariantCulture)); + + Assert.Contains(testSink.Writes, log => + log.LogLevel == LogLevel.Information && + log.Message == $"Endpoint 'NoPortTargetPortSet' on container resource 'database' was resolved before the container was created, so Aspire is assigning public port {desiredTargetPort} to match target port {desiredTargetPort} for proxyless access."); + } + + [Fact] + public async Task EndpointPortsContainerProxylessNoPortTargetPortSetUsesTargetPortFallbackWhenHostAndPortResolvedBeforeContainerCreation() + { + var builder = DistributedApplication.CreateBuilder(); + + const int desiredTargetPort = TestKubernetesService.StartOfAutoPortRange - 999; + var database = builder.AddContainer("database", "image") + .WithEndpoint(name: "NoPortTargetPortSet", targetPort: desiredTargetPort, isProxied: false); + database.WithEnvironment("PUBLIC_HOST_AND_PORT", database.GetEndpoint("NoPortTargetPortSet").Property(EndpointProperty.HostAndPort)); + + var kubernetesService = new TestKubernetesService(); + using var app = builder.Build(); + var distributedAppModel = app.Services.GetRequiredService(); + var appExecutor = CreateAppExecutor(distributedAppModel, kubernetesService: kubernetesService); + await appExecutor.RunApplicationAsync(); + + var dcpCtr = Assert.Single(kubernetesService.CreatedResources.OfType()); + var svc = kubernetesService.CreatedResources.OfType().Single(s => s.Name() == "database"); + + Assert.Equal(AddressAllocationModes.Proxyless, svc.Spec.AddressAllocationMode); + Assert.Equal(desiredTargetPort, svc.Status?.EffectivePort); + Assert.NotNull(dcpCtr.Spec.Ports); + Assert.Contains(dcpCtr.Spec.Ports!, p => p.HostPort == desiredTargetPort && p.ContainerPort == desiredTargetPort); + var envVarVal = dcpCtr.Spec.Env?.Single(v => v.Name == "PUBLIC_HOST_AND_PORT").Value; + Assert.Equal($"database.dev.internal:{desiredTargetPort}", envVarVal); } [Fact] @@ -4508,7 +4582,8 @@ private static DcpExecutor CreateAppExecutor( DcpOptions? dcpOptions = null, ResourceLoggerService? resourceLoggerService = null, DcpExecutorEvents? events = null, - Hosting.Eventing.IDistributedApplicationEventing? distributedApplicationEventing = null) + Hosting.Eventing.IDistributedApplicationEventing? distributedApplicationEventing = null, + ILogger? containerCreatorLogger = null) { if (configuration == null) { @@ -4573,7 +4648,7 @@ private static DcpExecutor CreateAppExecutor( resourceLoggerService, dcpDependencyCheckService, hostEnv, - NullLogger.Instance, + containerCreatorLogger ?? NullLogger.Instance, appResources); return new DcpExecutor( diff --git a/tests/Aspire.Hosting.Tests/Dcp/TestKubernetesService.cs b/tests/Aspire.Hosting.Tests/Dcp/TestKubernetesService.cs index 88387950ba7..52babc658f7 100644 --- a/tests/Aspire.Hosting.Tests/Dcp/TestKubernetesService.cs +++ b/tests/Aspire.Hosting.Tests/Dcp/TestKubernetesService.cs @@ -142,9 +142,11 @@ private List AllocateProxylessContainerServicePorts(CustomResour continue; } + var hostPort = container.Spec.Ports?.FirstOrDefault(port => port.ContainerPort == serviceProduced.Port)?.HostPort; + service.Status ??= new ServiceStatus(); service.Status.EffectiveAddress = service.Spec.Address ?? "localhost"; - service.Status.EffectivePort = Interlocked.Increment(ref _nextPort); + service.Status.EffectivePort = hostPort ?? Interlocked.Increment(ref _nextPort); modifiedResources.Add(service); }