diff --git a/src/Aspire.Hosting/ApplicationModel/EndpointReference.cs b/src/Aspire.Hosting/ApplicationModel/EndpointReference.cs index b12fd202c89..09126cf9502 100644 --- a/src/Aspire.Hosting/ApplicationModel/EndpointReference.cs +++ b/src/Aspire.Hosting/ApplicationModel/EndpointReference.cs @@ -221,61 +221,6 @@ public ReferenceExpression GetTlsValue(ReferenceExpression enabledValue, Referen GetAllocatedEndpoint() ?? throw new InvalidOperationException($"The endpoint `{EndpointName}` is not allocated for the resource `{Resource.Name}`."); - internal async Task GetAllocatedEndpointAsync(NetworkIdentifier networkId, ValueProviderContext context, CancellationToken cancellationToken = default) - { - var endpointAnnotation = EndpointAnnotation; - if (endpointAnnotation.AllAllocatedEndpoints.TryGetAllocatedEndpoint(networkId, out var endpoint)) - { - return endpoint; - } - - var allocationAnnotations = Resource.Annotations.OfType().ToArray(); - if (allocationAnnotations.Length > 0 && await ShouldAllocateEndpointOnDemandAsync(context, cancellationToken).ConfigureAwait(false)) - { - foreach (var allocationAnnotation in allocationAnnotations) - { - endpoint = allocationAnnotation.TryAllocate(endpointAnnotation, networkId); - if (endpoint is not null) - { - return endpoint; - } - } - } - - // Waiting here preserves late allocation for cases that don't need the on-demand fallback, - // such as proxyless container endpoints whose actual port is reported by DCP after startup. - return await endpointAnnotation.AllAllocatedEndpoints.GetAllocatedEndpointAsync(networkId, cancellationToken).ConfigureAwait(false); - } - - private async Task ShouldAllocateEndpointOnDemandAsync(ValueProviderContext context, CancellationToken cancellationToken) - { - if (context.Caller is not { } caller) - { - return true; - } - - if (Resource == caller) - { - return true; - } - - if (context.ExecutionContext is not { } executionContext) - { - return true; - } - - var dependencies = await Resource.GetResourceDependenciesAsync( - executionContext, - new ResourceDependencyDiscoveryOptions - { - DiscoveryMode = ResourceDependencyDiscoveryMode.Recursive, - CacheAnnotationCallbackResults = true - }, - cancellationToken).ConfigureAwait(false); - - return dependencies.Contains(caller); - } - private EndpointAnnotation? GetEndpointAnnotation() { if (_endpointAnnotation is not null) @@ -426,7 +371,7 @@ public class EndpointReferenceExpression(EndpointReference endpointReference, En async ValueTask ResolveValueWithAllocatedAddress() { - var allocatedEndpoint = await Endpoint.GetAllocatedEndpointAsync(networkContext, context, cancellationToken).ConfigureAwait(false); + var allocatedEndpoint = await Endpoint.EndpointAnnotation.AllAllocatedEndpoints.GetAllocatedEndpointAsync(networkContext, cancellationToken).ConfigureAwait(false); return Property switch { diff --git a/src/Aspire.Hosting/ApplicationModel/OnDemandEndpointAllocationAnnotation.cs b/src/Aspire.Hosting/ApplicationModel/OnDemandEndpointAllocationAnnotation.cs deleted file mode 100644 index ac1b266c32d..00000000000 --- a/src/Aspire.Hosting/ApplicationModel/OnDemandEndpointAllocationAnnotation.cs +++ /dev/null @@ -1,24 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace Aspire.Hosting.ApplicationModel; - -/// -/// Stores a resource-owned endpoint allocator that can run before normal allocation completes. -/// -internal sealed class OnDemandEndpointAllocationAnnotation(Func allocator) : IResourceAnnotation -{ - private Func? _allocator = allocator; - - public AllocatedEndpoint? TryAllocate(EndpointAnnotation endpoint, NetworkIdentifier networkId) - { - var allocator = _allocator; - - return allocator?.Invoke(endpoint, networkId); - } - - public void StopAllocating() - { - Interlocked.Exchange(ref _allocator, null); - } -} diff --git a/src/Aspire.Hosting/Dcp/ContainerCreator.cs b/src/Aspire.Hosting/Dcp/ContainerCreator.cs index a9eabcf36ff..68c49e4bc42 100644 --- a/src/Aspire.Hosting/Dcp/ContainerCreator.cs +++ b/src/Aspire.Hosting/Dcp/ContainerCreator.cs @@ -199,7 +199,7 @@ public IEnumerable> PrepareObjects() } var containerAppResource = new RenderedModelResource(container, ctr); - DcpModelUtilities.AddServicesProducedInfo(containerAppResource, _appResources.Get(), _logger); + DcpModelUtilities.AddServicesProducedInfo(containerAppResource, _appResources.Get()); _appResources.Add(containerAppResource); result.Add(containerAppResource); } @@ -316,11 +316,6 @@ private async Task BuildAndCreateContainerAsync(RenderedModelResource spec.RunArgs = runArgs; var (configuration, pemCertificates, createFiles) = await BuildContainerConfiguration(cr, logger, cToken).ConfigureAwait(false); - // Configuration callbacks are the last pre-creation point where on-demand allocation can run. - cr.ModelResource.Annotations - .OfType() - .SingleOrDefault() - ?.StopAllocating(); if (configuration.Exception is not null) { diff --git a/src/Aspire.Hosting/Dcp/DcpExecutor.cs b/src/Aspire.Hosting/Dcp/DcpExecutor.cs index 58a6a649ebe..fe62047f6f7 100644 --- a/src/Aspire.Hosting/Dcp/DcpExecutor.cs +++ b/src/Aspire.Hosting/Dcp/DcpExecutor.cs @@ -3,9 +3,11 @@ #pragma warning disable ASPIRECERTIFICATES001 #pragma warning disable ASPIRECONTAINERSHELLEXECUTION001 +#pragma warning disable ASPIREUSERSECRETS001 using System.Collections.Immutable; using System.Diagnostics; +using System.Globalization; using System.Text; using System.Text.Json; using System.Text.RegularExpressions; @@ -58,6 +60,7 @@ internal sealed partial class DcpExecutor : IDcpExecutor, IDcpObjectFactory, IAs private readonly IOptions _options; private readonly DistributedApplicationExecutionContext _executionContext; private readonly DcpAppResourceStore _appResources; + private readonly IUserSecretsManager _userSecretsManager; // Has an entry if we raised ResourceEndpointsAllocatedEvent for a resource with a given name. // We want to ensure we raise the event only once for each app model resource. @@ -72,6 +75,7 @@ internal sealed partial class DcpExecutor : IDcpExecutor, IDcpObjectFactory, IAs private readonly ExecutableCreator _executableCreator; private readonly ContainerCreator _containerCreator; + private readonly ProxylessEndpointPortAllocator _proxylessEndpointPortAllocator; // We need to preserve the container creation context from the application startup phase // so that container explicit start does not suffer from timing issues. @@ -98,7 +102,9 @@ public DcpExecutor(ILogger logger, DcpAppResourceStore appResources, ExecutableCreator executableCreator, ContainerCreator containerCreator, - ProfilingTelemetry profilingTelemetry) + ProfilingTelemetry profilingTelemetry, + ProxylessEndpointPortAllocator proxylessEndpointPortAllocator, + IUserSecretsManager userSecretsManager) { _distributedApplicationLogger = distributedApplicationLogger; _kubernetesService = kubernetesService; @@ -113,14 +119,16 @@ public DcpExecutor(ILogger logger, _options = options; _executionContext = executionContext; _appResources = appResources; + _userSecretsManager = userSecretsManager; - _resourceWatcher = new DcpResourceWatcher(logger, kubernetesService, loggerService, executorEvents, model, _appResources, _configuration, PublishEndpointAllocatedEventsAsync, profilingTelemetry, _shutdownCancellation.Token); + _resourceWatcher = new DcpResourceWatcher(logger, kubernetesService, loggerService, executorEvents, model, _appResources, _configuration, profilingTelemetry, _shutdownCancellation.Token); DeleteResourceRetryPipeline = DcpPipelineBuilder.BuildDeleteRetryPipeline(logger); _containerContextSource = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); _executableCreator = executableCreator; _containerCreator = containerCreator; + _proxylessEndpointPortAllocator = proxylessEndpointPortAllocator; } private string ContainerHostName => _configuration["AppHost:ContainerHostname"] ?? @@ -206,8 +214,7 @@ public async Task RunApplicationAsync(CancellationToken ct = default) if (DcpModelUtilities.TryAddWorkloadAllocatedEndpoints( executable, _options.Value.EnableAspireContainerTunnel, - ContainerHostName, - allowPendingDynamicProxylessContainerEndpoints: false)) + ContainerHostName)) { endpointAllocatedResources.Add(executable.ModelResource); } @@ -215,20 +222,10 @@ public async Task RunApplicationAsync(CancellationToken ct = default) foreach (var container in containers) { - // A dependent resource can resolve this container's proxyless endpoint during - // its starting callback. Commit required fallback ports before any container - // objects are submitted so the rendered container exposes the same port. - await DcpModelUtilities.TryAllocateDependentDynamicProxylessContainerEndpointsAsync( - container, - _executionContext, - _logger, - ct).ConfigureAwait(false); - if (DcpModelUtilities.TryAddWorkloadAllocatedEndpoints( container, _options.Value.EnableAspireContainerTunnel, - ContainerHostName, - allowPendingDynamicProxylessContainerEndpoints: true)) + ContainerHostName)) { endpointAllocatedResources.Add(container.ModelResource); } @@ -240,7 +237,7 @@ await DcpModelUtilities.TryAllocateDependentDynamicProxylessContainerEndpointsAs // make a valid cross-resource callback observe an unallocated endpoint. foreach (var resource in endpointAllocatedResources.Distinct()) { - await PublishEndpointAllocatedEventsAsync(resource, ct).ConfigureAwait(false); + await PublishEndpointsAllocatedEventAsync(resource, ct).ConfigureAwait(false); } }, ct); @@ -653,8 +650,30 @@ private void PrepareServices() var serviceProducers = _model.Resources .Select(r => (ModelResource: r, Endpoints: r.Annotations.OfType().ToArray())) - .Where(sp => sp.Endpoints.Any()); + .Where(sp => sp.Endpoints.Any()) + .ToArray(); + // Resolve endpoint behavior and exclude known public ports before any dynamic allocation can claim them. + foreach (var sp in serviceProducers) + { + foreach (var endpoint in sp.Endpoints) + { + endpoint.SetResolvedIsProxied(GetEffectiveIsProxied(sp.ModelResource, endpoint, _options.Value.RandomizePorts)); + ValidateEndpointBeforeDynamicPublicPortAllocation(sp.ModelResource, endpoint); + + if (TryGetEffectiveFixedPublicPort(sp.ModelResource, endpoint, _options.Value.RandomizePorts, out var fixedPublicPort)) + { + _proxylessEndpointPortAllocator.ExcludePort(fixedPublicPort); + } + + if (TryGetPersistedProxylessEndpointPort(sp.ModelResource, endpoint) is int persistedPort) + { + _proxylessEndpointPortAllocator.ExcludePort(persistedPort); + } + } + } + + // Create DCP services after known ports are excluded, allocating missing proxyless public ports as needed. foreach (var sp in serviceProducers) { var endpoints = sp.Endpoints; @@ -670,21 +689,13 @@ private void PrepareServices() var svc = Service.Create(serviceName); - endpoint.SetResolvedIsProxied(GetEffectiveIsProxied(sp.ModelResource, endpoint, _options.Value.RandomizePorts)); + EnsureProxylessEndpointPort(sp.ModelResource, endpoint); - int? port; - if (_options.Value.RandomizePorts && endpoint.IsProxied && endpoint.Port != null) + if (TryGetEffectiveFixedPublicPort(sp.ModelResource, endpoint, _options.Value.RandomizePorts, out var fixedPublicPort)) { - port = null; - _logger.LogDebug("Randomizing port for {ServiceName}. Original port: {OriginalPort}", serviceName, endpoint.Port); + svc.Spec.Port = fixedPublicPort; } - else - { - port = sp.ModelResource.IsContainer() && !endpoint.IsProxied - ? endpoint.SpecifiedPort - : endpoint.Port; - } - svc.Spec.Port = port; + svc.Spec.Protocol = PortProtocol.FromProtocolType(endpoint.Protocol); if (string.Equals(KnownHostNames.Localhost, endpoint.TargetHost, StringComparison.OrdinalIgnoreCase)) { @@ -709,26 +720,6 @@ private void PrepareServices() } } - static bool GetEffectiveIsProxied(IResource resource, EndpointAnnotation endpoint, bool randomizePorts) - { - if (!resource.SupportsProxy()) - { - return false; - } - - if (endpoint.IsExplicitlyProxied is bool isProxied) - { - return isProxied; - } - - if (randomizePorts) - { - return true; - } - - return !resource.HasPersistentLifetime(); - } - var containers = _model.Resources.Where(r => r.IsContainer()); if (!containers.Any()) { @@ -751,6 +742,136 @@ static bool GetEffectiveIsProxied(IResource resource, EndpointAnnotation endpoin } } + private static bool GetEffectiveIsProxied(IResource resource, EndpointAnnotation endpoint, bool randomizePorts) + { + if (!resource.SupportsProxy()) + { + return false; + } + + if (endpoint.IsExplicitlyProxied is bool isProxied) + { + return isProxied; + } + + if (randomizePorts) + { + return true; + } + + return !resource.HasPersistentLifetime(); + } + + /// + /// Determines whether an endpoint definition has a fixed public port DCP should reserve or pre-exclude. + /// + /// + /// Use this when deciding whether DCP should bind a service to a known public port. Proxied endpoints + /// with randomized ports deliberately do not report a fixed port so DCP can allocate the public port + /// instead of reserving the configured value. + /// Container endpoint definitions keep the public host port separate from the target container port, so + /// only an explicitly specified public port counts as fixed. Executable endpoint definitions use the same + /// port value for the process and the public endpoint, so the effective public port can come from either + /// the endpoint port or target port. + /// + private static bool TryGetEffectiveFixedPublicPort(IResource resource, EndpointAnnotation endpoint, bool randomizePorts, out int publicPort) + { + var effectivePublicPort = resource.IsContainer() ? endpoint.SpecifiedPort : endpoint.Port; + + // When port randomization is enabled, proxied endpoints intentionally ignore the defined public + // port so DCP can allocate one dynamically instead. + if (randomizePorts && endpoint.IsProxied && effectivePublicPort is not null) + { + publicPort = default; + return false; + } + + if (effectivePublicPort is int fixedPublicPort) + { + publicPort = fixedPublicPort; + return true; + } + + publicPort = default; + return false; + } + + private void EnsureProxylessEndpointPort(IResource resource, EndpointAnnotation endpoint) + { + if (!NeedsPublicPort(resource, endpoint)) + { + return; + } + + int publicPort; + if (TryGetPersistedProxylessEndpointPort(resource, endpoint) is int persistedPort) + { + publicPort = persistedPort; + _logger.LogDebug("Using persisted public port {Port} for proxyless endpoint '{EndpointName}' on persistent resource '{ResourceName}'.", persistedPort, endpoint.Name, resource.Name); + } + else + { + publicPort = _proxylessEndpointPortAllocator.AllocatePort(endpoint); + _logger.LogDebug("Allocated public port {Port} for proxyless endpoint '{EndpointName}' on resource '{ResourceName}'.", publicPort, endpoint.Name, resource.Name); + + if (resource.HasPersistentLifetime()) + { + var secretKey = GetPersistedProxylessEndpointPortKey(resource, endpoint); + if (!_userSecretsManager.TrySetSecret(secretKey, publicPort.ToString(CultureInfo.InvariantCulture))) + { + _logger.LogWarning("Failed to persist public port {Port} for proxyless endpoint '{EndpointName}' on persistent resource '{ResourceName}'. Enable user secrets, set a fixed public port, or configure the endpoint to use a proxy to avoid recreating the persistent resource each run.", publicPort, endpoint.Name, resource.Name); + } + } + } + + endpoint.Port = publicPort; + if (!resource.IsContainer()) + { + endpoint.TargetPort = publicPort; + } + } + + private static bool NeedsPublicPort(IResource resource, EndpointAnnotation endpoint) + { + return !endpoint.IsProxied && !TryGetEffectiveFixedPublicPort(resource, endpoint, randomizePorts: false, out _); + } + + private int? TryGetPersistedProxylessEndpointPort(IResource resource, EndpointAnnotation endpoint) + { + if (!resource.HasPersistentLifetime() || !NeedsPublicPort(resource, endpoint)) + { + return null; + } + + var configuredPort = _configuration[GetPersistedProxylessEndpointPortKey(resource, endpoint)]; + if (configuredPort is null) + { + return null; + } + + if (int.TryParse(configuredPort, NumberStyles.None, CultureInfo.InvariantCulture, out var port) && + port is >= 1 and <= 65535) + { + return port; + } + + _logger.LogDebug("Ignoring invalid persisted public port value '{Port}' for proxyless endpoint '{EndpointName}' on persistent resource '{ResourceName}'.", configuredPort, endpoint.Name, resource.Name); + return null; + } + + private static string GetPersistedProxylessEndpointPortKey(IResource resource, EndpointAnnotation endpoint) + { + return $"Aspire:ProxylessEndpointPorts:{resource.Name}:{endpoint.Name}"; + } + + private static void ValidateEndpointBeforeDynamicPublicPortAllocation(IResource resource, EndpointAnnotation endpoint) + { + if (resource.IsContainer() && endpoint.TargetPort is null) + { + throw new InvalidOperationException($"The endpoint '{endpoint.Name}' for container resource '{resource.Name}' must specify the {nameof(EndpointAnnotation.TargetPort)} value"); + } + } + internal static void SetInitialResourceState(IResource resource, IAnnotationHolder annotationHolder) { // Store the initial state of the resource @@ -1251,14 +1372,6 @@ private async Task PublishEndpointsAllocatedEventAsync(IResource resource, return true; } - private async Task PublishEndpointAllocatedEventsAsync(IResource resource, CancellationToken ct) - { - if (await PublishEndpointsAllocatedEventAsync(resource, ct).ConfigureAwait(false)) - { - await PublishConnectionStringAvailableEventAsync(resource, ct).ConfigureAwait(false); - } - } - private async Task PublishConnectionStringAvailableEventAsync(IResource resource, CancellationToken ct) { if (!DcpModelUtilities.AreResourceEndpointsAllocated(resource)) diff --git a/src/Aspire.Hosting/Dcp/DcpModelUtilities.cs b/src/Aspire.Hosting/Dcp/DcpModelUtilities.cs index 1aa88d935bf..26eb2cf3904 100644 --- a/src/Aspire.Hosting/Dcp/DcpModelUtilities.cs +++ b/src/Aspire.Hosting/Dcp/DcpModelUtilities.cs @@ -7,7 +7,6 @@ using System.Net; using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Dcp.Model; -using Microsoft.Extensions.Logging; namespace Aspire.Hosting.Dcp; @@ -34,8 +33,7 @@ internal static bool ShouldDeferCreateForExplicitStart(IResource modelResource, /// internal static void AddServicesProducedInfo( RenderedModelResource appResource, - IEnumerable appResources, - ILogger? logger = null) + IEnumerable appResources) where TDcpResource : CustomResource, IKubernetesStaticMetadata { var modelResource = appResource.ModelResource; @@ -89,16 +87,6 @@ internal static void AddServicesProducedInfo( appResource.ServicesProduced.Add(sp); } - if (appResource.ServicesProduced.Any(sp => IsDynamicProxylessContainerEndpoint(appResource, sp)) && - !modelResource.Annotations.OfType().Any()) - { - // These endpoints normally get their host port during container creation. If a - // reference needs the allocated endpoint while building the container configuration, - // commit the fallback port before waiting would deadlock resource creation. - modelResource.Annotations.Add(new OnDemandEndpointAllocationAnnotation( - (endpoint, networkId) => TryAllocateDynamicProxylessContainerEndpoint(appResource, endpoint, networkId, logger))); - } - static bool HasMultipleReplicas(CustomResource resource) { if (resource is Executable exe && exe.Metadata.Annotations.TryGetValue(CustomResource.ResourceReplicaCount, out var value) && int.TryParse(value, CultureInfo.InvariantCulture, out var replicas) && replicas > 1) @@ -117,22 +105,19 @@ internal static void AddWorkloadAllocatedEndpoints( { foreach (var res in resources) { - TryAddWorkloadAllocatedEndpoints(res, enableAspireContainerTunnel, containerHostName, allowPendingDynamicProxylessContainerEndpoints: false); + TryAddWorkloadAllocatedEndpoints(res, enableAspireContainerTunnel, containerHostName); } } internal static bool TryAddWorkloadAllocatedEndpoints( RenderedModelResource resource, bool enableAspireContainerTunnel, - string containerHostName, - bool allowPendingDynamicProxylessContainerEndpoints) + string containerHostName) where TDcpResource : CustomResource, IKubernetesStaticMetadata { foreach (var sp in resource.ServicesProduced) { - if (TryAddLocalhostAllocatedEndpoint( - sp, - allowPending: allowPendingDynamicProxylessContainerEndpoints && IsDynamicProxylessContainerEndpoint(resource, sp))) + if (TryAddLocalhostAllocatedEndpoint(sp, allowPending: false)) { AddContainerNetworkAllocatedEndpoint(resource, sp); AddExecutableContainerNetworkAllocatedEndpoint(resource, sp, enableAspireContainerTunnel, containerHostName); @@ -142,57 +127,20 @@ internal static bool TryAddWorkloadAllocatedEndpoints( return AreResourceEndpointsAllocated(resource.ModelResource); } - internal static async Task TryAllocateDependentDynamicProxylessContainerEndpointsAsync( - RenderedModelResource resource, - DistributedApplicationExecutionContext executionContext, - ILogger? logger, - CancellationToken cancellationToken) - where TDcpResource : CustomResource, IKubernetesStaticMetadata - { - if (resource.ServicesProduced.All(sp => !IsDynamicProxylessContainerEndpoint(resource, sp))) - { - return; - } - - var dependencies = await resource.ModelResource.GetResourceDependenciesAsync( - executionContext, - new ResourceDependencyDiscoveryOptions - { - DiscoveryMode = ResourceDependencyDiscoveryMode.DirectOnly, - CacheAnnotationCallbackResults = true - }, - cancellationToken).ConfigureAwait(false); - - if (!dependencies.Any()) - { - return; - } - - foreach (var sp in resource.ServicesProduced.Where(sp => IsDynamicProxylessContainerEndpoint(resource, sp))) - { - TryAllocateDynamicProxylessContainerEndpoint(resource, sp.EndpointAnnotation, KnownNetworkIdentifiers.LocalhostNetwork, logger); - } - } - - internal static bool TryApplyServiceAddressToEndpoint(Service observedService, IEnumerable appResources, [NotNullWhen(true)] out IResource? modelResource) + internal static void ApplyServiceAddressToEndpoint(Service observedService, IEnumerable appResources) { var serviceResource = appResources.OfType() .FirstOrDefault(swr => string.Equals(swr.DcpResource.Metadata.Name, observedService.Metadata.Name, StringComparison.Ordinal)); if (serviceResource is null) { - modelResource = null; - return false; + return; } serviceResource.Service.ApplyAddressInfoFrom(observedService); - var isDynamicProxylessContainerEndpoint = appResources.OfType>() - .Any(resource => ReferenceEquals(resource.ModelResource, serviceResource.ModelResource) && - IsDynamicProxylessContainerEndpoint(resource, serviceResource)); if (!TryAddLocalhostAllocatedEndpoint(serviceResource, allowPending: true)) { - modelResource = null; - return false; + return; } foreach (var containerResource in appResources.OfType>() @@ -200,9 +148,6 @@ internal static bool TryApplyServiceAddressToEndpoint(Service observedService, I { AddContainerNetworkAllocatedEndpoint(containerResource, serviceResource); } - - modelResource = serviceResource.ModelResource; - return isDynamicProxylessContainerEndpoint && AreResourceEndpointsAllocated(modelResource); } private static bool TryAddLocalhostAllocatedEndpoint(ServiceWithModelResource sp, bool allowPending, int? fallbackPort = null) @@ -315,50 +260,6 @@ internal static bool AreResourceEndpointsAllocated(IResource resource) return !resource.TryGetEndpoints(out var endpoints) || endpoints.All(e => e.AllocatedEndpoint is not null); } - private static bool IsDynamicProxylessContainerEndpoint(RenderedModelResource resource, ServiceWithModelResource sp) - where TDcpResource : CustomResource, IKubernetesStaticMetadata - { - return resource.DcpResource is Container && - !sp.EndpointAnnotation.IsProxied && - sp.EndpointAnnotation.SpecifiedPort is null; - } - - private static AllocatedEndpoint? TryAllocateDynamicProxylessContainerEndpoint( - RenderedModelResource resource, - EndpointAnnotation endpoint, - NetworkIdentifier networkId, - ILogger? logger) - where TDcpResource : CustomResource, IKubernetesStaticMetadata - { - var sp = resource.ServicesProduced.SingleOrDefault(sp => - ReferenceEquals(sp.EndpointAnnotation, endpoint) && - IsDynamicProxylessContainerEndpoint(resource, sp)); - if (sp is null) - { - return null; - } - - Debug.Assert(endpoint.TargetPort is not null); - - var targetPort = endpoint.TargetPort.Value; - endpoint.Port = targetPort; - logger?.LogInformation( - "Endpoint '{EndpointName}' on container resource '{ResourceName}' was resolved before the container was created, so Aspire is assigning public port {PublicPort} to match target port {TargetPort} for proxyless access.", - endpoint.Name, - sp.ModelResource.Name, - targetPort, - targetPort); - - if (TryAddLocalhostAllocatedEndpoint(sp, allowPending: false, fallbackPort: targetPort)) - { - AddContainerNetworkAllocatedEndpoint(resource, sp); - } - - return endpoint.AllAllocatedEndpoints.TryGetAllocatedEndpoint(networkId, out var allocatedEndpoint) - ? allocatedEndpoint - : null; - } - internal static void AddContainerTunnelAllocatedEndpoints( IEnumerable affectedResources, DcpAppResourceStore allAppResources, diff --git a/src/Aspire.Hosting/Dcp/DcpOptions.cs b/src/Aspire.Hosting/Dcp/DcpOptions.cs index 276330dcbec..97867c1bafb 100644 --- a/src/Aspire.Hosting/Dcp/DcpOptions.cs +++ b/src/Aspire.Hosting/Dcp/DcpOptions.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Diagnostics.CodeAnalysis; +using System.Globalization; using System.Reflection; using Aspire.Shared; using Microsoft.Extensions.Configuration; @@ -58,6 +60,21 @@ internal sealed class DcpOptions /// public bool RandomizePorts { get; set; } + /// + /// The first port in the range used to allocate unspecified public ports for proxyless endpoints. + /// + public int ProxylessEndpointPortRangeStart { get; set; } = 10000; + + /// + /// The last port in the range used to allocate unspecified public ports for proxyless endpoints. + /// + /// + /// The default leaves room for Aspire to persist stable allocated ports in the future while staying + /// compatible across supported OSes. Linux's default ephemeral range starts at 32768, which is the + /// most restrictive default among those OSes, so default allocations stop one port lower. + /// + public int ProxylessEndpointPortRangeEnd { get; set; } = 32767; + public int KubernetesConfigReadRetryCount { get; set; } = 300; public int KubernetesConfigReadRetryIntervalMilliseconds { get; set; } = 100; @@ -128,6 +145,21 @@ public ValidateOptionsResult Validate(string? name, DcpOptions options) builder.AddError("The path to the Aspire Dashboard binaries is missing.", "DashboardPath"); } + if (options.ProxylessEndpointPortRangeStart is < 1 or > 65535) + { + builder.AddError("The proxyless endpoint port range start must be between 1 and 65535.", nameof(options.ProxylessEndpointPortRangeStart)); + } + + if (options.ProxylessEndpointPortRangeEnd is < 1 or > 65535) + { + builder.AddError("The proxyless endpoint port range end must be between 1 and 65535.", nameof(options.ProxylessEndpointPortRangeEnd)); + } + + if (options.ProxylessEndpointPortRangeStart > options.ProxylessEndpointPortRangeEnd) + { + builder.AddError("The proxyless endpoint port range start must be less than or equal to the range end.", nameof(options.ProxylessEndpointPortRangeStart)); + } + return builder.Build(); } } @@ -224,6 +256,9 @@ public void Configure(DcpOptions options) } options.RandomizePorts = dcpPublisherConfiguration.GetValue(nameof(options.RandomizePorts), options.RandomizePorts); + options.ProxylessEndpointPortRangeStart = dcpPublisherConfiguration.GetValue(nameof(options.ProxylessEndpointPortRangeStart), options.ProxylessEndpointPortRangeStart); + options.ProxylessEndpointPortRangeEnd = dcpPublisherConfiguration.GetValue(nameof(options.ProxylessEndpointPortRangeEnd), options.ProxylessEndpointPortRangeEnd); + ApplyProxylessEndpointPortRangeOverride(options, configuration); options.WaitForResourceCleanup = dcpPublisherConfiguration.GetValue(nameof(options.WaitForResourceCleanup), options.WaitForResourceCleanup); options.ServiceStartupWatchTimeout = configuration.GetValue(KnownConfigNames.ServiceStartupWatchTimeout, KnownConfigNames.Legacy.ServiceStartupWatchTimeout, options.ServiceStartupWatchTimeout); options.ContainerRuntimeInitializationTimeout = dcpPublisherConfiguration.GetValue(nameof(options.ContainerRuntimeInitializationTimeout), options.ContainerRuntimeInitializationTimeout); @@ -234,6 +269,42 @@ public void Configure(DcpOptions options) options.EnableAspireContainerTunnel = configuration.GetValue(KnownConfigNames.EnableContainerTunnel, options.EnableAspireContainerTunnel); } + private static void ApplyProxylessEndpointPortRangeOverride(DcpOptions options, IConfiguration configuration) + { + if (configuration[KnownConfigNames.ProxylessEndpointPortRange] is not { Length: > 0 } configuredRange) + { + return; + } + + var separatorIndex = configuredRange.IndexOf('-', StringComparison.Ordinal); + if (separatorIndex < 0 || separatorIndex != configuredRange.LastIndexOf('-')) + { + ThrowInvalidProxylessEndpointPortRange(configuredRange); + } + + var startText = configuredRange[..separatorIndex].Trim(); + var endText = configuredRange[(separatorIndex + 1)..].Trim(); + if (!int.TryParse(startText, NumberStyles.None, CultureInfo.InvariantCulture, out var start)) + { + ThrowInvalidProxylessEndpointPortRange(configuredRange); + } + + if (!int.TryParse(endText, NumberStyles.None, CultureInfo.InvariantCulture, out var end)) + { + ThrowInvalidProxylessEndpointPortRange(configuredRange); + } + + options.ProxylessEndpointPortRangeStart = start; + options.ProxylessEndpointPortRangeEnd = end; + } + + [DoesNotReturn] + private static void ThrowInvalidProxylessEndpointPortRange(string configuredRange) + { + throw new InvalidOperationException( + $"Invalid value \"{configuredRange}\" for \"{KnownConfigNames.ProxylessEndpointPortRange}\". Expected a port range formatted as \"start-end\", for example \"10000-32767\"."); + } + private static string? GetMetadataValue(IEnumerable? assemblyMetadata, string key) { return assemblyMetadata?.FirstOrDefault(m => string.Equals(m.Key, key, StringComparison.OrdinalIgnoreCase))?.Value; diff --git a/src/Aspire.Hosting/Dcp/DcpResourceWatcher.cs b/src/Aspire.Hosting/Dcp/DcpResourceWatcher.cs index 876933b6932..d70841b7ffe 100644 --- a/src/Aspire.Hosting/Dcp/DcpResourceWatcher.cs +++ b/src/Aspire.Hosting/Dcp/DcpResourceWatcher.cs @@ -30,7 +30,6 @@ internal sealed class DcpResourceWatcher : IConsoleLogsService, IAsyncDisposable private readonly DcpExecutorEvents _executorEvents; private readonly ILogger _logger; private readonly IConfiguration _configuration; - private readonly Func _publishEndpointsAllocatedEventAsync; private readonly ProfilingTelemetry _profilingTelemetry; private readonly CancellationToken _shutdownToken; @@ -57,7 +56,6 @@ public DcpResourceWatcher( DistributedApplicationModel model, DcpAppResourceStore appResources, IConfiguration configuration, - Func publishEndpointsAllocatedEventAsync, ProfilingTelemetry profilingTelemetry, CancellationToken shutdownToken) { @@ -66,7 +64,6 @@ public DcpResourceWatcher( _executorEvents = executorEvents; _logger = logger; _configuration = configuration; - _publishEndpointsAllocatedEventAsync = publishEndpointsAllocatedEventAsync; _profilingTelemetry = profilingTelemetry; _shutdownToken = shutdownToken; @@ -497,10 +494,9 @@ private async Task ProcessServiceChange(WatchEventType watchEventType, Service s return; } - if (watchEventType is WatchEventType.Added or WatchEventType.Modified && - DcpModelUtilities.TryApplyServiceAddressToEndpoint(service, _resourceState.AppResources, out var allocatedResource)) + if (watchEventType is WatchEventType.Added or WatchEventType.Modified) { - await _publishEndpointsAllocatedEventAsync(allocatedResource, _shutdownToken).ConfigureAwait(false); + DcpModelUtilities.ApplyServiceAddressToEndpoint(service, _resourceState.AppResources); } foreach (var ((resourceKind, resourceName), _) in _resourceState.ResourceAssociatedServicesMap.Where(e => e.Value.Contains(service.Metadata.Name))) diff --git a/src/Aspire.Hosting/Dcp/ProxylessEndpointPortAllocator.cs b/src/Aspire.Hosting/Dcp/ProxylessEndpointPortAllocator.cs new file mode 100644 index 00000000000..96baa539b9d --- /dev/null +++ b/src/Aspire.Hosting/Dcp/ProxylessEndpointPortAllocator.cs @@ -0,0 +1,362 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Net; +using System.Net.Sockets; +using Aspire.Hosting.ApplicationModel; +using Microsoft.Extensions.Options; + +namespace Aspire.Hosting.Dcp; + +/// +/// Allocates and tracks public ports for proxyless endpoints that do not specify one. +/// +/// +/// Uses a stateful hybrid scan over the configured non-ephemeral port range. The allocator starts +/// with an exhaustive pseudo-random walk to find a likely-free region, then walks incrementally after +/// each successful allocation so adjacent free ports are consumed efficiently. If a candidate is in +/// use, the allocator jumps back to the random walk instead of linearly scanning through a dense used +/// cluster. +/// +/// The random walk cursor is independent of incremental scanning. Incremental successes mark adjacent +/// candidates as visited and can therefore consume ports the random permutation would have reached +/// later, but the next random jump resumes from the previous permutation position and skips already +/// visited ports. This keeps the search exhaustive while opportunistically exploiting nearby free ports. +/// +/// This approach was tested against naive incremental allocation, pure random allocation, and +/// ephemeral port allocation. It was the fastest strategy tested while avoiding the worst-case +/// failure modes of naive incremental search. +/// +internal sealed class ProxylessEndpointPortAllocator : IDisposable +{ + private readonly object _lock = new(); + private readonly int _rangeStart; + private readonly int _rangeEnd; + private readonly int _rangeSize; + private readonly bool[] _visited; + private readonly Dictionary _reservedPorts = new(ReferenceEqualityComparer.Instance); + private readonly Func _tryProbe; + private int _visitedCount; + private int _randomWalkCursor; + private readonly int _randomWalkStep; + private int? _nextCandidate; + private bool _disposed; + + public ProxylessEndpointPortAllocator(IOptions options) + : this( + options.Value.ProxylessEndpointPortRangeStart, + options.Value.ProxylessEndpointPortRangeEnd, + Random.Shared, + TryProbePort) + { + } + + internal ProxylessEndpointPortAllocator(int rangeStart, int rangeEnd, Random random, Func tryProbe) + : this(rangeStart, rangeEnd, GetRandomOffset(random, rangeStart, rangeEnd), GetRandomCoprimeStep(random, GetRangeSize(rangeStart, rangeEnd)), tryProbe) + { + } + + internal ProxylessEndpointPortAllocator(int rangeStart, int rangeEnd, int randomWalkOffset, int randomWalkStep, Func tryProbe) + { + if (rangeStart is < 1 or > 65535) + { + throw new ArgumentOutOfRangeException(nameof(rangeStart), rangeStart, "Port range start must be between 1 and 65535."); + } + + if (rangeEnd is < 1 or > 65535) + { + throw new ArgumentOutOfRangeException(nameof(rangeEnd), rangeEnd, "Port range end must be between 1 and 65535."); + } + + if (rangeStart > rangeEnd) + { + throw new ArgumentException("Port range start must be less than or equal to the range end.", nameof(rangeStart)); + } + + _rangeStart = rangeStart; + _rangeEnd = rangeEnd; + _rangeSize = rangeEnd - rangeStart + 1; + + if (randomWalkOffset < 0 || randomWalkOffset >= _rangeSize) + { + throw new ArgumentOutOfRangeException(nameof(randomWalkOffset), randomWalkOffset, "Random walk offset must be within the configured range size."); + } + + if (randomWalkStep < 1 || randomWalkStep > _rangeSize || GreatestCommonDivisor(randomWalkStep, _rangeSize) != 1) + { + throw new ArgumentOutOfRangeException(nameof(randomWalkStep), randomWalkStep, "Random walk step must be coprime with the configured range size."); + } + + // The scan range is dense and bounded, so indexable visited state is cheaper and simpler than + // hashing individual ports. The default range is only about 23 KB while still giving O(1) + // lookups for both random-walk and incremental scans. + _visited = new bool[_rangeSize]; + _randomWalkCursor = randomWalkOffset; + _randomWalkStep = randomWalkStep; + _tryProbe = tryProbe; + } + + public int AllocatePort(EndpointAnnotation endpoint) + { + lock (_lock) + { + ObjectDisposedException.ThrowIf(_disposed, this); + + if (_reservedPorts.TryGetValue(endpoint, out var reservedPort)) + { + return reservedPort; + } + + var port = AllocatePortCore(endpoint.Protocol); + _reservedPorts.Add(endpoint, port); + return port; + } + } + + public void ExcludePort(int port) + { + lock (_lock) + { + ObjectDisposedException.ThrowIf(_disposed, this); + + if (TryGetPortIndex(port, out var index)) + { + MarkVisited(index); + } + } + } + + public void Dispose() + { + lock (_lock) + { + if (_disposed) + { + return; + } + + _disposed = true; + } + } + + private int AllocatePortCore(ProtocolType protocol) + { + _nextCandidate ??= GetNextRandomWalkCandidate(); + + while (_nextCandidate is int candidate) + { + var port = _rangeStart + candidate; + MarkVisited(candidate); + + // Bind only long enough to confirm the OS currently considers the port available. + // After that, the allocator's visited/reserved state prevents Aspire from handing + // the same port to another endpoint in this app model. + if (_tryProbe(port, protocol)) + { + _nextCandidate = _visitedCount == _rangeSize ? null : GetNextIncrementalCandidate(candidate); + return port; + } + + _nextCandidate = GetNextRandomWalkCandidate(); + } + + throw CreateNoAvailablePortsException(); + } + + private int GetNextIncrementalCandidate(int afterIndex) + { + if (_visitedCount == _rangeSize) + { + throw CreateNoAvailablePortsException(); + } + + for (var i = 1; i <= _rangeSize; i++) + { + var candidate = (afterIndex + i) % _rangeSize; + if (!_visited[candidate]) + { + return candidate; + } + } + + throw CreateNoAvailablePortsException(); + } + + private int GetNextRandomWalkCandidate() + { + if (_visitedCount == _rangeSize) + { + throw CreateNoAvailablePortsException(); + } + + for (var i = 0; i < _rangeSize; i++) + { + var candidate = _randomWalkCursor; + _randomWalkCursor = (_randomWalkCursor + _randomWalkStep) % _rangeSize; + + if (!_visited[candidate]) + { + return candidate; + } + } + + throw CreateNoAvailablePortsException(); + } + + private InvalidOperationException CreateNoAvailablePortsException() + { + return new InvalidOperationException($"No available ports were found in the configured proxyless endpoint port range {_rangeStart}-{_rangeEnd}."); + } + + private void MarkVisited(int index) + { + if (_visited[index]) + { + return; + } + + _visited[index] = true; + _visitedCount++; + } + + private bool TryGetPortIndex(int port, out int index) + { + if (port < _rangeStart || port > _rangeEnd) + { + index = -1; + return false; + } + + index = port - _rangeStart; + return true; + } + + // Exposed for tests so port-availability checks use the exact same IPv4+IPv6 probe as + // production allocation. A test helper that probed a different address family could hand + // back a port the allocator then rejects, producing spurious "no available ports" failures. + internal static bool TryProbePort(int port, ProtocolType protocol) + { + return protocol == ProtocolType.Udp + ? TryProbePort(port, SocketType.Dgram, ProtocolType.Udp) + : TryProbePort(port, SocketType.Stream, ProtocolType.Tcp); + } + + private static bool TryProbePort(int port, SocketType socketType, ProtocolType protocolType) + { + var sockets = new List(); + + try + { + sockets.Add(CreateBoundSocket(AddressFamily.InterNetwork, socketType, protocolType, new IPEndPoint(IPAddress.Any, port))); + + if (Socket.OSSupportsIPv6) + { + sockets.Add(CreateBoundSocket(AddressFamily.InterNetworkV6, socketType, protocolType, new IPEndPoint(IPAddress.IPv6Any, port))); + } + + DisposeSockets(sockets); + return true; + } + catch (SocketException ex) when (ex.SocketErrorCode is SocketError.AddressAlreadyInUse or SocketError.AccessDenied) + { + DisposeSockets(sockets); + return false; + } + catch + { + DisposeSockets(sockets); + throw; + } + } + + private static Socket CreateBoundSocket(AddressFamily addressFamily, SocketType socketType, ProtocolType protocolType, IPEndPoint endPoint) + { + var socket = new Socket(addressFamily, socketType, protocolType) + { + ExclusiveAddressUse = true + }; + + var socketReturned = false; + try + { + if (addressFamily == AddressFamily.InterNetworkV6) + { + socket.DualMode = false; + } + + socket.Bind(endPoint); + socketReturned = true; + return socket; + } + finally + { + if (!socketReturned) + { + socket.Dispose(); + } + } + } + + private static int GetRandomCoprimeStep(Random random, int rangeSize) + { + if (rangeSize == 1) + { + return 1; + } + + while (true) + { + var step = random.Next(1, rangeSize); + if (GreatestCommonDivisor(step, rangeSize) == 1) + { + return step; + } + } + } + + private static int GetRandomOffset(Random random, int rangeStart, int rangeEnd) + { + return random.Next(GetRangeSize(rangeStart, rangeEnd)); + } + + private static int GetRangeSize(int rangeStart, int rangeEnd) + { + if (rangeStart is < 1 or > 65535) + { + throw new ArgumentOutOfRangeException(nameof(rangeStart), rangeStart, "Port range start must be between 1 and 65535."); + } + + if (rangeEnd is < 1 or > 65535) + { + throw new ArgumentOutOfRangeException(nameof(rangeEnd), rangeEnd, "Port range end must be between 1 and 65535."); + } + + if (rangeStart > rangeEnd) + { + throw new ArgumentException("Port range start must be less than or equal to the range end.", nameof(rangeStart)); + } + + return rangeEnd - rangeStart + 1; + } + + private static int GreatestCommonDivisor(int a, int b) + { + while (b != 0) + { + var temp = b; + b = a % b; + a = temp; + } + + return Math.Abs(a); + } + + private static void DisposeSockets(IEnumerable sockets) + { + foreach (var socket in sockets) + { + socket.Dispose(); + } + } + +} diff --git a/src/Aspire.Hosting/DistributedApplicationBuilder.cs b/src/Aspire.Hosting/DistributedApplicationBuilder.cs index 10d42e2b569..1f47dcdcfe8 100644 --- a/src/Aspire.Hosting/DistributedApplicationBuilder.cs +++ b/src/Aspire.Hosting/DistributedApplicationBuilder.cs @@ -512,6 +512,7 @@ public DistributedApplicationBuilder(DistributedApplicationOptions options) // DCP stuff _innerBuilder.Services.AddSingleton(); + _innerBuilder.Services.AddSingleton(); _innerBuilder.Services.AddSingleton(); _innerBuilder.Services.AddSingleton(); _innerBuilder.Services.AddSingleton(); diff --git a/src/Aspire.Hosting/ResourceBuilderExtensions.cs b/src/Aspire.Hosting/ResourceBuilderExtensions.cs index d583bc09143..60aad14628c 100644 --- a/src/Aspire.Hosting/ResourceBuilderExtensions.cs +++ b/src/Aspire.Hosting/ResourceBuilderExtensions.cs @@ -2821,7 +2821,6 @@ public static IResourceBuilder WithHttpHealthCheck(this IResourceBuilder { if (!endpoint.Exists) @@ -2829,6 +2828,12 @@ public static IResourceBuilder WithHttpHealthCheck(this IResourceBuilder + { var baseUri = new Uri(endpoint.Url, UriKind.Absolute); uri = new Uri(baseUri, path); return Task.CompletedTask; diff --git a/src/Shared/KnownConfigNames.cs b/src/Shared/KnownConfigNames.cs index 07866266964..d7e971e3b25 100644 --- a/src/Shared/KnownConfigNames.cs +++ b/src/Shared/KnownConfigNames.cs @@ -26,6 +26,7 @@ internal static class KnownConfigNames public const string ContainerRuntime = "ASPIRE_CONTAINER_RUNTIME"; public const string DependencyCheckTimeout = "ASPIRE_DEPENDENCY_CHECK_TIMEOUT"; + public const string ProxylessEndpointPortRange = "ASPIRE_PROXYLESS_ENDPOINT_PORT_RANGE"; public const string ServiceStartupWatchTimeout = "ASPIRE_SERVICE_STARTUP_WATCH_TIMEOUT"; public const string WaitForDebugger = "ASPIRE_WAIT_FOR_DEBUGGER"; public const string WaitForDebuggerTimeout = "ASPIRE_DEBUGGER_TIMEOUT"; diff --git a/tests/Aspire.Cli.EndToEnd.Tests/Helpers/KubernetesDeployTestHelpers.cs b/tests/Aspire.Cli.EndToEnd.Tests/Helpers/KubernetesDeployTestHelpers.cs index 71f90ede8a6..92749d5ed41 100644 --- a/tests/Aspire.Cli.EndToEnd.Tests/Helpers/KubernetesDeployTestHelpers.cs +++ b/tests/Aspire.Cli.EndToEnd.Tests/Helpers/KubernetesDeployTestHelpers.cs @@ -250,7 +250,7 @@ await auto.WaitUntilAsync( { await auto.TypeAsync($"dotnet add {projectName}.ApiService package {package} --prerelease"); await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); } // Step 5: Inject custom AppHost.cs and ApiService/Program.cs into the template-created project diff --git a/tests/Aspire.Hosting.Azure.Tests/AzureStorageEmulatorFunctionalTests.cs b/tests/Aspire.Hosting.Azure.Tests/AzureStorageEmulatorFunctionalTests.cs index 8d7a6b4d758..95cf13a762b 100644 --- a/tests/Aspire.Hosting.Azure.Tests/AzureStorageEmulatorFunctionalTests.cs +++ b/tests/Aspire.Hosting.Azure.Tests/AzureStorageEmulatorFunctionalTests.cs @@ -287,12 +287,17 @@ public async Task VerifyAzureStorageEmulator_queue_auto_created() } [Fact] [RequiresFeature(TestFeature.Docker)] - public Task AzureStorageEmulator_WithPersistentLifetime_ReusesContainer() + public Task AzureStorageEmulator_WithPersistentLifetime_ReusesContainersAndPorts() { - return PersistentContainerTestHelpers.AssertResourceReusesContainerAsync( + return PersistentContainerTestHelpers.AssertResourcesReuseContainersAsync( testOutputHelper, - builder => builder.AddAzureStorage("storage").RunAsEmulator(container => container.WithPersistentLifetime()), - "storage"); + builder => + { + builder.AddAzureStorage("storage1").RunAsEmulator(container => container.WithPersistentLifetime()); + builder.AddAzureStorage("storage2").RunAsEmulator(container => container.WithPersistentLifetime()); + }, + ["storage1", "storage2"], + compareUrls: true); } } diff --git a/tests/Aspire.Hosting.Tests/Dcp/DcpCliArgsTests.cs b/tests/Aspire.Hosting.Tests/Dcp/DcpCliArgsTests.cs index 2a59b8502ab..6da8ad87b80 100644 --- a/tests/Aspire.Hosting.Tests/Dcp/DcpCliArgsTests.cs +++ b/tests/Aspire.Hosting.Tests/Dcp/DcpCliArgsTests.cs @@ -134,6 +134,53 @@ public void DcpOptionsValidationFailsForWhitespacePaths() Assert.Contains("The path to the Aspire Dashboard binaries is missing.", result.FailureMessage); } + [Fact] + public void DcpOptionsValidationFailsForInvalidProxylessEndpointPortRange() + { + var validator = new ValidateDcpOptions(); + var result = validator.Validate(null, new DcpOptions + { + CliPath = "dcp", + DashboardPath = "dashboard", + ProxylessEndpointPortRangeStart = 32767, + ProxylessEndpointPortRangeEnd = 10000, + }); + + Assert.True(result.Failed); + Assert.Contains("The proxyless endpoint port range start must be less than or equal to the range end.", result.FailureMessage); + } + + [Fact] + public void KnownConfigProxylessEndpointPortRangeOverridesDcpPublisherConfiguration() + { + var builder = DistributedApplication.CreateBuilder(); + builder.Configuration.AddInMemoryCollection(new Dictionary + { + ["DcpPublisher:ProxylessEndpointPortRangeStart"] = "10000", + ["DcpPublisher:ProxylessEndpointPortRangeEnd"] = "10001", + [KnownConfigNames.ProxylessEndpointPortRange] = "20000-20001", + }); + + using var app = builder.Build(); + var dcpOptions = app.Services.GetRequiredService>().Value; + + Assert.Equal(20000, dcpOptions.ProxylessEndpointPortRangeStart); + Assert.Equal(20001, dcpOptions.ProxylessEndpointPortRangeEnd); + } + + [Fact] + public void KnownConfigProxylessEndpointPortRangeRequiresStartEndFormat() + { + var builder = DistributedApplication.CreateBuilder(); + builder.Configuration[KnownConfigNames.ProxylessEndpointPortRange] = "20000"; + using var app = builder.Build(); + + var exception = Assert.Throws(() => app.Services.GetRequiredService>().Value); + + Assert.Contains(KnownConfigNames.ProxylessEndpointPortRange, exception.Message); + Assert.Contains("start-end", exception.Message); + } + private static void AddDcpPublisherPathConfigurationOverride(ConfigurationManager configuration, string cliPath, string dashboardPath) { configuration.AddInMemoryCollection(new Dictionary diff --git a/tests/Aspire.Hosting.Tests/Dcp/DcpExecutorTests.cs b/tests/Aspire.Hosting.Tests/Dcp/DcpExecutorTests.cs index 640055e77c6..c3b43865d2d 100644 --- a/tests/Aspire.Hosting.Tests/Dcp/DcpExecutorTests.cs +++ b/tests/Aspire.Hosting.Tests/Dcp/DcpExecutorTests.cs @@ -4,10 +4,12 @@ #pragma warning disable ASPIREEXTENSION001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. #pragma warning disable ASPIRECERTIFICATES001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. #pragma warning disable ASPIREPERSISTENCE001 // Resource lifetime APIs are experimental. +#pragma warning disable ASPIREUSERSECRETS001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. using System.Collections.Concurrent; using System.Diagnostics; using System.Globalization; using System.IO.Pipelines; +using System.Net.Sockets; using System.Security.Cryptography; using System.Security.Cryptography.X509Certificates; using System.Text; @@ -19,6 +21,7 @@ using Aspire.Hosting.Diagnostics; using Aspire.Hosting.Publishing; using Aspire.Hosting.Tests.Utils; +using Aspire.Hosting.UserSecrets; using k8s.Models; using Microsoft.AspNetCore.InternalTesting; using Microsoft.Extensions.Configuration; @@ -27,7 +30,6 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Logging.Testing; using Microsoft.Extensions.Options; using Polly; using Polly.Retry; @@ -885,6 +887,7 @@ public async Task EndpointPortsExecutableNotReplicatedProxylessNoPortTargetPortS var svc = kubernetesService.CreatedResources.OfType().Single(s => s.Name() == "CoolProgram"); Assert.Equal(AddressAllocationModes.Proxyless, svc.Spec.AddressAllocationMode); Assert.Equal(desiredPort, svc.Status?.EffectivePort); + Assert.Equal(desiredPort, svc.Spec.Port); // Desired port should be part of the service producer annotation. Assert.Equal(desiredPort, spAnnList.Single(ann => ann.ServiceName == "CoolProgram").Port); var envVarVal = dcpExe.Spec.Env?.Single(v => v.Name == "NO_PORT_TARGET_PORT_SET").Value; @@ -923,6 +926,258 @@ public async Task EndpointPortsExecutableNotReplicatedProxylessPortAndTargetPort Assert.Equal(desiredPort, int.Parse(envVarVal, CultureInfo.InvariantCulture)); } + [Fact] + public async Task EndpointPortsExecutableNotReplicatedProxylessNoPortNoTargetPortAllocated() + { + var builder = DistributedApplication.CreateBuilder(); + + builder.AddExecutable("CoolProgram", "cool", Environment.CurrentDirectory, "--alpha", "--bravo") + .WithEndpoint(name: "NoPortNoTargetPort", env: "NO_PORT_NO_TARGET_PORT", isProxied: false); + + var kubernetesService = new TestKubernetesService(); + using var app = builder.Build(); + var distributedAppModel = app.Services.GetRequiredService(); + var appExecutor = CreateAppExecutor(distributedAppModel, kubernetesService: kubernetesService); + await appExecutor.RunApplicationAsync(); + + var dcpExe = Assert.Single(kubernetesService.CreatedResources.OfType()); + Assert.True(dcpExe.TryGetAnnotationAsObjectList(CustomResource.ServiceProducerAnnotation, out var spAnnList)); + + var svc = kubernetesService.CreatedResources.OfType().Single(s => s.Name() == "CoolProgram"); + var allocatedPort = Assert.IsType(svc.Status?.EffectivePort); + Assert.Equal(AddressAllocationModes.Proxyless, svc.Spec.AddressAllocationMode); + AssertPortAllocatedFromProxylessEndpointAllocatorRange(allocatedPort); + Assert.Equal(allocatedPort, svc.Spec.Port); + Assert.Equal(allocatedPort, spAnnList.Single(ann => ann.ServiceName == "CoolProgram").Port); + + var envVarVal = dcpExe.Spec.Env?.Single(v => v.Name == "NO_PORT_NO_TARGET_PORT").Value; + Assert.False(string.IsNullOrWhiteSpace(envVarVal)); + Assert.Equal(allocatedPort, int.Parse(envVarVal, CultureInfo.InvariantCulture)); + } + + [Fact] + public async Task ProxylessPortAllocatorExcludesFixedPublicPorts() + { + var (fixedPort, allocatedPort) = GetAvailableConsecutivePortPair(); + var builder = DistributedApplication.CreateBuilder(); + + builder.AddExecutable("FixedProgram", "fixed", Environment.CurrentDirectory) + .WithEndpoint(name: "fixed", port: fixedPort, isProxied: false); + builder.AddExecutable("DynamicProgram", "dynamic", Environment.CurrentDirectory) + .WithEndpoint(name: "dynamic", env: "DYNAMIC_PORT", isProxied: false); + + var dcpOptions = new DcpOptions + { + DashboardPath = "./dashboard", + ProxylessEndpointPortRangeStart = fixedPort, + ProxylessEndpointPortRangeEnd = allocatedPort + }; + var kubernetesService = new TestKubernetesService(); + using var app = builder.Build(); + var distributedAppModel = app.Services.GetRequiredService(); + var appExecutor = CreateAppExecutor(distributedAppModel, kubernetesService: kubernetesService, dcpOptions: dcpOptions); + await appExecutor.RunApplicationAsync(); + + var fixedService = kubernetesService.CreatedResources.OfType().Single(s => s.Name() == "FixedProgram"); + var dynamicService = kubernetesService.CreatedResources.OfType().Single(s => s.Name() == "DynamicProgram"); + var dynamicExecutable = kubernetesService.CreatedResources.OfType().Single(e => e.AppModelResourceName == "DynamicProgram"); + + Assert.Equal(fixedPort, fixedService.Status?.EffectivePort); + Assert.Equal(allocatedPort, dynamicService.Status?.EffectivePort); + Assert.Equal(allocatedPort, dynamicService.Spec.Port); + + var envVarVal = dynamicExecutable.Spec.Env?.Single(v => v.Name == "DYNAMIC_PORT").Value; + Assert.False(string.IsNullOrWhiteSpace(envVarVal)); + Assert.Equal(allocatedPort, int.Parse(envVarVal, CultureInfo.InvariantCulture)); + } + + [Fact] + public async Task PersistentProxylessExecutableWithoutPortPersistsAllocatedPort() + { + var (allocatedPort, _) = GetAvailableConsecutivePortPair(); + var builder = DistributedApplication.CreateBuilder(); + + builder.AddExecutable("CoolProgram", "cool", Environment.CurrentDirectory, "--alpha", "--bravo") + .WithPersistentLifetime() + .WithEndpoint(name: "http", env: "HTTP_PORT", isProxied: false); + + var configDict = new Dictionary + { + ["AppHost:Sha256"] = "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef" + }; + var configuration = new ConfigurationBuilder().AddInMemoryCollection(configDict).Build(); + var userSecretsManager = new MockUserSecretsManager(); + var dcpOptions = new DcpOptions + { + DashboardPath = "./dashboard", + ProxylessEndpointPortRangeStart = allocatedPort, + ProxylessEndpointPortRangeEnd = allocatedPort + }; + + var kubernetesService = new TestKubernetesService(); + using var app = builder.Build(); + var distributedAppModel = app.Services.GetRequiredService(); + var appExecutor = CreateAppExecutor(distributedAppModel, kubernetesService: kubernetesService, configuration: configuration, dcpOptions: dcpOptions, userSecretsManager: userSecretsManager); + await appExecutor.RunApplicationAsync(); + + var svc = kubernetesService.CreatedResources.OfType().Single(s => s.Name() == "CoolProgram"); + Assert.Equal(allocatedPort, svc.Status?.EffectivePort); + Assert.Equal(allocatedPort.ToString(CultureInfo.InvariantCulture), userSecretsManager.Secrets["Aspire:ProxylessEndpointPorts:CoolProgram:http"]); + } + + [Fact] + public async Task PersistentProxylessContainerWithoutPortPersistsAllocatedPort() + { + var (allocatedPort, _) = GetAvailableConsecutivePortPair(); + var builder = DistributedApplication.CreateBuilder(); + + const int targetPort = TestKubernetesService.StartOfAutoPortRange - 999; + builder.AddContainer("database", "image") + .WithPersistentLifetime() + .WithEndpoint(name: "http", targetPort: targetPort, isProxied: false); + + var configDict = new Dictionary + { + ["AppHost:Sha256"] = "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef" + }; + var configuration = new ConfigurationBuilder().AddInMemoryCollection(configDict).Build(); + var userSecretsManager = new MockUserSecretsManager(); + var dcpOptions = new DcpOptions + { + DashboardPath = "./dashboard", + ProxylessEndpointPortRangeStart = allocatedPort, + ProxylessEndpointPortRangeEnd = allocatedPort + }; + + var kubernetesService = new TestKubernetesService(); + using var app = builder.Build(); + var distributedAppModel = app.Services.GetRequiredService(); + var appExecutor = CreateAppExecutor(distributedAppModel, kubernetesService: kubernetesService, configuration: configuration, dcpOptions: dcpOptions, userSecretsManager: userSecretsManager); + await appExecutor.RunApplicationAsync(); + + var svc = kubernetesService.CreatedResources.OfType().Single(s => s.Name() == "database"); + Assert.Equal(allocatedPort, svc.Status?.EffectivePort); + Assert.Equal(allocatedPort, svc.Spec.Port); + Assert.Equal(allocatedPort.ToString(CultureInfo.InvariantCulture), userSecretsManager.Secrets["Aspire:ProxylessEndpointPorts:database:http"]); + } + + [Fact] + public async Task ProxylessExecutableAllocatedPortIsStableOnResourceRestart() + { + var (allocatedPort, _) = GetAvailableConsecutivePortPair(); + var builder = DistributedApplication.CreateBuilder(); + + builder.AddExecutable("CoolProgram", "cool", Environment.CurrentDirectory, "--alpha", "--bravo") + .WithEndpoint(name: "http", env: "HTTP_PORT", isProxied: false); + + var dcpOptions = new DcpOptions + { + DashboardPath = "./dashboard", + ProxylessEndpointPortRangeStart = allocatedPort, + ProxylessEndpointPortRangeEnd = allocatedPort + }; + + var kubernetesService = new TestKubernetesService(); + using var app = builder.Build(); + var distributedAppModel = app.Services.GetRequiredService(); + var appExecutor = CreateAppExecutor(distributedAppModel, kubernetesService: kubernetesService, dcpOptions: dcpOptions); + await appExecutor.RunApplicationAsync(); + + var service = kubernetesService.CreatedResources.OfType().Single(s => s.Name() == "CoolProgram"); + var firstExecutable = Assert.Single(GetCreatedExecutablesForResource(kubernetesService, "CoolProgram")); + Assert.Equal(allocatedPort, service.Status?.EffectivePort); + Assert.Equal(allocatedPort.ToString(CultureInfo.InvariantCulture), firstExecutable.Spec.Env?.Single(v => v.Name == "HTTP_PORT").Value); + + var reference = appExecutor.GetResource(firstExecutable.Metadata.Name); + await appExecutor.StopResourceAsync(reference, CancellationToken.None); + await appExecutor.StartResourceAsync(reference, CancellationToken.None); + + var executables = GetCreatedExecutablesForResource(kubernetesService, "CoolProgram"); + Assert.Equal(2, executables.Count); + Assert.Equal(allocatedPort, service.Status?.EffectivePort); + Assert.Equal(allocatedPort.ToString(CultureInfo.InvariantCulture), executables[1].Spec.Env?.Single(v => v.Name == "HTTP_PORT").Value); + } + + [Fact] + public async Task ProxylessContainerAllocatedHostPortIsStableOnResourceRestart() + { + var (allocatedPort, _) = GetAvailableConsecutivePortPair(); + var builder = DistributedApplication.CreateBuilder(); + + const int targetPort = TestKubernetesService.StartOfAutoPortRange - 999; + builder.AddContainer("database", "image") + .WithEndpoint(name: "http", targetPort: targetPort, isProxied: false); + + var dcpOptions = new DcpOptions + { + DashboardPath = "./dashboard", + ProxylessEndpointPortRangeStart = allocatedPort, + ProxylessEndpointPortRangeEnd = allocatedPort + }; + + var kubernetesService = new TestKubernetesService(); + using var app = builder.Build(); + var distributedAppModel = app.Services.GetRequiredService(); + var appExecutor = CreateAppExecutor(distributedAppModel, kubernetesService: kubernetesService, dcpOptions: dcpOptions); + await appExecutor.RunApplicationAsync(); + + var service = kubernetesService.CreatedResources.OfType().Single(s => s.Name() == "database"); + var firstContainer = Assert.Single(GetCreatedContainersForResource(kubernetesService, "database")); + Assert.Equal(allocatedPort, service.Status?.EffectivePort); + Assert.Contains(firstContainer.Spec.Ports!, p => p.HostPort == allocatedPort && p.ContainerPort == targetPort); + + var reference = appExecutor.GetResource(firstContainer.Metadata.Name); + await appExecutor.StopResourceAsync(reference, CancellationToken.None); + await appExecutor.StartResourceAsync(reference, CancellationToken.None); + + var containers = GetCreatedContainersForResource(kubernetesService, "database"); + Assert.Equal(2, containers.Count); + Assert.Equal(allocatedPort, service.Status?.EffectivePort); + Assert.Contains(containers[1].Spec.Ports!, p => p.HostPort == allocatedPort && p.ContainerPort == targetPort); + } + + [Fact] + public async Task PersistedProxylessEndpointPortIsReusedAndExcludedFromDynamicAllocation() + { + var (persistedPort, allocatedPort) = GetAvailableConsecutivePortPair(); + var builder = DistributedApplication.CreateBuilder(); + + builder.AddExecutable("PersistentProgram", "persistent", Environment.CurrentDirectory) + .WithPersistentLifetime() + .WithEndpoint(name: "http", env: "PERSISTENT_PORT", isProxied: false); + builder.AddExecutable("DynamicProgram", "dynamic", Environment.CurrentDirectory) + .WithEndpoint(name: "http", env: "DYNAMIC_PORT", isProxied: false); + + var configDict = new Dictionary + { + ["AppHost:Sha256"] = "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + ["Aspire:ProxylessEndpointPorts:PersistentProgram:http"] = persistedPort.ToString(CultureInfo.InvariantCulture) + }; + var configuration = new ConfigurationBuilder().AddInMemoryCollection(configDict).Build(); + var userSecretsManager = new MockUserSecretsManager(); + var dcpOptions = new DcpOptions + { + DashboardPath = "./dashboard", + ProxylessEndpointPortRangeStart = persistedPort, + ProxylessEndpointPortRangeEnd = allocatedPort + }; + + var kubernetesService = new TestKubernetesService(); + using var app = builder.Build(); + var distributedAppModel = app.Services.GetRequiredService(); + var appExecutor = CreateAppExecutor(distributedAppModel, kubernetesService: kubernetesService, configuration: configuration, dcpOptions: dcpOptions, userSecretsManager: userSecretsManager); + await appExecutor.RunApplicationAsync(); + + var persistentService = kubernetesService.CreatedResources.OfType().Single(s => s.Name() == "PersistentProgram"); + var dynamicService = kubernetesService.CreatedResources.OfType().Single(s => s.Name() == "DynamicProgram"); + + Assert.Equal(persistedPort, persistentService.Status?.EffectivePort); + Assert.Equal(persistedPort, persistentService.Spec.Port); + Assert.Equal(allocatedPort, dynamicService.Status?.EffectivePort); + Assert.Equal(allocatedPort, dynamicService.Spec.Port); + Assert.Empty(userSecretsManager.Secrets); + } + /// /// Verifies that applying unsupported endpoint port configuration to non-replicated, proxy-less Executables /// results in an error @@ -934,23 +1189,6 @@ public async Task UnsupportedEndpointPortsExecutableNotReplicatedProxyless() const int desiredPortTwo = TestKubernetesService.StartOfAutoPortRange - 999; (Action> AddEndpoint, string ErrorMessageFragment)[] testcases = [ - // Note: this configuration (neither Endpoint.Port, nor Endpoint.TargetPort set) COULD be supported as follows: - // Clients connect directly to the program, MAY have the program port injected. - // Program gets autogenerated port that MUST be injected via env var/startup param. - // - // BUT - // - // as of Aspire GA (May 2024) this is not supported due to how Aspire app model consumes autogenerated ports. - // Namely, the Aspire ApplicationExecutor creates Services and waits for Services to have ports allocated (by DCP) - // before creating Executables and Containers that implement these services. - // This does not work for proxy-less Services backed by Executables with auto-generated ports, because these Services - // get their ports from Executables that are backing them, and those Executables, in turn, get their ports when they get started. - // Delaying Executable creation like Aspire ApplicationExecutor does means the Services will never get their ports. - ( - er => er.WithEndpoint(name: "NoPortNoTargetPort", env: "NO_PORT_NO_TARGET_PORT", isProxied: false), - "needs to specify a port for endpoint" - ), - // Invalid configuration: both Port and TargetPort set, but to different values. ( er => er.WithEndpoint(name: "PortAndTargetPortSetDifferently", port: desiredPortOne, targetPort: desiredPortTwo, env: "PORT_AND_TARGET_PORT_SET_DIFFERENTLY", isProxied: false), @@ -1771,15 +2009,13 @@ public async Task EndpointPortsContainerProxylessNoPortTargetPortSet() var dcpCtr = Assert.Single(kubernetesService.CreatedResources.OfType()); Assert.True(dcpCtr.TryGetAnnotationAsObjectList(CustomResource.ServiceProducerAnnotation, out var spAnnList)); - // Port is empty, TargetPort is set. - // Clients connect directly to the container host port, MAY have the container host port injected. - // DCP allocates the container host port after the container is created. var svc = kubernetesService.CreatedResources.OfType().Single(s => s.Name() == "database"); + var allocatedPort = Assert.IsType(svc.Status?.EffectivePort); Assert.Equal(AddressAllocationModes.Proxyless, svc.Spec.AddressAllocationMode); - Assert.Null(svc.Spec.Port); - Assert.True(svc.Status?.EffectivePort >= TestKubernetesService.StartOfAutoPortRange); + Assert.Equal(allocatedPort, svc.Spec.Port); + AssertPortAllocatedFromProxylessEndpointAllocatorRange(allocatedPort); Assert.NotNull(dcpCtr.Spec.Ports); - Assert.Contains(dcpCtr.Spec.Ports!, p => p.HostPort is null && p.ContainerPort == desiredTargetPort); + Assert.Contains(dcpCtr.Spec.Ports!, p => p.HostPort == allocatedPort && p.ContainerPort == desiredTargetPort); // Desired port should be part of the service producer annotation. Assert.Equal(desiredTargetPort, spAnnList.Single(ann => ann.ServiceName == "database").Port); var envVarVal = dcpCtr.Spec.Env?.Single(v => v.Name == "NO_PORT_TARGET_PORT_SET").Value; @@ -1788,7 +2024,7 @@ public async Task EndpointPortsContainerProxylessNoPortTargetPortSet() } [Fact] - public async Task EndpointPortsContainerProxylessNoPortTargetPortSetPublishesAllocatedEndpointAfterServiceUpdate() + public async Task EndpointPortsContainerProxylessNoPortTargetPortSetPublishesAllocatedEndpoint() { var builder = DistributedApplication.CreateBuilder(); @@ -1798,11 +2034,13 @@ public async Task EndpointPortsContainerProxylessNoPortTargetPortSetPublishesAll var allocatedPortChannel = Channel.CreateUnbounded(); var connectionStringAvailableChannel = Channel.CreateUnbounded(); + var observedEvents = new ConcurrentQueue(); var eventing = new Hosting.Eventing.DistributedApplicationEventing(); eventing.Subscribe((@event, ct) => { if (@event.Resource.Name == "database") { + observedEvents.Enqueue(nameof(ResourceEndpointsAllocatedEvent)); var endpoint = ((IResourceWithEndpoints)@event.Resource).GetEndpoint("NoPortTargetPortSet"); if (endpoint.AllocatedEndpoint is { } allocatedEndpoint) { @@ -1817,11 +2055,21 @@ public async Task EndpointPortsContainerProxylessNoPortTargetPortSetPublishesAll { if (context.Resource.Name == "database") { + observedEvents.Enqueue(nameof(OnConnectionStringAvailableContext)); connectionStringAvailableChannel.Writer.TryWrite(context.Resource); } return Task.CompletedTask; }); + events.Subscribe(context => + { + if (context.Resource.Name == "database") + { + observedEvents.Enqueue(nameof(OnResourceStartingContext)); + } + + return Task.CompletedTask; + }); var kubernetesService = new TestKubernetesService(); using var app = builder.Build(); @@ -1836,15 +2084,21 @@ public async Task EndpointPortsContainerProxylessNoPortTargetPortSetPublishesAll Assert.Same(database.Resource, connectionStringAvailableResource); Assert.NotNull(dcpCtr.Spec.Ports); - Assert.Contains(dcpCtr.Spec.Ports!, p => p.HostPort is null && p.ContainerPort == desiredTargetPort); + Assert.Contains(dcpCtr.Spec.Ports!, p => p.HostPort == allocatedPort && p.ContainerPort == desiredTargetPort); Assert.Equal(allocatedPort, svc.Status?.EffectivePort); + Assert.Equal(allocatedPort, svc.Spec.Port); Assert.NotEqual(desiredTargetPort, allocatedPort); - Assert.True(allocatedPort >= TestKubernetesService.StartOfAutoPortRange); + AssertPortAllocatedFromProxylessEndpointAllocatorRange(allocatedPort); Assert.Equal(allocatedPort.ToString(CultureInfo.InvariantCulture), await database.GetEndpoint("NoPortTargetPortSet").Property(EndpointProperty.Port).GetValueAsync()); + Assert.Collection( + observedEvents, + eventName => Assert.Equal(nameof(ResourceEndpointsAllocatedEvent), eventName), + eventName => Assert.Equal(nameof(OnConnectionStringAvailableContext), eventName), + eventName => Assert.Equal(nameof(OnResourceStartingContext), eventName)); } [Fact] - public async Task EndpointPortsContainerProxylessNoPortTargetPortSetUsesTargetPortFallbackWhenResolvedBeforeContainerCreation() + public async Task EndpointPortsContainerProxylessNoPortTargetPortSetAllocatesHostPortAndInjectsTargetPortForContainerSelfReference() { var builder = DistributedApplication.CreateBuilder(); @@ -1867,11 +2121,9 @@ public async Task EndpointPortsContainerProxylessNoPortTargetPortSetUsesTargetPo }); var kubernetesService = new TestKubernetesService(); - var testSink = new TestSink(); - var containerCreatorLogger = new TestLogger(new TestLoggerFactory(testSink, enabled: true)); using var app = builder.Build(); var distributedAppModel = app.Services.GetRequiredService(); - var appExecutor = CreateAppExecutor(distributedAppModel, kubernetesService: kubernetesService, events: events, containerCreatorLogger: containerCreatorLogger); + var appExecutor = CreateAppExecutor(distributedAppModel, kubernetesService: kubernetesService, events: events); await appExecutor.RunApplicationAsync(); var connectionStringAvailableResource = await connectionStringAvailableChannel.Reader.ReadAsync().AsTask().DefaultTimeout(); @@ -1880,23 +2132,21 @@ public async Task EndpointPortsContainerProxylessNoPortTargetPortSetUsesTargetPo Assert.Same(database.Resource, connectionStringAvailableResource); Assert.Equal(AddressAllocationModes.Proxyless, svc.Spec.AddressAllocationMode); - Assert.Equal(desiredTargetPort, svc.Status?.EffectivePort); + var allocatedPort = Assert.IsType(svc.Status?.EffectivePort); + Assert.Equal(allocatedPort, svc.Spec.Port); + AssertPortAllocatedFromProxylessEndpointAllocatorRange(allocatedPort); Assert.NotNull(dcpCtr.Spec.Ports); - Assert.Contains(dcpCtr.Spec.Ports!, p => p.HostPort == desiredTargetPort && p.ContainerPort == desiredTargetPort); + Assert.Contains(dcpCtr.Spec.Ports!, p => p.HostPort == allocatedPort && p.ContainerPort == desiredTargetPort); var envVarVal = dcpCtr.Spec.Env?.Single(v => v.Name == "PUBLIC_PORT").Value; Assert.False(string.IsNullOrWhiteSpace(envVarVal)); Assert.Equal(desiredTargetPort, int.Parse(envVarVal, CultureInfo.InvariantCulture)); var secondEnvVarVal = dcpCtr.Spec.Env?.Single(v => v.Name == "PUBLIC_PORT_AGAIN").Value; Assert.False(string.IsNullOrWhiteSpace(secondEnvVarVal)); Assert.Equal(desiredTargetPort, int.Parse(secondEnvVarVal, CultureInfo.InvariantCulture)); - - Assert.Contains(testSink.Writes, log => - log.LogLevel == LogLevel.Information && - log.Message == $"Endpoint 'NoPortTargetPortSet' on container resource 'database' was resolved before the container was created, so Aspire is assigning public port {desiredTargetPort} to match target port {desiredTargetPort} for proxyless access."); } [Fact] - public async Task EndpointPortsContainerProxylessNoPortTargetPortSetUsesTargetPortFallbackWhenHostAndPortResolvedBeforeContainerCreation() + public async Task EndpointPortsContainerProxylessNoPortTargetPortSetAllocatesHostPortAndInjectsTargetHostAndPortForContainerSelfReference() { var builder = DistributedApplication.CreateBuilder(); @@ -1915,9 +2165,11 @@ public async Task EndpointPortsContainerProxylessNoPortTargetPortSetUsesTargetPo var svc = kubernetesService.CreatedResources.OfType().Single(s => s.Name() == "database"); Assert.Equal(AddressAllocationModes.Proxyless, svc.Spec.AddressAllocationMode); - Assert.Equal(desiredTargetPort, svc.Status?.EffectivePort); + var allocatedPort = Assert.IsType(svc.Status?.EffectivePort); + Assert.Equal(allocatedPort, svc.Spec.Port); + AssertPortAllocatedFromProxylessEndpointAllocatorRange(allocatedPort); Assert.NotNull(dcpCtr.Spec.Ports); - Assert.Contains(dcpCtr.Spec.Ports!, p => p.HostPort == desiredTargetPort && p.ContainerPort == desiredTargetPort); + Assert.Contains(dcpCtr.Spec.Ports!, p => p.HostPort == allocatedPort && p.ContainerPort == desiredTargetPort); var envVarVal = dcpCtr.Spec.Env?.Single(v => v.Name == "PUBLIC_HOST_AND_PORT").Value; Assert.Equal($"database.dev.internal:{desiredTargetPort}", envVarVal); } @@ -1960,10 +2212,13 @@ public async Task EndpointPortsContainerProxylessNoPortTargetPortSetCanBeResolve var dcpCtr = kubernetesService.CreatedResources.OfType().Single(c => c.AppModelResourceName == "database"); var svc = kubernetesService.CreatedResources.OfType().Single(s => s.Name() == "database"); + var allocatedPort = Assert.IsType(svc.Status?.EffectivePort); Assert.Equal($"http://database.dev.internal:{desiredTargetPort}", resolvedUrl); Assert.Equal(AddressAllocationModes.Proxyless, svc.Spec.AddressAllocationMode); + Assert.Equal(allocatedPort, svc.Spec.Port); + AssertPortAllocatedFromProxylessEndpointAllocatorRange(allocatedPort); Assert.NotNull(dcpCtr.Spec.Ports); - Assert.Contains(dcpCtr.Spec.Ports!, p => p.HostPort == desiredTargetPort && p.ContainerPort == desiredTargetPort); + Assert.Contains(dcpCtr.Spec.Ports!, p => p.HostPort == allocatedPort && p.ContainerPort == desiredTargetPort); } [Fact] @@ -1999,10 +2254,13 @@ public async Task EndpointPortsContainerProxylessNoPortTargetPortSetCanBeResolve var dcpCtr = kubernetesService.CreatedResources.OfType().Single(c => c.AppModelResourceName == "database"); var svc = kubernetesService.CreatedResources.OfType().Single(s => s.Name() == "database"); - Assert.Equal($"http://localhost:{desiredTargetPort}", resolvedUrl); + var allocatedPort = Assert.IsType(svc.Status?.EffectivePort); + Assert.Equal($"http://localhost:{allocatedPort}", resolvedUrl); Assert.Equal(AddressAllocationModes.Proxyless, svc.Spec.AddressAllocationMode); + Assert.Equal(allocatedPort, svc.Spec.Port); + AssertPortAllocatedFromProxylessEndpointAllocatorRange(allocatedPort); Assert.NotNull(dcpCtr.Spec.Ports); - Assert.Contains(dcpCtr.Spec.Ports!, p => p.HostPort == desiredTargetPort && p.ContainerPort == desiredTargetPort); + Assert.Contains(dcpCtr.Spec.Ports!, p => p.HostPort == allocatedPort && p.ContainerPort == desiredTargetPort); } [Fact] @@ -4810,12 +5068,20 @@ private static List GetCreatedExecutablesForResource(TestKubernetesS .Where(e => e.AppModelResourceName == appModelResourceName)]; } + private static List GetCreatedContainersForResource(TestKubernetesService kubernetesService, string appModelResourceName) + { + return [.. kubernetesService.CreatedResources + .OfType() + .Where(c => c.AppModelResourceName == appModelResourceName)]; + } + private static DcpExecutor CreateAppExecutor( DistributedApplicationModel distributedAppModel, IHostEnvironment? hostEnvironment = null, IConfiguration? configuration = null, IKubernetesService? kubernetesService = null, DcpOptions? dcpOptions = null, + IUserSecretsManager? userSecretsManager = null, ResourceLoggerService? resourceLoggerService = null, DcpExecutorEvents? events = null, Hosting.Eventing.IDistributedApplicationEventing? distributedApplicationEventing = null, @@ -4863,6 +5129,7 @@ private static DcpExecutor CreateAppExecutor( var dcpDependencyCheckService = new TestDcpDependencyCheckService(); var appResources = new DcpAppResourceStore(); + var proxylessEndpointPortAllocator = new ProxylessEndpointPortAllocator(Options.Create(dcpOptions)); var executableCreator = new ExecutableCreator( configuration, @@ -4903,7 +5170,42 @@ private static DcpExecutor CreateAppExecutor( appResources, executableCreator, containerCreator, - new ProfilingTelemetry(configuration)); + new ProfilingTelemetry(configuration), + proxylessEndpointPortAllocator, + userSecretsManager ?? NoopUserSecretsManager.Instance); + } + + private static void AssertPortAllocatedFromProxylessEndpointAllocatorRange(int port) + { + var defaultOptions = new DcpOptions(); + Assert.InRange(port, defaultOptions.ProxylessEndpointPortRangeStart, defaultOptions.ProxylessEndpointPortRangeEnd); + } + + private static (int First, int Second) GetAvailableConsecutivePortPair() + { + // Tests configure single-port allocation ranges, so the helper must agree exactly with + // the allocator on what "available" means. Reuse the allocator's IPv4+IPv6 probe instead + // of a separate IPv4-only bind that could return a port the allocator later rejects. + // + // Scan from a random offset rather than always starting at the bottom of the range. Test + // classes run in parallel and a deterministic start makes concurrent runs converge on the + // same low ports, where a transient probe collision throws against a zero-slack range. + const int rangeStart = 10000; + const int rangeEndExclusive = 32767; // Leave room for port + 1 within the proxyless default range. + var span = rangeEndExclusive - rangeStart; + var offset = Random.Shared.Next(span); + + for (var i = 0; i < span; i++) + { + var port = rangeStart + ((offset + i) % span); + if (ProxylessEndpointPortAllocator.TryProbePort(port, ProtocolType.Tcp) && + ProxylessEndpointPortAllocator.TryProbePort(port + 1, ProtocolType.Tcp)) + { + return (port, port + 1); + } + } + + throw new InvalidOperationException("Could not find two consecutive available ports."); } private static bool RetryTillTrueOrTimeout(Func check, int timeoutMilliseconds) diff --git a/tests/Aspire.Hosting.Tests/Dcp/ProxylessEndpointPortAllocatorTests.cs b/tests/Aspire.Hosting.Tests/Dcp/ProxylessEndpointPortAllocatorTests.cs new file mode 100644 index 00000000000..c922f2d41df --- /dev/null +++ b/tests/Aspire.Hosting.Tests/Dcp/ProxylessEndpointPortAllocatorTests.cs @@ -0,0 +1,87 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Net.Sockets; +using Aspire.Hosting.Dcp; + +namespace Aspire.Hosting.Tests.Dcp; + +public class ProxylessEndpointPortAllocatorTests +{ + [Fact] + public void AllocatePortUsesIncrementalCandidatesAfterSuccess() + { + var allocator = new ProxylessEndpointPortAllocator( + rangeStart: 10000, + rangeEnd: 10004, + randomWalkOffset: 2, + randomWalkStep: 3, + tryProbe: static (_, _) => true); + + Assert.Equal(10002, allocator.AllocatePort(CreateEndpoint("a"))); + Assert.Equal(10003, allocator.AllocatePort(CreateEndpoint("b"))); + Assert.Equal(10004, allocator.AllocatePort(CreateEndpoint("c"))); + } + + [Fact] + public void AllocatePortJumpsToRandomWalkCandidateAfterFailure() + { + var allocator = new ProxylessEndpointPortAllocator( + rangeStart: 10000, + rangeEnd: 10004, + randomWalkOffset: 0, + randomWalkStep: 2, + tryProbe: static (port, _) => port != 10000); + + Assert.Equal(10002, allocator.AllocatePort(CreateEndpoint("a"))); + Assert.Equal(10003, allocator.AllocatePort(CreateEndpoint("b"))); + } + + [Fact] + public void AllocatePortSkipsExcludedPortsAndExhaustsRangeWithoutRepeats() + { + var allocator = new ProxylessEndpointPortAllocator( + rangeStart: 10000, + rangeEnd: 10004, + randomWalkOffset: 1, + randomWalkStep: 2, + tryProbe: static (_, _) => true); + + allocator.ExcludePort(10001); + + var allocatedPorts = new[] + { + allocator.AllocatePort(CreateEndpoint("a")), + allocator.AllocatePort(CreateEndpoint("b")), + allocator.AllocatePort(CreateEndpoint("c")), + allocator.AllocatePort(CreateEndpoint("d")) + }; + + Assert.Equal(new[] { 10003, 10004, 10000, 10002 }, allocatedPorts); + Assert.Throws(() => + { + allocator.AllocatePort(CreateEndpoint("e")); + }); + } + + [Fact] + public void AllocatePortReturnsSameReservedPortForSameEndpoint() + { + var allocator = new ProxylessEndpointPortAllocator( + rangeStart: 10000, + rangeEnd: 10001, + randomWalkOffset: 0, + randomWalkStep: 1, + tryProbe: static (_, _) => true); + var endpoint = CreateEndpoint("endpoint"); + + Assert.Equal(10000, allocator.AllocatePort(endpoint)); + Assert.Equal(10000, allocator.AllocatePort(endpoint)); + Assert.Equal(10001, allocator.AllocatePort(CreateEndpoint("other"))); + } + + private static EndpointAnnotation CreateEndpoint(string name) + { + return new EndpointAnnotation(ProtocolType.Tcp, name: name, isProxied: false); + } +} diff --git a/tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs b/tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs index bb08e172538..4cdf75dd632 100644 --- a/tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs +++ b/tests/Aspire.Hosting.Tests/DistributedApplicationTests.cs @@ -1648,7 +1648,7 @@ public async Task ReplicasAndProxylessEndpointThrows() } [Fact] - public async Task ProxylessEndpointWithoutPortThrows() + public async Task ProxylessEndpointWithoutPortIsAllocated() { const string testName = "proxyess-endpoint-without-port"; using var testProgram = CreateTestProgram(testName); @@ -1660,9 +1660,12 @@ public async Task ProxylessEndpointWithoutPortThrows() await using var app = testProgram.Build(); - var ex = await Assert.ThrowsAsync(async () => await app.StartAsync().DefaultTimeout(TestConstants.DefaultOrchestratorTestLongTimeout)); - var suffix = app.Services.GetRequiredService>().Value.ResourceNameSuffix; - Assert.Equal($"Service '{testName}-servicea-{suffix}' needs to specify a port for endpoint 'http' since it isn't using a proxy.", ex.Message); + await app.StartAsync().DefaultTimeout(TestConstants.DefaultOrchestratorTestLongTimeout); + + var endpoint = testProgram.ServiceABuilder.Resource.GetEndpoint("http"); + var allocatedEndpointSnapshot = Assert.Single(endpoint.EndpointAnnotation.AllAllocatedEndpoints); + var allocatedEndpoint = await allocatedEndpointSnapshot.Snapshot.GetValueAsync().DefaultTimeout(); + Assert.InRange(allocatedEndpoint.Port, 10000, 32767); } [Fact] @@ -1786,7 +1789,7 @@ public async Task ProxylessContainerCanBeReferenced() endpoint.IsProxied = false; }); - // Since port is not specified, the container runtime will assign the host port after the container is created. + // Since port is not specified, Aspire will assign the host port before the container is created. var redisNoPort = builder.AddRedis($"{testName}-redisNoPort").WithEndpoint("tcp", endpoint => { endpoint.IsProxied = false; @@ -1829,7 +1832,7 @@ public async Task ProxylessContainerCanBeReferenced() } var otherRedisService = GetEndpointService(serviceList, redisNoPort.Resource, redisNoPort.Resource.PrimaryEndpoint); - var otherRedisPort = AssertRuntimeAssignedProxylessPort(otherRedisService); + var otherRedisPort = AssertAllocatedProxylessPort(otherRedisService); var otherRedisEnv = Assert.Single(service.Spec.Env!, e => e.Name == $"ConnectionStrings__{testName}-redisNoPort"); sslVal = redisNoPort.Resource.TlsEnabled ? ",ssl=true" : string.Empty; #pragma warning disable CS0618 // Type or member is obsolete @@ -1839,13 +1842,13 @@ public async Task ProxylessContainerCanBeReferenced() if (redisNoPort.Resource.TlsEnabled) { Assert.Equal(2, otherRedisContainer.Spec.Ports!.Count); - Assert.Contains(otherRedisContainer.Spec.Ports!, p => p.HostPort is null && p.ContainerPort == 6379); + Assert.Contains(otherRedisContainer.Spec.Ports!, p => p.HostPort == otherRedisPort && p.ContainerPort == 6379); } else { var portSpec = Assert.Single(otherRedisContainer.Spec.Ports!); Assert.Equal(6379, portSpec.ContainerPort); - Assert.Null(portSpec.HostPort); + Assert.Equal(otherRedisPort, portSpec.HostPort); } await app.StopAsync().DefaultTimeout(TestConstants.DefaultOrchestratorTestLongTimeout); @@ -1862,7 +1865,7 @@ public async Task WithEndpointProxySupportDisablesProxies() var port = await Network.GetAvailablePortAsync(); var redis = builder.AddRedis($"{testName}-redis", port).WithEndpointProxySupport(false); - // Since port is not specified, the container runtime will assign the host port after the container is created. + // Since port is not specified, Aspire will assign the host port before the container is created. var redisNoPort = builder.AddRedis($"{testName}-redisNoPort").WithEndpointProxySupport(false); var servicea = builder.AddProject($"{testName}-servicea") @@ -1906,7 +1909,7 @@ public async Task WithEndpointProxySupportDisablesProxies() } var otherRedisService = GetEndpointService(serviceList, redisNoPort.Resource, redisNoPort.Resource.PrimaryEndpoint); - var otherRedisPort = AssertRuntimeAssignedProxylessPort(otherRedisService); + var otherRedisPort = AssertAllocatedProxylessPort(otherRedisService); var otherRedisEnv = Assert.Single(service.Spec.Env!, e => e.Name == $"ConnectionStrings__{testName}-redisNoPort"); sslVal = redisNoPort.Resource.TlsEnabled ? ",ssl=true" : string.Empty; #pragma warning disable CS0618 // Type or member is obsolete @@ -1917,13 +1920,13 @@ public async Task WithEndpointProxySupportDisablesProxies() if (redisNoPort.Resource.TlsEnabled) { Assert.Equal(2, otherRedisContainer.Spec.Ports!.Count); - Assert.Contains(otherRedisContainer.Spec.Ports!, p => p.HostPort is null && p.ContainerPort == 6379); + Assert.Contains(otherRedisContainer.Spec.Ports!, p => p.HostPort == otherRedisPort && p.ContainerPort == 6379); } else { var portSpec = Assert.Single(otherRedisContainer.Spec.Ports!); Assert.Equal(6379, portSpec.ContainerPort); - Assert.Null(portSpec.HostPort); + Assert.Equal(otherRedisPort, portSpec.HostPort); } await app.StopAsync().DefaultTimeout(TestConstants.DefaultOrchestratorTestLongTimeout); @@ -2308,14 +2311,15 @@ private static Service GetEndpointService(IEnumerable services, RedisRe return Assert.Single(services, s => string.Equals(s.Metadata.Name, expectedServiceName, StringComparison.Ordinal)); } - private static int AssertRuntimeAssignedProxylessPort(Service service) + private static int AssertAllocatedProxylessPort(Service service) { Assert.Equal(AddressAllocationModes.Proxyless, service.Spec.AddressAllocationMode); - Assert.Null(service.Spec.Port); - var effectivePort = service.Status?.EffectivePort.GetValueOrDefault() ?? 0; - Assert.InRange(effectivePort, 1, 65535); - return effectivePort; + var port = Assert.IsType(service.Spec.Port); + var effectivePort = Assert.IsType(service.Status?.EffectivePort); + Assert.Equal(port, effectivePort); + Assert.InRange(port, 10000, 32767); + return port; } private static object? GetResourcePropertyValue(ResourceEvent resourceEvent, string propertyName) diff --git a/tests/Aspire.Hosting.Tests/EndpointReferenceTests.cs b/tests/Aspire.Hosting.Tests/EndpointReferenceTests.cs index 3f8ad369f9c..20ebd634d62 100644 --- a/tests/Aspire.Hosting.Tests/EndpointReferenceTests.cs +++ b/tests/Aspire.Hosting.Tests/EndpointReferenceTests.cs @@ -408,94 +408,6 @@ await Task.WhenAll ).WaitAsync(TimeSpan.FromSeconds(10)); } - [Fact] - public async Task GetValueAsync_AllocatesEndpointOnDemandWhenCallerIsUnknown() - { - var (_, _, expression, allocationCount) = CreateOnDemandEndpointExpression(); - - var url = await expression.GetValueAsync(new ValueProviderContext()); - - Assert.Equal("http://localhost:8080", url); - Assert.Equal(1, allocationCount()); - } - - [Fact] - public async Task GetValueAsync_AllocatesEndpointOnDemandForSelfReference() - { - var (resource, _, expression, allocationCount) = CreateOnDemandEndpointExpression(); - - var url = await expression.GetValueAsync(new ValueProviderContext { Caller = resource, ExecutionContext = CreateExecutionContext() }); - - Assert.Equal("http://localhost:8080", url); - Assert.Equal(1, allocationCount()); - } - - [Fact] - public async Task GetValueAsync_AllocatesEndpointOnDemandWhenEndpointResourceWaitsOnCaller() - { - var caller = new TestResource("caller"); - var (resource, _, expression, allocationCount) = CreateOnDemandEndpointExpression(); - resource.Annotations.Add(new WaitAnnotation(caller, WaitType.WaitUntilStarted)); - - var url = await expression.GetValueAsync(new ValueProviderContext { Caller = caller, ExecutionContext = CreateExecutionContext() }); - - Assert.Equal("http://localhost:8080", url); - Assert.Equal(1, allocationCount()); - } - - [Fact] - public async Task GetValueAsync_AllocatesEndpointOnDemandWhenEndpointResourceReferencesCaller() - { - var caller = new TestResource("caller"); - var (resource, _, expression, allocationCount) = CreateOnDemandEndpointExpression(); - var callerEndpoint = new EndpointAnnotation(ProtocolType.Tcp, uriScheme: "http", name: "http"); - caller.Annotations.Add(callerEndpoint); - resource.Annotations.Add(new EnvironmentCallbackAnnotation(context => - { - context.EnvironmentVariables["CALLER_URL"] = new EndpointReference(caller, callerEndpoint); - })); - - var url = await expression.GetValueAsync(new ValueProviderContext { Caller = caller, ExecutionContext = CreateExecutionContext() }); - - Assert.Equal("http://localhost:8080", url); - Assert.Equal(1, allocationCount()); - } - - [Fact] - public async Task GetValueAsync_AllocatesEndpointOnDemandWhenEndpointResourceTransitivelyDependsOnCaller() - { - var caller = new TestResource("caller"); - var intermediate = new TestResource("intermediate"); - var (resource, _, expression, allocationCount) = CreateOnDemandEndpointExpression(); - - resource.Annotations.Add(new WaitAnnotation(intermediate, WaitType.WaitUntilStarted)); - intermediate.Annotations.Add(new WaitAnnotation(caller, WaitType.WaitUntilStarted)); - - var url = await expression.GetValueAsync(new ValueProviderContext { Caller = caller, ExecutionContext = CreateExecutionContext() }); - - Assert.Equal("http://localhost:8080", url); - Assert.Equal(1, allocationCount()); - } - - [Fact] - public async Task GetValueAsync_WaitsForEndpointAllocationWhenContainerEndpointResourceDoesNotDependOnCaller() - { - var caller = new TestResource("caller"); - var (_, annotation, expression, allocationCount) = CreateOnDemandEndpointExpression(isContainerEndpoint: true); - - var getValueTask = expression.GetValueAsync(new ValueProviderContext { Caller = caller, ExecutionContext = CreateExecutionContext() }); - - Assert.False(getValueTask.IsCompleted); - Assert.Equal(0, allocationCount()); - - annotation.AllocatedEndpoint = new AllocatedEndpoint(annotation, "localhost", 8081); - - var url = await getValueTask; - - Assert.Equal("http://localhost:8081", url); - Assert.Equal(0, allocationCount()); - } - [Fact] public void EndpointAnnotation_ThrowsWhenEndpointNameNotDefined_ListsAvailableEndpoints() { @@ -562,29 +474,6 @@ private sealed class TestResource(string name) : Resource(name), IResourceWithEn { } - private static (TestResource Resource, EndpointAnnotation Endpoint, EndpointReferenceExpression Expression, Func AllocationCount) CreateOnDemandEndpointExpression(bool isContainerEndpoint = false) - { - var resource = new TestResource("test"); - var annotation = new EndpointAnnotation(ProtocolType.Tcp, uriScheme: "http", name: "http"); - var allocationCount = 0; - - if (isContainerEndpoint) - { - resource.Annotations.Add(new ContainerImageAnnotation { Image = "test-image" }); - } - - resource.Annotations.Add(annotation); - resource.Annotations.Add(new OnDemandEndpointAllocationAnnotation((endpoint, networkId) => - { - allocationCount++; - return new AllocatedEndpoint(endpoint, "localhost", 8080, EndpointBindingMode.SingleAddress, networkId: networkId); - })); - - return (resource, annotation, new EndpointReference(resource, annotation).Property(EndpointProperty.Url), () => allocationCount); - } - - private static DistributedApplicationExecutionContext CreateExecutionContext() => new(DistributedApplicationOperation.Run); - private struct WithWaitStartedNotification { private readonly WaitStartedNotificationAwaiter _awaiter; diff --git a/tests/Aspire.Hosting.Tests/HealthCheckTests.cs b/tests/Aspire.Hosting.Tests/HealthCheckTests.cs index ffb4b8ddc3e..d02ba018837 100644 --- a/tests/Aspire.Hosting.Tests/HealthCheckTests.cs +++ b/tests/Aspire.Hosting.Tests/HealthCheckTests.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using Aspire.TestUtilities; +using Aspire.Hosting.Eventing; using Aspire.Hosting.Utils; using Microsoft.AspNetCore.InternalTesting; using Microsoft.Extensions.DependencyInjection; @@ -75,6 +76,35 @@ public void WithHttpsHealthCheckThrowsIfReferencingEndpointThatIsNotHttpsScheme( ); } + [Fact] + public async Task WithHttpHealthCheckInitializesUriOnBeforeResourceStartedEvent() + { + using var builder = TestDistributedApplicationBuilder.Create(testOutputHelper); + + var resource = builder.AddContainer("resource", "dummycontainer") + .WithHttpEndpoint(port: 49217, targetPort: 80) + .WithHttpHealthCheck(); + + using var app = builder.Build(); + + var endpoint = resource.GetEndpoint("http").EndpointAnnotation; + endpoint.AllocatedEndpoint = new AllocatedEndpoint(endpoint, KnownHostNames.Localhost, 49217, EndpointBindingMode.SingleAddress, targetPortExpression: null, networkId: null); + + var eventing = app.Services.GetRequiredService(); + var registration = app.Services.GetRequiredService>().Value.Registrations + .Single(r => r.Name == "resource_http_/_200_check"); + + await eventing.PublishAsync(new ResourceEndpointsAllocatedEvent(resource.Resource, app.Services)); + + // The health check URI is intentionally initialized on BeforeResourceStartedEvent (not on + // ResourceEndpointsAllocatedEvent) so the URI reflects the final allocated endpoint. Once that + // event has been published the health check factory can build a valid check. + await eventing.PublishAsync(new BeforeResourceStartedEvent(resource.Resource, app.Services)); + + var healthCheck = registration.Factory(app.Services); + Assert.NotNull(healthCheck); + } + [Fact] [RequiresFeature(TestFeature.Docker)] public async Task VerifyWithHttpHealthCheckBlocksDependentResources() diff --git a/tests/Aspire.Hosting.Tests/Orchestrator/ApplicationOrchestratorTests.cs b/tests/Aspire.Hosting.Tests/Orchestrator/ApplicationOrchestratorTests.cs index 8326cccc0b1..b9edf0face5 100644 --- a/tests/Aspire.Hosting.Tests/Orchestrator/ApplicationOrchestratorTests.cs +++ b/tests/Aspire.Hosting.Tests/Orchestrator/ApplicationOrchestratorTests.cs @@ -408,6 +408,45 @@ public async Task GrandChildResourceWithConnectionString() Assert.True(grandChildConnectionStringAvailable); } + [Fact] + public async Task ConnectionStringAvailableEventPublishesBeforeBeforeResourceStartedEvent() + { + var builder = DistributedApplication.CreateBuilder(); + builder.WithTestAndResourceLogging(testOutputHelper); + + var resource = builder.AddResource(new TestResourceWithConnectionString("test-resource", "Server=localhost:5432;Database=testdb")); + + using var app = builder.Build(); + var distributedAppModel = app.Services.GetRequiredService(); + + var events = new DcpExecutorEvents(); + var resourceNotificationService = ResourceNotificationServiceTestHelpers.Create(); + var applicationEventing = new DistributedApplicationEventing(); + var observedEvents = new List(); + + applicationEventing.Subscribe(resource.Resource, (_, _) => + { + observedEvents.Add(nameof(ConnectionStringAvailableEvent)); + return Task.CompletedTask; + }); + applicationEventing.Subscribe(resource.Resource, (_, _) => + { + observedEvents.Add(nameof(BeforeResourceStartedEvent)); + return Task.CompletedTask; + }); + + var appOrchestrator = CreateOrchestrator(distributedAppModel, notificationService: resourceNotificationService, dcpEvents: events, applicationEventing: applicationEventing); + await appOrchestrator.RunApplicationAsync(); + + await events.PublishAsync(new OnConnectionStringAvailableContext(CancellationToken.None, resource.Resource)); + await events.PublishAsync(new OnResourceStartingContext(CancellationToken.None, KnownResourceTypes.Executable, resource.Resource, "test-resource-dcp")); + + Assert.Collection( + observedEvents, + eventName => Assert.Equal(nameof(ConnectionStringAvailableEvent), eventName), + eventName => Assert.Equal(nameof(BeforeResourceStartedEvent), eventName)); + } + [Fact] public async Task ConnectionStringAvailableEventPublishesUpdateWithConnectionStringValue() { diff --git a/tests/Aspire.Hosting.Tests/Utils/MockUserSecretsManager.cs b/tests/Aspire.Hosting.Tests/Utils/MockUserSecretsManager.cs index b501ffc1756..abc2fd0ba46 100644 --- a/tests/Aspire.Hosting.Tests/Utils/MockUserSecretsManager.cs +++ b/tests/Aspire.Hosting.Tests/Utils/MockUserSecretsManager.cs @@ -10,13 +10,22 @@ namespace Aspire.Hosting.Tests.Utils; internal sealed class MockUserSecretsManager : IUserSecretsManager { + public Dictionary Secrets { get; } = new(StringComparer.OrdinalIgnoreCase); + public bool IsAvailable => true; public string FilePath => "/mock/path/secrets.json"; - public bool TrySetSecret(string name, string value) => true; + public bool TrySetSecret(string name, string value) + { + Secrets[name] = value; + return true; + } - public bool TryDeleteSecret(string name) => true; + public bool TryDeleteSecret(string name) + { + return Secrets.Remove(name); + } public void GetOrSetSecret(IConfigurationManager configuration, string name, Func valueGenerator) { diff --git a/tests/Aspire.Hosting.Tests/Utils/PersistentContainerTestHelpers.cs b/tests/Aspire.Hosting.Tests/Utils/PersistentContainerTestHelpers.cs index 82a51efeefa..80b7bdc6060 100644 --- a/tests/Aspire.Hosting.Tests/Utils/PersistentContainerTestHelpers.cs +++ b/tests/Aspire.Hosting.Tests/Utils/PersistentContainerTestHelpers.cs @@ -28,6 +28,35 @@ public static async Task AssertResourceReusesContainerAsync( bool useTestContainerRegistry = false, bool randomizePorts = false, TimeSpan? timeout = null) + { + await AssertResourcesReuseContainersAsync( + testOutputHelper, + configureResource, + [resourceName], + useTestContainerRegistry, + randomizePorts, + compareUrls: false, + timeout); + } + + /// + /// Verifies that resources configured with persistent lifetimes use the same Docker containers across AppHost runs. + /// + /// The xUnit output helper used for test and resource logging. + /// Configures the persistent resources on each AppHost run. + /// The resource names whose persistent Docker container identities should be compared. + /// Whether to apply the test container registry override for integrations that require CI-mirrored images. + /// Whether to force DCP to randomize ports for the AppHost runs. + /// Whether to compare the resource URLs across runs. This also verifies stable public ports. + /// The timeout for starting, stopping, and observing the resources. Defaults to 10 minutes because some container integrations have slow cold starts. + public static async Task AssertResourcesReuseContainersAsync( + ITestOutputHelper testOutputHelper, + Action configureResources, + string[] resourceNames, + bool useTestContainerRegistry = false, + bool randomizePorts = false, + bool compareUrls = false, + TimeSpan? timeout = null) { using var cts = new CancellationTokenSource(timeout ?? TimeSpan.FromMinutes(10)); using var aspireStore = new TestTempDirectory(); @@ -56,7 +85,7 @@ public static async Task AssertResourceReusesContainerAsync( } } - async Task RunContainerAsync() + async Task RunContainerAsync() { var args = new[] { @@ -76,30 +105,32 @@ async Task RunContainerAsync() Assert.True(builder.UserSecretsManager.IsAvailable); - configureResource(builder); + configureResources(builder); using var app = builder.Build(); await app.StartAsync(cts.Token); var resourceNotificationService = app.Services.GetRequiredService(); - var containerIdentity = await GetContainerIdentityAsync(resourceNotificationService, resourceName, cts.Token); + var resourceSnapshots = await Task.WhenAll( + resourceNames.Select(resourceName => GetContainerIdentityAsync(resourceNotificationService, resourceName, compareUrls, cts.Token))); await app.StopAsync(cts.Token).WaitAsync(cts.Token); - return containerIdentity; + return resourceSnapshots.OrderBy(snapshot => snapshot.ResourceName, StringComparer.Ordinal).ToArray(); } } /// /// Gets the Docker container identity for a persistent resource after it becomes healthy. /// - private static async Task GetContainerIdentityAsync(ResourceNotificationService resourceNotificationService, string resourceName, CancellationToken cancellationToken) + private static async Task GetContainerIdentityAsync(ResourceNotificationService resourceNotificationService, string resourceName, bool includeUrls, CancellationToken cancellationToken) { await resourceNotificationService.WaitForResourceHealthyAsync(resourceName, cancellationToken); var resourceEvent = await resourceNotificationService.WaitForResourceAsync(resourceName, evt => { return GetPropertyValue(evt, ContainerLifetimePropertyName) is ContainerLifetime.Persistent && - GetPropertyValue(evt, ContainerIdPropertyName) is string { Length: > 0 }; + GetPropertyValue(evt, ContainerIdPropertyName) is string { Length: > 0 } && + (!includeUrls || evt.Snapshot.Urls.Length > 0); }, cancellationToken); var containerLifetime = GetPropertyValue(resourceEvent, ContainerLifetimePropertyName); @@ -108,9 +139,23 @@ private static async Task GetContainerIdentityAsync(ResourceNotification var containerId = Assert.IsType(GetPropertyValue(resourceEvent, ContainerIdPropertyName)); Assert.NotEmpty(containerId); - return containerId; + var urls = includeUrls + ? string.Join(Environment.NewLine, resourceEvent.Snapshot.Urls + .OrderBy(url => url.Name, StringComparer.Ordinal) + .ThenBy(url => url.Url, StringComparer.Ordinal) + .Select(url => $"{url.Name}:{url.Url}")) + : string.Empty; + + if (includeUrls) + { + Assert.NotEmpty(urls); + } + + return new(resourceName, containerId, urls); } private static object? GetPropertyValue(ResourceEvent resourceEvent, string propertyName) => resourceEvent.Snapshot.Properties.FirstOrDefault(x => x.Name == propertyName)?.Value; + + private sealed record ResourceRunSnapshot(string ResourceName, string ContainerId, string Urls); }