diff --git a/Directory.Packages.props b/Directory.Packages.props index c65cbc34..fab781ef 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,10 +4,11 @@ - + + diff --git a/Goa.sln b/Goa.sln index b0d138e1..88a2ca83 100644 --- a/Goa.sln +++ b/Goa.sln @@ -141,6 +141,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Goa.Clients.Sns.Benchmarks" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Goa.Clients.EventBridge.Benchmarks", "tests\Clients\Goa.Clients.EventBridge.Benchmarks\Goa.Clients.EventBridge.Benchmarks.csproj", "{71D5E069-1555-4C11-B45C-399BC26FC258}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Goa.Clients.S3.Tests", "tests\Clients\Goa.Clients.S3.Tests\Goa.Clients.S3.Tests.csproj", "{CC3715F8-340F-4F1E-BB46-1078CD764E99}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -691,6 +693,18 @@ Global {71D5E069-1555-4C11-B45C-399BC26FC258}.Release|x64.Build.0 = Release|Any CPU {71D5E069-1555-4C11-B45C-399BC26FC258}.Release|x86.ActiveCfg = Release|Any CPU {71D5E069-1555-4C11-B45C-399BC26FC258}.Release|x86.Build.0 = Release|Any CPU + {CC3715F8-340F-4F1E-BB46-1078CD764E99}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CC3715F8-340F-4F1E-BB46-1078CD764E99}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CC3715F8-340F-4F1E-BB46-1078CD764E99}.Debug|x64.ActiveCfg = Debug|Any CPU + {CC3715F8-340F-4F1E-BB46-1078CD764E99}.Debug|x64.Build.0 = Debug|Any CPU + {CC3715F8-340F-4F1E-BB46-1078CD764E99}.Debug|x86.ActiveCfg = Debug|Any CPU + {CC3715F8-340F-4F1E-BB46-1078CD764E99}.Debug|x86.Build.0 = Debug|Any CPU + {CC3715F8-340F-4F1E-BB46-1078CD764E99}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CC3715F8-340F-4F1E-BB46-1078CD764E99}.Release|Any CPU.Build.0 = Release|Any CPU + {CC3715F8-340F-4F1E-BB46-1078CD764E99}.Release|x64.ActiveCfg = Release|Any CPU + {CC3715F8-340F-4F1E-BB46-1078CD764E99}.Release|x64.Build.0 = Release|Any CPU + {CC3715F8-340F-4F1E-BB46-1078CD764E99}.Release|x86.ActiveCfg = Release|Any CPU + {CC3715F8-340F-4F1E-BB46-1078CD764E99}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -747,5 +761,6 @@ Global {22A37049-1E03-4D6A-817C-47B74BF6DF1B} = {B5DC1548-387B-4ABB-B4E8-CED73AFD8863} {C3D8BB6D-BAA5-4458-B835-1E44F5826A43} = {B5DC1548-387B-4ABB-B4E8-CED73AFD8863} {71D5E069-1555-4C11-B45C-399BC26FC258} = {B5DC1548-387B-4ABB-B4E8-CED73AFD8863} + {CC3715F8-340F-4F1E-BB46-1078CD764E99} = {B5DC1548-387B-4ABB-B4E8-CED73AFD8863} EndGlobalSection EndGlobal diff --git a/src/Clients/Goa.Clients.Core/Http/HttpOptions.cs b/src/Clients/Goa.Clients.Core/Http/HttpOptions.cs index c2ccd959..546056a8 100644 --- a/src/Clients/Goa.Clients.Core/Http/HttpOptions.cs +++ b/src/Clients/Goa.Clients.Core/Http/HttpOptions.cs @@ -8,25 +8,31 @@ public static class HttpOptions /// /// HTTP request option key for specifying the AWS region. /// - public static HttpRequestOptionsKey Region = new(nameof(Region)); - + public static readonly HttpRequestOptionsKey Region = new(nameof(Region)); + /// /// HTTP request option key for specifying the AWS service name. /// - public static HttpRequestOptionsKey Service = new(nameof(Service)); - + public static readonly HttpRequestOptionsKey Service = new(nameof(Service)); + /// /// HTTP request option key for specifying the operation target. /// - public static HttpRequestOptionsKey Target = new(nameof(Target)); - + public static readonly HttpRequestOptionsKey Target = new(nameof(Target)); + /// /// HTTP request option key for specifying the API version. /// - public static HttpRequestOptionsKey ApiVersion = new(nameof(ApiVersion)); - + public static readonly HttpRequestOptionsKey ApiVersion = new(nameof(ApiVersion)); + /// /// HTTP request option key for specifying the request payload as UTF-8 bytes. /// - public static HttpRequestOptionsKey> Payload = new(nameof(Payload)); + public static readonly HttpRequestOptionsKey> Payload = new(nameof(Payload)); + + /// + /// HTTP request option key indicating that the request path is already URI-encoded and must be + /// used as-is as the SigV4 canonical URI (S3-style single encoding) instead of being re-encoded. + /// + public static readonly HttpRequestOptionsKey UseSingleUriEncoding = new(nameof(UseSingleUriEncoding)); } \ No newline at end of file diff --git a/src/Clients/Goa.Clients.Core/Http/RequestSigner.cs b/src/Clients/Goa.Clients.Core/Http/RequestSigner.cs index 4ee4e666..3901f47a 100644 --- a/src/Clients/Goa.Clients.Core/Http/RequestSigner.cs +++ b/src/Clients/Goa.Clients.Core/Http/RequestSigner.cs @@ -166,9 +166,16 @@ private async ValueTask SignRequestWithCredentialsAsync( time.TryFormat(shortDate, out _, "yyyyMMdd".AsSpan()); time.TryFormat(longDate, out _, "yyyyMMdd'T'HHmmss'Z'".AsSpan()); - // Add required headers to the request for immediate usage - if (request.RequestUri != null) - request.Headers.Host = request.RequestUri.Host; + // Add required headers to the request for immediate usage. + // Match the canonical host written by WriteHost: include a non-default port so the Host + // header sent on the wire agrees with the signed canonical host (otherwise SigV4 validation + // fails against endpoints on non-default ports, e.g. LocalStack's mapped port). + if (request.RequestUri is { } uri) + { + // Mirror WriteHost exactly: use IdnHost and append a non-default port. + var hostName = uri.IdnHost ?? uri.Host; + request.Headers.Host = uri.IsDefaultPort ? hostName : $"{hostName}:{uri.Port}"; + } request.Headers.TryAddWithoutValidation(RequestHeaders.AmzDate, new string(longDate)); Span payloadHex = stackalloc char[64]; @@ -306,9 +313,18 @@ private async ValueTask SignRequestWithCredentialsAsync( else Array.Sort(headerRefs, 0, totalHeaders, HeaderRefComparer.OrdinalIgnoreCase); var pathStr = request.RequestUri?.AbsolutePath ?? RootPath; - var canonUriLen = MeasureCanonicalUri(pathStr); - // Process query parameters with efficient encoding + // S3-style signing uses the already-encoded request path as-is for the canonical URI, + // whereas other services require the path to be URI-encoded a second time. + var useSingleUriEncoding = request.Options.TryGetValue(HttpOptions.UseSingleUriEncoding, out var singleEncode) && singleEncode; + var canonUriLen = useSingleUriEncoding + ? Math.Max(pathStr.Length, 1) + : MeasureCanonicalUri(pathStr); + + // Process query parameters with efficient encoding. + // NOTE: query values are RFC 3986 encoded once here. If/when an S3 list operation + // (e.g. ListObjectsV2) is added, ensure its query string is built pre-encoded so the + // signed canonical query matches the value sent on the wire. var rawQuery = request.RequestUri?.Query; QueryPart[]? qparts = null; char[]? qEncodedBuf = null; @@ -363,7 +379,22 @@ private async ValueTask SignRequestWithCredentialsAsync( var m = request.Method.Method ?? "GET"; m.AsSpan().CopyTo(canonical.Slice(pos)); pos += m.Length; canonical[pos++] = '\n'; - pos += WriteCanonicalUri(canonical.Slice(pos), pathStr); + if (useSingleUriEncoding) + { + if (pathStr.Length == 0) + { + canonical[pos++] = '/'; + } + else + { + pathStr.AsSpan().CopyTo(canonical.Slice(pos)); + pos += pathStr.Length; + } + } + else + { + pos += WriteCanonicalUri(canonical.Slice(pos), pathStr); + } canonical[pos++] = '\n'; if (qparts is null || qEncodedBuf is null) diff --git a/src/Clients/Goa.Clients.S3/Class1.cs b/src/Clients/Goa.Clients.S3/Class1.cs deleted file mode 100644 index d28e6071..00000000 --- a/src/Clients/Goa.Clients.S3/Class1.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Goa.Clients.S3; - -/// -/// -public class Class1 -{ -} diff --git a/src/Clients/Goa.Clients.S3/Errors/S3ErrorCodes.cs b/src/Clients/Goa.Clients.S3/Errors/S3ErrorCodes.cs new file mode 100644 index 00000000..d29082b1 --- /dev/null +++ b/src/Clients/Goa.Clients.S3/Errors/S3ErrorCodes.cs @@ -0,0 +1,62 @@ +namespace Goa.Clients.S3.Errors; + +/// +/// Common error codes returned by S3 operations. +/// +public static class S3ErrorCodes +{ + /// + /// The specified key does not exist. + /// + public const string NoSuchKey = "NoSuchKey"; + + /// + /// The specified bucket does not exist. + /// + public const string NoSuchBucket = "NoSuchBucket"; + + /// + /// Access to the resource is denied. + /// + public const string AccessDenied = "AccessDenied"; + + /// + /// The requested range is not satisfiable. + /// + public const string InvalidRange = "InvalidRange"; + + /// + /// The proposed upload exceeds the maximum allowed object size. + /// + public const string EntityTooLarge = "EntityTooLarge"; + + /// + /// At least one of the preconditions specified did not hold. + /// + public const string PreconditionFailed = "PreconditionFailed"; + + /// + /// The operation is not valid for the current state of the object. + /// + public const string InvalidObjectState = "InvalidObjectState"; + + /// + /// Reduce your request rate. + /// + public const string SlowDown = "SlowDown"; + + /// + /// The supplied object key is not valid (empty, contains dot-segments, or control characters). + /// + public const string InvalidKey = "S3.InvalidKey"; + + /// + /// The supplied bucket name does not satisfy S3 bucket naming rules. + /// + public const string InvalidBucketName = "S3.InvalidBucketName"; + + /// + /// A user-defined metadata name is not a valid HTTP token. + /// + public const string InvalidMetadata = "S3.InvalidMetadata"; +} diff --git a/src/Clients/Goa.Clients.S3/Goa.Clients.S3.csproj b/src/Clients/Goa.Clients.S3/Goa.Clients.S3.csproj index 469799ea..d0642845 100644 --- a/src/Clients/Goa.Clients.S3/Goa.Clients.S3.csproj +++ b/src/Clients/Goa.Clients.S3/Goa.Clients.S3.csproj @@ -4,4 +4,12 @@ S3 client for object storage operations in high-performance AWS Lambda functions $(PackageBaseTags);s3;client;storage;objects + + + + + + + + diff --git a/src/Clients/Goa.Clients.S3/IS3Client.cs b/src/Clients/Goa.Clients.S3/IS3Client.cs new file mode 100644 index 00000000..53a7129d --- /dev/null +++ b/src/Clients/Goa.Clients.S3/IS3Client.cs @@ -0,0 +1,51 @@ +using ErrorOr; +using Goa.Clients.S3.Operations.DeleteObject; +using Goa.Clients.S3.Operations.GetObject; +using Goa.Clients.S3.Operations.HeadObject; +using Goa.Clients.S3.Operations.PutObject; + +namespace Goa.Clients.S3; + +/// +/// High-performance S3 client interface optimized for AWS Lambda usage. +/// All operations use strongly-typed request objects and return ErrorOr results. +/// +public interface IS3Client +{ + /// + /// Uploads an object to the specified bucket. + /// + /// The put object request. + /// A cancellation token to cancel the operation. + /// The put object response, or an error if the operation failed. + Task> PutObjectAsync(PutObjectRequest request, CancellationToken cancellationToken = default); + + /// + /// Retrieves an object from the specified bucket. + /// + /// + /// The entire object body is buffered into memory before the response is returned. For large + /// objects, set to fetch the object in bounded slices and + /// avoid excessive memory pressure. + /// + /// The get object request. + /// A cancellation token to cancel the operation. + /// The get object response, or an error if the operation failed. + Task> GetObjectAsync(GetObjectRequest request, CancellationToken cancellationToken = default); + + /// + /// Retrieves the metadata of an object without returning the object body. + /// + /// The head object request. + /// A cancellation token to cancel the operation. + /// The head object response, or an error if the operation failed. + Task> HeadObjectAsync(HeadObjectRequest request, CancellationToken cancellationToken = default); + + /// + /// Deletes an object from the specified bucket. Deleting a key that does not exist succeeds. + /// + /// The delete object request. + /// A cancellation token to cancel the operation. + /// The delete object response, or an error if the operation failed. + Task> DeleteObjectAsync(DeleteObjectRequest request, CancellationToken cancellationToken = default); +} diff --git a/src/Clients/Goa.Clients.S3/Log.cs b/src/Clients/Goa.Clients.S3/Log.cs new file mode 100644 index 00000000..8db8f1c2 --- /dev/null +++ b/src/Clients/Goa.Clients.S3/Log.cs @@ -0,0 +1,18 @@ +using Microsoft.Extensions.Logging; + +namespace Goa.Clients.S3; + +internal static partial class Log +{ + [LoggerMessage(EventId = 1, Level = LogLevel.Error, Message = "Failed to put object {Key} to S3 bucket {Bucket}")] + public static partial void PutObjectFailed(this ILogger logger, Exception exception, string bucket, string key); + + [LoggerMessage(EventId = 2, Level = LogLevel.Error, Message = "Failed to get object {Key} from S3 bucket {Bucket}")] + public static partial void GetObjectFailed(this ILogger logger, Exception exception, string bucket, string key); + + [LoggerMessage(EventId = 3, Level = LogLevel.Error, Message = "Failed to head object {Key} in S3 bucket {Bucket}")] + public static partial void HeadObjectFailed(this ILogger logger, Exception exception, string bucket, string key); + + [LoggerMessage(EventId = 4, Level = LogLevel.Error, Message = "Failed to delete object {Key} from S3 bucket {Bucket}")] + public static partial void DeleteObjectFailed(this ILogger logger, Exception exception, string bucket, string key); +} diff --git a/src/Clients/Goa.Clients.S3/Operations/DeleteObject/DeleteObjectRequest.cs b/src/Clients/Goa.Clients.S3/Operations/DeleteObject/DeleteObjectRequest.cs new file mode 100644 index 00000000..bc9dd7f7 --- /dev/null +++ b/src/Clients/Goa.Clients.S3/Operations/DeleteObject/DeleteObjectRequest.cs @@ -0,0 +1,17 @@ +namespace Goa.Clients.S3.Operations.DeleteObject; + +/// +/// Request for the DeleteObject operation. +/// +public sealed class DeleteObjectRequest +{ + /// + /// The name of the bucket containing the object. + /// + public required string Bucket { get; set; } + + /// + /// The key of the object to delete. + /// + public required string Key { get; set; } +} diff --git a/src/Clients/Goa.Clients.S3/Operations/DeleteObject/DeleteObjectResponse.cs b/src/Clients/Goa.Clients.S3/Operations/DeleteObject/DeleteObjectResponse.cs new file mode 100644 index 00000000..33478ee2 --- /dev/null +++ b/src/Clients/Goa.Clients.S3/Operations/DeleteObject/DeleteObjectResponse.cs @@ -0,0 +1,10 @@ +namespace Goa.Clients.S3.Operations.DeleteObject; + +/// +/// Response from the DeleteObject operation. +/// +public sealed class DeleteObjectResponse +{ + // DeleteObject returns an empty response on success. S3 responds with 204 No Content + // even when the specified key does not exist on an unversioned bucket. +} diff --git a/src/Clients/Goa.Clients.S3/Operations/GetObject/GetObjectRequest.cs b/src/Clients/Goa.Clients.S3/Operations/GetObject/GetObjectRequest.cs new file mode 100644 index 00000000..df39978e --- /dev/null +++ b/src/Clients/Goa.Clients.S3/Operations/GetObject/GetObjectRequest.cs @@ -0,0 +1,22 @@ +namespace Goa.Clients.S3.Operations.GetObject; + +/// +/// Request for the GetObject operation. +/// +public sealed class GetObjectRequest +{ + /// + /// The name of the bucket containing the object. + /// + public required string Bucket { get; set; } + + /// + /// The key of the object to retrieve. + /// + public required string Key { get; set; } + + /// + /// The byte range of the object to retrieve (e.g. "bytes=0-63"). + /// + public string? Range { get; set; } +} diff --git a/src/Clients/Goa.Clients.S3/Operations/GetObject/GetObjectResponse.cs b/src/Clients/Goa.Clients.S3/Operations/GetObject/GetObjectResponse.cs new file mode 100644 index 00000000..0093062f --- /dev/null +++ b/src/Clients/Goa.Clients.S3/Operations/GetObject/GetObjectResponse.cs @@ -0,0 +1,37 @@ +namespace Goa.Clients.S3.Operations.GetObject; + +/// +/// Response from the GetObject operation. +/// +public sealed class GetObjectResponse +{ + /// + /// The object content. + /// + public byte[] Body { get; set; } = []; + + /// + /// The MIME content type of the object. + /// + public string? ContentType { get; set; } + + /// + /// The size of the returned content in bytes. + /// + public long ContentLength { get; set; } + + /// + /// The entity tag of the object, as returned by S3 (including surrounding quotes). + /// + public string? ETag { get; set; } + + /// + /// The date and time the object was last modified. + /// + public DateTimeOffset? LastModified { get; set; } + + /// + /// User-defined metadata stored with the object ("x-amz-meta-*" headers, prefix stripped), if any. + /// + public Dictionary? Metadata { get; set; } +} diff --git a/src/Clients/Goa.Clients.S3/Operations/HeadObject/HeadObjectRequest.cs b/src/Clients/Goa.Clients.S3/Operations/HeadObject/HeadObjectRequest.cs new file mode 100644 index 00000000..bb668380 --- /dev/null +++ b/src/Clients/Goa.Clients.S3/Operations/HeadObject/HeadObjectRequest.cs @@ -0,0 +1,17 @@ +namespace Goa.Clients.S3.Operations.HeadObject; + +/// +/// Request for the HeadObject operation. +/// +public sealed class HeadObjectRequest +{ + /// + /// The name of the bucket containing the object. + /// + public required string Bucket { get; set; } + + /// + /// The key of the object to inspect. + /// + public required string Key { get; set; } +} diff --git a/src/Clients/Goa.Clients.S3/Operations/HeadObject/HeadObjectResponse.cs b/src/Clients/Goa.Clients.S3/Operations/HeadObject/HeadObjectResponse.cs new file mode 100644 index 00000000..e5d268f0 --- /dev/null +++ b/src/Clients/Goa.Clients.S3/Operations/HeadObject/HeadObjectResponse.cs @@ -0,0 +1,32 @@ +namespace Goa.Clients.S3.Operations.HeadObject; + +/// +/// Response from the HeadObject operation. +/// +public sealed class HeadObjectResponse +{ + /// + /// The MIME content type of the object. + /// + public string? ContentType { get; set; } + + /// + /// The size of the object in bytes. + /// + public long ContentLength { get; set; } + + /// + /// The entity tag of the object, as returned by S3 (including surrounding quotes). + /// + public string? ETag { get; set; } + + /// + /// The date and time the object was last modified. + /// + public DateTimeOffset? LastModified { get; set; } + + /// + /// User-defined metadata stored with the object ("x-amz-meta-*" headers, prefix stripped), if any. + /// + public Dictionary? Metadata { get; set; } +} diff --git a/src/Clients/Goa.Clients.S3/Operations/PutObject/PutObjectBuilder.cs b/src/Clients/Goa.Clients.S3/Operations/PutObject/PutObjectBuilder.cs new file mode 100644 index 00000000..d755b9e9 --- /dev/null +++ b/src/Clients/Goa.Clients.S3/Operations/PutObject/PutObjectBuilder.cs @@ -0,0 +1,126 @@ +using Goa.Clients.S3; + +namespace Goa.Clients.S3.Operations.PutObject; + +/// +/// Builder for creating PutObject requests with a fluent API. +/// +public sealed class PutObjectBuilder +{ + private string? _bucket; + private string? _key; + private ReadOnlyMemory _body; + private string? _contentType; + private string? _serverSideEncryption; + private string? _sseKmsKeyId; + private Dictionary? _metadata; + + /// + /// Sets the bucket name. + /// + /// The bucket name. + /// The builder instance for method chaining. + public PutObjectBuilder WithBucket(string bucket) + { + ArgumentException.ThrowIfNullOrWhiteSpace(bucket); + _bucket = bucket; + return this; + } + + /// + /// Sets the object key. + /// + /// The object key. + /// The builder instance for method chaining. + public PutObjectBuilder WithKey(string key) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + _key = key; + return this; + } + + /// + /// Sets the object content. + /// + /// The object content. + /// The builder instance for method chaining. + public PutObjectBuilder WithBody(ReadOnlyMemory body) + { + _body = body; + return this; + } + + /// + /// Sets the MIME content type of the object. + /// + /// The content type (e.g. "application/json"). + /// The builder instance for method chaining. + public PutObjectBuilder WithContentType(string contentType) + { + ArgumentException.ThrowIfNullOrWhiteSpace(contentType); + _contentType = contentType; + return this; + } + + /// + /// Sets the server-side encryption algorithm and optional KMS key ID. + /// + /// The server-side encryption algorithm (e.g. "aws:kms"). + /// The AWS KMS key ID to use when the algorithm is "aws:kms". + /// The builder instance for method chaining. + public PutObjectBuilder WithServerSideEncryption(string serverSideEncryption, string? sseKmsKeyId = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(serverSideEncryption); + _serverSideEncryption = serverSideEncryption; + _sseKmsKeyId = sseKmsKeyId; + return this; + } + + /// + /// Adds a user-defined metadata entry, sent as an "x-amz-meta-{name}" header. + /// + /// The metadata name. Must be a valid HTTP token (no spaces, colons or control characters). + /// The metadata value. + /// The builder instance for method chaining. + /// Thrown when is not a valid HTTP token. + public PutObjectBuilder AddMetadata(string name, string value) + { + ArgumentException.ThrowIfNullOrWhiteSpace(name); + ArgumentNullException.ThrowIfNull(value); + + if (!S3RequestValidation.IsValidHttpToken(name)) + throw new ArgumentException( + $"Metadata name '{name}' is not a valid HTTP token. Names must not contain spaces, colons, control or separator characters.", + nameof(name)); + + _metadata ??= new Dictionary(StringComparer.OrdinalIgnoreCase); + _metadata[name] = value; + return this; + } + + /// + /// Builds the PutObject request. + /// + /// A configured PutObjectRequest. + /// Thrown when required fields are missing. + public PutObjectRequest Build() + { + if (string.IsNullOrWhiteSpace(_bucket)) + throw new InvalidOperationException("Bucket is required."); + + if (string.IsNullOrWhiteSpace(_key)) + throw new InvalidOperationException("Key is required."); + + return new PutObjectRequest + { + Bucket = _bucket, + Key = _key, + Body = _body, + ContentType = _contentType, + ServerSideEncryption = _serverSideEncryption, + SseKmsKeyId = _sseKmsKeyId, + // Copy so a built request keeps its own metadata, independent of later builder mutations. + Metadata = _metadata is null ? null : new Dictionary(_metadata, StringComparer.OrdinalIgnoreCase) + }; + } +} diff --git a/src/Clients/Goa.Clients.S3/Operations/PutObject/PutObjectRequest.cs b/src/Clients/Goa.Clients.S3/Operations/PutObject/PutObjectRequest.cs new file mode 100644 index 00000000..f8e53c9f --- /dev/null +++ b/src/Clients/Goa.Clients.S3/Operations/PutObject/PutObjectRequest.cs @@ -0,0 +1,42 @@ +namespace Goa.Clients.S3.Operations.PutObject; + +/// +/// Request for the PutObject operation. +/// +public sealed class PutObjectRequest +{ + /// + /// The name of the bucket to which the object is uploaded. + /// + public required string Bucket { get; set; } + + /// + /// The key of the object to upload. + /// + public required string Key { get; set; } + + /// + /// The object content to upload. + /// + public ReadOnlyMemory Body { get; set; } + + /// + /// The MIME content type of the object (e.g. "application/json"). + /// + public string? ContentType { get; set; } + + /// + /// The server-side encryption algorithm to use when storing the object (e.g. "aws:kms"). + /// + public string? ServerSideEncryption { get; set; } + + /// + /// The AWS KMS key ID to use for object encryption when is "aws:kms". + /// + public string? SseKmsKeyId { get; set; } + + /// + /// User-defined metadata to store with the object. Each entry is sent as an "x-amz-meta-{key}" header. + /// + public Dictionary? Metadata { get; set; } +} diff --git a/src/Clients/Goa.Clients.S3/Operations/PutObject/PutObjectResponse.cs b/src/Clients/Goa.Clients.S3/Operations/PutObject/PutObjectResponse.cs new file mode 100644 index 00000000..3c094548 --- /dev/null +++ b/src/Clients/Goa.Clients.S3/Operations/PutObject/PutObjectResponse.cs @@ -0,0 +1,12 @@ +namespace Goa.Clients.S3.Operations.PutObject; + +/// +/// Response from the PutObject operation. +/// +public sealed class PutObjectResponse +{ + /// + /// The entity tag of the uploaded object, as returned by S3 (including surrounding quotes). + /// + public string? ETag { get; set; } +} diff --git a/src/Clients/Goa.Clients.S3/README.md b/src/Clients/Goa.Clients.S3/README.md index 0be203ee..e7fc5e84 100644 --- a/src/Clients/Goa.Clients.S3/README.md +++ b/src/Clients/Goa.Clients.S3/README.md @@ -1,5 +1,169 @@ # Goa.Clients.S3 -_Coming Soon_ +S3 client for object storage operations in high-performance AWS Lambda functions. This package provides a lightweight, AOT-ready S3 client optimized for minimal cold start times. -S3 client for AWS Simple Storage Service. +## Installation + +```bash +dotnet add package Goa.Clients.S3 +``` + +## Features + +- Native AOT support for faster Lambda cold starts +- Minimal dependencies and memory allocations +- Built-in error handling with ErrorOr pattern +- Virtual-host and path-style addressing (path-style is used automatically for custom endpoints such as LocalStack) +- Server-side encryption support (SSE-KMS) +- User-defined object metadata +- Ranged downloads + +## Usage + +### Basic Setup + +```csharp +using Goa.Clients.S3; +using Microsoft.Extensions.DependencyInjection; + +// Register S3 client +services.AddS3(); + +// Or with a custom endpoint (e.g. LocalStack) +services.AddS3(config => +{ + config.ServiceUrl = "http://localhost:4566"; + config.Region = "us-east-1"; +}); +``` + +### Uploading Objects + +```csharp +using Goa.Clients.S3.Operations.PutObject; +using System.Text; + +public class StorageService +{ + private readonly IS3Client _s3; + + public StorageService(IS3Client s3) + { + _s3 = s3; + } + + public async Task UploadAsync(string bucket, string key, string json) + { + var request = new PutObjectRequest + { + Bucket = bucket, + Key = key, + Body = Encoding.UTF8.GetBytes(json), + ContentType = "application/json" + }; + + var result = await _s3.PutObjectAsync(request); + return result.IsError ? null : result.Value.ETag; + } +} +``` + +Or with the fluent builder, including SSE-KMS encryption and metadata: + +```csharp +var request = new PutObjectBuilder() + .WithBucket("my-bucket") + .WithKey("documents/report.pdf") + .WithBody(fileBytes) + .WithContentType("application/pdf") + .WithServerSideEncryption("aws:kms", "alias/my-key") + .AddMetadata("uploaded-by", "lambda") + .Build(); + +var result = await _s3.PutObjectAsync(request); +``` + +### Downloading Objects + +```csharp +using Goa.Clients.S3.Operations.GetObject; + +public async Task DownloadAsync(string bucket, string key) +{ + var result = await _s3.GetObjectAsync(new GetObjectRequest + { + Bucket = bucket, + Key = key + }); + + if (result.IsError) + { + Console.WriteLine($"Download failed: {result.FirstError}"); + return null; + } + + return result.Value.Body; +} + +// Ranged download - first 64 bytes only +var rangeResult = await _s3.GetObjectAsync(new GetObjectRequest +{ + Bucket = "my-bucket", + Key = "large-file.bin", + Range = "bytes=0-63" +}); +``` + +> **Memory note:** `GetObjectAsync` buffers the entire object body into memory before returning. +> For large objects, use the `Range` property to fetch the object in bounded slices rather than +> downloading the whole object in a single call. + +### Checking Object Metadata + +```csharp +using Goa.Clients.S3.Operations.HeadObject; + +var head = await _s3.HeadObjectAsync(new HeadObjectRequest +{ + Bucket = "my-bucket", + Key = "documents/report.pdf" +}); + +if (!head.IsError) +{ + Console.WriteLine($"Size: {head.Value.ContentLength}, ETag: {head.Value.ETag}"); +} +``` + +### Deleting Objects + +```csharp +using Goa.Clients.S3.Operations.DeleteObject; + +var result = await _s3.DeleteObjectAsync(new DeleteObjectRequest +{ + Bucket = "my-bucket", + Key = "documents/report.pdf" +}); + +// Deleting a missing key also succeeds - S3 deletes are idempotent on unversioned buckets. +``` + +### Error Handling + +Missing objects are surfaced as `ErrorType.NotFound` rather than exceptions: + +```csharp +using ErrorOr; + +var result = await _s3.GetObjectAsync(new GetObjectRequest { Bucket = "my-bucket", Key = "missing" }); + +if (result.IsError && result.FirstError.Type == ErrorType.NotFound) +{ + // Handle missing object +} +``` + +## Documentation + +For more information and examples, visit the [main Goa documentation](https://github.com/im5tu/goa). diff --git a/src/Clients/Goa.Clients.S3/S3RequestValidation.cs b/src/Clients/Goa.Clients.S3/S3RequestValidation.cs new file mode 100644 index 00000000..febd689a --- /dev/null +++ b/src/Clients/Goa.Clients.S3/S3RequestValidation.cs @@ -0,0 +1,128 @@ +using ErrorOr; +using Goa.Clients.S3.Errors; + +namespace Goa.Clients.S3; + +/// +/// Shared validation for S3 bucket names and object keys. +/// Keys and bucket names are interpolated directly into the request URI, so they must be +/// validated before signing to prevent path traversal (dot-segments collapsing the path) +/// and malformed host names. +/// +internal static class S3RequestValidation +{ + /// + /// Validates an object key. Rejects empty keys, keys whose '/'-split produces an empty, + /// "." or ".." segment (these are normalized away by and can escape the + /// intended object, e.g. turning DeleteObject into DeleteBucket), and keys containing + /// control characters. + /// + public static ErrorOr ValidateKey(string? key) + { + if (string.IsNullOrEmpty(key)) + return Error.Validation(S3ErrorCodes.InvalidKey, "Key is required."); + + var start = 0; + for (var i = 0; i <= key.Length; i++) + { + if (i == key.Length || key[i] == '/') + { + var segLen = i - start; + if (segLen == 0) + return Error.Validation(S3ErrorCodes.InvalidKey, "Key must not contain empty path segments."); + if (segLen == 1 && key[start] == '.') + return Error.Validation(S3ErrorCodes.InvalidKey, "Key must not contain '.' path segments."); + if (segLen == 2 && key[start] == '.' && key[start + 1] == '.') + return Error.Validation(S3ErrorCodes.InvalidKey, "Key must not contain '..' path segments."); + start = i + 1; + } + else if (char.IsControl(key[i])) + { + return Error.Validation(S3ErrorCodes.InvalidKey, "Key must not contain control characters."); + } + } + + return Result.Success; + } + + /// + /// Validates a bucket name against S3 naming rules: 3-63 characters, only lowercase + /// letters, digits, hyphens and dots, must start and end with a letter or digit, and + /// must not contain consecutive dots. + /// + public static ErrorOr ValidateBucketName(string? bucket) + { + if (string.IsNullOrEmpty(bucket)) + return Error.Validation(S3ErrorCodes.InvalidBucketName, "Bucket is required."); + + if (bucket.Length is < 3 or > 63) + return Error.Validation(S3ErrorCodes.InvalidBucketName, "Bucket name must be between 3 and 63 characters."); + + if (!IsBucketBoundaryChar(bucket[0]) || !IsBucketBoundaryChar(bucket[^1])) + return Error.Validation(S3ErrorCodes.InvalidBucketName, "Bucket name must start and end with a lowercase letter or digit."); + + for (var i = 0; i < bucket.Length; i++) + { + var ch = bucket[i]; + if (!IsBucketChar(ch)) + return Error.Validation(S3ErrorCodes.InvalidBucketName, "Bucket name may only contain lowercase letters, digits, hyphens and dots."); + + if (ch == '.' && i + 1 < bucket.Length && bucket[i + 1] == '.') + return Error.Validation(S3ErrorCodes.InvalidBucketName, "Bucket name must not contain consecutive dots."); + } + + return Result.Success; + } + + /// + /// Validates user-defined metadata. Each name is sent as an "x-amz-meta-{name}" header, so it + /// must be a valid HTTP token; invalid tokens would otherwise be silently dropped by the HTTP + /// stack. This mirrors the validation enforced by PutObjectBuilder.AddMetadata so that + /// directly-constructed requests are held to the same rules. + /// + public static ErrorOr ValidateMetadata(IReadOnlyDictionary? metadata) + { + if (metadata is null || metadata.Count == 0) + return Result.Success; + + foreach (var entry in metadata) + { + if (string.IsNullOrWhiteSpace(entry.Key) || !IsValidHttpToken(entry.Key)) + return Error.Validation( + S3ErrorCodes.InvalidMetadata, + $"Metadata name '{entry.Key}' is not a valid HTTP token. Names must not contain spaces, colons, control or separator characters."); + } + + return Result.Success; + } + + /// + /// Determines whether a string is a valid HTTP token per RFC 7230 (used for header names). + /// Invalid tokens would be silently dropped by the HTTP stack, so they are rejected up-front. + /// + public static bool IsValidHttpToken(string value) + { + foreach (var ch in value) + { + if (ch <= ' ' || ch >= 0x7F) + return false; + + switch (ch) + { + case '(' or ')' or '<' or '>' or '@' + or ',' or ';' or ':' or '\\' or '"' + or '/' or '[' or ']' or '?' or '=' + or '{' or '}': + return false; + } + } + + return true; + } + + private static bool IsBucketBoundaryChar(char ch) => + ch is (>= 'a' and <= 'z') or (>= '0' and <= '9'); + + private static bool IsBucketChar(char ch) => + ch is (>= 'a' and <= 'z') or (>= '0' and <= '9') or '-' or '.'; +} diff --git a/src/Clients/Goa.Clients.S3/S3ServiceClient.cs b/src/Clients/Goa.Clients.S3/S3ServiceClient.cs new file mode 100644 index 00000000..49ff6b94 --- /dev/null +++ b/src/Clients/Goa.Clients.S3/S3ServiceClient.cs @@ -0,0 +1,353 @@ +using ErrorOr; +using Goa.Clients.Core; +using Goa.Clients.Core.Http; +using Goa.Clients.Core.Logging; +using Goa.Clients.S3.Errors; +using Goa.Clients.S3.Operations.DeleteObject; +using Goa.Clients.S3.Operations.GetObject; +using Goa.Clients.S3.Operations.HeadObject; +using Goa.Clients.S3.Operations.PutObject; +using Microsoft.Extensions.Logging; +using System.Net; +using System.Net.Http.Headers; +using System.Text; + +namespace Goa.Clients.S3; + +internal sealed class S3ServiceClient : AwsServiceClient, IS3Client +{ + private const string MetadataHeaderPrefix = "x-amz-meta-"; + + private string? _cachedPathStyleBaseUrl; + + public S3ServiceClient( + IHttpClientFactory httpClientFactory, + S3ServiceClientConfiguration configuration, + ILogger logger) + : base(httpClientFactory, logger, configuration) + { + } + + public async Task> PutObjectAsync(PutObjectRequest request, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + var validation = ValidateBucketAndKey(request.Bucket, request.Key); + if (validation.IsError) + return validation.Errors; + + // Validate metadata for directly-constructed requests so invalid names cannot reach + // TryAddWithoutValidation (where they would be silently dropped), matching the builder. + var metadataValidation = S3RequestValidation.ValidateMetadata(request.Metadata); + if (metadataValidation.IsError) + return metadataValidation.Errors; + + try + { + using var requestMessage = CreateObjectRequest(HttpMethod.Put, request.Bucket, request.Key); + + var content = new ReadOnlyMemoryContent(request.Body); + if (!string.IsNullOrWhiteSpace(request.ContentType)) + content.Headers.ContentType = MediaTypeHeaderValue.Parse(request.ContentType); + requestMessage.Content = content; + + if (request.Body.Length > 0) + requestMessage.Options.Set(HttpOptions.Payload, request.Body); + + if (!string.IsNullOrWhiteSpace(request.ServerSideEncryption)) + requestMessage.Headers.TryAddWithoutValidation("x-amz-server-side-encryption", request.ServerSideEncryption); + + if (!string.IsNullOrWhiteSpace(request.SseKmsKeyId)) + requestMessage.Headers.TryAddWithoutValidation("x-amz-server-side-encryption-aws-kms-key-id", request.SseKmsKeyId); + + if (request.Metadata is { Count: > 0 }) + { + foreach (var entry in request.Metadata) + requestMessage.Headers.TryAddWithoutValidation(MetadataHeaderPrefix + entry.Key, entry.Value); + } + + using var response = await SendAsync(requestMessage, "PutObject", cancellationToken); + + if (!response.IsSuccessStatusCode) + return await ReadErrorAsync(response, "PutObject", cancellationToken); + + return new PutObjectResponse + { + ETag = GetETag(response) + }; + } + catch (Exception ex) + { + Logger.PutObjectFailed(ex, request.Bucket, request.Key); + return Error.Failure("S3.PutObject.Failed", $"Failed to put object {request.Key} to S3 bucket {request.Bucket}"); + } + } + + public async Task> GetObjectAsync(GetObjectRequest request, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + var validation = ValidateBucketAndKey(request.Bucket, request.Key); + if (validation.IsError) + return validation.Errors; + + try + { + using var requestMessage = CreateObjectRequest(HttpMethod.Get, request.Bucket, request.Key); + + if (!string.IsNullOrWhiteSpace(request.Range)) + requestMessage.Headers.TryAddWithoutValidation("Range", request.Range); + + using var response = await SendAsync(requestMessage, "GetObject", cancellationToken); + + if (!response.IsSuccessStatusCode) + return await ReadErrorAsync(response, "GetObject", cancellationToken); + + var body = await response.Content.ReadAsByteArrayAsync(cancellationToken); + + return new GetObjectResponse + { + Body = body, + ContentType = response.Content.Headers.ContentType?.ToString(), + ContentLength = response.Content.Headers.ContentLength ?? body.Length, + ETag = GetETag(response), + LastModified = response.Content.Headers.LastModified, + Metadata = ReadMetadata(response) + }; + } + catch (Exception ex) + { + Logger.GetObjectFailed(ex, request.Bucket, request.Key); + return Error.Failure("S3.GetObject.Failed", $"Failed to get object {request.Key} from S3 bucket {request.Bucket}"); + } + } + + public async Task> HeadObjectAsync(HeadObjectRequest request, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + var validation = ValidateBucketAndKey(request.Bucket, request.Key); + if (validation.IsError) + return validation.Errors; + + try + { + using var requestMessage = CreateObjectRequest(HttpMethod.Head, request.Bucket, request.Key); + using var response = await SendAsync(requestMessage, "HeadObject", cancellationToken); + + if (!response.IsSuccessStatusCode) + return await ReadErrorAsync(response, "HeadObject", cancellationToken); + + return new HeadObjectResponse + { + ContentType = response.Content.Headers.ContentType?.ToString(), + ContentLength = response.Content.Headers.ContentLength ?? 0, + ETag = GetETag(response), + LastModified = response.Content.Headers.LastModified, + Metadata = ReadMetadata(response) + }; + } + catch (Exception ex) + { + Logger.HeadObjectFailed(ex, request.Bucket, request.Key); + return Error.Failure("S3.HeadObject.Failed", $"Failed to head object {request.Key} in S3 bucket {request.Bucket}"); + } + } + + public async Task> DeleteObjectAsync(DeleteObjectRequest request, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + var validation = ValidateBucketAndKey(request.Bucket, request.Key); + if (validation.IsError) + return validation.Errors; + + try + { + using var requestMessage = CreateObjectRequest(HttpMethod.Delete, request.Bucket, request.Key); + using var response = await SendAsync(requestMessage, "DeleteObject", cancellationToken); + + // S3 returns 204 No Content even when the key does not exist on an unversioned bucket. + if (!response.IsSuccessStatusCode) + return await ReadErrorAsync(response, "DeleteObject", cancellationToken); + + return new DeleteObjectResponse(); + } + catch (Exception ex) + { + Logger.DeleteObjectFailed(ex, request.Bucket, request.Key); + return Error.Failure("S3.DeleteObject.Failed", $"Failed to delete object {request.Key} from S3 bucket {request.Bucket}"); + } + } + + /// + /// Creates an HTTP request message targeting the specified object, marked for S3-style SigV4 signing. + /// + private HttpRequestMessage CreateObjectRequest(HttpMethod method, string bucket, string key) + { + var requestMessage = new HttpRequestMessage(method, BuildObjectUri(bucket, key)); + + // S3 requires the canonical URI to be the once-encoded request path; other AWS services double-encode. + requestMessage.Options.Set(HttpOptions.UseSingleUriEncoding, true); + + return requestMessage; + } + + /// + /// Builds the request URI for an object using virtual-host style addressing by default, + /// or path-style addressing when a custom service URL is configured or ForcePathStyle is enabled. + /// The bucket name and key are assumed to have already been validated via + /// . + /// + private Uri BuildObjectUri(string bucket, string key) + { + var encodedKey = EncodeKey(key); + + if (!string.IsNullOrWhiteSpace(Configuration.ServiceUrl) || Configuration.ForcePathStyle) + { + var baseUrl = _cachedPathStyleBaseUrl ??= + Configuration.ServiceUrl?.TrimEnd('/') ?? $"https://s3.{Configuration.Region}.amazonaws.com"; + return new Uri($"{baseUrl}/{bucket}/{encodedKey}"); + } + + return new Uri($"https://{bucket}.s3.{Configuration.Region}.amazonaws.com/{encodedKey}"); + } + + /// + /// URI-encodes an object key for use as the S3 canonical URI, preserving '/' separators. + /// Each character is encoded exactly as the AWS SDKs encode S3 object keys: the RFC 3986 + /// unreserved set plus the sub-delimiters S3 leaves unescaped are passed through, everything + /// else is percent-encoded from its UTF-8 bytes. Matching the AWS encoding byte-for-byte keeps + /// the path Goa signs and sends identical to what AWS produces, so SigV4 validation succeeds. + /// + private static string EncodeKey(string key) + { + var requiresEncoding = false; + foreach (var ch in key) + { + if (!IsS3UnreservedPathChar(ch)) + { + requiresEncoding = true; + break; + } + } + + if (!requiresEncoding) + return key; + + var builder = new StringBuilder(key.Length + 16); + Span utf8 = stackalloc byte[4]; + foreach (var rune in key.EnumerateRunes()) + { + if (rune.IsAscii && IsS3UnreservedPathChar((char)rune.Value)) + { + builder.Append((char)rune.Value); + continue; + } + + var written = rune.EncodeToUtf8(utf8); + for (var i = 0; i < written; i++) + { + builder.Append('%'); + builder.Append(HexUpper[utf8[i] >> 4]); + builder.Append(HexUpper[utf8[i] & 0xF]); + } + } + + return builder.ToString(); + } + + private const string HexUpper = "0123456789ABCDEF"; + + /// + /// Characters that are left unescaped inside an S3 canonical-URI path segment. This is the + /// RFC 3986 unreserved set plus the sub-delimiters and '/' that the AWS SDKs preserve when + /// encoding S3 object keys. + /// + private static bool IsS3UnreservedPathChar(char ch) => + ch is (>= 'A' and <= 'Z') or (>= 'a' and <= 'z') or (>= '0' and <= '9') + or '-' or '.' or '_' or '~' or '/' + or '!' or '$' or '&' or '\'' or '(' or ')' or '*' or '+' or ',' or ';' or '='; + + /// + /// Validates the bucket name and object key before they are interpolated into the request URI. + /// Rejects malformed bucket names and dot-segment keys that would otherwise + /// normalize away (which can escape the intended object and target the bucket itself). + /// + private static ErrorOr ValidateBucketAndKey(string? bucket, string? key) + { + var bucketResult = S3RequestValidation.ValidateBucketName(bucket); + if (bucketResult.IsError) + return bucketResult.Errors; + + return S3RequestValidation.ValidateKey(key); + } + + /// + /// Reads an error response body and maps the S3 XML error document to an . + /// HEAD responses carry no body, so the status code alone is used in that case. + /// + private async Task ReadErrorAsync(HttpResponseMessage response, string operation, CancellationToken cancellationToken) + { + string? code = null; + string? message = null; + + using var errorBuffer = await ReadResponseBytesAsync(response, cancellationToken); + if (errorBuffer.Length > 0) + { + var errorPayload = Encoding.UTF8.GetString(errorBuffer.Span); + var xmlError = new XmlApiError(); + xmlError.DeserializeFromXml(errorPayload); + + if (!string.IsNullOrWhiteSpace(xmlError.Code)) + code = xmlError.Code; + if (!string.IsNullOrWhiteSpace(xmlError.Message)) + message = xmlError.Message; + } + + Logger.RequestFailed($"Operation: {operation}, StatusCode: {(int)response.StatusCode}, Code: {code ?? "Unknown"}, Message: {message ?? "Unknown"}"); + + if (response.StatusCode == HttpStatusCode.NotFound || + string.Equals(code, S3ErrorCodes.NoSuchKey, StringComparison.Ordinal) || + string.Equals(code, S3ErrorCodes.NoSuchBucket, StringComparison.Ordinal)) + { + return Error.NotFound($"S3.{code ?? "NotFound"}", message ?? "The specified resource does not exist."); + } + + if (response.StatusCode == HttpStatusCode.Forbidden) + { + return Error.Forbidden($"S3.{code ?? S3ErrorCodes.AccessDenied}", message ?? "Access denied."); + } + + return Error.Failure( + $"S3.{code ?? "Unknown"}", + message ?? $"S3 {operation} request failed with status code {(int)response.StatusCode}."); + } + + private static string? GetETag(HttpResponseMessage response) + { + if (response.Headers.TryGetValues("ETag", out var values)) + { + foreach (var value in values) + return value; + } + + return null; + } + + private static Dictionary? ReadMetadata(HttpResponseMessage response) + { + Dictionary? metadata = null; + + foreach (var header in response.Headers.NonValidated) + { + if (header.Key.StartsWith(MetadataHeaderPrefix, StringComparison.OrdinalIgnoreCase)) + { + metadata ??= new Dictionary(StringComparer.OrdinalIgnoreCase); + metadata[header.Key[MetadataHeaderPrefix.Length..]] = header.Value.ToString(); + } + } + + return metadata; + } +} diff --git a/src/Clients/Goa.Clients.S3/S3ServiceClientConfiguration.cs b/src/Clients/Goa.Clients.S3/S3ServiceClientConfiguration.cs new file mode 100644 index 00000000..032ea697 --- /dev/null +++ b/src/Clients/Goa.Clients.S3/S3ServiceClientConfiguration.cs @@ -0,0 +1,25 @@ +using Goa.Clients.Core.Configuration; + +namespace Goa.Clients.S3; + +/// +/// Configuration class for S3 service providing S3-specific settings. +/// +public sealed class S3ServiceClientConfiguration : AwsServiceConfiguration +{ + /// + /// Gets or sets a value indicating whether path-style addressing should be used + /// (e.g. https://s3.us-east-1.amazonaws.com/bucket/key) instead of the default + /// virtual-host style addressing (e.g. https://bucket.s3.us-east-1.amazonaws.com/key). + /// Path-style addressing is always used when is set. + /// + public bool ForcePathStyle { get; set; } + + /// + /// Initializes a new instance of the S3ServiceClientConfiguration class. + /// + public S3ServiceClientConfiguration() : base("s3") + { + ApiVersion = "2006-03-01"; + } +} diff --git a/src/Clients/Goa.Clients.S3/ServiceExtensions.cs b/src/Clients/Goa.Clients.S3/ServiceExtensions.cs new file mode 100644 index 00000000..ded6f39a --- /dev/null +++ b/src/Clients/Goa.Clients.S3/ServiceExtensions.cs @@ -0,0 +1,74 @@ +using Goa.Clients.Core.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; + +namespace Goa.Clients.S3; + +/// +/// Extension methods for configuring S3 services in dependency injection containers. +/// +public static class ServiceExtensions +{ + /// + /// Adds S3 client services to the specified service collection with custom endpoint configuration. + /// + /// The service collection to add services to. + /// The custom S3 service endpoint URL. + /// The AWS region to use for S3 operations. Defaults to us-east-1. + /// The minimum log level for S3 operations. Defaults to Information. + /// The service collection for method chaining. + public static IServiceCollection AddS3(this IServiceCollection services, string? serviceUrl = null, string? region = null, LogLevel logLevel = LogLevel.Information) + { + ArgumentNullException.ThrowIfNull(services); + + return services.AddS3(config => + { + config.ServiceUrl = serviceUrl; + config.Region = region ?? Environment.GetEnvironmentVariable("AWS_REGION") ?? "us-east-1"; + config.LogLevel = logLevel; + }); + } + + /// + /// Adds S3 client services to the specified service collection with configuration action. + /// + /// The service collection to add services to. + /// An action to configure the S3 service options. + /// The service collection for method chaining. + public static IServiceCollection AddS3(this IServiceCollection services, Action configureOptions) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configureOptions); + + var configuration = new S3ServiceClientConfiguration(); + configureOptions(configuration); + + if (string.IsNullOrWhiteSpace(configuration.Region) && string.IsNullOrWhiteSpace(configuration.ServiceUrl)) + throw new Exception("Either region or service url must be provided"); + + // SigV4 signing always needs a region, even when only a ServiceUrl is configured. Apply the + // same fallback as the other AddS3 overload so requests sign correctly when AWS_REGION is unset. + if (string.IsNullOrWhiteSpace(configuration.Region)) + configuration.Region = Environment.GetEnvironmentVariable("AWS_REGION") ?? "us-east-1"; + + return services.AddS3Core(configuration); + } + + private static IServiceCollection AddS3Core(this IServiceCollection services, S3ServiceClientConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + // Add the Goa service infrastructure for S3 + services.AddGoaService(nameof(S3ServiceClient)); + + // Register the configuration + services.TryAddSingleton(configuration); + + // Register the S3 client + services.TryAddTransient(); + + return services; + } +} diff --git a/tests/Clients/Goa.Clients.Core.Tests/Goa.Clients.Core.Tests.csproj b/tests/Clients/Goa.Clients.Core.Tests/Goa.Clients.Core.Tests.csproj index b8c9724f..96a49dd8 100644 --- a/tests/Clients/Goa.Clients.Core.Tests/Goa.Clients.Core.Tests.csproj +++ b/tests/Clients/Goa.Clients.Core.Tests/Goa.Clients.Core.Tests.csproj @@ -6,6 +6,7 @@ + diff --git a/tests/Clients/Goa.Clients.Core.Tests/Http/RequestSignerComparisonTests.cs b/tests/Clients/Goa.Clients.Core.Tests/Http/RequestSignerComparisonTests.cs index 38b88994..0a35006b 100644 --- a/tests/Clients/Goa.Clients.Core.Tests/Http/RequestSignerComparisonTests.cs +++ b/tests/Clients/Goa.Clients.Core.Tests/Http/RequestSignerComparisonTests.cs @@ -6,6 +6,7 @@ using Amazon.Runtime; using Amazon.Runtime.Internal; using Amazon.Runtime.Internal.Auth; +using Amazon.S3; using Amazon.SQS; namespace Goa.Clients.Core.Tests.Http; @@ -609,4 +610,117 @@ public async Task Debug_Signature_Differences_Simple_Case() // If signatures don't match, this helps debug the canonical request construction await Assert.That(goaSignature).IsEqualTo(awsSignature); } + + [Test] + public async Task RequestSigner_SingleUriEncoding_Matches_AwsSdk_For_PercentEncoded_S3_Path() + { + // Proves Goa's single-encoding (S3-style) signature is byte-identical to AWS SDK's signer + // for a key that requires percent-encoding including the sub-delimiters S3 leaves literal + // ('+', '(' , ')' and a space). The S3 client encodes object keys with the same character + // set the AWS SDKs use, so the canonical URI Goa signs matches AWS exactly. This guards + // against signing regressions that LocalStack's lax validation would not catch. + + // Arrange + const string key = "folder name/file+v1 (final).txt"; + + // Mirror the S3 client's key encoding by using the AWS SDK's S3 path encoder, which leaves + // sub-delimiters such as '+' '(' ')' literal. The resulting path is used as the Goa request URI. + var encodedPath = Amazon.Util.AWSSDKUtils.UrlEncode(key, true); + + var awsCredentials = new BasicAWSCredentials("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + var awsSigner = new AWS4Signer(); + + // Use credentials without a session token so the signed-header sets match the AWS SDK + // (the shared _goaSigner carries a session token, which would add x-amz-security-token). + var goaCredentials = new AwsCredentials("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", null, DateTime.UtcNow.AddHours(1)); + var mockCredProvider = new Mock(); + mockCredProvider.Setup(x => x.GetCredentialsAsync()) + .ReturnsAsync(ErrorOrFactory.From(goaCredentials)); + var goaSigner = new RequestSigner(mockCredProvider.Object); + + var awsRequest = new DefaultRequest(new Amazon.S3.Model.GetObjectRequest(), "S3"); + awsRequest.HttpMethod = "GET"; + awsRequest.Endpoint = new Uri("https://my-bucket.s3.us-east-1.amazonaws.com/"); + awsRequest.ResourcePath = "/" + key; // AWS SDK encodes this itself + + // AWS SDK signs with its own timestamp; capture it and reuse for Goa. + var awsSigningResult = awsSigner.SignRequest( + awsRequest, + new Amazon.S3.AmazonS3Config { RegionEndpoint = Amazon.RegionEndpoint.USEast1 }, + null, + awsCredentials.GetCredentials().AccessKey, + awsCredentials.GetCredentials().SecretKey); + var awsSignature = awsSigningResult.Signature; + + var awsTimestamp = awsRequest.Headers["X-Amz-Date"]; + var awsTime = DateTime.ParseExact(awsTimestamp, "yyyyMMddTHHmmssZ", null, System.Globalization.DateTimeStyles.AdjustToUniversal); + + var goaRequest = new HttpRequestMessage(HttpMethod.Get, $"https://my-bucket.s3.us-east-1.amazonaws.com/{encodedPath}"); + goaRequest.Options.Set(HttpOptions.Region, "us-east-1"); + goaRequest.Options.Set(HttpOptions.Service, "s3"); + goaRequest.Options.Set(HttpOptions.UseSingleUriEncoding, true); + goaRequest.Headers.Host = goaRequest.RequestUri?.Host; + goaRequest.Headers.TryAddWithoutValidation("X-Amz-Date", awsTimestamp); + goaRequest.Headers.TryAddWithoutValidation("X-Amz-Content-SHA256", awsRequest.Headers["X-Amz-Content-SHA256"]); + + // Act + var goaSignature = await goaSigner.SignRequestAsync(goaRequest, awsTime); + + // Assert + await Assert.That(goaSignature).IsEqualTo(awsSignature); + } + + [Test] + public async Task RequestSigner_SingleUriEncoding_Uses_Encoded_Path_AsIs() + { + // S3-style signing must use the once-encoded request path as the canonical URI. + // Without the option, the signer re-encodes '%' characters (double encoding), so the + // signatures for a percent-encoded path must differ between the two modes. + + // Arrange - path contains percent-encoded characters + var requestUri = "https://s3.us-east-1.amazonaws.com/my-bucket/folder%20name/object%2Bkey"; + + var doubleEncodedRequest = new HttpRequestMessage(HttpMethod.Get, requestUri); + doubleEncodedRequest.Options.Set(HttpOptions.Region, "us-east-1"); + doubleEncodedRequest.Options.Set(HttpOptions.Service, "s3"); + + var singleEncodedRequest = new HttpRequestMessage(HttpMethod.Get, requestUri); + singleEncodedRequest.Options.Set(HttpOptions.Region, "us-east-1"); + singleEncodedRequest.Options.Set(HttpOptions.Service, "s3"); + singleEncodedRequest.Options.Set(HttpOptions.UseSingleUriEncoding, true); + + // Act + var doubleEncodedSignature = await _goaSigner.SignRequestAsync(doubleEncodedRequest, _goaCredentials, _fixedDateTime); + var singleEncodedSignature = await _goaSigner.SignRequestAsync(singleEncodedRequest, _goaCredentials, _fixedDateTime); + + // Assert + await Assert.That(singleEncodedSignature).IsNotNull(); + await Assert.That(singleEncodedSignature).IsNotEqualTo(doubleEncodedSignature); + } + + [Test] + public async Task RequestSigner_SingleUriEncoding_Matches_Default_For_Unreserved_Paths() + { + // For paths containing only unreserved characters and '/', single and double encoding + // produce the same canonical URI, so the signatures must match. + + // Arrange + var requestUri = "https://my-bucket.s3.us-east-1.amazonaws.com/folder/object-key_1.txt"; + + var defaultRequest = new HttpRequestMessage(HttpMethod.Get, requestUri); + defaultRequest.Options.Set(HttpOptions.Region, "us-east-1"); + defaultRequest.Options.Set(HttpOptions.Service, "s3"); + + var singleEncodedRequest = new HttpRequestMessage(HttpMethod.Get, requestUri); + singleEncodedRequest.Options.Set(HttpOptions.Region, "us-east-1"); + singleEncodedRequest.Options.Set(HttpOptions.Service, "s3"); + singleEncodedRequest.Options.Set(HttpOptions.UseSingleUriEncoding, true); + + // Act + var defaultSignature = await _goaSigner.SignRequestAsync(defaultRequest, _goaCredentials, _fixedDateTime); + var singleEncodedSignature = await _goaSigner.SignRequestAsync(singleEncodedRequest, _goaCredentials, _fixedDateTime); + + // Assert + await Assert.That(singleEncodedSignature).IsEqualTo(defaultSignature); + } } diff --git a/tests/Clients/Goa.Clients.S3.Tests/Goa.Clients.S3.Tests.csproj b/tests/Clients/Goa.Clients.S3.Tests/Goa.Clients.S3.Tests.csproj new file mode 100644 index 00000000..c4907c66 --- /dev/null +++ b/tests/Clients/Goa.Clients.S3.Tests/Goa.Clients.S3.Tests.csproj @@ -0,0 +1,18 @@ + + + + Exe + + + + + + + + + + + + + + diff --git a/tests/Clients/Goa.Clients.S3.Tests/LocalStackFixture.cs b/tests/Clients/Goa.Clients.S3.Tests/LocalStackFixture.cs new file mode 100644 index 00000000..13e8798f --- /dev/null +++ b/tests/Clients/Goa.Clients.S3.Tests/LocalStackFixture.cs @@ -0,0 +1,49 @@ +using DotNet.Testcontainers.Builders; +using DotNet.Testcontainers.Containers; +using DotNet.Testcontainers.Images; +using TUnit.Core.Interfaces; + +namespace Goa.Clients.S3.Tests; + +public class LocalStackFixture : IAsyncInitializer, IAsyncDisposable +{ + private IContainer? _container; + + public string ServiceUrl { get; private set; } = string.Empty; + + public async Task InitializeAsync() + { + // Pinned to the v4 community line: localstack/localstack:latest (2026+) requires a licence. + _container = new ContainerBuilder(new DockerImage("localstack/localstack:4")) + .WithEnvironment("SERVICES", "s3") + // Ask LocalStack to validate SigV4 signatures rather than skipping them, so the + // integration tests exercise real request signing end-to-end. (Note: the v4 community + // image does not strictly enforce this for every request, but the signed requests are + // additionally proven correct against the AWS SDK in RequestSignerComparisonTests.) + .WithEnvironment("S3_SKIP_SIGNATURE_VALIDATION", "0") + .WithEnvironment("DEBUG", "1") + .WithEnvironment("DOCKER_HOST", "unix:///var/run/docker.sock") + .WithEnvironment("LOCALSTACK_HOST", "localhost") + .WithPortBinding(4566, true) + .WithWaitStrategy(Wait.ForUnixContainer() + .UntilHttpRequestIsSucceeded(r => r.ForPort(4566).ForPath("/_localstack/health"))) + .Build(); + + await _container.StartAsync(); + + ServiceUrl = $"http://localhost:{_container.GetMappedPublicPort(4566)}"; + + Console.WriteLine($"LocalStack started at: {ServiceUrl}"); + } + + public async ValueTask DisposeAsync() + { + if (_container != null) + { + await _container.StopAsync(); + await _container.DisposeAsync(); + } + + GC.SuppressFinalize(this); + } +} diff --git a/tests/Clients/Goa.Clients.S3.Tests/PutObjectBuilderTests.cs b/tests/Clients/Goa.Clients.S3.Tests/PutObjectBuilderTests.cs new file mode 100644 index 00000000..3cc76149 --- /dev/null +++ b/tests/Clients/Goa.Clients.S3.Tests/PutObjectBuilderTests.cs @@ -0,0 +1,35 @@ +using Goa.Clients.S3.Operations.PutObject; + +namespace Goa.Clients.S3.Tests; + +/// +/// Unit tests for . These do not require LocalStack. +/// +public class PutObjectBuilderTests +{ + [Test] + [Arguments("bad name")] // space + [Arguments("bad:name")] // colon + [Arguments("bad/name")] // separator + [Arguments("bad@name")] // separator + public async Task AddMetadata_WithInvalidHttpToken_Throws(string name) + { + var builder = new PutObjectBuilder(); + + await Assert.That(() => builder.AddMetadata(name, "value")) + .Throws(); + } + + [Test] + [Arguments("category")] + [Arguments("uploaded-by")] + [Arguments("x-custom_token.1")] + public async Task AddMetadata_WithValidHttpToken_Succeeds(string name) + { + var builder = new PutObjectBuilder(); + + var result = builder.AddMetadata(name, "value"); + + await Assert.That(result).IsSameReferenceAs(builder); + } +} diff --git a/tests/Clients/Goa.Clients.S3.Tests/S3ClientIntegrationTests.cs b/tests/Clients/Goa.Clients.S3.Tests/S3ClientIntegrationTests.cs new file mode 100644 index 00000000..09efccd2 --- /dev/null +++ b/tests/Clients/Goa.Clients.S3.Tests/S3ClientIntegrationTests.cs @@ -0,0 +1,398 @@ +using ErrorOr; +using Goa.Clients.S3.Operations.DeleteObject; +using Goa.Clients.S3.Operations.GetObject; +using Goa.Clients.S3.Operations.HeadObject; +using Goa.Clients.S3.Operations.PutObject; +using System.Text; + +namespace Goa.Clients.S3.Tests; + +[ClassDataSource(Shared = SharedType.PerAssembly)] +public class S3ClientIntegrationTests +{ + private readonly S3TestFixture _fixture; + + public S3ClientIntegrationTests(S3TestFixture fixture) + { + _fixture = fixture; + } + + private static string NewKey() => $"{Guid.NewGuid()}/{Guid.NewGuid()}/{Guid.NewGuid()}"; + + [Test] + public async Task PutObjectAsync_WithValidRequest_ShouldReturnETag() + { + // Arrange + var request = new PutObjectBuilder() + .WithBucket(_fixture.BucketName) + .WithKey(NewKey()) + .WithBody(Encoding.UTF8.GetBytes("Hello, S3!")) + .WithContentType("text/plain") + .Build(); + + // Act + var result = await _fixture.S3Client.PutObjectAsync(request); + + // Assert + await Assert.That(result.IsError).IsFalse(); + await Assert.That(result.Value.ETag).IsNotNull(); + } + + [Test] + public async Task GetObjectAsync_AfterPut_ShouldRoundTripBodyAndContentType() + { + // Arrange + var key = NewKey(); + var body = Encoding.UTF8.GetBytes("{\"message\":\"round trip\"}"); + + var putResult = await _fixture.S3Client.PutObjectAsync(new PutObjectRequest + { + Bucket = _fixture.BucketName, + Key = key, + Body = body, + ContentType = "application/json" + }); + + await Assert.That(putResult.IsError).IsFalse(); + + // Act + var result = await _fixture.S3Client.GetObjectAsync(new GetObjectRequest + { + Bucket = _fixture.BucketName, + Key = key + }); + + // Assert + await Assert.That(result.IsError).IsFalse(); + await Assert.That(result.Value.Body).IsEquivalentTo(body); + await Assert.That(result.Value.ContentType).IsEqualTo("application/json"); + await Assert.That(result.Value.ContentLength).IsEqualTo(body.Length); + await Assert.That(result.Value.ETag).IsNotNull(); + } + + [Test] + public async Task HeadObjectAsync_AfterPut_ShouldReturnSizeAndETag() + { + // Arrange + var key = NewKey(); + var body = Encoding.UTF8.GetBytes("head object content"); + + var putResult = await _fixture.S3Client.PutObjectAsync(new PutObjectRequest + { + Bucket = _fixture.BucketName, + Key = key, + Body = body, + ContentType = "text/plain" + }); + + await Assert.That(putResult.IsError).IsFalse(); + + // Act + var result = await _fixture.S3Client.HeadObjectAsync(new HeadObjectRequest + { + Bucket = _fixture.BucketName, + Key = key + }); + + // Assert + await Assert.That(result.IsError).IsFalse(); + await Assert.That(result.Value.ContentLength).IsEqualTo(body.Length); + await Assert.That(result.Value.ETag).IsEqualTo(putResult.Value.ETag); + await Assert.That(result.Value.LastModified).IsNotNull(); + } + + [Test] + public async Task GetObjectAsync_WithRange_ShouldReturnExactSlice() + { + // Arrange + var key = NewKey(); + var body = new byte[256]; + for (var i = 0; i < body.Length; i++) + body[i] = (byte)i; + + var putResult = await _fixture.S3Client.PutObjectAsync(new PutObjectRequest + { + Bucket = _fixture.BucketName, + Key = key, + Body = body + }); + + await Assert.That(putResult.IsError).IsFalse(); + + // Act + var result = await _fixture.S3Client.GetObjectAsync(new GetObjectRequest + { + Bucket = _fixture.BucketName, + Key = key, + Range = "bytes=0-63" + }); + + // Assert + await Assert.That(result.IsError).IsFalse(); + await Assert.That(result.Value.Body).IsEquivalentTo(body.AsSpan(0, 64).ToArray()); + } + + [Test] + public async Task GetObjectAsync_WithMissingKey_ShouldReturnNotFound() + { + // Act + var result = await _fixture.S3Client.GetObjectAsync(new GetObjectRequest + { + Bucket = _fixture.BucketName, + Key = NewKey() + }); + + // Assert + await Assert.That(result.IsError).IsTrue(); + await Assert.That(result.FirstError.Type).IsEqualTo(ErrorType.NotFound); + } + + [Test] + public async Task HeadObjectAsync_WithMissingKey_ShouldReturnNotFound() + { + // Act + var result = await _fixture.S3Client.HeadObjectAsync(new HeadObjectRequest + { + Bucket = _fixture.BucketName, + Key = NewKey() + }); + + // Assert + await Assert.That(result.IsError).IsTrue(); + await Assert.That(result.FirstError.Type).IsEqualTo(ErrorType.NotFound); + } + + [Test] + public async Task DeleteObjectAsync_WithExistingKey_ShouldSucceed() + { + // Arrange + var key = NewKey(); + + var putResult = await _fixture.S3Client.PutObjectAsync(new PutObjectRequest + { + Bucket = _fixture.BucketName, + Key = key, + Body = Encoding.UTF8.GetBytes("delete me") + }); + + await Assert.That(putResult.IsError).IsFalse(); + + // Act + var result = await _fixture.S3Client.DeleteObjectAsync(new DeleteObjectRequest + { + Bucket = _fixture.BucketName, + Key = key + }); + + // Assert + await Assert.That(result.IsError).IsFalse(); + + var getResult = await _fixture.S3Client.GetObjectAsync(new GetObjectRequest + { + Bucket = _fixture.BucketName, + Key = key + }); + + await Assert.That(getResult.IsError).IsTrue(); + await Assert.That(getResult.FirstError.Type).IsEqualTo(ErrorType.NotFound); + } + + [Test] + public async Task DeleteObjectAsync_WithMissingKey_ShouldSucceed() + { + // Act - S3 returns 204 even when the key does not exist on an unversioned bucket + var result = await _fixture.S3Client.DeleteObjectAsync(new DeleteObjectRequest + { + Bucket = _fixture.BucketName, + Key = NewKey() + }); + + // Assert + await Assert.That(result.IsError).IsFalse(); + } + + [Test] + public async Task PutObjectAsync_With4MBBody_ShouldRoundTrip() + { + // Arrange + var key = NewKey(); + var body = new byte[4 * 1024 * 1024]; + Random.Shared.NextBytes(body); + + var putResult = await _fixture.S3Client.PutObjectAsync(new PutObjectRequest + { + Bucket = _fixture.BucketName, + Key = key, + Body = body, + ContentType = "application/octet-stream" + }); + + await Assert.That(putResult.IsError).IsFalse(); + + // Act + var result = await _fixture.S3Client.GetObjectAsync(new GetObjectRequest + { + Bucket = _fixture.BucketName, + Key = key + }); + + // Assert + await Assert.That(result.IsError).IsFalse(); + await Assert.That(result.Value.ContentLength).IsEqualTo(body.Length); + await Assert.That(result.Value.Body).IsEquivalentTo(body); + } + + [Test] + public async Task PutObjectAsync_WithMetadata_ShouldRoundTripMetadata() + { + // Arrange + var key = NewKey(); + var request = new PutObjectBuilder() + .WithBucket(_fixture.BucketName) + .WithKey(key) + .WithBody(Encoding.UTF8.GetBytes("metadata content")) + .AddMetadata("category", "test") + .AddMetadata("origin", "integration") + .Build(); + + var putResult = await _fixture.S3Client.PutObjectAsync(request); + await Assert.That(putResult.IsError).IsFalse(); + + // Act + var result = await _fixture.S3Client.HeadObjectAsync(new HeadObjectRequest + { + Bucket = _fixture.BucketName, + Key = key + }); + + // Assert + await Assert.That(result.IsError).IsFalse(); + await Assert.That(result.Value.Metadata).IsNotNull(); + await Assert.That(result.Value.Metadata!["category"]).IsEqualTo("test"); + await Assert.That(result.Value.Metadata!["origin"]).IsEqualTo("integration"); + } + + [Test] + [Arguments("folder name/file+v1 (final).txt")] + [Arguments("emoji/日本語-ключ-clé.txt")] + public async Task PutAndGet_WithKeyRequiringPercentEncoding_ShouldRoundTrip(string keySuffix) + { + // Exercises the %XX key-encoding path end-to-end with signature validation ON. + // The unique prefix avoids collisions while keeping the encoded segment intact. + var key = $"{Guid.NewGuid():N}/{keySuffix}"; + var body = Encoding.UTF8.GetBytes("encoded key round trip"); + + var putResult = await _fixture.S3Client.PutObjectAsync(new PutObjectRequest + { + Bucket = _fixture.BucketName, + Key = key, + Body = body, + ContentType = "text/plain" + }); + + await Assert.That(putResult.IsError).IsFalse(); + + var result = await _fixture.S3Client.GetObjectAsync(new GetObjectRequest + { + Bucket = _fixture.BucketName, + Key = key + }); + + await Assert.That(result.IsError).IsFalse(); + await Assert.That(result.Value.Body).IsEquivalentTo(body); + } + + [Test] + public async Task PutObjectAsync_WithZeroByteBody_ShouldRoundTrip() + { + // Exercises the empty-payload-hash path with signature validation ON. + var key = NewKey(); + + var putResult = await _fixture.S3Client.PutObjectAsync(new PutObjectRequest + { + Bucket = _fixture.BucketName, + Key = key, + Body = ReadOnlyMemory.Empty + }); + + await Assert.That(putResult.IsError).IsFalse(); + + var result = await _fixture.S3Client.GetObjectAsync(new GetObjectRequest + { + Bucket = _fixture.BucketName, + Key = key + }); + + await Assert.That(result.IsError).IsFalse(); + await Assert.That(result.Value.ContentLength).IsEqualTo(0L); + await Assert.That(result.Value.Body.Length).IsEqualTo(0); + } + + [Test] + [Arguments("../escape")] + [Arguments("a/../b")] + [Arguments("a/./b")] + [Arguments("/leading")] + [Arguments("a//b")] + public async Task GetObjectAsync_WithDotSegmentKey_ShouldReturnValidationError_VirtualHost(string key) + { + var result = await _fixture.S3Client.GetObjectAsync(new GetObjectRequest + { + Bucket = _fixture.BucketName, + Key = key + }); + + await Assert.That(result.IsError).IsTrue(); + await Assert.That(result.FirstError.Type).IsEqualTo(ErrorType.Validation); + } + + [Test] + [Arguments("../escape")] + [Arguments("a/../b")] + [Arguments("a/./b")] + [Arguments("/leading")] + [Arguments("a//b")] + public async Task DeleteObjectAsync_WithDotSegmentKey_ShouldReturnValidationError_PathStyle(string key) + { + var client = _fixture.CreatePathStyleClient(); + + var result = await client.DeleteObjectAsync(new DeleteObjectRequest + { + Bucket = _fixture.BucketName, + Key = key + }); + + await Assert.That(result.IsError).IsTrue(); + await Assert.That(result.FirstError.Type).IsEqualTo(ErrorType.Validation); + } + + [Test] + public async Task PathStyleClient_PutAndGet_ShouldSucceed() + { + // Arrange + var client = _fixture.CreatePathStyleClient(); + var key = NewKey(); + var body = Encoding.UTF8.GetBytes("path-style addressing"); + + var putResult = await client.PutObjectAsync(new PutObjectRequest + { + Bucket = _fixture.BucketName, + Key = key, + Body = body, + ContentType = "text/plain" + }); + + await Assert.That(putResult.IsError).IsFalse(); + + // Act + var result = await client.GetObjectAsync(new GetObjectRequest + { + Bucket = _fixture.BucketName, + Key = key + }); + + // Assert + await Assert.That(result.IsError).IsFalse(); + await Assert.That(result.Value.Body).IsEquivalentTo(body); + } +} diff --git a/tests/Clients/Goa.Clients.S3.Tests/S3KeyValidationTests.cs b/tests/Clients/Goa.Clients.S3.Tests/S3KeyValidationTests.cs new file mode 100644 index 00000000..1c5ade99 --- /dev/null +++ b/tests/Clients/Goa.Clients.S3.Tests/S3KeyValidationTests.cs @@ -0,0 +1,161 @@ +using ErrorOr; +using Goa.Clients.Core.Configuration; +using Goa.Clients.S3.Operations.DeleteObject; +using Goa.Clients.S3.Operations.GetObject; +using Goa.Clients.S3.Operations.HeadObject; +using Goa.Clients.S3.Operations.PutObject; +using Microsoft.Extensions.DependencyInjection; + +namespace Goa.Clients.S3.Tests; + +public class S3KeyValidationTests +{ + private const string InvalidKeyErrorCode = "S3.InvalidKey"; + + // Kept alive for the lifetime of the test class - the client resolves an IHttpClientFactory from + // this provider, so it must not be disposed until all tests have run (see DisposeProvider). + private static readonly ServiceProvider Provider = BuildProvider(); + + // Points at an unroutable endpoint - invalid keys must be rejected before any request is sent. + private static readonly IS3Client Client = Provider.GetRequiredService(); + + private static ServiceProvider BuildProvider() + { + var services = new ServiceCollection(); + + services.AddLogging(); + services.AddStaticCredentials("test", "test"); + services.AddS3(config => + { + config.ServiceUrl = "http://127.0.0.1:1"; + config.Region = "us-east-1"; + }); + + return services.BuildServiceProvider(); + } + + [After(Class)] + public static async Task DisposeProvider() => await Provider.DisposeAsync(); + + [Test] + [Arguments(".")] + [Arguments("..")] + [Arguments("../etc/passwd")] + [Arguments("foo/../bar")] + [Arguments("foo/./bar")] + [Arguments("foo/..")] + [Arguments("/foo")] + [Arguments("foo//bar")] + [Arguments("foo/")] + public async Task PutObjectAsync_WithTraversalKey_ShouldReturnValidationError(string key) + { + // Act + var result = await Client.PutObjectAsync(new PutObjectRequest + { + Bucket = "test-bucket", + Key = key, + Body = new byte[] { 1 } + }); + + // Assert + await Assert.That(result.IsError).IsTrue(); + await Assert.That(result.FirstError.Type).IsEqualTo(ErrorType.Validation); + await Assert.That(result.FirstError.Code).IsEqualTo(InvalidKeyErrorCode); + } + + [Test] + [Arguments(".")] + [Arguments("..")] + [Arguments("../etc/passwd")] + [Arguments("foo/../bar")] + [Arguments("foo/./bar")] + [Arguments("foo/..")] + [Arguments("/foo")] + [Arguments("foo//bar")] + [Arguments("foo/")] + public async Task GetObjectAsync_WithTraversalKey_ShouldReturnValidationError(string key) + { + // Act + var result = await Client.GetObjectAsync(new GetObjectRequest + { + Bucket = "test-bucket", + Key = key + }); + + // Assert + await Assert.That(result.IsError).IsTrue(); + await Assert.That(result.FirstError.Type).IsEqualTo(ErrorType.Validation); + await Assert.That(result.FirstError.Code).IsEqualTo(InvalidKeyErrorCode); + } + + [Test] + [Arguments(".")] + [Arguments("..")] + [Arguments("../etc/passwd")] + [Arguments("foo/../bar")] + [Arguments("foo/./bar")] + [Arguments("foo/..")] + [Arguments("/foo")] + [Arguments("foo//bar")] + [Arguments("foo/")] + public async Task HeadObjectAsync_WithTraversalKey_ShouldReturnValidationError(string key) + { + // Act + var result = await Client.HeadObjectAsync(new HeadObjectRequest + { + Bucket = "test-bucket", + Key = key + }); + + // Assert + await Assert.That(result.IsError).IsTrue(); + await Assert.That(result.FirstError.Type).IsEqualTo(ErrorType.Validation); + await Assert.That(result.FirstError.Code).IsEqualTo(InvalidKeyErrorCode); + } + + [Test] + [Arguments(".")] + [Arguments("..")] + [Arguments("../etc/passwd")] + [Arguments("foo/../bar")] + [Arguments("foo/./bar")] + [Arguments("foo/..")] + [Arguments("/foo")] + [Arguments("foo//bar")] + [Arguments("foo/")] + public async Task DeleteObjectAsync_WithTraversalKey_ShouldReturnValidationError(string key) + { + // Act + var result = await Client.DeleteObjectAsync(new DeleteObjectRequest + { + Bucket = "test-bucket", + Key = key + }); + + // Assert + await Assert.That(result.IsError).IsTrue(); + await Assert.That(result.FirstError.Type).IsEqualTo(ErrorType.Validation); + await Assert.That(result.FirstError.Code).IsEqualTo(InvalidKeyErrorCode); + } + + [Test] + [Arguments(".hidden")] + [Arguments("..config")] + [Arguments("file.name.txt")] + [Arguments("v1.0/archive..tar.gz")] + [Arguments("a/b.c/d")] + public async Task GetObjectAsync_WithDottedButValidKey_ShouldNotReturnInvalidKeyError(string key) + { + // Act - the unroutable endpoint means a key that passes validation fails at transport instead + var result = await Client.GetObjectAsync(new GetObjectRequest + { + Bucket = "test-bucket", + Key = key + }); + + // Assert + await Assert.That(result.IsError).IsTrue(); + await Assert.That(result.FirstError.Code).IsNotEqualTo(InvalidKeyErrorCode); + await Assert.That(result.FirstError.Type).IsNotEqualTo(ErrorType.Validation); + } +} diff --git a/tests/Clients/Goa.Clients.S3.Tests/S3RequestValidationTests.cs b/tests/Clients/Goa.Clients.S3.Tests/S3RequestValidationTests.cs new file mode 100644 index 00000000..d4df13e9 --- /dev/null +++ b/tests/Clients/Goa.Clients.S3.Tests/S3RequestValidationTests.cs @@ -0,0 +1,89 @@ +using ErrorOr; +using Goa.Clients.S3; +using Goa.Clients.S3.Errors; + +namespace Goa.Clients.S3.Tests; + +/// +/// Pure unit tests for bucket name and object key validation. These do not require LocalStack. +/// +public class S3RequestValidationTests +{ + [Test] + [Arguments("../etc/passwd")] + [Arguments("..")] + [Arguments("a/../b")] + [Arguments("a/./b")] + [Arguments("./a")] + [Arguments("a/.")] + [Arguments("/leading")] + [Arguments("a//b")] + [Arguments("trailing/")] + [Arguments("")] + public async Task ValidateKey_WithDotSegmentOrEmptyKey_ReturnsValidationError(string key) + { + var result = S3RequestValidation.ValidateKey(key); + + await Assert.That(result.IsError).IsTrue(); + await Assert.That(result.FirstError.Type).IsEqualTo(ErrorType.Validation); + await Assert.That(result.FirstError.Code).IsEqualTo(S3ErrorCodes.InvalidKey); + } + + [Test] + public async Task ValidateKey_WithControlCharacter_ReturnsValidationError() + { + var key = "folder/name"; // embedded BEL control character + + var result = S3RequestValidation.ValidateKey(key); + + await Assert.That(result.IsError).IsTrue(); + await Assert.That(result.FirstError.Code).IsEqualTo(S3ErrorCodes.InvalidKey); + } + + [Test] + [Arguments("simple.txt")] + [Arguments("folder/sub/object.json")] + [Arguments("folder name/file+v1 (final).txt")] + [Arguments("a..b/c")] + [Arguments("...")] + [Arguments("café/naïve.txt")] + public async Task ValidateKey_WithValidKey_ReturnsSuccess(string key) + { + var result = S3RequestValidation.ValidateKey(key); + + await Assert.That(result.IsError).IsFalse(); + } + + [Test] + [Arguments("ab")] + [Arguments("this-bucket-name-is-far-too-long-to-be-valid-because-it-exceeds-63ch")] + [Arguments("MyBucket")] + [Arguments("bucket_name")] + [Arguments("bucket name")] + [Arguments("bucket/name")] + [Arguments("-bucket")] + [Arguments("bucket-")] + [Arguments(".bucket")] + [Arguments("bucket..name")] + [Arguments("")] + public async Task ValidateBucketName_WithInvalidName_ReturnsValidationError(string bucket) + { + var result = S3RequestValidation.ValidateBucketName(bucket); + + await Assert.That(result.IsError).IsTrue(); + await Assert.That(result.FirstError.Type).IsEqualTo(ErrorType.Validation); + await Assert.That(result.FirstError.Code).IsEqualTo(S3ErrorCodes.InvalidBucketName); + } + + [Test] + [Arguments("my-bucket")] + [Arguments("my.bucket.name")] + [Arguments("bucket123")] + [Arguments("123bucket")] + public async Task ValidateBucketName_WithValidName_ReturnsSuccess(string bucket) + { + var result = S3RequestValidation.ValidateBucketName(bucket); + + await Assert.That(result.IsError).IsFalse(); + } +} diff --git a/tests/Clients/Goa.Clients.S3.Tests/S3TestFixture.cs b/tests/Clients/Goa.Clients.S3.Tests/S3TestFixture.cs new file mode 100644 index 00000000..c64f185e --- /dev/null +++ b/tests/Clients/Goa.Clients.S3.Tests/S3TestFixture.cs @@ -0,0 +1,132 @@ +using Amazon.Runtime; +using Amazon.S3; +using Amazon.S3.Model; +using Goa.Clients.Core.Configuration; +using Microsoft.Extensions.DependencyInjection; +using TUnit.Core.Interfaces; + +namespace Goa.Clients.S3.Tests; + +public class S3TestFixture : IAsyncInitializer, IAsyncDisposable +{ + private LocalStackFixture _localStack = null!; + private ServiceProvider _serviceProvider = null!; + private ServiceProvider _pathStyleServiceProvider = null!; + private IS3Client _pathStyleClient = null!; + + public IS3Client S3Client { get; private set; } = null!; + + public string BucketName { get; } = $"s3-test-bucket-{Guid.NewGuid():N}"; + + public async Task InitializeAsync() + { + _localStack = new LocalStackFixture(); + await _localStack.InitializeAsync(); + + var services = new ServiceCollection(); + + services.AddLogging(); + services.AddStaticCredentials("test", "test"); + services.AddS3(config => + { + config.ServiceUrl = _localStack.ServiceUrl; + config.Region = "us-east-1"; + }); + + _serviceProvider = services.BuildServiceProvider(); + S3Client = _serviceProvider.GetRequiredService(); + + // Eagerly build the path-style client during single-threaded initialization so + // CreatePathStyleClient is safe to call from tests running in parallel under TUnit. + var pathStyleServices = new ServiceCollection(); + pathStyleServices.AddLogging(); + pathStyleServices.AddStaticCredentials("test", "test"); + pathStyleServices.AddS3(config => + { + config.ServiceUrl = _localStack.ServiceUrl; + config.Region = "us-east-1"; + config.ForcePathStyle = true; + }); + + _pathStyleServiceProvider = pathStyleServices.BuildServiceProvider(); + _pathStyleClient = _pathStyleServiceProvider.GetRequiredService(); + + await CreateTestBucketAsync(BucketName); + } + + public IS3Client CreatePathStyleClient() => _pathStyleClient; + + public async Task CreateTestBucketAsync(string bucketName) + { + var tempClient = new AmazonS3Client(new BasicAWSCredentials("XXX", "XXX"), new AmazonS3Config + { + ServiceURL = _localStack.ServiceUrl, + UseHttp = true, + ForcePathStyle = true + }); + + try + { + await tempClient.PutBucketAsync(new PutBucketRequest { BucketName = bucketName }); + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to create test S3 bucket: {ex.Message}", ex); + } + finally + { + tempClient.Dispose(); + } + } + + public async ValueTask DisposeAsync() + { + try + { + var tempClient = new AmazonS3Client(new BasicAWSCredentials("XXX", "XXX"), new AmazonS3Config + { + ServiceURL = _localStack.ServiceUrl, + UseHttp = true, + ForcePathStyle = true + }); + + try + { + string? continuationToken = null; + do + { + var objects = await tempClient.ListObjectsV2Async(new ListObjectsV2Request + { + BucketName = BucketName, + ContinuationToken = continuationToken + }); + + foreach (var s3Object in objects.S3Objects ?? []) + { + await tempClient.DeleteObjectAsync(BucketName, s3Object.Key); + } + + continuationToken = objects.IsTruncated == true ? objects.NextContinuationToken : null; + } + while (continuationToken is not null); + + await tempClient.DeleteBucketAsync(BucketName); + } + finally + { + tempClient.Dispose(); + } + } + catch (Exception ex) + { + Console.WriteLine($"Warning: Failed to delete test S3 bucket: {ex.Message}"); + } + + if (_localStack != null) + await _localStack.DisposeAsync(); + + _serviceProvider?.Dispose(); + _pathStyleServiceProvider?.Dispose(); + GC.SuppressFinalize(this); + } +}