diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml
index 902697b..fec1bcd 100644
--- a/.github/workflows/dotnet.yml
+++ b/.github/workflows/dotnet.yml
@@ -44,6 +44,20 @@ jobs:
PASV_MIN_PORT: 21000
PASV_MAX_PORT: 21010
+ localstack:
+ image: localstack/localstack:latest
+ ports:
+ - 4566:4566
+ options: >-
+ --health-cmd "curl -f http://localhost:4566/_localstack/health || exit 1"
+ --health-interval 10s
+ --health-timeout 5s
+ --health-retries 5
+ env:
+ SERVICES: s3
+ DEBUG: 0
+ EDGE_PORT: 4566
+
steps:
- uses: actions/checkout@v5
- name: Setup .NET
@@ -56,6 +70,9 @@ jobs:
run: dotnet format --verify-no-changes
- name: Build
run: dotnet build --no-restore
+ - name: Create S3 test bucket
+ run: |
+ docker exec $(docker ps -q -f ancestor=localstack/localstack:latest) awslocal s3 mb s3://test-bucket
- name: Test
run: dotnet test --no-build --verbosity normal
env:
@@ -69,3 +86,8 @@ jobs:
FTP_TEST_USER: testuser
FTP_TEST_PASS: testpass
FTP_TEST_ROOT: ""
+ S3_TEST_BUCKET: test-bucket
+ S3_TEST_ACCESS_KEY: test
+ S3_TEST_SECRET_KEY: test
+ S3_TEST_ENDPOINT: http://localhost:4566
+ S3_TEST_PREFIX: sharpsync-tests
diff --git a/CLAUDE.md b/CLAUDE.md
index ae1f1a8..6d910ae 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -33,18 +33,20 @@ dotnet test --logger trx --results-directory TestResults
```
#### Running Integration Tests
-Integration tests require external services (SFTP server). Use the provided scripts:
+Integration tests require external services (SFTP, FTP, S3 via LocalStack). Use the provided scripts:
```bash
-# Linux/macOS - automatically starts Docker SFTP server
+# Linux/macOS - automatically starts Docker test servers
./scripts/run-integration-tests.sh
-# Windows - automatically starts Docker SFTP server
+# Windows - automatically starts Docker test servers
.\scripts\run-integration-tests.ps1
# Or manually with Docker Compose
docker-compose -f docker-compose.test.yml up -d
export SFTP_TEST_HOST=localhost SFTP_TEST_PORT=2222 SFTP_TEST_USER=testuser SFTP_TEST_PASS=testpass SFTP_TEST_ROOT=/home/testuser/upload
+export FTP_TEST_HOST=localhost FTP_TEST_PORT=21 FTP_TEST_USER=testuser FTP_TEST_PASS=testpass FTP_TEST_ROOT=/
+export S3_TEST_BUCKET=test-bucket S3_TEST_ACCESS_KEY=test S3_TEST_SECRET_KEY=test S3_TEST_ENDPOINT=http://localhost:4566
dotnet test --verbosity normal
docker-compose -f docker-compose.test.yml down
```
@@ -67,7 +69,7 @@ dotnet pack --configuration Release --version-suffix preview
The project uses GitHub Actions for CI/CD. The pipeline currently:
- Builds on Ubuntu only (multi-platform testing planned)
- Runs tests with format checking
-- Includes SFTP and FTP integration tests using Docker-based servers
+- Includes SFTP, FTP, and S3 integration tests using Docker-based servers (LocalStack for S3)
- Automatically configures test environment variables for integration tests
## High-Level Architecture
@@ -89,7 +91,7 @@ SharpSync is a **pure .NET file synchronization library** with no native depende
- `WebDavStorage` - WebDAV with OAuth2, chunking, and platform-specific optimizations (implemented, needs tests)
- `SftpStorage` - SFTP with password and key-based authentication (fully implemented and tested)
- `FtpStorage` - FTP/FTPS with secure connections support (fully implemented and tested)
- - `StorageType` enum includes: S3 (planned for future versions)
+ - `S3Storage` - Amazon S3 and S3-compatible storage (MinIO, LocalStack) with multipart uploads (fully implemented and tested)
3. **Authentication** (`src/SharpSync/Auth/`)
- `IOAuth2Provider` - OAuth2 authentication abstraction (no UI dependencies)
@@ -109,14 +111,15 @@ SharpSync is a **pure .NET file synchronization library** with no native depende
### Key Features
-- **Multi-Protocol Support**: Local, WebDAV, SFTP, and FTP/FTPS storage (extensible to S3)
+- **Multi-Protocol Support**: Local, WebDAV, SFTP, FTP/FTPS, and S3-compatible storage
- **OAuth2 Authentication**: Full OAuth2 flow support without UI dependencies (WebDAV)
- **SSH Key & Password Auth**: Secure SFTP authentication with private keys or passwords
- **FTP/FTPS Support**: Plain FTP, explicit FTPS, and implicit FTPS with password authentication
+- **S3 Compatibility**: Full AWS S3 support plus S3-compatible services (MinIO, LocalStack, etc.)
- **Smart Conflict Resolution**: Rich conflict analysis for UI integration
- **Selective Sync**: Include/exclude patterns for files and folders
- **Progress Reporting**: Real-time progress events for UI binding
-- **Large File Support**: Chunked uploads with platform-specific optimizations
+- **Large File Support**: Chunked/multipart uploads with platform-specific optimizations
- **Network Resilience**: Retry logic and error handling with automatic reconnection
- **Parallel Processing**: Configurable parallelism with intelligent prioritization
@@ -127,6 +130,7 @@ SharpSync is a **pure .NET file synchronization library** with no native depende
- `WebDav.Client` (2.9.0) - WebDAV protocol
- `SSH.NET` (2025.1.0) - SFTP protocol implementation
- `FluentFTP` (52.0.2) - FTP/FTPS protocol implementation
+- `AWSSDK.S3` (3.7.*) - Amazon S3 and S3-compatible storage
- Target Framework: .NET 8.0
### Platform-Specific Optimizations
@@ -285,11 +289,11 @@ The core library is production-ready, but several critical items must be address
### π CAN DEFER TO v1.1+
-10. **~~SFTP~~/~~FTP~~/S3 Implementations** (SFTP β
DONE, FTP β
DONE)
+10. **~~SFTP~~/~~FTP~~/~~S3~~ Implementations** β
**ALL DONE!**
- β
SFTP now fully implemented with comprehensive tests
- β
FTP/FTPS now fully implemented with comprehensive tests
- - S3 remains future feature for v1.1+
- - StorageType enum prepared for future S3 implementation
+ - β
S3 now fully implemented with comprehensive tests and LocalStack integration
+ - All major storage backends are now complete!
11. **Performance Benchmarks**
- BenchmarkDotNet suite for sync operations
@@ -325,30 +329,28 @@ The core library is production-ready, but several critical items must be address
**Minimum Acceptance Criteria:**
- β
Core sync engine tested (achieved)
-- β οΈ All storage implementations tested (LocalFileStorage β
, SftpStorage β
, FtpStorage β
, WebDavStorage β)
+- β οΈ All storage implementations tested (LocalFileStorage β
, SftpStorage β
, FtpStorage β
, S3Storage β
, WebDavStorage β)
- β README matches actual API (completely wrong)
- β
No TODOs/FIXMEs in code (achieved)
- β Examples directory exists (missing)
-- β
Package metadata accurate (SFTP and FTP now implemented!)
-- β
Integration test infrastructure (Docker-based CI testing for SFTP and FTP)
+- β
Package metadata accurate (SFTP, FTP, and S3 now implemented!)
+- β
Integration test infrastructure (Docker-based CI testing for SFTP, FTP, and S3)
-**Current Score: 5/9 (56%)** - Improved from 33%! (FTP implementation complete)
+**Current Score: 5/9 (56%)** - S3 implementation complete!
### π― Post-v1.0 Roadmap (Future Versions)
-**v1.0** β
SFTP and FTP Implemented!
+**v1.0** β
All Major Storage Backends Implemented!
- β
SFTP storage implementation (DONE!)
- β
FTP/FTPS storage implementation (DONE!)
-- β
Integration test infrastructure with Docker for SFTP and FTP (DONE!)
+- β
S3 storage implementation with AWS S3 and S3-compatible services (DONE!)
+- β
Integration test infrastructure with Docker for SFTP, FTP, and S3/LocalStack (DONE!)
**v1.1**
- Code coverage reporting
- Performance benchmarks
- Multi-platform CI testing (Windows, macOS)
- Additional conflict resolution strategies
-
-**v1.2**
-- S3-compatible storage implementation
- Advanced filtering (regex support)
**v2.0**
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index ace699b..9404726 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -31,6 +31,24 @@ services:
timeout: 5s
retries: 5
+ localstack:
+ image: localstack/localstack:latest
+ ports:
+ - "4566:4566"
+ environment:
+ SERVICES: s3
+ DEBUG: 0
+ DATA_DIR: /tmp/localstack/data
+ EDGE_PORT: 4566
+ volumes:
+ - localstack-data:/tmp/localstack
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://localhost:4566/_localstack/health"]
+ interval: 10s
+ timeout: 5s
+ retries: 5
+
volumes:
sftp-data:
ftp-data:
+ localstack-data:
diff --git a/scripts/run-integration-tests.ps1 b/scripts/run-integration-tests.ps1
index cbf4da5..c33b81c 100644
--- a/scripts/run-integration-tests.ps1
+++ b/scripts/run-integration-tests.ps1
@@ -1,4 +1,4 @@
-# Script to run SharpSync integration tests with Docker-based SFTP server
+# Script to run SharpSync integration tests with Docker-based test servers (SFTP, FTP, S3/LocalStack)
# Usage: .\scripts\run-integration-tests.ps1 [-TestFilter "filter"]
param(
@@ -10,7 +10,7 @@ $ErrorActionPreference = "Stop"
$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
$ProjectRoot = Split-Path -Parent $ScriptDir
-Write-Host "π Starting SFTP test server..." -ForegroundColor Cyan
+Write-Host "π Starting test servers (SFTP, FTP, S3/LocalStack)..." -ForegroundColor Cyan
Set-Location $ProjectRoot
docker-compose -f docker-compose.test.yml up -d
@@ -32,7 +32,7 @@ while ($elapsed -lt $timeout) {
Write-Host " Waiting... ($elapsed`s/$timeout`s)" -ForegroundColor Gray
}
-# Final check
+# Final check for SFTP
if (-not $isHealthy) {
Write-Host "β SFTP server failed to become healthy within $timeout`s" -ForegroundColor Red
Write-Host "π Server logs:" -ForegroundColor Yellow
@@ -41,6 +41,42 @@ if (-not $isHealthy) {
exit 1
}
+Write-Host "β³ Waiting for FTP server to be ready..." -ForegroundColor Yellow
+$elapsed = 0
+$isHealthy = $false
+
+while ($elapsed -lt $timeout) {
+ $containerStatus = docker-compose -f docker-compose.test.yml ps ftp
+ if ($containerStatus -match "healthy") {
+ Write-Host "β
FTP server is ready" -ForegroundColor Green
+ $isHealthy = $true
+ break
+ }
+ Start-Sleep -Seconds 2
+ $elapsed += 2
+ Write-Host " Waiting... ($elapsed`s/$timeout`s)" -ForegroundColor Gray
+}
+
+Write-Host "β³ Waiting for LocalStack (S3) to be ready..." -ForegroundColor Yellow
+$elapsed = 0
+$isHealthy = $false
+
+while ($elapsed -lt $timeout) {
+ $containerStatus = docker-compose -f docker-compose.test.yml ps localstack
+ if ($containerStatus -match "healthy") {
+ Write-Host "β
LocalStack is ready" -ForegroundColor Green
+ $isHealthy = $true
+ break
+ }
+ Start-Sleep -Seconds 2
+ $elapsed += 2
+ Write-Host " Waiting... ($elapsed`s/$timeout`s)" -ForegroundColor Gray
+}
+
+# Create S3 test bucket in LocalStack
+Write-Host "π¦ Creating S3 test bucket..." -ForegroundColor Cyan
+docker-compose -f docker-compose.test.yml exec -T localstack awslocal s3 mb s3://test-bucket 2>$null
+
# Set environment variables for tests
$env:SFTP_TEST_HOST = "localhost"
$env:SFTP_TEST_PORT = "2222"
@@ -48,6 +84,18 @@ $env:SFTP_TEST_USER = "testuser"
$env:SFTP_TEST_PASS = "testpass"
$env:SFTP_TEST_ROOT = "/home/testuser/upload"
+$env:FTP_TEST_HOST = "localhost"
+$env:FTP_TEST_PORT = "21"
+$env:FTP_TEST_USER = "testuser"
+$env:FTP_TEST_PASS = "testpass"
+$env:FTP_TEST_ROOT = "/"
+
+$env:S3_TEST_BUCKET = "test-bucket"
+$env:S3_TEST_ACCESS_KEY = "test"
+$env:S3_TEST_SECRET_KEY = "test"
+$env:S3_TEST_ENDPOINT = "http://localhost:4566"
+$env:S3_TEST_PREFIX = "sharpsync-tests"
+
Write-Host "π§ͺ Running tests..." -ForegroundColor Cyan
# Run tests with optional filter
diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index 4f497a7..cdc5197 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -1,6 +1,6 @@
#!/bin/bash
-# Script to run SharpSync integration tests with Docker-based SFTP server
+# Script to run SharpSync integration tests with Docker-based test servers (SFTP, FTP, S3/LocalStack)
# Usage: ./scripts/run-integration-tests.sh [test-filter]
set -e
@@ -8,7 +8,7 @@ set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
-echo "π Starting SFTP test server..."
+echo "π Starting test servers (SFTP, FTP, S3/LocalStack)..."
docker-compose -f "$PROJECT_ROOT/docker-compose.test.yml" up -d
echo "β³ Waiting for SFTP server to be ready..."
@@ -25,7 +25,7 @@ while [ $ELAPSED -lt $TIMEOUT ]; do
echo " Waiting... (${ELAPSED}s/${TIMEOUT}s)"
done
-# Final check
+# Final check for SFTP
if ! docker-compose -f "$PROJECT_ROOT/docker-compose.test.yml" ps sftp | grep -q "healthy"; then
echo "β SFTP server failed to become healthy within ${TIMEOUT}s"
echo "π Server logs:"
@@ -34,6 +34,34 @@ if ! docker-compose -f "$PROJECT_ROOT/docker-compose.test.yml" ps sftp | grep -q
exit 1
fi
+echo "β³ Waiting for FTP server to be ready..."
+ELAPSED=0
+while [ $ELAPSED -lt $TIMEOUT ]; do
+ if docker-compose -f "$PROJECT_ROOT/docker-compose.test.yml" ps ftp | grep -q "healthy"; then
+ echo "β
FTP server is ready"
+ break
+ fi
+ sleep 2
+ ELAPSED=$((ELAPSED + 2))
+ echo " Waiting... (${ELAPSED}s/${TIMEOUT}s)"
+done
+
+echo "β³ Waiting for LocalStack (S3) to be ready..."
+ELAPSED=0
+while [ $ELAPSED -lt $TIMEOUT ]; do
+ if docker-compose -f "$PROJECT_ROOT/docker-compose.test.yml" ps localstack | grep -q "healthy"; then
+ echo "β
LocalStack is ready"
+ break
+ fi
+ sleep 2
+ ELAPSED=$((ELAPSED + 2))
+ echo " Waiting... (${ELAPSED}s/${TIMEOUT}s)"
+done
+
+# Create S3 test bucket in LocalStack
+echo "π¦ Creating S3 test bucket..."
+docker-compose -f "$PROJECT_ROOT/docker-compose.test.yml" exec -T localstack awslocal s3 mb s3://test-bucket 2>/dev/null || true
+
# Set environment variables for tests
export SFTP_TEST_HOST=localhost
export SFTP_TEST_PORT=2222
@@ -41,6 +69,18 @@ export SFTP_TEST_USER=testuser
export SFTP_TEST_PASS=testpass
export SFTP_TEST_ROOT=/home/testuser/upload
+export FTP_TEST_HOST=localhost
+export FTP_TEST_PORT=21
+export FTP_TEST_USER=testuser
+export FTP_TEST_PASS=testpass
+export FTP_TEST_ROOT=/
+
+export S3_TEST_BUCKET=test-bucket
+export S3_TEST_ACCESS_KEY=test
+export S3_TEST_SECRET_KEY=test
+export S3_TEST_ENDPOINT=http://localhost:4566
+export S3_TEST_PREFIX=sharpsync-tests
+
echo "π§ͺ Running tests..."
cd "$PROJECT_ROOT"
diff --git a/src/SharpSync/SharpSync.csproj b/src/SharpSync/SharpSync.csproj
index 423c89a..b9a39ac 100644
--- a/src/SharpSync/SharpSync.csproj
+++ b/src/SharpSync/SharpSync.csproj
@@ -15,14 +15,14 @@
AndrΓ© Polykanine
Oire Software
SharpSync
- A pure .NET file synchronization library supporting WebDAV, SFTP, FTP/FTPS, and local storage with conflict resolution, selective sync, and progress reporting.
+ A pure .NET file synchronization library supporting WebDAV, SFTP, FTP/FTPS, S3, and local storage with conflict resolution, selective sync, and progress reporting.
Copyright Β© 2025 AndrΓ© Polykanine, Oire Software
Apache-2.0
false
https://github.com/Oire/sharp-sync
https://github.com/Oire/sharp-sync
git
- file-sync;synchronization;webdav;sftp;ftp;ftps;nextcloud;owncloud;backup;sync
+ file-sync;synchronization;webdav;sftp;ftp;ftps;s3;aws;nextcloud;owncloud;backup;sync
1.0.0
false
@@ -39,6 +39,7 @@
+
diff --git a/src/SharpSync/Storage/S3Storage.cs b/src/SharpSync/Storage/S3Storage.cs
new file mode 100644
index 0000000..bbe6892
--- /dev/null
+++ b/src/SharpSync/Storage/S3Storage.cs
@@ -0,0 +1,787 @@
+using System.Security.Cryptography;
+using Amazon.Runtime;
+using Amazon.S3;
+using Amazon.S3.Model;
+using Amazon.S3.Transfer;
+using Oire.SharpSync.Core;
+
+namespace Oire.SharpSync.Storage;
+
+///
+/// Amazon S3 and S3-compatible storage implementation
+/// Provides file synchronization over S3 protocol with support for AWS S3, MinIO, LocalStack, and other S3-compatible services
+///
+public class S3Storage: ISyncStorage, IDisposable {
+ private readonly AmazonS3Client _client;
+ private readonly string _bucketName;
+ private readonly string _prefix;
+
+ // Configuration
+ private readonly int _chunkSize;
+ private readonly int _maxRetries;
+ private readonly TimeSpan _retryDelay;
+
+ private readonly SemaphoreSlim _transferSemaphore;
+ private bool _disposed;
+
+ ///
+ /// Gets the storage type (always returns )
+ ///
+ public StorageType StorageType => StorageType.S3;
+
+ ///
+ /// Gets the root path (prefix) within the S3 bucket
+ ///
+ public string RootPath { get; }
+
+ ///
+ /// Creates S3 storage with AWS credentials
+ ///
+ /// S3 bucket name
+ /// AWS access key ID
+ /// AWS secret access key
+ /// AWS region (e.g., "us-east-1")
+ /// Prefix (folder path) within the bucket
+ /// Optional AWS session token for temporary credentials
+ /// Chunk size for multipart uploads (default 10MB)
+ /// Maximum retry attempts (default 3)
+ public S3Storage(
+ string bucketName,
+ string accessKey,
+ string secretKey,
+ string region = "us-east-1",
+ string prefix = "",
+ string? sessionToken = null,
+ int chunkSizeBytes = 10 * 1024 * 1024,
+ int maxRetries = 3) {
+ if (string.IsNullOrWhiteSpace(bucketName)) {
+ throw new ArgumentException("Bucket name cannot be empty", nameof(bucketName));
+ }
+
+ if (string.IsNullOrWhiteSpace(accessKey)) {
+ throw new ArgumentException("Access key cannot be empty", nameof(accessKey));
+ }
+
+ if (string.IsNullOrWhiteSpace(secretKey)) {
+ throw new ArgumentException("Secret key cannot be empty", nameof(secretKey));
+ }
+
+ _bucketName = bucketName;
+ _prefix = NormalizePath(prefix);
+ RootPath = _prefix;
+
+ _chunkSize = chunkSizeBytes;
+ _maxRetries = maxRetries;
+ _retryDelay = TimeSpan.FromSeconds(1);
+
+ // Create AWS credentials
+ AWSCredentials credentials = string.IsNullOrEmpty(sessionToken)
+ ? new BasicAWSCredentials(accessKey, secretKey)
+ : new SessionAWSCredentials(accessKey, secretKey, sessionToken);
+
+ // Create S3 client configuration
+ var config = new AmazonS3Config {
+ RegionEndpoint = Amazon.RegionEndpoint.GetBySystemName(region),
+ Timeout = TimeSpan.FromSeconds(300),
+ MaxErrorRetry = maxRetries
+ };
+
+ _client = new AmazonS3Client(credentials, config);
+ _transferSemaphore = new SemaphoreSlim(10, 10); // Allow up to 10 concurrent transfers
+ }
+
+ ///
+ /// Creates S3 storage with custom endpoint (for S3-compatible services like MinIO, LocalStack)
+ ///
+ /// S3 bucket name
+ /// Access key ID
+ /// Secret access key
+ /// Service endpoint URL (e.g., "http://localhost:9000" for MinIO)
+ /// Prefix (folder path) within the bucket
+ /// Force path-style URLs (required for MinIO and some S3-compatible services)
+ /// Chunk size for multipart uploads (default 10MB)
+ /// Maximum retry attempts (default 3)
+ public S3Storage(
+ string bucketName,
+ string accessKey,
+ string secretKey,
+ Uri serviceUrl,
+ string prefix = "",
+ bool forcePathStyle = true,
+ int chunkSizeBytes = 10 * 1024 * 1024,
+ int maxRetries = 3) {
+ if (string.IsNullOrWhiteSpace(bucketName)) {
+ throw new ArgumentException("Bucket name cannot be empty", nameof(bucketName));
+ }
+
+ if (string.IsNullOrWhiteSpace(accessKey)) {
+ throw new ArgumentException("Access key cannot be empty", nameof(accessKey));
+ }
+
+ if (string.IsNullOrWhiteSpace(secretKey)) {
+ throw new ArgumentException("Secret key cannot be empty", nameof(secretKey));
+ }
+
+ ArgumentNullException.ThrowIfNull(serviceUrl);
+
+ _bucketName = bucketName;
+ _prefix = NormalizePath(prefix);
+ RootPath = _prefix;
+
+ _chunkSize = chunkSizeBytes;
+ _maxRetries = maxRetries;
+ _retryDelay = TimeSpan.FromSeconds(1);
+
+ // Create AWS credentials
+ var credentials = new BasicAWSCredentials(accessKey, secretKey);
+
+ // Create S3 client configuration for custom endpoint
+ var config = new AmazonS3Config {
+ ServiceURL = serviceUrl.ToString(),
+ ForcePathStyle = forcePathStyle,
+ Timeout = TimeSpan.FromSeconds(300),
+ MaxErrorRetry = maxRetries
+ };
+
+ _client = new AmazonS3Client(credentials, config);
+ _transferSemaphore = new SemaphoreSlim(10, 10);
+ }
+
+ ///
+ /// Event raised when upload/download progress changes
+ ///
+ public event EventHandler? ProgressChanged;
+
+ ///
+ /// Tests the connection to the S3 service
+ ///
+ /// Cancellation token to cancel the operation
+ /// True if connection is successful and bucket is accessible, false otherwise
+ public async Task TestConnectionAsync(CancellationToken cancellationToken = default) {
+ try {
+ // Try to list objects with max keys 1 to verify bucket access
+ var request = new ListObjectsV2Request {
+ BucketName = _bucketName,
+ MaxKeys = 1,
+ Prefix = _prefix
+ };
+
+ await _client.ListObjectsV2Async(request, cancellationToken);
+ return true;
+ } catch {
+ return false;
+ }
+ }
+
+ ///
+ /// Lists all items (objects and "directories") in the specified path
+ ///
+ /// The relative path to list items from
+ /// Cancellation token to cancel the operation
+ /// A collection of sync items representing objects and directories
+ /// Thrown when authentication fails
+ public async Task> ListItemsAsync(string path, CancellationToken cancellationToken = default) {
+ var items = new List();
+ var directories = new HashSet();
+
+ return await ExecuteWithRetry(async () => {
+ // Build the S3 prefix with trailing slash for directory listing
+ string listPrefix;
+ if (string.IsNullOrEmpty(path)) {
+ // Listing root - use prefix with trailing slash if prefix exists
+ listPrefix = string.IsNullOrEmpty(_prefix) ? "" : _prefix + "/";
+ } else {
+ // Listing subdirectory - ensure trailing slash
+ var fullPath = GetFullPath(path);
+ listPrefix = fullPath.EndsWith('/') ? fullPath : fullPath + "/";
+ }
+
+ var request = new ListObjectsV2Request {
+ BucketName = _bucketName,
+ Prefix = listPrefix,
+ Delimiter = "/" // Use delimiter to get directory-like structure
+ };
+
+ ListObjectsV2Response? response;
+ do {
+ response = await _client.ListObjectsV2Async(request, cancellationToken);
+
+ // Add files (objects)
+ foreach (var s3Object in response.S3Objects) {
+ cancellationToken.ThrowIfCancellationRequested();
+
+ // Skip directory marker objects (keys ending with '/')
+ if (s3Object.Key.EndsWith('/')) {
+ continue;
+ }
+
+ var relativePath = GetRelativePath(s3Object.Key);
+
+ items.Add(new SyncItem {
+ Path = relativePath,
+ IsDirectory = false,
+ Size = s3Object.Size,
+ LastModified = s3Object.LastModified.ToUniversalTime(),
+ ETag = s3Object.ETag?.Trim('"'),
+ MimeType = GetMimeType(s3Object.Key),
+ Metadata = new Dictionary {
+ ["StorageClass"] = s3Object.StorageClass?.Value ?? "STANDARD"
+ }
+ });
+ }
+
+ // Add directories (common prefixes)
+ foreach (var commonPrefix in response.CommonPrefixes) {
+ cancellationToken.ThrowIfCancellationRequested();
+
+ // CommonPrefix includes trailing slash, remove it for relative path
+ var relativePath = GetRelativePath(commonPrefix.TrimEnd('/'));
+
+ // Avoid duplicates using HashSet.Add's return value (CA1868)
+ if (directories.Add(relativePath)) {
+ items.Add(new SyncItem {
+ Path = relativePath,
+ IsDirectory = true,
+ Size = 0,
+ LastModified = DateTime.UtcNow, // S3 doesn't track directory timestamps
+ MimeType = null
+ });
+ }
+ }
+
+ request.ContinuationToken = response.NextContinuationToken;
+ } while (response.IsTruncated);
+
+ return (IEnumerable)items;
+ }, cancellationToken);
+ }
+
+ ///
+ /// Gets metadata for a specific item (object or directory)
+ ///
+ /// The relative path to the item
+ /// Cancellation token to cancel the operation
+ /// The sync item if it exists, null otherwise
+ public async Task GetItemAsync(string path, CancellationToken cancellationToken = default) {
+ var fullPath = GetFullPath(path);
+
+ return await ExecuteWithRetry(async () => {
+ try {
+ // Try to get object metadata
+ var request = new GetObjectMetadataRequest {
+ BucketName = _bucketName,
+ Key = fullPath
+ };
+
+ var response = await _client.GetObjectMetadataAsync(request, cancellationToken);
+
+ return new SyncItem {
+ Path = path,
+ IsDirectory = false,
+ Size = response.ContentLength,
+ LastModified = response.LastModified.ToUniversalTime(),
+ ETag = response.ETag?.Trim('"'),
+ MimeType = response.Headers.ContentType,
+ Metadata = new Dictionary {
+ ["StorageClass"] = response.StorageClass?.Value ?? "STANDARD"
+ }
+ };
+ } catch (AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound) {
+ // Object doesn't exist, check if it's a directory (prefix)
+ var listRequest = new ListObjectsV2Request {
+ BucketName = _bucketName,
+ Prefix = fullPath.EndsWith('/') ? fullPath : fullPath + "/",
+ MaxKeys = 1
+ };
+
+ var listResponse = await _client.ListObjectsV2Async(listRequest, cancellationToken);
+
+ if (listResponse.S3Objects.Count > 0 || listResponse.CommonPrefixes.Count > 0) {
+ return new SyncItem {
+ Path = path,
+ IsDirectory = true,
+ Size = 0,
+ LastModified = DateTime.UtcNow,
+ MimeType = null
+ };
+ }
+
+ return null;
+ }
+ }, cancellationToken);
+ }
+
+ ///
+ /// Reads the contents of an object from S3
+ ///
+ /// The relative path to the object
+ /// Cancellation token to cancel the operation
+ /// A stream containing the object contents
+ /// Thrown when the object does not exist
+ /// Thrown when attempting to read a directory as a file
+ /// Thrown when authentication fails
+ public async Task ReadFileAsync(string path, CancellationToken cancellationToken = default) {
+ var fullPath = GetFullPath(path);
+
+ return await ExecuteWithRetry(async () => {
+ try {
+ var request = new GetObjectRequest {
+ BucketName = _bucketName,
+ Key = fullPath
+ };
+
+ var response = await _client.GetObjectAsync(request, cancellationToken);
+
+ // Read the entire stream into memory
+ var memoryStream = new MemoryStream();
+ var totalBytes = response.ContentLength;
+ var bytesRead = 0L;
+
+ var buffer = new byte[_chunkSize];
+ int read;
+
+ while ((read = await response.ResponseStream.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken)) > 0) {
+ await memoryStream.WriteAsync(new ReadOnlyMemory(buffer, 0, read), cancellationToken);
+ bytesRead += read;
+
+ if (totalBytes > _chunkSize) {
+ RaiseProgressChanged(path, bytesRead, totalBytes, StorageOperation.Download);
+ }
+ }
+
+ memoryStream.Position = 0;
+ return (Stream)memoryStream;
+ } catch (AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound) {
+ throw new FileNotFoundException($"File not found: {path}");
+ }
+ }, cancellationToken);
+ }
+
+ ///
+ /// Writes content to an object in S3, creating parent "directories" as needed
+ ///
+ /// The relative path to the object
+ /// The stream containing the object content to write
+ /// Cancellation token to cancel the operation
+ /// Thrown when authentication fails
+ ///
+ /// If the object already exists, it will be overwritten. For large files,
+ /// multipart upload is used automatically with progress reporting
+ ///
+ public async Task WriteFileAsync(string path, Stream content, CancellationToken cancellationToken = default) {
+ var fullPath = GetFullPath(path);
+
+ await _transferSemaphore.WaitAsync(cancellationToken);
+ try {
+ await ExecuteWithRetry(async () => {
+ var fileSize = content.CanSeek ? content.Length : -1;
+
+ if (fileSize >= 0 && fileSize > _chunkSize) {
+ // Use multipart upload for large files
+ using var transferUtility = new TransferUtility(_client);
+
+ var uploadRequest = new TransferUtilityUploadRequest {
+ BucketName = _bucketName,
+ Key = fullPath,
+ InputStream = content,
+ PartSize = _chunkSize,
+ AutoCloseStream = false
+ };
+
+ // Track progress
+ uploadRequest.UploadProgressEvent += (sender, args) => {
+ RaiseProgressChanged(path, args.TransferredBytes, args.TotalBytes, StorageOperation.Upload);
+ };
+
+ await transferUtility.UploadAsync(uploadRequest, cancellationToken);
+ } else {
+ // Use simple put for small files
+ var putRequest = new PutObjectRequest {
+ BucketName = _bucketName,
+ Key = fullPath,
+ InputStream = content,
+ AutoCloseStream = false
+ };
+
+ await _client.PutObjectAsync(putRequest, cancellationToken);
+
+ if (fileSize > 0) {
+ RaiseProgressChanged(path, fileSize, fileSize, StorageOperation.Upload);
+ }
+ }
+
+ return true;
+ }, cancellationToken);
+ } finally {
+ _transferSemaphore.Release();
+ }
+ }
+
+ ///
+ /// Creates a "directory" in S3 by creating a marker object
+ ///
+ /// The relative path to the directory to create
+ /// Cancellation token to cancel the operation
+ ///
+ /// S3 doesn't have real directories, but we create a zero-byte object with a trailing slash
+ /// to simulate directory structure. This is optional in S3 as paths are just key prefixes
+ ///
+ public async Task CreateDirectoryAsync(string path, CancellationToken cancellationToken = default) {
+ // In S3, directories are virtual - they don't need to be explicitly created
+ // However, we can create a marker object if needed
+ var fullPath = GetFullPath(path);
+
+ // Skip if path is empty or root
+ if (string.IsNullOrEmpty(fullPath)) {
+ return;
+ }
+
+ await ExecuteWithRetry(async () => {
+ // Ensure path ends with /
+ var directoryKey = fullPath.EndsWith('/') ? fullPath : fullPath + "/";
+
+ // Check if marker already exists
+ try {
+ var headRequest = new GetObjectMetadataRequest {
+ BucketName = _bucketName,
+ Key = directoryKey
+ };
+
+ await _client.GetObjectMetadataAsync(headRequest, cancellationToken);
+ return true; // Marker already exists
+ } catch (AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound) {
+ // Marker doesn't exist, create it
+ var putRequest = new PutObjectRequest {
+ BucketName = _bucketName,
+ Key = directoryKey,
+ InputStream = new MemoryStream(Array.Empty()),
+ ContentType = "application/x-directory"
+ };
+
+ await _client.PutObjectAsync(putRequest, cancellationToken);
+ return true;
+ }
+ }, cancellationToken);
+ }
+
+ ///
+ /// Deletes an object or "directory" from S3
+ ///
+ /// The relative path to the object or directory to delete
+ /// Cancellation token to cancel the operation
+ ///
+ /// If the path is a directory, all objects under that prefix will be deleted recursively
+ ///
+ public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) {
+ var fullPath = GetFullPath(path);
+
+ await ExecuteWithRetry(async () => {
+ // First, try to get the item to determine if it's a file or directory
+ var item = await GetItemAsync(path, cancellationToken);
+
+ if (item == null) {
+ return true; // Already deleted
+ }
+
+ if (item.IsDirectory) {
+ // Delete all objects with this prefix
+ await DeleteDirectoryRecursive(fullPath, cancellationToken);
+ } else {
+ // Delete single object
+ var deleteRequest = new DeleteObjectRequest {
+ BucketName = _bucketName,
+ Key = fullPath
+ };
+
+ await _client.DeleteObjectAsync(deleteRequest, cancellationToken);
+ }
+
+ return true;
+ }, cancellationToken);
+ }
+
+ ///
+ /// Recursively deletes all objects under a prefix
+ ///
+ private async Task DeleteDirectoryRecursive(string prefix, CancellationToken cancellationToken) {
+ var directoryPrefix = prefix.EndsWith('/') ? prefix : prefix + "/";
+
+ var listRequest = new ListObjectsV2Request {
+ BucketName = _bucketName,
+ Prefix = directoryPrefix
+ };
+
+ ListObjectsV2Response? response;
+ do {
+ response = await _client.ListObjectsV2Async(listRequest, cancellationToken);
+
+ if (response.S3Objects.Count > 0) {
+ // Delete objects in batches (S3 allows up to 1000 objects per request)
+ var deleteRequest = new DeleteObjectsRequest {
+ BucketName = _bucketName,
+ Objects = response.S3Objects.Select(obj => new KeyVersion { Key = obj.Key }).ToList()
+ };
+
+ await _client.DeleteObjectsAsync(deleteRequest, cancellationToken);
+ }
+
+ listRequest.ContinuationToken = response.NextContinuationToken;
+ } while (response.IsTruncated);
+
+ // Also delete the directory marker if it exists
+ try {
+ var deleteMarkerRequest = new DeleteObjectRequest {
+ BucketName = _bucketName,
+ Key = directoryPrefix
+ };
+
+ await _client.DeleteObjectAsync(deleteMarkerRequest, cancellationToken);
+ } catch (AmazonS3Exception) {
+ // Ignore if marker doesn't exist
+ }
+ }
+
+ ///
+ /// Moves or renames an object in S3
+ ///
+ /// The relative path to the source object
+ /// The relative path to the target location
+ /// Cancellation token to cancel the operation
+ /// Thrown when the source does not exist
+ ///
+ /// S3 doesn't have a native move operation, so this copies the object and deletes the source
+ ///
+ public async Task MoveAsync(string sourcePath, string targetPath, CancellationToken cancellationToken = default) {
+ var sourceFullPath = GetFullPath(sourcePath);
+ var targetFullPath = GetFullPath(targetPath);
+
+ await ExecuteWithRetry(async () => {
+ // Check if source exists
+ try {
+ var headRequest = new GetObjectMetadataRequest {
+ BucketName = _bucketName,
+ Key = sourceFullPath
+ };
+
+ await _client.GetObjectMetadataAsync(headRequest, cancellationToken);
+ } catch (AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound) {
+ throw new FileNotFoundException($"Source not found: {sourcePath}");
+ }
+
+ // Copy object to new location
+ var copyRequest = new CopyObjectRequest {
+ SourceBucket = _bucketName,
+ SourceKey = sourceFullPath,
+ DestinationBucket = _bucketName,
+ DestinationKey = targetFullPath
+ };
+
+ await _client.CopyObjectAsync(copyRequest, cancellationToken);
+
+ // Delete source object
+ var deleteRequest = new DeleteObjectRequest {
+ BucketName = _bucketName,
+ Key = sourceFullPath
+ };
+
+ await _client.DeleteObjectAsync(deleteRequest, cancellationToken);
+
+ return true;
+ }, cancellationToken);
+ }
+
+ ///
+ /// Checks whether an object or directory exists in S3
+ ///
+ /// The relative path to check
+ /// Cancellation token to cancel the operation
+ /// True if the object or directory exists, false otherwise
+ public async Task ExistsAsync(string path, CancellationToken cancellationToken = default) {
+ var item = await GetItemAsync(path, cancellationToken);
+ return item != null;
+ }
+
+ ///
+ /// Gets storage space information for the S3 bucket
+ ///
+ /// Cancellation token to cancel the operation
+ /// Storage information (returns -1 for unknown values as S3 doesn't provide quota information via API)
+ ///
+ /// S3 doesn't provide bucket size or quota information through standard APIs.
+ /// To get accurate bucket size, you would need to sum all object sizes, which is expensive.
+ /// This method returns -1 for both total and used space
+ ///
+ public async Task GetStorageInfoAsync(CancellationToken cancellationToken = default) {
+ // S3 doesn't provide bucket-level quota/size information
+ // We could calculate it by listing all objects, but that's expensive
+ // Return unknown values
+ return await Task.FromResult(new StorageInfo {
+ TotalSpace = -1,
+ UsedSpace = -1
+ });
+ }
+
+ ///
+ /// Computes the SHA256 hash of an object in S3
+ ///
+ /// The relative path to the object
+ /// Cancellation token to cancel the operation
+ /// Base64-encoded SHA256 hash of the object contents
+ /// Thrown when the object does not exist
+ ///
+ /// S3 ETags are MD5 hashes for simple uploads, but for multipart uploads they use a different algorithm.
+ /// This method downloads the object and computes SHA256 locally for consistency with other storage implementations
+ ///
+ public async Task ComputeHashAsync(string path, CancellationToken cancellationToken = default) {
+ // S3 ETag is MD5-based and complex for multipart uploads
+ // Download and compute SHA256 for consistency
+ using var stream = await ReadFileAsync(path, cancellationToken);
+ using var sha256 = SHA256.Create();
+
+ var hashBytes = await sha256.ComputeHashAsync(stream, cancellationToken);
+ return Convert.ToBase64String(hashBytes);
+ }
+
+ #region Helper Methods
+
+ ///
+ /// Normalizes a path for S3 (removes leading/trailing slashes)
+ ///
+ private static string NormalizePath(string path) {
+ if (string.IsNullOrWhiteSpace(path)) {
+ return "";
+ }
+
+ // Convert backslashes to forward slashes
+ path = path.Replace('\\', '/');
+
+ // Remove leading and trailing slashes
+ path = path.Trim('/');
+
+ return path;
+ }
+
+ ///
+ /// Gets the full S3 key (path with prefix)
+ ///
+ private string GetFullPath(string relativePath) {
+ if (string.IsNullOrEmpty(relativePath) || relativePath == "/") {
+ return _prefix;
+ }
+
+ relativePath = NormalizePath(relativePath);
+
+ if (string.IsNullOrEmpty(_prefix)) {
+ return relativePath;
+ }
+
+ return $"{_prefix}/{relativePath}";
+ }
+
+ ///
+ /// Gets the relative path from a full S3 key
+ ///
+ private string GetRelativePath(string fullKey) {
+ if (string.IsNullOrEmpty(_prefix)) {
+ return fullKey;
+ }
+
+ var prefix = _prefix + "/";
+ if (fullKey.StartsWith(prefix)) {
+ return fullKey.Substring(prefix.Length);
+ }
+
+ return fullKey;
+ }
+
+ ///
+ /// Gets MIME type based on file extension
+ ///
+ private static string GetMimeType(string key) {
+ var extension = Path.GetExtension(key).ToLowerInvariant();
+ return extension switch {
+ ".txt" => "text/plain",
+ ".pdf" => "application/pdf",
+ ".jpg" or ".jpeg" => "image/jpeg",
+ ".png" => "image/png",
+ ".gif" => "image/gif",
+ ".zip" => "application/zip",
+ ".json" => "application/json",
+ ".xml" => "application/xml",
+ ".html" or ".htm" => "text/html",
+ ".css" => "text/css",
+ ".js" => "application/javascript",
+ ".mp3" => "audio/mpeg",
+ ".mp4" => "video/mp4",
+ ".doc" => "application/msword",
+ ".docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ ".xls" => "application/vnd.ms-excel",
+ ".xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+ _ => "application/octet-stream"
+ };
+ }
+
+ ///
+ /// Executes an operation with retry logic
+ ///
+ private async Task ExecuteWithRetry(Func> operation, CancellationToken cancellationToken) {
+ Exception? lastException = null;
+
+ for (int attempt = 0; attempt <= _maxRetries; attempt++) {
+ try {
+ cancellationToken.ThrowIfCancellationRequested();
+ return await operation();
+ } catch (Exception ex) when (attempt < _maxRetries && IsRetriableException(ex)) {
+ lastException = ex;
+ await Task.Delay(_retryDelay * (attempt + 1), cancellationToken);
+ }
+ }
+
+ throw lastException ?? new InvalidOperationException("Operation failed");
+ }
+
+ ///
+ /// Determines if an exception is retriable
+ ///
+ private static bool IsRetriableException(Exception ex) {
+ // Retry on network errors, timeouts, and throttling
+ return ex is AmazonS3Exception s3Ex &&
+ (s3Ex.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable ||
+ s3Ex.StatusCode == System.Net.HttpStatusCode.RequestTimeout ||
+ s3Ex.ErrorCode == "RequestTimeout" ||
+ s3Ex.ErrorCode == "SlowDown" ||
+ s3Ex.ErrorCode == "InternalError");
+ }
+
+ ///
+ /// Raises progress changed event
+ ///
+ private void RaiseProgressChanged(string path, long completed, long total, StorageOperation operation) {
+ ProgressChanged?.Invoke(this, new StorageProgressEventArgs {
+ Path = path,
+ BytesTransferred = completed,
+ TotalBytes = total,
+ Operation = operation,
+ PercentComplete = total > 0 ? (int)((completed * 100L) / total) : 0
+ });
+ }
+
+ #endregion
+
+ #region IDisposable
+
+ ///
+ /// Releases all resources used by the S3 storage instance
+ ///
+ public void Dispose() {
+ if (!_disposed) {
+ _client?.Dispose();
+ _transferSemaphore?.Dispose();
+ _disposed = true;
+ }
+
+ GC.SuppressFinalize(this);
+ }
+
+ #endregion
+}
diff --git a/tests/SharpSync.Tests/Storage/S3StorageTests.cs b/tests/SharpSync.Tests/Storage/S3StorageTests.cs
new file mode 100644
index 0000000..3b3a395
--- /dev/null
+++ b/tests/SharpSync.Tests/Storage/S3StorageTests.cs
@@ -0,0 +1,573 @@
+namespace Oire.SharpSync.Tests.Storage;
+
+///
+/// Unit and integration tests for S3Storage
+/// NOTE: Integration tests require an S3-compatible service (AWS S3 or LocalStack). Set up environment variables:
+/// - S3_TEST_BUCKET: S3 bucket name
+/// - S3_TEST_ACCESS_KEY: AWS access key ID
+/// - S3_TEST_SECRET_KEY: AWS secret access key
+/// - S3_TEST_REGION: AWS region (default: us-east-1) - optional for LocalStack
+/// - S3_TEST_ENDPOINT: Custom endpoint URL (e.g., http://localhost:4566 for LocalStack)
+/// - S3_TEST_PREFIX: Prefix (folder path) within bucket (default: sharpsync-tests)
+///
+public class S3StorageTests: IDisposable {
+ private readonly string? _testBucket;
+ private readonly string? _testAccessKey;
+ private readonly string? _testSecretKey;
+ private readonly string _testRegion;
+ private readonly string? _testEndpoint;
+ private readonly string _testPrefix;
+ private readonly bool _integrationTestsEnabled;
+ private S3Storage? _storage;
+
+ public S3StorageTests() {
+ // Read environment variables for integration tests
+ _testBucket = Environment.GetEnvironmentVariable("S3_TEST_BUCKET");
+ _testAccessKey = Environment.GetEnvironmentVariable("S3_TEST_ACCESS_KEY");
+ _testSecretKey = Environment.GetEnvironmentVariable("S3_TEST_SECRET_KEY");
+ _testRegion = Environment.GetEnvironmentVariable("S3_TEST_REGION") ?? "us-east-1";
+ _testEndpoint = Environment.GetEnvironmentVariable("S3_TEST_ENDPOINT");
+ _testPrefix = Environment.GetEnvironmentVariable("S3_TEST_PREFIX") ?? "sharpsync-tests";
+
+ _integrationTestsEnabled = !string.IsNullOrEmpty(_testBucket) &&
+ !string.IsNullOrEmpty(_testAccessKey) &&
+ !string.IsNullOrEmpty(_testSecretKey);
+ }
+
+ public void Dispose() {
+ _storage?.Dispose();
+ }
+
+ #region Unit Tests (No S3 Service Required)
+
+ [Fact]
+ public void Constructor_AwsRegion_ValidParameters_CreatesStorage() {
+ // Act
+ using var storage = new S3Storage("test-bucket", "access-key", "secret-key", "us-east-1");
+
+ // Assert
+ Assert.Equal(StorageType.S3, storage.StorageType);
+ }
+
+ [Fact]
+ public void Constructor_AwsRegion_EmptyBucket_ThrowsException() {
+ // Act & Assert
+ Assert.Throws(() => new S3Storage("", "access-key", "secret-key"));
+ }
+
+ [Fact]
+ public void Constructor_AwsRegion_EmptyAccessKey_ThrowsException() {
+ // Act & Assert
+ Assert.Throws(() => new S3Storage("test-bucket", "", "secret-key"));
+ }
+
+ [Fact]
+ public void Constructor_AwsRegion_EmptySecretKey_ThrowsException() {
+ // Act & Assert
+ Assert.Throws(() => new S3Storage("test-bucket", "access-key", ""));
+ }
+
+ [Fact]
+ public void Constructor_CustomEndpoint_ValidParameters_CreatesStorage() {
+ // Arrange
+ var endpoint = new Uri("http://localhost:4566");
+
+ // Act
+ using var storage = new S3Storage("test-bucket", "access-key", "secret-key", endpoint);
+
+ // Assert
+ Assert.Equal(StorageType.S3, storage.StorageType);
+ }
+
+ [Fact]
+ public void Constructor_CustomEndpoint_NullEndpoint_ThrowsException() {
+ // Act & Assert
+ Assert.Throws(() => new S3Storage("test-bucket", "access-key", "secret-key", (Uri)null!));
+ }
+
+ [Fact]
+ public void RootPath_Property_ReturnsCorrectPath() {
+ // Arrange
+ var prefix = "test/path";
+ using var storage = new S3Storage("test-bucket", "access-key", "secret-key", prefix: prefix);
+
+ // Assert
+ Assert.Equal(prefix, storage.RootPath);
+ }
+
+ [Fact]
+ public void RootPath_Property_NormalizesPath() {
+ // Arrange
+ var prefix = "/test/path/";
+ using var storage = new S3Storage("test-bucket", "access-key", "secret-key", prefix: prefix);
+
+ // Assert - should remove leading and trailing slashes
+ Assert.Equal("test/path", storage.RootPath);
+ }
+
+ [Fact]
+ public void StorageType_Property_ReturnsS3() {
+ // Arrange
+ using var storage = new S3Storage("test-bucket", "access-key", "secret-key");
+
+ // Assert
+ Assert.Equal(StorageType.S3, storage.StorageType);
+ }
+
+ #endregion
+
+ #region Integration Tests (Require S3 Service)
+
+ private void SkipIfIntegrationTestsDisabled() {
+ if (!_integrationTestsEnabled) {
+ throw new SkipException("Integration tests disabled. Set S3_TEST_BUCKET, S3_TEST_ACCESS_KEY, and S3_TEST_SECRET_KEY environment variables.");
+ }
+ }
+
+ private S3Storage CreateStorage() {
+ SkipIfIntegrationTestsDisabled();
+
+ // Use unique prefix for each test to avoid conflicts
+ var uniquePrefix = $"{_testPrefix}/{Guid.NewGuid()}";
+
+ if (!string.IsNullOrEmpty(_testEndpoint)) {
+ // Custom endpoint (LocalStack, MinIO, etc.)
+ return new S3Storage(_testBucket!, _testAccessKey!, _testSecretKey!, new Uri(_testEndpoint), uniquePrefix);
+ } else {
+ // AWS S3
+ return new S3Storage(_testBucket!, _testAccessKey!, _testSecretKey!, _testRegion, uniquePrefix);
+ }
+ }
+
+ [Fact]
+ public async Task TestConnectionAsync_ValidCredentials_ReturnsTrue() {
+ SkipIfIntegrationTestsDisabled();
+
+ // Arrange
+ using var storage = CreateStorage();
+
+ // Act
+ var result = await storage.TestConnectionAsync();
+
+ // Assert
+ Assert.True(result);
+ }
+
+ [Fact]
+ public async Task CreateDirectoryAsync_CreatesDirectoryMarker() {
+ // Arrange
+ _storage = CreateStorage();
+ var dirPath = "test/subdir";
+
+ // Act
+ await _storage.CreateDirectoryAsync(dirPath);
+ var exists = await _storage.ExistsAsync(dirPath);
+
+ // Assert
+ Assert.True(exists);
+ }
+
+ [Fact]
+ public async Task WriteFileAsync_CreatesObject() {
+ // Arrange
+ _storage = CreateStorage();
+ var filePath = "test.txt";
+ var content = "Hello, S3 World!";
+
+ // Act
+ using var stream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(content));
+ await _storage.WriteFileAsync(filePath, stream);
+
+ // Assert
+ var exists = await _storage.ExistsAsync(filePath);
+ Assert.True(exists);
+ }
+
+ [Fact]
+ public async Task WriteFileAsync_LargeFile_UsesMultipartUpload() {
+ // Arrange
+ _storage = CreateStorage();
+ var filePath = "large_file.bin";
+
+ // Create a 15MB file (larger than default chunk size of 10MB)
+ var largeContent = new byte[15 * 1024 * 1024];
+ new Random().NextBytes(largeContent);
+
+ // Act
+ using var stream = new MemoryStream(largeContent);
+ await _storage.WriteFileAsync(filePath, stream);
+
+ // Assert
+ var exists = await _storage.ExistsAsync(filePath);
+ Assert.True(exists);
+
+ var item = await _storage.GetItemAsync(filePath);
+ Assert.NotNull(item);
+ Assert.Equal(largeContent.Length, item.Size);
+ }
+
+ [Fact]
+ public async Task ReadFileAsync_ReturnsObjectContent() {
+ // Arrange
+ _storage = CreateStorage();
+ var filePath = "test_read.txt";
+ var content = "Hello, S3 World!";
+
+ using var writeStream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(content));
+ await _storage.WriteFileAsync(filePath, writeStream);
+
+ // Act
+ using var readStream = await _storage.ReadFileAsync(filePath);
+ using var reader = new StreamReader(readStream);
+ var readContent = await reader.ReadToEndAsync();
+
+ // Assert
+ Assert.Equal(content, readContent);
+ }
+
+ [Fact]
+ public async Task ReadFileAsync_NonexistentFile_ThrowsFileNotFoundException() {
+ // Arrange
+ _storage = CreateStorage();
+ var filePath = "nonexistent.txt";
+
+ // Act & Assert
+ await Assert.ThrowsAsync(async () => await _storage.ReadFileAsync(filePath));
+ }
+
+ [Fact]
+ public async Task ListItemsAsync_EmptyDirectory_ReturnsEmpty() {
+ // Arrange
+ _storage = CreateStorage();
+
+ // Act
+ var items = await _storage.ListItemsAsync("");
+
+ // Assert
+ Assert.Empty(items);
+ }
+
+ [Fact]
+ public async Task ListItemsAsync_WithFiles_ReturnsFiles() {
+ // Arrange
+ _storage = CreateStorage();
+
+ // Create test files
+ await CreateTestFile(_storage, "file1.txt", "Content 1");
+ await CreateTestFile(_storage, "file2.txt", "Content 2");
+ await CreateTestFile(_storage, "subdir/file3.txt", "Content 3");
+
+ // Act
+ var items = (await _storage.ListItemsAsync("")).ToList();
+
+ // Assert
+ Assert.Contains(items, i => i.Path == "file1.txt" && !i.IsDirectory);
+ Assert.Contains(items, i => i.Path == "file2.txt" && !i.IsDirectory);
+ Assert.Contains(items, i => i.Path == "subdir" && i.IsDirectory);
+ }
+
+ [Fact]
+ public async Task ListItemsAsync_Subdirectory_ReturnsOnlySubdirectoryContents() {
+ // Arrange
+ _storage = CreateStorage();
+
+ // Create test files
+ await CreateTestFile(_storage, "root_file.txt", "Root");
+ await CreateTestFile(_storage, "subdir/file1.txt", "Content 1");
+ await CreateTestFile(_storage, "subdir/file2.txt", "Content 2");
+
+ // Act
+ var items = (await _storage.ListItemsAsync("subdir")).ToList();
+
+ // Assert
+ Assert.Equal(2, items.Count);
+ Assert.All(items, item => Assert.False(item.IsDirectory));
+ Assert.Contains(items, i => i.Path == "subdir/file1.txt");
+ Assert.Contains(items, i => i.Path == "subdir/file2.txt");
+ }
+
+ [Fact]
+ public async Task GetItemAsync_ExistingFile_ReturnsMetadata() {
+ // Arrange
+ _storage = CreateStorage();
+ var filePath = "test_metadata.txt";
+ var content = "Test metadata";
+
+ using var stream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(content));
+ await _storage.WriteFileAsync(filePath, stream);
+
+ // Act
+ var item = await _storage.GetItemAsync(filePath);
+
+ // Assert
+ Assert.NotNull(item);
+ Assert.Equal(filePath, item.Path);
+ Assert.False(item.IsDirectory);
+ Assert.Equal(content.Length, item.Size);
+ Assert.NotNull(item.ETag);
+ }
+
+ [Fact]
+ public async Task GetItemAsync_NonexistentFile_ReturnsNull() {
+ // Arrange
+ _storage = CreateStorage();
+ var filePath = "nonexistent.txt";
+
+ // Act
+ var item = await _storage.GetItemAsync(filePath);
+
+ // Assert
+ Assert.Null(item);
+ }
+
+ [Fact]
+ public async Task GetItemAsync_Directory_ReturnsDirectoryMetadata() {
+ // Arrange
+ _storage = CreateStorage();
+ var dirPath = "test_dir";
+
+ await _storage.CreateDirectoryAsync(dirPath);
+
+ // Act
+ var item = await _storage.GetItemAsync(dirPath);
+
+ // Assert
+ Assert.NotNull(item);
+ Assert.True(item.IsDirectory);
+ Assert.Equal(0, item.Size);
+ }
+
+ [Fact]
+ public async Task ExistsAsync_ExistingFile_ReturnsTrue() {
+ // Arrange
+ _storage = CreateStorage();
+ var filePath = "exists_test.txt";
+
+ await CreateTestFile(_storage, filePath, "Test");
+
+ // Act
+ var exists = await _storage.ExistsAsync(filePath);
+
+ // Assert
+ Assert.True(exists);
+ }
+
+ [Fact]
+ public async Task ExistsAsync_NonexistentFile_ReturnsFalse() {
+ // Arrange
+ _storage = CreateStorage();
+ var filePath = "does_not_exist.txt";
+
+ // Act
+ var exists = await _storage.ExistsAsync(filePath);
+
+ // Assert
+ Assert.False(exists);
+ }
+
+ [Fact]
+ public async Task DeleteAsync_ExistingFile_DeletesFile() {
+ // Arrange
+ _storage = CreateStorage();
+ var filePath = "delete_test.txt";
+
+ await CreateTestFile(_storage, filePath, "Test");
+
+ // Act
+ await _storage.DeleteAsync(filePath);
+
+ // Assert
+ var exists = await _storage.ExistsAsync(filePath);
+ Assert.False(exists);
+ }
+
+ [Fact]
+ public async Task DeleteAsync_Directory_DeletesAllContents() {
+ // Arrange
+ _storage = CreateStorage();
+ var dirPath = "delete_dir";
+
+ await CreateTestFile(_storage, $"{dirPath}/file1.txt", "Content 1");
+ await CreateTestFile(_storage, $"{dirPath}/file2.txt", "Content 2");
+ await CreateTestFile(_storage, $"{dirPath}/subdir/file3.txt", "Content 3");
+
+ // Act
+ await _storage.DeleteAsync(dirPath);
+
+ // Assert
+ var exists1 = await _storage.ExistsAsync($"{dirPath}/file1.txt");
+ var exists2 = await _storage.ExistsAsync($"{dirPath}/file2.txt");
+ var exists3 = await _storage.ExistsAsync($"{dirPath}/subdir/file3.txt");
+
+ Assert.False(exists1);
+ Assert.False(exists2);
+ Assert.False(exists3);
+ }
+
+ [Fact]
+ public async Task DeleteAsync_NonexistentFile_CompletesSuccessfully() {
+ // Arrange
+ _storage = CreateStorage();
+ var filePath = "does_not_exist.txt";
+
+ // Act & Assert - should not throw
+ await _storage.DeleteAsync(filePath);
+ }
+
+ [Fact]
+ public async Task MoveAsync_ExistingFile_MovesFile() {
+ // Arrange
+ _storage = CreateStorage();
+ var sourcePath = "source.txt";
+ var targetPath = "target.txt";
+ var content = "Move test";
+
+ await CreateTestFile(_storage, sourcePath, content);
+
+ // Act
+ await _storage.MoveAsync(sourcePath, targetPath);
+
+ // Assert
+ var sourceExists = await _storage.ExistsAsync(sourcePath);
+ var targetExists = await _storage.ExistsAsync(targetPath);
+
+ Assert.False(sourceExists);
+ Assert.True(targetExists);
+
+ // Verify content
+ using var stream = await _storage.ReadFileAsync(targetPath);
+ using var reader = new StreamReader(stream);
+ var readContent = await reader.ReadToEndAsync();
+ Assert.Equal(content, readContent);
+ }
+
+ [Fact]
+ public async Task MoveAsync_NonexistentFile_ThrowsFileNotFoundException() {
+ // Arrange
+ _storage = CreateStorage();
+ var sourcePath = "does_not_exist.txt";
+ var targetPath = "target.txt";
+
+ // Act & Assert
+ await Assert.ThrowsAsync(async () => await _storage.MoveAsync(sourcePath, targetPath));
+ }
+
+ [Fact]
+ public async Task ComputeHashAsync_ExistingFile_ReturnsHash() {
+ // Arrange
+ _storage = CreateStorage();
+ var filePath = "hash_test.txt";
+ var content = "Test hash computation";
+
+ await CreateTestFile(_storage, filePath, content);
+
+ // Act
+ var hash = await _storage.ComputeHashAsync(filePath);
+
+ // Assert
+ Assert.NotNull(hash);
+ Assert.NotEmpty(hash);
+
+ // Verify hash is consistent
+ var hash2 = await _storage.ComputeHashAsync(filePath);
+ Assert.Equal(hash, hash2);
+ }
+
+ [Fact]
+ public async Task ComputeHashAsync_NonexistentFile_ThrowsFileNotFoundException() {
+ // Arrange
+ _storage = CreateStorage();
+ var filePath = "nonexistent.txt";
+
+ // Act & Assert
+ await Assert.ThrowsAsync(async () => await _storage.ComputeHashAsync(filePath));
+ }
+
+ [Fact]
+ public async Task GetStorageInfoAsync_ReturnsInfo() {
+ // Arrange
+ _storage = CreateStorage();
+
+ // Act
+ var info = await _storage.GetStorageInfoAsync();
+
+ // Assert
+ Assert.NotNull(info);
+ // S3 doesn't provide quota info, so these should be -1
+ Assert.Equal(-1, info.TotalSpace);
+ Assert.Equal(-1, info.UsedSpace);
+ }
+
+ [Fact]
+ public async Task ProgressChanged_LargeFileUpload_RaisesEvents() {
+ // Arrange
+ _storage = CreateStorage();
+ var filePath = "progress_test.bin";
+
+ // Create a 12MB file (larger than chunk size)
+ var content = new byte[12 * 1024 * 1024];
+ new Random().NextBytes(content);
+
+ var progressEvents = new List();
+ _storage.ProgressChanged += (sender, args) => progressEvents.Add(args);
+
+ // Act
+ using var stream = new MemoryStream(content);
+ await _storage.WriteFileAsync(filePath, stream);
+
+ // Assert
+ Assert.NotEmpty(progressEvents);
+ Assert.All(progressEvents, evt => {
+ Assert.Equal(StorageOperation.Upload, evt.Operation);
+ Assert.Equal(filePath, evt.Path);
+ });
+
+ // Should have final 100% progress
+ var finalEvent = progressEvents.LastOrDefault();
+ Assert.NotNull(finalEvent);
+ Assert.Equal(100, finalEvent.PercentComplete);
+ }
+
+ [Fact]
+ public async Task ProgressChanged_LargeFileDownload_RaisesEvents() {
+ // Arrange
+ _storage = CreateStorage();
+ var filePath = "download_progress_test.bin";
+
+ // Create a 12MB file
+ var content = new byte[12 * 1024 * 1024];
+ new Random().NextBytes(content);
+
+ using (var stream = new MemoryStream(content)) {
+ await _storage.WriteFileAsync(filePath, stream);
+ }
+
+ var progressEvents = new List();
+ _storage.ProgressChanged += (sender, args) => progressEvents.Add(args);
+
+ // Act
+ using var readStream = await _storage.ReadFileAsync(filePath);
+
+ // Consume the stream to trigger progress events
+ using var ms = new MemoryStream();
+ await readStream.CopyToAsync(ms);
+
+ // Assert
+ Assert.NotEmpty(progressEvents);
+ Assert.All(progressEvents, evt => {
+ Assert.Equal(StorageOperation.Download, evt.Operation);
+ Assert.Equal(filePath, evt.Path);
+ });
+ }
+
+ #endregion
+
+ #region Helper Methods
+
+ private static async Task CreateTestFile(S3Storage storage, string path, string content) {
+ using var stream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(content));
+ await storage.WriteFileAsync(path, stream);
+ }
+
+ #endregion
+}