diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml index 14e4f06..902697b 100644 --- a/.github/workflows/dotnet.yml +++ b/.github/workflows/dotnet.yml @@ -27,6 +27,23 @@ jobs: env: SFTP_USERS: testuser:testpass:1001:100:upload + ftp: + image: fauria/vsftpd:latest + ports: + - 21:21 + - 21000-21010:21000-21010 + options: >- + --health-cmd "pgrep vsftpd" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + FTP_USER: testuser + FTP_PASS: testpass + PASV_ADDRESS: localhost + PASV_MIN_PORT: 21000 + PASV_MAX_PORT: 21010 + steps: - uses: actions/checkout@v5 - name: Setup .NET @@ -47,3 +64,8 @@ jobs: SFTP_TEST_USER: testuser SFTP_TEST_PASS: testpass SFTP_TEST_ROOT: "upload" + FTP_TEST_HOST: localhost + FTP_TEST_PORT: 21 + FTP_TEST_USER: testuser + FTP_TEST_PASS: testpass + FTP_TEST_ROOT: "" diff --git a/CLAUDE.md b/CLAUDE.md index af1ea73..ae1f1a8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -67,7 +67,7 @@ dotnet pack --configuration Release --version-suffix preview The project uses GitHub Actions for CI/CD. The pipeline currently: - Builds on Ubuntu only (multi-platform testing planned) - Runs tests with format checking -- Includes SFTP integration tests using Docker-based SFTP server +- Includes SFTP and FTP integration tests using Docker-based servers - Automatically configures test environment variables for integration tests ## High-Level Architecture @@ -88,7 +88,8 @@ SharpSync is a **pure .NET file synchronization library** with no native depende - `LocalFileStorage` - Local filesystem operations (fully implemented and tested) - `WebDavStorage` - WebDAV with OAuth2, chunking, and platform-specific optimizations (implemented, needs tests) - `SftpStorage` - SFTP with password and key-based authentication (fully implemented and tested) - - `StorageType` enum includes: FTP, S3 (planned for future versions) + - `FtpStorage` - FTP/FTPS with secure connections support (fully implemented and tested) + - `StorageType` enum includes: S3 (planned for future versions) 3. **Authentication** (`src/SharpSync/Auth/`) - `IOAuth2Provider` - OAuth2 authentication abstraction (no UI dependencies) @@ -108,9 +109,10 @@ SharpSync is a **pure .NET file synchronization library** with no native depende ### Key Features -- **Multi-Protocol Support**: Local, WebDAV, and SFTP storage (extensible to FTP, S3) +- **Multi-Protocol Support**: Local, WebDAV, SFTP, and FTP/FTPS storage (extensible to S3) - **OAuth2 Authentication**: Full OAuth2 flow support without UI dependencies (WebDAV) - **SSH Key & Password Auth**: Secure SFTP authentication with private keys or passwords +- **FTP/FTPS Support**: Plain FTP, explicit FTPS, and implicit FTPS with password authentication - **Smart Conflict Resolution**: Rich conflict analysis for UI integration - **Selective Sync**: Include/exclude patterns for files and folders - **Progress Reporting**: Real-time progress events for UI binding @@ -123,7 +125,8 @@ SharpSync is a **pure .NET file synchronization library** with no native depende - `sqlite-net-pcl` (1.9.172) - SQLite database - `SQLitePCLRaw.bundle_e_sqlite3` (3.0.2) - SQLite native binaries - `WebDav.Client` (2.9.0) - WebDAV protocol -- `SSH.NET` (2025.1.0) - Currently unused, planned for future SFTP implementation +- `SSH.NET` (2025.1.0) - SFTP protocol implementation +- `FluentFTP` (52.0.2) - FTP/FTPS protocol implementation - Target Framework: .NET 8.0 ### Platform-Specific Optimizations @@ -282,10 +285,11 @@ The core library is production-ready, but several critical items must be address ### πŸ”„ CAN DEFER TO v1.1+ -10. **~~SFTP~~/FTP/S3 Implementations** (SFTP βœ… DONE) +10. **~~SFTP~~/~~FTP~~/S3 Implementations** (SFTP βœ… DONE, FTP βœ… DONE) - βœ… SFTP now fully implemented with comprehensive tests - - FTP/S3 remain future features for v1.1+ - - StorageType enum prepared for future implementations + - βœ… FTP/FTPS now fully implemented with comprehensive tests + - S3 remains future feature for v1.1+ + - StorageType enum prepared for future S3 implementation 11. **Performance Benchmarks** - BenchmarkDotNet suite for sync operations @@ -321,31 +325,29 @@ The core library is production-ready, but several critical items must be address **Minimum Acceptance Criteria:** - βœ… Core sync engine tested (achieved) -- ⚠️ All storage implementations tested (LocalFileStorage βœ…, SftpStorage βœ…, WebDavStorage ❌) +- ⚠️ All storage implementations tested (LocalFileStorage βœ…, SftpStorage βœ…, FtpStorage βœ…, WebDavStorage ❌) - ❌ README matches actual API (completely wrong) - βœ… No TODOs/FIXMEs in code (achieved) - ❌ Examples directory exists (missing) -- βœ… Package metadata accurate (SFTP now implemented!) -- βœ… Integration test infrastructure (Docker-based CI testing) +- βœ… Package metadata accurate (SFTP and FTP now implemented!) +- βœ… Integration test infrastructure (Docker-based CI testing for SFTP and FTP) -**Current Score: 5/9 (56%)** - Improved from 33%! +**Current Score: 5/9 (56%)** - Improved from 33%! (FTP implementation complete) ### 🎯 Post-v1.0 Roadmap (Future Versions) -**v1.0** βœ… SFTP Implemented! +**v1.0** βœ… SFTP and FTP Implemented! - βœ… SFTP storage implementation (DONE!) -- βœ… Integration test infrastructure with Docker (DONE!) +- βœ… FTP/FTPS storage implementation (DONE!) +- βœ… Integration test infrastructure with Docker for SFTP and FTP (DONE!) **v1.1** - Code coverage reporting - Performance benchmarks - Multi-platform CI testing (Windows, macOS) - -**v1.2** -- FTP/FTPS storage implementation - Additional conflict resolution strategies -**v1.3** +**v1.2** - S3-compatible storage implementation - Advanced filtering (regex support) diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 4750294..ace699b 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -12,5 +12,25 @@ services: timeout: 5s retries: 5 + ftp: + image: fauria/vsftpd:latest + ports: + - "21:21" + - "21000-21010:21000-21010" + environment: + FTP_USER: testuser + FTP_PASS: testpass + PASV_ADDRESS: localhost + PASV_MIN_PORT: 21000 + PASV_MAX_PORT: 21010 + volumes: + - ftp-data:/home/vsftpd + healthcheck: + test: ["CMD", "pgrep", "vsftpd"] + interval: 10s + timeout: 5s + retries: 5 + volumes: sftp-data: + ftp-data: diff --git a/src/SharpSync/SharpSync.csproj b/src/SharpSync/SharpSync.csproj index 402ad75..423c89a 100644 --- a/src/SharpSync/SharpSync.csproj +++ b/src/SharpSync/SharpSync.csproj @@ -15,14 +15,14 @@ AndrΓ© Polykanine Oire Software SharpSync - A pure .NET file synchronization library supporting WebDAV, SFTP, and local storage with conflict resolution, selective sync, and progress reporting. + A pure .NET file synchronization library supporting WebDAV, SFTP, FTP/FTPS, and local storage with conflict resolution, selective sync, and progress reporting. Copyright Β© 2025 AndrΓ© Polykanine, Oire Software Apache-2.0 false https://github.com/Oire/sharp-sync https://github.com/Oire/sharp-sync git - file-sync;synchronization;webdav;sftp;nextcloud;owncloud;backup;sync + file-sync;synchronization;webdav;sftp;ftp;ftps;nextcloud;owncloud;backup;sync 1.0.0 false @@ -38,6 +38,7 @@ + diff --git a/src/SharpSync/Storage/FtpStorage.cs b/src/SharpSync/Storage/FtpStorage.cs new file mode 100644 index 0000000..3c4b1d3 --- /dev/null +++ b/src/SharpSync/Storage/FtpStorage.cs @@ -0,0 +1,668 @@ +using System.Security.Cryptography; +using FluentFTP; +using Oire.SharpSync.Core; + +namespace Oire.SharpSync.Storage; + +/// +/// FTP/FTPS storage implementation with support for secure connections +/// Provides file synchronization over File Transfer Protocol +/// +public class FtpStorage: ISyncStorage, IDisposable { + private AsyncFtpClient? _client; + private readonly string _host; + private readonly int _port; + private readonly string _username; + private readonly string _password; + private readonly FtpEncryptionMode _encryptionMode; + private readonly FtpConfig _config; + + // Configuration + private readonly int _chunkSize; + private readonly int _maxRetries; + private readonly TimeSpan _retryDelay; + private readonly TimeSpan _connectionTimeout; + + private readonly SemaphoreSlim _connectionSemaphore; + private bool _disposed; + + /// + /// Gets the storage type (always returns ) + /// + public StorageType StorageType => StorageType.Ftp; + + /// + /// Gets the root path on the FTP server + /// + public string RootPath { get; } + + /// + /// Creates FTP storage with password authentication + /// + /// FTP server hostname + /// FTP server port (default 21) + /// Username for authentication + /// Password for authentication + /// Root path on the FTP server + /// Use FTPS (explicit SSL/TLS) instead of plain FTP + /// Use implicit FTPS (SSL from connection start) + /// Chunk size for large file uploads (default 10MB) + /// Maximum retry attempts (default 3) + /// Connection timeout in seconds (default 30) + public FtpStorage( + string host, + int port = 21, + string username = "anonymous", + string password = "anonymous@example.com", + string rootPath = "", + bool useFtps = false, + bool useImplicitFtps = false, + int chunkSizeBytes = 10 * 1024 * 1024, // 10MB + int maxRetries = 3, + int connectionTimeoutSeconds = 30) { + if (string.IsNullOrWhiteSpace(host)) { + throw new ArgumentException("Host cannot be empty", nameof(host)); + } + + if (port <= 0 || port > 65535) { + throw new ArgumentException("Port must be between 1 and 65535", nameof(port)); + } + + if (string.IsNullOrWhiteSpace(username)) { + throw new ArgumentException("Username cannot be empty", nameof(username)); + } + + if (string.IsNullOrWhiteSpace(password)) { + throw new ArgumentException("Password cannot be empty", nameof(password)); + } + + _host = host; + _port = port; + _username = username; + _password = password; + RootPath = NormalizePath(rootPath); + + // Determine encryption mode + if (useImplicitFtps) { + _encryptionMode = FtpEncryptionMode.Implicit; + } else if (useFtps) { + _encryptionMode = FtpEncryptionMode.Explicit; + } else { + _encryptionMode = FtpEncryptionMode.None; + } + + _chunkSize = chunkSizeBytes; + _maxRetries = maxRetries; + _retryDelay = TimeSpan.FromSeconds(1); + _connectionTimeout = TimeSpan.FromSeconds(connectionTimeoutSeconds); + + // Configure FluentFTP client + _config = new FtpConfig { + EncryptionMode = _encryptionMode, + ValidateAnyCertificate = true, // Accept any certificate (can be configured for production) + ConnectTimeout = (int)_connectionTimeout.TotalMilliseconds, + DataConnectionType = FtpDataConnectionType.AutoPassive, + TransferChunkSize = _chunkSize + }; + + _connectionSemaphore = new SemaphoreSlim(1, 1); + } + + /// + /// Event raised when upload/download progress changes + /// + public event EventHandler? ProgressChanged; + + /// + /// Establishes connection to FTP server + /// + private async Task EnsureConnectedAsync(CancellationToken cancellationToken = default) { + if (_client?.IsConnected == true) { + return; + } + + await _connectionSemaphore.WaitAsync(cancellationToken); + try { + if (_client?.IsConnected == true) { + return; + } + + // Dispose old client if exists + if (_client != null) { + await _client.Disconnect(cancellationToken); + _client.Dispose(); + } + + // Create and connect client + _client = new AsyncFtpClient(_host, _username, _password, _port, _config); + + await _client.Connect(cancellationToken); + } finally { + _connectionSemaphore.Release(); + } + } + + /// + /// Tests the connection to the FTP server + /// + /// Cancellation token to cancel the operation + /// True if connection is successful, false otherwise + public async Task TestConnectionAsync(CancellationToken cancellationToken = default) { + try { + await EnsureConnectedAsync(cancellationToken); + return _client?.IsConnected == true; + } catch { + return false; + } + } + + /// + /// Lists all items (files and directories) in the specified path + /// + /// The relative path to list items from + /// Cancellation token to cancel the operation + /// A collection of sync items representing files and directories + /// Thrown when authentication fails + public async Task> ListItemsAsync(string path, CancellationToken cancellationToken = default) { + await EnsureConnectedAsync(cancellationToken); + + var fullPath = GetFullPath(path); + + return await ExecuteWithRetry(async () => { + var items = new List(); + + if (!await _client!.DirectoryExists(fullPath, cancellationToken)) { + return items; + } + + var ftpItems = await _client.GetListing(fullPath, cancellationToken); + + foreach (var item in ftpItems) { + cancellationToken.ThrowIfCancellationRequested(); + + // Skip parent directory entries + if (item.Name == ".." || item.Name == ".") { + continue; + } + + var relativePath = GetRelativePath(item.FullName); + + items.Add(new SyncItem { + Path = relativePath, + IsDirectory = item.Type == FtpObjectType.Directory, + Size = item.Size, + LastModified = item.Modified.ToUniversalTime(), + Permissions = ConvertPermissionsToString(item), + MimeType = item.Type == FtpObjectType.Directory ? null : GetMimeType(item.Name) + }); + } + + return (IEnumerable)items; + }, cancellationToken); + } + + /// + /// Gets metadata for a specific item (file or directory) + /// + /// The relative path to the item + /// Cancellation token to cancel the operation + /// The sync item if it exists, null otherwise + public async Task GetItemAsync(string path, CancellationToken cancellationToken = default) { + await EnsureConnectedAsync(cancellationToken); + + var fullPath = GetFullPath(path); + + return await ExecuteWithRetry(async () => { + if (!await _client!.FileExists(fullPath, cancellationToken) && + !await _client.DirectoryExists(fullPath, cancellationToken)) { + return null; + } + + var item = await _client.GetObjectInfo(fullPath); + if (item == null) { + return null; + } + + return new SyncItem { + Path = path, + IsDirectory = item.Type == FtpObjectType.Directory, + Size = item.Size, + LastModified = item.Modified.ToUniversalTime(), + Permissions = ConvertPermissionsToString(item), + MimeType = item.Type == FtpObjectType.Directory ? null : GetMimeType(item.Name) + }; + }, cancellationToken); + } + + /// + /// Reads the contents of a file from the FTP server + /// + /// The relative path to the file + /// Cancellation token to cancel the operation + /// A stream containing the file contents + /// Thrown when the file does not exist + /// Thrown when attempting to read a directory as a file + /// Thrown when authentication fails + /// + /// For files larger than the configured chunk size, progress events will be raised via + /// + public async Task ReadFileAsync(string path, CancellationToken cancellationToken = default) { + await EnsureConnectedAsync(cancellationToken); + + var fullPath = GetFullPath(path); + + return await ExecuteWithRetry(async () => { + if (!await _client!.FileExists(fullPath, cancellationToken)) { + throw new FileNotFoundException($"File not found: {path}"); + } + + var memoryStream = new MemoryStream(); + + // Get file size for progress reporting + var fileInfo = await _client.GetObjectInfo(fullPath); + var needsProgress = fileInfo?.Size > _chunkSize; + + if (needsProgress && fileInfo != null) { + // Download with progress reporting + var totalBytes = fileInfo.Size; + var progress = new Progress(p => { + RaiseProgressChanged(path, p.TransferredBytes, totalBytes, StorageOperation.Download); + }); + + await _client.DownloadStream(memoryStream, fullPath, progress: progress, token: cancellationToken); + } else { + // Download without progress + await _client.DownloadStream(memoryStream, fullPath, token: cancellationToken); + } + + memoryStream.Position = 0; + return (Stream)memoryStream; + }, cancellationToken); + } + + /// + /// Writes content to a file on the FTP server, creating parent directories as needed + /// + /// The relative path to the file + /// The stream containing the file content to write + /// Cancellation token to cancel the operation + /// Thrown when authentication fails + /// + /// If the file already exists, it will be overwritten. For files larger than the configured chunk size, + /// progress events will be raised via + /// + public async Task WriteFileAsync(string path, Stream content, CancellationToken cancellationToken = default) { + await EnsureConnectedAsync(cancellationToken); + + var fullPath = GetFullPath(path); + + // Ensure parent directories exist + var directory = GetParentDirectory(fullPath); + if (!string.IsNullOrEmpty(directory)) { + await CreateDirectoryAsync(GetRelativePath(directory), cancellationToken); + } + + await ExecuteWithRetry(async () => { + var needsProgress = content.CanSeek && content.Length > _chunkSize; + + if (needsProgress) { + // Upload with progress reporting + var totalBytes = content.Length; + var progress = new Progress(p => { + RaiseProgressChanged(path, p.TransferredBytes, totalBytes, StorageOperation.Upload); + }); + + await _client!.UploadStream(content, fullPath, FtpRemoteExists.Overwrite, true, progress, cancellationToken); + } else { + // Upload without progress + await _client!.UploadStream(content, fullPath, FtpRemoteExists.Overwrite, true, token: cancellationToken); + } + + return true; + }, cancellationToken); + } + + /// + /// Creates a directory on the FTP server, including all parent directories if needed + /// + /// The relative path to the directory to create + /// Cancellation token to cancel the operation + /// Thrown when authentication fails + /// + /// If the directory already exists, this method completes successfully without error + /// + public async Task CreateDirectoryAsync(string path, CancellationToken cancellationToken = default) { + await EnsureConnectedAsync(cancellationToken); + + var fullPath = GetFullPath(path); + + // Don't attempt to create root directory + if (fullPath == "/" || string.IsNullOrEmpty(fullPath)) { + return; + } + + await ExecuteWithRetry(async () => { + if (await _client!.DirectoryExists(fullPath, cancellationToken)) { + return true; // Directory already exists + } + + // Create directory with parent directories + await _client.CreateDirectory(fullPath, cancellationToken); + + return true; + }, cancellationToken); + } + + /// + /// Deletes a file or directory from the FTP server + /// + /// The relative path to the file or directory to delete + /// Cancellation token to cancel the operation + /// Thrown when authentication fails + /// + /// If the path is a directory, it will be deleted recursively along with all its contents. + /// If the item does not exist, this method completes successfully without error + /// + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) { + await EnsureConnectedAsync(cancellationToken); + + var fullPath = GetFullPath(path); + + await ExecuteWithRetry(async () => { + if (await _client!.DirectoryExists(fullPath, cancellationToken)) { + // Delete directory recursively + await _client.DeleteDirectory(fullPath, cancellationToken); + } else if (await _client.FileExists(fullPath, cancellationToken)) { + // Delete file + await _client.DeleteFile(fullPath, cancellationToken); + } + + return true; + }, cancellationToken); + } + + /// + /// Moves or renames a file or directory on the FTP server + /// + /// The relative path to the source file or directory + /// The relative path to the target location + /// Cancellation token to cancel the operation + /// Thrown when the source does not exist + /// Thrown when authentication fails + /// + /// Parent directories of the target path will be created if they don't exist + /// + public async Task MoveAsync(string sourcePath, string targetPath, CancellationToken cancellationToken = default) { + await EnsureConnectedAsync(cancellationToken); + + var sourceFullPath = GetFullPath(sourcePath); + var targetFullPath = GetFullPath(targetPath); + + // Ensure target parent directory exists + var targetParentRelative = GetParentDirectory(NormalizePath(targetPath)); + if (!string.IsNullOrEmpty(targetParentRelative)) { + await CreateDirectoryAsync(targetParentRelative, cancellationToken); + } + + await ExecuteWithRetry(async () => { + if (!await _client!.FileExists(sourceFullPath, cancellationToken) && + !await _client.DirectoryExists(sourceFullPath, cancellationToken)) { + throw new FileNotFoundException($"Source not found: {sourcePath}"); + } + + await _client.MoveFile(sourceFullPath, targetFullPath, FtpRemoteExists.Overwrite, cancellationToken); + + return true; + }, cancellationToken); + } + + /// + /// Checks whether a file or directory exists on the FTP server + /// + /// The relative path to check + /// Cancellation token to cancel the operation + /// True if the file or directory exists, false otherwise + public async Task ExistsAsync(string path, CancellationToken cancellationToken = default) { + await EnsureConnectedAsync(cancellationToken); + + var fullPath = GetFullPath(path); + + return await ExecuteWithRetry(async () => { + return await _client!.FileExists(fullPath, cancellationToken) || + await _client.DirectoryExists(fullPath, cancellationToken); + }, cancellationToken); + } + + /// + /// Gets storage space information for the FTP server + /// + /// Cancellation token to cancel the operation + /// Storage information (may return -1 for unknown values as not all FTP servers support this) + /// + /// FTP protocol does not have standardized disk space reporting. This method returns + /// best-effort values which may be -1 if the server doesn't support the SIZE command + /// + public async Task GetStorageInfoAsync(CancellationToken cancellationToken = default) { + await EnsureConnectedAsync(cancellationToken); + + // FTP doesn't have a standard way to get disk space + // Return unknown values + return await Task.FromResult(new StorageInfo { + TotalSpace = -1, + UsedSpace = -1 + }); + } + + /// + /// Computes the SHA256 hash of a file on the FTP server + /// + /// The relative path to the file + /// Cancellation token to cancel the operation + /// Base64-encoded SHA256 hash of the file contents + /// Thrown when the file does not exist + /// + /// Since FTP protocol doesn't provide native hash computation, this method downloads + /// the file and computes the hash locally. For large files, consider the performance implications + /// + public async Task ComputeHashAsync(string path, CancellationToken cancellationToken = default) { + // FTP doesn't have native hash support, so we download and hash + using var stream = await ReadFileAsync(path, cancellationToken); + using var sha256 = SHA256.Create(); + + var hashBytes = await sha256.ComputeHashAsync(stream, cancellationToken); + return Convert.ToBase64String(hashBytes); + } + + #region Helper Methods + + /// + /// Normalizes a path for FTP (uses forward slashes) + /// + private static string NormalizePath(string path) { + if (string.IsNullOrWhiteSpace(path)) { + return ""; + } + + // Convert backslashes to forward slashes + path = path.Replace('\\', '/'); + + // Remove trailing slashes + path = path.TrimEnd('/'); + + // Ensure path doesn't start with slash (unless it's root) + if (path == "/") { + return "/"; + } + + return path.TrimStart('/'); + } + + /// + /// Gets the full path on the FTP server + /// + private string GetFullPath(string relativePath) { + if (string.IsNullOrEmpty(relativePath) || relativePath == "/") { + return string.IsNullOrEmpty(RootPath) ? "/" : $"/{RootPath}"; + } + + relativePath = NormalizePath(relativePath); + + if (string.IsNullOrEmpty(RootPath)) { + return $"/{relativePath}"; + } + + return $"/{RootPath}/{relativePath}"; + } + + /// + /// Gets the relative path from a full FTP path + /// + private string GetRelativePath(string fullPath) { + var prefix = string.IsNullOrEmpty(RootPath) || RootPath == "/" + ? "/" + : $"/{RootPath}/"; + + if (fullPath.StartsWith(prefix)) { + var relativePath = fullPath.Substring(prefix.Length); + return string.IsNullOrEmpty(relativePath) ? "/" : relativePath; + } + + return fullPath; + } + + /// + /// Gets the parent directory of a path + /// + private static string GetParentDirectory(string path) { + if (string.IsNullOrEmpty(path)) { + return string.Empty; + } + + var lastSlash = path.LastIndexOf('/'); + if (lastSlash <= 0) { + return string.Empty; + } + + return path.Substring(0, lastSlash); + } + + /// + /// Converts FTP file permissions to a string representation + /// + private static string ConvertPermissionsToString(FtpListItem item) { + // If Chmod is 0 (unknown/unset), return empty; otherwise return numeric string (e.g., "755") + return item.Chmod != 0 ? item.Chmod.ToString() : string.Empty; + } + + /// + /// Gets MIME type based on file extension + /// + private static string GetMimeType(string fileName) { + var extension = Path.GetExtension(fileName).ToLowerInvariant(); + return extension switch { + ".txt" => "text/plain", + ".pdf" => "application/pdf", + ".jpg" or ".jpeg" => "image/jpeg", + ".png" => "image/png", + ".gif" => "image/gif", + ".zip" => "application/zip", + ".json" => "application/json", + ".xml" => "application/xml", + ".html" or ".htm" => "text/html", + ".css" => "text/css", + ".js" => "application/javascript", + ".mp3" => "audio/mpeg", + ".mp4" => "video/mp4", + ".doc" => "application/msword", + ".docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ".xls" => "application/vnd.ms-excel", + ".xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + _ => "application/octet-stream" + }; + } + + /// + /// Executes an operation with retry logic + /// + private async Task ExecuteWithRetry(Func> operation, CancellationToken cancellationToken) { + Exception? lastException = null; + + for (int attempt = 0; attempt <= _maxRetries; attempt++) { + try { + cancellationToken.ThrowIfCancellationRequested(); + return await operation(); + } catch (Exception ex) when (attempt < _maxRetries && IsRetriableException(ex)) { + lastException = ex; + + // Reconnect if connection was lost + if (ex is IOException || ex is TimeoutException) { + try { + if (_client != null) { + await _client.Disconnect(cancellationToken); + _client.Dispose(); + _client = null; + } + await EnsureConnectedAsync(cancellationToken); + } catch { + // Ignore reconnection errors, will retry + } + } + + await Task.Delay(_retryDelay * (attempt + 1), cancellationToken); + } + } + + throw lastException ?? new InvalidOperationException("Operation failed"); + } + + /// + /// Determines if an exception is retriable + /// + private static bool IsRetriableException(Exception ex) { + return ex is IOException || + ex is TimeoutException || + ex is UnauthorizedAccessException == false; // Don't retry auth errors + } + + /// + /// Raises progress changed event + /// + private void RaiseProgressChanged(string path, long completed, long total, StorageOperation operation) { + ProgressChanged?.Invoke(this, new StorageProgressEventArgs { + Path = path, + BytesTransferred = completed, + TotalBytes = total, + Operation = operation, + PercentComplete = total > 0 ? (int)((completed * 100L) / total) : 0 + }); + } + + #endregion + + #region IDisposable + + /// + /// Releases all resources used by the FTP storage instance + /// + /// + /// Disconnects from the FTP server and disposes of the underlying FTP client and connection semaphore. + /// This method can be called multiple times safely + /// + public void Dispose() { + if (!_disposed) { + try { + _client?.Disconnect(); + } catch { + // Ignore disconnection errors during disposal + } + + _client?.Dispose(); + _connectionSemaphore?.Dispose(); + _disposed = true; + } + + GC.SuppressFinalize(this); + } + + #endregion +} diff --git a/tests/SharpSync.Tests/Storage/FtpStorageTests.cs b/tests/SharpSync.Tests/Storage/FtpStorageTests.cs new file mode 100644 index 0000000..971d47f --- /dev/null +++ b/tests/SharpSync.Tests/Storage/FtpStorageTests.cs @@ -0,0 +1,566 @@ +namespace Oire.SharpSync.Tests.Storage; + +/// +/// Unit and integration tests for FtpStorage +/// NOTE: Integration tests require a real FTP server. Set up environment variables: +/// - FTP_TEST_HOST: FTP server hostname +/// - FTP_TEST_PORT: FTP server port (default 21) +/// - FTP_TEST_USER: FTP username +/// - FTP_TEST_PASS: FTP password +/// - FTP_TEST_ROOT: Root path on server (default: /tmp/sharpsync-tests) +/// - FTP_TEST_USE_FTPS: Use FTPS/explicit SSL (default: false) +/// - FTP_TEST_USE_IMPLICIT_FTPS: Use implicit FTPS (default: false) +/// +public class FtpStorageTests: IDisposable { + private readonly string? _testHost; + private readonly int _testPort; + private readonly string? _testUser; + private readonly string? _testPass; + private readonly string _testRoot; + private readonly bool _useFtps; + private readonly bool _useImplicitFtps; + private readonly bool _integrationTestsEnabled; + private FtpStorage? _storage; + + public FtpStorageTests() { + // Read environment variables for integration tests + _testHost = Environment.GetEnvironmentVariable("FTP_TEST_HOST"); + _testUser = Environment.GetEnvironmentVariable("FTP_TEST_USER"); + _testPass = Environment.GetEnvironmentVariable("FTP_TEST_PASS"); + _testRoot = Environment.GetEnvironmentVariable("FTP_TEST_ROOT") ?? "/tmp/sharpsync-tests"; + + var portStr = Environment.GetEnvironmentVariable("FTP_TEST_PORT"); + _testPort = int.TryParse(portStr, out var port) ? port : 21; + + var useFtpsStr = Environment.GetEnvironmentVariable("FTP_TEST_USE_FTPS"); + _useFtps = bool.TryParse(useFtpsStr, out var useFtps) && useFtps; + + var useImplicitFtpsStr = Environment.GetEnvironmentVariable("FTP_TEST_USE_IMPLICIT_FTPS"); + _useImplicitFtps = bool.TryParse(useImplicitFtpsStr, out var useImplicitFtps) && useImplicitFtps; + + _integrationTestsEnabled = !string.IsNullOrEmpty(_testHost) && + !string.IsNullOrEmpty(_testUser) && + !string.IsNullOrEmpty(_testPass); + } + + public void Dispose() { + _storage?.Dispose(); + } + + #region Unit Tests (No Server Required) + + [Fact] + public void Constructor_ValidParameters_CreatesStorage() { + // Act + using var storage = new FtpStorage("example.com", 21, "user", "password"); + + // Assert + Assert.Equal(StorageType.Ftp, storage.StorageType); + } + + [Fact] + public void Constructor_EmptyHost_ThrowsException() { + // Act & Assert + Assert.Throws(() => new FtpStorage("", 21, "user", "password")); + } + + [Fact] + public void Constructor_InvalidPort_ThrowsException() { + // Act & Assert + Assert.Throws(() => new FtpStorage("example.com", 0, "user", "password")); + Assert.Throws(() => new FtpStorage("example.com", 70000, "user", "password")); + } + + [Fact] + public void Constructor_EmptyUsername_ThrowsException() { + // Act & Assert + Assert.Throws(() => new FtpStorage("example.com", 21, "", "password")); + } + + [Fact] + public void Constructor_EmptyPassword_ThrowsException() { + // Act & Assert + Assert.Throws(() => new FtpStorage("example.com", 21, "user", "")); + } + + [Fact] + public void RootPath_Property_ReturnsCorrectPath() { + // Arrange + var rootPath = "test/path"; + using var storage = new FtpStorage("example.com", 21, "user", "password", rootPath: rootPath); + + // Assert + Assert.Equal(rootPath, storage.RootPath); + } + + [Fact] + public void StorageType_Property_ReturnsFtp() { + // Arrange + using var storage = new FtpStorage("example.com", 21, "user", "password"); + + // Assert + Assert.Equal(StorageType.Ftp, storage.StorageType); + } + + [Fact] + public void Constructor_WithFtps_CreatesStorage() { + // Act + using var storage = new FtpStorage("example.com", 21, "user", "password", useFtps: true); + + // Assert + Assert.Equal(StorageType.Ftp, storage.StorageType); + } + + [Fact] + public void Constructor_WithImplicitFtps_CreatesStorage() { + // Act + using var storage = new FtpStorage("example.com", 990, "user", "password", useImplicitFtps: true); + + // Assert + Assert.Equal(StorageType.Ftp, storage.StorageType); + } + + #endregion + + #region Integration Tests (Require FTP Server) + + private void SkipIfIntegrationTestsDisabled() { + if (!_integrationTestsEnabled) { + throw new SkipException("Integration tests disabled. Set FTP_TEST_HOST, FTP_TEST_USER, and FTP_TEST_PASS environment variables."); + } + } + + private FtpStorage CreateStorage() { + SkipIfIntegrationTestsDisabled(); + + return new FtpStorage( + _testHost!, + _testPort, + _testUser!, + _testPass!, + rootPath: $"{_testRoot}/{Guid.NewGuid()}", + useFtps: _useFtps, + useImplicitFtps: _useImplicitFtps); + } + + [Fact] + public async Task TestConnectionAsync_ValidCredentials_ReturnsTrue() { + SkipIfIntegrationTestsDisabled(); + + // Arrange + using var storage = CreateStorage(); + + // Act + var result = await storage.TestConnectionAsync(); + + // Assert + Assert.True(result); + } + + [Fact] + public async Task TestConnectionAsync_InvalidCredentials_ReturnsFalse() { + SkipIfIntegrationTestsDisabled(); + + // Arrange + using var storage = new FtpStorage(_testHost!, _testPort, _testUser!, "wrong_password"); + + // Act + var result = await storage.TestConnectionAsync(); + + // Assert + Assert.False(result); + } + + [Fact] + public async Task CreateDirectoryAsync_CreatesDirectory() { + // Arrange + _storage = CreateStorage(); + var dirPath = "test/subdir"; + + // Act + await _storage.CreateDirectoryAsync(dirPath); + var exists = await _storage.ExistsAsync(dirPath); + + // Assert + Assert.True(exists); + } + + [Fact] + public async Task WriteFileAsync_CreatesFile() { + // Arrange + _storage = CreateStorage(); + var filePath = "test.txt"; + var content = "Hello, FTP World!"; + + // Act + using var stream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(content)); + await _storage.WriteFileAsync(filePath, stream); + + // Assert + var exists = await _storage.ExistsAsync(filePath); + Assert.True(exists); + } + + [Fact] + public async Task ReadFileAsync_ReturnsFileContent() { + // Arrange + _storage = CreateStorage(); + var filePath = "test_read.txt"; + var content = "Hello, FTP World!"; + + using var writeStream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(content)); + await _storage.WriteFileAsync(filePath, writeStream); + + // Act + using var readStream = await _storage.ReadFileAsync(filePath); + using var reader = new StreamReader(readStream); + var result = await reader.ReadToEndAsync(); + + // Assert + Assert.Equal(content, result); + } + + [Fact] + public async Task ReadFileAsync_NonexistentFile_ThrowsException() { + // Arrange + _storage = CreateStorage(); + + // Act & Assert + await Assert.ThrowsAsync(() => _storage.ReadFileAsync("nonexistent.txt")); + } + + [Fact] + public async Task ExistsAsync_ExistingFile_ReturnsTrue() { + // Arrange + _storage = CreateStorage(); + var filePath = "exists_test.txt"; + using var stream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes("test")); + await _storage.WriteFileAsync(filePath, stream); + + // Act + var result = await _storage.ExistsAsync(filePath); + + // Assert + Assert.True(result); + } + + [Fact] + public async Task ExistsAsync_NonexistentFile_ReturnsFalse() { + // Arrange + _storage = CreateStorage(); + + // Act + var result = await _storage.ExistsAsync("nonexistent.txt"); + + // Assert + Assert.False(result); + } + + [Fact] + public async Task DeleteAsync_ExistingFile_DeletesFile() { + // Arrange + _storage = CreateStorage(); + var filePath = "delete_test.txt"; + using var stream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes("test")); + await _storage.WriteFileAsync(filePath, stream); + + // Act + await _storage.DeleteAsync(filePath); + + // Assert + var exists = await _storage.ExistsAsync(filePath); + Assert.False(exists); + } + + [Fact] + public async Task DeleteAsync_Directory_DeletesDirectory() { + // Arrange + _storage = CreateStorage(); + var dirPath = "delete_dir_test"; + await _storage.CreateDirectoryAsync(dirPath); + + // Act + await _storage.DeleteAsync(dirPath); + + // Assert + var exists = await _storage.ExistsAsync(dirPath); + Assert.False(exists); + } + + [Fact] + public async Task DeleteAsync_DirectoryWithContents_DeletesRecursively() { + // Arrange + _storage = CreateStorage(); + var dirPath = "delete_recursive"; + var filePath = "delete_recursive/file.txt"; + + await _storage.CreateDirectoryAsync(dirPath); + using var stream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes("test")); + await _storage.WriteFileAsync(filePath, stream); + + // Act + await _storage.DeleteAsync(dirPath); + + // Assert + var exists = await _storage.ExistsAsync(dirPath); + Assert.False(exists); + } + + [Fact] + public async Task MoveAsync_MovesFile() { + // Arrange + _storage = CreateStorage(); + var sourcePath = "source.txt"; + var targetPath = "target.txt"; + var content = "test content"; + + using var stream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(content)); + await _storage.WriteFileAsync(sourcePath, stream); + + // Act + await _storage.MoveAsync(sourcePath, targetPath); + + // Assert + var sourceExists = await _storage.ExistsAsync(sourcePath); + var targetExists = await _storage.ExistsAsync(targetPath); + + Assert.False(sourceExists); + Assert.True(targetExists); + + // Verify content + using var readStream = await _storage.ReadFileAsync(targetPath); + using var reader = new StreamReader(readStream); + var result = await reader.ReadToEndAsync(); + Assert.Equal(content, result); + } + + [Fact] + public async Task MoveAsync_ToSubdirectory_MovesCorrectly() { + // Arrange + _storage = CreateStorage(); + var sourcePath = "move_source.txt"; + var targetPath = "subdir/move_target.txt"; + var content = "test content"; + + using var stream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(content)); + await _storage.WriteFileAsync(sourcePath, stream); + + // Act + await _storage.MoveAsync(sourcePath, targetPath); + + // Assert + var sourceExists = await _storage.ExistsAsync(sourcePath); + var targetExists = await _storage.ExistsAsync(targetPath); + + Assert.False(sourceExists); + Assert.True(targetExists); + } + + [Fact] + public async Task ListItemsAsync_ReturnsItems() { + // Arrange + _storage = CreateStorage(); + + // Create test files and directories + using var stream1 = new MemoryStream(System.Text.Encoding.UTF8.GetBytes("content1")); + await _storage.WriteFileAsync("file1.txt", stream1); + + await _storage.CreateDirectoryAsync("subdir"); + + using var stream2 = new MemoryStream(System.Text.Encoding.UTF8.GetBytes("content2")); + await _storage.WriteFileAsync("subdir/file2.txt", stream2); + + // Act + var items = await _storage.ListItemsAsync(""); + + // Assert + var itemsList = items.ToList(); + Assert.True(itemsList.Count >= 2); + + var file = itemsList.FirstOrDefault(i => i.Path.Contains("file1.txt")); + var directory = itemsList.FirstOrDefault(i => i.Path.Contains("subdir") && i.IsDirectory); + + Assert.NotNull(file); + Assert.NotNull(directory); + Assert.False(file.IsDirectory); + Assert.True(directory.IsDirectory); + } + + [Fact] + public async Task GetItemAsync_ExistingFile_ReturnsItem() { + // Arrange + _storage = CreateStorage(); + var filePath = "get_item_test.txt"; + var content = "Hello, World!"; + + using var stream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(content)); + await _storage.WriteFileAsync(filePath, stream); + + // Act + var item = await _storage.GetItemAsync(filePath); + + // Assert + Assert.NotNull(item); + Assert.Equal(filePath, item.Path); + Assert.False(item.IsDirectory); + Assert.Equal(content.Length, item.Size); + } + + [Fact] + public async Task GetItemAsync_Directory_ReturnsDirectoryItem() { + // Arrange + _storage = CreateStorage(); + var dirPath = "get_dir_test"; + await _storage.CreateDirectoryAsync(dirPath); + + // Act + var item = await _storage.GetItemAsync(dirPath); + + // Assert + Assert.NotNull(item); + Assert.Equal(dirPath, item.Path); + Assert.True(item.IsDirectory); + } + + [Fact] + public async Task GetItemAsync_NonexistentItem_ReturnsNull() { + // Arrange + _storage = CreateStorage(); + + // Act + var item = await _storage.GetItemAsync("nonexistent.txt"); + + // Assert + Assert.Null(item); + } + + [Fact] + public async Task ComputeHashAsync_ReturnsConsistentHash() { + // Arrange + _storage = CreateStorage(); + var filePath = "hash_test.txt"; + var content = "Hello, World!"; + + using var stream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(content)); + await _storage.WriteFileAsync(filePath, stream); + + // Act + var hash1 = await _storage.ComputeHashAsync(filePath); + var hash2 = await _storage.ComputeHashAsync(filePath); + + // Assert + Assert.NotNull(hash1); + Assert.NotEmpty(hash1); + Assert.Equal(hash1, hash2); + } + + [Fact] + public async Task GetStorageInfoAsync_ReturnsStorageInfo() { + // Arrange + _storage = CreateStorage(); + + // Act + var info = await _storage.GetStorageInfoAsync(); + + // Assert + Assert.NotNull(info); + // FTP doesn't support storage info, so we expect -1 values + Assert.Equal(-1, info.TotalSpace); + Assert.Equal(-1, info.UsedSpace); + } + + [Fact] + public async Task WriteFileAsync_LargeFile_SupportsProgressReporting() { + // Arrange + _storage = CreateStorage(); + var filePath = "large_file.bin"; + var content = new byte[15 * 1024 * 1024]; // 15MB (larger than default chunk size) + new Random().NextBytes(content); + + var progressEvents = new List(); + _storage.ProgressChanged += (sender, args) => progressEvents.Add(args); + + // Act + using var stream = new MemoryStream(content); + await _storage.WriteFileAsync(filePath, stream); + + // Assert + var exists = await _storage.ExistsAsync(filePath); + Assert.True(exists); + + // Verify progress events were raised + Assert.NotEmpty(progressEvents); + Assert.All(progressEvents, e => Assert.Equal(StorageOperation.Upload, e.Operation)); + } + + [Fact] + public async Task ReadFileAsync_LargeFile_SupportsProgressReporting() { + // Arrange + _storage = CreateStorage(); + var filePath = "large_read.bin"; + var content = new byte[15 * 1024 * 1024]; // 15MB + new Random().NextBytes(content); + + using var writeStream = new MemoryStream(content); + await _storage.WriteFileAsync(filePath, writeStream); + + var progressEvents = new List(); + _storage.ProgressChanged += (sender, args) => progressEvents.Add(args); + + // Act + using var readStream = await _storage.ReadFileAsync(filePath); + var buffer = new byte[readStream.Length]; + + // Read the entire stream, handling partial reads + int totalRead = 0; + while (totalRead < buffer.Length) { + int bytesRead = await readStream.ReadAsync(buffer.AsMemory(totalRead, buffer.Length - totalRead)); + if (bytesRead == 0) { + break; // EOF + } + totalRead += bytesRead; + } + + // Assert + Assert.Equal(content.Length, totalRead); + Assert.NotEmpty(progressEvents); + Assert.All(progressEvents, e => Assert.Equal(StorageOperation.Download, e.Operation)); + } + + [Fact] + public async Task WriteFileAsync_CreatesParentDirectories() { + // Arrange + _storage = CreateStorage(); + var filePath = "parent/child/file.txt"; + var content = "nested file"; + + // Act + using var stream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(content)); + await _storage.WriteFileAsync(filePath, stream); + + // Assert + var exists = await _storage.ExistsAsync(filePath); + Assert.True(exists); + } + + [Fact] + public async Task DeleteAsync_NonexistentItem_DoesNotThrow() { + // Arrange + _storage = CreateStorage(); + + // Act & Assert - should not throw + await _storage.DeleteAsync("nonexistent_file.txt"); + } + + [Fact] + public async Task MoveAsync_NonexistentSource_ThrowsException() { + // Arrange + _storage = CreateStorage(); + + // Act & Assert + await Assert.ThrowsAsync(() => + _storage.MoveAsync("nonexistent_source.txt", "target.txt")); + } + + #endregion +}