Fully working monitoring + lots of bugfixes

.github/workflows/main.yml → .github/workflows/docker.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+      - develop
 
 jobs:
   build-and-push:
@@ -35,6 +36,7 @@ jobs:
             type=ref,event=branch
             type=semver,pattern={{version}}
             type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
+            type=raw,value=develop,enable=${{ github.ref == 'refs/heads/develop' }}
 
       - name: Build and push Docker image
         uses: docker/build-push-action@v6

README.md

@@ -0,0 +1,49 @@
+# Tilework
+
+## About
+
+Tilework is a fully integrated reverse proxying and load balancing platform. It is designed with the main objective of being simple and fast to configure rather than being in the way.
+
+## Features
+- Deployment of HTTP/TCP/UDP load balancers with multiple backends
+- HTTP rules based routing, including hostname, URL path, query string
+- Certificate issuing via popular services, lifecycle management, auto-renewal
+- Realtime and historical service statistics
+- Docker based service deployment - no disruption of the host environement
+
+
+## Install
+1. Install [docker engine](https://docs.docker.com/engine/install/) or [docker desktop](https://docs.docker.com/get-started/get-docker/). Be sure that docker compose is also installed.
+
+
+
+
+2. Create a docker-compose.yml file as follows:
+
+```yaml
+services:
+  tileworkui:
+    image: tilework/tilework:latest
+    ports:
+      - 5180:5180
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Docker
+    volumes:
+      - tilework_data:/var/lib/tilework
+      - /var/run/docker.sock:/var/run/docker.sock
+
+volumes:
+  tilework_data:
+    external: false
+```
+
+3. Start the service up
+```
+# If using the docker-compose command
+docker-compose up -d
+
+# If using docker-compose-plugin
+docker compose up -d
+```
+
+4. Navigate your browser to http://\<host>:5180

tilework.core/Attributes/CumulativeAttribute.cs

@@ -0,0 +1,4 @@
+namespace Tilework.Core.Attributes;
+
+[AttributeUsage(AttributeTargets.Property)]
+public sealed class CumulativeAttribute : Attribute { }

tilework.core/Enums/Core/ContainerExceptionType.cs

@@ -0,0 +1,5 @@
+namespace Tilework.Core.Enums;
+public enum ContainerExceptionType
+{
+    PORT_CONFLICT
+}

tilework.core/Enums/Core/ContainerRestartType.cs

@@ -0,0 +1,7 @@
+namespace Tilework.Core.Enums;
+public enum ContainerRestartType
+{
+    SIGNAL,
+    RESTART,
+    RECREATE
+}

tilework.core/Enums/Monitoring/MonitoringPersistenceType.cs

@@ -0,0 +1,6 @@
+namespace Tilework.Monitoring.Enums;
+
+public enum MonitoringPersistenceType
+{
+    INFLUXDB
+}

tilework.core/Enums/Monitoring/MonitoringSourceType.cs

@@ -0,0 +1,6 @@
+namespace Tilework.Monitoring.Enums;
+
+public enum MonitoringSourceType
+{
+    HAPROXY
+}

tilework.core/Events/Events/CertificateRenewed.cs

@@ -0,0 +1,12 @@
+using Coravel.Events.Interfaces;
+
+using Tilework.CertificateManagement.Models;
+
+namespace Tilework.Events;
+
+public class CertificateRenewed : IEvent
+{
+    public CertificateDTO Certificate { get; set; }
+
+    public CertificateRenewed(CertificateDTO certificate) => Certificate = certificate;
+}

tilework.core/Events/Listeners/LoadBalancerCertificateListener.cs

@@ -0,0 +1,26 @@
+using Microsoft.Extensions.Logging;
+
+using Coravel.Events.Interfaces;
+
+using Tilework.LoadBalancing.Interfaces;
+
+namespace Tilework.Events;
+
+public class LoadBalancerCertificateListener : IListener<CertificateRenewed>
+{
+    private readonly ILoadBalancerService _loadBalancer;
+    private readonly ILogger<LoadBalancerCertificateListener> _logger;
+
+    public LoadBalancerCertificateListener(ILoadBalancerService loadBalancer,
+                                           ILogger<LoadBalancerCertificateListener> logger)
+    {
+        _loadBalancer = loadBalancer;
+        _logger = logger;
+    }
+
+    public async Task HandleAsync(CertificateRenewed evt)
+    {
+        _logger.LogInformation($"Applying load balancer configuration due to renewal of certificate {evt.Certificate.Name}");
+        await _loadBalancer.ApplyConfiguration();
+    }
+}

tilework.core/Exceptions/Core/DockerException.cs

@@ -1,11 +1,37 @@
 using System.Text.Json;
+using System.Text.RegularExpressions;
+using Tilework.Core.Enums;
 
 namespace Tilework.Exceptions.Core;
 
 public class DockerException : Exception
 {
-    public DockerException(string message) : base(ParseResponseBody(message))
+    public ContainerExceptionType? Type { get; set; } = null;
+
+    private static readonly Regex PortConflictRegex = new(
+        @"failed\s+to\s+bind\s+host\s+port\s+[^\s:]+:\d+(?:/\w+)?:\s+address\s+already\s+in\s+use",
+        RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.CultureInvariant);
+
+    public DockerException(string message) : base(BuildExceptionMessage(message, out var parsedMessage))
     {
+        Type = DetectExceptionType(parsedMessage);
+    }
+
+    private static ContainerExceptionType? DetectExceptionType(string? message)
+    {
+        if (string.IsNullOrWhiteSpace(message))
+            return null;
+
+        if (PortConflictRegex.IsMatch(message))
+            return ContainerExceptionType.PORT_CONFLICT;
+
+        return null;
+    }
+
+    private static string BuildExceptionMessage(string message, out string? parsedMessage)
+    {
+        parsedMessage = ParseResponseBody(message);
+        return parsedMessage ?? message;
     }
 
     private static string? ParseResponseBody(string message)
@@ -19,4 +45,4 @@ public DockerException(string message) : base(ParseResponseBody(message))
 public class ResponseBody
 {
     public string message { get; set; }
-}
+}

tilework.core/Exceptions/Core/PortConflictException.cs

@@ -0,0 +1,12 @@
+using System.Text.Json;
+using System.Text.RegularExpressions;
+using Tilework.Core.Enums;
+
+namespace Tilework.Exceptions.Core;
+
+public class PortConfictException : Exception
+{
+    public PortConfictException(string? message = null) : base(message)
+    {
+    }
+}

tilework.core/Initializers/CertificateManagementInitializer.cs

@@ -1,8 +1,9 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Hosting;
-
 using Microsoft.Extensions.DependencyInjection;
 
+using Coravel;
+using Tilework.Core.Jobs.CertificateManagement;
 
 
 namespace Tilework.CertificateManagement.Services;
@@ -18,11 +19,22 @@ public CertificateManagementInitializer(ILogger<CertificateManagementInitializer
         _serviceProvider = serviceProvider;
     }
 
-    public Task StartAsync(CancellationToken ct) => Task.CompletedTask;
+    public async Task StartAsync(CancellationToken ct)
+    {
+        _logger.LogInformation($"Initiating startup for module: CertificateManagement");
+        await using var scope = _serviceProvider.CreateAsyncScope();
+
+        scope.ServiceProvider.UseScheduler(s =>
+        {
+            s.Schedule<CertificateRenewalJob>()
+             .Hourly()
+             .PreventOverlapping("CertificateRenewalJob");
+        });   
+    }
 
     public async Task StopAsync(CancellationToken ct)
     {
-        _logger.LogInformation($"Initiating shutdown for module: LoadBalancing");
+        _logger.LogInformation($"Initiating shutdown for module: CertificateManagement");
         await using var scope = _serviceProvider.CreateAsyncScope();
         var loadBalancerService = scope.ServiceProvider.GetRequiredService<AcmeVerificationService>();
 

tilework.core/Initializers/CoreInitializer.cs

@@ -3,7 +3,10 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.EntityFrameworkCore;
 
+using Coravel;
+
 using Tilework.Core.Persistence;
+using Tilework.Core.Interfaces;
 
 namespace Tilework.Core.Services;
 
@@ -27,6 +30,11 @@ public async Task StartAsync(CancellationToken ct)
         _logger.LogInformation($"Running migrations for context: TileworkContext");
         var dbContext = scope.ServiceProvider.GetRequiredService<TileworkContext>();
         await dbContext.Database.MigrateAsync(ct);
+
+        scope.ServiceProvider.ConfigureEvents();
+
+        var containerManager = scope.ServiceProvider.GetRequiredService<IContainerManager>();
+        containerManager.Initialize();
     }
 
 

tilework.core/Initializers/LoadBalancingInitializer.cs

@@ -2,7 +2,10 @@
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.DependencyInjection;
 
+using Coravel;
+
 using Tilework.LoadBalancing.Interfaces;
+using Tilework.Events;
 
 namespace Tilework.LoadBalancing.Services;
 
@@ -25,6 +28,10 @@ public async Task StartAsync(CancellationToken ct)
 
         var loadBalancerService = scope.ServiceProvider.GetRequiredService<ILoadBalancerService>();
         await loadBalancerService.ApplyConfiguration();
+
+        var events = scope.ServiceProvider.ConfigureEvents();
+        events.Register<CertificateRenewed>()
+              .Subscribe<LoadBalancerCertificateListener>();
     }
 
     public async Task StopAsync(CancellationToken ct)
@@ -35,4 +42,4 @@ public async Task StopAsync(CancellationToken ct)
 
         await loadBalancerService.Shutdown();
     }
-}
+}

tilework.core/Initializers/MonitoringInitializer.cs

@@ -0,0 +1,40 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.DependencyInjection;
+
+using Coravel;
+
+using Tilework.Monitoring.Services;
+
+namespace Tilework.LoadBalancing.Services;
+
+public sealed class MonitoringInitializer : IHostedService
+{
+    private readonly IServiceProvider _serviceProvider;
+    private readonly ILogger<MonitoringInitializer> _logger;
+
+    public MonitoringInitializer(ILogger<MonitoringInitializer> logger,
+                                 IServiceProvider serviceProvider)
+    {
+        _logger = logger;
+        _serviceProvider = serviceProvider;
+    }
+
+    public async Task StartAsync(CancellationToken ct)
+    {
+        _logger.LogInformation($"Initiating startup for module: Monitoring");
+        await using var scope = _serviceProvider.CreateAsyncScope();
+
+        var dataCollectorService = scope.ServiceProvider.GetRequiredService<DataCollectorService>();
+        await dataCollectorService.ApplyConfiguration();
+    }
+
+    public async Task StopAsync(CancellationToken ct)
+    {
+        _logger.LogInformation($"Initiating shutdown for module: Monitoring");
+        await using var scope = _serviceProvider.CreateAsyncScope();
+        var dataCollectorService = scope.ServiceProvider.GetRequiredService<DataCollectorService>();
+
+        await dataCollectorService.Shutdown();
+    }
+}

tilework.core/Interfaces/CertificateManagement/ICertificateManagementService.cs

@@ -14,6 +14,8 @@ public interface ICertificateManagementService
     public Task<List<CertificateDTO>> GetCertificates();
     public Task<CertificateDTO?> GetCertificate(Guid Id);
     public Task<CertificateDTO> AddCertificate(string name, string fqdn, KeyAlgorithm algorithm, Guid authorityId);
+    public Task<CertificateDTO> RenewCertificate(Guid Id);
+    public Task RenewExpiringCertificates();
     public Task RevokeCertificate(Guid Id);
     public Task DeleteCertificate(Guid Id);
 

tilework.core/Interfaces/Core/IContainerManager.cs

@@ -1,14 +1,21 @@
+using System.Net;
+
 using Tilework.Core.Enums;
 using Tilework.Core.Models;
 
 namespace Tilework.Core.Interfaces;
 
 public interface IContainerManager
 {
+    public Task Initialize();
+
     public Task<List<ContainerNetwork>> ListNetworks();
     public Task<ContainerNetwork> CreateNetwork(string name);
     public Task DeleteNetwork(string id);
 
+    public Task<IPAddress> GetContainerAddress(string id);
+    public Task<List<ContainerPort>> GetContainerPorts(string id);
+
     public Task<List<Container>> ListContainers(string? module);
     public Task<Container> CreateContainer(string name, string image, string module, List<ContainerPort>? ports);
     public Task DeleteContainer(string id);
@@ -17,4 +24,6 @@ public interface IContainerManager
     public Task StartContainer(string id);
     public Task StopContainer(string id);
     public Task KillContainer(string id, UnixSignal signal);
+
+    public Task<ContainerCommandResult> ExecuteContainerCommand(string id, string command);
 }

tilework.core/Interfaces/LoadBalancing/ILoadBalancerService.cs

@@ -14,6 +14,8 @@ public interface ILoadBalancerService
     public Task EnableLoadBalancer(Guid Id);
     public Task DisableLoadBalancer(Guid Id);
 
+
+
     public Task<List<RuleDTO>> GetRules(ApplicationLoadBalancerDTO balancer);
     public Task AddRule(ApplicationLoadBalancerDTO balancer, RuleDTO rule);
     public Task UpdateRule(ApplicationLoadBalancerDTO balancer, RuleDTO rule);
@@ -40,6 +42,9 @@ public interface ILoadBalancerService
     public Task UpdateTarget(TargetGroupDTO group, TargetDTO target);
     public Task RemoveTarget(TargetGroupDTO group, TargetDTO target);
 
+    public Task<List<LoadBalancingMonitorData>> GetLoadBalancerMonitoringData(Guid Id, TimeSpan interval, DateTimeOffset start, DateTimeOffset end);
+
+    public Task ApplyConfiguration(Guid Id);
     public Task ApplyConfiguration();
     public Task Shutdown();
-}
+}