feat(lambda): add observability

.github/actions/deploy/action.yml

@@ -54,6 +54,8 @@ runs:
         SPOTIFY_REFRESH_TOKEN: op://ci-cd/lho-lambda/SPOTIFY_REFRESH_TOKEN
         LASTFM_API_KEY: op://ci-cd/lho-lambda/LASTFM_API_KEY
         LASTFM_USERNAME: op://ci-cd/lho-lambda/LASTFM_USERNAME
+        SENTRY_DSN: op://ci-cd/lho-lambda/SENTRY_DSN
+        PUSHGATEWAY_AUTH_HEADER: op://ci-cd/lho-lambda/PUSHGATEWAY_AUTH_HEADER
         OP_ENV_FILE: ".env"
 
     - name: Set Terraform variables
@@ -64,6 +66,8 @@ runs:
         echo "TF_VAR_spotify_refresh_token=$SPOTIFY_REFRESH_TOKEN" >> $GITHUB_ENV
         echo "TF_VAR_lastfm_api_key=$LASTFM_API_KEY" >> $GITHUB_ENV
         echo "TF_VAR_lastfm_username=$LASTFM_USERNAME" >> $GITHUB_ENV
+        echo "TF_VAR_sentry_dsn=$SENTRY_DSN" >> $GITHUB_ENV
+        echo "TF_VAR_pushgateway_auth_header=$PUSHGATEWAY_AUTH_HEADER" >> $GITHUB_ENV
 
     - name: Terraform init
       shell: bash
@@ -118,6 +122,26 @@ runs:
           echo "APP_VERSION=$APP_VERSION" >> $GITHUB_ENV
         fi
 
+    - name: Import existing authorizer log group
+      shell: bash
+      working-directory: terraform
+      run: |
+        set -euo pipefail
+
+        project_name="${TF_VAR_project_name:-now-playing}"
+        log_group_name="/aws/lambda/${project_name}-api-authorizer-${{ inputs.environment }}"
+
+        if terraform state show aws_cloudwatch_log_group.auth_logs >/dev/null 2>&1; then
+          echo "Authorizer log group is already managed by Terraform"
+          exit 0
+        fi
+
+        if aws logs describe-log-groups --log-group-name-prefix "$log_group_name" --query 'logGroups[].logGroupName' --output text | tr '\t' '\n' | grep -Fxq "$log_group_name"; then
+          terraform import -input=false aws_cloudwatch_log_group.auth_logs "$log_group_name"
+        else
+          echo "No existing authorizer log group to import"
+        fi
+
     - name: Terraform plan
       id: plan
       shell: bash

Lho.Lambda.slnx

@@ -2,6 +2,7 @@
   <Folder Name="/src/">
     <Project Path="src/Lho.Lambda.Authorizer/Lho.Lambda.Authorizer.csproj" />
     <Project Path="src/Lho.Lambda.Local/Lho.Lambda.Local.csproj" />
+    <Project Path="src/Lho.Lambda.Observability/Lho.Lambda.Observability.csproj" />
     <Project Path="src/Lho.Lambda.Tests/Lho.Lambda.Tests.csproj" />
     <Project Path="src/Lho.Lambda/Lho.Lambda.csproj" />
   </Folder>

src/Lho.Lambda.Authorizer/Functions/AuthorizerFunction.cs

@@ -1,36 +1,110 @@
+using System.Diagnostics;
 using System.Security.Cryptography;
 using System.Text;
 using Amazon.Lambda.Core;
 using Lho.Lambda.Authorizer.Models;
+using Lho.Lambda.Observability;
 
 namespace Lho.Lambda.Authorizer.Functions;
 
 public class AuthorizerFunction
 {
     private static readonly HashSet<string> ValidConsumers = ["lhowsam-dev", "lhowsam-prod", "lhowsam-local"];
 
-    public Task<AuthorizerSimpleResponse> FunctionHandler(AuthorizerRequest request, ILambdaContext context)
+    public async Task<AuthorizerSimpleResponse> FunctionHandler(AuthorizerRequest request, ILambdaContext context)
     {
-        context.Logger.LogLine("Authorizer invoked");
-
-        var apiKey = GetHeaderValue(request.Headers, "x-api-key");
-        var validKey = Environment.GetEnvironmentVariable("API_KEY");
+        var stopwatch = Stopwatch.StartNew();
+        var consumer = NormaliseConsumer(GetHeaderValue(request.Headers, "x-consumer"));
+        var route = request.RouteKey ?? request.RequestContext?.Http?.Path ?? "authorizer";
+        var method = request.RequestContext?.Http?.Method ?? "AUTH";
+        var reason = "allowed";
+        var isAuthorized = false;
+        Exception? capturedException = null;
+        SentryTelemetry.Initialise(context.Logger);
+        using var transaction = SentryTelemetry.StartTransaction(context.Logger, $"{method} {route}", "http.server", new Dictionary<string, string?>
+        {
+            ["function"] = context.FunctionName,
+            ["request_id"] = context.AwsRequestId,
+            ["operation"] = "authorizer",
+            ["route"] = route,
+            ["method"] = method,
+            ["consumer"] = consumer
+        });
 
-        if (!SecureCompare(apiKey, validKey))
+        try
         {
-            context.Logger.LogLine("Deny - API key invalid");
-            return Task.FromResult(new AuthorizerSimpleResponse(false));
-        }
+            var apiKey = GetHeaderValue(request.Headers, "x-api-key");
+            var validKey = Environment.GetEnvironmentVariable("API_KEY");
+
+            if (!SecureCompare(apiKey, validKey))
+            {
+                reason = "invalid_api_key";
+                return new AuthorizerSimpleResponse(false);
+            }
 
-        var consumer = GetHeaderValue(request.Headers, "x-consumer");
-        if (consumer is not null && !ValidConsumers.Contains(consumer))
+            if (consumer is not null && !ValidConsumers.Contains(consumer))
+            {
+                reason = "invalid_consumer";
+                return new AuthorizerSimpleResponse(false);
+            }
+
+            isAuthorized = true;
+            return new AuthorizerSimpleResponse(true);
+        }
+        catch (Exception exception)
         {
-            context.Logger.LogLine("Deny - Invalid consumer");
-            return Task.FromResult(new AuthorizerSimpleResponse(false));
+            capturedException = exception;
+            reason = "exception";
+            StructuredLog.Error(context.Logger, "authorizer.error", exception, new Dictionary<string, object?>
+            {
+                ["requestId"] = context.AwsRequestId,
+                ["function"] = context.FunctionName,
+                ["route"] = route,
+                ["method"] = method,
+                ["consumer"] = consumer
+            });
+            await SentryTelemetry.CaptureExceptionAsync(exception, context.Logger, new Dictionary<string, string?>
+            {
+                ["function"] = context.FunctionName,
+                ["request_id"] = context.AwsRequestId,
+                ["operation"] = "authorizer",
+                ["route"] = route,
+                ["consumer"] = consumer
+            });
+            throw;
         }
+        finally
+        {
+            stopwatch.Stop();
+            var statusCode = isAuthorized ? 200 : 401;
+            var outcome = isAuthorized ? "success" : "denied";
+            StructuredLog.Info(context.Logger, "authorizer.request", new Dictionary<string, object?>
+            {
+                ["requestId"] = context.AwsRequestId,
+                ["function"] = context.FunctionName,
+                ["route"] = route,
+                ["method"] = method,
+                ["statusCode"] = statusCode,
+                ["durationMs"] = Math.Round(stopwatch.Elapsed.TotalMilliseconds, 2),
+                ["outcome"] = outcome,
+                ["reason"] = reason,
+                ["consumer"] = consumer
+            });
+
+            var metric = new InvocationMetric(
+                FunctionName: context.FunctionName,
+                Operation: "authorizer",
+                Route: route,
+                Method: method,
+                StatusCode: statusCode,
+                DurationMs: stopwatch.Elapsed.TotalMilliseconds,
+                Outcome: outcome,
+                Consumer: consumer);
 
-        context.Logger.LogLine("Allow");
-        return Task.FromResult(new AuthorizerSimpleResponse(true));
+            transaction?.Finish(statusCode, capturedException);
+            await PrometheusMetrics.PushInvocationAsync(metric, context.Logger);
+            await SentryTelemetry.RecordInvocationAsync(metric, context.Logger);
+        }
     }
 
     private static string? GetHeaderValue(IReadOnlyDictionary<string, string>? headers, string key)
@@ -55,7 +129,7 @@ private static bool SecureCompare(string? first, string? second)
     {
         if (first is null || second is null)
         {
-            return first is null && second is null;
+            return false;
         }
 
         var firstBytes = Encoding.UTF8.GetBytes(first);
@@ -64,4 +138,14 @@ private static bool SecureCompare(string? first, string? second)
         return firstBytes.Length == secondBytes.Length &&
                CryptographicOperations.FixedTimeEquals(firstBytes, secondBytes);
     }
+
+    private static string? NormaliseConsumer(string? consumer)
+    {
+        return consumer switch
+        {
+            "lhowsam-prod" or "lhowsam-dev" or "lhowsam-local" => consumer,
+            null or "" => null,
+            _ => "unknown"
+        };
+    }
 }

src/Lho.Lambda.Authorizer/Lho.Lambda.Authorizer.csproj

@@ -8,4 +8,8 @@
     <PackageReference Include="Amazon.Lambda.Core" Version="2.8.0" />
     <PackageReference Include="Amazon.Lambda.Serialization.SystemTextJson" Version="2.4.4" />
   </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Lho.Lambda.Observability\Lho.Lambda.Observability.csproj" />
+  </ItemGroup>
 </Project>

src/Lho.Lambda.Observability/Lho.Lambda.Observability.csproj

@@ -0,0 +1,10 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Amazon.Lambda.Core" Version="2.8.0" />
+    <PackageReference Include="Sentry" Version="6.5.0" />
+  </ItemGroup>
+</Project>

src/Lho.Lambda.Observability/ObservabilityConfig.cs

@@ -0,0 +1,51 @@
+namespace Lho.Lambda.Observability;
+
+public static class ObservabilityConfig
+{
+  public static string ServiceName => String("SERVICE_NAME", "now-playing");
+
+  public static string EnvironmentName => String("ENVIRONMENT", "local");
+
+  public static string Version => String("VERSION", "unknown");
+
+  public static string GitSha => String("GIT_SHA", "unknown");
+
+  public static string SentryEnvironment => String("SENTRY_ENVIRONMENT", EnvironmentName);
+
+  public static string SentryRelease => String("SENTRY_RELEASE", Version);
+
+  public static string? SentryDsn => OptionalString("SENTRY_DSN");
+
+  public static string? PushgatewayUrl => OptionalString("PUSHGATEWAY_URL");
+
+  public static string? PushgatewayAuthHeader => OptionalString("PUSHGATEWAY_AUTH_HEADER");
+
+  public static string PushgatewayJob => String("PROMETHEUS_JOB", ServiceName);
+
+  public static bool MetricsEnabled => Bool("METRICS_ENABLED", defaultValue: true) && !string.IsNullOrEmpty(PushgatewayUrl);
+
+  public static double SentryTracesSampleRate => Double("SENTRY_TRACES_SAMPLE_RATE", defaultValue: 0.5);
+
+  private static string String(string key, string defaultValue)
+  {
+    return Environment.GetEnvironmentVariable(key) ?? defaultValue;
+  }
+
+  private static string? OptionalString(string key)
+  {
+    var value = Environment.GetEnvironmentVariable(key);
+    return string.IsNullOrWhiteSpace(value) ? null : value;
+  }
+
+  private static bool Bool(string key, bool defaultValue)
+  {
+    var value = Environment.GetEnvironmentVariable(key);
+    return value is null ? defaultValue : string.Equals(value, "true", StringComparison.OrdinalIgnoreCase) || value == "1";
+  }
+
+  private static double Double(string key, double defaultValue)
+  {
+    var value = Environment.GetEnvironmentVariable(key);
+    return double.TryParse(value, out var parsed) ? Math.Clamp(parsed, 0, 1) : defaultValue;
+  }
+}