diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml index fb61d54..7d89c22 100644 --- a/.github/actions/deploy/action.yml +++ b/.github/actions/deploy/action.yml @@ -54,6 +54,8 @@ runs: SPOTIFY_REFRESH_TOKEN: op://ci-cd/lho-lambda/SPOTIFY_REFRESH_TOKEN LASTFM_API_KEY: op://ci-cd/lho-lambda/LASTFM_API_KEY LASTFM_USERNAME: op://ci-cd/lho-lambda/LASTFM_USERNAME + SENTRY_DSN: op://ci-cd/lho-lambda/SENTRY_DSN + PUSHGATEWAY_AUTH_HEADER: op://ci-cd/lho-lambda/PUSHGATEWAY_AUTH_HEADER OP_ENV_FILE: ".env" - name: Set Terraform variables @@ -64,6 +66,8 @@ runs: echo "TF_VAR_spotify_refresh_token=$SPOTIFY_REFRESH_TOKEN" >> $GITHUB_ENV echo "TF_VAR_lastfm_api_key=$LASTFM_API_KEY" >> $GITHUB_ENV echo "TF_VAR_lastfm_username=$LASTFM_USERNAME" >> $GITHUB_ENV + echo "TF_VAR_sentry_dsn=$SENTRY_DSN" >> $GITHUB_ENV + echo "TF_VAR_pushgateway_auth_header=$PUSHGATEWAY_AUTH_HEADER" >> $GITHUB_ENV - name: Terraform init shell: bash @@ -118,6 +122,26 @@ runs: echo "APP_VERSION=$APP_VERSION" >> $GITHUB_ENV fi + - name: Import existing authorizer log group + shell: bash + working-directory: terraform + run: | + set -euo pipefail + + project_name="${TF_VAR_project_name:-now-playing}" + log_group_name="/aws/lambda/${project_name}-api-authorizer-${{ inputs.environment }}" + + if terraform state show aws_cloudwatch_log_group.auth_logs >/dev/null 2>&1; then + echo "Authorizer log group is already managed by Terraform" + exit 0 + fi + + if aws logs describe-log-groups --log-group-name-prefix "$log_group_name" --query 'logGroups[].logGroupName' --output text | tr '\t' '\n' | grep -Fxq "$log_group_name"; then + terraform import -input=false aws_cloudwatch_log_group.auth_logs "$log_group_name" + else + echo "No existing authorizer log group to import" + fi + - name: Terraform plan id: plan shell: bash diff --git a/Lho.Lambda.slnx b/Lho.Lambda.slnx index 4f3c8ea..a119ad5 100644 --- a/Lho.Lambda.slnx +++ b/Lho.Lambda.slnx @@ -2,6 +2,7 @@ + diff --git a/src/Lho.Lambda.Authorizer/Functions/AuthorizerFunction.cs b/src/Lho.Lambda.Authorizer/Functions/AuthorizerFunction.cs index 278992f..35d552a 100644 --- a/src/Lho.Lambda.Authorizer/Functions/AuthorizerFunction.cs +++ b/src/Lho.Lambda.Authorizer/Functions/AuthorizerFunction.cs @@ -1,7 +1,9 @@ +using System.Diagnostics; using System.Security.Cryptography; using System.Text; using Amazon.Lambda.Core; using Lho.Lambda.Authorizer.Models; +using Lho.Lambda.Observability; namespace Lho.Lambda.Authorizer.Functions; @@ -9,28 +11,100 @@ public class AuthorizerFunction { private static readonly HashSet ValidConsumers = ["lhowsam-dev", "lhowsam-prod", "lhowsam-local"]; - public Task FunctionHandler(AuthorizerRequest request, ILambdaContext context) + public async Task FunctionHandler(AuthorizerRequest request, ILambdaContext context) { - context.Logger.LogLine("Authorizer invoked"); - - var apiKey = GetHeaderValue(request.Headers, "x-api-key"); - var validKey = Environment.GetEnvironmentVariable("API_KEY"); + var stopwatch = Stopwatch.StartNew(); + var consumer = NormaliseConsumer(GetHeaderValue(request.Headers, "x-consumer")); + var route = request.RouteKey ?? request.RequestContext?.Http?.Path ?? "authorizer"; + var method = request.RequestContext?.Http?.Method ?? "AUTH"; + var reason = "allowed"; + var isAuthorized = false; + Exception? capturedException = null; + SentryTelemetry.Initialise(context.Logger); + using var transaction = SentryTelemetry.StartTransaction(context.Logger, $"{method} {route}", "http.server", new Dictionary + { + ["function"] = context.FunctionName, + ["request_id"] = context.AwsRequestId, + ["operation"] = "authorizer", + ["route"] = route, + ["method"] = method, + ["consumer"] = consumer + }); - if (!SecureCompare(apiKey, validKey)) + try { - context.Logger.LogLine("Deny - API key invalid"); - return Task.FromResult(new AuthorizerSimpleResponse(false)); - } + var apiKey = GetHeaderValue(request.Headers, "x-api-key"); + var validKey = Environment.GetEnvironmentVariable("API_KEY"); + + if (!SecureCompare(apiKey, validKey)) + { + reason = "invalid_api_key"; + return new AuthorizerSimpleResponse(false); + } - var consumer = GetHeaderValue(request.Headers, "x-consumer"); - if (consumer is not null && !ValidConsumers.Contains(consumer)) + if (consumer is not null && !ValidConsumers.Contains(consumer)) + { + reason = "invalid_consumer"; + return new AuthorizerSimpleResponse(false); + } + + isAuthorized = true; + return new AuthorizerSimpleResponse(true); + } + catch (Exception exception) { - context.Logger.LogLine("Deny - Invalid consumer"); - return Task.FromResult(new AuthorizerSimpleResponse(false)); + capturedException = exception; + reason = "exception"; + StructuredLog.Error(context.Logger, "authorizer.error", exception, new Dictionary + { + ["requestId"] = context.AwsRequestId, + ["function"] = context.FunctionName, + ["route"] = route, + ["method"] = method, + ["consumer"] = consumer + }); + await SentryTelemetry.CaptureExceptionAsync(exception, context.Logger, new Dictionary + { + ["function"] = context.FunctionName, + ["request_id"] = context.AwsRequestId, + ["operation"] = "authorizer", + ["route"] = route, + ["consumer"] = consumer + }); + throw; } + finally + { + stopwatch.Stop(); + var statusCode = isAuthorized ? 200 : 401; + var outcome = isAuthorized ? "success" : "denied"; + StructuredLog.Info(context.Logger, "authorizer.request", new Dictionary + { + ["requestId"] = context.AwsRequestId, + ["function"] = context.FunctionName, + ["route"] = route, + ["method"] = method, + ["statusCode"] = statusCode, + ["durationMs"] = Math.Round(stopwatch.Elapsed.TotalMilliseconds, 2), + ["outcome"] = outcome, + ["reason"] = reason, + ["consumer"] = consumer + }); + + var metric = new InvocationMetric( + FunctionName: context.FunctionName, + Operation: "authorizer", + Route: route, + Method: method, + StatusCode: statusCode, + DurationMs: stopwatch.Elapsed.TotalMilliseconds, + Outcome: outcome, + Consumer: consumer); - context.Logger.LogLine("Allow"); - return Task.FromResult(new AuthorizerSimpleResponse(true)); + transaction?.Finish(statusCode, capturedException); + await PrometheusMetrics.PushInvocationAsync(metric, context.Logger); + await SentryTelemetry.RecordInvocationAsync(metric, context.Logger); + } } private static string? GetHeaderValue(IReadOnlyDictionary? headers, string key) @@ -55,7 +129,7 @@ private static bool SecureCompare(string? first, string? second) { if (first is null || second is null) { - return first is null && second is null; + return false; } var firstBytes = Encoding.UTF8.GetBytes(first); @@ -64,4 +138,14 @@ private static bool SecureCompare(string? first, string? second) return firstBytes.Length == secondBytes.Length && CryptographicOperations.FixedTimeEquals(firstBytes, secondBytes); } + + private static string? NormaliseConsumer(string? consumer) + { + return consumer switch + { + "lhowsam-prod" or "lhowsam-dev" or "lhowsam-local" => consumer, + null or "" => null, + _ => "unknown" + }; + } } diff --git a/src/Lho.Lambda.Authorizer/Lho.Lambda.Authorizer.csproj b/src/Lho.Lambda.Authorizer/Lho.Lambda.Authorizer.csproj index ff5922f..a76e48b 100644 --- a/src/Lho.Lambda.Authorizer/Lho.Lambda.Authorizer.csproj +++ b/src/Lho.Lambda.Authorizer/Lho.Lambda.Authorizer.csproj @@ -8,4 +8,8 @@ + + + + diff --git a/src/Lho.Lambda.Observability/Lho.Lambda.Observability.csproj b/src/Lho.Lambda.Observability/Lho.Lambda.Observability.csproj new file mode 100644 index 0000000..b3e1165 --- /dev/null +++ b/src/Lho.Lambda.Observability/Lho.Lambda.Observability.csproj @@ -0,0 +1,10 @@ + + + net8.0 + + + + + + + diff --git a/src/Lho.Lambda.Observability/ObservabilityConfig.cs b/src/Lho.Lambda.Observability/ObservabilityConfig.cs new file mode 100644 index 0000000..35504c7 --- /dev/null +++ b/src/Lho.Lambda.Observability/ObservabilityConfig.cs @@ -0,0 +1,51 @@ +namespace Lho.Lambda.Observability; + +public static class ObservabilityConfig +{ + public static string ServiceName => String("SERVICE_NAME", "now-playing"); + + public static string EnvironmentName => String("ENVIRONMENT", "local"); + + public static string Version => String("VERSION", "unknown"); + + public static string GitSha => String("GIT_SHA", "unknown"); + + public static string SentryEnvironment => String("SENTRY_ENVIRONMENT", EnvironmentName); + + public static string SentryRelease => String("SENTRY_RELEASE", Version); + + public static string? SentryDsn => OptionalString("SENTRY_DSN"); + + public static string? PushgatewayUrl => OptionalString("PUSHGATEWAY_URL"); + + public static string? PushgatewayAuthHeader => OptionalString("PUSHGATEWAY_AUTH_HEADER"); + + public static string PushgatewayJob => String("PROMETHEUS_JOB", ServiceName); + + public static bool MetricsEnabled => Bool("METRICS_ENABLED", defaultValue: true) && !string.IsNullOrEmpty(PushgatewayUrl); + + public static double SentryTracesSampleRate => Double("SENTRY_TRACES_SAMPLE_RATE", defaultValue: 0.5); + + private static string String(string key, string defaultValue) + { + return Environment.GetEnvironmentVariable(key) ?? defaultValue; + } + + private static string? OptionalString(string key) + { + var value = Environment.GetEnvironmentVariable(key); + return string.IsNullOrWhiteSpace(value) ? null : value; + } + + private static bool Bool(string key, bool defaultValue) + { + var value = Environment.GetEnvironmentVariable(key); + return value is null ? defaultValue : string.Equals(value, "true", StringComparison.OrdinalIgnoreCase) || value == "1"; + } + + private static double Double(string key, double defaultValue) + { + var value = Environment.GetEnvironmentVariable(key); + return double.TryParse(value, out var parsed) ? Math.Clamp(parsed, 0, 1) : defaultValue; + } +} diff --git a/src/Lho.Lambda.Observability/PrometheusMetrics.cs b/src/Lho.Lambda.Observability/PrometheusMetrics.cs new file mode 100644 index 0000000..4d10e76 --- /dev/null +++ b/src/Lho.Lambda.Observability/PrometheusMetrics.cs @@ -0,0 +1,149 @@ +using System.Collections.Concurrent; +using System.Globalization; +using System.Net.Http.Headers; +using System.Text; +using Amazon.Lambda.Core; + +namespace Lho.Lambda.Observability; + +public static class PrometheusMetrics +{ + private static readonly HttpClient HttpClient = new() + { + Timeout = TimeSpan.FromSeconds(2) + }; + private static readonly ConcurrentDictionary Counters = new(); + + public static async Task PushInvocationAsync(InvocationMetric metric, ILambdaLogger logger) + { + if (!ObservabilityConfig.MetricsEnabled) + { + return; + } + + try + { + var counterKey = $"{metric.FunctionName}|{metric.Operation}|{metric.Route}|{metric.Method}|{metric.StatusCode}|{metric.Outcome}|{metric.Consumer}|{metric.Provider}"; + var count = Counters.AddOrUpdate(counterKey, 1, (_, current) => current + 1); + var body = BuildBody(metric, count); + var endpoint = BuildPushgatewayEndpoint(metric.FunctionName); + + using var request = new HttpRequestMessage(HttpMethod.Post, endpoint); + request.Content = new StringContent(body, Encoding.UTF8); + request.Content.Headers.ContentType = new MediaTypeHeaderValue("text/plain") + { + CharSet = "utf-8" + }; + AddConfiguredAuthHeader(request); + + using var response = await HttpClient.SendAsync(request); + if (!response.IsSuccessStatusCode) + { + logger.LogLine($"Pushgateway rejected metrics with status {(int)response.StatusCode}"); + } + } + catch (Exception exception) + { + logger.LogLine($"Failed to push Prometheus metrics: {exception.Message}"); + } + } + + private static string BuildBody(InvocationMetric metric, long count) + { + var labels = Labels(metric); + var durationSeconds = metric.DurationMs / 1000d; + var timestamp = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); + + return string.Join('\n', [ + "# TYPE lho_lambda_invocations_total counter", + $"lho_lambda_invocations_total{{{labels}}} {count.ToString(CultureInfo.InvariantCulture)}", + "# TYPE lho_lambda_invocation_duration_seconds gauge", + $"lho_lambda_invocation_duration_seconds{{{labels}}} {durationSeconds.ToString("0.###", CultureInfo.InvariantCulture)}", + "# TYPE lho_lambda_last_invocation_timestamp_seconds gauge", + $"lho_lambda_last_invocation_timestamp_seconds{{{labels}}} {timestamp.ToString(CultureInfo.InvariantCulture)}", + "" + ]); + } + + private static string Labels(InvocationMetric metric) + { + var labels = new Dictionary + { + ["service"] = ObservabilityConfig.ServiceName, + ["environment"] = ObservabilityConfig.EnvironmentName, + ["version"] = ObservabilityConfig.Version, + ["git_sha"] = ObservabilityConfig.GitSha, + ["function"] = metric.FunctionName, + ["operation"] = metric.Operation, + ["route"] = metric.Route, + ["method"] = metric.Method, + ["status"] = metric.StatusCode.ToString(CultureInfo.InvariantCulture), + ["outcome"] = metric.Outcome + }; + + if (!string.IsNullOrWhiteSpace(metric.Consumer)) + { + labels["consumer"] = metric.Consumer; + } + + if (!string.IsNullOrWhiteSpace(metric.Provider)) + { + labels["provider"] = metric.Provider; + } + + return string.Join(",", labels.Select(label => $"{label.Key}=\"{EscapeLabelValue(label.Value)}\"")); + } + + private static Uri BuildPushgatewayEndpoint(string functionName) + { + var baseUri = ObservabilityConfig.PushgatewayUrl!.TrimEnd('/'); + var job = Uri.EscapeDataString(ObservabilityConfig.PushgatewayJob); + var environment = Uri.EscapeDataString(ObservabilityConfig.EnvironmentName); + var function = Uri.EscapeDataString(functionName); + + return new Uri($"{baseUri}/metrics/job/{job}/environment/{environment}/function/{function}"); + } + + private static void AddConfiguredAuthHeader(HttpRequestMessage request) + { + var authHeader = ObservabilityConfig.PushgatewayAuthHeader; + if (string.IsNullOrWhiteSpace(authHeader)) + { + return; + } + + var separator = authHeader.IndexOf('=', StringComparison.Ordinal); + if (separator <= 0 || separator == authHeader.Length - 1) + { + return; + } + + var name = authHeader[..separator].Trim(); + var value = authHeader[(separator + 1)..].Trim(); + if (string.IsNullOrWhiteSpace(name) || string.IsNullOrWhiteSpace(value)) + { + return; + } + + request.Headers.TryAddWithoutValidation(name, value); + } + + private static string EscapeLabelValue(string value) + { + return value + .Replace("\\", "\\\\", StringComparison.Ordinal) + .Replace("\n", "\\n", StringComparison.Ordinal) + .Replace("\"", "\\\"", StringComparison.Ordinal); + } +} + +public sealed record InvocationMetric( + string FunctionName, + string Operation, + string Route, + string Method, + int StatusCode, + double DurationMs, + string Outcome, + string? Consumer = null, + string? Provider = null); diff --git a/src/Lho.Lambda.Observability/SentryTelemetry.cs b/src/Lho.Lambda.Observability/SentryTelemetry.cs new file mode 100644 index 0000000..96e98d4 --- /dev/null +++ b/src/Lho.Lambda.Observability/SentryTelemetry.cs @@ -0,0 +1,215 @@ +using Amazon.Lambda.Core; +using Sentry; + +namespace Lho.Lambda.Observability; + +public static class SentryTelemetry +{ + private static readonly object InitLock = new(); + private static bool _initialised; + + public static bool Initialise(ILambdaLogger logger) + { + return EnsureInitialised(logger); + } + + public static SentryTransactionScope? StartTransaction( + ILambdaLogger logger, + string name, + string operation, + IReadOnlyDictionary tags) + { + if (!EnsureInitialised(logger)) + { + return null; + } + + var transaction = SentrySdk.StartTransaction(name, operation); + ApplyTags(transaction, tags); + SentrySdk.ConfigureScope(scope => + { + scope.Transaction = transaction; + ApplyTags(scope, tags); + }); + + return new SentryTransactionScope(transaction); + } + + public static async Task CaptureExceptionAsync( + Exception exception, + ILambdaLogger logger, + IReadOnlyDictionary tags) + { + if (!EnsureInitialised(logger)) + { + return; + } + + SentrySdk.CaptureException(exception, scope => + { + ApplyTags(scope, tags); + }); + + await SentrySdk.FlushAsync(TimeSpan.FromSeconds(2)); + } + + public static async Task RecordInvocationAsync(InvocationMetric metric, ILambdaLogger logger) + { + if (!EnsureInitialised(logger)) + { + return; + } + + var attributes = MetricAttributes(metric); + + SentrySdk.Metrics.EmitCounter("lambda.invocation", 1, attributes, null); + SentrySdk.Metrics.EmitDistribution("lambda.invocation.duration", metric.DurationMs, MeasurementUnit.Duration.Millisecond, attributes, null); + await SentrySdk.FlushAsync(TimeSpan.FromSeconds(2)); + } + + public static async Task FlushAsync(ILambdaLogger logger) + { + if (!EnsureInitialised(logger)) + { + return; + } + + await SentrySdk.FlushAsync(TimeSpan.FromSeconds(2)); + } + + private static bool EnsureInitialised(ILambdaLogger logger) + { + lock (InitLock) + { + if (_initialised) + { + return true; + } + } + + var dsn = ObservabilityConfig.SentryDsn; + if (string.IsNullOrWhiteSpace(dsn)) + { + return false; + } + + lock (InitLock) + { + if (_initialised) + { + return true; + } + + try + { + SentrySdk.Init(options => + { + options.Dsn = dsn; + options.Environment = ObservabilityConfig.SentryEnvironment; + options.Release = ObservabilityConfig.SentryRelease; + options.AttachStacktrace = true; + options.SampleRate = 1.0f; + options.EnableMetrics = true; + options.MaxBreadcrumbs = 50; + options.TracesSampleRate = ObservabilityConfig.SentryTracesSampleRate; + options.SendDefaultPii = false; + }); + _initialised = true; + } + catch (Exception exception) + { + logger.LogLine($"Failed to initialise Sentry: {exception.Message}"); + return false; + } + } + + return true; + } + + private static Dictionary MetricAttributes(InvocationMetric metric) + { + var attributes = new Dictionary + { + ["service"] = ObservabilityConfig.ServiceName, + ["environment"] = ObservabilityConfig.EnvironmentName, + ["version"] = ObservabilityConfig.Version, + ["git_sha"] = ObservabilityConfig.GitSha, + ["function"] = metric.FunctionName, + ["operation"] = metric.Operation, + ["route"] = metric.Route, + ["method"] = metric.Method, + ["status"] = metric.StatusCode.ToString(), + ["outcome"] = metric.Outcome + }; + + if (!string.IsNullOrWhiteSpace(metric.Consumer)) + { + attributes["consumer"] = metric.Consumer; + } + + if (!string.IsNullOrWhiteSpace(metric.Provider)) + { + attributes["provider"] = metric.Provider; + } + + return attributes; + } + + private static void ApplyTags(IHasTags target, IReadOnlyDictionary tags) + { + target.SetTag("service", ObservabilityConfig.ServiceName); + target.SetTag("environment", ObservabilityConfig.SentryEnvironment); + target.SetTag("version", ObservabilityConfig.SentryRelease); + target.SetTag("git_sha", ObservabilityConfig.GitSha); + + foreach (var (key, value) in tags) + { + if (!string.IsNullOrWhiteSpace(value)) + { + target.SetTag(key, value); + } + } + } + + private static SpanStatus StatusFromHttpStatus(int statusCode) + { + return statusCode switch + { + >= 500 => SpanStatus.InternalError, + 404 => SpanStatus.NotFound, + 401 => SpanStatus.Unauthenticated, + 403 => SpanStatus.PermissionDenied, + >= 400 => SpanStatus.InvalidArgument, + _ => SpanStatus.Ok + }; + } + + public sealed class SentryTransactionScope(ITransactionTracer transaction) : IDisposable + { + private bool _finished; + + public void Finish(int statusCode, Exception? exception = null) + { + if (_finished) + { + return; + } + + _finished = true; + var status = StatusFromHttpStatus(statusCode); + + if (exception is null) + { + transaction.Finish(status); + return; + } + + transaction.Finish(exception, status); + } + + public void Dispose() + { + Finish(200); + } + } +} diff --git a/src/Lho.Lambda.Observability/StructuredLog.cs b/src/Lho.Lambda.Observability/StructuredLog.cs new file mode 100644 index 0000000..cece61c --- /dev/null +++ b/src/Lho.Lambda.Observability/StructuredLog.cs @@ -0,0 +1,49 @@ +using System.Text.Json; +using Amazon.Lambda.Core; + +namespace Lho.Lambda.Observability; + +public static class StructuredLog +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }; + + public static void Info(ILambdaLogger logger, string eventName, IReadOnlyDictionary fields) + { + Write(logger, "info", eventName, fields); + } + + public static void Error(ILambdaLogger logger, string eventName, Exception exception, IReadOnlyDictionary fields) + { + var enrichedFields = new Dictionary(fields) + { + ["errorType"] = exception.GetType().Name, + ["errorMessage"] = exception.Message + }; + + Write(logger, "error", eventName, enrichedFields); + } + + private static void Write(ILambdaLogger logger, string level, string eventName, IReadOnlyDictionary fields) + { + var payload = new Dictionary + { + ["timestamp"] = DateTimeOffset.UtcNow.ToString("O"), + ["level"] = level, + ["event"] = eventName, + ["service"] = ObservabilityConfig.ServiceName, + ["environment"] = ObservabilityConfig.EnvironmentName, + ["version"] = ObservabilityConfig.Version, + ["gitSha"] = ObservabilityConfig.GitSha + }; + + foreach (var (key, value) in fields) + { + payload[key] = value; + } + + logger.LogLine(JsonSerializer.Serialize(payload, JsonOptions)); + } +} diff --git a/src/Lho.Lambda.Tests/ApiFunctionTests.cs b/src/Lho.Lambda.Tests/ApiFunctionTests.cs index b9d01d5..99416fa 100644 --- a/src/Lho.Lambda.Tests/ApiFunctionTests.cs +++ b/src/Lho.Lambda.Tests/ApiFunctionTests.cs @@ -1,4 +1,5 @@ using System.Net; +using System.Net.Sockets; using System.Text.Json; using Amazon.Lambda.APIGatewayEvents; using Lho.Lambda.Functions; @@ -52,7 +53,44 @@ public async Task VersionEndpointUsesDeploymentEnvironmentValues() Assert.Equal("abc123", body.GetProperty("gitSha").GetString()); } - private static APIGatewayHttpApiV2ProxyRequest CreateRequest(string path) + [Fact] + public async Task ApiHealthAndVersionRoutesPushInvocationMetrics() + { + var port = GetFreePort(); + using var listener = new HttpListener(); + listener.Prefixes.Add($"http://127.0.0.1:{port}/"); + listener.Start(); + + ConfigureMetrics(port); + try + { + var function = new ApiFunction(); + + var healthGetMetric = await InvokeAndReadMetric(listener, function, CreateRequest("/api/health")); + var healthHeadMetric = await InvokeAndReadMetric(listener, function, CreateRequest("/api/health", "HEAD")); + var versionMetric = await InvokeAndReadMetric(listener, function, CreateRequest("/api/version")); + + Assert.Contains("route=\"/api/health\"", healthGetMetric); + Assert.Contains("method=\"GET\"", healthGetMetric); + Assert.Contains("status=\"200\"", healthGetMetric); + Assert.Contains("route=\"/api/health\"", healthHeadMetric); + Assert.Contains("method=\"HEAD\"", healthHeadMetric); + Assert.Contains("status=\"200\"", healthHeadMetric); + Assert.Contains("route=\"/api/version\"", versionMetric); + Assert.Contains("method=\"GET\"", versionMetric); + Assert.Contains("status=\"200\"", versionMetric); + } + finally + { + Environment.SetEnvironmentVariable("METRICS_ENABLED", null); + Environment.SetEnvironmentVariable("PUSHGATEWAY_URL", null); + Environment.SetEnvironmentVariable("PUSHGATEWAY_AUTH_HEADER", null); + Environment.SetEnvironmentVariable("PROMETHEUS_JOB", null); + Environment.SetEnvironmentVariable("ENVIRONMENT", null); + } + } + + private static APIGatewayHttpApiV2ProxyRequest CreateRequest(string path, string method = "GET") { return new APIGatewayHttpApiV2ProxyRequest { @@ -62,7 +100,7 @@ private static APIGatewayHttpApiV2ProxyRequest CreateRequest(string path) { Http = new APIGatewayHttpApiV2ProxyRequest.HttpDescription { - Method = "GET", + Method = method, Path = path } } @@ -73,4 +111,40 @@ private static void SetEnvironment(string key, string value) { Environment.SetEnvironmentVariable(key, value); } + + private static void ConfigureMetrics(int port) + { + Environment.SetEnvironmentVariable("METRICS_ENABLED", "true"); + Environment.SetEnvironmentVariable("PUSHGATEWAY_URL", $"http://127.0.0.1:{port}"); + Environment.SetEnvironmentVariable("PUSHGATEWAY_AUTH_HEADER", null); + Environment.SetEnvironmentVariable("PROMETHEUS_JOB", "test-job"); + Environment.SetEnvironmentVariable("ENVIRONMENT", "test"); + } + + private static async Task InvokeAndReadMetric( + HttpListener listener, + ApiFunction function, + APIGatewayHttpApiV2ProxyRequest request) + { + var requestTask = listener.GetContextAsync(); + var responseTask = function.FunctionHandler(request, new TestLambdaContext()); + + var context = await requestTask.WaitAsync(TimeSpan.FromSeconds(2)); + using var reader = new StreamReader(context.Request.InputStream, context.Request.ContentEncoding); + var body = await reader.ReadToEndAsync(); + context.Response.StatusCode = (int)HttpStatusCode.Accepted; + context.Response.Close(); + + var response = await responseTask.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal((int)HttpStatusCode.OK, response.StatusCode); + + return body; + } + + private static int GetFreePort() + { + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + return ((IPEndPoint)listener.LocalEndpoint).Port; + } } diff --git a/src/Lho.Lambda.Tests/AuthorizerFunctionTests.cs b/src/Lho.Lambda.Tests/AuthorizerFunctionTests.cs index f90488a..708715e 100644 --- a/src/Lho.Lambda.Tests/AuthorizerFunctionTests.cs +++ b/src/Lho.Lambda.Tests/AuthorizerFunctionTests.cs @@ -56,6 +56,19 @@ public async Task DeniesMismatchedApiKey() Assert.False(response.IsAuthorized); } + [Fact] + public async Task DeniesMissingApiKeyConfiguration() + { + Environment.SetEnvironmentVariable("API_KEY", null); + var function = new AuthorizerFunction(); + + var response = await function.FunctionHandler( + CreateRequest([]), + new TestLambdaContext()); + + Assert.False(response.IsAuthorized); + } + private static AuthorizerRequest CreateRequest(Dictionary headers) { return new AuthorizerRequest diff --git a/src/Lho.Lambda.Tests/PrometheusMetricsTests.cs b/src/Lho.Lambda.Tests/PrometheusMetricsTests.cs new file mode 100644 index 0000000..fcb9a9c --- /dev/null +++ b/src/Lho.Lambda.Tests/PrometheusMetricsTests.cs @@ -0,0 +1,126 @@ +using System.Net; +using System.Net.Sockets; +using Lho.Lambda.Observability; +using Xunit; + +[assembly: CollectionBehavior(DisableTestParallelization = true)] + +namespace Lho.Lambda.Tests; + +public class PrometheusMetricsTests +{ + [Fact] + public async Task PushInvocationCountsEachProviderSeriesSeparately() + { + var port = GetFreePort(); + using var listener = new HttpListener(); + listener.Prefixes.Add($"http://127.0.0.1:{port}/"); + listener.Start(); + + ConfigureMetrics(port); + + var lastFmBody = await PushAndReadBody(listener, new InvocationMetric( + FunctionName: "provider-counter-test-function", + Operation: "now-playing", + Route: "/api/now-playing", + Method: "GET", + StatusCode: 200, + DurationMs: 12, + Outcome: "success", + Provider: "lastfm")); + + var spotifyBody = await PushAndReadBody(listener, new InvocationMetric( + FunctionName: "provider-counter-test-function", + Operation: "now-playing", + Route: "/api/now-playing", + Method: "GET", + StatusCode: 200, + DurationMs: 14, + Outcome: "success", + Provider: "spotify")); + + Assert.Equal("1", InvocationTotalValue(lastFmBody)); + Assert.Contains("provider=\"lastfm\"", InvocationTotalLine(lastFmBody)); + Assert.Equal("1", InvocationTotalValue(spotifyBody)); + Assert.Contains("provider=\"spotify\"", InvocationTotalLine(spotifyBody)); + } + + [Fact] + public async Task PushInvocationAddsConfiguredAuthorizationHeader() + { + var port = GetFreePort(); + using var listener = new HttpListener(); + listener.Prefixes.Add($"http://127.0.0.1:{port}/"); + listener.Start(); + + ConfigureMetrics(port); + Environment.SetEnvironmentVariable("PUSHGATEWAY_AUTH_HEADER", "Authorization=Basic dXNlcjpwYXNz"); + + var requestTask = listener.GetContextAsync(); + + var pushTask = PrometheusMetrics.PushInvocationAsync( + new InvocationMetric( + FunctionName: "test-function", + Operation: "test-operation", + Route: "/test", + Method: "GET", + StatusCode: 200, + DurationMs: 12, + Outcome: "success"), + new TestLambdaLogger()); + + var context = await requestTask.WaitAsync(TimeSpan.FromSeconds(2)); + var authorizationHeader = context.Request.Headers["Authorization"]; + context.Response.StatusCode = (int)HttpStatusCode.Accepted; + context.Response.Close(); + + await pushTask.WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.Equal("Basic dXNlcjpwYXNz", authorizationHeader); + } + + private static void ConfigureMetrics(int port) + { + Environment.SetEnvironmentVariable("METRICS_ENABLED", "true"); + Environment.SetEnvironmentVariable("PUSHGATEWAY_URL", $"http://127.0.0.1:{port}"); + Environment.SetEnvironmentVariable("PUSHGATEWAY_AUTH_HEADER", null); + Environment.SetEnvironmentVariable("PROMETHEUS_JOB", "test-job"); + Environment.SetEnvironmentVariable("ENVIRONMENT", "test"); + } + + private static async Task PushAndReadBody(HttpListener listener, InvocationMetric metric) + { + var requestTask = listener.GetContextAsync(); + var pushTask = PrometheusMetrics.PushInvocationAsync(metric, new TestLambdaLogger()); + + var context = await requestTask.WaitAsync(TimeSpan.FromSeconds(2)); + using var reader = new StreamReader(context.Request.InputStream, context.Request.ContentEncoding); + var body = await reader.ReadToEndAsync(); + context.Response.StatusCode = (int)HttpStatusCode.Accepted; + context.Response.Close(); + + await pushTask.WaitAsync(TimeSpan.FromSeconds(2)); + + return body; + } + + private static string InvocationTotalValue(string body) + { + var line = InvocationTotalLine(body); + return line[(line.LastIndexOf(' ') + 1)..]; + } + + private static string InvocationTotalLine(string body) + { + return body + .Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Single(line => line.StartsWith("lho_lambda_invocations_total{", StringComparison.Ordinal)); + } + + private static int GetFreePort() + { + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + return ((IPEndPoint)listener.LocalEndpoint).Port; + } +} diff --git a/src/Lho.Lambda/Functions/ApiFunction.cs b/src/Lho.Lambda/Functions/ApiFunction.cs index c8986e0..807eb45 100644 --- a/src/Lho.Lambda/Functions/ApiFunction.cs +++ b/src/Lho.Lambda/Functions/ApiFunction.cs @@ -1,7 +1,9 @@ +using System.Diagnostics; using Amazon.Lambda.APIGatewayEvents; using Amazon.Lambda.Core; using Lho.Lambda.Clients.LastFm; using Lho.Lambda.Clients.Spotify; +using Lho.Lambda.Observability; using Lho.Lambda.Services; using Lho.Lambda.Utils; @@ -36,35 +38,79 @@ public async Task FunctionHandler( ILambdaContext ctx ) { + var stopwatch = Stopwatch.StartNew(); var method = request.RequestContext?.Http?.Method ?? "GET"; var path = NormalisePath(request.RawPath); - ctx.Logger.LogLine($"Request {method} {path}"); + var consumer = NormaliseConsumer(GetHeaderValue(request.Headers, "x-consumer")); + var provider = path == "/api/now-playing" + ? QueryStringParser.Get(request.RawQueryString, "provider", "lastfm", ["lastfm", "spotify"]) + : null; + APIGatewayHttpApiV2ProxyResponse response; + Exception? capturedException = null; + SentryTelemetry.Initialise(ctx.Logger); + using var transaction = SentryTelemetry.StartTransaction(ctx.Logger, $"{method} {path}", "http.server", new Dictionary + { + ["function"] = ctx.FunctionName, + ["request_id"] = ctx.AwsRequestId, + ["route"] = path, + ["method"] = method, + ["consumer"] = consumer, + ["provider"] = provider + }); - if (!string.Equals(method, "GET", StringComparison.OrdinalIgnoreCase)) + try { - return ResponseBuilder.ErrorResponse(404, "Not Found"); + if (!IsSupportedMethod(method, path)) + { + response = ResponseBuilder.ErrorResponse(404, "Not Found"); + } + else + { + response = path switch + { + "/api/health" => ResponseBuilder.CreateResponse(new { status = "OK" }, includeCacheControl: false), + "/api/version" => ResponseBuilder.CreateResponse(VersionService.GetVersion(), includeCacheControl: false), + "/api/now-playing" => await HandleNowPlaying(request, ctx, provider!), + "/api/top-tracks" => await HandleTopTracks(request, ctx), + _ => ResponseBuilder.ErrorResponse(404, "Not Found") + }; + } } - - return path switch + catch (Exception exception) { - "/api/health" => ResponseBuilder.CreateResponse(new { status = "OK" }, includeCacheControl: false), - "/api/version" => ResponseBuilder.CreateResponse(VersionService.GetVersion(), includeCacheControl: false), - "/api/now-playing" => await HandleNowPlaying(request, ctx), - "/api/top-tracks" => await HandleTopTracks(request, ctx), - _ => ResponseBuilder.ErrorResponse(404, "Not Found") - }; + capturedException = exception; + response = ResponseBuilder.ErrorResponse(500, "Internal Server Error"); + StructuredLog.Error(ctx.Logger, "api.request.error", exception, new Dictionary + { + ["requestId"] = ctx.AwsRequestId, + ["method"] = method, + ["route"] = path, + ["consumer"] = consumer, + ["provider"] = provider + }); + await SentryTelemetry.CaptureExceptionAsync(exception, ctx.Logger, new Dictionary + { + ["function"] = ctx.FunctionName, + ["request_id"] = ctx.AwsRequestId, + ["route"] = path, + ["method"] = method, + ["consumer"] = consumer, + ["provider"] = provider + }); + } + + stopwatch.Stop(); + transaction?.Finish(response.StatusCode, capturedException); + await RecordInvocation(ctx, method, path, response.StatusCode, stopwatch.Elapsed.TotalMilliseconds, consumer, provider); + return response; } private async Task HandleNowPlaying( APIGatewayHttpApiV2ProxyRequest request, - ILambdaContext context) + ILambdaContext context, + string provider) { - var provider = QueryStringParser.Get( - request.RawQueryString, - "provider", - "lastfm", - ["lastfm", "spotify"]); var response = await new NowPlayingService(_cache, _spotifyApi, _lastFmApi, context.Logger).HandleNowPlaying(provider); return ResponseBuilder.CreateResponse(response, revalidateSeconds: 3); @@ -82,16 +128,46 @@ private async Task HandleTopTracks( var rawLimit = QueryStringParser.Get(request.RawQueryString, "limit", "20"); var limit = int.TryParse(rawLimit, out var parsedLimit) ? Math.Clamp(parsedLimit, 1, 50) : 20; - try - { - var response = await new TopTracksService(_spotifyApi, context.Logger).HandleTopTracks(timeRange, limit); - return ResponseBuilder.CreateResponse(response, revalidateSeconds: 300); - } - catch (Exception exception) + var response = await new TopTracksService(_spotifyApi, context.Logger).HandleTopTracks(timeRange, limit); + return ResponseBuilder.CreateResponse(response, revalidateSeconds: 300); + } + + private static async Task RecordInvocation( + ILambdaContext context, + string method, + string path, + int statusCode, + double durationMs, + string? consumer, + string? provider) + { + var outcome = statusCode >= 500 ? "error" : statusCode >= 400 ? "client_error" : "success"; + StructuredLog.Info(context.Logger, "api.request", new Dictionary { - context.Logger.LogLine($"Top tracks failed: {exception}"); - return ResponseBuilder.ErrorResponse(500, "Unable to fetch top tracks"); - } + ["requestId"] = context.AwsRequestId, + ["function"] = context.FunctionName, + ["method"] = method, + ["route"] = path, + ["statusCode"] = statusCode, + ["durationMs"] = Math.Round(durationMs, 2), + ["outcome"] = outcome, + ["consumer"] = consumer, + ["provider"] = provider + }); + + var metric = new InvocationMetric( + FunctionName: context.FunctionName, + Operation: "api", + Route: path, + Method: method, + StatusCode: statusCode, + DurationMs: durationMs, + Outcome: outcome, + Consumer: consumer, + Provider: provider); + + await PrometheusMetrics.PushInvocationAsync(metric, context.Logger); + await SentryTelemetry.RecordInvocationAsync(metric, context.Logger); } private static string NormalisePath(string? rawPath) @@ -127,4 +203,42 @@ var value when value.StartsWith("/", StringComparison.Ordinal) => value, _ => "/" + path }; } + + private static string? GetHeaderValue(IDictionary? headers, string key) + { + if (headers is null) + { + return null; + } + + foreach (var (headerKey, value) in headers) + { + if (string.Equals(headerKey, key, StringComparison.OrdinalIgnoreCase)) + { + return value; + } + } + + return null; + } + + private static string? NormaliseConsumer(string? consumer) + { + return consumer switch + { + "lhowsam-prod" or "lhowsam-dev" or "lhowsam-local" => consumer, + null or "" => null, + _ => "unknown" + }; + } + + private static bool IsSupportedMethod(string method, string path) + { + if (string.Equals(method, "GET", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + return string.Equals(method, "HEAD", StringComparison.OrdinalIgnoreCase) && path == "/api/health"; + } } diff --git a/src/Lho.Lambda/Lho.Lambda.csproj b/src/Lho.Lambda/Lho.Lambda.csproj index a11fcc5..a73feb8 100644 --- a/src/Lho.Lambda/Lho.Lambda.csproj +++ b/src/Lho.Lambda/Lho.Lambda.csproj @@ -9,4 +9,8 @@ + + + + diff --git a/src/Lho.Lambda/Services/NowPlayingService.cs b/src/Lho.Lambda/Services/NowPlayingService.cs index ef1997c..eded961 100644 --- a/src/Lho.Lambda/Services/NowPlayingService.cs +++ b/src/Lho.Lambda/Services/NowPlayingService.cs @@ -2,6 +2,7 @@ using Lho.Lambda.Clients.LastFm; using Lho.Lambda.Clients.Spotify; using Lho.Lambda.Models; +using Lho.Lambda.Observability; using Lho.Lambda.Utils; namespace Lho.Lambda.Services; @@ -39,6 +40,11 @@ public async Task HandleNowPlaying(string provider = LastFmP catch (Exception exception) { logger.LogLine($"Error fetching now playing data from {provider}: {exception}"); + await SentryTelemetry.CaptureExceptionAsync(exception, logger, new Dictionary + { + ["operation"] = "now-playing", + ["provider"] = provider + }); return EmptyResponse(maintenance: null, status: 500); } } diff --git a/src/Lho.Lambda/Services/TopTracksService.cs b/src/Lho.Lambda/Services/TopTracksService.cs index 7b6a3dd..4268d1c 100644 --- a/src/Lho.Lambda/Services/TopTracksService.cs +++ b/src/Lho.Lambda/Services/TopTracksService.cs @@ -1,6 +1,7 @@ using Amazon.Lambda.Core; using Lho.Lambda.Clients.Spotify; using Lho.Lambda.Models; +using Lho.Lambda.Observability; namespace Lho.Lambda.Services; @@ -25,6 +26,11 @@ public async Task HandleTopTracks(string timeRange, int li catch (Exception exception) { logger.LogLine($"Top tracks fetch failed: {exception}"); + await SentryTelemetry.CaptureExceptionAsync(exception, logger, new Dictionary + { + ["operation"] = "top-tracks", + ["time_range"] = timeRange + }); throw; } } diff --git a/src/Lho.Lambda/Utils/ResponseBuilder.cs b/src/Lho.Lambda/Utils/ResponseBuilder.cs index e410753..37cb2bd 100644 --- a/src/Lho.Lambda/Utils/ResponseBuilder.cs +++ b/src/Lho.Lambda/Utils/ResponseBuilder.cs @@ -16,7 +16,7 @@ public static class ResponseBuilder { ["content-type"] = "application/json", ["Access-Control-Allow-Origin"] = "*", - ["Access-Control-Allow-Methods"] = "GET,OPTIONS,POST,PUT,DELETE" + ["Access-Control-Allow-Methods"] = "GET,HEAD,OPTIONS,POST,PUT,DELETE" }; public static APIGatewayHttpApiV2ProxyResponse CreateResponse( diff --git a/terraform/authorizer.tf b/terraform/authorizer.tf index 12937f1..31499b5 100644 --- a/terraform/authorizer.tf +++ b/terraform/authorizer.tf @@ -4,9 +4,13 @@ data "archive_file" "auth_archive" { output_path = "${path.module}/../authorizer.zip" } +locals { + authorizer_function_name = "${var.project_name}-api-authorizer-${var.env}" +} + resource "aws_lambda_function" "api_authorizer" { filename = "${path.module}/../authorizer.zip" - function_name = "${var.project_name}-api-authorizer-${var.env}" + function_name = local.authorizer_function_name role = aws_iam_role.lambda_exec.arn handler = "Lho.Lambda.Authorizer::Lho.Lambda.Authorizer.Functions.AuthorizerFunction::FunctionHandler" source_code_hash = data.archive_file.auth_archive.output_base64sha256 @@ -15,17 +19,44 @@ resource "aws_lambda_function" "api_authorizer" { architectures = ["x86_64"] timeout = 10 + tracing_config { + mode = "Active" + } environment { variables = { - API_KEY = var.api_key - ENVIRONMENT = var.env + API_KEY = var.api_key + SERVICE_NAME = "now-playing" + ENVIRONMENT = var.env + VERSION = var.app_version + GIT_SHA = var.git_sha + SENTRY_DSN = var.sentry_dsn + SENTRY_ENVIRONMENT = var.env + SENTRY_RELEASE = var.app_version + PUSHGATEWAY_URL = var.pushgateway_url + PUSHGATEWAY_AUTH_HEADER = var.pushgateway_auth_header + PROMETHEUS_JOB = "now-playing-authorizer" + METRICS_ENABLED = tostring(var.monitoring_enabled) } } tags = merge(var.tags, { ENVIRONMENT = var.env }) + + depends_on = [aws_cloudwatch_log_group.auth_logs] +} + +resource "aws_cloudwatch_log_group" "auth_logs" { + name = "/aws/lambda/${local.authorizer_function_name}" + retention_in_days = 1 + log_group_class = "STANDARD" + + tags = { + Environment = var.env + Service = "nowplaying" + s3export = "true" + } } resource "aws_apigatewayv2_authorizer" "api_key" { diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 74b052a..e756483 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -26,13 +26,10 @@ resource "aws_iam_role_policy_attachment" "lambda_policy" { policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" } -# data "aws_iam_policy" "aws_xray_write_only_access" { -# arn = "arn:aws:iam::aws:policy/AWSXrayWriteOnlyAccess" -# } -# resource "aws_iam_role_policy_attachment" "aws_xray_write_only_access" { -# role = aws_iam_role.lambda_exec.name -# policy_arn = data.aws_iam_policy.aws_xray_write_only_access.arn -# } +resource "aws_iam_role_policy_attachment" "aws_xray_write_only_access" { + role = aws_iam_role.lambda_exec.name + policy_arn = "arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess" +} resource "aws_lambda_function" "lambda" { function_name = "${var.project_name}-lambda-${var.env}" @@ -42,23 +39,33 @@ resource "aws_lambda_function" "lambda" { filename = "${path.module}/../lambda.zip" source_code_hash = data.archive_file.lambda_archive.output_base64sha256 timeout = 30 - # tracing_config { - # mode = "Active" - # } + tracing_config { + mode = "Active" + } + description = "Now playing Lambda ${var.env}" memory_size = 256 architectures = ["x86_64"] environment { variables = { - SPOTIFY_CLIENT_ID = var.spotify_client_id - SPOTIFY_CLIENT_SECRET = var.spotify_client_secret - SPOTIFY_REFRESH_TOKEN = var.spotify_refresh_token - LASTFM_API_KEY = var.lastfm_api_key - LASTFM_USERNAME = var.lastfm_username - VERSION = var.app_version - DEPLOYED_AT = timestamp() - DEPLOYED_BY = var.deployed_by - GIT_SHA = var.git_sha + SPOTIFY_CLIENT_ID = var.spotify_client_id + SPOTIFY_CLIENT_SECRET = var.spotify_client_secret + SPOTIFY_REFRESH_TOKEN = var.spotify_refresh_token + LASTFM_API_KEY = var.lastfm_api_key + LASTFM_USERNAME = var.lastfm_username + SERVICE_NAME = "now-playing" + ENVIRONMENT = var.env + VERSION = var.app_version + DEPLOYED_AT = timestamp() + DEPLOYED_BY = var.deployed_by + GIT_SHA = var.git_sha + SENTRY_DSN = var.sentry_dsn + SENTRY_ENVIRONMENT = var.env + SENTRY_RELEASE = var.app_version + PUSHGATEWAY_URL = var.pushgateway_url + PUSHGATEWAY_AUTH_HEADER = var.pushgateway_auth_header + PROMETHEUS_JOB = "now-playing" + METRICS_ENABLED = tostring(var.monitoring_enabled) } } tags = merge(var.tags, { diff --git a/terraform/variables.tf b/terraform/variables.tf index fa61135..bb0ff71 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -96,6 +96,32 @@ variable "git_sha" { default = "unknown" } +variable "sentry_dsn" { + type = string + description = "Sentry DSN for Lambda error monitoring" + sensitive = true + default = "" +} + +variable "pushgateway_url" { + type = string + description = "Prometheus Pushgateway endpoint for Lambda invocation metrics" + default = "https://pushgateway.lhowsam.com" +} + +variable "pushgateway_auth_header" { + type = string + description = "Pushgateway auth header in Header=Value form, for example Authorization=Basic " + sensitive = true + default = "" +} + +variable "monitoring_enabled" { + type = bool + description = "Whether Lambda functions should push invocation metrics to Pushgateway" + default = true +} + variable "api_key" { description = "API key for securing the API Gateway endpoints" type = string