Bau/handle error

.github/workflows/deploy.yml

@@ -98,14 +98,24 @@ jobs:
 
       - name: Deploy aws Lambda functions
         run: |
+          deploy_if_exists() {
+            local function_name="$1"
+            local zip_path="$2"
+            if aws lambda get-function --function-name "$function_name" >/dev/null 2>&1; then
+              aws lambda update-function-code --function-name "$function_name" --zip-file "fileb://$zip_path"
+            else
+              echo "Skipping missing Lambda: $function_name"
+            fi
+          }
+
           aws lambda update-function-code --function-name ${{ matrix.environment }}-main-slack-notifier --zip-file fileb://./notification-code.zip
           aws lambda update-function-code --function-name ${{ matrix.environment }}-status-slack-notifier --zip-file fileb://./notification-code.zip
           aws lambda update-function-code --function-name ${{ matrix.environment }}-datasette-slack-notifier --zip-file fileb://./notification-code.zip
           aws lambda update-function-code --function-name ${{ matrix.environment }}-data-val-fe-slack-notifier --zip-file fileb://./notification-code.zip
           aws lambda update-function-code --function-name ${{ matrix.environment }}-pipeline-int-api-slack-notifier --zip-file fileb://./notification-code.zip
           aws lambda update-function-code --function-name ${{ matrix.environment }}-data-design-slack-notifier --zip-file fileb://./notification-code.zip
           aws lambda update-function-code --function-name ${{ matrix.environment }}-dataset-editor-slack-notifier --zip-file fileb://./notification-code.zip
-          aws lambda update-function-code --function-name ${{ matrix.environment }}-config-manager-slack-notifier --zip-file fileb://./notification-code.zip
+          deploy_if_exists "${{ matrix.environment }}-config-manager-slack-notifier" "./notification-code.zip"
           aws lambda update-function-code --function-name ${{ matrix.environment }}-send-alarms-to-slack --zip-file fileb://./cloudwatch-alarms-to-slack.zip
 
       - name: Deploy aws Process log Lambda functions

notification-code/main.js

@@ -58,62 +58,98 @@ const handleCodeDeployLifeCycleEvent = async (event) => {
     const slackBotToken = await getSlackBotToken();
     const DeploymentId = event.DeploymentId;
     const lifecycleEventHookExecutionId = event.LifecycleEventHookExecutionId;
+    let hookStatus = 'Succeeded';
+    let failureError = null;
+
+    try {
+        const deploymentTargets = await codedeploy.listDeploymentTargets({
+            deploymentId: DeploymentId,
+        }).promise();
+
+        console.log(JSON.stringify(deploymentTargets));
+
+        if (!deploymentTargets.targetIds?.length) {
+            console.warn(`No deployment targets found for deployment ${DeploymentId}. Skipping notification update.`);
+        } else {
+            const deploymentTarget = await codedeploy.getDeploymentTarget({
+                deploymentId: DeploymentId,
+                targetId: deploymentTargets.targetIds[0],
+            }).promise();
+
+            console.log(JSON.stringify(deploymentTarget));
+
+            const lifecycleEvents = deploymentTarget.deploymentTarget?.ecsTarget?.lifecycleEvents || [];
+            const latestLifecycleEvent = lifecycleEvents[lifecycleEvents.length - 1];
+
+            // Only fail the deployment when CodeDeploy reports an actual failed lifecycle event.
+            if (latestLifecycleEvent?.status === 'Failed') {
+                const lifecycleFailure = new Error(
+                    `Lifecycle event ${latestLifecycleEvent.lifecycleEventName} failed for deployment ${DeploymentId}`
+                );
+                lifecycleFailure.isLifecycleValidationFailure = true;
+                throw lifecycleFailure;
+            }
+
+            let Type = 'deployment';
+            let Application = process.env.APPLICATION_NAME;
+            const DeploymentDetails = await getDeploymentDetails(DeploymentId);
+            const Events = DeploymentDetails ? DeploymentDetails.Events : [];
+            const lastSuccessEvent = [...lifecycleEvents].reverse().find(e => e.status === 'Succeeded');
+            let State = lastSuccessEvent?.lifecycleEventName || latestLifecycleEvent?.lifecycleEventName || 'in progress';
+
+            switch (State) {
+                case 'Install':
+                    State = 'replacement service running';
+                    break;
+                case 'AfterInstall':
+                    State = 'replacement service accepting 20% traffic';
+                    break;
+                case 'AllowTraffic':
+                    State = 'replacement service accepting 100% traffic';
+                    break;
+                default:
+                    break;
+            }
+
+            Events.push({
+                Region: Events[0]?.Region || process.env.AWS_REGION,
+                State,
+                Type,
+                Timestamp: Date.now(),
+            });
+
+            try {
+                await sendMessage(slackBotToken, Application, DeploymentId, Events, Type);
+                await setDeploymentDetails({DeploymentId, Events});
+            } catch (notificationError) {
+                console.error('Non-blocking lifecycle notification error', notificationError);
+            }
+        }
 
-    const deploymentTargets = await codedeploy.listDeploymentTargets({
-        deploymentId: DeploymentId,
-    }).promise();
-
-    console.log(JSON.stringify(deploymentTargets));
-
-    const deploymentTarget = await codedeploy.getDeploymentTarget({
-        deploymentId: DeploymentId,
-        targetId: deploymentTargets.targetIds[0],
-    }).promise();
-
-    console.log(JSON.stringify(deploymentTarget));
-
-    let Type = 'deployment';
-    let Application = process.env.APPLICATION_NAME;
-
-    const DeploymentDetails = await getDeploymentDetails(DeploymentId);
-    const Events = DeploymentDetails ? DeploymentDetails.Events : [];
-
-    const lastSuccessEvent = deploymentTarget.deploymentTarget.ecsTarget.lifecycleEvents
-        .reverse()
-        .find(e => e.status === 'Succeeded');
-
-    let State = lastSuccessEvent.lifecycleEventName;
-
-    switch (State) {
-        case 'Install':
-            State = 'replacement service running';
-            break;
-        case 'AfterInstall':
-            State = 'replacement service accepting 20% traffic';
-            break;
-        case 'AllowTraffic':
-            State = 'replacement service accepting 100% traffic';
-            break;
-        default:
-            break;
+    } catch (err) {
+        if (err.isLifecycleValidationFailure) {
+            console.error(err);
+            hookStatus = 'Failed';
+            failureError = err;
+        } else {
+            console.error('Non-blocking lifecycle hook error', err);
+        }
+    } finally {
+        try {
+            await codedeploy.putLifecycleEventHookExecutionStatus({
+                deploymentId: DeploymentId,
+                lifecycleEventHookExecutionId,
+                status: hookStatus,
+            }).promise();
+        } catch (statusError) {
+            console.error('Failed to report lifecycle hook execution status', statusError);
+            throw statusError;
+        }
     }
 
-    Events.push({
-        Region: Events[0].Region,
-        State,
-        Type,
-        Timestamp: Date.now(),
-    });
-
-    await sendMessage(slackBotToken, Application, DeploymentId, Events, Type);
-
-    await setDeploymentDetails({DeploymentId, Events});
-
-    await codedeploy.putLifecycleEventHookExecutionStatus({
-        deploymentId: DeploymentId,
-        lifecycleEventHookExecutionId,
-        status: 'Succeeded',
-    }).promise();
+    if (failureError) {
+        throw failureError;
+    }
 };
 
 module.exports = {

process-log/index.js

@@ -4,28 +4,80 @@ const util = require('util');
 
 const gunzip = util.promisify(zlib.gunzip);
 
+const toLogEvent = (line) => {
+    let timestamp = null;
+
+    if (line.timestamp !== undefined && line.timestamp !== null) {
+        const numericTimestamp = Number(line.timestamp);
+        if (Number.isFinite(numericTimestamp)) {
+            timestamp = numericTimestamp;
+        } else {
+            const parsedTimestamp = Date.parse(line.timestamp);
+            if (!Number.isNaN(parsedTimestamp)) {
+                timestamp = parsedTimestamp;
+            }
+        }
+    }
+
+    if ((timestamp === null || !Number.isFinite(timestamp)) && line.date && line.time) {
+        const parsedTimestamp = Date.parse(`${line.date}T${line.time}Z`);
+        if (!Number.isNaN(parsedTimestamp)) {
+            timestamp = parsedTimestamp;
+        }
+    }
+
+    if (timestamp === null || !Number.isFinite(timestamp)) {
+        timestamp = Date.now();
+    }
+
+    return {timestamp, message: JSON.stringify(line)};
+};
+
+const parseLegacyCloudFrontLogs = (content, fieldsLine) => {
+    const fields = fieldsLine
+        .replace('#Fields: ', '')
+        .split(' ');
+
+    return content
+        .split('\n')
+        .filter(line => !line.startsWith('#'))
+        .filter(line => !!line)
+        .map((line) => Object.fromEntries(line.split('\t').map((column, index) => ([fields[index], column]))))
+        .map(toLogEvent);
+};
+
+const parseJsonCloudFrontLogs = (content) => {
+    return content
+        .split('\n')
+        .map(line => line.trim())
+        .filter(line => !!line)
+        .filter(line => !line.startsWith('#'))
+        .map((line) => {
+            try {
+                return JSON.parse(line);
+            } catch (err) {
+                console.error('Failed to parse JSON CloudFront log line', err);
+                return null;
+            }
+        })
+        .filter(line => line !== null)
+        .map(toLogEvent);
+};
+
 const getLogEvents = async (s3, Bucket, Key) => {
     try {
         const s3Object = await s3.getObject({Bucket, Key}).promise();
         const log = await gunzip(s3Object.Body);
-
-        const fields = log.toString()
+        const content = log.toString();
+        const fieldsLine = content
             .split('\n')
-            .find(line => line.startsWith('#Fields:'))
-            .replace('#Fields: ', '')
-            .split(' ');
+            .find(line => line.startsWith('#Fields:'));
 
-        return log.toString()
-            .split('\n')
-            .filter(line => !line.startsWith('#'))
-            .filter(line => !!line)
-            .map((line) => Object.fromEntries(line.split('\t').map((column, index) => ([fields[index], column]))))
-            .map(line => {
-                const date = line.date.split('-').map(d => parseInt(d));
-                const time = line.time.split(':').map(d => parseInt(d));
-                const timestamp = new Date(date[0], date[1] - 1, date[2], time[0], time[1], time[2]).getTime();
-                return {timestamp, message: JSON.stringify(line)};
-            });
+        if (fieldsLine) {
+            return parseLegacyCloudFrontLogs(content, fieldsLine);
+        }
+
+        return parseJsonCloudFrontLogs(content);
     } catch (err) {
         console.error(err);
         return [];