From 5b2823330bb940784cf4c458abb33cb923a6606c Mon Sep 17 00:00:00 2001 From: Craig Thacker Date: Sat, 8 Aug 2026 16:46:54 +0100 Subject: [PATCH 1/5] feat: add terraform for the Message Center workload identity Message Center is the one source with no feed: it is Graph only, at /admin/serviceAnnouncement/messages, and needs an authenticated tenant-scoped caller holding ServiceMessage.Read.All. This adds the Entra application that provides it, composing libre-devops/service-principal/azuread. Federated to GitHub Actions by OIDC with no client secret and no certificate, so nothing secret lands in the repository, in Actions, or in state. Four subjects are trusted: the environment form (what the feed job presents, since it declares environment: github-pages for Pages) and the ref form, each in both the plain and the immutable org@id format GitHub forces on repositories created or renamed after 2026-07-15. Verified with fmt, validate and a read-only plan against the tenant: 9 to add, subjects and the Graph role resolve as intended. --- .gitignore | 12 +++++ terraform/.terraform.lock.hcl | 49 +++++++++++++++++++ terraform/README.md | 88 +++++++++++++++++++++++++++++++++++ terraform/main.tf | 81 ++++++++++++++++++++++++++++++++ terraform/outputs.tf | 48 +++++++++++++++++++ terraform/providers.tf | 10 ++++ terraform/terraform.tf | 10 ++++ terraform/variables.tf | 70 ++++++++++++++++++++++++++++ 8 files changed, 368 insertions(+) create mode 100644 terraform/.terraform.lock.hcl create mode 100644 terraform/README.md create mode 100644 terraform/main.tf create mode 100644 terraform/outputs.tf create mode 100644 terraform/providers.tf create mode 100644 terraform/terraform.tf create mode 100644 terraform/variables.tf diff --git a/.gitignore b/.gitignore index 0965674..2c7fce1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,15 @@ __pycache__/ .DS_Store Thumbs.db .idea + +# Terraform. The lock file is deliberately NOT ignored: it pins provider checksums, so it is +# committed per the Libre DevOps Terraform standard. +terraform/.terraform/ +terraform/*.tfstate +terraform/*.tfstate.* +terraform/*.tfvars +terraform/*.tfvars.json +terraform/*_override.tf +terraform/override.tf +terraform/crash.log +terraform/tfplan* diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..e769f04 --- /dev/null +++ b/terraform/.terraform.lock.hcl @@ -0,0 +1,49 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azuread" { + version = "3.9.0" + constraints = ">= 3.0.0, < 4.0.0" + hashes = [ + "h1:+ZknnMPMLJ1dIVqxto9ZWoakX4ljsek5cmajhUfEwN4=", + "h1:BUs7c1/enLL68Qh0qcb95elIFPQzRdPnQTcwl08my7U=", + "h1:c7GIN2qq1Un6Vu4AbvyrS9+PPid7lO8TbpRkAgXm800=", + "h1:caKVAk5GOECNATz8XPruo39n2y6OcntxPblPgl+6QaY=", + "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", + "zh:39b11a075e4baa4f6ed5c72a8427013d50f43eecc1a7603b73bccf80f952f758", + "zh:41484c196c943b39411f561e70a308bd2a71da18155bfec7381ba0bd61361d34", + "zh:42068e5da223494beea5f7fcb9057c308cbfa92f96e53c50083e2639216479d8", + "zh:464d7da44682443a4b64bfdaf3d0eb53011c6e1471f244f6354c4d5bca18edce", + "zh:49f597ea3fac39931ff91e55afd5b5cc91e449920a03716f82509d588aaab708", + "zh:6092c376accfc50b555b7a0cd56b76c09abc3d65ac9dd5069063d6f9f1e76d3b", + "zh:65326a9f3ac0783c16e05c16422d191f0a926b8d021fd5303c1fdf8dc42f16e9", + "zh:784214ed809347d74562bb38194c0cef57831eaa621ba3b7cdd3fe7a7a76d844", + "zh:b4233f9bc791adc7d6643507fa5b47360a21125763a072d953586151cacb65f9", + "zh:c4ecdd995ff99b7e362e087c45f080816bcf097da5be257c87b912210e45dd3e", + "zh:f0122771f71cb98248e70cdd6c2ccd3bffb34e79d19897fa28b785c86b2312ed", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.9.0" + constraints = ">= 3.5.0, < 4.0.0" + hashes = [ + "h1:OO+IuvQJSPmWdN8AyyIEvPJbLvDQpgX/zbktoa9KsJE=", + "h1:UlBuNVuCGJ39tTv2c5gz2NRZnQbXfbIWbTzWcth5o74=", + "h1:o0s5Mk9NXMP60nlheO1r0LsDGGratFb3oL0t7bD2QnM=", + "h1:q/uaUTBdKgAmZESrwsoeDQff9uUA/cI/N5ZKNgVwa9c=", + "zh:161ad0bd9a75768c82f53fb6e7172a9d8be2d4889b012645a34795031aaf1bf1", + "zh:19dc9a5b17729725ccfc4f45b0500af0ee5bc6b6b160c7adb8f2bf617d2c80ea", + "zh:269eda8fe42daa7974d5a34d166c3ba9defe80cde86c01e4dadcfdf2e1f05e5f", + "zh:373f7c65566f8f2cc7f45d698654feb9d988996957e1266a69ca00c52d6d16d0", + "zh:5599d16804c41c83009ec621b6d6b6f74e102f5827678a4750f8809055546b61", + "zh:583be0440469a22bff70dcfa56593b01566860b29607437264adb51060cf46fc", + "zh:5f211d8ec3f2e1f414870d9584bfe26e6995560ef81c748f8447a48164767398", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7b547fd16216761ef86efc3ed516ac5ac0c5c42b7c7eb24a08cef2d93f69ed5e", + "zh:7e7c0679daf2a382151d05068c8c3f0dae6b7b7dccf818827b73dd08638df2ef", + "zh:8089dec888a8038b9b4fb23b3df7e1057293dbc5b60b42cc47ff690d69d4b61b", + "zh:c51f15a031edfd6f23ce8ced3446ca7f8d8d647e2499890d7d5d10d5016d7257", + "zh:c94784f005708890dc6895afd53636ec00ec1e430b15d41e5aebfb1d4b39bd04", + ] +} diff --git a/terraform/README.md b/terraform/README.md new file mode 100644 index 0000000..4694eed --- /dev/null +++ b/terraform/README.md @@ -0,0 +1,88 @@ +# Message Center workload identity + +Terraform for the one Entra ID application that lets the feed job read the Microsoft 365 Message +Center. + +Every other source in `scripts/fetch_feeds.py` is a public RSS or Atom feed and needs no +credentials. Message Center is the exception: it has no feed at all, only Microsoft Graph at +`/admin/serviceAnnouncement/messages`, and that endpoint is tenant scoped and requires an +authenticated caller holding `ServiceMessage.Read.All`. + +This stack creates that caller. It creates nothing else: the site is static and served by GitHub +Pages, so there is no Azure infrastructure behind it. + +## What it creates + +One application registration and its service principal, named to the Libre DevOps convention +(`svp-${short}-${loc}-${env}-mc-001`, so `svp-ldo-uks-prd-mc-001` by default), carrying: + +- The `ServiceMessage.Read.All` Microsoft Graph **application** role, with tenant-wide admin consent + granted by default. Application rather than delegated because the feed job runs unattended on a + schedule, with no signed-in user. +- Federated identity credentials trusting GitHub Actions OIDC from `libre-devops/security-news`. + +There is **no client secret and no certificate**, by design. The federated credentials are the only +way to authenticate as this application, so nothing secret exists to commit, store in Actions, or +rotate. That also means this stack's state file holds no credentials. + +## The subject claim gotcha + +A GitHub Actions job that declares an `environment:` presents a different OIDC subject from one that +does not: + +| Job | `sub` claim | +| --- | --- | +| With `environment: github-pages` | `repo:libre-devops/security-news:environment:github-pages` | +| Without an environment | `repo:libre-devops/security-news:ref:refs/heads/master` | + +`fetch-feeds.yml` declares `environment: github-pages` for the Pages deployment, so the environment +form is the one that matters today. Both are registered anyway, so splitting the Graph fetch into +its own job later needs no Entra change. + +Both are also registered in GitHub's **immutable** subject format (`repo:libre-devops@101948202/...`), +which GitHub forces on repositories created or renamed after 2026-07-15. This repository predates +that, so it presents the plain form, but a rename would flip the format and fail the run with +`AADSTS7002131 No matching federated identity record`. Registering both up front makes that a +non-event. Set `register_immutable_subjects = false` to skip them. + +If a run ever does fail with `AADSTS7002131`, compare the subject in the run log against the +`federated_credential_subjects` output. + +## Applying + +State is local and auth is Azure CLI user auth, matching the other single-tenant helper stacks in +this estate. This is applied by hand, occasionally, and is not wired into CI. + +```bash +az login +terraform -chdir=terraform init +terraform -chdir=terraform plan +terraform -chdir=terraform apply +``` + +The applier needs: + +- `Application.ReadWrite.All`, or the Application Administrator role, to create the registration. +- `AppRoleAssignment.ReadWrite.All`, or Privileged Role Administrator, **when + `grant_admin_consent` is true** (the default). Granting a Graph application role is tenant-wide + admin consent, which is a privileged action. + +If you cannot consent yourself, apply with `-var 'grant_admin_consent=false'`. The roles are still +requested on the application, and the `grant_admin_consent_commands` output prints ready-to-run +`az rest` commands for whoever can grant them. + +## After applying + +Publish the two identifiers the workflow needs. Neither is a secret, so they are Actions **variables** +rather than secrets, and the `terraform output github_variable_commands` output prints both commands: + +```bash +gh variable set AZURE_CLIENT_ID --repo libre-devops/security-news --body +gh variable set AZURE_TENANT_ID --repo libre-devops/security-news --body +``` + +`fetch-feeds.yml` already sets `id-token: write`, so no permission change is needed there. + +## Cost + +Nothing. Entra ID application registrations are free. diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000..9d08123 --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,81 @@ +# The workload identity behind the Microsoft 365 Message Center ingestion. Message Center has no +# RSS: it is Graph only, at /admin/serviceAnnouncement/messages, and that endpoint needs an +# authenticated tenant-scoped caller. This stack creates the one Entra application that provides it, +# federated to GitHub Actions by OIDC so the repository holds no client secret and there is nothing +# to rotate. +# +# Nothing else is provisioned. The site itself is static, built by Actions and served by GitHub +# Pages, so there is no Azure infrastructure to speak of. +locals { + spn_name = "svp-${var.short}-${var.loc}-${var.env}-mc-001" + + github_issuer = "https://token.actions.githubusercontent.com" + + # A job that declares an `environment:` presents the environment form of the OIDC subject, not the + # ref form. The feed job declares one (github-pages, for the Pages deployment), so the environment + # subject is the one that actually matters today. The ref subject is registered alongside it so + # that splitting the Graph fetch into its own environment-less job later needs no Entra change. + github_subjects = { + environment = "repo:${var.github_org}/${var.github_repo}:environment:${var.github_environment}" + branch = "repo:${var.github_org}/${var.github_repo}:ref:refs/heads/${var.github_branch}" + } + + # GitHub forces an immutable subject (the org@id form) on repositories created or renamed after + # 2026-07-15. This repository was created before that date, so the plain subjects above are what + # the runner presents, but a rename would silently flip the format and fail the run with + # AADSTS7002131. Registering both forms up front makes that a non-event. + github_immutable_subjects = var.register_immutable_subjects ? { + for key, subject in local.github_subjects : + "${key}-immutable" => replace( + subject, + "repo:${var.github_org}/", + "repo:${var.github_org}@${var.github_org_id}/", + ) + } : {} + + federated_subjects = merge(local.github_subjects, local.github_immutable_subjects) +} + +data "azuread_client_config" "current" {} + +# Microsoft Graph's own service principal in this tenant, used only to resolve app role ids for the +# manual consent commands in the outputs. The module resolves the same ids internally when it grants +# consent itself, so this is not on the critical path. +data "azuread_service_principal" "microsoft_graph" { + client_id = "00000003-0000-0000-c000-000000000000" +} + +module "message_center_spn" { + source = "libre-devops/service-principal/azuread" + version = "~> 4.0" + + service_principals = { + (local.spn_name) = { + description = "Reads the Microsoft 365 Message Center for the security.libredevops.org feed." + sign_in_audience = "AzureADMyOrg" + notes = "Managed by Terraform in libre-devops/security-news (terraform/). Read only, no credentials: the sole credential is a GitHub Actions federated identity." + + owners = [data.azuread_client_config.current.object_id] + service_principal_owners = [data.azuread_client_config.current.object_id] + service_principal_tags = ["security-news", "message-center"] + + # Application roles, not delegated scopes: the feed job runs unattended on a schedule, so + # there is no signed-in user whose consent could be carried. + microsoft_graph_application_roles = var.graph_application_roles + microsoft_graph_grant_admin_consent = var.grant_admin_consent + + # No client_secrets and no client_certificates, deliberately. The federated credentials below + # are the only way to authenticate as this application, which means nothing secret ever lands + # in the repository, in Actions secrets, or in this stack's state. + federated_credentials = { + for key, subject in local.federated_subjects : key => { + display_name = "github-${key}" + issuer = local.github_issuer + subject = subject + audiences = ["api://AzureADTokenExchange"] + description = "GitHub Actions OIDC for ${var.github_org}/${var.github_repo}, subject ${subject}." + } + } + } + } +} diff --git a/terraform/outputs.tf b/terraform/outputs.tf new file mode 100644 index 0000000..389242d --- /dev/null +++ b/terraform/outputs.tf @@ -0,0 +1,48 @@ +output "application_client_id" { + description = "The application (client) id. Not a secret: publish it as the AZURE_CLIENT_ID Actions variable, see github_variable_commands." + value = module.message_center_spn.client_ids[local.spn_name] +} + +output "application_name" { + description = "Display name of the application registration, as it appears in Entra ID." + value = local.spn_name +} + +output "application_object_id" { + description = "Object id of the application registration, for the Entra portal deep link and any manual follow-up." + value = module.message_center_spn.application_object_ids[local.spn_name] +} + +output "federated_credential_subjects" { + description = "The OIDC subjects trusted by this application, keyed by credential name. A GitHub Actions run whose sub claim is not in this list cannot authenticate, so compare against the run log when azure/login fails with AADSTS7002131." + value = local.federated_subjects +} + +output "github_variable_commands" { + description = "The two Actions variables the feed workflow reads. Neither is a secret (a client id and a tenant id are identifiers, not credentials), so they are variables rather than secrets and are safe to show in run logs." + value = [ + "gh variable set AZURE_CLIENT_ID --repo ${var.github_org}/${var.github_repo} --body ${module.message_center_spn.client_ids[local.spn_name]}", + "gh variable set AZURE_TENANT_ID --repo ${var.github_org}/${var.github_repo} --body ${data.azuread_client_config.current.tenant_id}", + ] +} + +output "grant_admin_consent_commands" { + description = "The az CLI alternative to grant_admin_consent: one tenant-wide admin consent grant per requested Graph application role, ready to run by someone holding AppRoleAssignment.ReadWrite.All. Only needed when grant_admin_consent is false." + value = [ + for role in var.graph_application_roles : join(" ", [ + "az rest --method POST", + "--url https://graph.microsoft.com/v1.0/servicePrincipals/${module.message_center_spn.service_principal_object_ids[local.spn_name]}/appRoleAssignments", + "--body '{\"principalId\":\"${module.message_center_spn.service_principal_object_ids[local.spn_name]}\",\"resourceId\":\"${data.azuread_service_principal.microsoft_graph.object_id}\",\"appRoleId\":\"${data.azuread_service_principal.microsoft_graph.app_role_ids[role]}\"}'", + ]) + ] +} + +output "service_principal_object_id" { + description = "Object id of the service principal (the enterprise application). This is the principal that holds the Graph role grants." + value = module.message_center_spn.service_principal_object_ids[local.spn_name] +} + +output "tenant_id" { + description = "Tenant the application belongs to. Not a secret: publish it as the AZURE_TENANT_ID Actions variable." + value = data.azuread_client_config.current.tenant_id +} diff --git a/terraform/providers.tf b/terraform/providers.tf new file mode 100644 index 0000000..da7b4b8 --- /dev/null +++ b/terraform/providers.tf @@ -0,0 +1,10 @@ +# Local, personal-tenant deployment: Azure CLI user auth and local state, matching the other +# single-tenant helper stacks in this estate. Run `az login` before applying. +# +# The applier needs Application.ReadWrite.All (or Application Administrator) to create the +# registration. When grant_admin_consent is true it also needs AppRoleAssignment.ReadWrite.All (or +# Privileged Role Administrator), because granting a Graph application role IS tenant-wide admin +# consent. Set grant_admin_consent to false to apply as a plain Application Administrator and hand +# the consent step to someone who holds it, using the commands in the grant_admin_consent_commands +# output. +provider "azuread" {} diff --git a/terraform/terraform.tf b/terraform/terraform.tf new file mode 100644 index 0000000..846d2b5 --- /dev/null +++ b/terraform/terraform.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.9.0, < 2.0.0" + + required_providers { + azuread = { + source = "hashicorp/azuread" + version = ">= 3.0.0, < 4.0.0" + } + } +} diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000..1972f6e --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,70 @@ +variable "env" { + description = "Suffix: environment code used in the application name. The feed job runs against the live site, so this defaults to prd." + type = string + default = "prd" +} + +variable "github_branch" { + description = "Branch the scheduled feed job runs on. Used to build the ref form of the OIDC subject, which is what a job WITHOUT an environment presents." + type = string + default = "master" +} + +variable "github_environment" { + description = "Deployment environment the feed job declares. Used to build the environment form of the OIDC subject, which is what a job WITH an environment presents (and the feed job does, for Pages)." + type = string + default = "github-pages" +} + +variable "github_org" { + description = "GitHub organisation owning the repository, as it appears in the OIDC subject claim." + type = string + default = "libre-devops" +} + +variable "github_org_id" { + description = "Numeric GitHub organisation id, used only to build the immutable (org@id) form of the OIDC subject. See register_immutable_subjects." + type = number + default = 101948202 +} + +variable "github_repo" { + description = "GitHub repository running the feed job, as it appears in the OIDC subject claim." + type = string + default = "security-news" +} + +variable "grant_admin_consent" { + description = "Grant the requested Graph application roles in this stack. Each grant IS tenant-wide admin consent, so the applier needs AppRoleAssignment.ReadWrite.All. Set false to request the roles without consenting, then use the grant_admin_consent_commands output." + type = bool + default = true +} + +variable "graph_application_roles" { + description = "Microsoft Graph APPLICATION roles the identity requests. ServiceMessage.Read.All is the only one the Message Center ingestion needs: it is read only and grants nothing beyond the service announcement surface. Keep this list minimal." + type = set(string) + default = ["ServiceMessage.Read.All"] + + validation { + condition = length(var.graph_application_roles) > 0 + error_message = "graph_application_roles must request at least one role, otherwise the identity cannot read the Message Center." + } +} + +variable "loc" { + description = "Outfix: short Azure region code used in the application name. Entra ID objects are not regional, so this is naming consistency only." + type = string + default = "uks" +} + +variable "register_immutable_subjects" { + description = "Also register the immutable (org@id) form of each OIDC subject. GitHub forces that format on repositories created or renamed after 2026-07-15. This repository predates it, so the plain form is what the runner presents today, but registering both means a later rename cannot break the scheduled run with AADSTS7002131." + type = bool + default = true +} + +variable "short" { + description = "Infix: short product code used in the application name." + type = string + default = "ldo" +} From cb276f5509fed4e843a5ae671115929f78147bce Mon Sep 17 00:00:00 2001 From: Craig Thacker Date: Sat, 8 Aug 2026 16:56:55 +0100 Subject: [PATCH 2/5] refactor: apply the Message Center stack from the pipeline, not a laptop Local state for something a scheduled production job depends on was the wrong call: no history, no locking, and the app registration goes unmanaged the moment that laptop does. Moves to the estate convention instead. - azurerm backend, configured at init from the org TFSTATE_* secrets, with a backend_override.tf.example for local plan-only work. - Provider on use_oidc, so CI authenticates as the org CI identity with no client secret. Local az login still works, verified by plan. - terraform.yml: plan on any PR touching terraform/, apply only on a manual dispatch. Fork PRs are skipped, since GitHub issues them no OIDC token. grant_admin_consent now defaults to false. Granting a Graph application role needs AppRoleAssignment.ReadWrite.All, which lets its holder assign any app role of any API to any principal, including granting itself directory write. On the shared org CI identity that turns a workflow-file edit into a tenant escalation path, so the pipeline requests the role and a human consents once out of band. Also renames the published Actions variables to MESSAGE_CENTER_CLIENT_ID and MESSAGE_CENTER_TENANT_ID. AZURE_CLIENT_ID and AZURE_TENANT_ID already exist as org variables for the CI identity, and a repository variable of the same name shadows the org one, which would have broken this repository's own pipeline. Verified with fmt, validate and a local plan: 8 to add, the consent grant correctly absent while the permission request stays managed. --- .github/workflows/terraform.yml | 66 +++++++++++++++++++++++++ terraform/README.md | 69 +++++++++++++++++++-------- terraform/backend_override.tf.example | 9 ++++ terraform/outputs.tf | 12 ++--- terraform/providers.tf | 17 ++++--- terraform/terraform.tf | 5 ++ terraform/variables.tf | 4 +- 7 files changed, 144 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/terraform.yml create mode 100644 terraform/backend_override.tf.example diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml new file mode 100644 index 0000000..8609453 --- /dev/null +++ b/.github/workflows/terraform.yml @@ -0,0 +1,66 @@ +name: Terraform + +# Plan and apply the Message Center workload identity in terraform/, through the estate's +# libre-devops/terraform-azure action against the shared remote state. Plans automatically on any +# pull request that touches the stack; applies only on a manual dispatch with apply ticked, so +# nothing reaches Entra ID without someone asking for it. +# +# Auth is OIDC via the org CI identity (the AZURE_* org variables), never a client secret. State +# lives in the org's firewalled tfstate account (the TFSTATE_* org secrets), so there is no local +# state and drift is visible on the next plan. + +on: + pull_request: + paths: + - "terraform/**" + - ".github/workflows/terraform.yml" + + workflow_dispatch: + inputs: + apply: + description: Apply the plan. Leave unticked to plan only. + type: boolean + default: false + +permissions: + id-token: write + contents: read + +# Terraform state takes a blob lease, so never let two runs overlap. Not cancel-in-progress: killing +# a run mid-apply would leave the lease held and the next run blocked. +concurrency: + group: terraform-message-center + cancel-in-progress: false + +jobs: + terraform: + name: ${{ (github.event_name == 'workflow_dispatch' && inputs.apply) && 'Plan and apply' || 'Plan' }} + runs-on: ubuntu-latest + timeout-minutes: 20 + + # A pull request from a fork is issued no OIDC token, so it could never authenticate. Skip it + # rather than fail the check red on an outside contribution that happens to touch terraform/. + if: >- + github.event_name == 'workflow_dispatch' || + github.event.pull_request.head.repo.full_name == github.repository + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Terraform + uses: libre-devops/terraform-azure@v1 + with: + terraform-code-location: . + terraform-stack-to-run-json: '["terraform"]' + terraform-workspace: prd + run-terraform-init: true + run-terraform-validate: true + run-terraform-plan: true + run-terraform-apply: ${{ github.event_name == 'workflow_dispatch' && inputs.apply }} + arm-client-id: ${{ vars.AZURE_CLIENT_ID }} + arm-tenant-id: ${{ vars.AZURE_TENANT_ID }} + arm-subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + firewall-storage-account-name: ${{ secrets.TFSTATE_STORAGE_ACCOUNT }} + firewall-storage-resource-group: ${{ secrets.TFSTATE_RESOURCE_GROUP }} + terraform-init-extra-args-json: '["-reconfigure","-upgrade","-backend-config=resource_group_name=${{ secrets.TFSTATE_RESOURCE_GROUP }}","-backend-config=storage_account_name=${{ secrets.TFSTATE_STORAGE_ACCOUNT }}","-backend-config=container_name=${{ secrets.TFSTATE_BLOB_CONTAINER }}"]' diff --git a/terraform/README.md b/terraform/README.md index 4694eed..22b36e4 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -16,9 +16,9 @@ Pages, so there is no Azure infrastructure behind it. One application registration and its service principal, named to the Libre DevOps convention (`svp-${short}-${loc}-${env}-mc-001`, so `svp-ldo-uks-prd-mc-001` by default), carrying: -- The `ServiceMessage.Read.All` Microsoft Graph **application** role, with tenant-wide admin consent - granted by default. Application rather than delegated because the feed job runs unattended on a - schedule, with no signed-in user. +- A request for the `ServiceMessage.Read.All` Microsoft Graph **application** role. Application + rather than delegated because the feed job runs unattended on a schedule, with no signed-in user. + The request is managed here; the consent is not (see below). - Federated identity credentials trusting GitHub Actions OIDC from `libre-devops/security-news`. There is **no client secret and no certificate**, by design. The federated credentials are the only @@ -50,37 +50,64 @@ If a run ever does fail with `AADSTS7002131`, compare the subject in the run log ## Applying -State is local and auth is Azure CLI user auth, matching the other single-tenant helper stacks in -this estate. This is applied by hand, occasionally, and is not wired into CI. +Through the pipeline, like everything else in the estate. `.github/workflows/terraform.yml` runs +the `libre-devops/terraform-azure` action against the shared remote state: + +- **Plan** happens automatically on any pull request touching `terraform/**`. +- **Apply** happens only on a manual dispatch with the `apply` input ticked: + +```bash +gh workflow run terraform.yml --repo libre-devops/security-news -f apply=true +``` + +Auth is OIDC through the org CI identity (`AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, +`AZURE_SUBSCRIPTION_ID` org variables) and state lives in the org's firewalled tfstate account +(`TFSTATE_*` org secrets). No local state, no client secret, and drift shows up on the next plan. + +Fork pull requests are skipped rather than failed: GitHub issues them no OIDC token, so they could +never authenticate. + +### Consent is not in the pipeline, deliberately + +`grant_admin_consent` defaults to **false**, so the stack requests `ServiceMessage.Read.All` but +does not consent to it. Granting a Graph application role requires `AppRoleAssignment.ReadWrite.All`, +and that permission lets its holder assign **any** app role of **any** API to **any** principal, +including granting itself directory write. Handing that to the shared org CI identity, which every +repository in the org can use, would turn a workflow-file change into a tenant escalation path. + +So consent is a one-off human act. After the first apply, run what +`terraform output grant_admin_consent_commands` prints, as someone who holds the permission: ```bash -az login -terraform -chdir=terraform init -terraform -chdir=terraform plan -terraform -chdir=terraform apply +az rest --method POST \ + --url https://graph.microsoft.com/v1.0/servicePrincipals//appRoleAssignments \ + --body '{"principalId":"","resourceId":"","appRoleId":""}' ``` -The applier needs: +Terraform does not manage that grant, so it will not fight you over it or try to remove it. If you +would rather manage it here, set `grant_admin_consent = true` and apply interactively as someone +who already holds the permission. -- `Application.ReadWrite.All`, or the Application Administrator role, to create the registration. -- `AppRoleAssignment.ReadWrite.All`, or Privileged Role Administrator, **when - `grant_admin_consent` is true** (the default). Granting a Graph application role is tenant-wide - admin consent, which is a privileged action. +### Local plans -If you cannot consent yourself, apply with `-var 'grant_admin_consent=false'`. The roles are still -requested on the application, and the `grant_admin_consent_commands` output prints ready-to-run -`az rest` commands for whoever can grant them. +For a local plan without touching the shared state, copy `backend_override.tf.example` to +`backend_override.tf` (gitignored) and re-run `terraform init`. Plan only: applying from a local +state file would fork the real one. ## After applying -Publish the two identifiers the workflow needs. Neither is a secret, so they are Actions **variables** -rather than secrets, and the `terraform output github_variable_commands` output prints both commands: +Publish the two identifiers the feed workflow needs. Neither is a secret, so they are Actions +**variables**, and `terraform output github_variable_commands` prints both: ```bash -gh variable set AZURE_CLIENT_ID --repo libre-devops/security-news --body -gh variable set AZURE_TENANT_ID --repo libre-devops/security-news --body +gh variable set MESSAGE_CENTER_CLIENT_ID --repo libre-devops/security-news --body +gh variable set MESSAGE_CENTER_TENANT_ID --repo libre-devops/security-news --body ``` +They are **not** called `AZURE_CLIENT_ID` and `AZURE_TENANT_ID` on purpose. Those already exist as +org variables for the CI identity, and a repository variable of the same name silently shadows the +org one, which would break this repository's own Terraform pipeline. + `fetch-feeds.yml` already sets `id-token: write`, so no permission change is needed there. ## Cost diff --git a/terraform/backend_override.tf.example b/terraform/backend_override.tf.example new file mode 100644 index 0000000..b7cd103 --- /dev/null +++ b/terraform/backend_override.tf.example @@ -0,0 +1,9 @@ +# Local development only. Copy to backend_override.tf (gitignored) to use local state instead of +# the azurerm backend, so you can plan without remote backend access. Re-run `terraform init` +# after adding it. Do not commit it, and do not apply from local state: the pipeline owns the real +# state, and applying from a local copy would fork it. +terraform { + backend "local" { + path = "terraform.tfstate.local" + } +} diff --git a/terraform/outputs.tf b/terraform/outputs.tf index 389242d..f701e45 100644 --- a/terraform/outputs.tf +++ b/terraform/outputs.tf @@ -1,5 +1,5 @@ output "application_client_id" { - description = "The application (client) id. Not a secret: publish it as the AZURE_CLIENT_ID Actions variable, see github_variable_commands." + description = "The application (client) id. Not a secret: publish it as the MESSAGE_CENTER_CLIENT_ID Actions variable, see github_variable_commands." value = module.message_center_spn.client_ids[local.spn_name] } @@ -19,15 +19,15 @@ output "federated_credential_subjects" { } output "github_variable_commands" { - description = "The two Actions variables the feed workflow reads. Neither is a secret (a client id and a tenant id are identifiers, not credentials), so they are variables rather than secrets and are safe to show in run logs." + description = "The two Actions variables the feed workflow reads. Neither is a secret (a client id and a tenant id are identifiers, not credentials), so they are variables rather than secrets and are safe to show in run logs. They are deliberately NOT called AZURE_CLIENT_ID and AZURE_TENANT_ID: those already exist as ORG variables for the CI identity, and a repository variable of the same name silently shadows the org one, which would break this repository's own Terraform pipeline." value = [ - "gh variable set AZURE_CLIENT_ID --repo ${var.github_org}/${var.github_repo} --body ${module.message_center_spn.client_ids[local.spn_name]}", - "gh variable set AZURE_TENANT_ID --repo ${var.github_org}/${var.github_repo} --body ${data.azuread_client_config.current.tenant_id}", + "gh variable set MESSAGE_CENTER_CLIENT_ID --repo ${var.github_org}/${var.github_repo} --body ${module.message_center_spn.client_ids[local.spn_name]}", + "gh variable set MESSAGE_CENTER_TENANT_ID --repo ${var.github_org}/${var.github_repo} --body ${data.azuread_client_config.current.tenant_id}", ] } output "grant_admin_consent_commands" { - description = "The az CLI alternative to grant_admin_consent: one tenant-wide admin consent grant per requested Graph application role, ready to run by someone holding AppRoleAssignment.ReadWrite.All. Only needed when grant_admin_consent is false." + description = "One tenant-wide admin consent grant per requested Graph application role, ready to run by someone holding AppRoleAssignment.ReadWrite.All. This is the normal path: grant_admin_consent defaults to false so the pipeline never holds that permission, and consent stays a deliberate one-off human act. Run these once after the first apply." value = [ for role in var.graph_application_roles : join(" ", [ "az rest --method POST", @@ -43,6 +43,6 @@ output "service_principal_object_id" { } output "tenant_id" { - description = "Tenant the application belongs to. Not a secret: publish it as the AZURE_TENANT_ID Actions variable." + description = "Tenant the application belongs to. Not a secret: publish it as the MESSAGE_CENTER_TENANT_ID Actions variable." value = data.azuread_client_config.current.tenant_id } diff --git a/terraform/providers.tf b/terraform/providers.tf index da7b4b8..0ff8dca 100644 --- a/terraform/providers.tf +++ b/terraform/providers.tf @@ -1,10 +1,9 @@ -# Local, personal-tenant deployment: Azure CLI user auth and local state, matching the other -# single-tenant helper stacks in this estate. Run `az login` before applying. +# OIDC in the pipeline, per the Libre DevOps Terraform standard: ARM_CLIENT_ID, ARM_TENANT_ID and +# ARM_SUBSCRIPTION_ID come from the org Actions variables and ARM_OIDC_TOKEN is injected by the +# runner, so no client secret exists anywhere in this repository. # -# The applier needs Application.ReadWrite.All (or Application Administrator) to create the -# registration. When grant_admin_consent is true it also needs AppRoleAssignment.ReadWrite.All (or -# Privileged Role Administrator), because granting a Graph application role IS tenant-wide admin -# consent. Set grant_admin_consent to false to apply as a plain Application Administrator and hand -# the consent step to someone who holds it, using the commands in the grant_admin_consent_commands -# output. -provider "azuread" {} +# The provider still falls back to Azure CLI auth when no OIDC token is present, so `az login` and +# a local plan work unchanged. +provider "azuread" { + use_oidc = true +} diff --git a/terraform/terraform.tf b/terraform/terraform.tf index 846d2b5..de7e021 100644 --- a/terraform/terraform.tf +++ b/terraform/terraform.tf @@ -7,4 +7,9 @@ terraform { version = ">= 3.0.0, < 4.0.0" } } + + # Deliberately empty: the backend is configured at init time by the terraform-azure action, from + # the org TFSTATE_* secrets. For local work, copy backend_override.tf.example to + # backend_override.tf (gitignored) to fall back to local state. + backend "azurerm" {} } diff --git a/terraform/variables.tf b/terraform/variables.tf index 1972f6e..2ca2deb 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -35,9 +35,9 @@ variable "github_repo" { } variable "grant_admin_consent" { - description = "Grant the requested Graph application roles in this stack. Each grant IS tenant-wide admin consent, so the applier needs AppRoleAssignment.ReadWrite.All. Set false to request the roles without consenting, then use the grant_admin_consent_commands output." + description = "Grant the requested Graph application roles from this stack. Defaults to false because granting one IS tenant-wide admin consent, which needs AppRoleAssignment.ReadWrite.All: a permission that lets its holder assign ANY app role of ANY API to ANY principal, including granting itself directory write. That is not something the shared org CI identity should carry, so the pipeline requests the roles and a human consents once using the grant_admin_consent_commands output. Set true only when applying interactively as someone who already holds it." type = bool - default = true + default = false } variable "graph_application_roles" { From fc490414782cb69d400b69d3d27867cf3e05825d Mon Sep 17 00:00:00 2001 From: Craig Thacker Date: Sat, 8 Aug 2026 17:00:30 +0100 Subject: [PATCH 3/5] fix: let a human own the application, not just the applier The pipeline plan showed the owner resolving to the org CI service principal, because owners came from azuread_client_config and that is whoever applied. Applied from CI the application would end up under nobody's owned applications and need a directory role to touch in the portal. Adds additional_owner_object_ids, unioned with the applier so the pipeline can still manage what it created. --- terraform/README.md | 11 +++++++++++ terraform/main.tf | 9 +++++++-- terraform/variables.tf | 6 ++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/terraform/README.md b/terraform/README.md index 22b36e4..b234502 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -67,6 +67,17 @@ Auth is OIDC through the org CI identity (`AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, Fork pull requests are skipped rather than failed: GitHub issues them no OIDC token, so they could never authenticate. +### Ownership + +The application is owned by whoever applied, which from the pipeline means the **org CI service +principal**, not you. Set `additional_owner_object_ids` to your own object id as well, otherwise the +application shows up under nobody's owned applications and needs a directory role to touch in the +portal: + +```bash +az ad signed-in-user show --query id -o tsv +``` + ### Consent is not in the pipeline, deliberately `grant_admin_consent` defaults to **false**, so the stack requests `ServiceMessage.Read.All` but diff --git a/terraform/main.tf b/terraform/main.tf index 9d08123..ea2382e 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -34,6 +34,11 @@ locals { } : {} federated_subjects = merge(local.github_subjects, local.github_immutable_subjects) + + # Whoever applies, plus anyone named explicitly. The applier differs by path: a human locally, the + # org CI service principal from the pipeline. Keeping the applier in the set means the pipeline can + # always manage what it created; additional_owner_object_ids is how a human stays able to as well. + owners = setunion([data.azuread_client_config.current.object_id], var.additional_owner_object_ids) } data "azuread_client_config" "current" {} @@ -55,8 +60,8 @@ module "message_center_spn" { sign_in_audience = "AzureADMyOrg" notes = "Managed by Terraform in libre-devops/security-news (terraform/). Read only, no credentials: the sole credential is a GitHub Actions federated identity." - owners = [data.azuread_client_config.current.object_id] - service_principal_owners = [data.azuread_client_config.current.object_id] + owners = local.owners + service_principal_owners = local.owners service_principal_tags = ["security-news", "message-center"] # Application roles, not delegated scopes: the feed job runs unattended on a schedule, so diff --git a/terraform/variables.tf b/terraform/variables.tf index 2ca2deb..a5cc23c 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -1,3 +1,9 @@ +variable "additional_owner_object_ids" { + description = "Extra Entra object ids to own the application and its service principal, alongside whoever applied. Worth setting at least one human: applied from the pipeline the applier is the org CI service principal, so without this the application appears under nobody's owned applications and needs a directory role to touch in the portal." + type = set(string) + default = [] +} + variable "env" { description = "Suffix: environment code used in the application name. The feed job runs against the live site, so this defaults to prd." type = string From d93eb93f04b9407d0fdd6f058267e6e8fc3eaa7c Mon Sep 17 00:00:00 2001 From: Craig Thacker Date: Sat, 8 Aug 2026 17:15:27 +0100 Subject: [PATCH 4/5] feat: ingest the Message Center, filter the broad feeds, add a justfile Source gains include_categories and exclude_categories, matched against an entry's own categories. The filter runs BEFORE the max_entries cut, which is the whole point: slicing first would take the newest 25 items and then filter them, so a broad feed like the roadmap would usually yield nothing. Three sources on top of that: - Microsoft 365 Roadmap, filtered to the Defender, Sentinel, Purview, Entra and Intune categories. Copilot is deliberately excluded, being the largest category by far and almost none of it security. 1819 raw items in, 30 kept. - Azure Service Updates, filtered to Security, Compliance, Identity and the security services. 200 raw in, 22 kept. - Message Center over Graph, the one source with no feed of any kind. The Graph path takes a token from whichever route is available: an explicit MESSAGE_CENTER_ACCESS_TOKEN, the GitHub Actions OIDC exchange, or the Azure CLI for local work. When none is available, or consent has not been granted, it logs and returns nothing, so the site degrades to its public feeds rather than failing the run. Verified: 403 from Graph is caught and named as a consent problem, and the run still produced 150 articles. Graph timestamps carry up to seven fractional-second digits, which fromisoformat rejects, so they are trimmed before parsing. The justfile wraps the LibreDevOpsHelpers engine so local work mirrors the action: the same fmt, validate, tflint and trivy gates, the same remote backend and the same storage firewall dance, plus feed and site recipes. The state key is now pinned in both the workflow and the justfile. The helper derives one from the folder layout, which resolves differently on a runner (GitHub nests the repo inside a directory of the same name) than on a laptop, so leaving it automatic would have given CI and local two separate states, and two application registrations. --- .github/workflows/fetch-feeds.yml | 10 + .github/workflows/terraform.yml | 7 +- justfile | 196 ++++++++++++++ scripts/fetch_feeds.py | 419 +++++++++++++++++++++++++++++- 4 files changed, 629 insertions(+), 3 deletions(-) create mode 100644 justfile diff --git a/.github/workflows/fetch-feeds.yml b/.github/workflows/fetch-feeds.yml index 5fffac7..b9350ac 100644 --- a/.github/workflows/fetch-feeds.yml +++ b/.github/workflows/fetch-feeds.yml @@ -46,6 +46,16 @@ jobs: pip install -r scripts/requirements.txt - name: Fetch Microsoft security feeds + env: + # Message Center is the one source that needs credentials. These are identifiers, not + # secrets, so they are Actions variables. They are deliberately not AZURE_CLIENT_ID or + # AZURE_TENANT_ID: those exist as ORG variables for the CI identity, and a repository + # variable of the same name would shadow them and break the Terraform pipeline. + # + # When they are unset (or consent has not been granted yet) fetch_feeds.py logs and skips + # that one source, so the site still builds from the public feeds. + MESSAGE_CENTER_CLIENT_ID: ${{ vars.MESSAGE_CENTER_CLIENT_ID }} + MESSAGE_CENTER_TENANT_ID: ${{ vars.MESSAGE_CENTER_TENANT_ID }} run: | python scripts/fetch_feeds.py | tee feed-output.log diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index 8609453..3de7960 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -63,4 +63,9 @@ jobs: arm-subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} firewall-storage-account-name: ${{ secrets.TFSTATE_STORAGE_ACCOUNT }} firewall-storage-resource-group: ${{ secrets.TFSTATE_RESOURCE_GROUP }} - terraform-init-extra-args-json: '["-reconfigure","-upgrade","-backend-config=resource_group_name=${{ secrets.TFSTATE_RESOURCE_GROUP }}","-backend-config=storage_account_name=${{ secrets.TFSTATE_STORAGE_ACCOUNT }}","-backend-config=container_name=${{ secrets.TFSTATE_BLOB_CONTAINER }}"]' + # The state key is pinned rather than left to the action's auto-computed one. That key is + # derived from the folder layout, which resolves differently here (GitHub nests the repo + # inside a directory of the same name) than on a laptop, so an auto key would quietly + # give CI and local two separate states, and two application registrations. The justfile + # pins the same value. + terraform-init-extra-args-json: '["-reconfigure","-upgrade","-backend-config=resource_group_name=${{ secrets.TFSTATE_RESOURCE_GROUP }}","-backend-config=storage_account_name=${{ secrets.TFSTATE_STORAGE_ACCOUNT }}","-backend-config=container_name=${{ secrets.TFSTATE_BLOB_CONTAINER }}","-backend-config=key=security-news-message-center.tfstate"]' diff --git a/justfile b/justfile new file mode 100644 index 0000000..533b202 --- /dev/null +++ b/justfile @@ -0,0 +1,196 @@ +# Local development task runner for security-news. Run `just` to list recipes. +# +# Install just with either: +# brew install just +# uv tool add rust-just # then call recipes as: uv run just +# +# The terraform recipes wrap the LibreDevOpsHelpers engine functions so local work mirrors the +# libre-devops/terraform-azure action: the same fmt, validate, tflint and trivy gates, the same +# remote azurerm backend, and the same storage firewall open-before close-after dance. They read +# the state coordinates from the environment the tenant bootstrap publishes: +# $env:TFSTATE_RESOURCE_GROUP, $env:TFSTATE_STORAGE_ACCOUNT, $env:TFSTATE_BLOB_CONTAINER +# Authenticate first with `az login`. A local .env file next to this justfile is loaded too. +# +# The feed recipes need no credentials at all, except the Message Center, which borrows your Azure +# CLI token. Every other source is a public feed. + +set shell := ["pwsh", "-NoProfile", "-Command"] +set dotenv-load + +workspace := env_var_or_default("TF_WORKSPACE", "prd") + +# The backend key is pinned rather than left to the helper's auto-computed one. That key is derived +# from the folder layout, which resolves differently on a runner (where GitHub nests the repo +# inside a directory of the same name) than on a laptop. An auto key would quietly give local and +# CI two separate states, and therefore two application registrations. +state_key := "security-news-message-center.tfstate" + +# List available recipes. +default: + just --list + +# Install or force-update LibreDevOpsHelpers (the engine the terraform recipes wrap) from PSGallery. +update-ldo-pwsh: + if (Get-Module -ListAvailable LibreDevOpsHelpers) { Update-Module LibreDevOpsHelpers -Force; Write-Host 'Updated LibreDevOpsHelpers to the latest from PSGallery.' } else { Install-Module LibreDevOpsHelpers -Scope CurrentUser -Force -AllowClobber; Write-Host 'Installed LibreDevOpsHelpers from PSGallery.' } + +# --- Terraform ---------------------------------------------------------------------------- + +# Format every Terraform file in place. +fmt: + terraform fmt -recursive + +# Offline gates for the stack: format check, validate, tflint, trivy. No cloud access needed. +validate: + #!/usr/bin/env pwsh + Set-StrictMode -Version Latest + $ErrorActionPreference = 'Stop' + Import-Module LibreDevOpsHelpers -Force + Set-LdoLogFormat -Format Text + Clear-LdoFinding + Invoke-LdoTerraformFmtCheck -CodePath ./terraform + terraform -chdir=terraform init -backend=false -input=false | Out-Null + Invoke-LdoTerraformValidate -CodePath ./terraform + Invoke-LdoTfLint -CodePath ./terraform + Invoke-LdoTrivy -CodePath ./terraform + Show-LdoFindingsSummary + +# Trivy config scan only, gating on HIGH and CRITICAL like the action does. +scan: + #!/usr/bin/env pwsh + Set-StrictMode -Version Latest + $ErrorActionPreference = 'Stop' + Import-Module LibreDevOpsHelpers -Force + Set-LdoLogFormat -Format Text + Clear-LdoFinding + Invoke-LdoTrivy -CodePath ./terraform + Show-LdoFindingsSummary + +# Plan against the real remote state. Read only: safe to run any time. +plan: + just _run plan {{ workspace }} + +# Apply against the real remote state. Prefer the pipeline (`just dispatch`) for anything shared. +apply: + just _run apply {{ workspace }} + +# Show the stack outputs, including the consent command and the gh variable commands. +output: + just _run output {{ workspace }} + +# Shared terraform driver: firewall the state account open, run, close it again whatever happens. +_run op ws: + #!/usr/bin/env pwsh + Set-StrictMode -Version Latest + $ErrorActionPreference = 'Stop' + Import-Module LibreDevOpsHelpers -Force + Set-LdoLogFormat -Format Text + Set-LdoTraceContext -Generate + Clear-LdoFinding + + $rg = $env:TFSTATE_RESOURCE_GROUP + $sa = $env:TFSTATE_STORAGE_ACCOUNT + $cn = $env:TFSTATE_BLOB_CONTAINER + if (-not ($rg -and $sa -and $cn)) { + throw 'Set TFSTATE_RESOURCE_GROUP, TFSTATE_STORAGE_ACCOUNT and TFSTATE_BLOB_CONTAINER (the values the tenant bootstrap publishes).' + } + + $path = './terraform' + $added = $false + try { + Add-LdoStorageCurrentIpRule -ResourceGroup $rg -StorageAccountName $sa + $added = $true + + Invoke-LdoTerraformInit -CodePath $path -InitArgs @( + '-reconfigure', + "-backend-config=resource_group_name=$rg", + "-backend-config=storage_account_name=$sa", + "-backend-config=container_name=$cn", + "-backend-config=key={{ state_key }}" + ) + Invoke-LdoTerraformWorkspaceSelect -CodePath $path -WorkspaceName '{{ ws }}' + + switch ('{{ op }}') { + 'output' { + terraform -chdir=terraform output + } + default { + Invoke-LdoTerraformFmtCheck -CodePath $path + Invoke-LdoTerraformValidate -CodePath $path + Invoke-LdoTfLint -CodePath $path + Invoke-LdoTrivy -CodePath $path + Invoke-LdoTerraformPlan -CodePath $path + Show-LdoFindingsSummary + if ('{{ op }}' -eq 'apply') { + Invoke-LdoTerraformApply -CodePath $path -SkipApprove + } + } + } + } + finally { + if ($added) { Remove-LdoStorageCurrentIpRule -ResourceGroup $rg -StorageAccountName $sa } + } + +# Run the pipeline instead of applying locally. This is the preferred path for a shared change. +dispatch: + gh workflow run terraform.yml --repo libre-devops/security-news -f apply=true + +# --- Message Center identity -------------------------------------------------------------- + +# Print the one-off tenant-wide admin consent command (needs AppRoleAssignment.ReadWrite.All). +consent: + #!/usr/bin/env pwsh + Set-StrictMode -Version Latest + $ErrorActionPreference = 'Stop' + Write-Host 'Run this once, as someone holding AppRoleAssignment.ReadWrite.All:' -ForegroundColor Cyan + just _run output {{ workspace }} | Select-String -Pattern 'az rest' + +# Publish the client and tenant ids as repository Actions VARIABLES (neither is a secret). +publish-vars: + #!/usr/bin/env pwsh + Set-StrictMode -Version Latest + $ErrorActionPreference = 'Stop' + $clientId = (terraform -chdir=terraform output -raw application_client_id) + $tenantId = (terraform -chdir=terraform output -raw tenant_id) + gh variable set MESSAGE_CENTER_CLIENT_ID --repo libre-devops/security-news --body $clientId + gh variable set MESSAGE_CENTER_TENANT_ID --repo libre-devops/security-news --body $tenantId + Write-Host 'Published MESSAGE_CENTER_CLIENT_ID and MESSAGE_CENTER_TENANT_ID.' -ForegroundColor Green + +# --- Feeds --------------------------------------------------------------------------------- + +# Regenerate data/feeds.json and data/feed.xml in place, as the scheduled job does (az login first). +feeds: + uv run --with-requirements scripts/requirements.txt python scripts/fetch_feeds.py + +# Same, but into a scratch directory, so the committed data files are left alone. +feeds-dry: + #!/usr/bin/env pwsh + Set-StrictMode -Version Latest + $ErrorActionPreference = 'Stop' + $scratch = Join-Path ([System.IO.Path]::GetTempPath()) ("security-news-" + [guid]::NewGuid()) + New-Item -ItemType Directory -Path $scratch | Out-Null + try { + Push-Location $scratch + uv run --with-requirements '{{ justfile_directory() }}/scripts/requirements.txt' python '{{ justfile_directory() }}/scripts/fetch_feeds.py' + } + finally { + Pop-Location + Remove-Item $scratch -Recurse -Force -ErrorAction SilentlyContinue + } + +# Format the Python with black, in place. +py-fmt: + uv run --with black black scripts/fetch_feeds.py + +# Offline gates for the Python: black format check plus an import smoke test. +py-check: + uv run --with black black --check scripts/fetch_feeds.py + uv run --with-requirements scripts/requirements.txt python -c "import ast, pathlib; ast.parse(pathlib.Path('scripts/fetch_feeds.py').read_text()); print('fetch_feeds.py parses clean')" + +# --- Site ---------------------------------------------------------------------------------- + +# Serve the site locally at http://localhost:8000 for a quick look before pushing. +serve: + python3 -m http.server 8000 + +# Everything that can run offline: Python gates plus the Terraform gates. +check: py-check validate diff --git a/scripts/fetch_feeds.py b/scripts/fetch_feeds.py index 70072e4..0c115e1 100644 --- a/scripts/fetch_feeds.py +++ b/scripts/fetch_feeds.py @@ -9,6 +9,10 @@ import os import re import socket +import subprocess # nosec B404 +import urllib.error +import urllib.parse +import urllib.request from dataclasses import dataclass from datetime import datetime, timedelta, timezone from html import unescape @@ -32,6 +36,26 @@ # the whole run until the CI job-level timeout kills it. FEED_TIMEOUT_SECONDS = 20 +# Microsoft Graph, for the Message Center. It is the one source with no feed of +# any kind: no RSS, no Atom, no anonymous endpoint. Everything else here is a +# public feed that needs no credentials at all. +GRAPH_MESSAGE_CENTER_URL = ( + "https://graph.microsoft.com/v1.0/admin/serviceAnnouncement/messages" +) +GRAPH_RESOURCE = "https://graph.microsoft.com" +GRAPH_SCOPE = f"{GRAPH_RESOURCE}/.default" +GRAPH_PAGE_SIZE = 100 + +# Message Center posts are only readable in the admin centre, by someone with +# admin access to the tenant they were published to. There is no public +# permalink, so this link is a pointer for the maintainer rather than something +# a general reader can follow. +MESSAGE_CENTER_LINK = "https://admin.microsoft.com/#/MessageCenter/:/messages/" + +# The audience GitHub must mint its OIDC token for. It has to match the audience +# on the federated identity credential in terraform/main.tf. +ENTRA_TOKEN_AUDIENCE = "api://AzureADTokenExchange" + @dataclass(frozen=True) class Source: @@ -46,6 +70,19 @@ class Source: board_id: Optional[str] = None max_entries: int = 25 + # Category filters, matched case-insensitively against an entry's own + # categories: the elements of an RSS item, or the services a + # Message Center post applies to. Empty include_categories means keep + # everything; otherwise an entry must carry at least one listed category. + # exclude_categories always wins. + # + # This exists because the broad Microsoft feeds are not security feeds. The + # M365 roadmap publishes every Outlook, Teams and OneDrive change alongside + # the handful worth showing here, so an unfiltered source would bury the + # site. Filters run BEFORE max_entries, or the cut would starve them. + include_categories: Tuple[str, ...] = () + exclude_categories: Tuple[str, ...] = () + SOURCES: List[Source] = [ Source( @@ -195,6 +232,80 @@ class Source: board_id="azurenetworksecurityblog", category="Network Security", ), + # Roadmap and service update feeds. Both are change announcements rather + # than security news, so both are filtered hard: the roadmap publishes over + # 1800 items covering every Outlook, Teams and OneDrive tweak, of which only + # a small tail is relevant here. Copilot is deliberately not on the + # allowlist; it is the single largest category and almost none of it is + # security. The ?filters= parameter these APIs advertise is ignored server + # side (it returns the identical item count), so filtering has to be ours. + Source( + id="m365_roadmap", + name="Microsoft 365 Roadmap", + url="https://www.microsoft.com/releasecommunications/api/v1/m365/rss", + vendor="Microsoft", + default_author="Microsoft", + source_group="Official Microsoft", + category="Roadmap", + max_entries=30, + include_categories=( + "Microsoft Defender XDR", + "Microsoft Defender for Endpoint", + "Microsoft Defender for Office 365", + "Microsoft Defender for Identity", + "Microsoft Defender for Cloud Apps", + "Microsoft Sentinel", + "Microsoft Purview", + "Microsoft Entra", + "Microsoft Intune", + ), + ), + Source( + id="azure_updates", + name="Azure Service Updates", + url="https://www.microsoft.com/releasecommunications/api/v2/azure/rss", + vendor="Microsoft", + default_author="Microsoft", + source_group="Official Microsoft", + category="Service Updates", + max_entries=25, + include_categories=( + "Security", + "Compliance", + "Identity", + "Microsoft Defender for Cloud", + "Microsoft Sentinel", + "Microsoft Entra ID", + "Azure Firewall", + "Azure Key Vault", + ), + ), + # The only authenticated source. See fetch_message_center: when no token can + # be obtained it logs and returns nothing, so a credential problem degrades + # the site to its public feeds rather than failing the run. + Source( + id="message_center", + name="Microsoft 365 Message Center", + url=GRAPH_MESSAGE_CENTER_URL, + vendor="Microsoft", + default_author="Microsoft", + source_group="Official Microsoft", + source_kind="graph", + category="Service Announcements", + max_entries=40, + include_categories=( + "Microsoft Defender XDR", + "Microsoft 365 Defender", + "Microsoft Defender for Endpoint", + "Microsoft Defender for Office 365", + "Microsoft Defender for Identity", + "Microsoft Defender for Cloud Apps", + "Microsoft Sentinel", + "Microsoft Purview", + "Microsoft Entra", + "Microsoft Intune", + ), + ), Source( id="aws_security", name="AWS Security Bulletins", @@ -556,20 +667,58 @@ def normalize_entry(entry: Any, source: Source) -> Optional[dict]: } +def entry_categories(entry: Any) -> List[str]: + """The terms on a feed entry, as plain strings.""" + return [ + (tag.get("term") or "").strip() + for tag in (entry.get("tags") or []) + if (tag.get("term") or "").strip() + ] + + +def passes_category_filter(source: Source, categories: List[str]) -> bool: + """Whether an entry's own categories satisfy the source's filters.""" + if not source.include_categories and not source.exclude_categories: + return True + + lowered = {category.lower() for category in categories} + + if any(excluded.lower() in lowered for excluded in source.exclude_categories): + return False + + if not source.include_categories: + return True + + return any(included.lower() in lowered for included in source.include_categories) + + def fetch_feed(source: Source) -> List[dict]: print(f"Fetching: {source.name}") try: feed = feedparser.parse(source.url) articles = [] + filtered_out = 0 + + # Filter first, slice second. Doing it the other way round would cut the + # newest max_entries items and then filter what survived, so a broad + # feed like the roadmap would usually yield nothing at all. + for entry in feed.entries: + if not passes_category_filter(source, entry_categories(entry)): + filtered_out += 1 + continue - for entry in feed.entries[: source.max_entries]: article = normalize_entry(entry, source) if article: articles.append(article) + if len(articles) >= source.max_entries: + break + print(f" Found {len(articles)} articles") print(f" Feed contains {len(feed.entries)} raw entries") + if filtered_out: + print(f" Filtered out {filtered_out} entries by category") return articles except Exception as ex: @@ -577,6 +726,269 @@ def fetch_feed(source: Source) -> List[dict]: return [] +def http_json( + url: str, + *, + data: Optional[bytes] = None, + headers: Optional[Dict[str, str]] = None, +) -> dict: + """A small JSON HTTP helper, so the Graph path needs no extra dependency.""" + request = urllib.request.Request( # nosec B310 + url, + data=data, + headers=headers or {}, + method="POST" if data else "GET", + ) + + if not url.lower().startswith("https://"): + raise ValueError(f"Refusing to call a non-HTTPS URL: {url}") + + with urllib.request.urlopen( # nosec B310 + request, timeout=FEED_TIMEOUT_SECONDS + ) as response: + return json.load(response) + + +def github_oidc_token() -> Optional[str]: + """The GitHub Actions OIDC token, when running inside Actions.""" + request_url = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_URL") + request_token = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_TOKEN") + + if not request_url or not request_token: + return None + + url = f"{request_url}&audience={urllib.parse.quote(ENTRA_TOKEN_AUDIENCE)}" + + payload = http_json(url, headers={"Authorization": f"Bearer {request_token}"}) + return payload.get("value") + + +def entra_token_from_github_oidc() -> Optional[str]: + """ + Exchange the GitHub OIDC token for a Graph token, the client credentials + flow with a client assertion. No client secret is involved: the assertion IS + the credential, and it is minted per job and expires in minutes. + """ + client_id = os.environ.get("MESSAGE_CENTER_CLIENT_ID") + tenant_id = os.environ.get("MESSAGE_CENTER_TENANT_ID") + + if not client_id or not tenant_id: + return None + + assertion = github_oidc_token() + if not assertion: + return None + + body = urllib.parse.urlencode( + { + "client_id": client_id, + "scope": GRAPH_SCOPE, + "grant_type": "client_credentials", + "client_assertion_type": ( + "urn:ietf:params:oauth:client-assertion-type:jwt-bearer" + ), + "client_assertion": assertion, + } + ).encode() + + payload = http_json( + f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token", + data=body, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + return payload.get("access_token") + + +def entra_token_from_azure_cli() -> Optional[str]: + """ + Local development fallback: borrow the signed-in user's Graph token from the + Azure CLI, so `just feeds` works on a laptop after `az login` without any + app registration involvement. + """ + try: + result = subprocess.run( # nosec B603 B607 + [ + "az", + "account", + "get-access-token", + "--resource", + GRAPH_RESOURCE, + "--output", + "json", + ], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return None + + if result.returncode != 0: + return None + + try: + return json.loads(result.stdout).get("accessToken") + except json.JSONDecodeError: + return None + + +def graph_access_token() -> Optional[str]: + """ + A Graph token, by whichever route is available. Returns None rather than + raising: the Message Center is one source among many, so losing it should + cost the site that section, not the whole run. + """ + explicit = os.environ.get("MESSAGE_CENTER_ACCESS_TOKEN") + if explicit: + print(" Using MESSAGE_CENTER_ACCESS_TOKEN from the environment") + return explicit + + try: + federated = entra_token_from_github_oidc() + if federated: + print(" Authenticated by GitHub Actions federated identity") + return federated + except (urllib.error.URLError, ValueError, KeyError) as ex: + print(f" Federated identity exchange failed: {ex}") + + cli = entra_token_from_azure_cli() + if cli: + print(" Authenticated by the Azure CLI (local development)") + return cli + + return None + + +def parse_graph_datetime(value: str) -> str: + """ + Graph timestamps to the isoformat the rest of the pipeline expects. + Graph emits up to seven fractional-second digits, which fromisoformat + rejects (it accepts three or six), so they are trimmed before parsing. + """ + text = (value or "").strip().replace("Z", "+00:00") + text = re.sub(r"(\.\d{6})\d+", r"\1", text) + + try: + parsed = datetime.fromisoformat(text) + except ValueError: + return datetime.now(timezone.utc).isoformat() + + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + + return parsed.isoformat() + + +def normalize_message(message: dict, source: Source) -> Optional[dict]: + """Turn one Graph serviceAnnouncement message into an article.""" + message_id = (message.get("id") or "").strip() + title = clean_html(message.get("title") or "Untitled") + + if not message_id: + return None + + body = (message.get("body") or {}).get("content") or "" + summary_raw = clean_html(body) + services = [service for service in (message.get("services") or []) if service] + + published = parse_graph_datetime( + message.get("lastModifiedDateTime") or message.get("startDateTime") or "" + ) + + # The services a post applies to are the strongest classification signal it + # carries, so they are fed to the classifier alongside the body text. + products = classify_products( + title, f"{summary_raw} {' '.join(services)}", source.name + ) + + return { + "title": title, + "link": f"{MESSAGE_CENTER_LINK}{message_id}", + "published": published, + "summary": truncate(summary_raw), + "author": source.default_author, + "source": source.name, + "source_id": source.id, + "source_group": source.source_group, + "source_kind": source.source_kind, + "vendor": source.vendor, + "source_category": source.category, + "board_id": source.board_id, + "message_id": message_id, + "services": services, + "products": products, + "tags": [product["name"] for product in products], + } + + +def fetch_message_center(source: Source) -> List[dict]: + """ + Message Center, over Graph. Unlike every other source this one is + authenticated and tenant scoped, so it is also the only one that can fail + for reasons that have nothing to do with the network. + """ + print(f"Fetching: {source.name}") + + token = graph_access_token() + if not token: + print(" No Graph token available, skipping the Message Center.") + print(" Locally: az login. In Actions: set the MESSAGE_CENTER_* variables.") + return [] + + since = ( + datetime.now(timezone.utc) - timedelta(days=MAX_ARTICLE_AGE_DAYS) + ).strftime("%Y-%m-%dT%H:%M:%SZ") + + query = urllib.parse.urlencode( + { + "$filter": f"lastModifiedDateTime ge {since}", + "$top": str(GRAPH_PAGE_SIZE), + "$orderby": "lastModifiedDateTime desc", + } + ) + + try: + payload = http_json( + f"{source.url}?{query}", + headers={"Authorization": f"Bearer {token}"}, + ) + except urllib.error.HTTPError as ex: + detail = ( + "forbidden: has ServiceMessage.Read.All been consented?" + if ex.code == 403 + else ex.reason + ) + print(f" Error fetching {source.name}: HTTP {ex.code} ({detail})") + return [] + except (urllib.error.URLError, ValueError) as ex: + print(f" Error fetching {source.name}: {ex}") + return [] + + messages = payload.get("value") or [] + articles = [] + filtered_out = 0 + + for message in messages: + if not passes_category_filter(source, message.get("services") or []): + filtered_out += 1 + continue + + article = normalize_message(message, source) + if article: + articles.append(article) + + if len(articles) >= source.max_entries: + break + + print(f" Found {len(articles)} articles") + print(f" Message Center returned {len(messages)} raw messages") + if filtered_out: + print(f" Filtered out {filtered_out} messages by service") + + return articles + + def deduplicate_articles(articles: List[dict]) -> Tuple[List[dict], dict]: cutoff = datetime.now(timezone.utc) - timedelta(days=MAX_ARTICLE_AGE_DAYS) @@ -662,7 +1074,10 @@ def main(): articles = [] for source in SOURCES: - articles.extend(fetch_feed(source)) + if source.source_kind == "graph": + articles.extend(fetch_message_center(source)) + else: + articles.extend(fetch_feed(source)) articles.sort(key=lambda x: x["published"], reverse=True) From 37c7ed680acebc99bb3fb76eee0beb305757e2fb Mon Sep 17 00:00:00 2001 From: Craig Thacker Date: Sat, 8 Aug 2026 17:22:25 +0100 Subject: [PATCH 5/5] docs: state the consent default as a policy choice, not a technical limit Pre-flight against the tenant showed the org CI identity already holds AppRoleAssignment.ReadWrite.All, alongside RoleManagement.ReadWrite.Directory and Directory.ReadWrite.All. The earlier wording justified the default on the CI identity not carrying that permission, which is simply not true, so it overstated the case. The default stays false because consent should be a deliberate act, not because the pipeline is incapable of it. --- terraform/README.md | 12 ++++++++---- terraform/variables.tf | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/terraform/README.md b/terraform/README.md index b234502..51d625e 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -81,10 +81,14 @@ az ad signed-in-user show --query id -o tsv ### Consent is not in the pipeline, deliberately `grant_admin_consent` defaults to **false**, so the stack requests `ServiceMessage.Read.All` but -does not consent to it. Granting a Graph application role requires `AppRoleAssignment.ReadWrite.All`, -and that permission lets its holder assign **any** app role of **any** API to **any** principal, -including granting itself directory write. Handing that to the shared org CI identity, which every -repository in the org can use, would turn a workflow-file change into a tenant escalation path. +does not consent to it. Granting a Graph application role is tenant-wide admin consent, and the +preference here is that consent stays a deliberate human act rather than something that happens as +a side effect of a merge. + +This is a policy choice rather than a technical limit. The org CI identity already holds +`AppRoleAssignment.ReadWrite.All`, alongside `RoleManagement.ReadWrite.Directory` and +`Directory.ReadWrite.All`, so flipping this to `true` would work and would grant the pipeline +nothing it cannot already do. So consent is a one-off human act. After the first apply, run what `terraform output grant_admin_consent_commands` prints, as someone who holds the permission: diff --git a/terraform/variables.tf b/terraform/variables.tf index a5cc23c..fc518f5 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -41,7 +41,7 @@ variable "github_repo" { } variable "grant_admin_consent" { - description = "Grant the requested Graph application roles from this stack. Defaults to false because granting one IS tenant-wide admin consent, which needs AppRoleAssignment.ReadWrite.All: a permission that lets its holder assign ANY app role of ANY API to ANY principal, including granting itself directory write. That is not something the shared org CI identity should carry, so the pipeline requests the roles and a human consents once using the grant_admin_consent_commands output. Set true only when applying interactively as someone who already holds it." + description = "Grant the requested Graph application roles from this stack. Defaults to false: granting one IS tenant-wide admin consent, and the preference here is that consent stays a deliberate human act rather than something a pipeline does on a merge. The pipeline therefore requests the roles and a human runs the grant_admin_consent_commands output once. Note this is a policy choice, not a technical limit: the org CI identity already holds AppRoleAssignment.ReadWrite.All, so setting this true would work. Doing so grants it nothing it does not already have." type = bool default = false }