From 0d44757e3a0442d02c85e15c2a97b391b9b7d37c Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Wed, 7 Jun 2023 11:49:00 +0300 Subject: [PATCH 01/49] Define the dependencies more exactly to get the integration tests running. --- .github/workflows/frontend_charm_integration_tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index 80670c320..f63681f6d 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -1,7 +1,7 @@ name: Run charm tests on: push: - branches: ["main"] + branches: ["main", "integration-test-fix"] tags: ["v*.*.*"] jobs: @@ -18,6 +18,9 @@ jobs: - name: Setup operator environment uses: charmed-kubernetes/actions-operator@main with: + juju-channel: 3.1/stable + charmcraft-channel: 2.x/stable + lxd-channel: 5.13/stable provider: microk8s - name: Run integration tests run: tox -e integration From 97e8d4c34906b6ba4e36fe7e2655880551d01913 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Wed, 7 Jun 2023 12:05:44 +0300 Subject: [PATCH 02/49] juju 3.1/stable -> 3.0/stable --- .github/workflows/frontend_charm_integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index f63681f6d..a09b06379 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -18,7 +18,7 @@ jobs: - name: Setup operator environment uses: charmed-kubernetes/actions-operator@main with: - juju-channel: 3.1/stable + juju-channel: 3.0/stable charmcraft-channel: 2.x/stable lxd-channel: 5.13/stable provider: microk8s From a05570116aedd84e4cbe7083d139b9654ceacb32 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Wed, 7 Jun 2023 12:26:59 +0300 Subject: [PATCH 03/49] Install strictly confined versions of juju and microk8s. --- .github/workflows/frontend_charm_integration_tests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index a09b06379..f201e1286 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -16,9 +16,11 @@ jobs: with: channel: 5.13/stable - name: Setup operator environment - uses: charmed-kubernetes/actions-operator@main + uses: mz2/actions-operator@4fcb55b99e2a0f69a5910870d3639d203aeb87aa with: - juju-channel: 3.0/stable + juju-channel: 3.1/stable + juju-classic-confinement: "false" + microk8s-classic-confinement: "false" charmcraft-channel: 2.x/stable lxd-channel: 5.13/stable provider: microk8s From 672ea62b07cd6ed73e35c2c6b6113816a4d25440 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Wed, 7 Jun 2023 12:32:24 +0300 Subject: [PATCH 04/49] Use microk8s 1.27-strict/stable --- .github/workflows/frontend_charm_integration_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index f201e1286..3326afc1b 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -23,6 +23,7 @@ jobs: microk8s-classic-confinement: "false" charmcraft-channel: 2.x/stable lxd-channel: 5.13/stable + channel: 1.27-strict/stable provider: microk8s - name: Run integration tests run: tox -e integration From 6c11eb5307c375c10eb0ad8e695694f1653338a3 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Wed, 7 Jun 2023 13:04:29 +0300 Subject: [PATCH 05/49] Adds package repo access to the integration test job --- .github/workflows/frontend_charm_integration_tests.yml | 4 ++++ frontend/charm/metadata.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index 3326afc1b..d362107b3 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -6,6 +6,10 @@ on: jobs: integration-test: + permissions: + contents: read + packages: read + name: Integration tests runs-on: ubuntu-22.04 steps: diff --git a/frontend/charm/metadata.yaml b/frontend/charm/metadata.yaml index 767c3d4c3..3a4857d29 100644 --- a/frontend/charm/metadata.yaml +++ b/frontend/charm/metadata.yaml @@ -21,4 +21,4 @@ resources: frontend-image: type: oci-image description: OCI image for test-observer-frontend - upstream-source: ghcr.io/canonical/test_observer/frontend:v0.0.10 + upstream-source: ghcr.io/canonical/test_observer/frontend:v0.0.10 \ No newline at end of file From 5c149e27f46698bc6190dfbe35e0406663329731 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Wed, 7 Jun 2023 14:22:47 +0300 Subject: [PATCH 06/49] Add the containerd template modification --- .../workflows/frontend_charm_integration_tests.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index d362107b3..999cc010c 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -29,6 +29,18 @@ jobs: lxd-channel: 5.13/stable channel: 1.27-strict/stable provider: microk8s + - name: Adjust containerd template to accept ghcr.io images + run: | + sudo mkdir -p /var/snap/microk8s/current/args + echo "" >> /var/snap/microk8s/current/args/containerd-template.toml + echo "[plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"ghcr.io\".auth]" >> /var/snap/microk8s/current/args/containerd-template.toml + echo " username = \"${GHCR_PACKAGE_API_USER}\" >> /var/snap/microk8s/current/args/containerd-template.toml + echo " password = \"${GHCR_PACKAGE_API_TOKEN}\" >> /var/snap/microk8s/current/args/containerd-template.toml + echo $GHCR_PACKAGE_API_TOKEN | docker login ghcr.io -u $GHCR_PACKAGE_API_USER --password-stdin + sudo systemctl restart snap.microk8s.daemon-containerd.service && sudo microk8s.stop && sudo microk8s.start + env: + GHCR_PACKAGE_API_USER: ${{ secrets.GHCR_PACKAGE_API_USER }} + GHCR_PACKAGE_API_TOKEN: ${{ secrets.GHCR_PACKAGE_API_TOKEN }} - name: Run integration tests run: tox -e integration - name: Archive Tested Charm From 11cf30b5454447a54e4f27fca43ff562b17f028a Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Wed, 7 Jun 2023 14:35:12 +0300 Subject: [PATCH 07/49] Adds some missing sudoing --- .github/workflows/frontend_charm_integration_tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index 999cc010c..807a329e9 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -32,11 +32,11 @@ jobs: - name: Adjust containerd template to accept ghcr.io images run: | sudo mkdir -p /var/snap/microk8s/current/args - echo "" >> /var/snap/microk8s/current/args/containerd-template.toml - echo "[plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"ghcr.io\".auth]" >> /var/snap/microk8s/current/args/containerd-template.toml - echo " username = \"${GHCR_PACKAGE_API_USER}\" >> /var/snap/microk8s/current/args/containerd-template.toml - echo " password = \"${GHCR_PACKAGE_API_TOKEN}\" >> /var/snap/microk8s/current/args/containerd-template.toml - echo $GHCR_PACKAGE_API_TOKEN | docker login ghcr.io -u $GHCR_PACKAGE_API_USER --password-stdin + sudo echo "" >> /var/snap/microk8s/current/args/containerd-template.toml + sudo echo "[plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"ghcr.io\".auth]" >> /var/snap/microk8s/current/args/containerd-template.toml + sudo echo " username = \"${GHCR_PACKAGE_API_USER}\" >> /var/snap/microk8s/current/args/containerd-template.toml + sudo echo " password = \"${GHCR_PACKAGE_API_TOKEN}\" >> /var/snap/microk8s/current/args/containerd-template.toml + sudo echo $GHCR_PACKAGE_API_TOKEN | docker login ghcr.io -u $GHCR_PACKAGE_API_USER --password-stdin sudo systemctl restart snap.microk8s.daemon-containerd.service && sudo microk8s.stop && sudo microk8s.start env: GHCR_PACKAGE_API_USER: ${{ secrets.GHCR_PACKAGE_API_USER }} From 6e8375a02cb94a02868fde88fa2443604c8344ce Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Wed, 7 Jun 2023 14:43:13 +0300 Subject: [PATCH 08/49] From >> to tee -a --- .github/workflows/frontend_charm_integration_tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index 807a329e9..c425fa982 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -32,11 +32,11 @@ jobs: - name: Adjust containerd template to accept ghcr.io images run: | sudo mkdir -p /var/snap/microk8s/current/args - sudo echo "" >> /var/snap/microk8s/current/args/containerd-template.toml - sudo echo "[plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"ghcr.io\".auth]" >> /var/snap/microk8s/current/args/containerd-template.toml - sudo echo " username = \"${GHCR_PACKAGE_API_USER}\" >> /var/snap/microk8s/current/args/containerd-template.toml - sudo echo " password = \"${GHCR_PACKAGE_API_TOKEN}\" >> /var/snap/microk8s/current/args/containerd-template.toml - sudo echo $GHCR_PACKAGE_API_TOKEN | docker login ghcr.io -u $GHCR_PACKAGE_API_USER --password-stdin + echo "" | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml + echo "[plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"ghcr.io\".auth]" | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml + echo " username = \"${GHCR_PACKAGE_API_USER}\" | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml + echo " password = \"${GHCR_PACKAGE_API_TOKEN}\" | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml + echo $GHCR_PACKAGE_API_TOKEN | docker login ghcr.io -u $GHCR_PACKAGE_API_USER --password-stdin sudo systemctl restart snap.microk8s.daemon-containerd.service && sudo microk8s.stop && sudo microk8s.start env: GHCR_PACKAGE_API_USER: ${{ secrets.GHCR_PACKAGE_API_USER }} From 97f1531ea6b9262b6f10ec34bd3189701dfa6f11 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Wed, 7 Jun 2023 14:51:39 +0300 Subject: [PATCH 09/49] Drop the docker login --- .github/workflows/frontend_charm_integration_tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index c425fa982..d3fdee856 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -36,7 +36,6 @@ jobs: echo "[plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"ghcr.io\".auth]" | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml echo " username = \"${GHCR_PACKAGE_API_USER}\" | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml echo " password = \"${GHCR_PACKAGE_API_TOKEN}\" | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml - echo $GHCR_PACKAGE_API_TOKEN | docker login ghcr.io -u $GHCR_PACKAGE_API_USER --password-stdin sudo systemctl restart snap.microk8s.daemon-containerd.service && sudo microk8s.stop && sudo microk8s.start env: GHCR_PACKAGE_API_USER: ${{ secrets.GHCR_PACKAGE_API_USER }} From 1478f455e8a2fcb3573605574c072dba373cd907 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Wed, 7 Jun 2023 16:34:18 +0300 Subject: [PATCH 10/49] Oops, escaping quotes did not work quite right there. --- .github/workflows/frontend_charm_integration_tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index d3fdee856..158b23813 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -31,11 +31,11 @@ jobs: provider: microk8s - name: Adjust containerd template to accept ghcr.io images run: | - sudo mkdir -p /var/snap/microk8s/current/args - echo "" | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml - echo "[plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"ghcr.io\".auth]" | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml - echo " username = \"${GHCR_PACKAGE_API_USER}\" | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml - echo " password = \"${GHCR_PACKAGE_API_TOKEN}\" | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml + sudo mkdir -p /var/snap/microk8s/current/args && + echo "" | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml && + echo '[plugins."io.containerd.grpc.v1.cri".registry.configs."ghcr.io".auth]' | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml && + echo ' username = "$GHCR_PACKAGE_API_USER"' | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml && + echo ' password = "$GHCR_PACKAGE_API_TOKEN"' | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml && sudo systemctl restart snap.microk8s.daemon-containerd.service && sudo microk8s.stop && sudo microk8s.start env: GHCR_PACKAGE_API_USER: ${{ secrets.GHCR_PACKAGE_API_USER }} From b03e01552f7bb7525618fd74d3d6ee8f313059da Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Wed, 7 Jun 2023 18:03:41 +0300 Subject: [PATCH 11/49] Adjust the integration test scenario, expecting rather than raising on blocked status (more scenarios needed to actually relate the frontend to the backend). --- .../workflows/frontend_charm_integration_tests.yml | 4 ++-- frontend/charm/src/charm.py | 14 ++++++++++++++ frontend/charm/tests/integration/test_charm.py | 11 ++++++----- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index 158b23813..ecf6d7311 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -22,13 +22,13 @@ jobs: - name: Setup operator environment uses: mz2/actions-operator@4fcb55b99e2a0f69a5910870d3639d203aeb87aa with: + channel: 1.27-strict/stable + provider: microk8s juju-channel: 3.1/stable juju-classic-confinement: "false" microk8s-classic-confinement: "false" charmcraft-channel: 2.x/stable lxd-channel: 5.13/stable - channel: 1.27-strict/stable - provider: microk8s - name: Adjust containerd template to accept ghcr.io images run: | sudo mkdir -p /var/snap/microk8s/current/args && diff --git a/frontend/charm/src/charm.py b/frontend/charm/src/charm.py index f85192eaf..a08424314 100755 --- a/frontend/charm/src/charm.py +++ b/frontend/charm/src/charm.py @@ -143,6 +143,20 @@ def _update_layer_and_restart(self, event): hostname = self._stored.backend_hostname port = self._stored.backend_port + # Blocked status is set without a pebble re-configuration or container restart here. + # Frontend charm _could_ instead make nginx present an error page when backend has not been configured. + if hostname is None: + self.unit.status = BlockedStatus( + "test-observer-rest-api relation not connected, or hostname not populated based on relation data." + ) + return + + if hostname is None: + self.unit.status = BlockedStatus( + "test-observer-rest-api relation not connected, or port not populated based on relation data." + ) + return + if int(port) == 80 or int(port) == 443: base_uri = f"{scheme}{hostname}" else: diff --git a/frontend/charm/tests/integration/test_charm.py b/frontend/charm/tests/integration/test_charm.py index 10742f61b..12a3f4905 100644 --- a/frontend/charm/tests/integration/test_charm.py +++ b/frontend/charm/tests/integration/test_charm.py @@ -17,20 +17,21 @@ @pytest.mark.abort_on_fail -async def test_build_and_deploy(ops_test: OpsTest): - """Build the charm-under-test and deploy it together with related charms. +async def test_build_and_deploy_without_backend(ops_test: OpsTest): + """Build the charm-under-test and deploy it together without related charms. Assert on the unit status before any relations/configurations take place. """ charm = await ops_test.build_charm(".") - resources = {"frontend-image": METADATA["resources"]["frontend-image"]["upstream-source"]} + resources = { + "frontend-image": METADATA["resources"]["frontend-image"]["upstream-source"] + } await asyncio.gather( ops_test.model.deploy(charm, resources=resources, application_name=APP_NAME), ops_test.model.wait_for_idle( apps=[APP_NAME], - status="active", - raise_on_blocked=True, + status="blocked", timeout=1000, ), ) From 55b943634219331a16cbd6f780e97e0f7b42cb01 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Wed, 7 Jun 2023 18:06:17 +0300 Subject: [PATCH 12/49] Drops the debugging intended branch from the list of branches to trigger the integration tests with. --- .github/workflows/frontend_charm_integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index ecf6d7311..6d442372c 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -1,7 +1,7 @@ name: Run charm tests on: push: - branches: ["main", "integration-test-fix"] + branches: ["main"] tags: ["v*.*.*"] jobs: From cde6b80bb3a39c0737a06adbf5f765c560d149b8 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Wed, 7 Jun 2023 18:28:24 +0300 Subject: [PATCH 13/49] Change to maintenance status instead for clarity, since configuration action is required. --- .github/workflows/frontend_charm_integration_tests.yml | 2 +- frontend/charm/src/charm.py | 4 ++-- frontend/charm/tests/integration/test_charm.py | 8 +++----- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index 6d442372c..ecf6d7311 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -1,7 +1,7 @@ name: Run charm tests on: push: - branches: ["main"] + branches: ["main", "integration-test-fix"] tags: ["v*.*.*"] jobs: diff --git a/frontend/charm/src/charm.py b/frontend/charm/src/charm.py index a08424314..af8e5d5ae 100755 --- a/frontend/charm/src/charm.py +++ b/frontend/charm/src/charm.py @@ -146,13 +146,13 @@ def _update_layer_and_restart(self, event): # Blocked status is set without a pebble re-configuration or container restart here. # Frontend charm _could_ instead make nginx present an error page when backend has not been configured. if hostname is None: - self.unit.status = BlockedStatus( + self.unit.status = MaintenanceStatus( "test-observer-rest-api relation not connected, or hostname not populated based on relation data." ) return if hostname is None: - self.unit.status = BlockedStatus( + self.unit.status = MaintenanceStatus( "test-observer-rest-api relation not connected, or port not populated based on relation data." ) return diff --git a/frontend/charm/tests/integration/test_charm.py b/frontend/charm/tests/integration/test_charm.py index 12a3f4905..77cc33f3a 100644 --- a/frontend/charm/tests/integration/test_charm.py +++ b/frontend/charm/tests/integration/test_charm.py @@ -23,15 +23,13 @@ async def test_build_and_deploy_without_backend(ops_test: OpsTest): Assert on the unit status before any relations/configurations take place. """ charm = await ops_test.build_charm(".") - resources = { - "frontend-image": METADATA["resources"]["frontend-image"]["upstream-source"] - } + resources = {"frontend-image": METADATA["resources"]["frontend-image"]["upstream-source"]} await asyncio.gather( ops_test.model.deploy(charm, resources=resources, application_name=APP_NAME), ops_test.model.wait_for_idle( apps=[APP_NAME], - status="blocked", - timeout=1000, + status="maintenance", + timeout=2000, ), ) From 1ed9d5f034cb84c1a8fbe481f3bbf3ec6f085d34 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Wed, 7 Jun 2023 18:34:24 +0300 Subject: [PATCH 14/49] Adds the "integration-test-fix" branch into the rule still --- .github/workflows/frontend_charm_analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/frontend_charm_analysis.yml b/.github/workflows/frontend_charm_analysis.yml index 751c14bef..84fe004dc 100644 --- a/.github/workflows/frontend_charm_analysis.yml +++ b/.github/workflows/frontend_charm_analysis.yml @@ -4,7 +4,7 @@ on: branches: - main push: - branches: ["main"] + branches: ["main", "integration-test-fix"] tags: ["v*.*.*"] jobs: From 0a30cee1668e1b284b2d9ea43b81dec8f3e1319f Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Wed, 7 Jun 2023 22:58:53 +0300 Subject: [PATCH 15/49] Adds a 503 response / maintenance status to the frontend application state where relation is not yet connected. Simplified the charm logic, no _stored state needed (fetching the hostname and port from the relation's application data bag). --- frontend/charm/src/charm.py | 109 ++++++++++++++++++++++++------------ 1 file changed, 72 insertions(+), 37 deletions(-) diff --git a/frontend/charm/src/charm.py b/frontend/charm/src/charm.py index af8e5d5ae..ec262cf45 100755 --- a/frontend/charm/src/charm.py +++ b/frontend/charm/src/charm.py @@ -30,8 +30,6 @@ class TestObserverFrontendCharm(ops.CharmBase): """The frontend charm operates serving the frontend through nginx.""" - _stored = StoredState() - def __init__(self, *args): super().__init__(*args) self.pebble_service_name = "test-observer-frontend" @@ -56,8 +54,6 @@ def __init__(self, *args): self.framework.observe(self.ingress.on.ready, self._on_ingress_ready) self.framework.observe(self.ingress.on.revoked, self._on_ingress_revoked) - self._stored.set_default(backend_hostname=None, backend_port=None) - def _on_frontend_pebble_ready(self, event: ops.PebbleReadyEvent): container = event.workload container.add_layer("frontend", self._pebble_layer, combine=True) @@ -98,20 +94,12 @@ def _config_is_valid(self, config) -> Tuple[bool, str]: def _on_rest_api_relation_update(self, event): api_hostname = event.relation.data[event.app].get("hostname") api_port = event.relation.data[event.app].get("port") - logger.debug(f"API hostname: {api_hostname} (app: {event.app})") - - if self.unit.is_leader(): - self._stored.backend_hostname = api_hostname - self._stored.backend_port = api_port - + logger.debug(f"API hostname: {api_hostname}, port: {api_port} (app: {event.app})") self._update_layer_and_restart(event) def _on_rest_api_relation_broken(self, event): - logger.debug("REST API relation broken -> removing backend hostname") - - if self.unit.is_leader(): - self._stored.backend_hostname = None - self._stored.backend_port = None + logger.debug("REST API relation broken") + self._update_layer_and_restart(event) def nginx_config(self, base_uri: str) -> str: """Return a config where the backend port `base_uri` is adjusted.""" @@ -136,44 +124,91 @@ def nginx_config(self, base_uri: str) -> str: }} """ + def nginx_503_config(self) -> str: + """Return a config for the situation when the backend is not yet available.""" + return """ + server { + listen 80 default_server; + server_name _; + return 503; + error_page 503 @maintenance; + + location @maintenance { + rewrite ^(.*)$ /503.html break; + root /usr/share/nginx/html; + } + } + """ + + def html_503(self) -> str: + """Return a 503 response page.""" + return """ + + + 503 Service Unavailable + + +

503 Service Unavailable

+

Backend not yet configured.

+ + + """ + def _update_layer_and_restart(self, event): self.unit.status = MaintenanceStatus(f"Updating {self.pebble_service_name} layer") - scheme = self.config["test-observer-api-scheme"] - hostname = self._stored.backend_hostname - port = self._stored.backend_port - - # Blocked status is set without a pebble re-configuration or container restart here. - # Frontend charm _could_ instead make nginx present an error page when backend has not been configured. - if hostname is None: - self.unit.status = MaintenanceStatus( - "test-observer-rest-api relation not connected, or hostname not populated based on relation data." - ) - return + api_relation = self.model.relations["test-observer-rest-api"][0] + + if api_relation is None: + if self.container.can_connect(): + self.container.add_layer( + self.pebble_service_name, self._pebble_layer, combine=True + ) + self.container.push( + "/etc/nginx/sites-available/test-observer-frontend", + self.nginx_503_config(), + make_dirs=True, + ) + self.container.push( + "/usr/share/nginx/html/503.html", + self.html_503(), + make_dirs=True, + ) + self.container.restart(self.pebble_service_name) + self.unit.status = MaintenanceStatus( + "test-observer-rest-api relation not connected." + ) + else: + self.unit.status = WaitingStatus( + "Waiting for Pebble for API to set maintenance state" + ) - if hostname is None: - self.unit.status = MaintenanceStatus( - "test-observer-rest-api relation not connected, or port not populated based on relation data." - ) return + hostname = api_relation.data[api_relation.app]["hostname"] + port = api_relation.data[api_relation.app]["port"] + + scheme = self.config["test-observer-api-scheme"] + + logger.info(f"Hostname: {hostname} (from relation: {hostname})") + logger.info(f"Port: {port} (from relation: {port})") + if int(port) == 80 or int(port) == 443: base_uri = f"{scheme}{hostname}" else: base_uri = f"{scheme}{hostname}:{port}" - self.container.push( - "/etc/nginx/sites-available/test-observer-frontend", - self.nginx_config(base_uri=base_uri), - make_dirs=True, - ) - if self.container.can_connect(): self.container.add_layer(self.pebble_service_name, self._pebble_layer, combine=True) + self.container.push( + "/etc/nginx/sites-available/test-observer-frontend", + self.nginx_config(base_uri=base_uri), + make_dirs=True, + ) self.container.restart(self.pebble_service_name) self.unit.status = ActiveStatus() else: - self.unit.status = WaitingStatus("Waiting for Pebble for API") + self.unit.status = WaitingStatus("Waiting for Pebble for API to set available state") @property def _pebble_layer(self): From bf2859c503eea435c75ec0e14f4c2dd721d93c3d Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Wed, 7 Jun 2023 23:09:10 +0300 Subject: [PATCH 16/49] Removs the redundant import StoredState --- frontend/charm/src/charm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/charm/src/charm.py b/frontend/charm/src/charm.py index ec262cf45..f9f997677 100755 --- a/frontend/charm/src/charm.py +++ b/frontend/charm/src/charm.py @@ -15,7 +15,6 @@ IngressPerAppRequirer, IngressPerAppRevokedEvent, ) -from ops.framework import StoredState from ops.model import ( ActiveStatus, BlockedStatus, From 2fc065fc47fd8af789220fac19cc4e01ac174333 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Wed, 7 Jun 2023 23:24:43 +0300 Subject: [PATCH 17/49] Add a `microk8s status --wait-ready` --- .github/workflows/frontend_charm_integration_tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index ecf6d7311..8d5a86baa 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -36,7 +36,10 @@ jobs: echo '[plugins."io.containerd.grpc.v1.cri".registry.configs."ghcr.io".auth]' | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml && echo ' username = "$GHCR_PACKAGE_API_USER"' | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml && echo ' password = "$GHCR_PACKAGE_API_TOKEN"' | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml && - sudo systemctl restart snap.microk8s.daemon-containerd.service && sudo microk8s.stop && sudo microk8s.start + sudo systemctl restart snap.microk8s.daemon-containerd.service && + sudo microk8s.stop && + sudo microk8s.start && + microk8s status --wait-ready env: GHCR_PACKAGE_API_USER: ${{ secrets.GHCR_PACKAGE_API_USER }} GHCR_PACKAGE_API_TOKEN: ${{ secrets.GHCR_PACKAGE_API_TOKEN }} From 06a258ecacc3b30612590ca19169161f66cb1994 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Wed, 7 Jun 2023 23:33:27 +0300 Subject: [PATCH 18/49] Adds missing sudo --- .github/workflows/frontend_charm_integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index 8d5a86baa..ff7b9d41f 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -39,7 +39,7 @@ jobs: sudo systemctl restart snap.microk8s.daemon-containerd.service && sudo microk8s.stop && sudo microk8s.start && - microk8s status --wait-ready + sudo microk8s status --wait-ready env: GHCR_PACKAGE_API_USER: ${{ secrets.GHCR_PACKAGE_API_USER }} GHCR_PACKAGE_API_TOKEN: ${{ secrets.GHCR_PACKAGE_API_TOKEN }} From a3a54049212ee2abbfa4d7456c73ea7c2a513006 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Sun, 2 Jul 2023 12:55:37 +0300 Subject: [PATCH 19/49] Let's try with the microk8s version + libpyjuju < 3.1 Let's try with the microk8s version and libpyjuju < 3.1 that seems to work for rout53-acme-operator where the integration test pipeline is pulled from --- .../frontend_charm_integration_tests.yml | 21 +++++++++++-------- frontend/charm/tox.ini | 2 +- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index ff7b9d41f..2fe787f85 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -18,15 +18,15 @@ jobs: - name: Setup LXD uses: canonical/setup-lxd@main with: - channel: 5.13/stable + channel: 5.12/stable - name: Setup operator environment - uses: mz2/actions-operator@4fcb55b99e2a0f69a5910870d3639d203aeb87aa + uses: charmed-kubernetes/actions-operator@main # mz2/actions-operator@4fcb55b99e2a0f69a5910870d3639d203aeb87aa with: - channel: 1.27-strict/stable + channel: 1.27-strict/stable # 1.26.4 provider: microk8s - juju-channel: 3.1/stable - juju-classic-confinement: "false" - microk8s-classic-confinement: "false" + juju-channel: 2.9/stable + # juju-classic-confinement: "false" + # microk8s-classic-confinement: "false" charmcraft-channel: 2.x/stable lxd-channel: 5.13/stable - name: Adjust containerd template to accept ghcr.io images @@ -37,9 +37,12 @@ jobs: echo ' username = "$GHCR_PACKAGE_API_USER"' | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml && echo ' password = "$GHCR_PACKAGE_API_TOKEN"' | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml && sudo systemctl restart snap.microk8s.daemon-containerd.service && - sudo microk8s.stop && - sudo microk8s.start && - sudo microk8s status --wait-ready + sudo microk8s.stop && + sleep 10 && + sudo microk8s.start && + sleep 10 && + sudo microk8s status --wait-ready && + sleep 30 env: GHCR_PACKAGE_API_USER: ${{ secrets.GHCR_PACKAGE_API_USER }} GHCR_PACKAGE_API_TOKEN: ${{ secrets.GHCR_PACKAGE_API_TOKEN }} diff --git a/frontend/charm/tox.ini b/frontend/charm/tox.ini index fb9bfa4f4..d205153ad 100644 --- a/frontend/charm/tox.ini +++ b/frontend/charm/tox.ini @@ -66,7 +66,7 @@ commands = description = Run integration tests deps = pytest - juju + juju<3.1 pytest-operator -r {tox_root}/requirements.txt commands = From 234b1aa3030a208368468435ad551bd9e1b38582 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Sun, 2 Jul 2023 13:11:11 +0300 Subject: [PATCH 20/49] Back to 5.13/stable for lxd --- .github/workflows/frontend_charm_integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index 2fe787f85..12a1bff29 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -18,7 +18,7 @@ jobs: - name: Setup LXD uses: canonical/setup-lxd@main with: - channel: 5.12/stable + channel: 5.13/stable - name: Setup operator environment uses: charmed-kubernetes/actions-operator@main # mz2/actions-operator@4fcb55b99e2a0f69a5910870d3639d203aeb87aa with: From fd3ed3534e31af9a256a26d8aa7352d45c3f5da8 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Sun, 2 Jul 2023 13:22:07 +0300 Subject: [PATCH 21/49] 2.9/stable -> 3.0/stable --- .github/workflows/frontend_charm_integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index 12a1bff29..95d6a8be3 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -24,7 +24,7 @@ jobs: with: channel: 1.27-strict/stable # 1.26.4 provider: microk8s - juju-channel: 2.9/stable + juju-channel: 3.0/stable # juju-classic-confinement: "false" # microk8s-classic-confinement: "false" charmcraft-channel: 2.x/stable From 63f4d6f2990ee3eb8ab9a6fc0121013a7d51b3fe Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Sun, 2 Jul 2023 14:02:37 +0300 Subject: [PATCH 22/49] Trying with the non-strictly confined microk8s --- .github/workflows/frontend_charm_integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index 95d6a8be3..aa1f5a110 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -22,7 +22,7 @@ jobs: - name: Setup operator environment uses: charmed-kubernetes/actions-operator@main # mz2/actions-operator@4fcb55b99e2a0f69a5910870d3639d203aeb87aa with: - channel: 1.27-strict/stable # 1.26.4 + channel: 1.27/stable # 1.26.4 provider: microk8s juju-channel: 3.0/stable # juju-classic-confinement: "false" From acf47ce5fa59ede1d6b391621999c6eacdf43ba3 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Sun, 2 Jul 2023 14:02:47 +0300 Subject: [PATCH 23/49] Adds "publish-charm" job --- .../frontend_charm_integration_tests.yml | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index aa1f5a110..52802c5a3 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -70,3 +70,29 @@ jobs: defaults: run: working-directory: ./frontend/charm + + publish-charm: + name: Publish Charm + needs: integration-test + runs-on: ubuntu-22.04 + if: ${{ github.ref_name == 'main' }} + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Install charmcraft + run: sudo snap install charmcraft --classic + - name: Fetch Tested Charm + uses: actions/download-artifact@v3 + with: + name: tested-charm + - name: Move charm in current directory + run: find ./ -name test-observer-frontend_ubuntu-22.04-amd64.charm -exec mv -t ./ {} \; + - name: Select Charmhub channel + uses: canonical/charming-actions/channel@2.2.2 + id: channel + - name: Upload charm to Charmhub + uses: canonical/charming-actions/upload-charm@2.2.2 + with: + credentials: "${{ secrets.CHARMHUB_AUTH_API }}" + github-token: "${{ secrets.GITHUB_TOKEN }}" + channel: "${{ steps.channel.outputs.name }}" From b03dddbabc24bddbb3c5890103473bc727203c8b Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Sun, 2 Jul 2023 14:03:28 +0300 Subject: [PATCH 24/49] Adds promotion action --- .github/workflows/frontend_charm_promote.yml | 43 ++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/frontend_charm_promote.yml diff --git a/.github/workflows/frontend_charm_promote.yml b/.github/workflows/frontend_charm_promote.yml new file mode 100644 index 000000000..617a6cd28 --- /dev/null +++ b/.github/workflows/frontend_charm_promote.yml @@ -0,0 +1,43 @@ +name: Promote Charm + +on: + workflow_dispatch: + inputs: + promotion: + type: choice + description: Channel to promote from + options: + - edge -> beta + - beta -> candidate + - candidate -> stable + +jobs: + promote: + name: Promote Charm + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set target channel + env: + PROMOTE_FROM: ${{ github.event.inputs.promotion }} + run: | + if [ "${PROMOTE_FROM}" == "edge -> beta" ]; then + echo "promote-from=edge" >> ${GITHUB_ENV} + echo "promote-to=beta" >> ${GITHUB_ENV} + elif [ "${PROMOTE_FROM}" == "beta -> candidate" ]; then + echo "promote-from=beta" >> ${GITHUB_ENV} + echo "promote-to=candidate" >> ${GITHUB_ENV} + elif [ "${PROMOTE_FROM}" == "candidate -> stable" ]; then + echo "promote-from=candidate" >> ${GITHUB_ENV} + echo "promote-to=stable" >> ${GITHUB_ENV} + fi + - name: Promote Charm + uses: canonical/charming-actions/release-charm@2.2.0 + with: + base-channel: 22.04 + credentials: ${{ secrets.CHARMHUB_TOKEN }} + github-token: ${{ secrets.GITHUB_TOKEN }} + destination-channel: latest/${{ env.promote-to }} + origin-channel: latest/${{ env.promote-from }} + charmcraft-channel: latest/stable From 73485591bb58af0b48811432adc223b130875624 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Thu, 26 Oct 2023 18:53:45 +0300 Subject: [PATCH 25/49] Swap to self-hosted runners --- .github/workflows/frontend_charm_analysis.yml | 6 +++--- .github/workflows/frontend_charm_integration_tests.yml | 4 ++-- .github/workflows/frontend_charm_promote.yml | 2 +- .github/workflows/public_backend_image.yml | 2 +- .github/workflows/publish_frontend_image.yml | 2 +- .github/workflows/release_charms.yml | 2 +- .github/workflows/test_backend.yml | 2 +- .github/workflows/test_frontend.yml | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/frontend_charm_analysis.yml b/.github/workflows/frontend_charm_analysis.yml index 84fe004dc..707d6b287 100644 --- a/.github/workflows/frontend_charm_analysis.yml +++ b/.github/workflows/frontend_charm_analysis.yml @@ -10,7 +10,7 @@ on: jobs: lint-report: name: Lint and format report - runs-on: ubuntu-22.04 + runs-on: [self-hosted, linux, large] steps: - uses: actions/checkout@v3 - name: Install tox @@ -23,7 +23,7 @@ jobs: static-analysis: name: Static analysis - runs-on: ubuntu-22.04 + runs-on: [self-hosted, linux] steps: - uses: actions/checkout@v3 - name: Install tox @@ -36,7 +36,7 @@ jobs: unit-tests-with-coverage: name: Unit tests - runs-on: ubuntu-22.04 + runs-on: [self-hosted, linux] steps: - uses: actions/checkout@v3 - name: Install tox diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index 52802c5a3..18ad95f6e 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -11,7 +11,7 @@ jobs: packages: read name: Integration tests - runs-on: ubuntu-22.04 + runs-on: [self-hosted, linux, large] steps: - name: Checkout uses: actions/checkout@v3 @@ -74,7 +74,7 @@ jobs: publish-charm: name: Publish Charm needs: integration-test - runs-on: ubuntu-22.04 + runs-on: [self-hosted, linux] if: ${{ github.ref_name == 'main' }} steps: - name: Checkout diff --git a/.github/workflows/frontend_charm_promote.yml b/.github/workflows/frontend_charm_promote.yml index 617a6cd28..a04866df9 100644 --- a/.github/workflows/frontend_charm_promote.yml +++ b/.github/workflows/frontend_charm_promote.yml @@ -14,7 +14,7 @@ on: jobs: promote: name: Promote Charm - runs-on: ubuntu-22.04 + runs-on: [self-hosted, linux] steps: - name: Checkout uses: actions/checkout@v3 diff --git a/.github/workflows/public_backend_image.yml b/.github/workflows/public_backend_image.yml index 0e74e6430..5f1653a9c 100644 --- a/.github/workflows/public_backend_image.yml +++ b/.github/workflows/public_backend_image.yml @@ -9,7 +9,7 @@ env: jobs: build-and-push-backend-image: - runs-on: ubuntu-latest + runs-on: [self-hosted, linux] permissions: contents: read packages: write diff --git a/.github/workflows/publish_frontend_image.yml b/.github/workflows/publish_frontend_image.yml index 3b901b821..982edb457 100644 --- a/.github/workflows/publish_frontend_image.yml +++ b/.github/workflows/publish_frontend_image.yml @@ -9,7 +9,7 @@ env: jobs: build-and-push-frontend-image: - runs-on: ubuntu-latest + runs-on: [self-hosted, linux] permissions: contents: read packages: write diff --git a/.github/workflows/release_charms.yml b/.github/workflows/release_charms.yml index b344a8106..148b8b753 100644 --- a/.github/workflows/release_charms.yml +++ b/.github/workflows/release_charms.yml @@ -9,7 +9,7 @@ on: jobs: build: name: Release test-observer-api charm to edge - runs-on: ubuntu-latest + runs-on: [self-hosted, linux] steps: - name: Checkout uses: actions/checkout@v2 diff --git a/.github/workflows/test_backend.yml b/.github/workflows/test_backend.yml index 223752b3b..a2718155a 100644 --- a/.github/workflows/test_backend.yml +++ b/.github/workflows/test_backend.yml @@ -6,7 +6,7 @@ concurrency: cancel-in-progress: true jobs: test: - runs-on: ubuntu-latest + runs-on: [self-hosted, linux, large] defaults: run: working-directory: backend diff --git a/.github/workflows/test_frontend.yml b/.github/workflows/test_frontend.yml index 5a99a692b..c2f76f534 100644 --- a/.github/workflows/test_frontend.yml +++ b/.github/workflows/test_frontend.yml @@ -6,7 +6,7 @@ concurrency: cancel-in-progress: true jobs: test: - runs-on: ubuntu-latest + runs-on: [self-hosted, linux, large] defaults: run: working-directory: frontend From 9521314ee5d489e1cf27ef35e24886228b5d5638 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Thu, 26 Oct 2023 19:22:10 +0300 Subject: [PATCH 26/49] LXD 5.13 => 5.19 --- .github/workflows/frontend_charm_integration_tests.yml | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index 18ad95f6e..a7b3350f6 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -18,7 +18,7 @@ jobs: - name: Setup LXD uses: canonical/setup-lxd@main with: - channel: 5.13/stable + channel: 5.19/stable - name: Setup operator environment uses: charmed-kubernetes/actions-operator@main # mz2/actions-operator@4fcb55b99e2a0f69a5910870d3639d203aeb87aa with: @@ -28,7 +28,7 @@ jobs: # juju-classic-confinement: "false" # microk8s-classic-confinement: "false" charmcraft-channel: 2.x/stable - lxd-channel: 5.13/stable + lxd-channel: 5.19/stable - name: Adjust containerd template to accept ghcr.io images run: | sudo mkdir -p /var/snap/microk8s/current/args && diff --git a/README.md b/README.md index aae884d89..67b9dc16e 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Observe the status and state of certification tests for various artefacts - `juju` 3.1 or later (`sudo snap install juju --channel=3.1/stable`) - `microk8s` 1.27 or later (`sudo snap install microk8s --channel=1.27-strict/stable`) + [permission setup steps after install](https://juju.is/docs/sdk/set-up-your-development-environment#heading--install-microk8s) - `terraform` 1.4.6 or later (`sudo snap install terraform --classic`) -- `lxd` 5.13 or later (`sudo snap install lxd --channel=5.13/stable` or `sudo snap refresh lxd --channel=5.13/stable` if already installed) + `lxd init --auto` after install. +- `lxd` 5.19 or later (`sudo snap install lxd --channel=5.19/stable` or `sudo snap refresh lxd --channel=5.19/stable` if already installed) + `lxd init --auto` after install. - `charmcraft` 2.3.0 or later (`sudo snap install charmcraft --channel=2.x/stable --classic`) - optional: `jhack` for all kinds of handy Juju and charm SDK development and debugging operations (`sudo snap install jhack`) From 6a53221e01eac98b19e12d737a4bfb610e20536d Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Thu, 26 Oct 2023 19:28:27 +0300 Subject: [PATCH 27/49] Run also the integration tests --- .github/workflows/frontend_charm_integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index a7b3350f6..cbb58d3e5 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -1,7 +1,7 @@ name: Run charm tests on: push: - branches: ["main", "integration-test-fix"] + branches: ["main", "integration-test-fix*"] tags: ["v*.*.*"] jobs: From bcd4cf5aa4f8ff680bb67779c3828824bfd4c5cb Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Thu, 26 Oct 2023 19:30:24 +0300 Subject: [PATCH 28/49] Move publishing charm to a separate workflow --- .../frontend_charm_integration_tests.yml | 25 --------------- .github/workflows/publish_charms.yml | 32 +++++++++++++++++++ 2 files changed, 32 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/publish_charms.yml diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index cbb58d3e5..f6e1d961e 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -71,28 +71,3 @@ jobs: run: working-directory: ./frontend/charm - publish-charm: - name: Publish Charm - needs: integration-test - runs-on: [self-hosted, linux] - if: ${{ github.ref_name == 'main' }} - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Install charmcraft - run: sudo snap install charmcraft --classic - - name: Fetch Tested Charm - uses: actions/download-artifact@v3 - with: - name: tested-charm - - name: Move charm in current directory - run: find ./ -name test-observer-frontend_ubuntu-22.04-amd64.charm -exec mv -t ./ {} \; - - name: Select Charmhub channel - uses: canonical/charming-actions/channel@2.2.2 - id: channel - - name: Upload charm to Charmhub - uses: canonical/charming-actions/upload-charm@2.2.2 - with: - credentials: "${{ secrets.CHARMHUB_AUTH_API }}" - github-token: "${{ secrets.GITHUB_TOKEN }}" - channel: "${{ steps.channel.outputs.name }}" diff --git a/.github/workflows/publish_charms.yml b/.github/workflows/publish_charms.yml new file mode 100644 index 000000000..cdf4983d4 --- /dev/null +++ b/.github/workflows/publish_charms.yml @@ -0,0 +1,32 @@ +name: Run charm tests +on: + push: + branches: ["main"] + tags: ["v*.*.*"] + +jobs: + publish-charm: + name: Publish Charm + needs: integration-test + runs-on: [self-hosted, linux] + if: ${{ github.ref_name == 'main' }} + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Install charmcraft + run: sudo snap install charmcraft --classic + - name: Fetch Tested Charm + uses: actions/download-artifact@v3 + with: + name: tested-charm + - name: Move charm in current directory + run: find ./ -name test-observer-frontend_ubuntu-22.04-amd64.charm -exec mv -t ./ {} \; + - name: Select Charmhub channel + uses: canonical/charming-actions/channel@2.2.2 + id: channel + - name: Upload charm to Charmhub + uses: canonical/charming-actions/upload-charm@2.2.2 + with: + credentials: "${{ secrets.CHARMHUB_AUTH_API }}" + github-token: "${{ secrets.GITHUB_TOKEN }}" + channel: "${{ steps.channel.outputs.name }}" From 08d8c5c79f98bdca4e4bebf4ce972a01957b81aa Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Thu, 26 Oct 2023 19:37:51 +0300 Subject: [PATCH 29/49] juju 3.0 => juju 3.2 --- .github/workflows/frontend_charm_integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index f6e1d961e..d7c4790da 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -24,7 +24,7 @@ jobs: with: channel: 1.27/stable # 1.26.4 provider: microk8s - juju-channel: 3.0/stable + juju-channel: 3.2/stable # juju-classic-confinement: "false" # microk8s-classic-confinement: "false" charmcraft-channel: 2.x/stable From 977550e29228ef9724380d52bf94034760d16066 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Thu, 26 Oct 2023 19:40:48 +0300 Subject: [PATCH 30/49] juju 3.0 -> 3.2 --- README.md | 2 +- backend/charm/metadata.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 67b9dc16e..210a7f3ff 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Observe the status and state of certification tests for various artefacts ## Prerequisites for developing and deploying locally -- `juju` 3.1 or later (`sudo snap install juju --channel=3.1/stable`) +- `juju` 3.2 or later (`sudo snap install juju --channel=3.2/stable`) - `microk8s` 1.27 or later (`sudo snap install microk8s --channel=1.27-strict/stable`) + [permission setup steps after install](https://juju.is/docs/sdk/set-up-your-development-environment#heading--install-microk8s) - `terraform` 1.4.6 or later (`sudo snap install terraform --classic`) - `lxd` 5.19 or later (`sudo snap install lxd --channel=5.19/stable` or `sudo snap refresh lxd --channel=5.19/stable` if already installed) + `lxd init --auto` after install. diff --git a/backend/charm/metadata.yaml b/backend/charm/metadata.yaml index b9d659ee1..54c0397a6 100644 --- a/backend/charm/metadata.yaml +++ b/backend/charm/metadata.yaml @@ -6,7 +6,7 @@ description: | summary: | API and dashboard to observe the status of artifact (snaps, debs, etc) test status assumes: - - juju >= 3.0 + - juju >= 3.2 - k8s-api containers: api: From 9f8ccec852b0a87167eb7ce8caed07a81cd61ba0 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Thu, 26 Oct 2023 19:54:52 +0300 Subject: [PATCH 31/49] Adds a now required trailing comma --- frontend/lib/ui/dashboard/dashboard_body.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/lib/ui/dashboard/dashboard_body.dart b/frontend/lib/ui/dashboard/dashboard_body.dart index 3a6a9252c..6828b7063 100644 --- a/frontend/lib/ui/dashboard/dashboard_body.dart +++ b/frontend/lib/ui/dashboard/dashboard_body.dart @@ -93,7 +93,7 @@ class _ArtefactCard extends StatelessWidget { const SizedBox(height: Spacing.level2), ], ) - .toList() + .toList(), ], ), ), From 422ee3b84d31e6ec1dcda52a4052bb6983602ab7 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Thu, 26 Oct 2023 19:56:11 +0300 Subject: [PATCH 32/49] microk8s 1.27 => 1.28 --- .github/workflows/frontend_charm_integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index d7c4790da..dccda6c32 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -22,7 +22,7 @@ jobs: - name: Setup operator environment uses: charmed-kubernetes/actions-operator@main # mz2/actions-operator@4fcb55b99e2a0f69a5910870d3639d203aeb87aa with: - channel: 1.27/stable # 1.26.4 + channel: 1.28/stable # 1.26.4 provider: microk8s juju-channel: 3.2/stable # juju-classic-confinement: "false" From 4068f041d2b325434c2e0eed89fe048a7c73e28e Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Thu, 26 Oct 2023 20:16:17 +0300 Subject: [PATCH 33/49] Use mz2 fork of the action operator --- .github/workflows/frontend_charm_integration_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index dccda6c32..78937150a 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -20,13 +20,13 @@ jobs: with: channel: 5.19/stable - name: Setup operator environment - uses: charmed-kubernetes/actions-operator@main # mz2/actions-operator@4fcb55b99e2a0f69a5910870d3639d203aeb87aa + uses: mz2/actions-operator@4fcb55b99e2a0f69a5910870d3639d203aeb87aa with: channel: 1.28/stable # 1.26.4 provider: microk8s juju-channel: 3.2/stable # juju-classic-confinement: "false" - # microk8s-classic-confinement: "false" + microk8s-classic-confinement: "false" charmcraft-channel: 2.x/stable lxd-channel: 5.19/stable - name: Adjust containerd template to accept ghcr.io images From 1b1e21205e1a648546e699e0dfce521b166d7343 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Thu, 26 Oct 2023 20:16:37 +0300 Subject: [PATCH 34/49] Juju classic confinement = off too --- .github/workflows/frontend_charm_integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index 78937150a..389b6c5ed 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -25,7 +25,7 @@ jobs: channel: 1.28/stable # 1.26.4 provider: microk8s juju-channel: 3.2/stable - # juju-classic-confinement: "false" + juju-classic-confinement: "false" microk8s-classic-confinement: "false" charmcraft-channel: 2.x/stable lxd-channel: 5.19/stable From fb4817acf08eb5b677e433c137cc98e0adc93d39 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Sat, 9 Dec 2023 23:38:49 +0200 Subject: [PATCH 35/49] Introduces a working integration testing workflow for the frontend charm The workflow depends on self hosted runner and composite actions introduced in https://github.com/canonical/certification-github-workflows --- .../frontend_charm_integration_tests.yml | 58 ++----------- .github/workflows/publish_api.yml | 4 +- README.md | 2 +- backend/charm/metadata.yaml | 2 +- backend/charm/requirements.txt | 2 +- frontend/charm/requirements.txt | 3 +- frontend/charm/src/charm.py | 82 ++++++------------- frontend/charm/tests/unit/test_charm.py | 4 +- frontend/charm/tox.ini | 2 +- terraform/test-observer.tf | 7 +- 10 files changed, 42 insertions(+), 124 deletions(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index 389b6c5ed..80a7df058 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -11,62 +11,16 @@ jobs: packages: read name: Integration tests - runs-on: [self-hosted, linux, large] + runs-on: [self-hosted, jammy, xlarge] steps: - name: Checkout - uses: actions/checkout@v3 - - name: Setup LXD - uses: canonical/setup-lxd@main - with: - channel: 5.19/stable - - name: Setup operator environment - uses: mz2/actions-operator@4fcb55b99e2a0f69a5910870d3639d203aeb87aa - with: - channel: 1.28/stable # 1.26.4 - provider: microk8s - juju-channel: 3.2/stable - juju-classic-confinement: "false" - microk8s-classic-confinement: "false" - charmcraft-channel: 2.x/stable - lxd-channel: 5.19/stable - - name: Adjust containerd template to accept ghcr.io images - run: | - sudo mkdir -p /var/snap/microk8s/current/args && - echo "" | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml && - echo '[plugins."io.containerd.grpc.v1.cri".registry.configs."ghcr.io".auth]' | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml && - echo ' username = "$GHCR_PACKAGE_API_USER"' | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml && - echo ' password = "$GHCR_PACKAGE_API_TOKEN"' | sudo tee -a /var/snap/microk8s/current/args/containerd-template.toml && - sudo systemctl restart snap.microk8s.daemon-containerd.service && - sudo microk8s.stop && - sleep 10 && - sudo microk8s.start && - sleep 10 && - sudo microk8s status --wait-ready && - sleep 30 - env: - GHCR_PACKAGE_API_USER: ${{ secrets.GHCR_PACKAGE_API_USER }} - GHCR_PACKAGE_API_TOKEN: ${{ secrets.GHCR_PACKAGE_API_TOKEN }} + uses: actions/checkout@v4 + - name: Set up microk8s + uses: canonical/certification-github-workflows/.github/actions/microk8s-setup@microk8s-setup-action - name: Run integration tests run: tox -e integration - - name: Archive Tested Charm - uses: actions/upload-artifact@v3 - if: ${{ github.ref_name == 'main' }} - with: - name: tested-charm - path: .tox/**/test-observer-frontend_ubuntu-22.04-amd64.charm - retention-days: 5 - - name: Archive charmcraft logs - if: failure() - uses: actions/upload-artifact@v3 - with: - name: charmcraft-logs - path: /home/runner/.local/state/charmcraft/log/*.log - - name: Archive juju crashdump - if: failure() - uses: actions/upload-artifact@v3 - with: - name: juju-crashdump - path: juju-crashdump-*.tar.xz + - name: Archive charm testing outputs + uses: canonical/certification-github-workflows/.github/actions/archive-charm-testing-artifacts@microk8s-setup-action defaults: run: working-directory: ./frontend/charm diff --git a/.github/workflows/publish_api.yml b/.github/workflows/publish_api.yml index efc307133..156926fb6 100644 --- a/.github/workflows/publish_api.yml +++ b/.github/workflows/publish_api.yml @@ -8,7 +8,7 @@ env: jobs: build-and-push-image: - runs-on: [self-hosted, linux] + runs-on: [self-hosted, linux, xlarge] permissions: contents: read packages: write @@ -43,8 +43,8 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-and-push-charm: + runs-on: [self-hosted, linux, xlarge] needs: build-and-push-image - runs-on: ubuntu-latest steps: - name: Checkout diff --git a/README.md b/README.md index 715f5d799..8f875b6ba 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Observe the status and state of certification tests for various artefacts ## Prerequisites for deploying locally -- `juju` 3.2 or later (`sudo snap install juju --channel=3.2/stable`) +- `juju` 3.1 or later (`sudo snap install juju --channel=3.1/stable`) - `microk8s` 1.27 or later (`sudo snap install microk8s --channel=1.27-strict/stable`) + [permission setup steps after install](https://juju.is/docs/sdk/set-up-your-development-environment#heading--install-microk8s) - `terraform` 1.4.6 or later (`sudo snap install terraform --classic`) - `lxd` 5.19 or later (`sudo snap install lxd --channel=5.19/stable` or `sudo snap refresh lxd --channel=5.19/stable` if already installed) + `lxd init --auto` after install. diff --git a/backend/charm/metadata.yaml b/backend/charm/metadata.yaml index 27183a88a..448bc2bcb 100644 --- a/backend/charm/metadata.yaml +++ b/backend/charm/metadata.yaml @@ -6,7 +6,7 @@ description: | summary: | API and dashboard to observe the status of artifact (snaps, debs, etc) test status assumes: - - juju >= 2.9 + - juju >=3.2 - k8s-api containers: api: diff --git a/backend/charm/requirements.txt b/backend/charm/requirements.txt index 398fee13d..60f27e45d 100644 --- a/backend/charm/requirements.txt +++ b/backend/charm/requirements.txt @@ -1,2 +1,2 @@ -ops >= 1.5.0 +ops >= 2.8.0 requests==2.31.0 diff --git a/frontend/charm/requirements.txt b/frontend/charm/requirements.txt index aaa16b155..c0f162079 100644 --- a/frontend/charm/requirements.txt +++ b/frontend/charm/requirements.txt @@ -1 +1,2 @@ -ops >= 2.2.0 +ops >= 2.8.0 +macaroonbakery==1.3.2 \ No newline at end of file diff --git a/frontend/charm/src/charm.py b/frontend/charm/src/charm.py index 34c822631..331482c81 100755 --- a/frontend/charm/src/charm.py +++ b/frontend/charm/src/charm.py @@ -30,7 +30,9 @@ def __init__(self, *args): self.pebble_service_name = "test-observer-frontend" self.container = self.unit.get_container("frontend") - self.framework.observe(self.on.frontend_pebble_ready, self._update_layer_and_restart) + self.framework.observe( + self.on.frontend_pebble_ready, self._update_layer_and_restart + ) self.framework.observe(self.on.config_changed, self._on_config_changed) self.framework.observe( self.on.test_observer_rest_api_relation_joined, @@ -83,16 +85,14 @@ def _config_is_valid(self, config) -> Tuple[bool, str]: def _on_rest_api_relation_update(self, event): api_hostname = event.relation.data[event.app].get("hostname") api_port = event.relation.data[event.app].get("port") - logger.debug(f"API hostname: {api_hostname}, port: {api_port} (app: {event.app})") + logger.debug( + f"API hostname: {api_hostname}, port: {api_port} (app: {event.app})" + ) self._update_layer_and_restart(event) def _on_rest_api_relation_broken(self, event): logger.debug("REST API relation broken") -<<<<<<< HEAD - self._update_layer_and_restart(event) -======= self._handle_no_api_relation() ->>>>>>> 5944c53a8531a0af407dfc20274774ec67d3d07d def nginx_config(self, base_uri: str) -> str: """Return a config where the backend port `base_uri` is adjusted.""" @@ -133,13 +133,10 @@ def nginx_503_config(self) -> str: location @maintenance { rewrite ^(.*)$ /503.html break; root /usr/share/nginx/html; -<<<<<<< HEAD -======= # Ensure no caching expires -1; add_header Cache-Control "no-store, no-cache, must-revalidate, post-check=0, pre-check=0"; ->>>>>>> 5944c53a8531a0af407dfc20274774ec67d3d07d } } """ @@ -159,40 +156,10 @@ def html_503(self) -> str: """ def _update_layer_and_restart(self, event): - self.unit.status = MaintenanceStatus(f"Updating {self.pebble_service_name} layer") - -<<<<<<< HEAD - api_relation = self.model.relations["test-observer-rest-api"][0] - - if api_relation is None: - if self.container.can_connect(): - self.container.add_layer( - self.pebble_service_name, self._pebble_layer, combine=True - ) - self.container.push( - "/etc/nginx/sites-available/test-observer-frontend", - self.nginx_503_config(), - make_dirs=True, - ) - self.container.push( - "/usr/share/nginx/html/503.html", - self.html_503(), - make_dirs=True, - ) - self.container.restart(self.pebble_service_name) - self.unit.status = MaintenanceStatus( - "test-observer-rest-api relation not connected." - ) - else: - self.unit.status = WaitingStatus( - "Waiting for Pebble for API to set maintenance state" - ) - - return + self.unit.status = MaintenanceStatus( + f"Updating {self.pebble_service_name} layer" + ) - hostname = api_relation.data[api_relation.app]["hostname"] - port = api_relation.data[api_relation.app]["port"] -======= if self.container.can_connect(): api_url = self._api_url if api_url: @@ -209,7 +176,9 @@ def _update_layer_and_restart(self, event): else: self._handle_no_api_relation() else: - self.unit.status = WaitingStatus("Waiting for Pebble for API to set available state") + self.unit.status = WaitingStatus( + "Waiting for Pebble for API to set available state" + ) @property def _api_url(self) -> str | None: @@ -221,12 +190,13 @@ def _api_url(self) -> str | None: relation_data = api_relation.data[api_relation.app] if not relation_data: - self.unit.status = WaitingStatus("Waiting for test observer api relation data") + self.unit.status = WaitingStatus( + "Waiting for test observer api relation data" + ) return hostname = relation_data["hostname"] port = relation_data["port"] ->>>>>>> 5944c53a8531a0af407dfc20274774ec67d3d07d scheme = self.config["test-observer-api-scheme"] @@ -239,10 +209,7 @@ def _api_url(self) -> str | None: base_uri = f"{scheme}{hostname}:{port}" return base_uri -<<<<<<< HEAD -======= def _handle_no_api_relation(self): ->>>>>>> 5944c53a8531a0af407dfc20274774ec67d3d07d if self.container.can_connect(): self.container.push( "/etc/nginx/sites-available/test-observer-frontend", @@ -254,20 +221,17 @@ def _handle_no_api_relation(self): self.html_503(), make_dirs=True, ) - self.container.add_layer(self.pebble_service_name, self._pebble_layer, combine=True) - self.container.push( - "/etc/nginx/sites-available/test-observer-frontend", - self.nginx_config(base_uri=base_uri), - make_dirs=True, + self.container.add_layer( + self.pebble_service_name, self._pebble_layer, combine=True ) self.container.restart(self.pebble_service_name) - self.unit.status = MaintenanceStatus("test-observer-rest-api relation not connected.") + self.unit.status = MaintenanceStatus( + "test-observer-rest-api relation not connected." + ) else: -<<<<<<< HEAD - self.unit.status = WaitingStatus("Waiting for Pebble for API to set available state") -======= - self.unit.status = WaitingStatus("Waiting for Pebble for API to set maintenance state") ->>>>>>> 5944c53a8531a0af407dfc20274774ec67d3d07d + self.unit.status = WaitingStatus( + "Waiting for Pebble for API to set maintenance state" + ) @property def _pebble_layer(self): diff --git a/frontend/charm/tests/unit/test_charm.py b/frontend/charm/tests/unit/test_charm.py index 6a3330c83..fa8a1ec3d 100644 --- a/frontend/charm/tests/unit/test_charm.py +++ b/frontend/charm/tests/unit/test_charm.py @@ -33,7 +33,9 @@ def test_relating(self): harness.update_config({"test-observer-api-scheme": "https://"}) harness.begin() rel_id = harness.add_relation("test-observer-rest-api", "backend") - harness.update_relation_data(rel_id, "backend", {"hostname": "teh-backend", "port": "443"}) + harness.update_relation_data( + rel_id, "backend", {"hostname": "teh-backend", "port": "443"} + ) service = harness.model.unit.get_container("frontend").get_service( "test-observer-frontend" diff --git a/frontend/charm/tox.ini b/frontend/charm/tox.ini index 0ec476cac..954ee8c0b 100644 --- a/frontend/charm/tox.ini +++ b/frontend/charm/tox.ini @@ -66,7 +66,7 @@ commands = description = Run integration tests deps = pytest - juju<3.1 + juju<=3.2 pytest-operator -r {tox_root}/requirements.txt commands = diff --git a/terraform/test-observer.tf b/terraform/test-observer.tf index ac39f2be4..a6aa6daf1 100644 --- a/terraform/test-observer.tf +++ b/terraform/test-observer.tf @@ -1,7 +1,7 @@ terraform { required_providers { juju = { - version = "~> 0.7.0" + version = "~> 0.10.0" source = "juju/juju" } } @@ -54,7 +54,7 @@ resource "juju_application" "ingress" { } config = { - tls-secret-name = var.tls_secret_name + tls-secret-name = var.tls_secret_name whitelist-source-range = var.nginx_ingress_integrator_charm_whitelist_source_range } } @@ -67,7 +67,6 @@ resource "juju_application" "pg" { charm { name = "postgresql-k8s" channel = "14/stable" - series = "jammy" } } @@ -78,7 +77,6 @@ resource "juju_application" "test-observer-api" { charm { name = "test-observer-api" channel = "latest/edge" - series = "jammy" } config = { @@ -97,7 +95,6 @@ resource "juju_application" "test-observer-frontend" { charm { name = "test-observer-frontend" channel = "latest/edge" - series = "jammy" } config = { From 5a75762adb5896c3ed5f3f5b9610cc9e7ca577e9 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Sun, 10 Dec 2023 00:16:11 +0200 Subject: [PATCH 36/49] Update .github/workflows/publish_charms.yml --- .github/workflows/publish_charms.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_charms.yml b/.github/workflows/publish_charms.yml index cdf4983d4..8e03080d2 100644 --- a/.github/workflows/publish_charms.yml +++ b/.github/workflows/publish_charms.yml @@ -15,7 +15,7 @@ jobs: uses: actions/checkout@v2 - name: Install charmcraft run: sudo snap install charmcraft --classic - - name: Fetch Tested Charm + - name: Fetch the tested charm uses: actions/download-artifact@v3 with: name: tested-charm From 418cdbd0f7519b5ee05e18e950c2607ff375e43d Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Sun, 10 Dec 2023 00:18:59 +0200 Subject: [PATCH 37/49] Remove publish_charms.yml that has been long since been separately added --- .github/workflows/publish_charms.yml | 32 ---------------------------- 1 file changed, 32 deletions(-) delete mode 100644 .github/workflows/publish_charms.yml diff --git a/.github/workflows/publish_charms.yml b/.github/workflows/publish_charms.yml deleted file mode 100644 index 8e03080d2..000000000 --- a/.github/workflows/publish_charms.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Run charm tests -on: - push: - branches: ["main"] - tags: ["v*.*.*"] - -jobs: - publish-charm: - name: Publish Charm - needs: integration-test - runs-on: [self-hosted, linux] - if: ${{ github.ref_name == 'main' }} - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Install charmcraft - run: sudo snap install charmcraft --classic - - name: Fetch the tested charm - uses: actions/download-artifact@v3 - with: - name: tested-charm - - name: Move charm in current directory - run: find ./ -name test-observer-frontend_ubuntu-22.04-amd64.charm -exec mv -t ./ {} \; - - name: Select Charmhub channel - uses: canonical/charming-actions/channel@2.2.2 - id: channel - - name: Upload charm to Charmhub - uses: canonical/charming-actions/upload-charm@2.2.2 - with: - credentials: "${{ secrets.CHARMHUB_AUTH_API }}" - github-token: "${{ secrets.GITHUB_TOKEN }}" - channel: "${{ steps.channel.outputs.name }}" From ff57b227a41b777a2a86cc98690b6c7fb9443c15 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Sun, 10 Dec 2023 00:19:48 +0200 Subject: [PATCH 38/49] Removes release-charms.yml that has long been superseded --- .github/workflows/release_charms.yml | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 .github/workflows/release_charms.yml diff --git a/.github/workflows/release_charms.yml b/.github/workflows/release_charms.yml deleted file mode 100644 index 148b8b753..000000000 --- a/.github/workflows/release_charms.yml +++ /dev/null @@ -1,28 +0,0 @@ -on: -# pull_request: -# branches: -# - main - push: - branches: ["a-branch-that-definitely-does-not-exist"] - # tags: ["v*.*.*"] - -jobs: - build: - name: Release test-observer-api charm to edge - runs-on: [self-hosted, linux] - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - #- name: Select charmhub channel - # uses: canonical/charming-actions/channel@2.2.0 - # id: channel - - name: Upload charm to charmhub - uses: canonical/charming-actions/upload-charm@2.2.0 - with: - charm-path: "backend/charm" - credentials: "${{ secrets.CHARMHUB_AUTH_API }}" - github-token: "${{ secrets.GITHUB_TOKEN }}" - upload-image: "true" - channel: edge # "${{ steps.channel.outputs.name }}" From 61e6d4fe80eca79de3ed566cc6e12626725420b1 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Sun, 10 Dec 2023 00:20:59 +0200 Subject: [PATCH 39/49] Removes also frontend_charm_promote.yml that is not really in scope of these changes --- .github/workflows/frontend_charm_promote.yml | 43 -------------------- 1 file changed, 43 deletions(-) delete mode 100644 .github/workflows/frontend_charm_promote.yml diff --git a/.github/workflows/frontend_charm_promote.yml b/.github/workflows/frontend_charm_promote.yml deleted file mode 100644 index a04866df9..000000000 --- a/.github/workflows/frontend_charm_promote.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Promote Charm - -on: - workflow_dispatch: - inputs: - promotion: - type: choice - description: Channel to promote from - options: - - edge -> beta - - beta -> candidate - - candidate -> stable - -jobs: - promote: - name: Promote Charm - runs-on: [self-hosted, linux] - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Set target channel - env: - PROMOTE_FROM: ${{ github.event.inputs.promotion }} - run: | - if [ "${PROMOTE_FROM}" == "edge -> beta" ]; then - echo "promote-from=edge" >> ${GITHUB_ENV} - echo "promote-to=beta" >> ${GITHUB_ENV} - elif [ "${PROMOTE_FROM}" == "beta -> candidate" ]; then - echo "promote-from=beta" >> ${GITHUB_ENV} - echo "promote-to=candidate" >> ${GITHUB_ENV} - elif [ "${PROMOTE_FROM}" == "candidate -> stable" ]; then - echo "promote-from=candidate" >> ${GITHUB_ENV} - echo "promote-to=stable" >> ${GITHUB_ENV} - fi - - name: Promote Charm - uses: canonical/charming-actions/release-charm@2.2.0 - with: - base-channel: 22.04 - credentials: ${{ secrets.CHARMHUB_TOKEN }} - github-token: ${{ secrets.GITHUB_TOKEN }} - destination-channel: latest/${{ env.promote-to }} - origin-channel: latest/${{ env.promote-from }} - charmcraft-channel: latest/stable From 9e0ff278ba87dce6277d2006e13636cf6b9710bc Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Sun, 10 Dec 2023 00:21:46 +0200 Subject: [PATCH 40/49] From 3.2 requirement => 2.9 requirement --- backend/charm/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/charm/metadata.yaml b/backend/charm/metadata.yaml index 448bc2bcb..27183a88a 100644 --- a/backend/charm/metadata.yaml +++ b/backend/charm/metadata.yaml @@ -6,7 +6,7 @@ description: | summary: | API and dashboard to observe the status of artifact (snaps, debs, etc) test status assumes: - - juju >=3.2 + - juju >= 2.9 - k8s-api containers: api: From 943e0e6eb2d16800716fa1c868baacaeeefb4a86 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Sun, 10 Dec 2023 00:25:56 +0200 Subject: [PATCH 41/49] Removes the stray reformatting of frontend charm.py --- frontend/charm/src/charm.py | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/frontend/charm/src/charm.py b/frontend/charm/src/charm.py index 331482c81..98880e6ff 100755 --- a/frontend/charm/src/charm.py +++ b/frontend/charm/src/charm.py @@ -30,9 +30,7 @@ def __init__(self, *args): self.pebble_service_name = "test-observer-frontend" self.container = self.unit.get_container("frontend") - self.framework.observe( - self.on.frontend_pebble_ready, self._update_layer_and_restart - ) + self.framework.observe(self.on.frontend_pebble_ready, self._update_layer_and_restart) self.framework.observe(self.on.config_changed, self._on_config_changed) self.framework.observe( self.on.test_observer_rest_api_relation_joined, @@ -85,9 +83,7 @@ def _config_is_valid(self, config) -> Tuple[bool, str]: def _on_rest_api_relation_update(self, event): api_hostname = event.relation.data[event.app].get("hostname") api_port = event.relation.data[event.app].get("port") - logger.debug( - f"API hostname: {api_hostname}, port: {api_port} (app: {event.app})" - ) + logger.debug(f"API hostname: {api_hostname}, port: {api_port} (app: {event.app})") self._update_layer_and_restart(event) def _on_rest_api_relation_broken(self, event): @@ -156,9 +152,7 @@ def html_503(self) -> str: """ def _update_layer_and_restart(self, event): - self.unit.status = MaintenanceStatus( - f"Updating {self.pebble_service_name} layer" - ) + self.unit.status = MaintenanceStatus(f"Updating {self.pebble_service_name} layer") if self.container.can_connect(): api_url = self._api_url @@ -176,9 +170,7 @@ def _update_layer_and_restart(self, event): else: self._handle_no_api_relation() else: - self.unit.status = WaitingStatus( - "Waiting for Pebble for API to set available state" - ) + self.unit.status = WaitingStatus("Waiting for Pebble for API to set available state") @property def _api_url(self) -> str | None: @@ -190,9 +182,7 @@ def _api_url(self) -> str | None: relation_data = api_relation.data[api_relation.app] if not relation_data: - self.unit.status = WaitingStatus( - "Waiting for test observer api relation data" - ) + self.unit.status = WaitingStatus("Waiting for test observer api relation data") return hostname = relation_data["hostname"] @@ -221,17 +211,11 @@ def _handle_no_api_relation(self): self.html_503(), make_dirs=True, ) - self.container.add_layer( - self.pebble_service_name, self._pebble_layer, combine=True - ) + self.container.add_layer(self.pebble_service_name, self._pebble_layer, combine=True) self.container.restart(self.pebble_service_name) - self.unit.status = MaintenanceStatus( - "test-observer-rest-api relation not connected." - ) + self.unit.status = MaintenanceStatus("test-observer-rest-api relation not connected.") else: - self.unit.status = WaitingStatus( - "Waiting for Pebble for API to set maintenance state" - ) + self.unit.status = WaitingStatus("Waiting for Pebble for API to set maintenance state") @property def _pebble_layer(self): From acb0dd891f5a2835f08610e5d107fcf86a11a040 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Sun, 10 Dec 2023 00:35:54 +0200 Subject: [PATCH 42/49] Remove another example of accidental reformatting --- frontend/charm/tests/unit/test_charm.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frontend/charm/tests/unit/test_charm.py b/frontend/charm/tests/unit/test_charm.py index fa8a1ec3d..6a3330c83 100644 --- a/frontend/charm/tests/unit/test_charm.py +++ b/frontend/charm/tests/unit/test_charm.py @@ -33,9 +33,7 @@ def test_relating(self): harness.update_config({"test-observer-api-scheme": "https://"}) harness.begin() rel_id = harness.add_relation("test-observer-rest-api", "backend") - harness.update_relation_data( - rel_id, "backend", {"hostname": "teh-backend", "port": "443"} - ) + harness.update_relation_data(rel_id, "backend", {"hostname": "teh-backend", "port": "443"}) service = harness.model.unit.get_container("frontend").get_service( "test-observer-frontend" From f9528e3cc09a18fed21102fe21e41309b03e06ef Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Sun, 10 Dec 2023 00:40:08 +0200 Subject: [PATCH 43/49] Run the integration test workflow manually, on main branch and when PR review has been submitted --- .github/workflows/frontend_charm_integration_tests.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index f17f434a7..b95af45de 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -1,8 +1,11 @@ -name: Run charm tests +name: Run charm integration tests on: + pull_request_review: + types: [submitted] push: - branches: ["main", "integration-test-fix*"] + branches: ["main"] tags: ["v*.*.*"] + workflow_dispatch: jobs: integration-test: From 3ded998ee7c6fc89b579397b69fe5efccb61fbb6 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Sun, 10 Dec 2023 00:47:22 +0200 Subject: [PATCH 44/49] Run frontend integration tests also on branches where there are changes to resources under frontend/charm --- .github/workflows/frontend_charm_integration_tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index b95af45de..cc7e6c8e2 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -2,6 +2,11 @@ name: Run charm integration tests on: pull_request_review: types: [submitted] + pull_request: + branches: + - '**' + paths: + - 'frontend/charm/**' push: branches: ["main"] tags: ["v*.*.*"] From 5bcbd1cdbfce9e9b073b60d6c507a36dd94cf4f4 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Sun, 10 Dec 2023 00:52:26 +0200 Subject: [PATCH 45/49] Removes a stray change to frontend_charm_analysis.yml --- .github/workflows/frontend_charm_analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/frontend_charm_analysis.yml b/.github/workflows/frontend_charm_analysis.yml index 707d6b287..b411f3a57 100644 --- a/.github/workflows/frontend_charm_analysis.yml +++ b/.github/workflows/frontend_charm_analysis.yml @@ -4,7 +4,7 @@ on: branches: - main push: - branches: ["main", "integration-test-fix"] + branches: ["main"] tags: ["v*.*.*"] jobs: From ad2ae8bf509f5a03c1a4f5d97530dec3f64b1d02 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Mon, 11 Dec 2023 13:53:51 +0200 Subject: [PATCH 46/49] An explanatory comment re: pinning macaroonbakery to 1.3.2 --- frontend/charm/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/charm/requirements.txt b/frontend/charm/requirements.txt index c0f162079..84a2a12dd 100644 --- a/frontend/charm/requirements.txt +++ b/frontend/charm/requirements.txt @@ -1,2 +1,2 @@ ops >= 2.8.0 -macaroonbakery==1.3.2 \ No newline at end of file +macaroonbakery==1.3.2 # workaround to failures such as https://github.com/canonical/test_observer/actions/runs/7142353669/job/19451476954 \ No newline at end of file From 09e1425f4a13d8d1fadd6accaa8aafe8702e0c55 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Mon, 11 Dec 2023 14:02:14 +0200 Subject: [PATCH 47/49] Adds docker.io mirroring --- .github/workflows/publish_api.yml | 6 +++++- .github/workflows/publish_frontend.yml | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish_api.yml b/.github/workflows/publish_api.yml index a109aef16..1cb2982f4 100644 --- a/.github/workflows/publish_api.yml +++ b/.github/workflows/publish_api.yml @@ -8,7 +8,7 @@ env: jobs: build-and-push-image: - runs-on: [self-hosted, linux, xlarge] + runs-on: [self-hosted, linux] permissions: contents: read packages: write @@ -19,6 +19,10 @@ jobs: steps: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + config-inline: | + [registry."docker.io"] + mirrors = ["https://github-runner-dockerhub-cache.canonical.com:5000"] - name: Checkout repository uses: actions/checkout@v3 diff --git a/.github/workflows/publish_frontend.yml b/.github/workflows/publish_frontend.yml index 655ac2653..7e6ecd926 100644 --- a/.github/workflows/publish_frontend.yml +++ b/.github/workflows/publish_frontend.yml @@ -19,6 +19,10 @@ jobs: steps: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + config-inline: | + [registry."docker.io"] + mirrors = ["https://github-runner-dockerhub-cache.canonical.com:5000"] - name: Checkout repository uses: actions/checkout@v3 From e465185314669ea9ebe44c1237cb113785aa5e9f Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Mon, 11 Dec 2023 15:06:45 +0200 Subject: [PATCH 48/49] Fix the trigger so the workflow triggers on approved PRs --- .github/workflows/frontend_charm_integration_tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index cc7e6c8e2..2407f526d 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -1,7 +1,9 @@ name: Run charm integration tests on: pull_request_review: - types: [submitted] + on: + pull_request_review: + types: [submitted] pull_request: branches: - '**' @@ -20,6 +22,7 @@ jobs: name: Integration tests runs-on: [self-hosted, jammy, xlarge] + if: github.event.review.state == 'APPROVED' steps: - name: Checkout uses: actions/checkout@v4 From 4814f969f91328b9f62c857ba7693815e1bdd0d4 Mon Sep 17 00:00:00 2001 From: Matias Piipari Date: Mon, 11 Dec 2023 15:20:12 +0200 Subject: [PATCH 49/49] Reference the setup steps from main --- .github/workflows/frontend_charm_integration_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/frontend_charm_integration_tests.yml b/.github/workflows/frontend_charm_integration_tests.yml index 2407f526d..4ac1338e9 100644 --- a/.github/workflows/frontend_charm_integration_tests.yml +++ b/.github/workflows/frontend_charm_integration_tests.yml @@ -27,11 +27,11 @@ jobs: - name: Checkout uses: actions/checkout@v4 - name: Set up microk8s - uses: canonical/certification-github-workflows/.github/actions/microk8s-setup@microk8s-setup-action + uses: canonical/certification-github-workflows/.github/actions/microk8s-setup@main - name: Run integration tests run: tox -e integration - name: Archive charm testing outputs - uses: canonical/certification-github-workflows/.github/actions/archive-charm-testing-artifacts@microk8s-setup-action + uses: canonical/certification-github-workflows/.github/actions/archive-charm-testing-artifacts@main defaults: run: working-directory: ./frontend/charm