[WIP] Compute resources & plugins

.github/workflows/test.yml

@@ -75,12 +75,18 @@ jobs:
       - name: Run deployment
         run: |
           ./foremanctl deploy --certificate-source=${{ matrix.certificate_source }} --foreman-initial-admin-password=changeme
-      - name: Setup hammer
+      - name: Add optional feature - hammer
         run: |
-          ./foremanctl setup-hammer
-      - name: Set up Foreman Proxy
+          ./foremanctl deploy --add-feature hammer
+      - name: Add optional feature - foreman-proxy
         run: |
-          ./foremanctl setup-foreman-proxy
+          ./foremanctl deploy --add-feature foreman-proxy
+      - name: Add optional feature - foreman_azure_rm
+        run: |
+          ./foremanctl deploy --add-feature foreman_azure_rm
+      - name: Add optional feature - foreman_google
+        run: |
+          ./foremanctl deploy --add-feature foreman_google
       - name: Run tests
         run: |
           ./forge test --pytest-args="--certificate-source=${{ matrix.certificate_source }}"
@@ -181,6 +187,12 @@ jobs:
       - name: Run deployment
         run: |
           ./foremanctl deploy --foreman-initial-admin-password=changeme
+      - name: Add optional feature - hammer
+        run: |
+          ./foremanctl deploy --add-feature hammer
+      - name: Add optional feature - foreman-proxy
+        run: |
+          ./foremanctl deploy --add-feature foreman-proxy
       - name: Stop services
         run:
           vagrant ssh quadlet -- sudo systemctl stop foreman.target
@@ -193,12 +205,6 @@ jobs:
       - name: Run deployment
         run: |
           ./foremanctl deploy
-      - name: Setup hammer
-        run: |
-          ./foremanctl setup-hammer
-      - name: Set up Foreman Proxy
-        run: |
-          ./foremanctl setup-foreman-proxy
       - name: Run tests
         run: |
           ./forge test

DEVELOPMENT.md

@@ -34,7 +34,7 @@ source .venv/bin/activate
 
 ```
 ./forge setup-repositories
-./foremanctl setup-hammer
+./foremanctl deploy --add-feature hammer
 ```
 To teardown the environment:
 

development/playbooks/deploy-dev/deploy-dev.yaml

@@ -4,6 +4,7 @@
   become: true
   vars_files:
     - "../../../src/vars/defaults.yml"
+    - "../../../src/vars/flavors/{{ flavor }}.yml"
     - "../../../src/vars/{{ certificate_source }}_certificates.yml"
     - "../../../src/vars/images.yml"
     - "../../../src/vars/database.yml"

docs/deployment.md

@@ -23,6 +23,25 @@ For example, pre-pulling images to reduce the core deployment utility runtime.
   6. Run deployment utility
   7. Post deploy checks
 
+### Features and Flavors
+
+To allow deployments with different sets of functionality enabled, the deployment utility supports features and flavors.
+
+- A feature is an abstract representation of "the deployed system can now do X", usually implemented by enabling a Foreman/Pulp/Hammer plugin (or a collection of these). 
+- A flavor is a set of features that are enabled by default and can not be disabled. This is to allow common deployment types like "vanilla foreman", "katello", "satellite" and similar.
+
+Additionally to the functionality offered by plugins, we define the following "base" features:
+- `foreman` to deploy the main Rails app and make the deployment a "Server"
+- `foreman-proxy` to deploy the Foreman Proxy code
+- `hammer` to deploy the base CLI
+
+These base features control which plugins are enabled when a feature is requested.
+- `foreman` + `remote_execution` = `foreman_remote_execution`
+- `foreman-proxy` + `remote_execution` = `smart_proxy_remote_execution_ssh`
+- `hammer` + `remote_execution` = `hammer_cli_foreman_remote_execution`
+
+A deployment can have multiple base features enabled.
+
 ### Authenticated Registry Handling
 
 In the non-default case where the image sources are supplied from an authenticated location users will need to inject a login step.

foremanctl.spec

@@ -10,7 +10,7 @@ URL:       https://github.com/theforeman/foremanctl
 Source:    https://github.com/theforeman/foremanctl/releases/download/%{version}/%{name}-%{version}.tar.gz
 
 BuildArch: noarch
-Requires:  python3-obsah >= 1.4
+Requires:  python3-obsah >= 1.6
 
 # These are needed on the target host, which is usually localhost
 Recommends:  podman

src/playbooks/_flavor_features/metadata.obsah.yaml

@@ -0,0 +1,13 @@
+---
+variables:
+  flavor:
+    help: Base flavor to use in this deployment.
+  features:
+    parameter: --add-feature
+    help: Additional features to enable in this deployment.
+    action: append_unique
+  remove_features:
+    parameter: --remove-feature
+    help: Additional features to disable in this deployment.
+    action: remove
+    dest: features

src/playbooks/deploy/deploy.yaml

@@ -5,6 +5,7 @@
   become: true
   vars_files:
     - "../../vars/defaults.yml"
+    - "../../vars/flavors/{{ flavor }}.yml"
     - "../../vars/{{ certificate_source }}_certificates.yml"
     - "../../vars/images.yml"
     - "../../vars/database.yml"
@@ -29,4 +30,10 @@
     - pulp
     - foreman
     - role: systemd_target
+    - role: foreman_proxy
+      when:
+        - "'foreman-proxy' in enabled_features"
+    - role: hammer
+      when:
+        - "'hammer' in enabled_features"
     - post_install

src/playbooks/deploy/metadata.obsah.yaml

@@ -16,3 +16,4 @@ include:
   - _certificate_source
   - _database_mode
   - _database_connection
+  - _flavor_features

src/playbooks/pull-images/metadata.obsah.yaml

@@ -4,3 +4,4 @@ help: |
 
 include:
   - _database_mode
+  - _flavor_features

src/playbooks/pull-images/pull-images.yaml

@@ -4,7 +4,9 @@
     - quadlet
   vars_files:
     - "../../vars/defaults.yml"
+    - "../../vars/flavors/{{ flavor }}.yml"
     - "../../vars/images.yml"
+    - "../../vars/base.yaml"
   become: true
   tasks:
     - name: Install podman
@@ -17,6 +19,13 @@
         name: "{{ item }}"
       loop: "{{ images }}"
 
+    - name: Pull foreman_proxy images
+      containers.podman.podman_image:
+        name: "{{ item }}"
+      loop: "{{ foreman_proxy_images }}"
+      when:
+        - "'foreman-proxy' in enabled_features"
+
     - name: Pull database images
       containers.podman.podman_image:
         name: "{{ item }}"

src/requirements.txt

@@ -1,2 +1,2 @@
-obsah >= 1.4
+obsah >= 1.6
 requests

src/requirements.yml

@@ -5,3 +5,4 @@ collections:
   - name: containers.podman
     version: ">=1.16.4"
   - name: theforeman.foreman
+  - name: theforeman.operations

src/roles/foreman/tasks/main.yaml

@@ -105,6 +105,7 @@
       - 'foreman-client-key,type=mount,target=/etc/foreman/client_key.pem'
     env:
       FOREMAN_PUMA_WORKERS: "{{ foreman_puma_workers }}"
+      FOREMAN_ENABLED_PLUGINS: "{{ foreman_plugins | join(' ') }}"
     quadlet_options:
       - |
         [Install]
@@ -136,6 +137,7 @@
       DYNFLOW_SIDEKIQ_SCRIPT: "/usr/share/foreman/extras/dynflow-sidekiq.rb"
       DYNFLOW_REDIS_URL: "redis://localhost:6379/6"
       REDIS_PROVIDER: "DYNFLOW_REDIS_URL"
+      FOREMAN_ENABLED_PLUGINS: "{{ foreman_plugins | join(' ') }}"
     command: "/usr/libexec/foreman/sidekiq-selinux -e production -r /usr/share/foreman/extras/dynflow-sidekiq.rb -C /etc/foreman/dynflow/%i.yml"
     quadlet_options:
       - |
@@ -175,6 +177,8 @@
     detach: false
     working_dir: /usr/share/foreman
     network: host
+    env:
+      FOREMAN_ENABLED_PLUGINS: "{{ foreman_plugins | join(' ') }}"
     secrets:
       - 'foreman-database-url,type=env,target=DATABASE_URL'
       - 'foreman-seed-admin-user,type=env,target=SEED_ADMIN_USER'

src/roles/hammer/defaults/main.yml

@@ -1,8 +1,7 @@
 ---
 hammer_foreman_server_url: "https://{{ ansible_facts['fqdn'] }}"
 hammer_ca_certificate: ""
-hammer_packages:
-  - hammer-cli-plugin-foreman
-  - hammer-cli-plugin-foreman_tasks
-  - hammer-cli-plugin-foreman_remote_execution
-  - hammer-cli-plugin-katello
+hammer_default_plugins:
+  - foreman
+hammer_plugins: []
+hammer_packages: "{{ (hammer_default_plugins+hammer_plugins) | map('regex_replace', '^', 'hammer-cli-plugin-') }}"

src/roles/hammer/tasks/main.yml

@@ -1,4 +1,11 @@
 ---
+- name: Setup Foreman repositories
+  ansible.builtin.include_role:
+    name: theforeman.operations.foreman_repositories
+  vars:
+    foreman_repositories_version: nightly
+    foreman_repositories_katello_version: nightly
+
 - name: Install Hammer and plugins
   ansible.builtin.package:
     name: "{{ hammer_packages }}"

src/roles/pulp/defaults/main.yaml

@@ -21,6 +21,15 @@ pulp_pulp_url: "http://{{ ansible_facts['fqdn'] }}:24817"
 
 pulp_enable_analytics: false
 
+pulp_default_plugins:
+  - pulp_certguard
+  - pulp_file
+  - pulp_smart_proxy
+pulp_plugins:
+  - pulp_container
+  - pulp_rpm
+pulp_enabled_plugins: "{{ pulp_default_plugins + pulp_plugins }}"
+
 pulp_database_name: pulp
 pulp_database_user: pulp
 pulp_database_host: localhost
@@ -35,6 +44,8 @@ pulp_settings_database_env:
   PULP_DATABASES__default__PORT: "{{ pulp_database_port }}"
   PULP_DATABASES__default__OPTIONS__sslmode: "{{ pulp_database_ssl_mode }}"
   PULP_DATABASES__default__OPTIONS__sslrootcert: "{{ pulp_database_ssl_ca }}"
+  PULP_ENABLED_PLUGINS: >-
+    {{ pulp_enabled_plugins }}
 
 # The arrays (AUTH_BACKENDS, AUTH_CLASSES) need to be strings
 # that get parsed to array by dynaconf, so we're using block scalars for them

src/vars/base.yaml

@@ -20,6 +20,7 @@ foreman_client_key: "{{ client_key }}"
 foreman_client_certificate: "{{ client_certificate }}"
 foreman_oauth_consumer_key: abcdefghijklmnopqrstuvwxyz123456
 foreman_oauth_consumer_secret: abcdefghijklmnopqrstuvwxyz123456
+foreman_plugins: "{{ enabled_features | reject('contains', 'content/') | reject('eq', 'hammer') | reject('eq', 'foreman-proxy') | reject('eq', 'foreman') }}"
 foreman_url: "https://{{ ansible_facts['fqdn'] }}"
 
 httpd_server_ca_certificate: "{{ server_ca_certificate }}"
@@ -29,3 +30,7 @@ httpd_server_key: "{{ server_key }}"
 
 pulp_content_origin: "https://{{ ansible_facts['fqdn'] }}"
 pulp_pulp_url: "https://{{ ansible_facts['fqdn'] }}"
+pulp_plugins: "{{ enabled_features | select('contains', 'content/') | map('replace', 'content/', 'pulp_') | list }}"
+
+hammer_ca_certificate: "{{ server_ca_certificate }}"
+hammer_plugins: "{{ foreman_plugins | map('replace', 'foreman-tasks', 'foreman_tasks') | list }}"

src/vars/defaults.yml

@@ -2,3 +2,6 @@
 certificate_source: default
 database_mode: internal
 tuning_profile: development
+flavor: katello
+features: []
+enabled_features: "{{ (flavor_features + features) }}"

src/vars/flavors/katello.yml

@@ -0,0 +1,6 @@
+---
+flavor_features:
+  - foreman
+  - katello
+  - content/container
+  - content/rpm

src/vars/images.yml

@@ -21,3 +21,6 @@ images:
 
 database_images:
  - "{{ postgresql_container_image }}:{{ postgresql_container_tag }}"
+
+foreman_proxy_images:
+ - "{{ foreman_proxy_container_image }}:{{ foreman_proxy_container_tag }}"

tests/foreman_compute_resources_test.py

@@ -0,0 +1,10 @@
+import pytest
+
+FOREMAN_HOST = 'localhost'
+FOREMAN_PORT = 3000
+
+@pytest.mark.parametrize("compute_resource", ['AzureRm', 'EC2', 'GCE', 'Libvirt', 'Openstack', 'Vmware'])
+def test_foreman_compute_resources(server, compute_resource):
+    hammer = server.run("hammer compute-resource create --help | grep provider")
+    assert hammer.succeeded
+    assert compute_resource in hammer.stdout