From d5887d0ca2f02b98ef4131fe4ecbdd844320abc1 Mon Sep 17 00:00:00 2001 From: Damjan Becirovic Date: Tue, 9 Jun 2026 18:06:53 +0200 Subject: [PATCH 1/5] feat(ppl): blobless + sparse checkout for init job without pre-flight checks The pipeline initialization (compilation) job only needs the pipeline YAML and the Git history (trees/commits, used by `change_in`) to compile the pipeline - not the full repository working tree. For large repositories the full checkout dominates the init job runtime (cloning hundreds of MiB and materializing tens of thousands of files). When there are no pre-flight checks, instruct `checkout` (via the SEMAPHORE_GIT_PARTIAL_CLONE_FILTER and SEMAPHORE_GIT_SPARSE_CHECKOUT_PATHS env vars) to perform a blobless partial clone with a sparse working tree limited to the pipeline directory. When pre-flight checks are configured, their custom commands run after the predefined ones and may rely on the full working tree, so the standard full checkout is kept. Note: this relies on the corresponding toolbox `checkout` support and the spc `commands_file` on-demand fetch. Per-organization feature-flag gating is added in a follow-up commit. Co-Authored-By: Claude Opus 4.8 --- .../stm_handler/compilation/definition.ex | 43 ++++++++++++-- .../compilation/definition_test.exs | 57 +++++++++++++++++++ 2 files changed, 94 insertions(+), 6 deletions(-) diff --git a/plumber/ppl/lib/ppl/ppl_sub_inits/stm_handler/compilation/definition.ex b/plumber/ppl/lib/ppl/ppl_sub_inits/stm_handler/compilation/definition.ex index 219b8c169..5a5dc3209 100644 --- a/plumber/ppl/lib/ppl/ppl_sub_inits/stm_handler/compilation/definition.ex +++ b/plumber/ppl/lib/ppl/ppl_sub_inits/stm_handler/compilation/definition.ex @@ -43,6 +43,8 @@ defmodule Ppl.PplSubInits.STMHandler.Compilation.Definition do end defp form_defintion_jobs(ppl_req = %{request_args: req_args}, pfcs) do + pfc_cmds = pfc_commands(pfcs) + [ %{ "name" => "Compilation", @@ -50,7 +52,7 @@ defmodule Ppl.PplSubInits.STMHandler.Compilation.Definition do "priority" => [%{"value" => @default_priority, "when" => true}], "env_vars" => ppl_env_vars(ppl_req) ++ [sem_yaml_file_path_env_var(req_args)], "secrets" => secrets_definition(pfcs, req_args), - "commands" => commands(pfcs), + "commands" => default_commands(req_args, _optimize_checkout? = pfc_cmds == []) ++ pfc_cmds, "epilogue_always_cmds" => epilogue_always_commands() } ] @@ -109,10 +111,6 @@ defmodule Ppl.PplSubInits.STMHandler.Compilation.Definition do test_commands(mix_env) ++ pfc_commands(pfcs) end - defp commands(pfcs) do - default_commands() ++ pfc_commands(pfcs) - end - defp pfc_commands(%{"organization_pfc" => org_pfc, "project_pfc" => prj_pfc}) when is_map(org_pfc) and is_map(prj_pfc), do: Map.get(org_pfc, "commands", []) ++ Map.get(prj_pfc, "commands", []) @@ -125,9 +123,31 @@ defmodule Ppl.PplSubInits.STMHandler.Compilation.Definition do defp pfc_commands(_ppl_req), do: [] - defp default_commands() do + # When there are no pre-flight checks, the initialization job only needs the + # pipeline YAML and the Git history (trees/commits, used by `change_in`), not + # the full repository working tree. In that case we instruct `checkout` to + # perform a blobless partial clone with a sparse working tree limited to the + # pipeline directory, which avoids downloading/materializing the whole repo. + # + # When pre-flight checks are configured, their custom commands run after the + # predefined ones and may rely on the full working tree being present, so we + # keep the standard full checkout. + @doc false + def default_commands(req_args, _optimize_checkout? = true) do [ ~s[export GIT_LFS_SKIP_SMUDGE=1], + ~s[export SEMAPHORE_GIT_PARTIAL_CLONE_FILTER="blob:none"], + ~s[export SEMAPHORE_GIT_SPARSE_CHECKOUT_PATHS="#{sparse_checkout_path(req_args)}"] + ] ++ checkout_and_compile_commands() + end + + @doc false + def default_commands(_req_args, _optimize_checkout? = false) do + [~s[export GIT_LFS_SKIP_SMUDGE=1]] ++ checkout_and_compile_commands() + end + + defp checkout_and_compile_commands() do + [ ~s[checkout], ~s[export INPUT_FILE="$SEMAPHORE_YAML_FILE_PATH"], ~s[export OUTPUT_FILE="${SEMAPHORE_YAML_FILE_PATH}.output.yml"], @@ -138,6 +158,17 @@ defmodule Ppl.PplSubInits.STMHandler.Compilation.Definition do ] end + # The pipeline lives in the working directory of the YAML file; sparse-checkout + # of that directory keeps the pipeline files available while skipping the rest + # of the repository. Falls back to the repository root when no working + # directory is set (no optimization, but always correct). + defp sparse_checkout_path(req_args) do + case (req_args["working_dir"] || "") |> String.trim() do + "" -> "." + working_dir -> working_dir + end + end + defp epilogue_always_commands() do [ ~s[export BASE_NAME=$SEMAPHORE_PIPELINE_ID-$(basename $INPUT_FILE)], diff --git a/plumber/ppl/test/ppl_sub_inits/stm_handler/compilation/definition_test.exs b/plumber/ppl/test/ppl_sub_inits/stm_handler/compilation/definition_test.exs index a833f2d9a..2ead25336 100644 --- a/plumber/ppl/test/ppl_sub_inits/stm_handler/compilation/definition_test.exs +++ b/plumber/ppl/test/ppl_sub_inits/stm_handler/compilation/definition_test.exs @@ -491,6 +491,56 @@ defmodule Ppl.PplSubInits.STMHandler.Compilation.Definition.Test do end end + describe "default_commands/2" do + test "without pre-flight checks it uses a blobless + sparse checkout of the pipeline dir" do + req_args = %{"working_dir" => ".semaphore", "file_name" => "semaphore.yml"} + + commands = Definition.default_commands(req_args, _optimize_checkout? = true) + + assert "export GIT_LFS_SKIP_SMUDGE=1" in commands + assert ~s[export SEMAPHORE_GIT_PARTIAL_CLONE_FILTER="blob:none"] in commands + assert ~s[export SEMAPHORE_GIT_SPARSE_CHECKOUT_PATHS=".semaphore"] in commands + assert "checkout" in commands + + # the optimization exports must come before checkout + filter_idx = Enum.find_index(commands, &(&1 =~ "SEMAPHORE_GIT_PARTIAL_CLONE_FILTER")) + sparse_idx = Enum.find_index(commands, &(&1 =~ "SEMAPHORE_GIT_SPARSE_CHECKOUT_PATHS")) + checkout_idx = Enum.find_index(commands, &(&1 == "checkout")) + assert filter_idx < checkout_idx + assert sparse_idx < checkout_idx + + assert List.last(commands) == + "spc compile --input $INPUT_FILE --output $OUTPUT_FILE --logs $LOGS_FILE" + end + + test "with pre-flight checks it keeps the standard full checkout" do + req_args = %{"working_dir" => ".semaphore", "file_name" => "semaphore.yml"} + + commands = Definition.default_commands(req_args, _optimize_checkout? = false) + + assert "export GIT_LFS_SKIP_SMUDGE=1" in commands + assert "checkout" in commands + refute Enum.any?(commands, &(&1 =~ "SEMAPHORE_GIT_PARTIAL_CLONE_FILTER")) + refute Enum.any?(commands, &(&1 =~ "SEMAPHORE_GIT_SPARSE_CHECKOUT_PATHS")) + end + + test "falls back to the repository root when the working dir is empty" do + req_args = %{"working_dir" => " ", "file_name" => "semaphore.yml"} + + commands = Definition.default_commands(req_args, _optimize_checkout? = true) + + assert ~s[export SEMAPHORE_GIT_SPARSE_CHECKOUT_PATHS="."] in commands + end + + test "uses a nested working dir as the sparse path" do + req_args = %{"working_dir" => ".semaphore/prod", "file_name" => "deploy.yml"} + + commands = Definition.default_commands(req_args, _optimize_checkout? = true) + + assert ~s[export SEMAPHORE_GIT_SPARSE_CHECKOUT_PATHS=".semaphore/prod"] in commands + end + end + @tag :integration test "when in prod environment => return compilation definition with proper env vars set" do System.put_env("INTERNAL_API_URL_PFC", "localhost:50053") @@ -531,6 +581,13 @@ defmodule Ppl.PplSubInits.STMHandler.Compilation.Definition.Test do yml_path_ev = Enum.find(list, fn map -> map["name"] == "SEMAPHORE_YAML_FILE_PATH" end) assert yml_path_ev["value"] == ".semaphore/semaphore.yml" + # With no pre-flight checks, the init job uses the optimized blobless + + # sparse checkout of the pipeline directory. + commands = Map.get(definition, "jobs") |> Enum.at(0) |> Map.get("commands") + assert ~s[export SEMAPHORE_GIT_PARTIAL_CLONE_FILTER="blob:none"] in commands + assert ~s[export SEMAPHORE_GIT_SPARSE_CHECKOUT_PATHS=".semaphore"] in commands + assert "checkout" in commands + on_exit(fn -> GRPC.Server.stop(PFCServiceMock) end) From 853357b5f32e937df28871a6a5ca0864a6531f3a Mon Sep 17 00:00:00 2001 From: Damjan Becirovic Date: Tue, 9 Jun 2026 19:02:08 +0200 Subject: [PATCH 2/5] feat(ppl): gate init-job optimized checkout behind a FeatureHub flag Wire the FeatureProvider/FeatureHub stack into plumber and gate the blobless + sparse init-job checkout behind the per-organization `sparse_checkout_init_job` feature flag. The optimization now applies only when both hold: there are no pre-flight checks AND the feature is enabled for the organization. The check fails closed (missing org id or unreachable Feature service => standard full checkout). - proto: generate InternalApi.Feature stubs; add INTERNAL_API_LOCAL_PATH support to the proto Makefile so they can be regenerated from a local internal_api checkout, plus a feature.proto generation step. - ppl: add feature_provider dep (pin yaml_elixir ~> 1.3 via override since definition_validator requires 1.x and we only use the FeatureHub provider). - Ppl.FeatureClient: gRPC client for the Feature service (INTERNAL_API_URL_FEATURE). - Ppl.FeatureHubProvider: FeatureProvider.Provider backed by the Feature service. - Ppl.Features: thin, fail-closed wrapper exposing sparse_checkout_init_job_enabled?/1. - Ppl.Application: init FeatureProvider and start the :feature_cache Cachex. - config: FeatureHub provider with CachexCache (no cache in test env). - tests: Feature gRPC mock + coverage for the gate decision and provider path. Co-Authored-By: Claude Opus 4.8 --- plumber/ppl/config/config.exs | 8 + plumber/ppl/config/test.exs | 4 + plumber/ppl/lib/ppl/application.ex | 26 +- plumber/ppl/lib/ppl/feature_client.ex | 100 +++++++ plumber/ppl/lib/ppl/feature_hub_provider.ex | 61 ++++ plumber/ppl/lib/ppl/features.ex | 22 ++ .../stm_handler/compilation/definition.ex | 16 +- plumber/ppl/mix.exs | 6 + plumber/ppl/test/features_test.exs | 35 +++ .../compilation/definition_test.exs | 22 ++ .../ppl/test/support/mocks/feature_server.ex | 33 +++ plumber/proto/Makefile | 54 ++-- plumber/proto/lib/internal_api/feature.pb.ex | 276 ++++++++++++++++++ 13 files changed, 634 insertions(+), 29 deletions(-) create mode 100644 plumber/ppl/lib/ppl/feature_client.ex create mode 100644 plumber/ppl/lib/ppl/feature_hub_provider.ex create mode 100644 plumber/ppl/lib/ppl/features.ex create mode 100644 plumber/ppl/test/features_test.exs create mode 100644 plumber/ppl/test/support/mocks/feature_server.ex create mode 100644 plumber/proto/lib/internal_api/feature.pb.ex diff --git a/plumber/ppl/config/config.exs b/plumber/ppl/config/config.exs index 8987a3a22..bd753a84f 100644 --- a/plumber/ppl/config/config.exs +++ b/plumber/ppl/config/config.exs @@ -31,6 +31,14 @@ config :vmstats, sink: Ppl.VmstatsWatchmanSink, interval: 10_000 +# Feature flags (FeatureHub). The Feature service address is read at call time +# from INTERNAL_API_URL_FEATURE (see Ppl.FeatureClient). Results are cached in +# the :feature_cache Cachex instance started by Ppl.Application. +config :ppl, + feature_provider: + {Ppl.FeatureHubProvider, + [cache: {FeatureProvider.CachexCache, name: :feature_cache, ttl_ms: :timer.minutes(10)}]} + # Mappings to function definitions for functions available in when condition DSL config :when, change_in: {Block.ChangeInResolver, :change_in, [1, 2]} diff --git a/plumber/ppl/config/test.exs b/plumber/ppl/config/test.exs index 9914319b7..766626adf 100644 --- a/plumber/ppl/config/test.exs +++ b/plumber/ppl/config/test.exs @@ -19,6 +19,10 @@ config :ppl, Ppl.Cache.OrganizationSettings, size_limit: 1_000, reclaim_coef: 0.5 +# In tests, use the FeatureHub provider without caching so the gRPC mock is +# always consulted. Tests that need to control the flag stub Ppl.Features. +config :ppl, feature_provider: {Ppl.FeatureHubProvider, []} + # Time to wait before pipeline status is reexamined # -2 means 'do not wait' or take all config :ppl, general_looper_cooling_time_sec: -2 diff --git a/plumber/ppl/lib/ppl/application.ex b/plumber/ppl/lib/ppl/application.ex index dd750f89f..002449662 100644 --- a/plumber/ppl/lib/ppl/application.ex +++ b/plumber/ppl/lib/ppl/application.ex @@ -15,17 +15,31 @@ defmodule Ppl.Application do Application.stop(:watchman) Application.ensure_all_started(:watchman) + init_feature_provider() + # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html # for other strategies and supported options opts = [strategy: :one_for_one, name: Ppl.Supervisor] (children(get_env()) ++ grpc_supervisor(get_env())) |> Supervisor.start_link(opts) end + defp init_feature_provider() do + case Application.get_env(:ppl, :feature_provider) do + nil -> :ok + provider -> FeatureProvider.init(provider) + end + end + def children(:test) do [ {Looper.Publisher.AMQP, amqp_url()}, - Supervisor.child_spec({Ppl.Grpc.InFlightCounter, in_flight_counter_args(:describe)}, id: InFlightCounterDescribe), - Supervisor.child_spec({Ppl.Grpc.InFlightCounter, in_flight_counter_args(:list)}, id: InFlightCounterList), + Supervisor.child_spec({Ppl.Grpc.InFlightCounter, in_flight_counter_args(:describe)}, + id: InFlightCounterDescribe + ), + Supervisor.child_spec({Ppl.Grpc.InFlightCounter, in_flight_counter_args(:list)}, + id: InFlightCounterList + ), + %{id: :feature_cache, start: {Cachex, :start_link, [:feature_cache, []]}}, supervisor(Ppl.Cache, []), supervisor(Ppl.EctoRepo, []) ] @@ -41,12 +55,14 @@ defmodule Ppl.Application do [ Test.Support.Mocks.UserServer, Test.Support.Mocks.PFCServer, - Test.Support.Mocks.OrgServer + Test.Support.Mocks.OrgServer, + Test.Support.Mocks.FeatureServer ] ++ grpc_servers() defp grpc_servers(_), do: grpc_servers() - defp grpc_servers, do: [Ppl.Grpc.Server, Plumber.WorkflowAPI.Server, Ppl.Admin.Server, Ppl.Grpc.HealthCheck] + defp grpc_servers, + do: [Ppl.Grpc.Server, Plumber.WorkflowAPI.Server, Ppl.Admin.Server, Ppl.Grpc.HealthCheck] def children_ do [ @@ -100,7 +116,7 @@ defmodule Ppl.Application do defp in_flight_counter_args(type), do: [type: type, limit: in_flight_counter_limit(type)] defp in_flight_counter_limit(type) do - up_type = type |> Atom.to_string |> String.upcase + up_type = type |> Atom.to_string() |> String.upcase() "IN_FLIGHT_#{up_type}_LIMIT" |> System.get_env() diff --git a/plumber/ppl/lib/ppl/feature_client.ex b/plumber/ppl/lib/ppl/feature_client.ex new file mode 100644 index 000000000..2fea0c2e6 --- /dev/null +++ b/plumber/ppl/lib/ppl/feature_client.ex @@ -0,0 +1,100 @@ +defmodule Ppl.FeatureClient do + @moduledoc """ + gRPC client consuming the Feature (FeatureHub) API. + + Used by `Ppl.FeatureHubProvider` to fetch the features enabled for an + organization. Failures are turned into `{:error, _}` so callers can fail + closed (treat the feature as disabled). + """ + + @watchman_prefix_key "Ppl.FeatureClient" + @url_env_var "INTERNAL_API_URL_FEATURE" + @timeout 3_000 + + alias InternalApi.Feature, as: API + alias API.FeatureService.Stub + require Logger + + @doc """ + Calls the Feature gRPC API to list the features enabled for an organization. + """ + @spec list_organization_features(String.t()) :: + {:ok, [InternalApi.Feature.OrganizationFeature.t()]} + | {:error, :timeout} + | {:error, any()} + def list_organization_features(organization_id) do + result = + Wormhole.capture(__MODULE__, :do_list_organization_features, [organization_id], + stacktrace: true, + timeout: @timeout + ) + + case result do + {:ok, features} -> + {:ok, features} + + {:error, {:timeout, timeout}} -> + log_timeout(organization_id, timeout) + {:error, :timeout} + + {:error, {:shutdown, {reason, _stacktrace}}} -> + log_shutdown(organization_id, reason) + {:error, reason} + end + end + + # + # gRPC connection + # + + @doc false + def do_list_organization_features(organization_id) do + Watchman.benchmark("#{@watchman_prefix_key}.list_organization_features.duration", fn -> + request = API.ListOrganizationFeaturesRequest.new(org_id: organization_id) + + case send_request(request) do + {:ok, response} -> + Watchman.increment("#{@watchman_prefix_key}.list_organization_features.success") + response.organization_features + + {:error, reason} -> + Watchman.increment("#{@watchman_prefix_key}.list_organization_features.failure") + raise reason + end + end) + end + + defp send_request(request) do + url = + System.get_env(@url_env_var) || raise "environment variable #{@url_env_var} was not found" + + with {:ok, channel} <- GRPC.Stub.connect(url) do + Watchman.increment("#{@watchman_prefix_key}.list_organization_features.connect") + + try do + Stub.list_organization_features(channel, request) + after + GRPC.Stub.disconnect(channel) + end + end + end + + # + # Logging functions + # + + defp log_timeout(organization_id, _timeout) do + metadata = log_metadata(organization_id: organization_id) + Logger.error("Ppl.FeatureClient.list_organization_features/1: TIMEOUT #{metadata}") + end + + defp log_shutdown(organization_id, reason) do + metadata = log_metadata(organization_id: organization_id, reason: reason) + Logger.error("Ppl.FeatureClient.list_organization_features/1: SHUTDOWN #{metadata}") + end + + defp log_metadata(metadata) do + formatter = &"#{elem(&1, 0)}=#{inspect(elem(&1, 1))}" + metadata |> Enum.map_join(" ", formatter) + end +end diff --git a/plumber/ppl/lib/ppl/feature_hub_provider.ex b/plumber/ppl/lib/ppl/feature_hub_provider.ex new file mode 100644 index 000000000..86fab5890 --- /dev/null +++ b/plumber/ppl/lib/ppl/feature_hub_provider.ex @@ -0,0 +1,61 @@ +defmodule Ppl.FeatureHubProvider do + @moduledoc """ + `FeatureProvider.Provider` implementation backed by the Feature (FeatureHub) + gRPC API. + + Only organization features are needed in plumber; machines are not used here. + On any error fetching features we return `{:error, _}`, which makes + `FeatureProvider.feature_enabled?/2` fail closed (return `false`). + """ + + use FeatureProvider.Provider + + alias Ppl.FeatureClient + alias InternalApi.Feature.{Availability, OrganizationFeature} + + @impl FeatureProvider.Provider + def provide_features(org_id, _opts \\ []) do + case FeatureClient.list_organization_features(org_id) do + {:ok, organization_features} -> + features = + organization_features + |> Enum.map(&feature_from_grpc/1) + |> Enum.filter(&FeatureProvider.Feature.visible?/1) + + {:ok, features} + + {:error, reason} -> + {:error, reason} + end + end + + @impl FeatureProvider.Provider + def provide_machines(_org_id, _opts \\ []) do + {:ok, []} + end + + defp feature_from_grpc(%OrganizationFeature{feature: feature, availability: availability}) do + %FeatureProvider.Feature{ + name: feature.name, + type: feature.type, + description: feature.description, + quantity: quantity_from_availability(availability), + state: state_from_availability(availability) + } + end + + defp quantity_from_availability(%Availability{quantity: quantity}), do: quantity + defp quantity_from_availability(_), do: 0 + + defp state_from_availability(%Availability{state: state}) do + state + |> Availability.State.key() + |> case do + :ENABLED -> :enabled + :HIDDEN -> :disabled + :ZERO_STATE -> :zero_state + end + end + + defp state_from_availability(_), do: :disabled +end diff --git a/plumber/ppl/lib/ppl/features.ex b/plumber/ppl/lib/ppl/features.ex new file mode 100644 index 000000000..3d897129a --- /dev/null +++ b/plumber/ppl/lib/ppl/features.ex @@ -0,0 +1,22 @@ +defmodule Ppl.Features do + @moduledoc """ + Organization feature-flag checks used by plumber. + + Thin wrapper around `FeatureProvider` so feature names live in one place and + call sites stay readable. All checks fail closed: an empty organization id or + any error reaching the Feature service results in `false`. + """ + + @sparse_checkout_init_job "sparse_checkout_init_job" + + @doc """ + Whether the initialization (compilation) job may use the optimized blobless + + sparse checkout for the given organization. + """ + @spec sparse_checkout_init_job_enabled?(String.t() | nil) :: boolean() + def sparse_checkout_init_job_enabled?(org_id) when is_binary(org_id) and org_id != "" do + FeatureProvider.feature_enabled?(@sparse_checkout_init_job, param: org_id) + end + + def sparse_checkout_init_job_enabled?(_org_id), do: false +end diff --git a/plumber/ppl/lib/ppl/ppl_sub_inits/stm_handler/compilation/definition.ex b/plumber/ppl/lib/ppl/ppl_sub_inits/stm_handler/compilation/definition.ex index 5a5dc3209..8912ad449 100644 --- a/plumber/ppl/lib/ppl/ppl_sub_inits/stm_handler/compilation/definition.ex +++ b/plumber/ppl/lib/ppl/ppl_sub_inits/stm_handler/compilation/definition.ex @@ -6,7 +6,6 @@ defmodule Ppl.PplSubInits.STMHandler.Compilation.Definition do alias Util.ToTuple alias Ppl.DefinitionReviser.BlocksReviser - @default_execution_limit_in_minutes 10 @default_priority 95 @@ -44,6 +43,7 @@ defmodule Ppl.PplSubInits.STMHandler.Compilation.Definition do defp form_defintion_jobs(ppl_req = %{request_args: req_args}, pfcs) do pfc_cmds = pfc_commands(pfcs) + org_id = Map.get(req_args, "organization_id", "") [ %{ @@ -52,12 +52,24 @@ defmodule Ppl.PplSubInits.STMHandler.Compilation.Definition do "priority" => [%{"value" => @default_priority, "when" => true}], "env_vars" => ppl_env_vars(ppl_req) ++ [sem_yaml_file_path_env_var(req_args)], "secrets" => secrets_definition(pfcs, req_args), - "commands" => default_commands(req_args, _optimize_checkout? = pfc_cmds == []) ++ pfc_cmds, + "commands" => + default_commands(req_args, optimize_checkout?(org_id, pfc_cmds)) ++ pfc_cmds, "epilogue_always_cmds" => epilogue_always_commands() } ] end + # The optimized blobless + sparse checkout is used only when both hold: + # * there are no pre-flight checks (their custom commands may need the full + # working tree), and + # * the `sparse_checkout_init_job` feature is enabled for the organization. + # The feature check fails closed, so a missing org id or an unreachable + # Feature service keeps the standard full checkout. + @doc false + def optimize_checkout?(org_id, pfc_cmds) do + pfc_cmds == [] and Ppl.Features.sparse_checkout_init_job_enabled?(org_id) + end + defp agent_definition(pre_flight_checks, settings) when is_map(pre_flight_checks) do pfc_agent = get_in(pre_flight_checks, ["project_pfc", "agent"]) || %{} diff --git a/plumber/ppl/mix.exs b/plumber/ppl/mix.exs index 760c48bea..3d5c95693 100644 --- a/plumber/ppl/mix.exs +++ b/plumber/ppl/mix.exs @@ -68,6 +68,12 @@ defmodule Ppl.Mixfile do {:definition_validator, path: "../definition_validator"}, {:block, path: "../block"}, {:job_matrix, path: "../job_matrix"}, + {:feature_provider, path: "../../feature_provider"}, + # feature_provider requests yaml_elixir >= 2.0, but plumber's + # definition_validator (pipeline YAML validation) pins ~> 1.1. We only use + # feature_provider's FeatureHub provider (not its YamlProvider), so pin the + # existing 1.3 line to keep a single version across the umbrella. + {:yaml_elixir, "~> 1.3", override: true}, {:dialyxir, "~> 0.5", only: [:dev], runtime: false}, {:amqp_client, "~> 3.9.2"}, {:tackle, github: "renderedtext/ex-tackle", tag: "v0.2.1"}, diff --git a/plumber/ppl/test/features_test.exs b/plumber/ppl/test/features_test.exs new file mode 100644 index 000000000..1a656dd81 --- /dev/null +++ b/plumber/ppl/test/features_test.exs @@ -0,0 +1,35 @@ +defmodule Ppl.FeaturesTest do + use ExUnit.Case, async: false + + @url_env_var "INTERNAL_API_URL_FEATURE" + + setup do + original = System.get_env(@url_env_var) + System.put_env(@url_env_var, "localhost:50053") + + on_exit(fn -> + case original do + nil -> System.delete_env(@url_env_var) + value -> System.put_env(@url_env_var, value) + end + end) + + :ok + end + + describe "sparse_checkout_init_job_enabled?/1" do + test "true when the feature is enabled for the org (via the FeatureHub mock)" do + assert Ppl.Features.sparse_checkout_init_job_enabled?("org-123") == true + end + + test "fails closed for a missing org id" do + assert Ppl.Features.sparse_checkout_init_job_enabled?("") == false + assert Ppl.Features.sparse_checkout_init_job_enabled?(nil) == false + end + + test "fails closed when the Feature service is unreachable" do + System.put_env(@url_env_var, "localhost:1") + assert Ppl.Features.sparse_checkout_init_job_enabled?("org-123") == false + end + end +end diff --git a/plumber/ppl/test/ppl_sub_inits/stm_handler/compilation/definition_test.exs b/plumber/ppl/test/ppl_sub_inits/stm_handler/compilation/definition_test.exs index 2ead25336..7547cc020 100644 --- a/plumber/ppl/test/ppl_sub_inits/stm_handler/compilation/definition_test.exs +++ b/plumber/ppl/test/ppl_sub_inits/stm_handler/compilation/definition_test.exs @@ -1,5 +1,6 @@ defmodule Ppl.PplSubInits.STMHandler.Compilation.Definition.Test do use ExUnit.Case, async: false + import Mock alias Ppl.PplSubInits.STMHandler.Compilation.Definition alias Ppl.PplRequests.Model.PplRequestsQueries @@ -541,10 +542,31 @@ defmodule Ppl.PplSubInits.STMHandler.Compilation.Definition.Test do end end + describe "optimize_checkout?/2" do + test "true when no pre-flight checks and the feature is enabled" do + with_mock Ppl.Features, sparse_checkout_init_job_enabled?: fn _ -> true end do + assert Definition.optimize_checkout?("org-1", []) == true + end + end + + test "false when the feature is disabled, even without pre-flight checks" do + with_mock Ppl.Features, sparse_checkout_init_job_enabled?: fn _ -> false end do + assert Definition.optimize_checkout?("org-1", []) == false + end + end + + test "false when pre-flight checks are present, even if the feature is enabled" do + with_mock Ppl.Features, sparse_checkout_init_job_enabled?: fn _ -> true end do + assert Definition.optimize_checkout?("org-1", ["./security_check.sh"]) == false + end + end + end + @tag :integration test "when in prod environment => return compilation definition with proper env vars set" do System.put_env("INTERNAL_API_URL_PFC", "localhost:50053") System.put_env("INTERNAL_API_URL_USER", "localhost:50053") + System.put_env("INTERNAL_API_URL_FEATURE", "localhost:50053") System.put_env("INTERNAL_API_URL_ORGANIZATION", "localhost:50053") not_trimmed_file_name = "semaphore.yml " diff --git a/plumber/ppl/test/support/mocks/feature_server.ex b/plumber/ppl/test/support/mocks/feature_server.ex new file mode 100644 index 000000000..aa492ee1a --- /dev/null +++ b/plumber/ppl/test/support/mocks/feature_server.ex @@ -0,0 +1,33 @@ +defmodule Test.Support.Mocks.FeatureServer do + @moduledoc """ + Test mock for the Feature (FeatureHub) gRPC service. + + Returns the `sparse_checkout_init_job` feature as ENABLED for every + organization, so the prod-path compilation definition test exercises the + optimized checkout branch. + """ + + use GRPC.Server, service: InternalApi.Feature.FeatureService.Service + + alias InternalApi.Feature.{ + ListOrganizationFeaturesResponse, + OrganizationFeature, + Feature, + Availability + } + + def list_organization_features(_request, _stream) do + ListOrganizationFeaturesResponse.new( + organization_features: [ + OrganizationFeature.new( + feature: + Feature.new( + type: "sparse_checkout_init_job", + name: "sparse_checkout_init_job" + ), + availability: Availability.new(state: Availability.State.value(:ENABLED), quantity: 1) + ) + ] + ) + end +end diff --git a/plumber/proto/Makefile b/plumber/proto/Makefile index 91fd5f18e..562d1c107 100644 --- a/plumber/proto/Makefile +++ b/plumber/proto/Makefile @@ -14,12 +14,19 @@ INTERACTIVE_SESSION=\ INTERNAL_API_BRANCH?=master TMP_INTERNAL_REPO_DIR?=/tmp/internal_api +# Set INTERNAL_API_LOCAL_PATH to regenerate the stubs from a local internal_api +# checkout instead of cloning the repository, e.g.: +# make pb.gen INTERNAL_API_LOCAL_PATH=/path/to/internal_api +INTERNAL_API_LOCAL_PATH ?= +INTERNAL_API_SOURCE_DIR=$(if $(INTERNAL_API_LOCAL_PATH),$(INTERNAL_API_LOCAL_PATH),$(TMP_INTERNAL_REPO_DIR)) RELATIVE_INTERNAL_PB_OUTPUT_DIR=lib/internal_api RT_PROTOC_IMG_VSN=1.6.6-3.3.0-0.5.4 pb.clone: +ifeq ($(INTERNAL_API_LOCAL_PATH),) rm -rf $(TMP_INTERNAL_REPO_DIR) git clone git@github.com:renderedtext/internal_api.git $(TMP_INTERNAL_REPO_DIR) && (cd $(TMP_INTERNAL_REPO_DIR) && git checkout $(INTERNAL_API_BRANCH) && cd -) +endif pb.gen: pb.clone ifeq ($(shell whoami), vagrant) @@ -27,29 +34,32 @@ ifeq ($(shell whoami), vagrant) else rm -rf $(RELATIVE_INTERNAL_PB_OUTPUT_DIR) && mkdir -p $(RELATIVE_INTERNAL_PB_OUTPUT_DIR) endif - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/include/google/protobuf/timestamp.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/include/google/rpc/status.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/include/google/rpc/code.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/gofer.switch.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/gofer.dt.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/organization.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/paparazzo.snapshot.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/plumber.pipeline.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/plumber.admin.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/plumber_w_f.workflow.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/repo_proxy.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/repository.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/include/internal_api/response_status.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/include/internal_api/status.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/task.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/repository_integrator.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/user.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/projecthub.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/artifacthub.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/health.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/pre_flight_checks_hub.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/usage.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/include/google/protobuf/timestamp.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/include/google/rpc/status.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/include/google/rpc/code.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/gofer.switch.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/gofer.dt.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/organization.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/paparazzo.snapshot.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/plumber.pipeline.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/plumber.admin.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/plumber_w_f.workflow.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/repo_proxy.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/repository.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/include/internal_api/response_status.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/include/internal_api/status.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/task.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/repository_integrator.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/user.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/projecthub.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/artifacthub.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/health.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/pre_flight_checks_hub.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/usage.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/feature.proto +ifeq ($(INTERNAL_API_LOCAL_PATH),) rm -rf $(TMP_INTERNAL_REPO_DIR) +endif console: docker run $(INTERACTIVE_SESSION) /bin/ash diff --git a/plumber/proto/lib/internal_api/feature.pb.ex b/plumber/proto/lib/internal_api/feature.pb.ex new file mode 100644 index 000000000..880b85d57 --- /dev/null +++ b/plumber/proto/lib/internal_api/feature.pb.ex @@ -0,0 +1,276 @@ +defmodule InternalApi.Feature.ListOrganizationFeaturesRequest do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + org_id: String.t() + } + defstruct [:org_id] + + field(:org_id, 1, type: :string) +end + +defmodule InternalApi.Feature.ListOrganizationFeaturesResponse do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + organization_features: [InternalApi.Feature.OrganizationFeature.t()] + } + defstruct [:organization_features] + + field(:organization_features, 1, repeated: true, type: InternalApi.Feature.OrganizationFeature) +end + +defmodule InternalApi.Feature.OrganizationFeature do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + feature: InternalApi.Feature.Feature.t(), + availability: InternalApi.Feature.Availability.t(), + project_ids: [String.t()], + requester_id: String.t(), + created_at: Google.Protobuf.Timestamp.t(), + updated_at: Google.Protobuf.Timestamp.t() + } + defstruct [:feature, :availability, :project_ids, :requester_id, :created_at, :updated_at] + + field(:feature, 1, type: InternalApi.Feature.Feature) + field(:availability, 2, type: InternalApi.Feature.Availability) + field(:project_ids, 3, repeated: true, type: :string) + field(:requester_id, 5, type: :string) + field(:created_at, 6, type: Google.Protobuf.Timestamp) + field(:updated_at, 7, type: Google.Protobuf.Timestamp) +end + +defmodule InternalApi.Feature.ListFeaturesRequest do + @moduledoc false + use Protobuf, syntax: :proto3 + + defstruct [] +end + +defmodule InternalApi.Feature.ListFeaturesResponse do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + features: [InternalApi.Feature.Feature.t()] + } + defstruct [:features] + + field(:features, 1, repeated: true, type: InternalApi.Feature.Feature) +end + +defmodule InternalApi.Feature.Feature do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + type: String.t(), + availability: InternalApi.Feature.Availability.t(), + name: String.t(), + description: String.t() + } + defstruct [:type, :availability, :name, :description] + + field(:type, 1, type: :string) + field(:availability, 2, type: InternalApi.Feature.Availability) + field(:name, 3, type: :string) + field(:description, 4, type: :string) +end + +defmodule InternalApi.Feature.ListOrganizationMachinesRequest do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + org_id: String.t() + } + defstruct [:org_id] + + field(:org_id, 1, type: :string) +end + +defmodule InternalApi.Feature.ListOrganizationMachinesResponse do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + organization_machines: [InternalApi.Feature.OrganizationMachine.t()], + default_type: String.t() + } + defstruct [:organization_machines, :default_type] + + field(:organization_machines, 1, repeated: true, type: InternalApi.Feature.OrganizationMachine) + field(:default_type, 2, type: :string) +end + +defmodule InternalApi.Feature.OrganizationMachine do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + machine: InternalApi.Feature.Machine.t(), + availability: InternalApi.Feature.Availability.t(), + requester_id: String.t(), + created_at: Google.Protobuf.Timestamp.t(), + updated_at: Google.Protobuf.Timestamp.t() + } + defstruct [:machine, :availability, :requester_id, :created_at, :updated_at] + + field(:machine, 1, type: InternalApi.Feature.Machine) + field(:availability, 2, type: InternalApi.Feature.Availability) + field(:requester_id, 3, type: :string) + field(:created_at, 4, type: Google.Protobuf.Timestamp) + field(:updated_at, 5, type: Google.Protobuf.Timestamp) +end + +defmodule InternalApi.Feature.ListMachinesRequest do + @moduledoc false + use Protobuf, syntax: :proto3 + + defstruct [] +end + +defmodule InternalApi.Feature.ListMachinesResponse do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + machines: [InternalApi.Feature.Machine.t()] + } + defstruct [:machines] + + field(:machines, 1, repeated: true, type: InternalApi.Feature.Machine) +end + +defmodule InternalApi.Feature.Machine do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + type: String.t(), + availability: InternalApi.Feature.Availability.t(), + platform: integer, + vcpu: String.t(), + ram: String.t(), + disk: String.t(), + default_os_image: String.t(), + os_images: [String.t()] + } + defstruct [:type, :availability, :platform, :vcpu, :ram, :disk, :default_os_image, :os_images] + + field(:type, 1, type: :string) + field(:availability, 2, type: InternalApi.Feature.Availability) + field(:platform, 3, type: InternalApi.Feature.Machine.Platform, enum: true) + field(:vcpu, 4, type: :string) + field(:ram, 5, type: :string) + field(:disk, 6, type: :string) + field(:default_os_image, 7, type: :string) + field(:os_images, 8, repeated: true, type: :string) +end + +defmodule InternalApi.Feature.Machine.Platform do + @moduledoc false + use Protobuf, enum: true, syntax: :proto3 + + field(:LINUX, 0) + field(:MAC, 1) +end + +defmodule InternalApi.Feature.Availability do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + state: integer, + quantity: non_neg_integer + } + defstruct [:state, :quantity] + + field(:state, 1, type: InternalApi.Feature.Availability.State, enum: true) + field(:quantity, 2, type: :uint32) +end + +defmodule InternalApi.Feature.Availability.State do + @moduledoc false + use Protobuf, enum: true, syntax: :proto3 + + field(:HIDDEN, 0) + field(:ZERO_STATE, 1) + field(:ENABLED, 2) +end + +defmodule InternalApi.Feature.MachinesChanged do + @moduledoc false + use Protobuf, syntax: :proto3 + + defstruct [] +end + +defmodule InternalApi.Feature.OrganizationMachinesChanged do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + org_id: String.t() + } + defstruct [:org_id] + + field(:org_id, 1, type: :string) +end + +defmodule InternalApi.Feature.FeaturesChanged do + @moduledoc false + use Protobuf, syntax: :proto3 + + defstruct [] +end + +defmodule InternalApi.Feature.OrganizationFeaturesChanged do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + org_id: String.t() + } + defstruct [:org_id] + + field(:org_id, 1, type: :string) +end + +defmodule InternalApi.Feature.FeatureService.Service do + @moduledoc false + use GRPC.Service, name: "InternalApi.Feature.FeatureService" + + rpc( + :ListOrganizationFeatures, + InternalApi.Feature.ListOrganizationFeaturesRequest, + InternalApi.Feature.ListOrganizationFeaturesResponse + ) + + rpc( + :ListFeatures, + InternalApi.Feature.ListFeaturesRequest, + InternalApi.Feature.ListFeaturesResponse + ) + + rpc( + :ListOrganizationMachines, + InternalApi.Feature.ListOrganizationMachinesRequest, + InternalApi.Feature.ListOrganizationMachinesResponse + ) + + rpc( + :ListMachines, + InternalApi.Feature.ListMachinesRequest, + InternalApi.Feature.ListMachinesResponse + ) +end + +defmodule InternalApi.Feature.FeatureService.Stub do + @moduledoc false + use GRPC.Stub, service: InternalApi.Feature.FeatureService.Service +end From 34a247b1ab790b6dc609e8824023984537d6a988 Mon Sep 17 00:00:00 2001 From: Damjan Becirovic Date: Wed, 10 Jun 2026 13:59:25 +0200 Subject: [PATCH 3/5] fix(ppl): build Docker image from repo-root context to include feature_provider feature_provider lives at the repository root, outside the plumber/ tree, so it was not reachable from ppl's previous Docker build context (plumber/), breaking `mix deps.get` in the image build. Move ppl's build context to the repository root (matching front/zebra/secrethub) and prefix the Dockerfile COPY paths with plumber/, plus COPY feature_provider into the image. Update the Makefile build path and docker-compose context/dockerfile accordingly. Validated locally by building both the dev (mix compile) and prod (mix release) image targets. Co-Authored-By: Claude Opus 4.8 --- plumber/ppl/Dockerfile | 68 ++++++++++++++++++---------------- plumber/ppl/Makefile | 5 ++- plumber/ppl/docker-compose.yml | 4 +- 3 files changed, 42 insertions(+), 35 deletions(-) diff --git a/plumber/ppl/Dockerfile b/plumber/ppl/Dockerfile index ca5d30251..e7e2d141e 100644 --- a/plumber/ppl/Dockerfile +++ b/plumber/ppl/Dockerfile @@ -26,51 +26,55 @@ RUN mix local.hex --force --if-missing && \ mix local.rebar --force --if-missing # install mix dependencies -COPY ppl/mix.exs ppl/mix.lock ./ -COPY block/mix.exs block/mix.lock ../block/ -COPY definition_validator/mix.exs definition_validator/mix.lock ../definition_validator/ -COPY gofer_client/mix.exs gofer_client/mix.lock ../gofer_client/ -COPY job_matrix/mix.exs job_matrix/mix.lock ../job_matrix/ -COPY looper/mix.exs looper/mix.lock ../looper/ -COPY proto/mix.exs proto/mix.lock ../proto/ -COPY spec/mix.exs spec/mix.lock ../spec/ +# NOTE: the build context is the repository root (see DOCKER_BUILD_PATH in the +# Makefile) so that feature_provider, which lives outside the plumber/ tree, +# can be copied into the image. +COPY plumber/ppl/mix.exs plumber/ppl/mix.lock ./ +COPY plumber/block/mix.exs plumber/block/mix.lock ../block/ +COPY plumber/definition_validator/mix.exs plumber/definition_validator/mix.lock ../definition_validator/ +COPY plumber/gofer_client/mix.exs plumber/gofer_client/mix.lock ../gofer_client/ +COPY plumber/job_matrix/mix.exs plumber/job_matrix/mix.lock ../job_matrix/ +COPY plumber/looper/mix.exs plumber/looper/mix.lock ../looper/ +COPY plumber/proto/mix.exs plumber/proto/mix.lock ../proto/ +COPY plumber/spec/mix.exs plumber/spec/mix.lock ../spec/ +COPY feature_provider ../feature_provider RUN mix deps.get --only $MIX_ENV RUN mkdir config # copy compile-time config files before we compile dependencies # to ensure any relevant config change will trigger the dependencies # to be re-compiled. -COPY ppl/config/config.exs ppl/config/${MIX_ENV}.exs config/ -COPY block/config/config.exs block/config/${MIX_ENV}.exs ../block/config/ -COPY definition_validator/config/config.exs ../definition_validator/config/ -COPY gofer_client/config/config.exs ../gofer_client/config/ -COPY job_matrix/config/config.exs ../job_matrix/config/ -COPY looper/config/config.exs ../looper/config/ -COPY proto/config/config.exs ../proto/config/ -COPY spec/config/config.exs ../spec/config/ +COPY plumber/ppl/config/config.exs plumber/ppl/config/${MIX_ENV}.exs config/ +COPY plumber/block/config/config.exs plumber/block/config/${MIX_ENV}.exs ../block/config/ +COPY plumber/definition_validator/config/config.exs ../definition_validator/config/ +COPY plumber/gofer_client/config/config.exs ../gofer_client/config/ +COPY plumber/job_matrix/config/config.exs ../job_matrix/config/ +COPY plumber/looper/config/config.exs ../looper/config/ +COPY plumber/proto/config/config.exs ../proto/config/ +COPY plumber/spec/config/config.exs ../spec/config/ RUN mix deps.compile # copy the rest of the config files -COPY ppl/config/ config/ +COPY plumber/ppl/config/ config/ # Compile the release -COPY ppl/lib lib -COPY ppl/priv/ecto_repo/migrations priv/ecto_repo/migrations -COPY block/lib ../block/lib -COPY block/priv/ecto_repo/migrations ../block/priv/ecto_repo/migrations -COPY block/priv/repos ../block/priv/repos -COPY definition_validator/lib ../definition_validator/lib -COPY gofer_client/lib ../gofer_client/lib -COPY job_matrix/lib ../job_matrix/lib -COPY looper/lib ../looper/lib -COPY proto/lib ../proto/lib -COPY spec/lib ../spec/lib -COPY spec/priv ../spec/priv +COPY plumber/ppl/lib lib +COPY plumber/ppl/priv/ecto_repo/migrations priv/ecto_repo/migrations +COPY plumber/block/lib ../block/lib +COPY plumber/block/priv/ecto_repo/migrations ../block/priv/ecto_repo/migrations +COPY plumber/block/priv/repos ../block/priv/repos +COPY plumber/definition_validator/lib ../definition_validator/lib +COPY plumber/gofer_client/lib ../gofer_client/lib +COPY plumber/job_matrix/lib ../job_matrix/lib +COPY plumber/looper/lib ../looper/lib +COPY plumber/proto/lib ../proto/lib +COPY plumber/spec/lib ../spec/lib +COPY plumber/spec/priv ../spec/priv FROM base AS dev -COPY ppl/.formatter.exs .formatter.exs -COPY ppl/.credo.exs .credo.exs -COPY ppl/test test +COPY plumber/ppl/.formatter.exs .formatter.exs +COPY plumber/ppl/.credo.exs .credo.exs +COPY plumber/ppl/test test RUN mix compile diff --git a/plumber/ppl/Makefile b/plumber/ppl/Makefile index d4f10366c..80ae7e8c3 100644 --- a/plumber/ppl/Makefile +++ b/plumber/ppl/Makefile @@ -2,7 +2,10 @@ export MIX_ENV?=dev include ../../Makefile -DOCKER_BUILD_PATH=.. +# Build context is the repository root (two levels up from plumber/ppl) so the +# Docker image can include feature_provider, which lives outside the plumber/ +# tree. The Dockerfile COPY paths are prefixed with plumber/ accordingly. +DOCKER_BUILD_PATH=../.. EX_CATCH_WARRNINGS_FLAG= APP_NAME=ppl diff --git a/plumber/ppl/docker-compose.yml b/plumber/ppl/docker-compose.yml index 4fb0b7267..eeedda8cf 100644 --- a/plumber/ppl/docker-compose.yml +++ b/plumber/ppl/docker-compose.yml @@ -4,10 +4,10 @@ services: app: image: ${IMAGE:-ppl}:${TAG:-latest} build: - context: .. + context: ../.. cache_from: - "${IMAGE:-ppl}:${IMAGE_TAG:-latest}" - dockerfile: ppl/Dockerfile + dockerfile: plumber/ppl/Dockerfile target: ${DOCKER_BUILD_TARGET:-dev} args: - BUILD_ENV=dev From 2cc4ca1a91a2f4d167519ebf1e9e4e3995f4a27c Mon Sep 17 00:00:00 2001 From: Damjan Becirovic Date: Fri, 12 Jun 2026 13:46:40 +0200 Subject: [PATCH 4/5] test(ppl): make global_job_config env restore nil-safe The integration test restored INTERNAL_API_URL_ARTIFACTHUB / _PROJECT in on_exit via System.put_env/2, which raises when the previous value was nil (the vars are not set globally; they depend on test ordering/sharding). Guard the restore so a nil previous value deletes the env var instead of crashing. Surfaced after a new test file shifted integration sharding. Co-Authored-By: Claude Opus 4.8 --- .../definition_reviser/global_job_config_test.exs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/plumber/ppl/test/definition_reviser/global_job_config_test.exs b/plumber/ppl/test/definition_reviser/global_job_config_test.exs index faac99258..909a11aa5 100644 --- a/plumber/ppl/test/definition_reviser/global_job_config_test.exs +++ b/plumber/ppl/test/definition_reviser/global_job_config_test.exs @@ -31,8 +31,16 @@ defmodule Ppl.DefinitionReviser.BlocksReviser.GlobalJobConfig.Test do Test.Support.GrpcServerHelper.setup_service_url(@url_env_name_2, project_port) on_exit(fn -> - System.put_env(@url_env_name, old_artifact_url) - System.put_env(@url_env_name_2, old_project_url) + # The env vars may have been unset before this test ran (depends on test + # ordering/sharding), so guard against restoring a nil value, which + # System.put_env/2 rejects. + if old_artifact_url, + do: System.put_env(@url_env_name, old_artifact_url), + else: System.delete_env(@url_env_name) + + if old_project_url, + do: System.put_env(@url_env_name_2, old_project_url), + else: System.delete_env(@url_env_name_2) end) Test.Helpers.truncate_db() From da8f121758fe8af4122dd3e1cad4f4cba7883bc9 Mon Sep 17 00:00:00 2001 From: Damjan Becirovic Date: Tue, 16 Jun 2026 13:43:54 +0200 Subject: [PATCH 5/5] fix(ppl): bound and cache the init-job feature check The feature flag check runs in the pipeline initialization hot path (the compile task is built and awaited inside a deadline-bounded looper step). Harden it so Feature service availability cannot couple into init latency: - Ppl.FeatureClient: add a 1s per-call gRPC deadline and tighten the Wormhole backstop to 1.5s, so the call fails closed fast instead of blocking the init path for seconds when feature-hub is slow/unreachable. - Ppl.Features: memoize the (fail-closed) boolean result for 30s in the :feature_cache. FeatureProvider only caches successful responses, so without this a feature-hub blip would make every init pay the timeout. Co-Authored-By: Claude Opus 4.8 --- plumber/ppl/lib/ppl/feature_client.ex | 9 +++++-- plumber/ppl/lib/ppl/features.ex | 36 ++++++++++++++++++++++++++- plumber/ppl/test/features_test.exs | 5 ++++ 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/plumber/ppl/lib/ppl/feature_client.ex b/plumber/ppl/lib/ppl/feature_client.ex index 2fea0c2e6..2a470d498 100644 --- a/plumber/ppl/lib/ppl/feature_client.ex +++ b/plumber/ppl/lib/ppl/feature_client.ex @@ -9,7 +9,12 @@ defmodule Ppl.FeatureClient do @watchman_prefix_key "Ppl.FeatureClient" @url_env_var "INTERNAL_API_URL_FEATURE" - @timeout 3_000 + # This call sits in the pipeline initialization hot path (the compile task is + # built and awaited inside a deadline-bounded looper step), so it must be + # tightly bounded and fail fast. @grpc_timeout is the per-call gRPC deadline; + # @timeout is a slightly larger Wormhole backstop covering connect hangs. + @grpc_timeout 1_000 + @timeout 1_500 alias InternalApi.Feature, as: API alias API.FeatureService.Stub @@ -72,7 +77,7 @@ defmodule Ppl.FeatureClient do Watchman.increment("#{@watchman_prefix_key}.list_organization_features.connect") try do - Stub.list_organization_features(channel, request) + Stub.list_organization_features(channel, request, timeout: @grpc_timeout) after GRPC.Stub.disconnect(channel) end diff --git a/plumber/ppl/lib/ppl/features.ex b/plumber/ppl/lib/ppl/features.ex index 3d897129a..43931effb 100644 --- a/plumber/ppl/lib/ppl/features.ex +++ b/plumber/ppl/lib/ppl/features.ex @@ -5,18 +5,52 @@ defmodule Ppl.Features do Thin wrapper around `FeatureProvider` so feature names live in one place and call sites stay readable. All checks fail closed: an empty organization id or any error reaching the Feature service results in `false`. + + The boolean result is memoized for a short time in the `:feature_cache` Cachex + instance. `FeatureProvider` only caches successful provider responses, so + without this a `feature-hub` outage would make every pipeline initialization + pay the gRPC timeout. Caching the (fail-closed) result briefly keeps a blip + from repeatedly stalling the init path. """ + require Logger + @sparse_checkout_init_job "sparse_checkout_init_job" + @cache :feature_cache + @result_ttl_ms :timer.seconds(30) + @doc """ Whether the initialization (compilation) job may use the optimized blobless + sparse checkout for the given organization. """ @spec sparse_checkout_init_job_enabled?(String.t() | nil) :: boolean() def sparse_checkout_init_job_enabled?(org_id) when is_binary(org_id) and org_id != "" do - FeatureProvider.feature_enabled?(@sparse_checkout_init_job, param: org_id) + cached_result(@sparse_checkout_init_job, org_id, fn -> + FeatureProvider.feature_enabled?(@sparse_checkout_init_job, param: org_id) + end) end def sparse_checkout_init_job_enabled?(_org_id), do: false + + # Memoize the boolean result (including the fail-closed false) for a short TTL. + # On any cache error we fall back to evaluating directly, so caching can never + # make the check less available than the bare provider call. + defp cached_result(feature, org_id, fun) do + key = {:ppl_features, feature, org_id} + + case Cachex.get(@cache, key) do + {:ok, nil} -> + result = fun.() + Cachex.put(@cache, key, result, ttl: @result_ttl_ms) + result + + {:ok, cached} -> + cached + + other -> + Logger.warning("Ppl.Features cache read failed for #{inspect(key)}: #{inspect(other)}") + fun.() + end + end end diff --git a/plumber/ppl/test/features_test.exs b/plumber/ppl/test/features_test.exs index 1a656dd81..8845b8cc1 100644 --- a/plumber/ppl/test/features_test.exs +++ b/plumber/ppl/test/features_test.exs @@ -4,6 +4,11 @@ defmodule Ppl.FeaturesTest do @url_env_var "INTERNAL_API_URL_FEATURE" setup do + # The result is memoized in :feature_cache; clear it so each test starts + # fresh (e.g. the "unreachable" case must not see a cached value from the + # "enabled" case for the same org id). + Cachex.clear(:feature_cache) + original = System.get_env(@url_env_var) System.put_env(@url_env_var, "localhost:50053")