diff --git a/plumber/ppl/Dockerfile b/plumber/ppl/Dockerfile index ca5d30251..e7e2d141e 100644 --- a/plumber/ppl/Dockerfile +++ b/plumber/ppl/Dockerfile @@ -26,51 +26,55 @@ RUN mix local.hex --force --if-missing && \ mix local.rebar --force --if-missing # install mix dependencies -COPY ppl/mix.exs ppl/mix.lock ./ -COPY block/mix.exs block/mix.lock ../block/ -COPY definition_validator/mix.exs definition_validator/mix.lock ../definition_validator/ -COPY gofer_client/mix.exs gofer_client/mix.lock ../gofer_client/ -COPY job_matrix/mix.exs job_matrix/mix.lock ../job_matrix/ -COPY looper/mix.exs looper/mix.lock ../looper/ -COPY proto/mix.exs proto/mix.lock ../proto/ -COPY spec/mix.exs spec/mix.lock ../spec/ +# NOTE: the build context is the repository root (see DOCKER_BUILD_PATH in the +# Makefile) so that feature_provider, which lives outside the plumber/ tree, +# can be copied into the image. +COPY plumber/ppl/mix.exs plumber/ppl/mix.lock ./ +COPY plumber/block/mix.exs plumber/block/mix.lock ../block/ +COPY plumber/definition_validator/mix.exs plumber/definition_validator/mix.lock ../definition_validator/ +COPY plumber/gofer_client/mix.exs plumber/gofer_client/mix.lock ../gofer_client/ +COPY plumber/job_matrix/mix.exs plumber/job_matrix/mix.lock ../job_matrix/ +COPY plumber/looper/mix.exs plumber/looper/mix.lock ../looper/ +COPY plumber/proto/mix.exs plumber/proto/mix.lock ../proto/ +COPY plumber/spec/mix.exs plumber/spec/mix.lock ../spec/ +COPY feature_provider ../feature_provider RUN mix deps.get --only $MIX_ENV RUN mkdir config # copy compile-time config files before we compile dependencies # to ensure any relevant config change will trigger the dependencies # to be re-compiled. -COPY ppl/config/config.exs ppl/config/${MIX_ENV}.exs config/ -COPY block/config/config.exs block/config/${MIX_ENV}.exs ../block/config/ -COPY definition_validator/config/config.exs ../definition_validator/config/ -COPY gofer_client/config/config.exs ../gofer_client/config/ -COPY job_matrix/config/config.exs ../job_matrix/config/ -COPY looper/config/config.exs ../looper/config/ -COPY proto/config/config.exs ../proto/config/ -COPY spec/config/config.exs ../spec/config/ +COPY plumber/ppl/config/config.exs plumber/ppl/config/${MIX_ENV}.exs config/ +COPY plumber/block/config/config.exs plumber/block/config/${MIX_ENV}.exs ../block/config/ +COPY plumber/definition_validator/config/config.exs ../definition_validator/config/ +COPY plumber/gofer_client/config/config.exs ../gofer_client/config/ +COPY plumber/job_matrix/config/config.exs ../job_matrix/config/ +COPY plumber/looper/config/config.exs ../looper/config/ +COPY plumber/proto/config/config.exs ../proto/config/ +COPY plumber/spec/config/config.exs ../spec/config/ RUN mix deps.compile # copy the rest of the config files -COPY ppl/config/ config/ +COPY plumber/ppl/config/ config/ # Compile the release -COPY ppl/lib lib -COPY ppl/priv/ecto_repo/migrations priv/ecto_repo/migrations -COPY block/lib ../block/lib -COPY block/priv/ecto_repo/migrations ../block/priv/ecto_repo/migrations -COPY block/priv/repos ../block/priv/repos -COPY definition_validator/lib ../definition_validator/lib -COPY gofer_client/lib ../gofer_client/lib -COPY job_matrix/lib ../job_matrix/lib -COPY looper/lib ../looper/lib -COPY proto/lib ../proto/lib -COPY spec/lib ../spec/lib -COPY spec/priv ../spec/priv +COPY plumber/ppl/lib lib +COPY plumber/ppl/priv/ecto_repo/migrations priv/ecto_repo/migrations +COPY plumber/block/lib ../block/lib +COPY plumber/block/priv/ecto_repo/migrations ../block/priv/ecto_repo/migrations +COPY plumber/block/priv/repos ../block/priv/repos +COPY plumber/definition_validator/lib ../definition_validator/lib +COPY plumber/gofer_client/lib ../gofer_client/lib +COPY plumber/job_matrix/lib ../job_matrix/lib +COPY plumber/looper/lib ../looper/lib +COPY plumber/proto/lib ../proto/lib +COPY plumber/spec/lib ../spec/lib +COPY plumber/spec/priv ../spec/priv FROM base AS dev -COPY ppl/.formatter.exs .formatter.exs -COPY ppl/.credo.exs .credo.exs -COPY ppl/test test +COPY plumber/ppl/.formatter.exs .formatter.exs +COPY plumber/ppl/.credo.exs .credo.exs +COPY plumber/ppl/test test RUN mix compile diff --git a/plumber/ppl/Makefile b/plumber/ppl/Makefile index d4f10366c..80ae7e8c3 100644 --- a/plumber/ppl/Makefile +++ b/plumber/ppl/Makefile @@ -2,7 +2,10 @@ export MIX_ENV?=dev include ../../Makefile -DOCKER_BUILD_PATH=.. +# Build context is the repository root (two levels up from plumber/ppl) so the +# Docker image can include feature_provider, which lives outside the plumber/ +# tree. The Dockerfile COPY paths are prefixed with plumber/ accordingly. +DOCKER_BUILD_PATH=../.. EX_CATCH_WARRNINGS_FLAG= APP_NAME=ppl diff --git a/plumber/ppl/config/config.exs b/plumber/ppl/config/config.exs index 8987a3a22..bd753a84f 100644 --- a/plumber/ppl/config/config.exs +++ b/plumber/ppl/config/config.exs @@ -31,6 +31,14 @@ config :vmstats, sink: Ppl.VmstatsWatchmanSink, interval: 10_000 +# Feature flags (FeatureHub). The Feature service address is read at call time +# from INTERNAL_API_URL_FEATURE (see Ppl.FeatureClient). Results are cached in +# the :feature_cache Cachex instance started by Ppl.Application. +config :ppl, + feature_provider: + {Ppl.FeatureHubProvider, + [cache: {FeatureProvider.CachexCache, name: :feature_cache, ttl_ms: :timer.minutes(10)}]} + # Mappings to function definitions for functions available in when condition DSL config :when, change_in: {Block.ChangeInResolver, :change_in, [1, 2]} diff --git a/plumber/ppl/config/test.exs b/plumber/ppl/config/test.exs index 9914319b7..766626adf 100644 --- a/plumber/ppl/config/test.exs +++ b/plumber/ppl/config/test.exs @@ -19,6 +19,10 @@ config :ppl, Ppl.Cache.OrganizationSettings, size_limit: 1_000, reclaim_coef: 0.5 +# In tests, use the FeatureHub provider without caching so the gRPC mock is +# always consulted. Tests that need to control the flag stub Ppl.Features. +config :ppl, feature_provider: {Ppl.FeatureHubProvider, []} + # Time to wait before pipeline status is reexamined # -2 means 'do not wait' or take all config :ppl, general_looper_cooling_time_sec: -2 diff --git a/plumber/ppl/docker-compose.yml b/plumber/ppl/docker-compose.yml index 4fb0b7267..eeedda8cf 100644 --- a/plumber/ppl/docker-compose.yml +++ b/plumber/ppl/docker-compose.yml @@ -4,10 +4,10 @@ services: app: image: ${IMAGE:-ppl}:${TAG:-latest} build: - context: .. + context: ../.. cache_from: - "${IMAGE:-ppl}:${IMAGE_TAG:-latest}" - dockerfile: ppl/Dockerfile + dockerfile: plumber/ppl/Dockerfile target: ${DOCKER_BUILD_TARGET:-dev} args: - BUILD_ENV=dev diff --git a/plumber/ppl/lib/ppl/application.ex b/plumber/ppl/lib/ppl/application.ex index dd750f89f..002449662 100644 --- a/plumber/ppl/lib/ppl/application.ex +++ b/plumber/ppl/lib/ppl/application.ex @@ -15,17 +15,31 @@ defmodule Ppl.Application do Application.stop(:watchman) Application.ensure_all_started(:watchman) + init_feature_provider() + # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html # for other strategies and supported options opts = [strategy: :one_for_one, name: Ppl.Supervisor] (children(get_env()) ++ grpc_supervisor(get_env())) |> Supervisor.start_link(opts) end + defp init_feature_provider() do + case Application.get_env(:ppl, :feature_provider) do + nil -> :ok + provider -> FeatureProvider.init(provider) + end + end + def children(:test) do [ {Looper.Publisher.AMQP, amqp_url()}, - Supervisor.child_spec({Ppl.Grpc.InFlightCounter, in_flight_counter_args(:describe)}, id: InFlightCounterDescribe), - Supervisor.child_spec({Ppl.Grpc.InFlightCounter, in_flight_counter_args(:list)}, id: InFlightCounterList), + Supervisor.child_spec({Ppl.Grpc.InFlightCounter, in_flight_counter_args(:describe)}, + id: InFlightCounterDescribe + ), + Supervisor.child_spec({Ppl.Grpc.InFlightCounter, in_flight_counter_args(:list)}, + id: InFlightCounterList + ), + %{id: :feature_cache, start: {Cachex, :start_link, [:feature_cache, []]}}, supervisor(Ppl.Cache, []), supervisor(Ppl.EctoRepo, []) ] @@ -41,12 +55,14 @@ defmodule Ppl.Application do [ Test.Support.Mocks.UserServer, Test.Support.Mocks.PFCServer, - Test.Support.Mocks.OrgServer + Test.Support.Mocks.OrgServer, + Test.Support.Mocks.FeatureServer ] ++ grpc_servers() defp grpc_servers(_), do: grpc_servers() - defp grpc_servers, do: [Ppl.Grpc.Server, Plumber.WorkflowAPI.Server, Ppl.Admin.Server, Ppl.Grpc.HealthCheck] + defp grpc_servers, + do: [Ppl.Grpc.Server, Plumber.WorkflowAPI.Server, Ppl.Admin.Server, Ppl.Grpc.HealthCheck] def children_ do [ @@ -100,7 +116,7 @@ defmodule Ppl.Application do defp in_flight_counter_args(type), do: [type: type, limit: in_flight_counter_limit(type)] defp in_flight_counter_limit(type) do - up_type = type |> Atom.to_string |> String.upcase + up_type = type |> Atom.to_string() |> String.upcase() "IN_FLIGHT_#{up_type}_LIMIT" |> System.get_env() diff --git a/plumber/ppl/lib/ppl/feature_client.ex b/plumber/ppl/lib/ppl/feature_client.ex new file mode 100644 index 000000000..2a470d498 --- /dev/null +++ b/plumber/ppl/lib/ppl/feature_client.ex @@ -0,0 +1,105 @@ +defmodule Ppl.FeatureClient do + @moduledoc """ + gRPC client consuming the Feature (FeatureHub) API. + + Used by `Ppl.FeatureHubProvider` to fetch the features enabled for an + organization. Failures are turned into `{:error, _}` so callers can fail + closed (treat the feature as disabled). + """ + + @watchman_prefix_key "Ppl.FeatureClient" + @url_env_var "INTERNAL_API_URL_FEATURE" + # This call sits in the pipeline initialization hot path (the compile task is + # built and awaited inside a deadline-bounded looper step), so it must be + # tightly bounded and fail fast. @grpc_timeout is the per-call gRPC deadline; + # @timeout is a slightly larger Wormhole backstop covering connect hangs. + @grpc_timeout 1_000 + @timeout 1_500 + + alias InternalApi.Feature, as: API + alias API.FeatureService.Stub + require Logger + + @doc """ + Calls the Feature gRPC API to list the features enabled for an organization. + """ + @spec list_organization_features(String.t()) :: + {:ok, [InternalApi.Feature.OrganizationFeature.t()]} + | {:error, :timeout} + | {:error, any()} + def list_organization_features(organization_id) do + result = + Wormhole.capture(__MODULE__, :do_list_organization_features, [organization_id], + stacktrace: true, + timeout: @timeout + ) + + case result do + {:ok, features} -> + {:ok, features} + + {:error, {:timeout, timeout}} -> + log_timeout(organization_id, timeout) + {:error, :timeout} + + {:error, {:shutdown, {reason, _stacktrace}}} -> + log_shutdown(organization_id, reason) + {:error, reason} + end + end + + # + # gRPC connection + # + + @doc false + def do_list_organization_features(organization_id) do + Watchman.benchmark("#{@watchman_prefix_key}.list_organization_features.duration", fn -> + request = API.ListOrganizationFeaturesRequest.new(org_id: organization_id) + + case send_request(request) do + {:ok, response} -> + Watchman.increment("#{@watchman_prefix_key}.list_organization_features.success") + response.organization_features + + {:error, reason} -> + Watchman.increment("#{@watchman_prefix_key}.list_organization_features.failure") + raise reason + end + end) + end + + defp send_request(request) do + url = + System.get_env(@url_env_var) || raise "environment variable #{@url_env_var} was not found" + + with {:ok, channel} <- GRPC.Stub.connect(url) do + Watchman.increment("#{@watchman_prefix_key}.list_organization_features.connect") + + try do + Stub.list_organization_features(channel, request, timeout: @grpc_timeout) + after + GRPC.Stub.disconnect(channel) + end + end + end + + # + # Logging functions + # + + defp log_timeout(organization_id, _timeout) do + metadata = log_metadata(organization_id: organization_id) + Logger.error("Ppl.FeatureClient.list_organization_features/1: TIMEOUT #{metadata}") + end + + defp log_shutdown(organization_id, reason) do + metadata = log_metadata(organization_id: organization_id, reason: reason) + Logger.error("Ppl.FeatureClient.list_organization_features/1: SHUTDOWN #{metadata}") + end + + defp log_metadata(metadata) do + formatter = &"#{elem(&1, 0)}=#{inspect(elem(&1, 1))}" + metadata |> Enum.map_join(" ", formatter) + end +end diff --git a/plumber/ppl/lib/ppl/feature_hub_provider.ex b/plumber/ppl/lib/ppl/feature_hub_provider.ex new file mode 100644 index 000000000..86fab5890 --- /dev/null +++ b/plumber/ppl/lib/ppl/feature_hub_provider.ex @@ -0,0 +1,61 @@ +defmodule Ppl.FeatureHubProvider do + @moduledoc """ + `FeatureProvider.Provider` implementation backed by the Feature (FeatureHub) + gRPC API. + + Only organization features are needed in plumber; machines are not used here. + On any error fetching features we return `{:error, _}`, which makes + `FeatureProvider.feature_enabled?/2` fail closed (return `false`). + """ + + use FeatureProvider.Provider + + alias Ppl.FeatureClient + alias InternalApi.Feature.{Availability, OrganizationFeature} + + @impl FeatureProvider.Provider + def provide_features(org_id, _opts \\ []) do + case FeatureClient.list_organization_features(org_id) do + {:ok, organization_features} -> + features = + organization_features + |> Enum.map(&feature_from_grpc/1) + |> Enum.filter(&FeatureProvider.Feature.visible?/1) + + {:ok, features} + + {:error, reason} -> + {:error, reason} + end + end + + @impl FeatureProvider.Provider + def provide_machines(_org_id, _opts \\ []) do + {:ok, []} + end + + defp feature_from_grpc(%OrganizationFeature{feature: feature, availability: availability}) do + %FeatureProvider.Feature{ + name: feature.name, + type: feature.type, + description: feature.description, + quantity: quantity_from_availability(availability), + state: state_from_availability(availability) + } + end + + defp quantity_from_availability(%Availability{quantity: quantity}), do: quantity + defp quantity_from_availability(_), do: 0 + + defp state_from_availability(%Availability{state: state}) do + state + |> Availability.State.key() + |> case do + :ENABLED -> :enabled + :HIDDEN -> :disabled + :ZERO_STATE -> :zero_state + end + end + + defp state_from_availability(_), do: :disabled +end diff --git a/plumber/ppl/lib/ppl/features.ex b/plumber/ppl/lib/ppl/features.ex new file mode 100644 index 000000000..43931effb --- /dev/null +++ b/plumber/ppl/lib/ppl/features.ex @@ -0,0 +1,56 @@ +defmodule Ppl.Features do + @moduledoc """ + Organization feature-flag checks used by plumber. + + Thin wrapper around `FeatureProvider` so feature names live in one place and + call sites stay readable. All checks fail closed: an empty organization id or + any error reaching the Feature service results in `false`. + + The boolean result is memoized for a short time in the `:feature_cache` Cachex + instance. `FeatureProvider` only caches successful provider responses, so + without this a `feature-hub` outage would make every pipeline initialization + pay the gRPC timeout. Caching the (fail-closed) result briefly keeps a blip + from repeatedly stalling the init path. + """ + + require Logger + + @sparse_checkout_init_job "sparse_checkout_init_job" + + @cache :feature_cache + @result_ttl_ms :timer.seconds(30) + + @doc """ + Whether the initialization (compilation) job may use the optimized blobless + + sparse checkout for the given organization. + """ + @spec sparse_checkout_init_job_enabled?(String.t() | nil) :: boolean() + def sparse_checkout_init_job_enabled?(org_id) when is_binary(org_id) and org_id != "" do + cached_result(@sparse_checkout_init_job, org_id, fn -> + FeatureProvider.feature_enabled?(@sparse_checkout_init_job, param: org_id) + end) + end + + def sparse_checkout_init_job_enabled?(_org_id), do: false + + # Memoize the boolean result (including the fail-closed false) for a short TTL. + # On any cache error we fall back to evaluating directly, so caching can never + # make the check less available than the bare provider call. + defp cached_result(feature, org_id, fun) do + key = {:ppl_features, feature, org_id} + + case Cachex.get(@cache, key) do + {:ok, nil} -> + result = fun.() + Cachex.put(@cache, key, result, ttl: @result_ttl_ms) + result + + {:ok, cached} -> + cached + + other -> + Logger.warning("Ppl.Features cache read failed for #{inspect(key)}: #{inspect(other)}") + fun.() + end + end +end diff --git a/plumber/ppl/lib/ppl/ppl_sub_inits/stm_handler/compilation/definition.ex b/plumber/ppl/lib/ppl/ppl_sub_inits/stm_handler/compilation/definition.ex index 219b8c169..8912ad449 100644 --- a/plumber/ppl/lib/ppl/ppl_sub_inits/stm_handler/compilation/definition.ex +++ b/plumber/ppl/lib/ppl/ppl_sub_inits/stm_handler/compilation/definition.ex @@ -6,7 +6,6 @@ defmodule Ppl.PplSubInits.STMHandler.Compilation.Definition do alias Util.ToTuple alias Ppl.DefinitionReviser.BlocksReviser - @default_execution_limit_in_minutes 10 @default_priority 95 @@ -43,6 +42,9 @@ defmodule Ppl.PplSubInits.STMHandler.Compilation.Definition do end defp form_defintion_jobs(ppl_req = %{request_args: req_args}, pfcs) do + pfc_cmds = pfc_commands(pfcs) + org_id = Map.get(req_args, "organization_id", "") + [ %{ "name" => "Compilation", @@ -50,12 +52,24 @@ defmodule Ppl.PplSubInits.STMHandler.Compilation.Definition do "priority" => [%{"value" => @default_priority, "when" => true}], "env_vars" => ppl_env_vars(ppl_req) ++ [sem_yaml_file_path_env_var(req_args)], "secrets" => secrets_definition(pfcs, req_args), - "commands" => commands(pfcs), + "commands" => + default_commands(req_args, optimize_checkout?(org_id, pfc_cmds)) ++ pfc_cmds, "epilogue_always_cmds" => epilogue_always_commands() } ] end + # The optimized blobless + sparse checkout is used only when both hold: + # * there are no pre-flight checks (their custom commands may need the full + # working tree), and + # * the `sparse_checkout_init_job` feature is enabled for the organization. + # The feature check fails closed, so a missing org id or an unreachable + # Feature service keeps the standard full checkout. + @doc false + def optimize_checkout?(org_id, pfc_cmds) do + pfc_cmds == [] and Ppl.Features.sparse_checkout_init_job_enabled?(org_id) + end + defp agent_definition(pre_flight_checks, settings) when is_map(pre_flight_checks) do pfc_agent = get_in(pre_flight_checks, ["project_pfc", "agent"]) || %{} @@ -109,10 +123,6 @@ defmodule Ppl.PplSubInits.STMHandler.Compilation.Definition do test_commands(mix_env) ++ pfc_commands(pfcs) end - defp commands(pfcs) do - default_commands() ++ pfc_commands(pfcs) - end - defp pfc_commands(%{"organization_pfc" => org_pfc, "project_pfc" => prj_pfc}) when is_map(org_pfc) and is_map(prj_pfc), do: Map.get(org_pfc, "commands", []) ++ Map.get(prj_pfc, "commands", []) @@ -125,9 +135,31 @@ defmodule Ppl.PplSubInits.STMHandler.Compilation.Definition do defp pfc_commands(_ppl_req), do: [] - defp default_commands() do + # When there are no pre-flight checks, the initialization job only needs the + # pipeline YAML and the Git history (trees/commits, used by `change_in`), not + # the full repository working tree. In that case we instruct `checkout` to + # perform a blobless partial clone with a sparse working tree limited to the + # pipeline directory, which avoids downloading/materializing the whole repo. + # + # When pre-flight checks are configured, their custom commands run after the + # predefined ones and may rely on the full working tree being present, so we + # keep the standard full checkout. + @doc false + def default_commands(req_args, _optimize_checkout? = true) do [ ~s[export GIT_LFS_SKIP_SMUDGE=1], + ~s[export SEMAPHORE_GIT_PARTIAL_CLONE_FILTER="blob:none"], + ~s[export SEMAPHORE_GIT_SPARSE_CHECKOUT_PATHS="#{sparse_checkout_path(req_args)}"] + ] ++ checkout_and_compile_commands() + end + + @doc false + def default_commands(_req_args, _optimize_checkout? = false) do + [~s[export GIT_LFS_SKIP_SMUDGE=1]] ++ checkout_and_compile_commands() + end + + defp checkout_and_compile_commands() do + [ ~s[checkout], ~s[export INPUT_FILE="$SEMAPHORE_YAML_FILE_PATH"], ~s[export OUTPUT_FILE="${SEMAPHORE_YAML_FILE_PATH}.output.yml"], @@ -138,6 +170,17 @@ defmodule Ppl.PplSubInits.STMHandler.Compilation.Definition do ] end + # The pipeline lives in the working directory of the YAML file; sparse-checkout + # of that directory keeps the pipeline files available while skipping the rest + # of the repository. Falls back to the repository root when no working + # directory is set (no optimization, but always correct). + defp sparse_checkout_path(req_args) do + case (req_args["working_dir"] || "") |> String.trim() do + "" -> "." + working_dir -> working_dir + end + end + defp epilogue_always_commands() do [ ~s[export BASE_NAME=$SEMAPHORE_PIPELINE_ID-$(basename $INPUT_FILE)], diff --git a/plumber/ppl/mix.exs b/plumber/ppl/mix.exs index 760c48bea..3d5c95693 100644 --- a/plumber/ppl/mix.exs +++ b/plumber/ppl/mix.exs @@ -68,6 +68,12 @@ defmodule Ppl.Mixfile do {:definition_validator, path: "../definition_validator"}, {:block, path: "../block"}, {:job_matrix, path: "../job_matrix"}, + {:feature_provider, path: "../../feature_provider"}, + # feature_provider requests yaml_elixir >= 2.0, but plumber's + # definition_validator (pipeline YAML validation) pins ~> 1.1. We only use + # feature_provider's FeatureHub provider (not its YamlProvider), so pin the + # existing 1.3 line to keep a single version across the umbrella. + {:yaml_elixir, "~> 1.3", override: true}, {:dialyxir, "~> 0.5", only: [:dev], runtime: false}, {:amqp_client, "~> 3.9.2"}, {:tackle, github: "renderedtext/ex-tackle", tag: "v0.2.1"}, diff --git a/plumber/ppl/test/definition_reviser/global_job_config_test.exs b/plumber/ppl/test/definition_reviser/global_job_config_test.exs index faac99258..909a11aa5 100644 --- a/plumber/ppl/test/definition_reviser/global_job_config_test.exs +++ b/plumber/ppl/test/definition_reviser/global_job_config_test.exs @@ -31,8 +31,16 @@ defmodule Ppl.DefinitionReviser.BlocksReviser.GlobalJobConfig.Test do Test.Support.GrpcServerHelper.setup_service_url(@url_env_name_2, project_port) on_exit(fn -> - System.put_env(@url_env_name, old_artifact_url) - System.put_env(@url_env_name_2, old_project_url) + # The env vars may have been unset before this test ran (depends on test + # ordering/sharding), so guard against restoring a nil value, which + # System.put_env/2 rejects. + if old_artifact_url, + do: System.put_env(@url_env_name, old_artifact_url), + else: System.delete_env(@url_env_name) + + if old_project_url, + do: System.put_env(@url_env_name_2, old_project_url), + else: System.delete_env(@url_env_name_2) end) Test.Helpers.truncate_db() diff --git a/plumber/ppl/test/features_test.exs b/plumber/ppl/test/features_test.exs new file mode 100644 index 000000000..8845b8cc1 --- /dev/null +++ b/plumber/ppl/test/features_test.exs @@ -0,0 +1,40 @@ +defmodule Ppl.FeaturesTest do + use ExUnit.Case, async: false + + @url_env_var "INTERNAL_API_URL_FEATURE" + + setup do + # The result is memoized in :feature_cache; clear it so each test starts + # fresh (e.g. the "unreachable" case must not see a cached value from the + # "enabled" case for the same org id). + Cachex.clear(:feature_cache) + + original = System.get_env(@url_env_var) + System.put_env(@url_env_var, "localhost:50053") + + on_exit(fn -> + case original do + nil -> System.delete_env(@url_env_var) + value -> System.put_env(@url_env_var, value) + end + end) + + :ok + end + + describe "sparse_checkout_init_job_enabled?/1" do + test "true when the feature is enabled for the org (via the FeatureHub mock)" do + assert Ppl.Features.sparse_checkout_init_job_enabled?("org-123") == true + end + + test "fails closed for a missing org id" do + assert Ppl.Features.sparse_checkout_init_job_enabled?("") == false + assert Ppl.Features.sparse_checkout_init_job_enabled?(nil) == false + end + + test "fails closed when the Feature service is unreachable" do + System.put_env(@url_env_var, "localhost:1") + assert Ppl.Features.sparse_checkout_init_job_enabled?("org-123") == false + end + end +end diff --git a/plumber/ppl/test/ppl_sub_inits/stm_handler/compilation/definition_test.exs b/plumber/ppl/test/ppl_sub_inits/stm_handler/compilation/definition_test.exs index a833f2d9a..7547cc020 100644 --- a/plumber/ppl/test/ppl_sub_inits/stm_handler/compilation/definition_test.exs +++ b/plumber/ppl/test/ppl_sub_inits/stm_handler/compilation/definition_test.exs @@ -1,5 +1,6 @@ defmodule Ppl.PplSubInits.STMHandler.Compilation.Definition.Test do use ExUnit.Case, async: false + import Mock alias Ppl.PplSubInits.STMHandler.Compilation.Definition alias Ppl.PplRequests.Model.PplRequestsQueries @@ -491,10 +492,81 @@ defmodule Ppl.PplSubInits.STMHandler.Compilation.Definition.Test do end end + describe "default_commands/2" do + test "without pre-flight checks it uses a blobless + sparse checkout of the pipeline dir" do + req_args = %{"working_dir" => ".semaphore", "file_name" => "semaphore.yml"} + + commands = Definition.default_commands(req_args, _optimize_checkout? = true) + + assert "export GIT_LFS_SKIP_SMUDGE=1" in commands + assert ~s[export SEMAPHORE_GIT_PARTIAL_CLONE_FILTER="blob:none"] in commands + assert ~s[export SEMAPHORE_GIT_SPARSE_CHECKOUT_PATHS=".semaphore"] in commands + assert "checkout" in commands + + # the optimization exports must come before checkout + filter_idx = Enum.find_index(commands, &(&1 =~ "SEMAPHORE_GIT_PARTIAL_CLONE_FILTER")) + sparse_idx = Enum.find_index(commands, &(&1 =~ "SEMAPHORE_GIT_SPARSE_CHECKOUT_PATHS")) + checkout_idx = Enum.find_index(commands, &(&1 == "checkout")) + assert filter_idx < checkout_idx + assert sparse_idx < checkout_idx + + assert List.last(commands) == + "spc compile --input $INPUT_FILE --output $OUTPUT_FILE --logs $LOGS_FILE" + end + + test "with pre-flight checks it keeps the standard full checkout" do + req_args = %{"working_dir" => ".semaphore", "file_name" => "semaphore.yml"} + + commands = Definition.default_commands(req_args, _optimize_checkout? = false) + + assert "export GIT_LFS_SKIP_SMUDGE=1" in commands + assert "checkout" in commands + refute Enum.any?(commands, &(&1 =~ "SEMAPHORE_GIT_PARTIAL_CLONE_FILTER")) + refute Enum.any?(commands, &(&1 =~ "SEMAPHORE_GIT_SPARSE_CHECKOUT_PATHS")) + end + + test "falls back to the repository root when the working dir is empty" do + req_args = %{"working_dir" => " ", "file_name" => "semaphore.yml"} + + commands = Definition.default_commands(req_args, _optimize_checkout? = true) + + assert ~s[export SEMAPHORE_GIT_SPARSE_CHECKOUT_PATHS="."] in commands + end + + test "uses a nested working dir as the sparse path" do + req_args = %{"working_dir" => ".semaphore/prod", "file_name" => "deploy.yml"} + + commands = Definition.default_commands(req_args, _optimize_checkout? = true) + + assert ~s[export SEMAPHORE_GIT_SPARSE_CHECKOUT_PATHS=".semaphore/prod"] in commands + end + end + + describe "optimize_checkout?/2" do + test "true when no pre-flight checks and the feature is enabled" do + with_mock Ppl.Features, sparse_checkout_init_job_enabled?: fn _ -> true end do + assert Definition.optimize_checkout?("org-1", []) == true + end + end + + test "false when the feature is disabled, even without pre-flight checks" do + with_mock Ppl.Features, sparse_checkout_init_job_enabled?: fn _ -> false end do + assert Definition.optimize_checkout?("org-1", []) == false + end + end + + test "false when pre-flight checks are present, even if the feature is enabled" do + with_mock Ppl.Features, sparse_checkout_init_job_enabled?: fn _ -> true end do + assert Definition.optimize_checkout?("org-1", ["./security_check.sh"]) == false + end + end + end + @tag :integration test "when in prod environment => return compilation definition with proper env vars set" do System.put_env("INTERNAL_API_URL_PFC", "localhost:50053") System.put_env("INTERNAL_API_URL_USER", "localhost:50053") + System.put_env("INTERNAL_API_URL_FEATURE", "localhost:50053") System.put_env("INTERNAL_API_URL_ORGANIZATION", "localhost:50053") not_trimmed_file_name = "semaphore.yml " @@ -531,6 +603,13 @@ defmodule Ppl.PplSubInits.STMHandler.Compilation.Definition.Test do yml_path_ev = Enum.find(list, fn map -> map["name"] == "SEMAPHORE_YAML_FILE_PATH" end) assert yml_path_ev["value"] == ".semaphore/semaphore.yml" + # With no pre-flight checks, the init job uses the optimized blobless + + # sparse checkout of the pipeline directory. + commands = Map.get(definition, "jobs") |> Enum.at(0) |> Map.get("commands") + assert ~s[export SEMAPHORE_GIT_PARTIAL_CLONE_FILTER="blob:none"] in commands + assert ~s[export SEMAPHORE_GIT_SPARSE_CHECKOUT_PATHS=".semaphore"] in commands + assert "checkout" in commands + on_exit(fn -> GRPC.Server.stop(PFCServiceMock) end) diff --git a/plumber/ppl/test/support/mocks/feature_server.ex b/plumber/ppl/test/support/mocks/feature_server.ex new file mode 100644 index 000000000..aa492ee1a --- /dev/null +++ b/plumber/ppl/test/support/mocks/feature_server.ex @@ -0,0 +1,33 @@ +defmodule Test.Support.Mocks.FeatureServer do + @moduledoc """ + Test mock for the Feature (FeatureHub) gRPC service. + + Returns the `sparse_checkout_init_job` feature as ENABLED for every + organization, so the prod-path compilation definition test exercises the + optimized checkout branch. + """ + + use GRPC.Server, service: InternalApi.Feature.FeatureService.Service + + alias InternalApi.Feature.{ + ListOrganizationFeaturesResponse, + OrganizationFeature, + Feature, + Availability + } + + def list_organization_features(_request, _stream) do + ListOrganizationFeaturesResponse.new( + organization_features: [ + OrganizationFeature.new( + feature: + Feature.new( + type: "sparse_checkout_init_job", + name: "sparse_checkout_init_job" + ), + availability: Availability.new(state: Availability.State.value(:ENABLED), quantity: 1) + ) + ] + ) + end +end diff --git a/plumber/proto/Makefile b/plumber/proto/Makefile index 91fd5f18e..562d1c107 100644 --- a/plumber/proto/Makefile +++ b/plumber/proto/Makefile @@ -14,12 +14,19 @@ INTERACTIVE_SESSION=\ INTERNAL_API_BRANCH?=master TMP_INTERNAL_REPO_DIR?=/tmp/internal_api +# Set INTERNAL_API_LOCAL_PATH to regenerate the stubs from a local internal_api +# checkout instead of cloning the repository, e.g.: +# make pb.gen INTERNAL_API_LOCAL_PATH=/path/to/internal_api +INTERNAL_API_LOCAL_PATH ?= +INTERNAL_API_SOURCE_DIR=$(if $(INTERNAL_API_LOCAL_PATH),$(INTERNAL_API_LOCAL_PATH),$(TMP_INTERNAL_REPO_DIR)) RELATIVE_INTERNAL_PB_OUTPUT_DIR=lib/internal_api RT_PROTOC_IMG_VSN=1.6.6-3.3.0-0.5.4 pb.clone: +ifeq ($(INTERNAL_API_LOCAL_PATH),) rm -rf $(TMP_INTERNAL_REPO_DIR) git clone git@github.com:renderedtext/internal_api.git $(TMP_INTERNAL_REPO_DIR) && (cd $(TMP_INTERNAL_REPO_DIR) && git checkout $(INTERNAL_API_BRANCH) && cd -) +endif pb.gen: pb.clone ifeq ($(shell whoami), vagrant) @@ -27,29 +34,32 @@ ifeq ($(shell whoami), vagrant) else rm -rf $(RELATIVE_INTERNAL_PB_OUTPUT_DIR) && mkdir -p $(RELATIVE_INTERNAL_PB_OUTPUT_DIR) endif - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/include/google/protobuf/timestamp.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/include/google/rpc/status.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/include/google/rpc/code.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/gofer.switch.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/gofer.dt.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/organization.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/paparazzo.snapshot.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/plumber.pipeline.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/plumber.admin.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/plumber_w_f.workflow.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/repo_proxy.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/repository.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/include/internal_api/response_status.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/include/internal_api/status.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/task.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/repository_integrator.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/user.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/projecthub.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/artifacthub.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/health.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/pre_flight_checks_hub.proto - docker run --rm -v $(PWD):/home/protoc/code -v $(TMP_INTERNAL_REPO_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/usage.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/include/google/protobuf/timestamp.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/include/google/rpc/status.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/include/google/rpc/code.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/gofer.switch.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/gofer.dt.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/organization.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/paparazzo.snapshot.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/plumber.pipeline.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/plumber.admin.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/plumber_w_f.workflow.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/repo_proxy.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/repository.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/include/internal_api/response_status.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/include/internal_api/status.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/task.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/repository_integrator.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/user.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/projecthub.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/artifacthub.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/health.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/pre_flight_checks_hub.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/usage.proto + docker run --rm -v $(PWD):/home/protoc/code -v $(INTERNAL_API_SOURCE_DIR):/home/protoc/source renderedtext/protoc:$(RT_PROTOC_IMG_VSN) protoc -I /home/protoc/source -I /home/protoc/source/include --elixir_out=plugins=grpc:$(RELATIVE_INTERNAL_PB_OUTPUT_DIR) --plugin=/root/.mix/escripts/protoc-gen-elixir /home/protoc/source/feature.proto +ifeq ($(INTERNAL_API_LOCAL_PATH),) rm -rf $(TMP_INTERNAL_REPO_DIR) +endif console: docker run $(INTERACTIVE_SESSION) /bin/ash diff --git a/plumber/proto/lib/internal_api/feature.pb.ex b/plumber/proto/lib/internal_api/feature.pb.ex new file mode 100644 index 000000000..880b85d57 --- /dev/null +++ b/plumber/proto/lib/internal_api/feature.pb.ex @@ -0,0 +1,276 @@ +defmodule InternalApi.Feature.ListOrganizationFeaturesRequest do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + org_id: String.t() + } + defstruct [:org_id] + + field(:org_id, 1, type: :string) +end + +defmodule InternalApi.Feature.ListOrganizationFeaturesResponse do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + organization_features: [InternalApi.Feature.OrganizationFeature.t()] + } + defstruct [:organization_features] + + field(:organization_features, 1, repeated: true, type: InternalApi.Feature.OrganizationFeature) +end + +defmodule InternalApi.Feature.OrganizationFeature do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + feature: InternalApi.Feature.Feature.t(), + availability: InternalApi.Feature.Availability.t(), + project_ids: [String.t()], + requester_id: String.t(), + created_at: Google.Protobuf.Timestamp.t(), + updated_at: Google.Protobuf.Timestamp.t() + } + defstruct [:feature, :availability, :project_ids, :requester_id, :created_at, :updated_at] + + field(:feature, 1, type: InternalApi.Feature.Feature) + field(:availability, 2, type: InternalApi.Feature.Availability) + field(:project_ids, 3, repeated: true, type: :string) + field(:requester_id, 5, type: :string) + field(:created_at, 6, type: Google.Protobuf.Timestamp) + field(:updated_at, 7, type: Google.Protobuf.Timestamp) +end + +defmodule InternalApi.Feature.ListFeaturesRequest do + @moduledoc false + use Protobuf, syntax: :proto3 + + defstruct [] +end + +defmodule InternalApi.Feature.ListFeaturesResponse do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + features: [InternalApi.Feature.Feature.t()] + } + defstruct [:features] + + field(:features, 1, repeated: true, type: InternalApi.Feature.Feature) +end + +defmodule InternalApi.Feature.Feature do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + type: String.t(), + availability: InternalApi.Feature.Availability.t(), + name: String.t(), + description: String.t() + } + defstruct [:type, :availability, :name, :description] + + field(:type, 1, type: :string) + field(:availability, 2, type: InternalApi.Feature.Availability) + field(:name, 3, type: :string) + field(:description, 4, type: :string) +end + +defmodule InternalApi.Feature.ListOrganizationMachinesRequest do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + org_id: String.t() + } + defstruct [:org_id] + + field(:org_id, 1, type: :string) +end + +defmodule InternalApi.Feature.ListOrganizationMachinesResponse do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + organization_machines: [InternalApi.Feature.OrganizationMachine.t()], + default_type: String.t() + } + defstruct [:organization_machines, :default_type] + + field(:organization_machines, 1, repeated: true, type: InternalApi.Feature.OrganizationMachine) + field(:default_type, 2, type: :string) +end + +defmodule InternalApi.Feature.OrganizationMachine do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + machine: InternalApi.Feature.Machine.t(), + availability: InternalApi.Feature.Availability.t(), + requester_id: String.t(), + created_at: Google.Protobuf.Timestamp.t(), + updated_at: Google.Protobuf.Timestamp.t() + } + defstruct [:machine, :availability, :requester_id, :created_at, :updated_at] + + field(:machine, 1, type: InternalApi.Feature.Machine) + field(:availability, 2, type: InternalApi.Feature.Availability) + field(:requester_id, 3, type: :string) + field(:created_at, 4, type: Google.Protobuf.Timestamp) + field(:updated_at, 5, type: Google.Protobuf.Timestamp) +end + +defmodule InternalApi.Feature.ListMachinesRequest do + @moduledoc false + use Protobuf, syntax: :proto3 + + defstruct [] +end + +defmodule InternalApi.Feature.ListMachinesResponse do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + machines: [InternalApi.Feature.Machine.t()] + } + defstruct [:machines] + + field(:machines, 1, repeated: true, type: InternalApi.Feature.Machine) +end + +defmodule InternalApi.Feature.Machine do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + type: String.t(), + availability: InternalApi.Feature.Availability.t(), + platform: integer, + vcpu: String.t(), + ram: String.t(), + disk: String.t(), + default_os_image: String.t(), + os_images: [String.t()] + } + defstruct [:type, :availability, :platform, :vcpu, :ram, :disk, :default_os_image, :os_images] + + field(:type, 1, type: :string) + field(:availability, 2, type: InternalApi.Feature.Availability) + field(:platform, 3, type: InternalApi.Feature.Machine.Platform, enum: true) + field(:vcpu, 4, type: :string) + field(:ram, 5, type: :string) + field(:disk, 6, type: :string) + field(:default_os_image, 7, type: :string) + field(:os_images, 8, repeated: true, type: :string) +end + +defmodule InternalApi.Feature.Machine.Platform do + @moduledoc false + use Protobuf, enum: true, syntax: :proto3 + + field(:LINUX, 0) + field(:MAC, 1) +end + +defmodule InternalApi.Feature.Availability do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + state: integer, + quantity: non_neg_integer + } + defstruct [:state, :quantity] + + field(:state, 1, type: InternalApi.Feature.Availability.State, enum: true) + field(:quantity, 2, type: :uint32) +end + +defmodule InternalApi.Feature.Availability.State do + @moduledoc false + use Protobuf, enum: true, syntax: :proto3 + + field(:HIDDEN, 0) + field(:ZERO_STATE, 1) + field(:ENABLED, 2) +end + +defmodule InternalApi.Feature.MachinesChanged do + @moduledoc false + use Protobuf, syntax: :proto3 + + defstruct [] +end + +defmodule InternalApi.Feature.OrganizationMachinesChanged do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + org_id: String.t() + } + defstruct [:org_id] + + field(:org_id, 1, type: :string) +end + +defmodule InternalApi.Feature.FeaturesChanged do + @moduledoc false + use Protobuf, syntax: :proto3 + + defstruct [] +end + +defmodule InternalApi.Feature.OrganizationFeaturesChanged do + @moduledoc false + use Protobuf, syntax: :proto3 + + @type t :: %__MODULE__{ + org_id: String.t() + } + defstruct [:org_id] + + field(:org_id, 1, type: :string) +end + +defmodule InternalApi.Feature.FeatureService.Service do + @moduledoc false + use GRPC.Service, name: "InternalApi.Feature.FeatureService" + + rpc( + :ListOrganizationFeatures, + InternalApi.Feature.ListOrganizationFeaturesRequest, + InternalApi.Feature.ListOrganizationFeaturesResponse + ) + + rpc( + :ListFeatures, + InternalApi.Feature.ListFeaturesRequest, + InternalApi.Feature.ListFeaturesResponse + ) + + rpc( + :ListOrganizationMachines, + InternalApi.Feature.ListOrganizationMachinesRequest, + InternalApi.Feature.ListOrganizationMachinesResponse + ) + + rpc( + :ListMachines, + InternalApi.Feature.ListMachinesRequest, + InternalApi.Feature.ListMachinesResponse + ) +end + +defmodule InternalApi.Feature.FeatureService.Stub do + @moduledoc false + use GRPC.Stub, service: InternalApi.Feature.FeatureService.Service +end