-
Notifications
You must be signed in to change notification settings - Fork 81
feat(ppl): optimize init-job checkout (blobless + sparse) behind a feature flag #1063
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
8 commits
Select commit
Hold shift + click to select a range
d5887d0
feat(ppl): blobless + sparse checkout for init job without pre-flight…
DamjanBecirovic 853357b
feat(ppl): gate init-job optimized checkout behind a FeatureHub flag
DamjanBecirovic 34a247b
fix(ppl): build Docker image from repo-root context to include featur…
DamjanBecirovic 2cc4ca1
test(ppl): make global_job_config env restore nil-safe
DamjanBecirovic da8f121
fix(ppl): bound and cache the init-job feature check
DamjanBecirovic 2abeb4d
Merge remote-tracking branch 'origin/main' into db/init-job-blobless-…
skipi c9e76c0
Merge remote-tracking branch 'origin/main' into db/init-job-blobless-…
skipi 317f53a
Merge remote-tracking branch 'origin/main' into db/init-job-blobless-…
skipi File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,105 @@ | ||
| defmodule Ppl.FeatureClient do | ||
| @moduledoc """ | ||
| gRPC client consuming the Feature (FeatureHub) API. | ||
|
|
||
| Used by `Ppl.FeatureHubProvider` to fetch the features enabled for an | ||
| organization. Failures are turned into `{:error, _}` so callers can fail | ||
| closed (treat the feature as disabled). | ||
| """ | ||
|
|
||
| @watchman_prefix_key "Ppl.FeatureClient" | ||
| @url_env_var "INTERNAL_API_URL_FEATURE" | ||
| # This call sits in the pipeline initialization hot path (the compile task is | ||
| # built and awaited inside a deadline-bounded looper step), so it must be | ||
| # tightly bounded and fail fast. @grpc_timeout is the per-call gRPC deadline; | ||
| # @timeout is a slightly larger Wormhole backstop covering connect hangs. | ||
| @grpc_timeout 1_000 | ||
| @timeout 1_500 | ||
|
|
||
| alias InternalApi.Feature, as: API | ||
| alias API.FeatureService.Stub | ||
| require Logger | ||
|
|
||
| @doc """ | ||
| Calls the Feature gRPC API to list the features enabled for an organization. | ||
| """ | ||
| @spec list_organization_features(String.t()) :: | ||
| {:ok, [InternalApi.Feature.OrganizationFeature.t()]} | ||
| | {:error, :timeout} | ||
| | {:error, any()} | ||
| def list_organization_features(organization_id) do | ||
| result = | ||
| Wormhole.capture(__MODULE__, :do_list_organization_features, [organization_id], | ||
| stacktrace: true, | ||
| timeout: @timeout | ||
| ) | ||
|
|
||
| case result do | ||
| {:ok, features} -> | ||
| {:ok, features} | ||
|
|
||
| {:error, {:timeout, timeout}} -> | ||
| log_timeout(organization_id, timeout) | ||
| {:error, :timeout} | ||
|
|
||
| {:error, {:shutdown, {reason, _stacktrace}}} -> | ||
| log_shutdown(organization_id, reason) | ||
| {:error, reason} | ||
| end | ||
| end | ||
|
|
||
| # | ||
| # gRPC connection | ||
| # | ||
|
|
||
| @doc false | ||
| def do_list_organization_features(organization_id) do | ||
| Watchman.benchmark("#{@watchman_prefix_key}.list_organization_features.duration", fn -> | ||
| request = API.ListOrganizationFeaturesRequest.new(org_id: organization_id) | ||
|
|
||
| case send_request(request) do | ||
| {:ok, response} -> | ||
| Watchman.increment("#{@watchman_prefix_key}.list_organization_features.success") | ||
| response.organization_features | ||
|
|
||
| {:error, reason} -> | ||
| Watchman.increment("#{@watchman_prefix_key}.list_organization_features.failure") | ||
| raise reason | ||
| end | ||
| end) | ||
| end | ||
|
|
||
| defp send_request(request) do | ||
| url = | ||
| System.get_env(@url_env_var) || raise "environment variable #{@url_env_var} was not found" | ||
|
|
||
| with {:ok, channel} <- GRPC.Stub.connect(url) do | ||
| Watchman.increment("#{@watchman_prefix_key}.list_organization_features.connect") | ||
|
|
||
| try do | ||
| Stub.list_organization_features(channel, request, timeout: @grpc_timeout) | ||
| after | ||
| GRPC.Stub.disconnect(channel) | ||
| end | ||
| end | ||
| end | ||
|
|
||
| # | ||
| # Logging functions | ||
| # | ||
|
|
||
| defp log_timeout(organization_id, _timeout) do | ||
| metadata = log_metadata(organization_id: organization_id) | ||
| Logger.error("Ppl.FeatureClient.list_organization_features/1: TIMEOUT #{metadata}") | ||
| end | ||
|
|
||
| defp log_shutdown(organization_id, reason) do | ||
| metadata = log_metadata(organization_id: organization_id, reason: reason) | ||
| Logger.error("Ppl.FeatureClient.list_organization_features/1: SHUTDOWN #{metadata}") | ||
| end | ||
|
|
||
| defp log_metadata(metadata) do | ||
| formatter = &"#{elem(&1, 0)}=#{inspect(elem(&1, 1))}" | ||
| metadata |> Enum.map_join(" ", formatter) | ||
| end | ||
| end |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| defmodule Ppl.FeatureHubProvider do | ||
| @moduledoc """ | ||
| `FeatureProvider.Provider` implementation backed by the Feature (FeatureHub) | ||
| gRPC API. | ||
|
|
||
| Only organization features are needed in plumber; machines are not used here. | ||
| On any error fetching features we return `{:error, _}`, which makes | ||
| `FeatureProvider.feature_enabled?/2` fail closed (return `false`). | ||
| """ | ||
|
|
||
| use FeatureProvider.Provider | ||
|
|
||
| alias Ppl.FeatureClient | ||
| alias InternalApi.Feature.{Availability, OrganizationFeature} | ||
|
|
||
| @impl FeatureProvider.Provider | ||
| def provide_features(org_id, _opts \\ []) do | ||
| case FeatureClient.list_organization_features(org_id) do | ||
| {:ok, organization_features} -> | ||
| features = | ||
| organization_features | ||
| |> Enum.map(&feature_from_grpc/1) | ||
| |> Enum.filter(&FeatureProvider.Feature.visible?/1) | ||
|
|
||
| {:ok, features} | ||
|
|
||
| {:error, reason} -> | ||
| {:error, reason} | ||
| end | ||
| end | ||
|
|
||
| @impl FeatureProvider.Provider | ||
| def provide_machines(_org_id, _opts \\ []) do | ||
| {:ok, []} | ||
| end | ||
|
|
||
| defp feature_from_grpc(%OrganizationFeature{feature: feature, availability: availability}) do | ||
| %FeatureProvider.Feature{ | ||
| name: feature.name, | ||
| type: feature.type, | ||
| description: feature.description, | ||
| quantity: quantity_from_availability(availability), | ||
| state: state_from_availability(availability) | ||
| } | ||
| end | ||
|
|
||
| defp quantity_from_availability(%Availability{quantity: quantity}), do: quantity | ||
| defp quantity_from_availability(_), do: 0 | ||
|
|
||
| defp state_from_availability(%Availability{state: state}) do | ||
| state | ||
| |> Availability.State.key() | ||
| |> case do | ||
| :ENABLED -> :enabled | ||
| :HIDDEN -> :disabled | ||
| :ZERO_STATE -> :zero_state | ||
| end | ||
| end | ||
|
|
||
| defp state_from_availability(_), do: :disabled | ||
| end |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.