diff --git a/concoursetools/additional.py b/concoursetools/additional.py index 0a14282..6d712b3 100644 --- a/concoursetools/additional.py +++ b/concoursetools/additional.py @@ -64,8 +64,8 @@ class InOnlyConcourseResource(ConcourseResource[DatetimeVersion]): :meth:`~concoursetools.resource.ConcourseResource.download_version`. The correct use case of this resource is to execute a - :concourse:`put-step.put-step`, and then place parameters - in the :concourse:`put-step.schema.put.get_params` section: + :concourse:`steps.put`, and then place parameters + in the :concourse:`steps.put#get_params` section: .. code:: yaml @@ -286,7 +286,7 @@ class MultiVersionConcourseResource(TriggerOnChangeConcourseResource[MultiVersio .. tip:: This resource class is best suited to resources used in conjunction - with the :concourse:`set-pipeline-step`. + with the :concourse:`steps.set_pipeline` step. """ def __init__(self, key: str, sub_version_class: type[SortableVersionT]): self.key = key diff --git a/concoursetools/metadata.py b/concoursetools/metadata.py index 32dbd78..dd4f855 100644 --- a/concoursetools/metadata.py +++ b/concoursetools/metadata.py @@ -10,7 +10,8 @@ Build metadata is deliberately not passed to :meth:`~concoursetools.resource.ConcourseResource.fetch_new_versions`, as none of this metadata is passed to the check environment by Concourse, to avoid antipatterns. -See the Concourse :concourse:`implementing-resource-types.resource-metadata` documentation for more information. +See the Concourse :concourse:`resource types metadata ` documentation +for more information. """ from __future__ import annotations @@ -32,8 +33,9 @@ class BuildMetadata: # pylint: disable=invalid-name :param BUILD_NAME: The build number within the build's job. :param BUILD_JOB_NAME: The name of the build's job. :param BUILD_PIPELINE_NAME: The name of the pipeline that the build's job lives in. - :param BUILD_PIPELINE_INSTANCE_VARS: The instance vars of the :concourse:`instanced pipeline ` - that the build's job lives in, serialized as JSON. + :param BUILD_PIPELINE_INSTANCE_VARS: The instance vars of the :concourse:`instanced pipeline + ` that the build's + job lives in, serialized as JSON. .. note:: A few variables are often present in the build environment, but are **not** documented by Concourse: @@ -66,14 +68,14 @@ def BUILD_CREATED_BY(self) -> str: .. warning:: By default this information is **not** available. To enable it, you need to set - :concourse:`resources.schema.resource.expose_build_created_by` in your resource schema. + :concourse:`resources#expose_build_created_by` in your resource schema. """ try: return os.environ["BUILD_CREATED_BY"] except KeyError as error: raise PermissionError("The 'BUILD_CREATED_BY' variable has not been made available. This must be enabled " "with the 'expose_build_created_by' variable within the resource schema: " - "https://concourse-ci.org/resources.html#schema.resource.expose_build_created_by") from error + "https://concourse-ci.org/docs/resources/#expose_build_created_by") from error @property def is_one_off_build(self) -> bool: @@ -81,7 +83,7 @@ def is_one_off_build(self) -> bool: Return :data:`True` if this build is one-off, and :data:`False` otherwise. A build is a "one-off" is it is triggered via the Concourse CLI - :concourse:`execute command `. + :concourse:`execute command `. It is determined by the absence of all of the following attributes: * ``BUILD_JOB_NAME`` @@ -96,14 +98,15 @@ def is_one_off_build(self) -> bool: @property def is_instanced_pipeline(self) -> bool: - """Return :data:`True` if this is an :concourse:`instanced pipeline `.""" + """Return :data:`True` if this is an :concourse:`instanced pipeline `.""" return self.BUILD_PIPELINE_INSTANCE_VARS is not None def instance_vars(self) -> dict[str, object]: """ Return the instance vars set on this pipeline as a mapping. - When working with an :concourse:`instanced pipeline `, it is much more convenient to + When working with an :concourse:`instanced pipeline + `, it is much more convenient to work with the instance vars as a mapping instead of a JSON string. .. note:: @@ -113,7 +116,7 @@ def instance_vars(self) -> dict[str, object]: :Example: If a instanced pipeline has been created from within another pipeline - (using the :concourse:`set-pipeline-step`), such as this: + (using the :concourse:`steps.set_pipeline` step), such as this: .. code:: yaml @@ -169,7 +172,7 @@ def format_string(self, string: str, additional_values: dict[str, str] | None = Format a string with metadata using standard bash ``$`` notation. Only a handful of "safe" values will be interpolated, not arbitrary attributes on the instance. - These are the :concourse:`original environment variables `, + These are the :concourse:`original environment variables `, including :attr:`BUILD_CREATED_BY` if it exists. object missing environment variable (such as in the case of a one-off build) will be empty. A ``$BUILD_URL`` variable is also added for ease. diff --git a/concoursetools/parsing.py b/concoursetools/parsing.py index b687525..facfd85 100644 --- a/concoursetools/parsing.py +++ b/concoursetools/parsing.py @@ -13,7 +13,7 @@ def parse_check_payload(raw_json: str) -> tuple[ResourceConfig, VersionConfig | None]: """ - Parse raw input JSON for a :concourse:`check payload `. + Parse raw input JSON for a :concourse:`check payload `. :param raw_json: A JSON string of the following form: @@ -46,7 +46,7 @@ def parse_check_payload(raw_json: str) -> tuple[ResourceConfig, VersionConfig | def parse_in_payload(raw_json: str) -> tuple[ResourceConfig, VersionConfig, Params]: """ - Parse raw input JSON for an :concourse:`in payload `. + Parse raw input JSON for an :concourse:`in payload `. :param raw_json: A JSON string of the following form: @@ -78,7 +78,7 @@ def parse_in_payload(raw_json: str) -> tuple[ResourceConfig, VersionConfig, Para def parse_out_payload(raw_json: str) -> tuple[ResourceConfig, Params]: """ - Parse raw input JSON for an :concourse:`out payload `. + Parse raw input JSON for an :concourse:`out payload `. :param raw_json: A JSON string of the following form: @@ -116,7 +116,7 @@ def parse_metadata(metadata_pairs: list[MetadataPair]) -> Metadata: def format_check_output(version_configs: list[VersionConfig], **json_kwargs: Any) -> str: """ - Format :concourse:`check output ` as a JSON string. + Format :concourse:`check output ` as a JSON string. :param version_configs: A list of version configurations. :param json_kwargs: Additional keyword arguments to pass to :func:`json.dumps`. @@ -136,8 +136,8 @@ def format_check_output(version_configs: list[VersionConfig], **json_kwargs: Any def format_in_out_output(version_config: VersionConfig, metadata: Metadata, **json_kwargs: Any) -> str: """ - Format :concourse:`in output ` or - :concourse:`out output ` as a JSON string. + Format :concourse:`in output ` or + :concourse:`out output ` as a JSON string. :param version_config: A version configuration. :param metadata: A key-value mapping of metadata. @@ -175,7 +175,7 @@ def format_metadata(metadata: Metadata) -> list[MetadataPair]: def format_check_input(resource_config: ResourceConfig, version_config: VersionConfig | None = None, **json_kwargs: Any) -> str: """ - Format :concourse:`check input ` as a JSON string. + Format :concourse:`check input ` as a JSON string. :param resource_config: A resource configuration. :param version_config: A version configuration, or :data:`None` if no version currently exists. @@ -201,7 +201,7 @@ def format_check_input(resource_config: ResourceConfig, version_config: VersionC def format_in_input(resource_config: ResourceConfig, version_config: VersionConfig, params: Params | None = None, **json_kwargs: Any) -> str: """ - Format :concourse:`in input ` as a JSON string. + Format :concourse:`in input ` as a JSON string. :param resource_config: A resource configuration. :param version_config: A version configuration. @@ -232,7 +232,7 @@ def format_in_input(resource_config: ResourceConfig, version_config: VersionConf def format_out_input(resource_config: ResourceConfig, params: Params | None = None, **json_kwargs: Any) -> str: """ - Format :concourse:`out input ` as a JSON string. + Format :concourse:`out input ` as a JSON string. :param resource_config: A resource configuration. :param params: Optional parameters to be passed. diff --git a/concoursetools/resource.py b/concoursetools/resource.py index cca39b6..70064df 100644 --- a/concoursetools/resource.py +++ b/concoursetools/resource.py @@ -7,12 +7,12 @@ Configuring the same resource in any pipeline on any Concourse cluster will behave the exact same way. Concourse will continuously check each configured resource to discover new versions. These versions then flow through the pipeline -via :concourse:`get steps ` configured on jobs. +via :concourse:`get steps ` configured on jobs. Find out more about resources in the :concourse:`Concourse resource documentation `. To learn more about how Concourse resource types are actually implemented under the hood, -check out :concourse:`implementing-resource-types` in Concourse. +check out :concourse:`implementing resource types ` in Concourse. """ from __future__ import annotations @@ -32,7 +32,7 @@ class ConcourseResource(ABC, Generic[VersionT]): """ Represents an external input or output to a pipeline. - The :concourse:`resource-types.schema.resource_type.source` + The :concourse:`resource-types#source` defined in a Concourse :concourse:`pipeline ` is parsed into JSON by Concourse, and will be passed to the initialiser of the :class:`ConcourseResource` class. @@ -94,7 +94,7 @@ def certs_dir(self) -> Path: This folder may not always exist, depending on how the Concourse runner was configured. - See the :concourse:`implementing-resource-types.resource-certs` documentation for more information. + See the :concourse:`resource-types.implementing#certificate-propagation` documentation for more information. """ return Path("/etc/ssl/certs") diff --git a/concoursetools/testing.py b/concoursetools/testing.py index fd7e1ea..618afa2 100644 --- a/concoursetools/testing.py +++ b/concoursetools/testing.py @@ -395,13 +395,13 @@ class FileTestResourceWrapper(TestResourceWrapper[Version]): This is best to use if your resource uses external scripts for any of the steps. :param inner_resource_config: The JSON configuration for the resource. - :param check_script: The path to the external script for the :concourse:`check `. + :param check_script: The path to the external script for the :concourse:`check `. Setting to :data:`None` (default) means that :meth:`fetch_new_versions` raises :class:`NotImplementedError`. - :param in_script: The path to the external script for the :concourse:`check `. + :param in_script: The path to the external script for the :concourse:`check `. Setting to :data:`None` (default) means that :meth:`download_version` raises :class:`NotImplementedError`. - :param out_script: The path to the external script for the :concourse:`check `. + :param out_script: The path to the external script for the :concourse:`check `. Setting to :data:`None` (default) means that :meth:`publish_new_version` raises :class:`NotImplementedError`. :param directory_dict: The initial state of the resource directory. See :class:`~concoursetools.mocking.TemporaryDirectoryState` diff --git a/concoursetools/typing.py b/concoursetools/typing.py index 7a223c5..e91ee72 100644 --- a/concoursetools/typing.py +++ b/concoursetools/typing.py @@ -18,13 +18,13 @@ class breaks the :wikipedia:`Liskov substitution principle`. If you wish to util ResourceConfig = dict[str, Any] """ Represents arbitrary configuration passed to a Concourse resource. -See the :concourse:`config-basics.schema.config` schema for more information. +See the :concourse:`config-basics#config-schema` for more information. """ Params = dict[str, Any] """ Represents arbitrary parameters passed to a Concourse resource. -See the :concourse:`config-basics.schema.config` schema for more information. +See the :concourse:`config-basics#config-schema` for more information. """ Metadata = dict[str, str] @@ -48,7 +48,7 @@ class MetadataPair(TypedDict): VersionConfig = dict[str, str] """ Represents a version of a Concourse resource. -See the :concourse:`config-basics.schema.version` schema for more information. +See the :concourse:`config-basics#version-schema` for more information. """ VersionT = TypeVar("VersionT", bound="VersionProtocol") diff --git a/concoursetools/version.py b/concoursetools/version.py index b857871..2fcb166 100644 --- a/concoursetools/version.py +++ b/concoursetools/version.py @@ -3,7 +3,7 @@ A resource version represents the exact state of a resource at a given point in time. The resource is responsible for defining what a version *actually is*, and this is defined with Concourse Tools using a :class:`Version` subclass. -More information on versions can be found in the :concourse:`Concourse documentation `. +More information on versions can be found in the :concourse:`Concourse documentation `. """ from __future__ import annotations diff --git a/docs/requirements.txt b/docs/requirements.txt index f80d8d3..d3fa0a0 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,6 +1,6 @@ accessible-pygments==0.0.5 alabaster==1.0.0 -babel==2.17.0 +babel==2.18.0 beautifulsoup4==4.14.3 certifi==2026.1.4 charset-normalizer==3.4.4 @@ -10,14 +10,14 @@ idna==3.11 imagesize==1.4.1 Jinja2==3.1.6 MarkupSafe==3.0.3 -packaging==25.0 +packaging==26.0 Pygments==2.19.2 requests==2.32.5 roman-numerals==4.1.0 snowballstemmer==3.0.1 -soupsieve==2.8.1 +soupsieve==2.8.3 Sphinx==9.1.0 -sphinx-autodoc-typehints==3.6.2 +sphinx-autodoc-typehints==3.6.3 sphinx-basic-ng==1.0.0b2 sphinxcontrib-applehelp==2.0.0 sphinxcontrib-devhelp==2.0.0 diff --git a/docs/source/changelog/0.8.0.rst b/docs/source/changelog/0.8.0.rst index bbaca3e..e7fc7ce 100644 --- a/docs/source/changelog/0.8.0.rst +++ b/docs/source/changelog/0.8.0.rst @@ -55,8 +55,8 @@ Build Metadata string formatting Both :meth:`~concoursetools.resource.ConcourseResource.download_version` and :meth:`~concoursetools.resource.ConcourseResource.publish_new_version` methods are passed an instance of :class:`~concoursetools.metadata.BuildMetadata` to interface with the -:concourse:`implementing-resource-types.resource-metadata`. This allows developers to easily interface with this -metadata from within the resource code. +:concourse:`resource metadata `. This allows developers to easily interface with +this metadata from within the resource code. However, many resources want to make this metadata available to the end user in their pipelines, by allowing them to reference the variables in commit messages, slack notifications and emails. Previously, resources would need to use @@ -245,7 +245,7 @@ Allowed ``source`` to be empty in resource configuration Recall that the parameters of the ``__init__`` method of your :class:`~concoursetools.resource.ConcourseResource` -subclass are taken from the :concourse:`resource-types.schema.resource_type.source` block of the pipeline YAML. +subclass are taken from the :concourse:`resource-types#source` block of the pipeline YAML. This means that a resource which looks like this: .. code:: python diff --git a/docs/source/examples/branches.rst b/docs/source/examples/branches.rst index 9806bbb..1105fdd 100644 --- a/docs/source/examples/branches.rst +++ b/docs/source/examples/branches.rst @@ -15,7 +15,7 @@ On occasion, you may wish your resource to emit versions when something has chan But what happens when a check yields two versions from different branches? We could figure out the "latest" version by sorting by commit date, but if we are tracking multiple branches then we don't want to ignore a commit on one just because a more recent commit has been made on another. Previous in Concourse this was "fixed" by passing ``version: every`` to the resource, but this means that if multiple commits are pushed to a branch at once, then each commit will emit a build, which may not be what we want. -What we really want is to iterate over all branches in our repository, and spin up a branch-specific pipeline for each one using the :concourse:`set-pipeline-step`. This is easy to do, but we need to make sure that new pipelines are added when new branches appear, and that old pipelines are deleted when their branches are removed. We specifically require a resource which will trigger a pipeline whenever something has changed. The generic resource for this pattern is the :class:`~concoursetools.additional.TriggerOnChangeConcourseResource`, but because the "state" is made up of several "sub versions", it makes sense to instead utilise the :class:`~concoursetools.additional.MultiVersionConcourseResource`, which takes care of much of the boiler plate for us, and also allow us to automatically download these subversions as JSON so that we may iterate over them. +What we really want is to iterate over all branches in our repository, and spin up a branch-specific pipeline for each one using the :concourse:`steps.set_pipeline` step. This is easy to do, but we need to make sure that new pipelines are added when new branches appear, and that old pipelines are deleted when their branches are removed. We specifically require a resource which will trigger a pipeline whenever something has changed. The generic resource for this pattern is the :class:`~concoursetools.additional.TriggerOnChangeConcourseResource`, but because the "state" is made up of several "sub versions", it makes sense to instead utilise the :class:`~concoursetools.additional.MultiVersionConcourseResource`, which takes care of much of the boiler plate for us, and also allow us to automatically download these subversions as JSON so that we may iterate over them. Branch Version -------------- @@ -66,7 +66,7 @@ We only need to implement the :meth:`~concoursetools.additional.MultiVersionConc The logic required here is incredibly minimal, and we only need to return each available subversion. The returned set will then be sorted by the resource when converting it to its final version, which is why we needed the subversion to be sortable and hashable. -When the branches change (either with new ones added or existing ones removed) a new version will be emitted and the pipeline will be triggered. Users can then iterate over them in their pipeline using a combination of the :concourse:`across-step` and the :concourse:`set-pipeline-step`: +When the branches change (either with new ones added or existing ones removed) a new version will be emitted and the pipeline will be triggered. Users can then iterate over them in their pipeline using a combination of the :concourse:`steps.modifier-and-hooks.across` step modifier and the :concourse:`steps.set_pipeline` step: .. code:: yaml diff --git a/docs/source/examples/build_status.rst b/docs/source/examples/build_status.rst index 260f117..770fdfa 100644 --- a/docs/source/examples/build_status.rst +++ b/docs/source/examples/build_status.rst @@ -91,7 +91,7 @@ Updating the Resource We can now write the code to update the resource, which involves implementing :meth:`~concoursetools.additional.OutOnlyConcourseResource.publish_new_version`. The method takes the usual parameters, and then the additional arguments offered by the original: .. note:: - The original configures a few variables via files instead of direct parameters. This is likely due to it being implemented before the :concourse:`load-var-step` was a thing. This implementation replaces them with direct variables. + The original configures a few variables via files instead of direct parameters. This is likely due to it being implemented before the :concourse:`steps.load_var` step was a thing. This implementation replaces them with direct variables. .. literalinclude:: ../../../examples/build_status.py :pyobject: Resource.publish_new_version diff --git a/docs/source/examples/pipeline.rst b/docs/source/examples/pipeline.rst index d8f2504..a802534 100644 --- a/docs/source/examples/pipeline.rst +++ b/docs/source/examples/pipeline.rst @@ -6,10 +6,10 @@ AWS SageMaker Pipeline In this example we will explore how to create a custom Concourse Resource using the Concourse Tools library. In this particular example we will consider a resource to interact with an `AWS SageMaker pipeline `_, which needs to keep track of `executions `_ of the pipeline, and to also be able to trigger new executions. In particular, we want the following behaviour: -* :concourse:`get-step`: Fetch the latest execution and download the metadata. -* :concourse:`put-step`: Start a new pipeline execution. +* :concourse:`steps.get` step: Fetch the latest execution and download the metadata. +* :concourse:`steps.put` step: Start a new pipeline execution. -The functionality of the resource will depend heavily on `boto3 `_ in order to reduce the amount of code needed to function. +The functionality of the resource will depend heavily on `boto3 `_ in order to reduce the amount of code needed to function. Execution Version @@ -23,7 +23,7 @@ When selecting the version for a custom resource, it is important to ensure that It is possible to break some of these assumptions, but to do so requires great care. For example, the `built-in git resource `_ uses a git commit to map to a version, but these are only unique within the context of a specific branch. Therefore, the resource requires a branch to be specified, otherwise the versions would not be linear. Although it would be possible to also include commit-specific information (such as author and timestamp) within the version, this is unnecessary. -Previously, some custom Concourse resources (such as the `Bitbucket pull request resource `_) attempt to use non-linear versions to be able to track both commits *and* pull request IDs, but this requires the user to set ``version: every`` on their resource in order to properly pick up *every* change, which can have some unintended consequences (such as multiple commits pushed to a branch at once being considered multiple versions). The correct way to deal with this now is to make use of :concourse:`instanced-pipelines` to correspond to each pull request, and to keep versions as linear. For more information on this pattern, see the :ref:`Github Branches` example. +Previously, some custom Concourse resources (such as the `Bitbucket pull request resource `_) attempt to use non-linear versions to be able to track both commits *and* pull request IDs, but this requires the user to set ``version: every`` on their resource in order to properly pick up *every* change, which can have some unintended consequences (such as multiple commits pushed to a branch at once being considered multiple versions). The correct way to deal with this now is to make use of :concourse:`instanced pipelines ` to correspond to each pull request, and to keep versions as linear. For more information on this pattern, see the :ref:`Github Branches` example. In this example, each execution has a corresponding `ARN `_, which uniquely defines it within the context of a pipeline (in fact, it uniquely defines it within an AWS account). Therefore, our version should contain this ARN as a string. This can be done by inheriting from the :class:`~concoursetools.version.Version` class, but it is generally easier (and requires less code) to instead make use of :mod:`dataclasses` and to inherit from :class:`~concoursetools.version.TypedVersion`: @@ -36,9 +36,9 @@ Passing around an instance of a class instead of a JSON object or even just a st Pipeline Resource ----------------- -The first thing to do is to establish what the :concourse:`resources.schema.resource.source` of the resource will be, and what we require of the user to properly configure it. For Concourse Tools, a resource is a subclass of the :class:`~concoursetools.resource.ConcourseResource` class, and the arguments in the ``__init__`` method correspond to the source. In this example, we require one configuration option, and an additional *optional* option: +The first thing to do is to establish what the :concourse:`resource-types#source` of the resource will be, and what we require of the user to properly configure it. For Concourse Tools, a resource is a subclass of the :class:`~concoursetools.resource.ConcourseResource` class, and the arguments in the ``__init__`` method correspond to the source. In this example, we require one configuration option, and an additional *optional* option: -* ``pipeline``: This will be the `ARN `_ of the pipeline itself. We can parse this for the pipeline name we use for the `boto3 `_ calls, as well as the region we need for constructing the client. It should be a string. +* ``pipeline``: This will be the `ARN `_ of the pipeline itself. We can parse this for the pipeline name we use for the `boto3 `_ calls, as well as the region we need for constructing the client. It should be a string. * ``statuses``: A pipeline execution can have one of five statuses at any time: ``Executing``, ``Stopping``, ``Stopped``, ``Failed`` and ``Succeeded``. By default, we don't want to trigger a version for executions which are still going, but we want the user to have this choice. Therefore, we pass a default value of a list containing the statuses which trigger a version, and allow the user to pass their own. These options correspond to the following ``__init__`` method: @@ -57,7 +57,7 @@ A few things are happening here: Checking for Executions ________________________ -The first behaviour we will implement is the :concourse:`check `, when the external resource is queried for new executions. There are a few cases that we need to handle: +The first behaviour we will implement is the :concourse:`check `, when the external resource is queried for new executions. There are a few cases that we need to handle: 1. If no previous version is passed (i.e., the parameter is :data:`None`), then we know that this is the first request and we need to return the *latest valid version*. 2. If no versions are available *at all* then an empty list should be returned. @@ -65,7 +65,7 @@ The first behaviour we will implement is the :concourse:`check `_. All we need be concerned with is that this method will yield instances of ``ExecutionVersion`` in newest-to-oldest order until the server runs out. +We start by defining a private method on the resource to yield "potential versions" from the external source. This is just to allow us to check equality with the previous version directly, which is cleaner than worrying about a potential :class:`AttributeError` if the previous version is :data:`None`. It also allows us to handle the way in which AWS batches up its response when calling `list_pipeline_executions `_. All we need be concerned with is that this method will yield instances of ``ExecutionVersion`` in newest-to-oldest order until the server runs out. .. literalinclude:: ../../../examples/pipeline.py :pyobject: PipelineResource._yield_potential_execution_versions @@ -82,10 +82,10 @@ We start by defining an iterator on the output of ``_yield_potential_execution_v Downloading Executions ______________________ -The next functionality to consider is the downloading of a version in a :concourse:`get-step`. This is implemented within Concourse Tools by overloading :meth:`~concoursetools.resource.ConcourseResource.download_version`. The behaviour we want for this step is: +The next functionality to consider is the downloading of a version in a :concourse:`steps.get` step. This is implemented within Concourse Tools by overloading :meth:`~concoursetools.resource.ConcourseResource.download_version`. The behaviour we want for this step is: -* Download the metadata of the execution (as described in `describe_pipeline_execution `_) to a JSON file. -* *Optionally* (but by default) download the definition of the pipeline to a different JSON file. This is done by calling `describe_pipeline_definition_for_execution `_. +* Download the metadata of the execution (as described in `describe_pipeline_execution `_) to a JSON file. +* *Optionally* (but by default) download the definition of the pipeline to a different JSON file. This is done by calling `describe_pipeline_definition_for_execution `_. * Return information about the pipeline as metadata to be displayed in the UI. The code for doing this is as follows: @@ -104,16 +104,16 @@ Next, we check if the user wishes to download the pipeline, and then write the d Publishing New Executions _________________________ -FInally, we consider the functionality of the :concourse:`put-step`, and creating new versions. To do this in Concourse Tools, we overload the :meth:`~concoursetools.resource.ConcourseResource.publish_new_version` method. The code will rely on a call to `start_pipeline_execution `_. +FInally, we consider the functionality of the :concourse:`steps.put` step, and creating new versions. To do this in Concourse Tools, we overload the :meth:`~concoursetools.resource.ConcourseResource.publish_new_version` method. The code will rely on a call to `start_pipeline_execution `_. .. literalinclude:: ../../../examples/pipeline.py :pyobject: PipelineResource.publish_new_version :dedent: -We start with the execution description. We include an optional parameter to allow the user to specify this in their :concourse:`get params `, but if they don't then we create a default description based on the :class:`~concoursetools.metadata.BuildMetadata`. In instance of this class is passed to the :meth:`~concoursetools.resource.ConcourseResource.download_version` and :meth:`~concoursetools.resource.ConcourseResource.publish_new_version` methods, and wraps the build environment to more easily make use of environment variables which are made available to resources, as well as some useful methods for computing the :meth:`~concoursetools.metadata.BuildMetadata.build_url` and working with the :meth:`~concoursetools.metadata.BuildMetadata.instance_vars`. In particular here we've made use of the ``BUILD_ID`` and ``BUILD_PIPELINE_NAME`` attributes. We also allow the user to pass in parameters for the execution. These are passed in as a mapping, but need to be converted to a list to fit with the function call. We also add them to the metadata for the user's benefit. Finally, we return the new version and the metadata. +We start with the execution description. We include an optional parameter to allow the user to specify this in their :concourse:`steps.put#get_params`, but if they don't then we create a default description based on the :class:`~concoursetools.metadata.BuildMetadata`. In instance of this class is passed to the :meth:`~concoursetools.resource.ConcourseResource.download_version` and :meth:`~concoursetools.resource.ConcourseResource.publish_new_version` methods, and wraps the build environment to more easily make use of environment variables which are made available to resources, as well as some useful methods for computing the :meth:`~concoursetools.metadata.BuildMetadata.build_url` and working with the :meth:`~concoursetools.metadata.BuildMetadata.instance_vars`. In particular here we've made use of the ``BUILD_ID`` and ``BUILD_PIPELINE_NAME`` attributes. We also allow the user to pass in parameters for the execution. These are passed in as a mapping, but need to be converted to a list to fit with the function call. We also add them to the metadata for the user's benefit. Finally, we return the new version and the metadata. .. note:: - The response from the `start_pipeline_execution `_ function contains only the ARN of the new execution, and is a big reason why the version doesn't contain the start time of the execution. To fill that in, we would need to make a second request to the server to fetch the information about the execution before creating and returning the version, which is not ideal. + The response from the `start_pipeline_execution `_ function contains only the ARN of the new execution, and is a big reason why the version doesn't contain the start time of the execution. To fill that in, we would need to make a second request to the server to fetch the information about the execution before creating and returning the version, which is not ideal. Pipeline Conclusion ------------------- diff --git a/docs/source/examples/s3.rst b/docs/source/examples/s3.rst index e7607c5..c269f0e 100644 --- a/docs/source/examples/s3.rst +++ b/docs/source/examples/s3.rst @@ -6,16 +6,16 @@ S3 Presigned URL This example will showcase the :class:`~concoursetools.additional.InOnlyConcourseResource`, and how to build a resource to fetch arbitrary data from an external resource **that is not stored externally**. In this particular example we will consider a resource to generate a new `presigned URL `_ for an object in an S3 bucket. -Traditionally, when Concourse users which to "run an external function" from a pipeline, they create an :class:`~concoursetools.additional.OutOnlyConcourseResource` to do it. Classic examples include setting a build status (as shown in the :ref:`Bitbucket Build Status` example), or to `send a message on Slack `_. Usually, the new version will only contain a small amount of placeholder information, such as the new build status. However, a :concourse:`put-step` is designed to "fetch" the information for further use in the pipeline, and this is only really possible when the new version represents a "state" which is stored with the external service. For example, the `Git resource `_ will push a new commit, and then emit the version corresponding to that commit so that - when the resource runs its implicit :concourse:`get-step` - the information is not overwritten. +Traditionally, when Concourse users which to "run an external function" from a pipeline, they create an :class:`~concoursetools.additional.OutOnlyConcourseResource` to do it. Classic examples include setting a build status (as shown in the :ref:`Bitbucket Build Status` example), or to `send a message on Slack `_. Usually, the new version will only contain a small amount of placeholder information, such as the new build status. However, a :concourse:`steps.put` step is designed to "fetch" the information for further use in the pipeline, and this is only really possible when the new version represents a "state" which is stored with the external service. For example, the `Git resource `_ will push a new commit, and then emit the version corresponding to that commit so that - when the resource runs its implicit :concourse:`steps.get` step - the information is not overwritten. -However, in the case of presigned URLs, AWS does **not** store these anywhere such that they are accessible from the server. When they are created, the user does not get given a UUID which allows them to "look up" the URL at a later date. Therefore, even if the :concourse:`put-step` of the resource created the URL and downloaded it to its resource directory, the :concourse:`get-step` would overwrite it with an empty folder. The two main solutions to this are: +However, in the case of presigned URLs, AWS does **not** store these anywhere such that they are accessible from the server. When they are created, the user does not get given a UUID which allows them to "look up" the URL at a later date. Therefore, even if the :concourse:`steps.put` step of the resource created the URL and downloaded it to its resource directory, the :concourse:`steps.get` step would overwrite it with an empty folder. The two main solutions to this are: 1. Pass the URL in the newly created version. -2. Have the :concourse:`put-step` write the URL to a *different* resource directory. +2. Have the :concourse:`steps.put` step write the URL to a *different* resource directory. -Option 2 requires another resource to be already available, as it isn't possible to request additional output directories like a :concourse:`task-step`. This is do-able but fragile, and requires care to make sure that no important files in the other resource are overwritten. Option 1 seems cleaner, but the URL might be sensitive, and storing it as plaintext within the version is definitely not ideal. We could consider encrypting it and having the user pass some sort of key, but this is complicating matters greatly. There is a better way. +Option 2 requires another resource to be already available, as it isn't possible to request additional output directories like a :concourse:`steps.task` step. This is do-able but fragile, and requires care to make sure that no important files in the other resource are overwritten. Option 1 seems cleaner, but the URL might be sensitive, and storing it as plaintext within the version is definitely not ideal. We could consider encrypting it and having the user pass some sort of key, but this is complicating matters greatly. There is a better way. -The :class:`~concoursetools.additional.InOnlyConcourseResource` is designed to run these "functions" in the :concourse:`get-step`, and to be *triggered* by a :concourse:`put-step`, like so: +The :class:`~concoursetools.additional.InOnlyConcourseResource` is designed to run these "functions" in the :concourse:`steps.get` step, and to be *triggered* by a :concourse:`steps.put` step, like so: .. code:: yaml @@ -46,7 +46,7 @@ All of the resource functionality comes from overloading the :meth:`~concourseto :pyobject: S3SignedURLConcourseResource.download_data :dedent: -This method takes a required ``file_path`` argument to indicate the object for which the URL should be generated. The ``expires_in`` parameter takes a mapping of arguments for :class:`datetime.timedelta` to allow users to specify expiration time more explicitly. Finally, passing a ``file_name`` instructs the URL to name the downloaded file something specific, rather than the original name from within S3. The URL itself is generated using the `generate_presigned_url `_ function. Finally, we don't return a version, so we only need to concern ourselves with the :ref:`Step Metadata`. +This method takes a required ``file_path`` argument to indicate the object for which the URL should be generated. The ``expires_in`` parameter takes a mapping of arguments for :class:`datetime.timedelta` to allow users to specify expiration time more explicitly. Finally, passing a ``file_name`` instructs the URL to name the downloaded file something specific, rather than the original name from within S3. The URL itself is generated using the `generate_presigned_url `_ function. Finally, we don't return a version, so we only need to concern ourselves with the :ref:`Step Metadata`. This resource can then be invoked like so: @@ -60,7 +60,7 @@ This resource can then be invoked like so: expires_in: hours: 24 -Once the implicit :concourse:`get-step` is completed, the URL can then be :concourse:`loaded easily ` and used in prior steps: +Once the implicit :concourse:`steps.get` step is completed, the URL can then be :concourse:`loaded easily ` and used in prior steps: .. code:: yaml diff --git a/docs/source/examples/secrets.rst b/docs/source/examples/secrets.rst index 314b8ac..566d77e 100644 --- a/docs/source/examples/secrets.rst +++ b/docs/source/examples/secrets.rst @@ -4,7 +4,7 @@ AWS Secrets .. caution:: This example is for reference only. It is not extensively tested, and it not intended to be a fully-fledged Concourse resource for production pipelines. Copy and paste at your own risk. -This example will showcase the :class:`~concoursetools.additional.TriggerOnChangeConcourseResource`, and how to build a resource to emit versions whenever something has changed, rather than when a new linear version becomes available. In this example, the resource will watch an secret in `AWS SecretsManager `_, and yield a new version whenever the value of that secret has changed. It will also allow the user to optionally download the secret value, and also to update the secret via a string or a file. The functionality will depend heavily on `boto3 `_ in order to reduce the amount of code needed to function. +This example will showcase the :class:`~concoursetools.additional.TriggerOnChangeConcourseResource`, and how to build a resource to emit versions whenever something has changed, rather than when a new linear version becomes available. In this example, the resource will watch an secret in `AWS SecretsManager `_, and yield a new version whenever the value of that secret has changed. It will also allow the user to optionally download the secret value, and also to update the secret via a string or a file. The functionality will depend heavily on `boto3 `_ in order to reduce the amount of code needed to function. Secrets Version @@ -33,7 +33,7 @@ With this resource type, we don't overload from :meth:`~concoursetools.resource. :pyobject: Resource.fetch_latest_version :dedent: -We use `list_secret_version_ids `_ with ``IncludeDeprecated=False`` to ensure that we only get versions which are either current or pending. We then iterate over the versions and find the one marked with ``AWSCURRENT``. If we don't find it then we raise an error to alert the user. +We use `list_secret_version_ids `_ with ``IncludeDeprecated=False`` to ensure that we only get versions which are either current or pending. We then iterate over the versions and find the one marked with ``AWSCURRENT``. If we don't find it then we raise an error to alert the user. Next, we overload :meth:`~concoursetools.resource.ConcourseResource.download_version` to allow us to download the metadata (and optionally the value) of the new secret: @@ -43,9 +43,9 @@ Next, we overload :meth:`~concoursetools.resource.ConcourseResource.download_ver The behaviour of the resource is as follows: -1. The metadata of the secret is fetched from AWS using `describe_secret `_. The response metadata is removed, but could potentially be output as :ref:`Step Metadata`. -2. The metadata is saved to a file. By default this is named ``metadata.json``, but the user can customise this with the parameters of the :concourse:`get-step`. -3. If the user has requested the secret value also (which is **not** the default behaviour), then this is fetched using `get_secret_value `_ to be saved to a file, which defaults to ``value`` but is again customisable by the user. +1. The metadata of the secret is fetched from AWS using `describe_secret `_. The response metadata is removed, but could potentially be output as :ref:`Step Metadata`. +2. The metadata is saved to a file. By default this is named ``metadata.json``, but the user can customise this with the parameters of the :concourse:`steps.get` step. +3. If the user has requested the secret value also (which is **not** the default behaviour), then this is fetched using `get_secret_value `_ to be saved to a file, which defaults to ``value`` but is again customisable by the user. 4. If the response contains a string, then this is written directly to the file using :meth:`~pathlib.Path.write_text`, but if it contains a binary then it is instead written using :meth:`~pathlib.Path.write_bytes`. The :mod:`json` module is imported as ``json_package`` to avoid a name collision with a future argument. The metadata of the secret contains some :class:`~datetime.datetime` objects, and so a custom :class:`~json.JSONEncoder` is required: @@ -62,7 +62,7 @@ Finally, we overload :meth:`~concoursetools.resource.ConcourseResource.publish_n The behaviour is as follows: 1. If the user has specified the secret value as JSON, then encode that as a string as pass it forward. -2. If the secret is specified as a string, then attempt to set the secret value with `put_secret_value `_. +2. If the secret is specified as a string, then attempt to set the secret value with `put_secret_value `_. 3. If the secret is specified as a file path, then use :meth:`~pathlib.Path.read_bytes` to get the contents of the file (more resilient than reading the text) and set the ``SecretBinary`` instead. 4. If none of these have been set, then raise an error. 5. Pull out the new ID from the response to establish the new version to return, and also pass the ``VersionStages`` as metadata to be output to the console. diff --git a/docs/source/extensions/concourse.py b/docs/source/extensions/concourse.py index 6ef594b..88920db 100644 --- a/docs/source/extensions/concourse.py +++ b/docs/source/extensions/concourse.py @@ -2,18 +2,13 @@ """ Minor Sphinx extension for creating Concourse documentation links, based on https://sphinx-toolbox.readthedocs.io/en/stable/extensions/wikipedia.html. -:concourse:`jobs` will create a link to /jobs.html with the text "jobs". -:concourse:`jobs.thing.thing2` will create a link to /jobs.html#thing1.thing2 with the text "thing2". -:concourse:`jobs-a-b-c` will create a link to /jobs-a-b-c.html with the text "jobs a b c". -:concourse:`job ` will create a link to /jobs.html with the text "job". -:concourse:`job ` will create a link to /jobs.html#thing1.thing2 with the text "job". +Calling :concourse:`jobs` will create a link to the Concourse documentation. Rules for parsing links and text can be +found in the docstring examples of the ``parse_link_and_title`` function. Set ``concourse_base_url`` in ``conf.py`` to change the URL used. It must contain "{target}" to be populated. """ from __future__ import annotations -from urllib.parse import quote - from docutils import nodes from docutils.nodes import system_message from docutils.parsers.rst.states import Inliner @@ -22,7 +17,7 @@ __all__ = ("make_concourse_link", "setup") -DEFAULT_BASE_URL = "https://concourse-ci.org/{target}" +DEFAULT_BASE_URL = "https://concourse-ci.org/docs/{target}" def make_concourse_link(name: str, rawtext: str, text: str, lineno: int, inliner: Inliner, options: dict[str, object] = {}, @@ -45,24 +40,50 @@ def make_concourse_link(name: str, rawtext: str, text: str, lineno: int, inliner :return: A list containing the created node, and a list containing any messages generated during the function. """ text = nodes.unescape(text) - _, title, target = split_explicit_title(text) - - title = title.split(".")[-1].replace("-", " ") - - page, *anchors = quote(target.replace(" ", "_"), safe="").split(".", maxsplit=1) - if anchors: - anchor = ".".join(anchors) - new_target = f"{page}.html#{anchor}" - else: - new_target = f"{page}.html" + target, title = parse_link_and_title(text) base_url: str = inliner.document.settings.env.config.concourse_base_url - ref = base_url.format(target=new_target) + ref = base_url.format(target=target) node = nodes.reference(rawtext, title, refuri=str(ref), **options) return [node], [] +def parse_link_and_title(text: str) -> tuple[str, str]: + """ + Parse the link and title from text. + + :Example: + >>> parse_link_and_title("steps") + ('steps/', 'steps') + >>> parse_link_and_title("steps.set_pipeline") + ('steps/set-pipeline/', 'set_pipeline') + >>> parse_link_and_title("steps.set_pipeline#instance_vars") + ('steps/set-pipeline/#instance_vars', 'instance_vars') + >>> parse_link_and_title("step ") + ('steps/', 'step') + >>> parse_link_and_title("setting pipeline ") + ('steps/set-pipeline/', 'setting pipeline') + >>> parse_link_and_title("pipeline vars ") + ('steps/set-pipeline/#instance_vars', 'pipeline vars') + """ + _, title, target = split_explicit_title(text) + + new_title = title.split(".")[-1].split("#")[-1].replace("-", " ") + + try: + page, anchor = target.split("#") + except ValueError: + page = target + anchor = "" + + new_page = page.replace("_", "-").replace(".", "/") + new_target = f"{new_page}/" + new_target = f"{new_page}/#{anchor}" if anchor else f"{new_page}/" + + return new_target, new_title + + def setup(app: Sphinx) -> dict[str, object]: """ Attach the extension to the application. diff --git a/docs/source/index.rst b/docs/source/index.rst index c251d77..9bc548f 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -3,7 +3,7 @@ Concourse Tools (|version|) .. admonition:: Description - A Python package for easily implementing Concourse :concourse:`resource types `, created by GCHQ. + A Python package for easily implementing Concourse :concourse:`resource types `, created by GCHQ. diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst index 17517c6..e835750 100644 --- a/docs/source/quickstart.rst +++ b/docs/source/quickstart.rst @@ -9,7 +9,7 @@ To begin, install **Concourse Tools** from source or from PyPI (see the footer f $ pip install concoursetools Start by familiarising yourself with the Concourse resource "rules" in the -:concourse:`documentation `. To recreate that example, start by creating a new +:concourse:`documentation `. To recreate that example, start by creating a new ``concourse.py`` file in your repository. The first step is to create a :class:`~concoursetools.version.Version` subclass: diff --git a/docs/source/resource.rst b/docs/source/resource.rst index 9baa6b3..e3b567a 100644 --- a/docs/source/resource.rst +++ b/docs/source/resource.rst @@ -20,7 +20,8 @@ The :meth:`~concoursetools.resource.ConcourseResource.download_version` and :met .. figure:: _static/step_metadata.png :scale: 75% - Concourse metadata example from the :concourse:`git-trigger-example.git-trigger-example`. The red outline shows the metadata. + Concourse metadata example from the `git trigger example `_. + The red outline shows the metadata. This metadata dictionary should contain string-only key/value pairs. This is enforced by Concourse Tools, which casts both to strings before emitting any JSON. This is not easily overloaded, as the metadata is only ever intended for visual consumption. diff --git a/docs/source/typing.rst b/docs/source/typing.rst index 8a62e3a..49a067a 100644 --- a/docs/source/typing.rst +++ b/docs/source/typing.rst @@ -6,7 +6,7 @@ Typing Concourse Types --------------- -The following types map onto Concourse :concourse:`config-basics.basic-schemas` used throughout the library. +The following types map onto Concourse :concourse:`config-basics#basic-schemas` used throughout the library. .. autodata:: concoursetools.typing.ResourceConfig