From af89bd873dea67ac80568b0668dc9f82ca1f5043 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Tue, 4 Aug 2026 18:00:47 -0700 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat(create):=20record=20environmen?= =?UTF-8?q?ts=20per=20PEP=20832?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Editors and type checkers have no standard way to find a project's environments without an activated shell, so each one hard-codes a search per tool that creates environments. PEP 832 replaces that with a .python-envs file whose last line names the default environment. virtualenv creates environments, so it covers the write half. After creation it records the destination in the .python-envs file of the parent folder and moves any existing entry for it to the end, making the fresh environment the default. A destination named .venv is left alone, since the PEP counts it as the implicit last line. A read or write failure warns and leaves the environment usable. Pass --no-python-envs to opt out. Recording rewrites the whole file, which loses entries when several environments land in one folder at once: eight concurrent creations dropped between one and four of them on every attempt. The read-modify-write therefore runs under a .python-envs.lock file next to the recorded one. Locking beside the resource rather than inside the app data folder holds the guarantee when the app data is read-only or disabled, and gives any other PEP 832 writer a location to agree on. Documented across all four Diataxis dimensions, including a new reference page cataloguing every file virtualenv writes inside and beside an environment. --- docs/changelog/3204.feature.rst | 3 + docs/explanation.rst | 55 ++++++++++++++- docs/how-to/usage.rst | 60 +++++++++++++++++ docs/index.rst | 2 + docs/plugin/tutorial.rst | 2 +- docs/reference/files.rst | 96 +++++++++++++++++++++++++++ docs/tutorial/getting-started.rst | 15 ++++- src/virtualenv/create/creator.py | 34 ++++++++++ tests/unit/create/test_python_envs.py | 85 ++++++++++++++++++++++++ 9 files changed, 348 insertions(+), 4 deletions(-) create mode 100644 docs/changelog/3204.feature.rst create mode 100644 docs/reference/files.rst create mode 100644 tests/unit/create/test_python_envs.py diff --git a/docs/changelog/3204.feature.rst b/docs/changelog/3204.feature.rst new file mode 100644 index 000000000..ac1747c91 --- /dev/null +++ b/docs/changelog/3204.feature.rst @@ -0,0 +1,3 @@ +Record the created environment as the default one of its parent folder within a ``.python-envs`` file, per `PEP 832 +`_, so editors and type checkers can find it. Parallel creations under one folder +serialize through a ``.python-envs.lock`` file next to it. Skip both with ``--no-python-envs`` - by :user:`gaborbernat`. diff --git a/docs/explanation.rst b/docs/explanation.rst index 845dc02c7..006dd92dd 100644 --- a/docs/explanation.rst +++ b/docs/explanation.rst @@ -147,7 +147,8 @@ virtualenv operates in two distinct phases: CreatePython --> SeedPackages[Install seed packages: pip, setuptools, wheel] SeedPackages --> ActivationScripts[Install activation scripts] ActivationScripts --> VCSIgnore[Create VCS ignore files] - VCSIgnore --> Complete([Virtual environment ready]) + VCSIgnore --> Record[Record in .python-envs] + Record --> Complete([Virtual environment ready]) style Start fill:#2563eb,stroke:#1d4ed8,color:#fff style Phase1 fill:#6366f1,stroke:#4f46e5,color:#fff @@ -161,12 +162,14 @@ virtualenv operates in two distinct phases: flag to specify a different interpreter. **Phase 2: Create the virtual environment** - Once the target interpreter is identified, virtualenv creates the environment in four steps: + Once the target interpreter is identified, virtualenv creates the environment in five steps: 1. Create a Python executable matching the target interpreter 2. Install seed packages (pip, setuptools, wheel) to enable package installation 3. Install activation scripts for various shells 4. Create VCS ignore files (currently Git's ``.gitignore``, skip with ``--no-vcs-ignore``) + 5. Record the environment in a ``.python-envs`` file so editors and type checkers can find it (`PEP 832 + `_, skip with ``--no-python-envs``) An important design principle: virtual environments are not self-contained. A complete Python installation consists of thousands of files, and copying all of them into every virtual environment would be wasteful. Instead, virtual @@ -549,6 +552,54 @@ For a deeper dive into how activation works under the hood, see Allison Kaptur's edition `_, which explains how virtualenv uses ``PATH`` and ``PYTHONHOME`` to isolate virtual environments. +*********************** + Environment discovery +*********************** + +Editors, type checkers and task runners need to find a project's environments, and they cannot count on an activated +shell to tell them where those are. Launching an editor on a fresh checkout leaves it guessing. Each tool has answered +that by hard-coding a search per environment manager it wants to support, which is why editor support for any new tool +lags behind the tool itself. + +`PEP 832 `_ replaces the guessing with a file. A project may carry a ``.python-envs`` +file listing one environment per line, and the tools that create environments keep it current. virtualenv creates +environments, so it writes the file; reading it belongs to the tools consuming it. + +After creating ``/``, virtualenv records ```` in ``/.python-envs``: + +.. code-block:: console + + $ virtualenv env + $ virtualenv other + $ cat .python-envs + env + other + +**The last line is the default** + A tool asking "which environment should I use" takes the final entry, so the environment created last wins. An entry + pointing at the same destination moves to the end instead of being repeated. + +**A ``.venv`` folder needs no line** + PEP 832 counts a ``.venv`` folder beside the file as its implicit last line. Creating ``.venv`` therefore writes + nothing, and where a ``.venv`` folder exists it stays the default whatever ``.python-envs`` lists. virtualenv logs a + debug message when it records an environment that a sibling ``.venv`` outranks. + +**Recording never breaks creation** + A read or write failure logs a warning and leaves you with a working environment. Discovery is a convenience for + other tools, so it does not get to fail the job you asked for. + +Why a lock file +=============== + +Recording rewrites the whole file, which loses entries when several environments land in one folder at once. A task +runner building a matrix does that. virtualenv serializes the read-modify-write through a ``.python-envs.lock`` file, +which is why a second file appears next to ``.python-envs`` on every platform except Windows, where the lock file goes +away as the lock releases. + +The lock sits beside the file it guards rather than in the app data folder for two reasons. It keeps working when the +app data is read-only or disabled, and it gives any other PEP 832 writer a location to agree on, which a path private to +virtualenv could not. + ********** See also ********** diff --git a/docs/how-to/usage.rst b/docs/how-to/usage.rst index 4c1fca11a..91e370cf7 100644 --- a/docs/how-to/usage.rst +++ b/docs/how-to/usage.rst @@ -286,6 +286,66 @@ Options are resolved in this order (highest to lowest priority): style C fill:#d97706,stroke:#b45309,color:#fff style D fill:#6366f1,stroke:#4f46e5,color:#fff +******************************** + Make environments discoverable +******************************** + +virtualenv records every environment it creates in a ``.python-envs`` file next to it, so editors and type checkers can +find them without an activated shell. See `PEP 832 `_ for the format and +:ref:`explanation:Environment discovery` for the reasoning. + +Point a tool at the right environment +===================================== + +The last line of ``.python-envs`` is the default environment, and the environment you create last takes that spot: + +.. code-block:: console + + $ virtualenv py313 --python 3.13 + $ virtualenv py314 --python 3.14 + $ cat .python-envs + py313 + py314 + +To promote ``py313`` back to the default, create it again over the existing folder. Its entry moves to the end rather +than repeating: + +.. code-block:: console + + $ virtualenv py313 --python 3.13 + $ cat .python-envs + py314 + py313 + +A ``.venv`` folder outranks every line in the file, so delete or rename it if you want another environment to win. + +Skip recording +============== + +Pass ``--no-python-envs`` when you do not want the ``.python-envs`` and ``.python-envs.lock`` files: + +.. code-block:: console + + $ virtualenv env --no-python-envs + +Set it once for every environment you create through the configuration file or an environment variable: + +.. code-block:: ini + + [virtualenv] + no_python_envs = true + +.. code-block:: console + + $ export VIRTUALENV_NO_PYTHON_ENVS=1 + +Commit the file or ignore it +============================ + +Commit ``.python-envs`` when the environment locations are the same for everyone on the project, such as a containerized +setup or a fixed tox layout. Add it to ``.gitignore`` when developers pick their own paths. Always ignore +``.python-envs.lock``, which is a local lock file rather than project configuration. + *********************** Control seed packages *********************** diff --git a/docs/index.rst b/docs/index.rst index 050eed9f5..e4dce04b3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -61,6 +61,7 @@ into the standard library under the ``venv`` module. For how ``virtualenv`` comp - :doc:`reference/compatibility` — Supported Python versions and operating systems - :doc:`reference/cli` — Command line options and flags +- :doc:`reference/files` — Files written inside and beside a created environment - :doc:`reference/api` — Programmatic Python API reference **Explanation** - Understand the concepts @@ -115,6 +116,7 @@ Learn more about virtualenv from these community resources: reference/compatibility reference/cli + reference/files reference/api .. toctree:: diff --git a/docs/plugin/tutorial.rst b/docs/plugin/tutorial.rst index b6f4a7e55..71e349ecc 100644 --- a/docs/plugin/tutorial.rst +++ b/docs/plugin/tutorial.rst @@ -112,7 +112,7 @@ The output should list ``pyenv`` as an available discovery mechanism. You can no $ virtualenv --discovery=pyenv myenv created virtual environment CPython3.11.0.final.0-64 in 234ms - creator CPython3Posix(dest=/path/to/myenv, clear=False, no_vcs_ignore=False, global=False) + creator CPython3Posix(dest=/path/to/myenv, clear=False, no_vcs_ignore=False, no_python_envs=False, global=False) seeder FromAppData(download=False, pip=bundle, setuptools=bundle, wheel=bundle, via=copy, app_data_dir=/path) added seed packages: pip==23.0, setuptools==65.5.0, wheel==0.38.4 activators BashActivator,CShellActivator,FishActivator,NushellActivator,PowerShellActivator,PythonActivator diff --git a/docs/reference/files.rst b/docs/reference/files.rst new file mode 100644 index 000000000..87879452e --- /dev/null +++ b/docs/reference/files.rst @@ -0,0 +1,96 @@ +################# + Generated files +################# + +Creating an environment writes files inside the destination folder and beside it. This page lists each one, what it +holds, and how to suppress it. + +******************************* + Inside the environment folder +******************************* + +``pyvenv.cfg`` +============== + +Marks the folder as a virtual environment and points the interpreter at the Python it was built from, per `PEP 405 +`_. Deleting it breaks the environment. + +.. code-block:: ini + + home = /usr/local/python-3.14/bin + implementation = CPython + version_info = 3.14.6.final.0 + version = 3.14.6 + executable = /usr/local/python-3.14/bin/python3.14 + command = /usr/bin/python3 -m virtualenv /home/user/env + virtualenv = 21.7.1 + include-system-site-packages = false + base-prefix = /usr/local/python-3.14 + base-exec-prefix = /usr/local/python-3.14 + base-executable = /usr/local/python-3.14/bin/python3.14 + +``prompt`` appears as an extra key when you pass ``--prompt``. The ``base-*`` keys come from the creator and vary by +creation method. + +``CACHEDIR.TAG`` +================ + +Marks the environment as regenerable cache content, following the `cache directory tagging specification +`_, so backup tools skip it. virtualenv leaves an existing file untouched. + +``.gitignore`` +============== + +Holds ``*``, keeping the environment out of Git. Skip it with ``--no-vcs-ignore``. virtualenv leaves an existing file +untouched, and writes nothing for Mercurial, Bazaar or Subversion, none of which honor ignore files in a subdirectory. + +``bin`` / ``Scripts`` +===================== + +The interpreter, the console scripts of any seeded package, and the activation scripts for each shell. See +:ref:`explanation:Activators` for the full list. + +******************************* + Beside the environment folder +******************************* + +``.python-envs`` +================ + +Lists the environments of the parent folder, one per line, with the last line naming the default one, per `PEP 832 +`_. Skip it with ``--no-python-envs``. + +.. code-block:: text + + py313 + py314 + +Format rules virtualenv follows when it rewrites the file: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + - - Rule + - Behavior + - - Encoding + - UTF-8. + - - Entry + - The destination folder name, since the file sits in the destination's parent. Absolute entries written by other + tools are read and preserved. + - - Default + - The last line. A new environment goes last, and an entry already pointing at it moves there rather than + repeating. + - - ``.venv`` + - Never written, because PEP 832 counts a ``.venv`` folder beside the file as its implicit last line. + - - Blank lines + - Dropped on rewrite. + - - Failure + - Logged as a warning; the environment is still created. + +``.python-envs.lock`` +===================== + +An empty lock file serializing concurrent rewrites of ``.python-envs``. It carries no content. On Windows it disappears +as the lock releases; on other platforms it stays behind, and you can delete it while no virtualenv is running. +Suppressed by ``--no-python-envs`` along with the file it guards, and worth adding to your VCS ignore list. diff --git a/docs/tutorial/getting-started.rst b/docs/tutorial/getting-started.rst index b92a5c4c6..ecda59a72 100644 --- a/docs/tutorial/getting-started.rst +++ b/docs/tutorial/getting-started.rst @@ -26,13 +26,24 @@ Let's create a virtual environment called ``myproject``: $ virtualenv myproject created virtual environment CPython3.13.2.final.0-64 in 200ms - creator CPython3Posix(dest=/home/user/myproject, clear=False, no_vcs_ignore=False, global=False) + creator CPython3Posix(dest=/home/user/myproject, clear=False, no_vcs_ignore=False, no_python_envs=False, global=False) seeder FromAppData(download=False, pip=bundle, setuptools=bundle, via=copy, app_data_dir=/home/user/.cache/virtualenv) activators BashActivator,CShellActivator,FishActivator,NushellActivator,PowerShellActivator,PythonActivator This creates a new directory called ``myproject`` containing a complete, isolated Python environment with its own copy of Python, pip, and other tools. +Alongside it you get a ``.python-envs`` file naming the environment you just made: + +.. code-block:: console + + $ cat .python-envs + myproject + +Editors and type checkers read that file to find your environment, so they can offer the right interpreter before you +activate anything. Create a second environment here and its name joins the list, with the newest one last. Pass +``--no-python-envs`` if you would rather virtualenv left no trace outside the environment folder. + ************************** Activate the environment ************************** @@ -232,6 +243,7 @@ In this tutorial, you learned how to: - Install packages in isolation from your system Python. - Save project dependencies with ``pip freeze``. - Reproduce environments using ``requirements.txt``. +- Let editors find your environments through the ``.python-envs`` file. ************ Next steps @@ -242,3 +254,4 @@ Now that you understand the basics, explore these topics: - :doc:`../how-to/usage` for selecting specific Python versions, configuring defaults, and advanced usage patterns. - :doc:`../explanation` for understanding how virtualenv works under the hood and how it compares to ``venv``. - :doc:`../reference/cli` for all available command line options and flags. +- :doc:`../reference/files` for every file virtualenv writes inside and beside an environment. diff --git a/src/virtualenv/create/creator.py b/src/virtualenv/create/creator.py index 800eb28d7..09113eafa 100644 --- a/src/virtualenv/create/creator.py +++ b/src/virtualenv/create/creator.py @@ -23,6 +23,7 @@ from os.path import commonpath +from virtualenv.util.lock import ReentrantFileLock from virtualenv.util.path import safe_delete from virtualenv.util.subprocess import LogCmd, run_cmd from virtualenv.version import __version__ @@ -54,6 +55,7 @@ def __init__(self, options: VirtualEnvOptions, interpreter: PythonInfo) -> None: self.dest = Path(options.dest) self.clear = options.clear self.no_vcs_ignore = options.no_vcs_ignore + self.no_python_envs = options.no_python_envs self.pyenv_cfg = PyEnvCfg.from_folder(self.dest) self.app_data = options.app_data self.env = options.env @@ -90,6 +92,7 @@ def _args(self) -> list[tuple[str, Any]]: ("dest", str(self.dest)), ("clear", self.clear), ("no_vcs_ignore", self.no_vcs_ignore), + ("no_python_envs", self.no_python_envs), ] @classmethod @@ -139,6 +142,13 @@ def add_parser_arguments( help="don't create VCS ignore directive in the destination directory", default=False, ) + parser.add_argument( + "--no-python-envs", + dest="no_python_envs", + action="store_true", + help="don't record the created environment within a PEP-832 .python-envs file next to the destination", + default=False, + ) @abstractmethod def create(self) -> None: @@ -204,6 +214,8 @@ def run(self) -> None: self.set_pyenv_cfg() if not self.no_vcs_ignore: self.setup_ignore_vcs() + if not self.no_python_envs: + self.record_python_envs() def add_cachedir_tag(self) -> None: """Generate a file indicating that this is not meant to be backed up.""" @@ -244,6 +256,28 @@ def setup_ignore_vcs(self) -> None: # Bazaar - does not support ignore files in sub-directories, only at root level via .bzrignore # Subversion - does not support ignore files, requires direct manipulation with the svn tool + def record_python_envs(self) -> None: + """Register the environment as the default one of its parent folder, per PEP-832.""" + if self.dest.name == ".venv": # implicitly the last line of .python-envs, no need to record it + return + root = self.dest.parent + if (root / ".venv" / "pyvenv.cfg").exists(): + LOGGER.debug(".venv keeps being the default environment of %s", root) + envs_file = root / ".python-envs" + target = os.path.normcase(os.path.normpath(str(self.dest))) + try: + # concurrent creations under one folder rewrite the same file, so serialize them across processes + with ReentrantFileLock(root).lock_for_key(envs_file.name): + # the last entry is the default environment, so an entry already pointing at us must move to the end + keep = [ + i + for i in (envs_file.read_text(encoding="utf-8") if envs_file.exists() else "").splitlines() + if i.strip() and os.path.normcase(os.path.normpath(os.path.join(str(root), i))) != target + ] + envs_file.write_text("".join(f"{i}\n" for i in [*keep, self.dest.name]), encoding="utf-8") + except OSError as exc: + LOGGER.warning("could not record %s within %s - %s", self.dest, envs_file, exc) + @property def debug(self) -> dict[str, Any] | None: """Debug information about the virtual environment (only valid after :meth:`create` has run).""" diff --git a/tests/unit/create/test_python_envs.py b/tests/unit/create/test_python_envs.py new file mode 100644 index 000000000..987b6a16e --- /dev/null +++ b/tests/unit/create/test_python_envs.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import logging +import os +from stat import S_IREAD, S_IWRITE +from threading import Thread +from typing import TYPE_CHECKING + +import pytest + +from virtualenv.run import cli_run + +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.fixture +def envs_file(tmp_path: Path) -> Path: + return tmp_path / ".python-envs" + + +@pytest.mark.parametrize( + ("before", "after"), + [ + pytest.param(None, "env\n", id="no-file-yet"), + pytest.param("other\n/opt/elsewhere\n", "other\n/opt/elsewhere\nenv\n", id="appended-last"), + pytest.param("env\nother\n", "other\nenv\n", id="relative-entry-moved-last"), + pytest.param("{dest}\nother\n", "other\nenv\n", id="absolute-entry-moved-last"), + pytest.param("other\r\n\r\n \r\n", "other\nenv\n", id="blank-lines-dropped"), + ], +) +def test_python_envs_recorded(tmp_path: Path, envs_file: Path, before: str | None, after: str) -> None: + dest = tmp_path / "env" + if before is not None: + envs_file.write_bytes(before.format(dest=dest.resolve()).encode("utf-8")) + _create(dest) + assert envs_file.read_text(encoding="utf-8") == after + + +def _create(dest: Path, *args: str) -> None: + cli_run([str(dest), "--without-pip", "--activators", "", *args], setup_logging=False) + + +@pytest.mark.parametrize( + ("name", "args"), + [ + pytest.param("env", ("--no-python-envs",), id="opted-out"), + pytest.param(".venv", (), id="dot-venv-is-implicit"), + ], +) +def test_python_envs_not_recorded(tmp_path: Path, envs_file: Path, name: str, args: tuple[str, ...]) -> None: + _create(tmp_path / name, *args) + assert not envs_file.exists() + + +def test_python_envs_records_parallel_creations(tmp_path: Path, envs_file: Path) -> None: + names = [f"env-{i}" for i in range(8)] + threads = [Thread(target=_create, args=(tmp_path / i,)) for i in names] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert sorted(envs_file.read_text(encoding="utf-8").splitlines()) == names + + +def test_python_envs_dot_venv_keeps_precedence(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + _create(tmp_path / ".venv") + with caplog.at_level(logging.DEBUG): + _create(tmp_path / "env") + assert ".venv keeps being the default environment" in caplog.text + + +def test_python_envs_not_write_able(tmp_path: Path, envs_file: Path, caplog: pytest.LogCaptureFixture) -> None: + if hasattr(os, "geteuid") and os.geteuid() == 0: + pytest.skip("root may write read-only files") + + envs_file.write_text("other\n", encoding="utf-8") + envs_file.chmod(S_IREAD) + try: + with caplog.at_level(logging.WARNING): + _create(tmp_path / "env") + finally: + envs_file.chmod(S_IREAD | S_IWRITE) + assert f"could not record {(tmp_path / 'env').resolve()} within {envs_file.resolve()}" in caplog.text + assert (tmp_path / "env" / "pyvenv.cfg").exists()