-
Notifications
You must be signed in to change notification settings - Fork 0
Add snapshot file locking #2
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
6 commits
Select commit
Hold shift + click to select a range
4e4c511
Add snapshot file locking
Mattie 6c7f0dc
Address PR review comments
Mattie 379b9cf
Address follow-up PR review comments
Mattie 57ae8c5
Address additional lock review comments
Mattie d5a5911
Address lock sidecar review comments
Mattie 58ff8da
Reserve lock snapshot suffix
Mattie File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Some comments aren't visible on the classic Files Changed page.
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -15,3 +15,5 @@ htmlcov/ | |
| .mypy_cache/ | ||
| .ruff_cache/ | ||
| .pyright/ | ||
|
|
||
| .todo/ | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,142 @@ | ||
| from __future__ import annotations | ||
|
|
||
| import errno | ||
| import os | ||
| import threading | ||
| import time | ||
| from collections.abc import Iterator | ||
| from contextlib import contextmanager | ||
| from pathlib import Path | ||
| from typing import BinaryIO | ||
|
|
||
|
|
||
| class _PathLockState: | ||
| def __init__(self) -> None: | ||
| self.lock = threading.RLock() | ||
| self.depth = 0 | ||
| self.handle: BinaryIO | None = None | ||
|
|
||
|
|
||
| _LOCKS: dict[Path, _PathLockState] = {} | ||
| _LOCKS_GUARD = threading.Lock() | ||
|
|
||
|
|
||
| def write_lock_for(path: Path) -> threading.RLock: | ||
| return _lock_state_for(path).lock | ||
|
|
||
|
|
||
| @contextmanager | ||
| def locked_path(path: Path) -> Iterator[None]: | ||
| normalized_path = _normalized_path(path) | ||
| state = _lock_state_for(normalized_path) | ||
| state.lock.acquire() | ||
| try: | ||
| if state.depth == 0: | ||
| state.handle = _acquire_os_lock(normalized_path) | ||
| state.depth += 1 | ||
| try: | ||
| yield | ||
| finally: | ||
| state.depth -= 1 | ||
| if state.depth == 0: | ||
| handle = state.handle | ||
| state.handle = None | ||
| if handle is not None: | ||
| _release_os_lock(handle) | ||
| finally: | ||
| state.lock.release() | ||
|
|
||
|
|
||
| def _lock_state_for(path: Path) -> _PathLockState: | ||
| key = _normalized_path(path) | ||
| with _LOCKS_GUARD: | ||
| state = _LOCKS.get(key) | ||
| if state is None: | ||
| state = _PathLockState() | ||
| _LOCKS[key] = state | ||
| return state | ||
|
|
||
|
|
||
| def _normalized_path(path: Path) -> Path: | ||
| absolute = path if path.is_absolute() else Path.cwd() / path | ||
| # Atomic replace writes to the final path itself, so resolve parent | ||
| # directories but keep the leaf name instead of following a leaf symlink. | ||
| return absolute.parent.resolve(strict=False) / absolute.name | ||
|
|
||
|
|
||
| def _lock_path_for(path: Path) -> Path: | ||
| return path.with_name(f"{path.name}.lock") | ||
|
Mattie marked this conversation as resolved.
|
||
|
|
||
|
|
||
| def _is_lock_path(path: Path) -> bool: | ||
| return path.name.lower().endswith(".lock") | ||
|
|
||
|
|
||
| def _acquire_os_lock(path: Path) -> BinaryIO: | ||
| lock_path = _lock_path_for(path) | ||
| lock_path.parent.mkdir(parents=True, exist_ok=True) | ||
| handle = lock_path.open("a+b") | ||
|
Mattie marked this conversation as resolved.
|
||
| try: | ||
| if os.name == "nt": | ||
| _acquire_windows_lock(handle) | ||
| else: | ||
| _acquire_posix_lock(handle) | ||
| except Exception: | ||
| handle.close() | ||
| raise | ||
| return handle | ||
|
|
||
|
|
||
| def _release_os_lock(handle: BinaryIO) -> None: | ||
| try: | ||
| if os.name == "nt": | ||
| _release_windows_lock(handle) | ||
| else: | ||
| _release_posix_lock(handle) | ||
| finally: | ||
| handle.close() | ||
|
|
||
|
|
||
| def _acquire_windows_lock(handle: BinaryIO) -> None: | ||
| import msvcrt | ||
|
|
||
| handle.seek(0, os.SEEK_END) | ||
| if handle.tell() == 0: | ||
| handle.write(b"\0") | ||
| handle.flush() | ||
| while True: | ||
| try: | ||
| handle.seek(0) | ||
| msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) | ||
| return | ||
| except OSError as exc: | ||
| if not _is_windows_lock_contention(exc): | ||
| raise | ||
| time.sleep(0.05) | ||
|
|
||
|
|
||
| def _is_windows_lock_contention(exc: OSError) -> bool: | ||
| winerror = getattr(exc, "winerror", None) | ||
| if winerror is not None: | ||
| return winerror in {32, 33} | ||
| # CPython's msvcrt.locking reports byte-range lock contention this way. | ||
| return exc.errno in {errno.EACCES, errno.EDEADLK} | ||
|
|
||
|
|
||
| def _release_windows_lock(handle: BinaryIO) -> None: | ||
| import msvcrt | ||
|
|
||
| handle.seek(0) | ||
| msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) | ||
|
|
||
|
|
||
| def _acquire_posix_lock(handle: BinaryIO) -> None: | ||
| import fcntl | ||
|
|
||
| fcntl.flock(handle.fileno(), fcntl.LOCK_EX) | ||
|
|
||
|
|
||
| def _release_posix_lock(handle: BinaryIO) -> None: | ||
| import fcntl | ||
|
|
||
| fcntl.flock(handle.fileno(), fcntl.LOCK_UN) | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.