From c8156e83d2dc538a737f9ba7baea5d649530a573 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sat, 11 Oct 2025 21:24:57 +0200 Subject: [PATCH 01/13] tests: adjust fsspec compat test --- upath/tests/cases.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/upath/tests/cases.py b/upath/tests/cases.py index 8b6e4fc0..b05a52ab 100644 --- a/upath/tests/cases.py +++ b/upath/tests/cases.py @@ -1,6 +1,5 @@ import os import pickle -import re import stat import sys import warnings @@ -380,12 +379,8 @@ def test_fsspec_compat(self): fs = self.path.fs content = b"a,b,c\n1,2,3\n4,5,6" - def strip_scheme(path): - root = "" if sys.platform.startswith("win") else "/" - return root + re.sub("^[a-z0-9]+:/*", "", str(path)) - upath1 = self.path / "output1.csv" - p1 = strip_scheme(upath1) + p1 = upath1.path upath1.write_bytes(content) assert fs is upath1.fs with fs.open(p1) as f: @@ -394,7 +389,7 @@ def strip_scheme(path): # write with fsspec, read with upath upath2 = self.path / "output2.csv" - p2 = strip_scheme(upath2) + p2 = upath2.path assert fs is upath2.fs with fs.open(p2, "wb") as f: f.write(content) From be563eac3cc2c1d84fbd9464f6cc7783c5ad02dc Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 12 Oct 2025 14:09:42 +0200 Subject: [PATCH 02/13] tests: adjust intended behavior for tests --- upath/tests/cases.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/upath/tests/cases.py b/upath/tests/cases.py index b05a52ab..56bb0141 100644 --- a/upath/tests/cases.py +++ b/upath/tests/cases.py @@ -268,7 +268,7 @@ def test_rename(self): assert not upath.exists() assert moved.exists() # reverse with an absolute path as str - back = moved.rename(str(upath)) + back = moved.rename(upath.path) assert back == upath assert not moved.exists() assert back.exists() @@ -382,7 +382,9 @@ def test_fsspec_compat(self): upath1 = self.path / "output1.csv" p1 = upath1.path upath1.write_bytes(content) - assert fs is upath1.fs + assert fs._fs_token == upath1.fs._fs_token + if fs.cachable: # codespell:ignore cachable + assert fs is upath1.fs with fs.open(p1) as f: assert f.read() == content upath1.unlink() @@ -390,7 +392,8 @@ def test_fsspec_compat(self): # write with fsspec, read with upath upath2 = self.path / "output2.csv" p2 = upath2.path - assert fs is upath2.fs + if fs.cachable: # codespell:ignore cachable + assert fs is upath2.fs with fs.open(p2, "wb") as f: f.write(content) assert upath2.read_bytes() == content @@ -419,8 +422,10 @@ def test_pickling_child_path(self): assert path.storage_options == recovered_path.storage_options def test_child_path(self): - path_str = str(self.path) - path_a = UPath(path_str, "folder", **self.path.storage_options) + path_str = self.path.__vfspath__() + path_a = UPath( + path_str, "folder", protocol=self.path.protocol, **self.path.storage_options + ) path_b = self.path / "folder" assert str(path_a) == str(path_b) From deb1bf38517ecd7708f0486e12cfdbda52125a88 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 12 Oct 2025 14:10:24 +0200 Subject: [PATCH 03/13] upath._chain: fix unchain behavior --- upath/_chain.py | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/upath/_chain.py b/upath/_chain.py index 63c7ffc3..627a4a75 100644 --- a/upath/_chain.py +++ b/upath/_chain.py @@ -172,8 +172,14 @@ def unchain(self, path: str, kwargs: dict[str, Any]) -> list[ChainSegment]: first_bit_protocol: str | None = kwargs.pop("protocol", None) it_bits = iter(path.split(self.link)) bits: list[str] - if first_bit_protocol is not None: - bits = [next(it_bits)] + if first_bit_protocol == (bit := next(it_bits)) and first_bit_protocol in { + "blockcache", + "filecache", + "simplecache", + }: + bits = [""] + elif first_bit_protocol is not None: + bits = [bit] else: bits = [] for p in it_bits: @@ -191,7 +197,6 @@ def unchain(self, path: str, kwargs: dict[str, Any]) -> list[ChainSegment]: # [[url, protocol, kwargs], ...] out: list[ChainSegment] = [] - previous_bit: str | None = None kwargs = kwargs.copy() first_bit_idx = len(bits) - 1 for idx, bit in enumerate(reversed(bits)): @@ -206,19 +211,22 @@ def unchain(self, path: str, kwargs: dict[str, Any]) -> list[ChainSegment]: kws.update(kwargs) kw = dict(**extra_kwargs) kw.update(kws) - if "target_protocol" in kw: - kw.setdefault("target_options", {}) - bit = flavour.strip_protocol(bit) or flavour.root_marker - if ( - protocol in {"blockcache", "filecache", "simplecache"} - and "target_protocol" not in kw - ): - out.append(ChainSegment(None, protocol, kw)) - if previous_bit is not None: - bit = previous_bit + if protocol in {"blockcache", "filecache", "simplecache"}: + # passthrough filesystems + if bit: + kw["fo"] = bit # codespell:ignore fo + bit = None else: - out.append(ChainSegment(bit, protocol, kw)) - previous_bit = bit + bit = flavour.strip_protocol(bit) or flavour.root_marker + if "target_protocol" in kw: + out.append( + ChainSegment( + kw.pop("fo", None), # codespell:ignore fo + kw.pop("target_protocol"), + kw.pop("target_options", {}), + ) + ) + out.append(ChainSegment(bit, protocol, kw)) out.reverse() return out From 401b4189d80f57f1160e1ce3ff1c6c271271fd95 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 12 Oct 2025 14:11:24 +0200 Subject: [PATCH 04/13] upath.implementations.cached: add simplecachepath --- upath/implementations/cached.py | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/upath/implementations/cached.py b/upath/implementations/cached.py index de01fb3e..762f3c97 100644 --- a/upath/implementations/cached.py +++ b/upath/implementations/cached.py @@ -1,19 +1,27 @@ from __future__ import annotations import sys +from types import MappingProxyType from typing import TYPE_CHECKING from upath.core import UPath from upath.types import JoinablePathLike if TYPE_CHECKING: + from collections.abc import Iterator + from collections.abc import Mapping + from typing import Any from typing import Literal if sys.version_info >= (3, 11): + from typing import Self from typing import Unpack else: + from typing_extensions import Self from typing_extensions import Unpack + from fsspec import AbstractFileSystem + from upath._chain import FSSpecChainParser from upath.types.storage_options import SimpleCacheStorageOptions @@ -33,3 +41,29 @@ def __init__( chain_parser: FSSpecChainParser = ..., **storage_options: Unpack[SimpleCacheStorageOptions], ) -> None: ... + + @classmethod + def _fs_factory( + cls, + urlpath: str, + protocol: str, + storage_options: Mapping[str, Any], + ) -> AbstractFileSystem: + so = storage_options.copy() + so.pop("fo", None) + return super()._fs_factory( + urlpath, + protocol, + so, + ) + + @property + def storage_options(self) -> Mapping[str, Any]: + so = self._storage_options.copy() + so.pop("fo", None) + return MappingProxyType(so) + + def iterdir(self) -> Iterator[Self]: + if self.is_file(): + raise NotADirectoryError(str(self)) + yield from super().iterdir() From 3831b97430ba83c4bb2767b83603fb8a46879e62 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 12 Oct 2025 14:12:48 +0200 Subject: [PATCH 05/13] upath.core: fix chain behavior --- upath/core.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/upath/core.py b/upath/core.py index ce309265..b86237cc 100644 --- a/upath/core.py +++ b/upath/core.py @@ -425,7 +425,8 @@ def __init__( segments = chain_parser.unchain( str_args0, {"protocol": protocol, **storage_options} ) - chain = Chain.from_list(segments) + # FIXME: normalization needs to happen in unchain already... + chain = Chain.from_list(Chain.from_list(segments).to_list()) if len(args) > 1: chain = chain.replace( path=WrappedFileSystemFlavour.from_protocol(protocol).join( @@ -1102,14 +1103,14 @@ def group(self) -> str: def absolute(self) -> Self: if self._relative_base is not None: - return self.cwd().joinpath(str(self)) + return self.cwd().joinpath(self.__vfspath__()) return self def is_absolute(self) -> bool: if self._relative_base is not None: return False else: - return self.parser.isabs(str(self)) + return self.parser.isabs(self.__vfspath__()) def __eq__(self, other: object) -> bool: """UPaths are considered equal if their protocol, path and @@ -1275,7 +1276,7 @@ def root(self) -> str: return self.parser.splitroot(str(self))[1] def __reduce__(self): - args = tuple(self._raw_urlpaths) + args = (self.__vfspath__(),) kwargs = { "protocol": self._protocol, **self._storage_options, From 21fa716182e771c6340af3585759297957825ce3 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Wed, 15 Oct 2025 21:22:42 +0200 Subject: [PATCH 06/13] upath._chain: refactor unchain --- upath/_chain.py | 175 ++++++++++++++++++++++++-------------- upath/core.py | 4 +- upath/tests/test_chain.py | 3 +- 3 files changed, 117 insertions(+), 65 deletions(-) diff --git a/upath/_chain.py b/upath/_chain.py index 627a4a75..f530d996 100644 --- a/upath/_chain.py +++ b/upath/_chain.py @@ -4,23 +4,28 @@ import warnings from collections import defaultdict from collections import deque +from collections.abc import Iterator from collections.abc import MutableMapping from collections.abc import Sequence from collections.abc import Set +from itertools import zip_longest from typing import TYPE_CHECKING from typing import Any from typing import NamedTuple +from upath._flavour import WrappedFileSystemFlavour +from upath._protocol import get_upath_protocol +from upath.registry import available_implementations +from upath.types import UNSET_DEFAULT + if TYPE_CHECKING: if sys.version_info >= (3, 11): + from typing import Never from typing import Self else: + from typing_extensions import Never from typing_extensions import Self -from upath._flavour import WrappedFileSystemFlavour -from upath._protocol import get_upath_protocol -from upath.registry import available_implementations - __all__ = [ "ChainSegment", "Chain", @@ -153,6 +158,23 @@ def nest(self) -> ChainSegment: return ChainSegment(urlpath, protocol, inkwargs) +def _iter_protocol_fileobject_options( + protocol: str, + fileobject: str | None, + storage_options: dict[str, Any], + /, +) -> Iterator[tuple[str, str | None, dict[str, Any]]]: + """yields protocol, fileobject, and remaining storage options""" + so = storage_options.copy() + while "target_protocol" in so: + t_protocol = so.pop("target_protocol", None) + t_fileobject = so.pop("fo", None) # codespell:ignore fo + t_so = so.pop("target_options", {}) + yield protocol, fileobject, so + protocol, fileobject, so = t_protocol, t_fileobject, t_so + yield protocol, fileobject, so + + class FSSpecChainParser: """parse an fsspec chained urlpath""" @@ -160,7 +182,15 @@ def __init__(self) -> None: self.link: str = "::" self.known_protocols: Set[str] = set() - def unchain(self, path: str, kwargs: dict[str, Any]) -> list[ChainSegment]: + def unchain( + self, + path: str, + _deprecated_storage_options: Never = UNSET_DEFAULT, + /, + *, + protocol: str | None = None, + storage_options: dict[str, Any] | None = None, + ) -> list[ChainSegment]: """implements same behavior as fsspec.core._un_chain two differences: @@ -168,67 +198,88 @@ def unchain(self, path: str, kwargs: dict[str, Any]) -> list[ChainSegment]: 2. it checks against the known protocols for exact matches """ - # TODO: upstream to fsspec - first_bit_protocol: str | None = kwargs.pop("protocol", None) - it_bits = iter(path.split(self.link)) - bits: list[str] - if first_bit_protocol == (bit := next(it_bits)) and first_bit_protocol in { - "blockcache", - "filecache", - "simplecache", - }: - bits = [""] - elif first_bit_protocol is not None: - bits = [bit] - else: - bits = [] - for p in it_bits: - if "://" in p: # uri-like, fast-path - bits.append(p) - elif "/" in p: # path-like, fast-path - bits.append(p) - elif p in self.known_protocols: # exact match a fsspec protocol - bits.append(f"{p}://") - elif p in (m := set(available_implementations(fallback=True))): + if _deprecated_storage_options is not UNSET_DEFAULT: + warnings.warn( + "passing storage_options as positional argument is deprecated, " + "pass as keyword argument instead", + DeprecationWarning, + stacklevel=2, + ) + if storage_options is not None: + raise ValueError( + "cannot pass storage_options both positionally and as keyword" + ) + storage_options = _deprecated_storage_options + protocol = protocol or storage_options.get("protocol") + if storage_options is None: + storage_options = {} + + segments = [] + next_path_overwrite: str | None = None + for proto0, bit in zip_longest([protocol], path.split(self.link)): + # get protocol and path_bit + if ( + "://" in bit # uri-like, fast-path (redundant) + or "/" in bit # path-like, fast-path + ): + proto = get_upath_protocol(bit, protocol=proto0) + flavour = WrappedFileSystemFlavour.from_protocol(proto) + path_bit = flavour.strip_protocol(bit) + extra_so = flavour.get_kwargs_from_url(bit) + elif bit in self.known_protocols and ( + proto0 is None or bit == proto0 + ): # exact match a fsspec protocol + proto = bit + path_bit = "" + extra_so = {} + elif bit in (m := set(available_implementations(fallback=True))) and ( + proto0 is None or bit == proto0 + ): self.known_protocols = m - bits.append(f"{p}://") - else: - bits.append(p) - - # [[url, protocol, kwargs], ...] - out: list[ChainSegment] = [] - kwargs = kwargs.copy() - first_bit_idx = len(bits) - 1 - for idx, bit in enumerate(reversed(bits)): - if idx == first_bit_idx: - protocol = first_bit_protocol or get_upath_protocol(bit) or "" + proto = bit + path_bit = "" + extra_so = {} else: - protocol = get_upath_protocol(bit) or "" - flavour = WrappedFileSystemFlavour.from_protocol(protocol) - extra_kwargs = flavour.get_kwargs_from_url(bit) - kws = kwargs.pop(protocol, {}) - if bit is bits[0]: - kws.update(kwargs) - kw = dict(**extra_kwargs) - kw.update(kws) - if protocol in {"blockcache", "filecache", "simplecache"}: - # passthrough filesystems - if bit: - kw["fo"] = bit # codespell:ignore fo - bit = None + proto = get_upath_protocol(bit, protocol=proto0) + flavour = WrappedFileSystemFlavour.from_protocol(proto) + path_bit = flavour.strip_protocol(bit) + extra_so = flavour.get_kwargs_from_url(bit) + if proto in {"blockcache", "filecache", "simplecache"}: + if path_bit: + next_path_overwrite = path_bit + path_bit = None + elif next_path_overwrite is not None: + path_bit = next_path_overwrite + next_path_overwrite = None + segments.append(ChainSegment(path_bit, proto, extra_so)) + + root_so = segments[0].storage_options + for segment, proto_fo_so in zip_longest( + segments, + _iter_protocol_fileobject_options( + protocol or "", + path_bit if segments else None, + storage_options, + ), + ): + t_proto, t_fo, t_so = proto_fo_so or (None, None, {}) + if segment is None: + if next_path_overwrite is not None: + t_fo = next_path_overwrite + next_path_overwrite = None + segments.append(ChainSegment(t_fo, t_proto, t_so)) else: - bit = flavour.strip_protocol(bit) or flavour.root_marker - if "target_protocol" in kw: - out.append( - ChainSegment( - kw.pop("fo", None), # codespell:ignore fo - kw.pop("target_protocol"), - kw.pop("target_options", {}), + proto = segment.protocol + # check if protocol is consistent with storage options + if t_proto and t_proto != proto: + raise ValueError( + f"protocol {proto!r} collides with target_protocol {t_proto!r}" ) - ) - out.append(ChainSegment(bit, protocol, kw)) - out.reverse() - return out + # update the storage_options + segment.storage_options.update(root_so.pop(proto, {})) + segment.storage_options.update(t_so) + + return segments def chain(self, segments: Sequence[ChainSegment]) -> tuple[str, dict[str, Any]]: """returns a chained urlpath from the segments""" diff --git a/upath/core.py b/upath/core.py index b86237cc..3dc33ed7 100644 --- a/upath/core.py +++ b/upath/core.py @@ -423,7 +423,9 @@ def __init__( str_args0 = "." segments = chain_parser.unchain( - str_args0, {"protocol": protocol, **storage_options} + str_args0, + protocol=protocol, + storage_options=storage_options, ) # FIXME: normalization needs to happen in unchain already... chain = Chain.from_list(Chain.from_list(segments).to_list()) diff --git a/upath/tests/test_chain.py b/upath/tests/test_chain.py index 54a5a696..8d91be4f 100644 --- a/upath/tests/test_chain.py +++ b/upath/tests/test_chain.py @@ -54,7 +54,6 @@ def test_chaining_upath_path(urlpath, expected): "simplecache::file:///tmp", { "target_protocol": "file", - "fo": Path("/tmp").absolute().as_posix(), "target_options": {}, }, ), @@ -130,7 +129,7 @@ def test_write_file(clear_memory_fs): ) def test_chain_parser_roundtrip(urlpath: str): parser = FSSpecChainParser() - segments = parser.unchain(urlpath, {}) + segments = parser.unchain(urlpath, protocol=None, storage_options={}) rechained, kw = parser.chain(segments) assert rechained == urlpath assert kw == {} From f48ec43e82e1348c8ca704b33d00efd537c21ad4 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Wed, 15 Oct 2025 21:42:31 +0200 Subject: [PATCH 07/13] upath.core: fix rename behavior for achored paths with empty root_marker --- upath/core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/upath/core.py b/upath/core.py index 3dc33ed7..6975851b 100644 --- a/upath/core.py +++ b/upath/core.py @@ -1227,7 +1227,10 @@ def rename( **kwargs: Any, ) -> Self: if isinstance(target, str) and self.storage_options: - target = UPath(target, **self.storage_options) + if self.anchor and target.startswith(self.anchor): + target = UPath(target, protocol=self.protocol, **self.storage_options) + else: + target = UPath(target, **self._storage_options) if target == self: return self if self._relative_base is not None: From b55a9e7288d49c7ff7fd1629e0c1f06c8143ebc6 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Wed, 15 Oct 2025 21:42:50 +0200 Subject: [PATCH 08/13] upath.core: fix pickling for relative paths --- upath/core.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/upath/core.py b/upath/core.py index 6975851b..724e5fa5 100644 --- a/upath/core.py +++ b/upath/core.py @@ -1281,14 +1281,20 @@ def root(self) -> str: return self.parser.splitroot(str(self))[1] def __reduce__(self): - args = (self.__vfspath__(),) - kwargs = { - "protocol": self._protocol, - **self._storage_options, - } - # Include _relative_base in the state if it's set - if self._relative_base is not None: - kwargs["_relative_base"] = self._relative_base + if self._relative_base is None: + args = (self.__vfspath__(),) + kwargs = { + "protocol": self._protocol, + **self._storage_options, + } + else: + args = (self._relative_base, self.__vfspath__()) + # Include _relative_base in the state if it's set + kwargs = { + "protocol": self._protocol, + **self._storage_options, + "_relative_base": self._relative_base, + } return _make_instance, (type(self), args, kwargs) @classmethod From 19bb3403474f19f21f0af6916841248009b2cdf7 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Wed, 15 Oct 2025 22:15:27 +0200 Subject: [PATCH 09/13] upath._chain: typing fixes --- upath/_chain.py | 27 ++++++++++++++------------- upath/implementations/cached.py | 2 +- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/upath/_chain.py b/upath/_chain.py index f530d996..83d3db5a 100644 --- a/upath/_chain.py +++ b/upath/_chain.py @@ -158,21 +158,21 @@ def nest(self) -> ChainSegment: return ChainSegment(urlpath, protocol, inkwargs) -def _iter_protocol_fileobject_options( - protocol: str, +def _iter_fileobject_protocol_options( fileobject: str | None, + protocol: str, storage_options: dict[str, Any], /, -) -> Iterator[tuple[str, str | None, dict[str, Any]]]: - """yields protocol, fileobject, and remaining storage options""" +) -> Iterator[tuple[str | None, str, dict[str, Any]]]: + """yields fileobject, protocol and remaining storage options""" so = storage_options.copy() while "target_protocol" in so: - t_protocol = so.pop("target_protocol", None) + t_protocol = so.pop("target_protocol", "") t_fileobject = so.pop("fo", None) # codespell:ignore fo t_so = so.pop("target_options", {}) - yield protocol, fileobject, so - protocol, fileobject, so = t_protocol, t_fileobject, t_so - yield protocol, fileobject, so + yield fileobject, protocol, so + fileobject, protocol, so = t_fileobject, t_protocol, t_so + yield fileobject, protocol, so class FSSpecChainParser: @@ -214,7 +214,8 @@ def unchain( if storage_options is None: storage_options = {} - segments = [] + segments: list[ChainSegment] = [] + path_bit: str | None next_path_overwrite: str | None = None for proto0, bit in zip_longest([protocol], path.split(self.link)): # get protocol and path_bit @@ -256,13 +257,13 @@ def unchain( root_so = segments[0].storage_options for segment, proto_fo_so in zip_longest( segments, - _iter_protocol_fileobject_options( - protocol or "", + _iter_fileobject_protocol_options( path_bit if segments else None, + protocol or "", storage_options, ), ): - t_proto, t_fo, t_so = proto_fo_so or (None, None, {}) + t_fo, t_proto, t_so = proto_fo_so or (None, "", {}) if segment is None: if next_path_overwrite is not None: t_fo = next_path_overwrite @@ -327,7 +328,7 @@ def chain(self, segments: Sequence[ChainSegment]) -> tuple[str, dict[str, Any]]: chained_kw = {"zip": {"allowZip64": False}} print(chained_path, chained_kw) out0 = _un_chain(chained_path, chained_kw) - out1 = FSSpecChainParser().unchain(chained_path, chained_kw) + out1 = FSSpecChainParser().unchain(chained_path, storage_options=chained_kw) pp(out0) pp(out1) diff --git a/upath/implementations/cached.py b/upath/implementations/cached.py index 762f3c97..0da528a6 100644 --- a/upath/implementations/cached.py +++ b/upath/implementations/cached.py @@ -49,7 +49,7 @@ def _fs_factory( protocol: str, storage_options: Mapping[str, Any], ) -> AbstractFileSystem: - so = storage_options.copy() + so = dict(storage_options) so.pop("fo", None) return super()._fs_factory( urlpath, From 4fad9f9715d4a3e7afb9f58d024b1b818f7b0de3 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Wed, 15 Oct 2025 22:28:30 +0200 Subject: [PATCH 10/13] upath._flavour_sources: add simplecache flavour --- dev/fsspec_inspector/generate_flavours.py | 2 +- upath/_flavour_sources.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/dev/fsspec_inspector/generate_flavours.py b/dev/fsspec_inspector/generate_flavours.py index 8a6b0a56..324f007a 100644 --- a/dev/fsspec_inspector/generate_flavours.py +++ b/dev/fsspec_inspector/generate_flavours.py @@ -105,7 +105,6 @@ def __init_subclass__(cls: Any, **kwargs): "dir", "blockcache", "cached", - "simplecache", "filecache", ] @@ -116,6 +115,7 @@ def __init_subclass__(cls: Any, **kwargs): FIX_METHODS = { "GCSFileSystem": ["_strip_protocol", "_get_kwargs_from_urls", "_split_path"], + "SimpleCacheFileSystem": [], } diff --git a/upath/_flavour_sources.py b/upath/_flavour_sources.py index e5ba196c..396d6017 100644 --- a/upath/_flavour_sources.py +++ b/upath/_flavour_sources.py @@ -23,7 +23,6 @@ # - cached # - dir # - filecache -# - simplecache # protocol import errors: # - gdrive (Please install gdrive_fs for access to Google Drive) # - generic (GenericFileSystem: '_strip_protocol' not a classmethod) @@ -926,6 +925,15 @@ def _get_kwargs_from_urls(path): return out +class SimpleCacheFileSystemFlavour(AbstractFileSystemFlavour): + __orig_class__ = 'fsspec.implementations.cached.SimpleCacheFileSystem' + __orig_version__ = '2025.9.0' + protocol = ('simplecache',) + root_marker = '' + sep = '/' + local_file = True + + class TarFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'fsspec.implementations.tar.TarFileSystem' __orig_version__ = '2025.9.0' From 7bcf96375958b281d28db8631bb0e3d42a5a5f03 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Wed, 15 Oct 2025 22:29:50 +0200 Subject: [PATCH 11/13] tests: add simple cache tests --- upath/tests/implementations/test_cached.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 upath/tests/implementations/test_cached.py diff --git a/upath/tests/implementations/test_cached.py b/upath/tests/implementations/test_cached.py new file mode 100644 index 00000000..e7b757f2 --- /dev/null +++ b/upath/tests/implementations/test_cached.py @@ -0,0 +1,19 @@ +import pytest + +from upath import UPath +from upath.implementations.cached import SimpleCachePath + +from ..cases import BaseTests + + +class TestSimpleCachePath(BaseTests): + @pytest.fixture(autouse=True) + def path(self, local_testdir): + if not local_testdir.startswith("/"): + local_testdir = "/" + local_testdir + path = f"simplecache::memory:{local_testdir}" + self.path = UPath(path) + self.prepare_file_system() + + def test_is_SimpleCachePath(self): + assert isinstance(self.path, SimpleCachePath) From 81a00adf25914421ac518c8b905dfb11994eb615 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Wed, 15 Oct 2025 22:54:41 +0200 Subject: [PATCH 12/13] upath.core: adjust rename logic --- upath/core.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/upath/core.py b/upath/core.py index 724e5fa5..8d3b03f9 100644 --- a/upath/core.py +++ b/upath/core.py @@ -1,6 +1,6 @@ from __future__ import annotations -import os +import posixpath import sys import warnings from abc import ABCMeta @@ -1226,25 +1226,22 @@ def rename( maxdepth: int | None = UNSET_DEFAULT, **kwargs: Any, ) -> Self: - if isinstance(target, str) and self.storage_options: - if self.anchor and target.startswith(self.anchor): - target = UPath(target, protocol=self.protocol, **self.storage_options) + target_protocol = get_upath_protocol(target) + if target_protocol and target_protocol != self.protocol: + raise ValueError( + f"expected protocol {self.protocol!r}, got: {target_protocol!r}" + ) + if not isinstance(target, UPath): + if target_protocol or self.anchor and target.startswith(self.anchor): + target = self.with_segments(target) else: - target = UPath(target, **self._storage_options) + target = UPath(target) if target == self: return self if self._relative_base is not None: self = self.absolute() - target_protocol = get_upath_protocol(target) if target_protocol: - if target_protocol != self.protocol: - raise ValueError( - f"expected protocol {self.protocol!r}, got: {target_protocol!r}" - ) - if not isinstance(target, UPath): - target_ = UPath(target, **self.storage_options) - else: - target_ = target + target_ = target # avoid calling .resolve for subclasses of UPath if ".." in target_.parts or "." in target_.parts: target_ = target_.resolve() @@ -1253,7 +1250,7 @@ def rename( # avoid calling .resolve for subclasses of UPath if ".." in parent.parts or "." in parent.parts: parent = parent.resolve() - target_ = parent.joinpath(os.path.normpath(str(target))) + target_ = parent.joinpath(posixpath.normpath(target.path)) if recursive is not UNSET_DEFAULT: kwargs["recursive"] = recursive if maxdepth is not UNSET_DEFAULT: From 6bd8438705df10a2326273381a8a31ce7c2d5087 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Wed, 15 Oct 2025 23:16:45 +0200 Subject: [PATCH 13/13] upath.core: fix rename for non-local targets --- upath/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/upath/core.py b/upath/core.py index 8d3b03f9..1a1ae198 100644 --- a/upath/core.py +++ b/upath/core.py @@ -1232,7 +1232,8 @@ def rename( f"expected protocol {self.protocol!r}, got: {target_protocol!r}" ) if not isinstance(target, UPath): - if target_protocol or self.anchor and target.startswith(self.anchor): + target = str(target) + if target_protocol or (self.anchor and target.startswith(self.anchor)): target = self.with_segments(target) else: target = UPath(target) @@ -1240,6 +1241,7 @@ def rename( return self if self._relative_base is not None: self = self.absolute() + target_protocol = get_upath_protocol(target) if target_protocol: target_ = target # avoid calling .resolve for subclasses of UPath