From 4af15a96409803ef4ca4197f2bd2c8ee290a1f2d Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sat, 4 Oct 2025 22:45:06 +0200 Subject: [PATCH 1/8] upath: add type overloads to type narrow on protocol --- upath/core.py | 131 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 126 insertions(+), 5 deletions(-) diff --git a/upath/core.py b/upath/core.py index 691c6675..398882c0 100644 --- a/upath/core.py +++ b/upath/core.py @@ -295,11 +295,10 @@ def __new__( protocol: str | None = None, chain_parser: FSSpecChainParser = DEFAULT_CHAIN_PARSER, **storage_options: Any, - ) -> UPath: + ) -> Self: # narrow type - assert issubclass( - cls, UPath - ), "UPath.__new__ can't instantiate non-UPath classes" + if not issubclass(cls, UPath): + raise TypeError("UPath.__new__ can't instantiate non-UPath classes") # deprecate 'scheme' if "scheme" in storage_options: @@ -438,13 +437,135 @@ class UPath(_UPathMixin, WritablePath, ReadablePath): "_relative_base", ) - if TYPE_CHECKING: + if TYPE_CHECKING: # noqa: C901 _chain: Chain _chain_parser: FSSpecChainParser _fs_cached: bool _raw_urlpaths: Sequence[JoinablePathLike] _relative_base: str | None + import upath.implementations as _uimpl + + @overload + def __new__( + cls, + *args: JoinablePathLike, + protocol: Literal["simplecache"], + chain_parser: FSSpecChainParser = ..., + **storage_options: Any, + ) -> _uimpl.cached.SimpleCachePath: ... + @overload # noqa: E301 + def __new__( + cls, + *args: JoinablePathLike, + protocol: Literal["gcs", "gs"], + chain_parser: FSSpecChainParser = ..., + **storage_options: Any, + ) -> _uimpl.cloud.GCSPath: ... + @overload # noqa: E301 + def __new__( + cls, + *args: JoinablePathLike, + protocol: Literal["s3", "s3a"], + chain_parser: FSSpecChainParser = ..., + **storage_options: Any, + ) -> _uimpl.cloud.S3Path: ... + @overload # noqa: E301 + def __new__( + cls, + *args: JoinablePathLike, + protocol: Literal["az", "abfs", "abfss", "adl"], + chain_parser: FSSpecChainParser = ..., + **storage_options: Any, + ) -> _uimpl.cloud.AzurePath: ... + @overload # noqa: E301 + def __new__( + cls, + *args: JoinablePathLike, + protocol: Literal["data"], + chain_parser: FSSpecChainParser = ..., + **storage_options: Any, + ) -> _uimpl.data.DataPath: ... + @overload # noqa: E301 + def __new__( + cls, + *args: JoinablePathLike, + protocol: Literal["github"], + chain_parser: FSSpecChainParser = ..., + **storage_options: Any, + ) -> _uimpl.github.GitHubPath: ... + @overload # noqa: E301 + def __new__( + cls, + *args: JoinablePathLike, + protocol: Literal["hdfs"], + chain_parser: FSSpecChainParser = ..., + **storage_options: Any, + ) -> _uimpl.hdfs.HDFSPath: ... + @overload # noqa: E301 + def __new__( + cls, + *args: JoinablePathLike, + protocol: Literal["http", "https"], + chain_parser: FSSpecChainParser = ..., + **storage_options: Any, + ) -> _uimpl.http.HTTPPath: ... + @overload # noqa: E301 + def __new__( + cls, + *args: JoinablePathLike, + protocol: Literal["file", "local"], + chain_parser: FSSpecChainParser = ..., + **storage_options: Any, + ) -> _uimpl.local.FilePath: ... + @overload # noqa: E301 + def __new__( + cls, + *args: JoinablePathLike, + protocol: Literal["memory"], + chain_parser: FSSpecChainParser = ..., + **storage_options: Any, + ) -> _uimpl.memory.MemoryPath: ... + @overload # noqa: E301 + def __new__( + cls, + *args: JoinablePathLike, + protocol: Literal["sftp", "ssh"], + chain_parser: FSSpecChainParser = ..., + **storage_options: Any, + ) -> _uimpl.sftp.SFTPPath: ... + @overload # noqa: E301 + def __new__( + cls, + *args: JoinablePathLike, + protocol: Literal["smb"], + chain_parser: FSSpecChainParser = ..., + **storage_options: Any, + ) -> _uimpl.smb.SMBPath: ... + @overload # noqa: E301 + def __new__( + cls, + *args: JoinablePathLike, + protocol: Literal["webdav"], + chain_parser: FSSpecChainParser = ..., + **storage_options: Any, + ) -> _uimpl.webdav.WebdavPath: ... + @overload # noqa: E301 + def __new__( + cls, + *args: JoinablePathLike, + protocol: str | None = ..., + chain_parser: FSSpecChainParser = ..., + **storage_options: Any, + ) -> Self: ... + def __new__( # noqa: E301 + cls, + *args: JoinablePathLike, + protocol: str | None = None, + chain_parser: FSSpecChainParser = DEFAULT_CHAIN_PARSER, + **storage_options: Any, + ) -> Self: ... + # === JoinablePath attributes ===================================== parser: UPathParser = LazyFlavourDescriptor() # type: ignore[assignment] From 89388157315394d099a5b35bfe50e6287f0a657b Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sat, 4 Oct 2025 23:25:54 +0200 Subject: [PATCH 2/8] typesafety: test UPath constructor protocol type narrowing --- typesafety/test_upath_types.yml | 56 +++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/typesafety/test_upath_types.yml b/typesafety/test_upath_types.yml index bc0a7b61..071c174c 100644 --- a/typesafety/test_upath_types.yml +++ b/typesafety/test_upath_types.yml @@ -90,3 +90,59 @@ path_cls = get_upath_class("some-unknown-protocol") reveal_type(path_cls) # N: Revealed type is "type[upath.core.UPath] | None" + +- case: upath__new__fsspec_protocols + disable_cache: false + parametrized: + - cls_fqn: upath.implementations.cached.SimpleCachePath + protocol: simplecache + - cls_fqn: upath.implementations.cloud.S3Path + protocol: s3 + - cls_fqn: upath.implementations.cloud.GCSPath + protocol: gcs + - cls_fqn: upath.implementations.cloud.AzurePath + protocol: abfs + - cls_fqn: upath.implementations.data.DataPath + protocol: data + - cls_fqn: upath.implementations.github.GitHubPath + protocol: github + - cls_fqn: upath.implementations.hdfs.HDFSPath + protocol: hdfs + - cls_fqn: upath.implementations.http.HTTPPath + protocol: http + - cls_fqn: upath.implementations.local.FilePath + protocol: file + - cls_fqn: upath.implementations.memory.MemoryPath + protocol: memory + - cls_fqn: upath.implementations.sftp.SFTPPath + protocol: sftp + - cls_fqn: upath.implementations.smb.SMBPath + protocol: smb + - cls_fqn: upath.implementations.webdav.WebdavPath + protocol: webdav + - cls_fqn: upath.core.UPath + protocol: unknown-protocol + main: | + from upath import UPath + + # without protocol kwarg resolve to UPath + reveal_type(UPath("{{ protocol }}://someuri/blah")) # N: Revealed type is "upath.core.UPath" + + # with protocol kwarg resolve to specific UPath subclass + reveal_type(UPath(".", protocol="{{ protocol }}")) # N: Revealed type is "{{ cls_fqn }}" + +- case: upath__new__empty_protocol + disable_cache: false + skip: sys.platform == "win32" + main: | + from upath import UPath + + reveal_type(UPath("", protocol="")) # N: Revealed type is "type[upath.implementations.local.PosixUPath]" + +- case: get_upath_class_pathlib_win + disable_cache: false + skip: sys.platform != "win32" + main: | + from upath import UPath + + reveal_type(UPath("", protocol="")) # N: Revealed type is "type[upath.implementations.local.WindowsUPath]" From b1310d6b0840ebcc8abbb7449be0c97388aed462 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 5 Oct 2025 02:22:58 +0200 Subject: [PATCH 3/8] typesafety: update tests --- typesafety/test_upath_types.yml | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/typesafety/test_upath_types.yml b/typesafety/test_upath_types.yml index 071c174c..f98366b6 100644 --- a/typesafety/test_upath_types.yml +++ b/typesafety/test_upath_types.yml @@ -123,26 +123,25 @@ - cls_fqn: upath.core.UPath protocol: unknown-protocol main: | - from upath import UPath - - # without protocol kwarg resolve to UPath - reveal_type(UPath("{{ protocol }}://someuri/blah")) # N: Revealed type is "upath.core.UPath" + import upath - # with protocol kwarg resolve to specific UPath subclass - reveal_type(UPath(".", protocol="{{ protocol }}")) # N: Revealed type is "{{ cls_fqn }}" + p = upath.UPath(".", protocol="{{ protocol }}") + reveal_type(p) # N: Revealed type is "{{ cls_fqn }}" - case: upath__new__empty_protocol - disable_cache: false + disable_cache: true skip: sys.platform == "win32" main: | - from upath import UPath + from upath.core import UPath - reveal_type(UPath("", protocol="")) # N: Revealed type is "type[upath.implementations.local.PosixUPath]" + p = UPath("asd", protocol="") + reveal_type(p) # N: Revealed type is "upath.implementations.local.PosixUPath" - case: get_upath_class_pathlib_win - disable_cache: false + disable_cache: true skip: sys.platform != "win32" main: | - from upath import UPath + from upath.core import UPath - reveal_type(UPath("", protocol="")) # N: Revealed type is "type[upath.implementations.local.WindowsUPath]" + p = UPath("", protocol="") + reveal_type(p) # N: Revealed type is "upath.implementations.local.WindowsUPath" From a155ba02390f43794238814e2606f119af8b407e Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 5 Oct 2025 02:25:09 +0200 Subject: [PATCH 4/8] upath: update typing of metaclass --- upath/core.py | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/upath/core.py b/upath/core.py index 398882c0..80b72015 100644 --- a/upath/core.py +++ b/upath/core.py @@ -48,6 +48,8 @@ from upath.types import WritablePathLike if TYPE_CHECKING: + import upath.implementations as _uimpl + if sys.version_info >= (3, 11): from typing import Self else: @@ -56,6 +58,7 @@ from pydantic import GetCoreSchemaHandler from pydantic_core.core_schema import CoreSchema + _MT = TypeVar("_MT") _WT = TypeVar("_WT", bound="WritablePath") __all__ = ["UPath"] @@ -109,7 +112,7 @@ class _UPathMeta(ABCMeta): def __getitem__(cls, key): return cls - def __call__(cls, *args, **kwargs): + def __call__(cls: type[_MT], *args: Any, **kwargs: Any) -> _MT: # create a copy if UPath class try: (arg0,) = args @@ -117,10 +120,8 @@ def __call__(cls, *args, **kwargs): pass else: if isinstance(arg0, UPath) and not kwargs: - return copy(arg0) - inst = cls.__new__(cls, *args, **kwargs) - inst.__init__(*args, **kwargs) - return inst + return copy(arg0) # type: ignore[return-value] + return ABCMeta.__call__(cls, *args, **kwargs) class _UPathMixin(metaclass=_UPathMeta): @@ -316,6 +317,7 @@ def __new__( storage_options=storage_options, ) # determine which UPath subclass to dispatch to + upath_cls: type[_UPathMixin] | None if cls._protocol_dispatch or cls._protocol_dispatch is None: upath_cls = get_upath_class(protocol=pth_protocol) if upath_cls is None: @@ -325,9 +327,12 @@ def __new__( # by setting MyUPathSubclass._protocol_dispatch to `False`. # This will effectively ignore the registered UPath # implementations and return an instance of MyUPathSubclass. - # This can be useful if a subclass wants to extend the UPath + # This be useful if a subclass wants to extend the UPath # api, and it is fine to rely on the default implementation # for all supported user protocols. + # + # THIS IS DEPRECATED! + # Use upath.extensions.ProxyUPath to extend the UPath API upath_cls = cls if issubclass(upath_cls, cls): @@ -444,8 +449,6 @@ class UPath(_UPathMixin, WritablePath, ReadablePath): _raw_urlpaths: Sequence[JoinablePathLike] _relative_base: str | None - import upath.implementations as _uimpl - @overload def __new__( cls, @@ -551,6 +554,14 @@ def __new__( **storage_options: Any, ) -> _uimpl.webdav.WebdavPath: ... @overload # noqa: E301 + def __new__( # type: ignore[misc] + cls, + *args: JoinablePathLike, + protocol: Literal[""], + chain_parser: FSSpecChainParser = ..., + **storage_options: Any, + ) -> _uimpl.local.PosixUPath: ... + @overload # noqa: E301 def __new__( cls, *args: JoinablePathLike, @@ -558,13 +569,21 @@ def __new__( chain_parser: FSSpecChainParser = ..., **storage_options: Any, ) -> Self: ... - def __new__( # noqa: E301 + + def __new__( # type: ignore[misc] cls, *args: JoinablePathLike, protocol: str | None = None, chain_parser: FSSpecChainParser = DEFAULT_CHAIN_PARSER, **storage_options: Any, - ) -> Self: ... + ) -> Self: + return super().__new__( + cls, + *args, + protocol=protocol, + chain_parser=chain_parser, + **storage_options, + ) # === JoinablePath attributes ===================================== From 194e1ed27513e87403c6679690e5dc158e737016 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 5 Oct 2025 11:52:01 +0200 Subject: [PATCH 5/8] upath.core: adjust overloads --- upath/core.py | 54 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/upath/core.py b/upath/core.py index 80b72015..9cce3aac 100644 --- a/upath/core.py +++ b/upath/core.py @@ -296,7 +296,7 @@ def __new__( protocol: str | None = None, chain_parser: FSSpecChainParser = DEFAULT_CHAIN_PARSER, **storage_options: Any, - ) -> Self: + ) -> UPath: # narrow type if not issubclass(cls, UPath): raise TypeError("UPath.__new__ can't instantiate non-UPath classes") @@ -317,7 +317,7 @@ def __new__( storage_options=storage_options, ) # determine which UPath subclass to dispatch to - upath_cls: type[_UPathMixin] | None + upath_cls: type[UPath] | None if cls._protocol_dispatch or cls._protocol_dispatch is None: upath_cls = get_upath_class(protocol=pth_protocol) if upath_cls is None: @@ -450,6 +450,10 @@ class UPath(_UPathMixin, WritablePath, ReadablePath): _relative_base: str | None @overload + def __new__( + cls, + ) -> Self: ... + @overload # noqa: E301 def __new__( cls, *args: JoinablePathLike, @@ -553,14 +557,29 @@ def __new__( chain_parser: FSSpecChainParser = ..., **storage_options: Any, ) -> _uimpl.webdav.WebdavPath: ... - @overload # noqa: E301 - def __new__( # type: ignore[misc] - cls, - *args: JoinablePathLike, - protocol: Literal[""], - chain_parser: FSSpecChainParser = ..., - **storage_options: Any, - ) -> _uimpl.local.PosixUPath: ... + + if sys.platform == "win32": + + @overload # noqa: E301 + def __new__( + cls, + *args: JoinablePathLike, + protocol: Literal[""], + chain_parser: FSSpecChainParser = ..., + **storage_options: Any, + ) -> _uimpl.local.WindowsUPath: ... + + else: + + @overload # noqa: E301 + def __new__( + cls, + *args: JoinablePathLike, + protocol: Literal[""], + chain_parser: FSSpecChainParser = ..., + **storage_options: Any, + ) -> _uimpl.local.PosixUPath: ... + @overload # noqa: E301 def __new__( cls, @@ -570,20 +589,13 @@ def __new__( **storage_options: Any, ) -> Self: ... - def __new__( # type: ignore[misc] + def __new__( cls, *args: JoinablePathLike, - protocol: str | None = None, - chain_parser: FSSpecChainParser = DEFAULT_CHAIN_PARSER, + protocol: str | None = ..., + chain_parser: FSSpecChainParser = ..., **storage_options: Any, - ) -> Self: - return super().__new__( - cls, - *args, - protocol=protocol, - chain_parser=chain_parser, - **storage_options, - ) + ) -> Self: ... # === JoinablePath attributes ===================================== From 3c4fb4952d42b797a6342b5bd6c0f7557c9d7b74 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 5 Oct 2025 12:01:49 +0200 Subject: [PATCH 6/8] upath.implementations.local: force mypy to believe PosixUPath/WindowsUPath are actual UPath subclasses --- upath/implementations/local.py | 75 ++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 30 deletions(-) diff --git a/upath/implementations/local.py b/upath/implementations/local.py index cc463b12..64b04c28 100644 --- a/upath/implementations/local.py +++ b/upath/implementations/local.py @@ -432,38 +432,53 @@ def _copy_from( UPath.register(LocalPath) -class WindowsUPath(LocalPath, pathlib.WindowsPath): - __slots__ = () - - if os.name != "nt": - - def __new__( - cls, - *args, - protocol: str | None = None, - chain_parser: FSSpecChainParser = DEFAULT_CHAIN_PARSER, - **storage_options: Any, - ) -> WindowsUPath: - raise NotImplementedError( - f"cannot instantiate {cls.__name__} on your system" - ) - +# Mypy will ignore the ABC.register call above, so we need to force it to +# think PosixUPath and WindowsUPath are subclasses of UPath. +# This is really not a good pattern, but it's the best we can do without +# either introducing a duck-type protocol for UPath or come up with a +# better solution for the UPath versions of the pathlib.Path subclasses. -class PosixUPath(LocalPath, pathlib.PosixPath): - __slots__ = () - - if os.name == "nt": +if TYPE_CHECKING: - def __new__( - cls, - *args, - protocol: str | None = None, - chain_parser: FSSpecChainParser = DEFAULT_CHAIN_PARSER, - **storage_options: Any, - ) -> PosixUPath: - raise NotImplementedError( - f"cannot instantiate {cls.__name__} on your system" - ) + class WindowsUPath(LocalPath, pathlib.WindowsPath, UPath): # type: ignore[misc] + __slots__ = () + + class PosixUPath(LocalPath, pathlib.PosixPath, UPath): # type: ignore[misc] + __slots__ = () + +else: + + class WindowsUPath(LocalPath, pathlib.WindowsPath): + __slots__ = () + + if os.name != "nt": + + def __new__( + cls, + *args, + protocol: str | None = None, + chain_parser: FSSpecChainParser = DEFAULT_CHAIN_PARSER, + **storage_options: Any, + ) -> WindowsUPath: + raise NotImplementedError( + f"cannot instantiate {cls.__name__} on your system" + ) + + class PosixUPath(LocalPath, pathlib.PosixPath): + __slots__ = () + + if os.name == "nt": + + def __new__( + cls, + *args, + protocol: str | None = None, + chain_parser: FSSpecChainParser = DEFAULT_CHAIN_PARSER, + **storage_options: Any, + ) -> PosixUPath: + raise NotImplementedError( + f"cannot instantiate {cls.__name__} on your system" + ) class FilePath(UPath): From 43c495a4a9a7312a0d29b1c8e1bf5f5f7f31fd18 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 5 Oct 2025 12:03:30 +0200 Subject: [PATCH 7/8] upath.registry: adjust overload formatting --- upath/registry.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/upath/registry.py b/upath/registry.py index 672c3746..706914fc 100644 --- a/upath/registry.py +++ b/upath/registry.py @@ -213,8 +213,7 @@ def get_upath_class(protocol: Literal["simplecache"]) -> type[_SimpleCachePath]: def get_upath_class(protocol: Literal["s3", "s3a"]) -> type[_S3Path]: ... @overload def get_upath_class(protocol: Literal["gcs", "gs"]) -> type[_GCSPath]: ... - - @overload + @overload # noqa: E301 def get_upath_class( protocol: Literal["abfs", "abfss", "adl", "az"], ) -> type[_AzurePath]: ... From 6df0d8934efeb034c8081fff2cd38d55c6867efa Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 5 Oct 2025 12:19:23 +0200 Subject: [PATCH 8/8] upath.core: restore metaclass.__call__ behavior --- upath/core.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/upath/core.py b/upath/core.py index 9cce3aac..59c08cac 100644 --- a/upath/core.py +++ b/upath/core.py @@ -121,7 +121,11 @@ def __call__(cls: type[_MT], *args: Any, **kwargs: Any) -> _MT: else: if isinstance(arg0, UPath) and not kwargs: return copy(arg0) # type: ignore[return-value] - return ABCMeta.__call__(cls, *args, **kwargs) + # We do this call manually, because cls could be a registered + # subclass of UPath that is not directly inheriting from UPath. + inst = cls.__new__(cls, *args, **kwargs) + inst.__init__(*args, **kwargs) # type: ignore[misc] + return inst class _UPathMixin(metaclass=_UPathMeta):