From 234165b409b600f7238d337466cfc746d43f03fe Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 5 Oct 2025 14:49:19 +0200 Subject: [PATCH 1/5] upath.types: add storage_options submodule --- upath/types/storage_options.py | 360 +++++++++++++++++++++++++++++++++ 1 file changed, 360 insertions(+) create mode 100644 upath/types/storage_options.py diff --git a/upath/types/storage_options.py b/upath/types/storage_options.py new file mode 100644 index 00000000..f156f221 --- /dev/null +++ b/upath/types/storage_options.py @@ -0,0 +1,360 @@ +"""Storage options types for various filesystems based on fsspec.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING +from typing import TypedDict + +if TYPE_CHECKING: + from asyncio import AbstractEventLoop + from typing import Any + from typing import Literal + + from fsspec.implementations.cache_mapper import AbstractCacheMapper + from fsspec.spec import AbstractFileSystem + + +__all__ = [ + "SimpleCacheStorageOptions", + "GCSStorageOptions", + "S3StorageOptions", + "AzureBlobStorageOptions", + "DataStorageOptions", + "GithubStorageOptions", + "HDFSStorageOptions", + "HTTPStorageOptions", + "FileStorageOptions", + "MemoryStorageOptions", + "SFTPStorageOptions", + "SMBStorageOptions", + "WebdavStorageOptions", +] + + +class _AbstractStorageOptions(TypedDict, total=False): + """Base storage options for fsspec-based filesystems""" + + # dircache related options + use_listings_cache: bool + listings_expiry_time: int | float | None + max_paths: int | None + # fs instance cache options + skip_instance_cache: bool + # async fs instance options + asynchronous: bool + loop: AbstractEventLoop | None + batch_size: int | None + + +class _ChainableStorageOptions(TypedDict, total=False): + """Storage options for filesystems supporting chaining""" + + target_protocol: str | None + target_options: dict[str, Any] | None + fs: AbstractFileSystem | None + + +class SimpleCacheStorageOptions( + _AbstractStorageOptions, + _ChainableStorageOptions, + total=False, +): + """Storage options for SimpleCache""" + + cache_storage: Literal["TMP"] | list[str] | str + cache_check: int | Literal[False] + check_files: bool + expiry_time: int | Literal[False] + same_names: bool | None + compression: str # todo: specify allowed values + cache_mapper: AbstractCacheMapper | None + + +class GCSStorageOptions(_AbstractStorageOptions, total=False): + """Storage options for Google Cloud Storage""" + + # Authentication and project settings + project: str + access: Literal["read_only", "read_write", "full_control"] + token: ( + None + | Literal["google_default", "cache", "anon", "browser", "cloud"] + | str + | dict[str, Any] + ) + + # Performance and caching + block_size: int | None + consistency: Literal["none", "size", "md5"] + cache_timeout: float | None # overrides listings_expiry_time if set + + # Request configuration + requests_timeout: float | None + requester_pays: bool | str + session_kwargs: dict[str, Any] | None # aiohttp.ClientSession kwargs + timeout: float | None # timeout used for .buckets? + + # Connection settings + endpoint_url: str | None + check_connection: bool | None + + # Storage configuration + default_location: str | None + version_aware: bool + + # Deprecated options + # secure_serialize: bool + + +class S3StorageOptions(_AbstractStorageOptions, total=False): + """Storage options for AWS S3 and S3-compatible services""" + + # Authentication + anon: bool + key: str | None + secret: str | None + token: str | None + username: str | None # alias for key + password: str | None # alias for secret + + # Connection settings + endpoint_url: str | None + use_ssl: bool + + # AWS-specific configuration + client_kwargs: dict[str, Any] | None # botocore Client kwargs + config_kwargs: dict[str, Any] | None # botocore Config kwargs + s3_additional_kwargs: dict[str, Any] | None # s3 api methods kwargs + session: Any | None # aiobotocore AioSession + + # Performance settings + requester_pays: bool + default_block_size: int | None + default_fill_cache: bool + default_cache_type: str # fsspec.caching Literal["readahead", "none", "bytes", ...] + max_concurrency: int + fixed_upload_size: bool + + # Feature flags + version_aware: bool + cache_regions: bool + + +class AzureBlobStorageOptions(_AbstractStorageOptions, total=False): + """Storage options for Azure Blob Storage and Azure Data Lake Gen2""" + + # Account and authentication + account_name: str | None + account_key: str | None + connection_string: str | None + credential: ( + str | Any | None + ) # azure.core.credentials_async.AsyncTokenCredential or SAS token + sas_token: str | None + anon: bool | None + + # Service Principal authentication + client_id: str | None + client_secret: str | None + tenant_id: str | None + + # Connection and networking + # request_session: Any | None # for http requests not used by anything ??? + # socket_timeout: int | None deprecated + account_host: str | None + location_mode: Literal["primary", "secondary"] + + # Performance settings + blocksize: int # block size for download/upload operations + default_fill_cache: bool + default_cache_type: str # fsspec cache type + max_concurrency: int | None + + # Timeout settings + timeout: int | None # server-side timeout for operations + connection_timeout: int | None # connection establishment timeout + read_timeout: int | None # read operation timeout + + # Feature flags + version_aware: bool # support blob versioning + assume_container_exists: bool | None # container existence assumptions + + +class DataStorageOptions(_AbstractStorageOptions, total=False): + """Storage options for Data URIs filesystem""" + + # No specific options for Data URIs at the moment + + +class GithubStorageOptions(_AbstractStorageOptions, total=False): + """Storage options for GitHub repository filesystem""" + + # Repository identification + org: str # GitHub organization or username + repo: str # Repository name + sha: str | None # Commit SHA, branch, or tag (default: current master) + + # Authentication + username: str | None # GitHub username for authenticated access + token: str | None # GitHub personal access token + + # Connection settings + timeout: tuple[int, int] | int | None # (connect, read) timeouts or single timeout + + +class HDFSStorageOptions(_AbstractStorageOptions, total=False): + """Storage options for Hadoop Distributed File System (HDFS)""" + + # Connection settings + host: str # Hostname, IP or "default" to try to read from Hadoop config + port: int # Port to connect on, or default from Hadoop config if 0 + + # Authentication + user: str | None # Username to connect as + kerb_ticket: str | None # Kerberos ticket for authentication + + # HDFS-specific settings + replication: int # Replication factor for write operations (default: 3) + extra_conf: ( + dict[str, Any] | None + ) # Additional configuration passed to HadoopFileSystem + + +class HTTPStorageOptions(_AbstractStorageOptions, total=False): + """Storage options for HTTP(S) filesystem""" + + # Performance settings + block_size: ( + int | None + ) # Block size for reading bytes; 0 = raw requests file-like objects + + # Link parsing behavior + simple_links: ( + bool # Consider both HTML tags and URL-like strings vs HTML tags only + ) + same_scheme: ( + bool # Only consider paths with matching http/https scheme during ls/glob + ) + + # Caching configuration + cache_type: str # Default cache type used in open() (e.g., "bytes") + cache_options: dict[str, Any] | None # Default cache options used in open() + + # HTTP client configuration + client_kwargs: dict[str, Any] | None # Passed to aiohttp.ClientSession + get_client: Any | None # Callable that constructs aiohttp.ClientSession + + # Encoding settings + encoded: bool # Whether URLs should be encoded + + # Deprecated options + # size_policy: Any # Deprecated parameter + + +class FileStorageOptions(_AbstractStorageOptions, total=False): + """Storage options for local filesystem (file:// and local:// protocols)""" + + # File system behavior + auto_mkdir: bool # Whether to create parent directories when opening files + + +class MemoryStorageOptions(_AbstractStorageOptions, total=False): + """Storage options for memory filesystem""" + + # No specific options for memory filesystem at the moment + + +class SFTPStorageOptions(_AbstractStorageOptions, total=False): + """Storage options for SFTP/SSH filesystem""" + + # Connection settings + host: str # Hostname or IP address (required) + port: int | None # SSH port (default: 22) + + # Authentication + username: str | None # Username to authenticate as + password: ( + str | None + ) # Password authentication; also used for private key decryption + passphrase: str | None # Used for decrypting private keys + pkey: Any | None # Private key for authentication (paramiko.PKey) + key_filename: str | list[str] | None # Filename(s) of private key(s)/certs to try + + # Connection behavior + timeout: float | None # TCP connect timeout in seconds + allow_agent: bool | None # Whether to connect to SSH agent (default: True) + look_for_keys: ( + bool | None + ) # Whether to search for private keys in ~/.ssh/ (default: True) + compress: bool | None # Whether to turn on compression + sock: Any | None # Socket or socket-like object for communication + + # GSS-API authentication + gss_auth: bool | None # Use GSS-API authentication + gss_kex: bool | None # Perform GSS-API Key Exchange and user authentication + gss_deleg_creds: bool | None # Delegate GSS-API client credentials + gss_host: str | None # Target name in kerberos database (default: hostname) + gss_trust_dns: bool | None # Trust DNS to canonicalize hostname (default: True) + + # Timeout settings + banner_timeout: float | None # SSH banner timeout in seconds + auth_timeout: float | None # Authentication response timeout in seconds + channel_timeout: float | None # Channel open response timeout in seconds + + # Advanced configuration + disabled_algorithms: ( + dict[str, Any] | None + ) # Algorithms to disable (passed to Transport) + transport_factory: Any | None # Callable to generate Transport instance + auth_strategy: Any | None # AuthStrategy instance for newer authentication + + # SFTP-specific settings + temppath: str # Remote temporary directory path (default: "/tmp") + + +class SMBStorageOptions(_AbstractStorageOptions, total=False): + """Storage options for SMB/Windows/Samba network shares""" + + # Connection settings + host: str # The remote server name/ip to connect to (required) + port: int | None # Port to connect with (usually 445, sometimes 139) + + # Authentication + username: str | None # Username to connect with (required if not using Kerberos) + password: str | None # User's password on the server + + # Connection behavior + timeout: int # Connection timeout in seconds (default: 60) + encrypt: bool | None # Whether to force encryption + + # File access control + share_access: Literal["r", "w", "d"] | None # Default access for file operations + # None (default): exclusively locks file until closed + # 'r': Allow other handles with read access + # 'w': Allow other handles with write access + # 'd': Allow other handles with delete access + + # Session retry configuration + register_session_retries: int # Number of session registration retries (default: 4) + register_session_retry_wait: ( + int # Wait time between retries in seconds (default: 1) + ) + register_session_retry_factor: int # Exponential backoff factor (default: 10) + + # File system behavior + auto_mkdir: bool # Whether to create parent directories when opening files + + +class WebdavStorageOptions(_AbstractStorageOptions, total=False): + """Storage options for WebDAV filesystem""" + + # Connection settings + base_url: str # Base URL of the WebDAV server (required) + + # Authentication + auth: ( + tuple[str, str] | Any | None + ) # Authentication (username, password) tuple or custom auth + + # Client configuration + client: Any | None # webdav4.client.Client instance From 19e4e263729d44c654c2fd4fdc2bc4104a29d847 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 5 Oct 2025 14:50:16 +0200 Subject: [PATCH 2/5] upath.types: fix *PathLike types --- upath/types/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/upath/types/__init__.py b/upath/types/__init__.py index f919697d..f164df13 100644 --- a/upath/types/__init__.py +++ b/upath/types/__init__.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING from typing import Any from typing import Protocol +from typing import Union from typing import runtime_checkable from upath.types._abc import JoinablePath @@ -19,7 +20,7 @@ if sys.version_info >= (3, 12): from typing import TypeAlias else: - TypeAlias = Any + from typing_extensions import TypeAlias __all__ = [ "JoinablePath", @@ -41,10 +42,10 @@ class VFSPathLike(Protocol): def __vfspath__(self) -> str: ... -SupportsPathLike: TypeAlias = "VFSPathLike | PathLike[str]" -JoinablePathLike: TypeAlias = "JoinablePath | SupportsPathLike | str" -ReadablePathLike: TypeAlias = "ReadablePath | SupportsPathLike | str" -WritablePathLike: TypeAlias = "WritablePath | SupportsPathLike | str" +SupportsPathLike: TypeAlias = Union[VFSPathLike, PathLike[str]] +JoinablePathLike: TypeAlias = Union[JoinablePath, SupportsPathLike, str] +ReadablePathLike: TypeAlias = Union[ReadablePath, SupportsPathLike, str] +WritablePathLike: TypeAlias = Union[WritablePath, SupportsPathLike, str] class _DefaultValue(enum.Enum): From f8ca798aa79d86451faff8e36ff6d18e4692e744 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 5 Oct 2025 14:51:18 +0200 Subject: [PATCH 3/5] upath.implementations: add storage-options types --- upath/implementations/cached.py | 30 +++++++++++++++++++++++++- upath/implementations/cloud.py | 37 +++++++++++++++++++++++++-------- upath/implementations/data.py | 22 +++++++++++++++++++- upath/implementations/github.py | 28 ++++++++++++++++++++++--- upath/implementations/hdfs.py | 20 +++++++++++++++++- upath/implementations/http.py | 18 ++++++++++++++++ upath/implementations/local.py | 16 +++++++++++++- upath/implementations/memory.py | 22 +++++++++++++++++++- upath/implementations/sftp.py | 23 +++++++++++++++++--- upath/implementations/smb.py | 18 ++++++++++++++++ upath/implementations/webdav.py | 27 +++++++++++++++++++++--- 11 files changed, 238 insertions(+), 23 deletions(-) diff --git a/upath/implementations/cached.py b/upath/implementations/cached.py index dc8800e9..de01fb3e 100644 --- a/upath/implementations/cached.py +++ b/upath/implementations/cached.py @@ -1,7 +1,35 @@ from __future__ import annotations +import sys +from typing import TYPE_CHECKING + from upath.core import UPath +from upath.types import JoinablePathLike + +if TYPE_CHECKING: + from typing import Literal + + if sys.version_info >= (3, 11): + from typing import Unpack + else: + from typing_extensions import Unpack + + from upath._chain import FSSpecChainParser + from upath.types.storage_options import SimpleCacheStorageOptions + + +__all__ = ["SimpleCachePath"] class SimpleCachePath(UPath): - pass + __slots__ = () + + if TYPE_CHECKING: + + def __init__( + self, + *args: JoinablePathLike, + protocol: Literal["simplecache"] | None = ..., + chain_parser: FSSpecChainParser = ..., + **storage_options: Unpack[SimpleCacheStorageOptions], + ) -> None: ... diff --git a/upath/implementations/cloud.py b/upath/implementations/cloud.py index 5c39108d..b2e8489c 100644 --- a/upath/implementations/cloud.py +++ b/upath/implementations/cloud.py @@ -5,15 +5,25 @@ from typing import TYPE_CHECKING from typing import Any +from upath._chain import DEFAULT_CHAIN_PARSER from upath._flavour import upath_strip_protocol from upath.core import UPath from upath.types import JoinablePathLike if TYPE_CHECKING: + from typing import Literal + if sys.version_info >= (3, 11): from typing import Self + from typing import Unpack else: from typing_extensions import Self + from typing_extensions import Unpack + + from upath._chain import FSSpecChainParser + from upath.types.storage_options import AzureBlobStorageOptions + from upath.types.storage_options import GCSStorageOptions + from upath.types.storage_options import S3StorageOptions __all__ = [ "CloudPath", @@ -84,10 +94,13 @@ class GCSPath(CloudPath): def __init__( self, *args: JoinablePathLike, - protocol: str | None = None, - **storage_options: Any, + protocol: Literal["gcs", "gs"] | None = None, + chain_parser: FSSpecChainParser = DEFAULT_CHAIN_PARSER, + **storage_options: Unpack[GCSStorageOptions], ) -> None: - super().__init__(*args, protocol=protocol, **storage_options) + super().__init__( + *args, protocol=protocol, chain_parser=chain_parser, **storage_options + ) if not self.drive and len(self.parts) > 1: raise ValueError("non key-like path provided (bucket/container missing)") @@ -114,10 +127,13 @@ class S3Path(CloudPath): def __init__( self, *args: JoinablePathLike, - protocol: str | None = None, - **storage_options: Any, + protocol: Literal["s3", "s3a"] | None = None, + chain_parser: FSSpecChainParser = DEFAULT_CHAIN_PARSER, + **storage_options: Unpack[S3StorageOptions], ) -> None: - super().__init__(*args, protocol=protocol, **storage_options) + super().__init__( + *args, protocol=protocol, chain_parser=chain_parser, **storage_options + ) if not self.drive and len(self.parts) > 1: raise ValueError("non key-like path provided (bucket/container missing)") @@ -128,9 +144,12 @@ class AzurePath(CloudPath): def __init__( self, *args: JoinablePathLike, - protocol: str | None = None, - **storage_options: Any, + protocol: Literal["abfs", "abfss", "adl", "az"] | None = None, + chain_parser: FSSpecChainParser = DEFAULT_CHAIN_PARSER, + **storage_options: Unpack[AzureBlobStorageOptions], ) -> None: - super().__init__(*args, protocol=protocol, **storage_options) + super().__init__( + *args, protocol=protocol, chain_parser=chain_parser, **storage_options + ) if not self.drive and len(self.parts) > 1: raise ValueError("non key-like path provided (bucket/container missing)") diff --git a/upath/implementations/data.py b/upath/implementations/data.py index 6ad50667..41d15cf0 100644 --- a/upath/implementations/data.py +++ b/upath/implementations/data.py @@ -8,13 +8,33 @@ from upath.types import JoinablePathLike if TYPE_CHECKING: - if sys.version_info > (3, 11): + from typing import Literal + + if sys.version_info >= (3, 11): from typing import Self + from typing import Unpack else: from typing_extensions import Self + from typing_extensions import Unpack + + from upath._chain import FSSpecChainParser + from upath.types.storage_options import DataStorageOptions + +__all__ = ["DataPath"] class DataPath(UPath): + __slots__ = () + + if TYPE_CHECKING: + + def __init__( + self, + *args: JoinablePathLike, + protocol: Literal["data"] | None = ..., + chain_parser: FSSpecChainParser = ..., + **storage_options: Unpack[DataStorageOptions], + ) -> None: ... @property def parts(self) -> Sequence[str]: diff --git a/upath/implementations/github.py b/upath/implementations/github.py index 692ed5fe..d72905a1 100644 --- a/upath/implementations/github.py +++ b/upath/implementations/github.py @@ -9,20 +9,42 @@ from collections.abc import Sequence from typing import TYPE_CHECKING -import upath.core +from upath.core import UPath +from upath.types import JoinablePathLike if TYPE_CHECKING: - if sys.version_info > (3, 11): + from typing import Literal + + if sys.version_info >= (3, 11): from typing import Self + from typing import Unpack else: from typing_extensions import Self + from typing_extensions import Unpack + + from upath._chain import FSSpecChainParser + from upath.types.storage_options import GithubStorageOptions + +__all__ = ["GitHubPath"] -class GitHubPath(upath.core.UPath): +class GitHubPath(UPath): """ GitHubPath supporting the fsspec.GitHubFileSystem """ + __slots__ = () + + if TYPE_CHECKING: + + def __init__( + self, + *args: JoinablePathLike, + protocol: Literal["github"] | None = ..., + chain_parser: FSSpecChainParser = ..., + **storage_options: Unpack[GithubStorageOptions], + ) -> None: ... + @property def path(self) -> str: pth = super().path diff --git a/upath/implementations/hdfs.py b/upath/implementations/hdfs.py index dab26167..72dae2a3 100644 --- a/upath/implementations/hdfs.py +++ b/upath/implementations/hdfs.py @@ -5,12 +5,20 @@ from typing import TYPE_CHECKING from upath.core import UPath +from upath.types import JoinablePathLike if TYPE_CHECKING: - if sys.version_info > (3, 11): + from typing import Literal + + if sys.version_info >= (3, 11): from typing import Self + from typing import Unpack else: from typing_extensions import Self + from typing_extensions import Unpack + + from upath._chain import FSSpecChainParser + from upath.types.storage_options import HDFSStorageOptions __all__ = ["HDFSPath"] @@ -18,6 +26,16 @@ class HDFSPath(UPath): __slots__ = () + if TYPE_CHECKING: + + def __init__( + self, + *args: JoinablePathLike, + protocol: Literal["hdfs"] | None = ..., + chain_parser: FSSpecChainParser = ..., + **storage_options: Unpack[HDFSStorageOptions], + ) -> None: ... + def mkdir( self, mode: int = 0o777, parents: bool = False, exist_ok: bool = False ) -> None: diff --git a/upath/implementations/http.py b/upath/implementations/http.py index b2c2ee58..f2019343 100644 --- a/upath/implementations/http.py +++ b/upath/implementations/http.py @@ -15,15 +15,33 @@ from upath.types import JoinablePathLike if TYPE_CHECKING: + from typing import Literal + if sys.version_info >= (3, 11): from typing import Self + from typing import Unpack else: from typing_extensions import Self + from typing_extensions import Unpack + + from upath._chain import FSSpecChainParser + from upath.types.storage_options import HTTPStorageOptions __all__ = ["HTTPPath"] class HTTPPath(UPath): + __slots__ = () + + if TYPE_CHECKING: + + def __init__( + self, + *args: JoinablePathLike, + protocol: Literal["http", "https"] | None = ..., + chain_parser: FSSpecChainParser = ..., + **storage_options: Unpack[HTTPStorageOptions], + ) -> None: ... @classmethod def _transform_init_args( diff --git a/upath/implementations/local.py b/upath/implementations/local.py index 64b04c28..704a974a 100644 --- a/upath/implementations/local.py +++ b/upath/implementations/local.py @@ -38,8 +38,12 @@ if sys.version_info >= (3, 11): from typing import Self + from typing import Unpack else: from typing_extensions import Self + from typing_extensions import Unpack + + from upath.types.storage_options import FileStorageOptions _WT = TypeVar("_WT", bound="WritablePath") @@ -416,7 +420,7 @@ def walk( def hardlink_to(self, target: ReadablePathLike) -> None: try: - os.link(target, self) + os.link(target, self) # type: ignore[arg-type] except AttributeError: raise NotImplementedError @@ -484,6 +488,16 @@ def __new__( class FilePath(UPath): __slots__ = () + if TYPE_CHECKING: + + def __init__( + self, + *args: JoinablePathLike, + protocol: Literal["file", "local"] | None = ..., + chain_parser: FSSpecChainParser = ..., + **storage_options: Unpack[FileStorageOptions], + ) -> None: ... + def __fspath__(self) -> str: return self.path diff --git a/upath/implementations/memory.py b/upath/implementations/memory.py index 33aaa8c9..052ed3bc 100644 --- a/upath/implementations/memory.py +++ b/upath/implementations/memory.py @@ -5,17 +5,37 @@ from typing import TYPE_CHECKING from upath.core import UPath +from upath.types import JoinablePathLike if TYPE_CHECKING: - if sys.version_info > (3, 11): + from typing import Literal + + if sys.version_info >= (3, 11): from typing import Self + from typing import Unpack else: from typing_extensions import Self + from typing_extensions import Unpack + + from upath._chain import FSSpecChainParser + from upath.types.storage_options import MemoryStorageOptions __all__ = ["MemoryPath"] class MemoryPath(UPath): + __slots__ = () + + if TYPE_CHECKING: + + def __init__( + self, + *args: JoinablePathLike, + protocol: Literal["memory"] | None = ..., + chain_parser: FSSpecChainParser = ..., + **storage_options: Unpack[MemoryStorageOptions], + ) -> None: ... + def iterdir(self) -> Iterator[Self]: if not self.is_dir(): raise NotADirectoryError(str(self)) diff --git a/upath/implementations/sftp.py b/upath/implementations/sftp.py index 1f1feb8f..018b8e8b 100644 --- a/upath/implementations/sftp.py +++ b/upath/implementations/sftp.py @@ -3,22 +3,39 @@ import sys from collections.abc import Iterator from typing import TYPE_CHECKING -from typing import Any + +from upath.core import UPath +from upath.types import JoinablePathLike if TYPE_CHECKING: + from typing import Literal + if sys.version_info >= (3, 11): from typing import Self + from typing import Unpack else: from typing_extensions import Self + from typing_extensions import Unpack -from upath import UPath + from upath._chain import FSSpecChainParser + from upath.types.storage_options import SFTPStorageOptions -_unset: Any = object() +__all__ = ["SFTPPath"] class SFTPPath(UPath): __slots__ = () + if TYPE_CHECKING: + + def __init__( + self, + *args: JoinablePathLike, + protocol: Literal["sftp"] | None = ..., + chain_parser: FSSpecChainParser = ..., + **storage_options: Unpack[SFTPStorageOptions], + ) -> None: ... + def iterdir(self) -> Iterator[Self]: if not self.is_dir(): raise NotADirectoryError(str(self)) diff --git a/upath/implementations/smb.py b/upath/implementations/smb.py index b5bcd376..685f4539 100644 --- a/upath/implementations/smb.py +++ b/upath/implementations/smb.py @@ -10,18 +10,36 @@ from upath.core import UPath from upath.types import UNSET_DEFAULT +from upath.types import JoinablePathLike from upath.types import WritablePathLike if TYPE_CHECKING: + from typing import Literal + if sys.version_info >= (3, 11): from typing import Self + from typing import Unpack else: from typing_extensions import Self + from typing_extensions import Unpack + + from upath._chain import FSSpecChainParser + from upath.types.storage_options import SMBStorageOptions class SMBPath(UPath): __slots__ = () + if TYPE_CHECKING: + + def __init__( + self, + *args: JoinablePathLike, + protocol: Literal["smb"] | None = ..., + chain_parser: FSSpecChainParser = ..., + **storage_options: Unpack[SMBStorageOptions], + ) -> None: ... + def mkdir( self, mode: int = 0o777, diff --git a/upath/implementations/webdav.py b/upath/implementations/webdav.py index 9ec47b8f..d0799923 100644 --- a/upath/implementations/webdav.py +++ b/upath/implementations/webdav.py @@ -1,7 +1,9 @@ from __future__ import annotations +import sys from collections.abc import Mapping from collections.abc import Sequence +from typing import TYPE_CHECKING from typing import Any from urllib.parse import urlsplit @@ -11,9 +13,18 @@ from upath.core import UPath from upath.types import JoinablePathLike -__all__ = [ - "WebdavPath", -] +if TYPE_CHECKING: + from typing import Literal + + if sys.version_info >= (3, 11): + from typing import Unpack + else: + from typing_extensions import Unpack + + from upath._chain import FSSpecChainParser + from upath.types.storage_options import WebdavStorageOptions + +__all__ = ["WebdavPath"] # webdav was only registered in fsspec>=2022.5.0 if "webdav" not in known_implementations: @@ -25,6 +36,16 @@ class WebdavPath(UPath): __slots__ = () + if TYPE_CHECKING: + + def __init__( + self, + *args: JoinablePathLike, + protocol: Literal["webdav+http", "webdav+https"] | None = ..., + chain_parser: FSSpecChainParser = ..., + **storage_options: Unpack[WebdavStorageOptions], + ) -> None: ... + @classmethod def _transform_init_args( cls, From 57b07431e7e2f0340a7664104cd6d0b2c4a287ab Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 5 Oct 2025 16:17:22 +0200 Subject: [PATCH 4/5] typesafety: add tests ensuring that storage_options are typed --- typesafety/test_upath_types.yml | 143 +++++++++++++++++++++++++++++++- 1 file changed, 141 insertions(+), 2 deletions(-) diff --git a/typesafety/test_upath_types.yml b/typesafety/test_upath_types.yml index f98366b6..333fdc20 100644 --- a/typesafety/test_upath_types.yml +++ b/typesafety/test_upath_types.yml @@ -129,7 +129,7 @@ reveal_type(p) # N: Revealed type is "{{ cls_fqn }}" - case: upath__new__empty_protocol - disable_cache: true + disable_cache: false skip: sys.platform == "win32" main: | from upath.core import UPath @@ -138,10 +138,149 @@ reveal_type(p) # N: Revealed type is "upath.implementations.local.PosixUPath" - case: get_upath_class_pathlib_win - disable_cache: true + disable_cache: false skip: sys.platform != "win32" main: | from upath.core import UPath p = UPath("", protocol="") reveal_type(p) # N: Revealed type is "upath.implementations.local.WindowsUPath" + +- case: upath_constructor_sopts_correct_type + disable_cache: false + parametrized: + - module: upath.implementations.cached + cls: SimpleCachePath + supported_example_name: check_files + supported_example_value: True + - module: upath.implementations.cloud + cls: GCSPath + supported_example_name: project + supported_example_value: '"my-project"' + - module: upath.implementations.cloud + cls: S3Path + supported_example_name: anon + supported_example_value: True + - module: upath.implementations.cloud + cls: AzurePath + supported_example_name: account_name + supported_example_value: '"myaccount"' + - module: upath.implementations.data + cls: DataPath + supported_example_name: use_listings_cache + supported_example_value: False + - module: upath.implementations.github + cls: GitHubPath + supported_example_name: org + supported_example_value: '"fsspec"' + - module: upath.implementations.hdfs + cls: HDFSPath + supported_example_name: host + supported_example_value: '"localhost"' + - module: upath.implementations.http + cls: HTTPPath + supported_example_name: simple_links + supported_example_value: True + - module: upath.implementations.local + cls: FilePath + supported_example_name: auto_mkdir + supported_example_value: True + - module: upath.implementations.memory + cls: MemoryPath + supported_example_name: use_listings_cache + supported_example_value: True + - module: upath.implementations.sftp + cls: SFTPPath + supported_example_name: host + supported_example_value: '"sftp.example.com"' + - module: upath.implementations.smb + cls: SMBPath + supported_example_name: timeout + supported_example_value: 60 + - module: upath.implementations.webdav + cls: WebdavPath + supported_example_name: base_url + supported_example_value: '"https://webdav.example.com"' + main: | + from {{ module }} import {{ cls }} + + p = {{ cls }}(".", {{ supported_example_name }}={{ supported_example_value }}) + reveal_type(p) # N: Revealed type is "{{ module }}.{{ cls }}" + +- case: upath_constructor_sopts_incorrect_type + disable_cache: false + parametrized: + - module: upath.implementations.cached + cls: SimpleCachePath + supported_example_name: check_files + unsupported_example_value: '"blub"' + - module: upath.implementations.cloud + cls: GCSPath + supported_example_name: version_aware + unsupported_example_value: '"yes"' + - module: upath.implementations.cloud + cls: S3Path + supported_example_name: anon + unsupported_example_value: '"blah"' + - module: upath.implementations.cloud + cls: AzurePath + supported_example_name: account_name + unsupported_example_value: '123' + - module: upath.implementations.data + cls: DataPath + supported_example_name: use_listings_cache + unsupported_example_value: '"blub"' + - module: upath.implementations.github + cls: GitHubPath + supported_example_name: repo + unsupported_example_value: 'True' + - module: upath.implementations.hdfs + cls: HDFSPath + supported_example_name: host + unsupported_example_value: '8020' + - module: upath.implementations.http + cls: HTTPPath + supported_example_name: simple_links + unsupported_example_value: '"no"' + - module: upath.implementations.local + cls: FilePath + supported_example_name: auto_mkdir + unsupported_example_value: '"abc"' + - module: upath.implementations.memory + cls: MemoryPath + supported_example_name: use_listings_cache + unsupported_example_value: '{}' + - module: upath.implementations.sftp + cls: SFTPPath + supported_example_name: host + unsupported_example_value: '[]' + - module: upath.implementations.smb + cls: SMBPath + supported_example_name: host + unsupported_example_value: 'False' + - module: upath.implementations.webdav + cls: WebdavPath + supported_example_name: base_url + unsupported_example_value: '123' + main: | + from {{ module }} import {{ cls }} + + p = {{ cls }}(".", {{ supported_example_name }}={{ unsupported_example_value }}) # ER: Argument "{{ supported_example_name }}" to "{{ cls }}" has incompatible type.* + +# FIXME: mypy emits a 'defined here' note when emitting the error below. +# seems to be a limitation/bug in pytest-mypy-plugins that I can't match it +# +# - case: upath_constructor_sopts_incorrect_name +# disable_cache: false +# regex: yes +# parametrized: +# - module: upath.implementations.cached +# cls: SimpleCachePath +# unsupported_example_name: idontexist +# main: | +# from {{ module }} import {{ cls }} +# +# p = {{ cls }}(".", {{ unsupported_example_name }}=1) +# out: | +# \\.\\..* +# main:3: error: Unexpected keyword argument "idontexist" to "{{ cls }}".* From fccd0b8748562d3d70f176925a6978a4692ed78a Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 5 Oct 2025 16:24:02 +0200 Subject: [PATCH 5/5] upath.types: fix missing import --- upath/types/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upath/types/__init__.py b/upath/types/__init__.py index f164df13..3f716972 100644 --- a/upath/types/__init__.py +++ b/upath/types/__init__.py @@ -2,6 +2,7 @@ import enum import sys +from os import PathLike from typing import TYPE_CHECKING from typing import Any from typing import Protocol @@ -15,7 +16,6 @@ from upath.types._abc import WritablePath if TYPE_CHECKING: - from os import PathLike # noqa: F401 if sys.version_info >= (3, 12): from typing import TypeAlias