diff --git a/.gitignore b/.gitignore index 4449592e..49636e18 100644 --- a/.gitignore +++ b/.gitignore @@ -70,6 +70,7 @@ instance/ # Sphinx documentation docs/_build/ +docs/changelog.md # PyBuilder .pybuilder/ @@ -143,3 +144,6 @@ upath/_version.py # vscode workspace settings .vscode/ + +# mac +**/.DS_Store diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eb7bcdcc..bdd58c35 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,6 +18,7 @@ repos: args: ['--assume-in-merge'] - id: check-toml - id: check-yaml + exclude: ^mkdocs\.yml$ - id: debug-statements - id: end-of-file-fixer - id: mixed-line-ending diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..8fbda54c --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,24 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version, and other tools you might need +build: + os: ubuntu-24.04 + tools: + python: "3.13" + jobs: + pre_create_environment: + - asdf plugin add uv + - asdf install uv latest + - asdf global uv latest + create_environment: + - uv venv "${READTHEDOCS_VIRTUALENV_PATH}" + install: + - UV_PROJECT_ENVIRONMENT="${READTHEDOCS_VIRTUALENV_PATH}" uv sync --group docs + +# Build documentation with Mkdocs +mkdocs: + configuration: mkdocs.yml diff --git a/README.md b/README.md index 747e4a55..3f8ee4e7 100644 --- a/README.md +++ b/README.md @@ -232,11 +232,11 @@ simplify interaction with `filesystem_spec`. Think of the `UPath` class in terms of the following code: ```python -from pathlib import Path +from pathlib_abc import ReadablePath, WritablePath from typing import Any, Mapping from fsspec import AbstractFileSystem -class UPath(Path): +class UPath(ReadablePath, WriteablePath): # the real implementation is more complex, but this is the general idea @property diff --git a/docs/_plugins/copy_changelog.py b/docs/_plugins/copy_changelog.py new file mode 100644 index 00000000..cb1cf5f6 --- /dev/null +++ b/docs/_plugins/copy_changelog.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from pathlib import Path + +THIS_DIR = Path(__file__).parent +DOCS_DIR = THIS_DIR.parent +PROJECT_ROOT = DOCS_DIR.parent + + +def on_pre_build(**_) -> None: + """Add changelog to docs/changelog.md""" + cl_now = PROJECT_ROOT.joinpath("CHANGELOG.md").read_text(encoding="utf-8") + + f_doc = DOCS_DIR.joinpath("changelog.md") + if not f_doc.is_file() or f_doc.read_text(encoding="utf-8") != cl_now: + f_doc.write_text(cl_now, encoding="utf-8") diff --git a/docs/api/extensions.md b/docs/api/extensions.md new file mode 100644 index 00000000..458baea3 --- /dev/null +++ b/docs/api/extensions.md @@ -0,0 +1,67 @@ +# Extensions :puzzle_piece: + +The extensions module provides a base class for extending UPath functionality while maintaining +compatibility with all filesystem implementations. + +## ProxyUPath + +::: upath.extensions.ProxyUPath + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: false + show_bases: true + +--- + +## Usage Example + +`ProxyUPath` allows you to extend the UPath interface with additional methods while +preserving compatibility with all supported filesystem implementations. It acts as a +wrapper around any UPath instance. + +### Creating a Custom Extension + +```python +from upath import UPath +from upath.extensions import ProxyUPath + +class MyCustomPath(ProxyUPath): + """Custom path with additional functionality""" + + def custom_method(self) -> str: + """Add your custom functionality here""" + return f"Custom processing for: {self.path}" + + def enhanced_read(self) -> str: + """Enhanced read with preprocessing""" + content = self.read_text() + # Add custom processing + return content.upper() + +# Use with any filesystem +s3_path = MyCustomPath("s3://bucket/file.txt") +local_path = MyCustomPath("/tmp/file.txt") +gcs_path = MyCustomPath("gs://bucket/file.txt") + +# All standard UPath methods work +print(s3_path.exists()) +print(local_path.parent) + +# Always a subclass of your class +assert isinstance(s3_path, MyCustomPath) +assert isinstance(local_path, MyCustomPath) + +# Plus your custom methods +print(s3_path.custom_method()) +content = local_path.enhanced_read() +``` + +--- + +## See Also :link: + +- [UPath](index.md) - Main UPath class documentation +- [Implementations](implementations.md) - Built-in UPath subclasses +- [Registry](registry.md) - Implementation registry diff --git a/docs/api/implementations.md b/docs/api/implementations.md new file mode 100644 index 00000000..db24cc1b --- /dev/null +++ b/docs/api/implementations.md @@ -0,0 +1,263 @@ +# Implementations :file_folder: + +Universal Pathlib provides specialized UPath subclasses for different filesystem protocols. +Each implementation is optimized for its respective filesystem and may provide additional +protocol-specific functionality. + +## upath.implementations.cloud + +::: upath.implementations.cloud.S3Path + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: [] + show_bases: true + +**Protocols:** `s3://`, `s3a://` + +Amazon S3 compatible object storage implementation. + +::: upath.implementations.cloud.GCSPath + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: [] + show_bases: true + +**Protocols:** `gs://`, `gcs://` + +Google Cloud Storage implementation. + +::: upath.implementations.cloud.AzurePath + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: [] + show_bases: true + +**Protocols:** `abfs://`, `abfss://`, `adl://`, `az://` + +Azure Blob Storage and Azure Data Lake implementation. + +--- + +## upath.implementations.local + +::: upath.implementations.local.PosixUPath + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: [] + show_bases: true + +POSIX-style local filesystem paths (Linux, macOS, Unix). + +::: upath.implementations.local.WindowsUPath + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: [] + show_bases: true + +Windows-style local filesystem paths. + +::: upath.implementations.local.FilePath + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: [] + show_bases: true + +**Protocols:** `file://`, `local://` + +File URI implementation for local filesystem. + +--- + +## upath.implementations.http + +::: upath.implementations.http.HTTPPath + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: [] + show_bases: true + +**Protocols:** `http://`, `https://` + +HTTP/HTTPS read-only filesystem implementation. + +--- + +## upath.implementations.sftp + +::: upath.implementations.sftp.SFTPPath + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: [] + show_bases: true + +**Protocols:** `sftp://`, `ssh://` + +SFTP (SSH File Transfer Protocol) implementation. + +--- + +## upath.implementations.smb + +::: upath.implementations.smb.SMBPath + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: [] + show_bases: true + +**Protocol:** `smb://` + +SMB/CIFS network filesystem implementation. + +--- + +## upath.implementations.webdav + +::: upath.implementations.webdav.WebdavPath + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: [] + show_bases: true + +**Protocols:** `webdav://`, `webdav+http://`, `webdav+https://` + +WebDAV protocol implementation. + +--- + +## upath.implementations.hdfs + +::: upath.implementations.hdfs.HDFSPath + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: [] + show_bases: true + +**Protocol:** `hdfs://` + +Hadoop Distributed File System implementation. + +--- + +## upath.implementations.github + +::: upath.implementations.github.GitHubPath + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: [] + show_bases: true + +**Protocol:** `github://` + +GitHub repository file access implementation. + +--- + +## upath.implementations.zip + +::: upath.implementations.zip.ZipPath + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: [] + show_bases: true + +**Protocol:** `zip://` + +ZIP archive filesystem implementation. + +--- + +## upath.implementations.tar + +::: upath.implementations.tar.TarPath + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: [] + show_bases: true + +**Protocol:** `tar://` + +TAR archive filesystem implementation. + +--- + +## upath.implementations.memory + +::: upath.implementations.memory.MemoryPath + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: [] + show_bases: true + +**Protocol:** `memory://` + +In-memory filesystem implementation for testing and temporary storage. + +--- + +## upath.implementations.data + +::: upath.implementations.data.DataPath + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: [] + show_bases: true + +**Protocol:** `data://` + +Data URL scheme implementation for embedded data. + +--- + +## upath.implementations.cached + +::: upath.implementations.cached.SimpleCachePath + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: [] + show_bases: true + +**Protocol:** `simplecache://` + +Local caching wrapper for remote filesystems. + +--- + +## See Also :link: + +- [UPath](index.md) - Main UPath class documentation +- [Registry](registry.md) - Implementation registry +- [Extensions](extensions.md) - Extending UPath functionality diff --git a/docs/api/index.md b/docs/api/index.md new file mode 100644 index 00000000..b1fb5a1f --- /dev/null +++ b/docs/api/index.md @@ -0,0 +1,96 @@ + + +# UPath ![upath](../assets/logo-128x128.svg){: #upath-logo } + +The `UPath` class is your default entry point for interacting with fsspec filesystems. +When instantiating UPath, a specific `UPath` subclass will be returned, dependent on the +detected or provided `protocol`. Here we document all methods and properties available on +UPath instances. + +!!! info "Compatibility" + All methods documented here work consistently across all supported Python versions, + even if they were introduced in later Python versions. We consider it a bug if they + don't :bug: so please report and issue if you run into inconsistencies. + + +```python +from upath import UPath +``` + +::: upath.core.UPath + options: + heading_level: 2 + merge_init_into_class: false + inherited_members: true + members: + - __init__ + - protocol + - storage_options + - fs + - path + - parts + - name + - stem + - drive + - root + - anchor + - suffix + - suffixes + - parent + - parents + - joinpath + - joinuri + - with_name + - with_stem + - with_suffix + - with_segments + - relative_to + - is_relative_to + - match + - full_match + - as_posix + - as_uri + - open + - read_text + - read_bytes + - write_text + - write_bytes + - iterdir + - glob + - rglob + - walk + - mkdir + - rmdir + - touch + - unlink + - rename + - replace + - copy + - move + - copy_into + - move_into + - exists + - is_file + - is_dir + - is_symlink + - is_absolute + - stat + - info + - absolute + - resolve + - expanduser + - cwd + - home + +--- + +## See Also :link: + +- [Registry](registry.md) - The upath implementation registry +- [Implementations](implementations.md) - UPath subclasses +- [Extensions](extensions.md) - Extending UPath functionality +- [Types](types.md) - Type hints and protocols diff --git a/docs/api/registry.md b/docs/api/registry.md new file mode 100644 index 00000000..6d93df2c --- /dev/null +++ b/docs/api/registry.md @@ -0,0 +1,33 @@ +# Registry :file_cabinet: + +The UPath registry system manages filesystem-specific path implementations. It allows you to +register custom UPath subclasses for different protocols and retrieve the appropriate +implementation for a given protocol. + +## Functions + +::: upath.registry.get_upath_class + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + +::: upath.registry.register_implementation + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + +::: upath.registry.available_implementations + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + +--- + +## See Also :link: + +- [UPath](index.md) - Main UPath class documentation +- [Implementations](implementations.md) - Built-in UPath subclasses +- [Extensions](extensions.md) - Extending UPath functionality diff --git a/docs/api/types.md b/docs/api/types.md new file mode 100644 index 00000000..35bbd15f --- /dev/null +++ b/docs/api/types.md @@ -0,0 +1,148 @@ +# Types :label: + +The types module provides type hints, protocols, and type aliases for working with UPath +and filesystem operations. This includes abstract base classes, type aliases for path-like +objects, and typed dictionaries for filesystem-specific storage options. + +## pathlib-abc base classes + +These abstract base classes and protocols are re-exported from [pathlib-abc](https://github.com/barneygale/pathlib-abc) +They define the core path interfaces that stdlib pathlib and UPath implementations conform to. + +::: upath.types.JoinablePath + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: false + show_bases: true + +::: upath.types.ReadablePath + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: false + show_bases: true + +::: upath.types.WritablePath + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: false + show_bases: true + +::: upath.types.PathInfo + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: false + show_bases: true + +::: upath.types.PathParser + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: false + show_bases: true + +--- + +## UPath specific protocols + +::: upath.types.UPathParser + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: false + show_bases: true + +--- + +## Type Aliases + +Convenient type aliases for path-like objects used throughout UPath. + +::: upath.types.JoinablePathLike + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + +Union of types that can be joined as path segments. + +::: upath.types.ReadablePathLike + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + +Union of types that can be read from. + +::: upath.types.WritablePathLike + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + +Union of types that can be written to. + +::: upath.types.SupportsPathLike + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + +Union of objects that support `__fspath__()` or `__vfspath__()` protocols. + +::: upath.types.StatResultType + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + members: false + +Protocol for `os.stat_result`-like objects. + +--- + +## Storage Options + +Typed dictionaries providing type hints for filesystem-specific configuration options. +These help ensure correct parameter names and types when configuring different filesystems. + +::: upath.types.storage_options + options: + heading_level: 3 + show_root_heading: true + show_root_full_path: false + show_bases: false + members: + - SimpleCacheStorageOptions + - GCSStorageOptions + - S3StorageOptions + - AzureStorageOptions + - DataStorageOptions + - GitHubStorageOptions + - HDFSStorageOptions + - HTTPStorageOptions + - FileStorageOptions + - MemoryStorageOptions + - SFTPStorageOptions + - SMBStorageOptions + - WebdavStorageOptions + - ZipStorageOptions + - TarStorageOptions + +--- + +## See Also :link: + +- [UPath](index.md) - Main UPath class documentation +- [Implementations](implementations.md) - Built-in UPath subclasses +- [Extensions](extensions.md) - Extending UPath functionality +- [Registry](registry.md) - Implementation registry diff --git a/docs/assets/favicon.png b/docs/assets/favicon.png new file mode 100644 index 00000000..772aba3f Binary files /dev/null and b/docs/assets/favicon.png differ diff --git a/docs/assets/logo-128x128-white.svg b/docs/assets/logo-128x128-white.svg new file mode 100644 index 00000000..01fd9bb2 --- /dev/null +++ b/docs/assets/logo-128x128-white.svg @@ -0,0 +1,115 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/assets/logo-128x128.svg b/docs/assets/logo-128x128.svg new file mode 100644 index 00000000..f351a32a --- /dev/null +++ b/docs/assets/logo-128x128.svg @@ -0,0 +1,166 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/assets/logo-text.svg b/docs/assets/logo-text.svg new file mode 100644 index 00000000..dfe471e6 --- /dev/null +++ b/docs/assets/logo-text.svg @@ -0,0 +1,212 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/concepts/fsspec.md b/docs/concepts/fsspec.md new file mode 100644 index 00000000..bd13e5e5 --- /dev/null +++ b/docs/concepts/fsspec.md @@ -0,0 +1,186 @@ +# Filesystem Spec :file_folder: + +[fsspec](https://filesystem-spec.readthedocs.io/) is a Python library that provides a unified, pythonic interface for working with different storage backends. It abstracts away the differences between various storage systems, allowing you to interact with local files, cloud storage, remote systems, and specialty filesystems using a consistent API. + +## What is fsspec? + +fsspec is both a **specification** and a **collection of implementations** for pythonic filesystems. The project defines a standard interface that filesystem implementations should follow, then provides concrete implementations for dozens of different storage backends. + +The core idea is simple: whether you're working with files on your local disk, objects in an S3 bucket, blobs in Azure storage, or data over HTTP, the API to interact with them should be the same. + +### Core Functionality + +fsspec provides filesystem objects with methods for common operations. All filesystem implementations inherit from `fsspec.spec.AbstractFileSystem`, which defines the standard interface that all filesystems must implement: + +```python +import fsspec + +# Create a filesystem instance +# Returns an AbstractFileSystem subclass for the specified protocol +fs = fsspec.filesystem('s3', anon=True) + +# List files +files = fs.ls('my-bucket/data/') + +# Check if file exists +exists = fs.exists('my-bucket/data/file.txt') + +# Get file info +info = fs.info('my-bucket/data/file.txt') + +# Read file +with fs.open('my-bucket/data/file.txt', 'r') as f: + content = f.read() + +# Write file +with fs.open('my-bucket/output.txt', 'w') as f: + f.write('Hello, World!') + +# Copy files +fs.cp('my-bucket/source.txt', 'my-bucket/dest.txt') + +# Delete files +fs.rm('my-bucket/file.txt') +``` + +### Protocols + +fsspec identifies filesystem types via **protocols**. Each protocol corresponds to a specific filesystem implementation: + +- `file://` - Local filesystem +- `memory://` - In-memory filesystem (temporary, non-persistent) +- `s3://` or `s3a://` - Amazon S3 +- `gs://` or `gcs://` - Google Cloud Storage +- `az://` or `abfs://` - Azure Blob Storage +- `adl://` - Azure Data Lake Gen1 +- `abfss://` - Azure Data Lake Gen2 (secure) +- `http://` or `https://` - HTTP(S) access +- `ftp://` - FTP +- `sftp://` or `ssh://` - SFTP over SSH +- `smb://` - Samba/Windows file shares +- `webdav://` or `webdav+http://` - WebDAV +- `hdfs://` - Hadoop Distributed File System +- `hf://` - Hugging Face Hub +- `github://` - GitHub repositories +- `zip://` - ZIP archives +- `tar://` - TAR archives +- `gzip://` - GZIP compressed files +- `cached://` - Caching layer over other filesystems + +And many more. See the [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations) for the complete list. + +### Storage Options + +Each filesystem implementation accepts different configuration parameters called **storage options**. These control authentication, connection settings, caching behavior, and more. +They are usually provided as keyword parameters to the +specific filesystem class on instantiation. + +Common storage option patterns: + +```python +import fsspec + +# Authentication credentials +fs = fsspec.filesystem('s3', key='...', secret='...') + +# Anonymous/public access +fs = fsspec.filesystem('s3', anon=True) + +# Tokens and service accounts +fs = fsspec.filesystem('gs', token='path/to/creds.json') + +# Connection settings +fs = fsspec.filesystem('sftp', host='...', port=22, username='...') + +# Behavioral options +fs = fsspec.filesystem('s3', use_ssl=True, default_block_size=5*2**20) +``` + +Refer to the [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations) for details on what each filesystem supports. + +### URI-Based Access: urlpaths + +fsspec supports opening files directly using URIs. Usually a +resource is clearly defined by its 'protocol', 'storage options', and 'path'. The protocol and path can usually be +combined to a urlpath string: + +```python +import fsspec + +# resource +protocol = "s3" +storage_options = {"anon": True} +path = "bucket/file.txt" + +# Create filesystem and open path +fs = fsspec.filesystem("s3", anon=True) +with fs.open("bucket/file.txt", "r") as f: + content = f.read() + +# Or open a file via its urlpath with storage_options +with fsspec.open('s3://bucket/file.txt', 'r', anon=True) as f: + content = f.read() +``` + +### Chained Filesystems + +fsspec supports composing filesystems together using the `::` separator. This allows one filesystem to be used as the target +filesystem for another: + +```python +import fsspec + +# Access a file inside a ZIP archive on S3 +with fsspec.open('zip://data.csv::s3://bucket/archive.zip', 'r', anon=True) as f: + content = f.read() + +# Read a compressed file +with fsspec.open('tar://file.txt::s3://bucket/archive.tar', 'r', anon=True) as f: + content = f.read() +``` + +### Caching + +fsspec includes powerful caching capabilities to improve performance when accessing remote files: + +```python +import fsspec + +# Simple caching +fs = fsspec.filesystem( + 's3', + anon=True, + use_listings_cache=True, + listings_expiry_time=600 # Cache for 10 minutes +) + +# File-level caching +cached_fs = fsspec.filesystem( + 'filecache', + target_protocol='s3', + target_options={'anon': True}, + cache_storage='/tmp/fsspec-cache' +) +``` + +## When to use fsspec directly + +You typically use fsspec directly when you: + +- Need filesystem-level operations (`ls`, `cp`, `rm`, `find`) +- Want to work with file-like objects without path abstractions +- Need low-level control over filesystem behavior +- Are integrating with data libraries that accept fsspec URLs +- Want to implement custom filesystem wrappers +- Want to avoid the overhead of UPath instance creation + +## Learn More + +For comprehensive information about fsspec: + +- **Official documentation**: [fsspec.readthedocs.io](https://filesystem-spec.readthedocs.io/) +- **API reference**: [Built-in filesystem implementations](https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations) +- **GitHub repository**: [fsspec/filesystem_spec](https://github.com/fsspec/filesystem_spec) +- **Usage guides**: [Examples and tutorials](https://filesystem-spec.readthedocs.io/en/latest/usage.html) + +For using fsspec with a pathlib-style interface, see [upath.md](upath.md). diff --git a/docs/concepts/index.md b/docs/concepts/index.md new file mode 100644 index 00000000..4ebef598 --- /dev/null +++ b/docs/concepts/index.md @@ -0,0 +1,9 @@ +# Overview :map: + +Universal Pathlib brings together fsspec and pathlib to provide a unified, pythonic interface for working with files across different storage systems. Understanding how these components work together will help you make the most of universal-pathlib. + +- **[Filesystem Spec](fsspec.md)** provides the foundation—a specification and collection of filesystem implementations that offer consistent access to local storage, cloud services, and remote systems. +- **[Pathlib](pathlib.md)** defines the familiar object-oriented API from Python's standard library for working with filesystem paths. +- **[Universal Pathlib](upath.md)** ties them together, implementing the [pathlib-abc](https://github.com/barneygale/pathlib-abc) interface on top of fsspec filesystems to give you a Path-like experience everywhere. + +Start with [fsspec filesystems](fsspec.md) to understand the available storage backends, then explore [stdlib pathlib](pathlib.md) to learn about the path interface, and finally see [upath](upath.md) to discover how universal-pathlib combines them into a powerful, unified API. diff --git a/docs/concepts/pathlib.md b/docs/concepts/pathlib.md new file mode 100644 index 00000000..c2c0286f --- /dev/null +++ b/docs/concepts/pathlib.md @@ -0,0 +1,144 @@ +# Pathlib :snake: + +[pathlib](https://docs.python.org/3/library/pathlib.html) is a Python standard library module that provides an object-oriented interface for working with filesystem paths. It's the modern, pythonic way to handle file paths and filesystem operations, replacing the older string-based `os.path` approach. + +## What is pathlib? + +Introduced in Python 3.4, pathlib represents filesystem paths as objects rather than strings. + +### Path Objects + +In pathlib, paths are instances of `Path` (or platform-specific subclasses) that represent local filesystem paths: + +```python +from pathlib import Path + +# Create path objects +p = Path("/home/user/documents") +p = Path("relative/path/to/file.txt") +p = Path.home() # User's home directory +p = Path.cwd() # Current working directory +``` + +### Pure vs. Concrete Paths + +pathlib distinguishes between two types of paths: + +**Pure Paths** (`PurePath`, `PurePosixPath`, `PureWindowsPath`): +- Only manipulate path strings +- Don't access the filesystem +- Work on any platform regardless of OS +- Useful for path manipulation without I/O + +```python +from pathlib import PurePath, PurePosixPath, PureWindowsPath + +# Pure path - string manipulation only +pure = PurePath("/home/user/file.txt") +parent = pure.parent # Works +name = pure.name # Works +# exists = pure.exists() # AttributeError - no filesystem access + +# Platform-specific pure paths +posix = PurePosixPath("/home/user/file.txt") # Always uses / +windows = PureWindowsPath("C:\\Users\\file.txt") # Always uses \ +``` + +**Concrete Paths** (`Path`, `PosixPath`, `WindowsPath`): +- Inherit from pure paths +- Actually access the filesystem +- Support operations like `.exists()`, `.stat()`, `.read_text()` +- Platform-specific: `PosixPath` on Unix, `WindowsPath` on Windows + +```python +from pathlib import Path + +# Concrete path - filesystem operations +p = Path("/home/user/file.txt") +exists = p.exists() # Checks filesystem +content = p.read_text() # Reads file +size = p.stat().st_size # Gets file size +``` + +## When to use pathlib + +Use pathlib when you: + +- Work with local filesystem paths in Python +- Need cross-platform path handling +- Want object-oriented path manipulation + +## What is pathlib-abc? + +[pathlib-abc](https://github.com/barneygale/pathlib-abc) is a Python library that defines abstract base classes (ABCs) for path-like objects. It provides a formal specification for the pathlib interface that can be implemented by different path types, not just local filesystem paths. + +### Abstract Base Classes for Paths + +pathlib-abc extracts the core concepts from Python's pathlib module into abstract base classes. This allows library authors and framework developers to: + +1. **Define path-like interfaces** that work across different storage backends +2. **Type hint** functions that accept any path-like object +3. **Implement custom path classes** that follow pathlib conventions +4. **Ensure compatibility** between different path implementations + +!!! info "Relationship to Python's pathlib" + Currently (as of Python 3.14), the standard library `pathlib.Path` does **not** inherit from public pathlib-abc classes. However, there is ongoing work to incorporate these ABCs into future Python releases. + +The library defines three main abstract base classes that represent different levels of path functionality: + +### JoinablePath + +`JoinablePath` is the most basic path abstraction. It represents paths that can be constructed, manipulated, and joined together, but cannot necessarily access any actual filesystem. + +**Key capabilities:** + +- Path construction and manipulation +- String operations on paths +- Path component access (name, stem, suffix, parent, etc.) +- Path joining with the `/` operator +- Pattern matching + +Think of `JoinablePath` as equivalent to pathlib's `PurePath` - it only manipulates path strings. + +### ReadablePath + +`ReadablePath` extends `JoinablePath` to add read-only filesystem operations. It represents paths where you can read data but not modify the filesystem. + +**Adds capabilities for:** + +- Reading file contents (`.read_text()`, `.read_bytes()`) +- Opening files for reading +- Checking file existence and type (`.exists()`, `.is_file()`, `.is_dir()`) +- Listing directory contents (`.iterdir()`) +- Globbing and pattern matching (`.glob()`, `.rglob()`) +- Walking directory trees (`.walk()`) +- Reading symlinks (`.readlink()`) +- Accessing file metadata (`.info` property) + +### WritablePath + +`WritablePath` extends `JoinablePath` (not `ReadablePath`) to add write operations. It represents paths where you can create, modify, and delete filesystem objects. + +**Adds capabilities for:** + +- Writing file contents (`.write_text()`, `.write_bytes()`) +- Opening files for writing +- Creating directories (`.mkdir()`) +- Creating symlinks (`.symlink_to()`) + +!!! note "WritablePath Does Not Inherit from ReadablePath" + `WritablePath` does NOT inherit from `ReadablePath`. A path that is writable is not automatically readable. In practice, most filesystem paths are both readable and writable (like `UPath` which inherits from both), but the separation allows for specialized use cases like write-only destinations or read-only sources. + +## Learn More + +For comprehensive information about pathlib: + +- **Official documentation**: [Python pathlib documentation](https://docs.python.org/3/library/pathlib.html) +- **PEP 428**: [The pathlib module – object-oriented filesystem paths](https://www.python.org/dev/peps/pep-0428/) +- **Comparison with os.path**: [Correspondence to tools in the os module](https://docs.python.org/3/library/pathlib.html#correspondence-to-tools-in-the-os-module) + +For comprehensive information about pathlib-abc: + +- **GitHub repository**: [barneygale/pathlib-abc](https://github.com/barneygale/pathlib-abc) + +For using pathlib-style paths with remote and cloud filesystems, see [upath.md](upath.md). diff --git a/docs/concepts/upath.md b/docs/concepts/upath.md new file mode 100644 index 00000000..db1a4b5d --- /dev/null +++ b/docs/concepts/upath.md @@ -0,0 +1,247 @@ + + +# Universal Pathlib ![upath](../assets/logo-128x128.svg){: #upath-logo } + +**universal-pathlib** (imported as `upath`) bridges Python's [pathlib](https://docs.python.org/3/library/pathlib.html) API with [fsspec](https://filesystem-spec.readthedocs.io/)'s filesystem implementations. It provides a familiar, pathlib-style interface for working with files across local storage, cloud services, and remote systems. + +## The Best of Both Worlds + +universal-pathlib combines: + +- **fsspec's filesystem support**: Access to S3, GCS, Azure, HDFS, HTTP, SFTP, and dozens more backends +- **pathlib's elegant API**: Object-oriented paths, `/` operator, `.exists()`, `.read_text()`, etc. + +This means you can write code using the pathlib syntax you already know, and it works seamlessly across any storage system that fsspec supports. + +## How UPath and Path Relate via pathlib-abc + +`UPath` and `pathlib.Path` are related through the abstract base classes defined in [pathlib-abc](https://github.com/barneygale/pathlib-abc). While they share a common API design, they serve different purposes and have distinct inheritance hierarchies. + +### The Class Hierarchy + +The following diagram shows how `UPath` implementations relate to `pathlib` classes through the `pathlib_abc` abstract base classes: + +```mermaid +flowchart TB + + subgraph p0[pathlib_abc] + X ----> Y + X ----> Z + end + + subgraph s0[pathlib] + X -.-> A + + A----> B + A--> AP + A--> AW + + Y -.-> B + Z -.-> B + + B--> BP + AP----> BP + B--> BW + AW----> BW + end + subgraph s1[upath] + Y ---> U + Z ---> U + + U --> UP + U --> UW + BP ---> UP + BW ---> UW + U --> UL + U --> US3 + U --> UH + U -.-> UO + end + + X(JoinablePath) + Y(WritablePath) + Z(ReadablePath) + + A(PurePath) + AP(PurePosixPath) + AW(PureWindowsPath) + B(Path) + BP(PosixPath) + BW(WindowsPath) + + U(UPath) + UP(PosixUPath) + UW(WindowsUPath) + UL(FilePath) + US3(S3Path) + UH(HttpPath) + UO(...Path) + + classDef na fill:#f7f7f7,stroke:#02a822,stroke-width:2px,color:#333 + classDef np fill:#f7f7f7,stroke:#2166ac,stroke-width:2px,color:#333 + classDef nu fill:#f7f7f7,stroke:#b2182b,stroke-width:2px,color:#333 + + class X,Y,Z na + class A,AP,AW,B,BP,BW,UP,UW np + class U,UL,US3,UH,UO nu + + style UO stroke-dasharray: 3 3 + + style p0 fill:none,stroke:#0a2,stroke-width:3px,stroke-dasharray:3,color:#0a2 + style s0 fill:none,stroke:#07b,stroke-width:3px,stroke-dasharray:3,color:#07b + style s1 fill:none,stroke:#d02,stroke-width:3px,stroke-dasharray:3,color:#d02 +``` + +**Legend:** + +- **Green (pathlib_abc)**: Abstract base classes defining the path interface +- **Blue (pathlib)**: Standard library path classes for local filesystems +- **Red (upath)**: Universal pathlib classes for all filesystems +- Solid lines: Direct inheritance +- Dotted lines: Conceptual relationship (not actual inheritance yet) + +### Understanding the Relationships + +**pathlib-abc Layer (Green):** + +- `JoinablePath` - Basic path manipulation without filesystem access +- `ReadablePath` - Adds read-only filesystem operations +- `WritablePath` - Adds write filesystem operations + +**pathlib Layer (Blue):** + +- `PurePath` - Pure path manipulation (similar to `JoinablePath` conceptually) +- `Path` - Concrete local filesystem paths (conceptually similar to `ReadablePath` + `WritablePath`) +- Platform-specific: `PosixPath`, `WindowsPath`, etc. + +**universal-pathlib Layer (Red):** + +- `UPath` - Universal path for any filesystem backend +- Local implementations: `PosixUPath`, `WindowsUPath`, `FilePath` +- Remote implementations: `S3Path`, `HttpPath`, and others + +### Key Differences + +**Current State (Python 3.9-3.13):** + +```python +from pathlib import Path +from upath import UPath +from upath.types import JoinablePath, ReadablePath, WritablePath + +# UPath explicitly implements pathlib-abc +path = UPath("s3://bucket/file.txt") +assert isinstance(path, JoinablePath) # True +assert isinstance(path, ReadablePath) # True +assert isinstance(path, WritablePath) # True + +# pathlib.Path does NOT (yet) inherit from pathlib-abc +local = Path("/home/user/file.txt") +assert isinstance(local, JoinablePath) # False +assert isinstance(local, ReadablePath) # False +assert isinstance(local, WritablePath) # False +``` + +**Important Note:** The dotted lines in the diagram represent a conceptual relationship. While `pathlib.Path` doesn't currently inherit from `pathlib_abc` classes, it implements a compatible API. Future Python versions may formalize this relationship. + +### Local Path Compatibility + +For local filesystem paths, `UPath` provides implementations that are 100% compatible with stdlib `pathlib`: + +```python +from pathlib import Path, PosixPath, WindowsPath +from upath import UPath + +# Without protocol -> returns platform-specific UPath +local = UPath("/home/user/file.txt") +assert isinstance(local, UPath) # True +assert isinstance(local, PosixPath) # True (on Unix systems) +assert isinstance(local, Path) # True + +# With file:// protocol -> returns FilePath (fsspec-based) +file_path = UPath("file:///home/user/file.txt") +assert isinstance(file_path, UPath) # True +assert not isinstance(file_path, Path) # False (uses fsspec instead) +``` + +**PosixUPath and WindowsUPath:** +- Subclass both `UPath` and `pathlib.Path` +- 100% compatible with stdlib pathlib for local paths +- Tested against CPython's pathlib test suite +- Implement `os.PathLike` protocol + +**FilePath:** +- Subclass of `UPath` only +- Uses fsspec's `LocalFileSystem` for file access +- Useful for consistent fsspec-based access across all backends +- Implements `os.PathLike` protocol + +### Remote and Cloud Paths + +For remote filesystems, `UPath` implementations provide the pathlib API backed by fsspec: + +```python +from upath import UPath + +# S3Path +s3 = UPath("s3://bucket/file.txt") +assert isinstance(s3, UPath) +assert not isinstance(s3, Path) # Not a local path + +# HttpPath +http = UPath("https://example.com/data.json") +assert isinstance(http, UPath) +assert not isinstance(http, Path) # Not a local path +``` + +### Why This Design? + +This architecture provides several benefits: + +1. **Unified API**: Same pathlib interface works across all backends +2. **Type Safety**: pathlib-abc provides formal type hints for path operations +3. **Local Compatibility**: `PosixUPath`/`WindowsUPath` maintain full stdlib compatibility +4. **Flexibility**: Easy to add new filesystem implementations +5. **Future-Proof**: Ready for potential stdlib integration of pathlib-abc + +### Writing Filesystem-Agnostic Code + +Use pathlib-abc types to write code that works with both `Path` and `UPath`: + +```python +from upath.types import ReadablePath, WritablePath + +def process_file(input_path: ReadablePath, output_path: WritablePath) -> None: + """Works with Path, UPath, or any ReadablePath/WritablePath implementation.""" + data = input_path.read_text() + processed = data.upper() + output_path.write_text(processed) + +# Works with stdlib Path +from pathlib import Path +process_file(Path("input.txt"), Path("output.txt")) + +# Works with UPath for cloud storage +from upath import UPath +process_file( + UPath("s3://input-bucket/data.txt", anon=True), + UPath("s3://output-bucket/result.txt") +) + +# Mix local and remote +process_file( + UPath("https://example.com/data.txt"), + Path("/tmp/result.txt") +) +``` + +## Learn More + +- **pathlib concepts**: See [pathlib.md](pathlib.md) for details on the pathlib API +- **fsspec backends**: See [filesystems.md](fsspec.md) for information about available filesystems +- **API reference**: Check the [API documentation](../api/index.md) for complete method details +- **fsspec details**: Visit [fsspec documentation](https://filesystem-spec.readthedocs.io/) for filesystem-specific options diff --git a/docs/contributing.md b/docs/contributing.md new file mode 100644 index 00000000..d71caf96 --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1,458 @@ +# Contributing to Universal Pathlib :heart: + +Thank you for your interest in contributing to Universal Pathlib! We're excited to have you here. :tada: + +This project thrives on community contributions, and we welcome all forms of participation - whether you're reporting bugs, suggesting features, improving documentation, or contributing code. + +--- + +## Why Contribute? :sparkles: + +Universal Pathlib is an open-source project that helps developers work seamlessly with different filesystems. By contributing, you: + +- :handshake: **Help the community** - Your contributions make it easier for others to work with cloud storage and remote filesystems +- :books: **Learn and grow** - Get hands-on experience with filesystems, Python internals, and testing +- :star2: **Build your profile** - Showcase your work in an active, project +- :people_holding_hands: **Join a welcoming community** - Work with friendly maintainers and contributors + +!!! tip "First Time Contributing?" + Don't worry! We're here to help. Everyone was a first-time contributor once, and we're happy to guide you through the process. + +--- + +## Ways to Contribute :gift: + +There are many ways to contribute to Universal Pathlib: + +
+ +- :bug: **Report Bugs** + + --- + + Found something broken? Let us know! Clear bug reports help us fix issues quickly. + + [:octicons-arrow-right-24: Report a bug](https://github.com/fsspec/universal_pathlib/issues/new) + +- :bulb: **Suggest Features** + + --- + + Have an idea for improvement? We'd love to hear it! Feature requests help shape the project's future. + + [:octicons-arrow-right-24: Request a feature](https://github.com/fsspec/universal_pathlib/issues/new) + +- :memo: **Improve Documentation** + + --- + + Spot a typo? Know a better way to explain something? Documentation improvements are always welcome! + + [:octicons-arrow-right-24: Edit the docs](https://github.com/fsspec/universal_pathlib) + +- :wrench: **Contribute Code** + + --- + + Ready to get your hands dirty? We welcome bug fixes, new features, and performance improvements. + + [:octicons-arrow-right-24: Development guide](#setting-up-your-development-environment) + +
+ +--- + +## Reporting Bugs :bug: + +Found a bug? We want to know about it! Here's how to report it effectively: + +### Before Reporting + +1. **Search existing issues** - Someone might have already reported it +2. **Try the latest version** - The bug might already be fixed +3. **Check the documentation** - Make sure it's actually a bug and not expected behavior + +### Creating a Bug Report + +When filing a bug report, please include: + +- **Operating system and Python version** - Which OS and Python version are you using? +- **Universal Pathlib version** - Which version of the project are you running? +- **Filesystem type** - S3, GCS, local, etc.? +- **What you did** - Clear steps to reproduce the issue +- **What you expected** - What should have happened? +- **What actually happened** - What did you see instead? +- **Code example** - A minimal reproducible example is extremely helpful! + +!!! example "Good Bug Report Example" + ````markdown + **Environment:** + - OS: macOS 14.0 + - Python: 3.11.5 + - universal_pathlib: 0.2.2 + - Filesystem: S3 (s3fs 2023.10.0) + + **Issue:** + `UPath.glob()` doesn't match files with spaces in their names on S3. + + **Reproduction:** + ```python + from upath import UPath + + path = UPath("s3://my-bucket/") + # This file exists: "my file.txt" + list(path.glob("my*.txt")) # Returns empty list + ``` + + **Expected:** Should find "my file.txt" + **Actual:** Returns empty list + ```` + +[:octicons-arrow-right-24: Report a bug on GitHub](https://github.com/fsspec/universal_pathlib/issues/new) + +--- + +## Requesting Features :bulb: + +Have an idea to make Universal Pathlib better? We'd love to hear it! + +### Before Requesting + +1. **Check existing issues** - Someone might have already suggested it +2. **Think about scope** - Does it fit with the project's goals? +3. **Consider alternatives** - Could it be implemented with existing features? + +### Creating a Feature Request + +When requesting a feature, please explain: + +- **The problem** - What are you trying to solve? +- **Your proposed solution** - How would you like it to work? +- **Alternatives considered** - What other approaches did you think about? +- **Use case** - When would this feature be useful? + +!!! example "Good Feature Request Example" + ````markdown + **Problem:** + When working with large directories, I need to filter paths by size + before loading them into memory. + + **Proposed Solution:** + Add a `glob_with_info()` method that yields tuples of (path, stat_info). + + **Use Case:** + Processing large S3 buckets where I only want files over 100MB: + ```python + for path, info in bucket.glob_with_info("**/*.parquet"): + if info.st_size > 100_000_000: + process(path) + ``` + + **Alternatives:** + Currently need to call `stat()` on every path, which is slow. + ```` + +[:octicons-arrow-right-24: Request a feature on GitHub](https://github.com/fsspec/universal_pathlib/issues/new) + +--- + +## Setting Up Your Development Environment :computer: + +Ready to contribute code? Here's how to set up your development environment. + +### Prerequisites + +You'll need: + +- **Python 3.9 or higher** +- **Git** - For version control +- **nox** - For running tests and other tasks + +### Installation Steps + +1. **Fork and clone the repository** + + ```bash + # Fork on GitHub first, then: + git clone https://github.com/YOUR-USERNAME/universal_pathlib.git + cd universal_pathlib + ``` + +2. **Install nox** + + ```bash + uv tool install nox + ``` + +3. **Verify your setup** + + ```bash + # List available nox sessions + nox --list-sessions + ``` + +That's it! You're ready to start developing. + +!!! tip "Using uv?" + If you prefer `uv`, you can use it as well: + ```bash + uv pip install nox + ``` + +--- + +## Running Tests :test_tube: + +Universal Pathlib has a comprehensive test suite to ensure quality and compatibility. + +### Run All Tests + +```bash +nox +``` + +This runs the full test suite across multiple Python versions. It may take a while! + +### Run Tests for Your Python Version + +```bash +nox --session=tests +``` + +This runs tests using your current Python version only. + +### Run Specific Test Files + +```bash +nox --session=tests -- tests/test_core.py +``` + +### Run Tests with Coverage + +```bash +nox --session=tests -- --cov=upath --cov-report=html +``` + +Then open `htmlcov/index.html` to view the coverage report. + +### List All Available Sessions + +```bash +nox --list-sessions +``` + +!!! info "Test Requirements" + Some tests require additional dependencies (like `s3fs` for S3 tests). Install extras as needed: + ```bash + pip install -e ".[dev,s3,gcs,azure]" + ``` + +--- + +## Code Quality :sparkles: + +We use automated tools to maintain code quality and consistency. + +### Running Linters + +```bash +nox --session=lint +``` + +This runs: +- **ruff** - Fast Python linter +- **mypy** - Type checking +- **Code formatting checks** + +### Type Checking + +```bash +nox --session=type_checking +nox --session=typesafety +``` + +!!! tip "Pre-commit Hooks" + Consider setting up pre-commit hooks to automatically check your code: + ```bash + pip install pre-commit + pre-commit install + ``` + +--- + +## Making Changes :wrench: + +Here's the workflow for contributing code: + +### 1. Create a Branch + +```bash +git checkout -b feature/my-awesome-feature +# or +git checkout -b fix/bug-description +``` + +### 2. Make Your Changes + +- Write clear, readable code +- Follow existing code style +- Add tests for new functionality +- Update documentation as needed + +### 3. Test Your Changes + +```bash +# Run tests +nox --session=tests + +# Run linters +nox --session=lint +``` + +### 4. Commit Your Changes + +```bash +git add . +git commit -m "Add feature: brief description of what you did" +``` + +!!! tip "Good Commit Messages" + - Start with a verb: "Add", "Fix", "Update", "Remove" + - Be concise but descriptive + - Reference issues: "Fix #123: Description" + +### 5. Push and Create a Pull Request + +```bash +git push origin feature/my-awesome-feature +``` + +Then open a pull request on GitHub! + +--- + +## Pull Request Guidelines :rocket: + +Your pull request should: + +- ✅ **Pass all tests** - The test suite must pass without errors +- ✅ **Maintain coverage** - Add tests for new functionality +- ✅ **Update documentation** - Document any API changes +- ✅ **Follow code style** - Pass linting checks +- ✅ **Have a clear description** - Explain what and why + +### Pull Request Template + +When creating a PR, please include: + +```markdown +## Description +Brief description of changes + +## Type of Change +- [ ] Bug fix +- [ ] New feature +- [ ] Documentation update +- [ ] Performance improvement + +## Testing +How was this tested? + +## Checklist +- [ ] Tests pass locally +- [ ] Linting passes +- [ ] Documentation updated +- [ ] CHANGELOG.md updated (if applicable) +``` + +!!! success "Don't Worry About Perfection" + It's okay to submit a work-in-progress PR! We can iterate and improve it together. Just mark it as a draft if it's not ready for review. + +--- + +## Development Tips :bulb: + +### Debugging Tests + +```bash +# Run specific test with verbose output +nox --session=tests -- tests/test_core.py::test_name -v + +# Drop into debugger on failure +nox --session=tests -- --pdb +``` +--- + +## Getting Help :question: + +Need help? We're here for you! + +- **GitHub Issues** - Ask questions, report bugs, suggest features +- **Documentation** - Check the docs for guides and examples +- **Code of Conduct** - Read our [Code of Conduct](https://github.com/fsspec/universal_pathlib/blob/main/CODE_OF_CONDUCT.rst) + +!!! info "Response Time" + We're a volunteer-driven project. We'll try to respond quickly, but please be patient if it takes a few days. + +--- + +## Recognition :star: + +We appreciate all contributions! Contributors are: + +- Listed in the project's contributor graph +- Mentioned in release notes for significant contributions +- Part of a welcoming, collaborative community + +Thank you for making Universal Pathlib better! :heart: + +--- + +## Quick Links :link: + +
+ +- :octicons-mark-github-16: **GitHub Repository** + + --- + + View the source code and contribute + + [:octicons-arrow-right-24: fsspec/universal_pathlib](https://github.com/fsspec/universal_pathlib) + +- :octicons-issue-opened-16: **Issue Tracker** + + --- + + Report bugs and request features + + [:octicons-arrow-right-24: View issues](https://github.com/fsspec/universal_pathlib/issues) + +- :octicons-git-pull-request-16: **Pull Requests** + + --- + + Submit your contributions + + [:octicons-arrow-right-24: Open a PR](https://github.com/fsspec/universal_pathlib/pulls) + +- :octicons-code-of-conduct-16: **Code of Conduct** + + --- + + Read our community guidelines + + [:octicons-arrow-right-24: View code of conduct](https://github.com/fsspec/universal_pathlib/blob/main/CODE_OF_CONDUCT.rst) + +
+ +--- + +
+ +**Ready to contribute?** :rocket: + +[Create an Issue](https://github.com/fsspec/universal_pathlib/issues/new){ .md-button .md-button--primary } +[Open a Pull Request](https://github.com/fsspec/universal_pathlib/compare){ .md-button } + +
diff --git a/docs/css/extra.css b/docs/css/extra.css new file mode 100644 index 00000000..58eaef8c --- /dev/null +++ b/docs/css/extra.css @@ -0,0 +1,11 @@ +:root { + --md-primary-fg-color: #4361EE; + scrollbar-gutter: stable; + overflow-y: scroll; +} + +.md-typeset table:not([class]) th:not(:first-child) { + min-width: 1em; + padding-left: 0.8em; + padding-right: 0.8em; +} diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..e9291d4e --- /dev/null +++ b/docs/index.md @@ -0,0 +1,167 @@ + + +![universal pathlib logo](assets/logo-text.svg){: #upath-logo } + +[![PyPI](https://img.shields.io/pypi/v/universal_pathlib.svg)](https://pypi.org/project/universal_pathlib/) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/universal_pathlib)](https://pypi.org/project/universal_pathlib/) +[![PyPI - License](https://img.shields.io/pypi/l/universal_pathlib)](https://github.com/fsspec/universal_pathlib/blob/main/LICENSE) +[![Conda (channel only)](https://img.shields.io/conda/vn/conda-forge/universal_pathlib?label=conda)](https://anaconda.org/conda-forge/universal_pathlib) + +[![Tests](https://github.com/fsspec/universal_pathlib/actions/workflows/tests.yml/badge.svg)](https://github.com/fsspec/universal_pathlib/actions/workflows/tests.yml) +[![GitHub issues](https://img.shields.io/github/issues/fsspec/universal_pathlib)](https://github.com/fsspec/universal_pathlib/issues) +[![Codestyle black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +[![Changelog](https://img.shields.io/badge/changelog-Keep%20a%20Changelog-%23E05735)](./changelog.md) + +--- + +**Universal Pathlib** is a Python library that extends the [`pathlib_abc.JoinablePath`][pathlib_abc], [`pathlib_abc.Readable`][pathlib_abc], and [`pathlib_abc.Writable`][pathlib_abc] API to give you a unified, Pythonic interface for working with files, whether they're on your local machine, in S3, on GitHub, or anywhere else. Built on top of [`filesystem_spec`][fsspec], it brings the convenienve of a [`pathlib.Path`][pathlib]-like interface to cloud storage, remote filesystems, and more! :sparkles: + +[pathlib_abc]: https://github.com/barneygale/pathlib-abc +[pathlib]: https://docs.python.org/3/library/pathlib.html +[fsspec]: https://filesystem-spec.readthedocs.io/en/latest/intro.html + +--- + +If you enjoy working with Python's [pathlib][pathlib] objects to operate on local file system paths, +universal pathlib provides the same interface for many supported [ filesystem_spec ][fsspec] +implementations, from cloud-native object storage like `Amazon's S3 Storage`, `Google Cloud Storage`, +`Azure Blob Storage`, to `http`, `sftp`, `memory` stores, and many more... + +If you're familiar with [ filesystem_spec ][fsspec], then universal pathlib provides a convenient +way to handle the path, protocol and storage options of a object stored on a fsspec filesystem in a +single container (`upath.UPath`). And it further provides a pathlib interface to do path operations on the +fsspec urlpath. + +The great part is, if you're familiar with the [pathlib.Path][pathlib] API, you can immediately +switch from working with local paths to working on remote and virtual filesystem by simply using +the `UPath` class: + +=== "The Problem" + + ```python + # Local files: use pathlib + from pathlib import Path + local_file = Path("data/file.txt") + content = local_file.read_text() + + # S3 files: use boto3/s3fs + import boto3 + s3 = boto3.client('s3') + obj = s3.get_object(Bucket='bucket', Key='data/file.txt') + content = obj['Body'].read().decode('utf-8') + + # Different APIs, different patterns 😫 + ``` + +=== "The Solution" + + ```python + # All files: use UPath! ✨ + from upath import UPath + + local_file = UPath("data/file.txt") + s3_file = UPath("s3://bucket/data/file.txt") + + # Same API everywhere! 🎉 + content = local_file.read_text() + content = s3_file.read_text() + ``` + +[Learn more about why you should use Universal Pathlib →](why.md){ .md-button } + +--- + +## Quick Start :rocket: + +### Installation + +```bash +pip install universal-pathlib +``` + +!!! tip "Installing for specific filesystems" + To use cloud storage or other remote filesystems, install the necessary fsspec extras: + + ```bash + pip install "universal-pathlib" "fsspec[s3,gcs,azure]" + ``` + + See the [Installation Guide](install.md) for more details. + +### TL;DR Examples + +```python +from upath import UPath + +# Works with local paths +local_path = UPath("documents/notes.txt") +local_path.write_text("Hello, World!") +print(local_path.read_text()) # "Hello, World!" + +# Works with S3 +s3_path = UPath("s3://my-bucket/data/processed/results.csv") +if s3_path.exists(): + data = s3_path.read_text() + +# Works with HTTP +http_path = UPath("https://example.com/data/file.json") +if http_path.exists(): + content = http_path.read_bytes() + +# Works with many more! 🌟 +``` + +--- + +## Currently supported filesystems + +- :fontawesome-solid-folder: `file:` and `local:` Local filesystem +- :fontawesome-solid-memory: `memory:` Ephemeral filesystem in RAM +- :fontawesome-brands-microsoft: `az:`, `adl:`, `abfs:` and `abfss:` Azure Storage _(requires `adlfs`)_ +- :fontawesome-solid-database: `data:` RFC 2397 style data URLs _(requires `fsspec>=2023.12.2`)_ +- :fontawesome-brands-github: `github:` GitHub repository filesystem +- :fontawesome-solid-globe: `http:` and `https:` HTTP(S)-based filesystem +- :fontawesome-solid-server: `hdfs:` Hadoop distributed filesystem +- :fontawesome-brands-google: `gs:` and `gcs:` Google Cloud Storage _(requires `gcsfs`)_ +- :fontawesome-brands-aws: `s3:` and `s3a:` AWS S3 _(requires `s3fs`)_ +- :fontawesome-solid-network-wired: `sftp:` and `ssh:` SFTP and SSH filesystems _(requires `paramiko`)_ +- :fontawesome-solid-share-nodes: `smb:` SMB filesystems _(requires `smbprotocol`)_ +- :fontawesome-solid-cloud: `webdav:`, `webdav+http:` and `webdav+https:` WebDAV _(requires `webdav4[fsspec]`)_ + +!!! info "Untested Filesystems" + Other fsspec-compatible filesystems likely work through the default implementation. If you encounter issues, please [report it our issue tracker](https://github.com/fsspec/universal_pathlib/issues)! We're happy to add official support! + +--- + +## Getting Help :question: + +Need help? We're here for you! + +- :fontawesome-brands-github: [GitHub Issues](https://github.com/fsspec/universal_pathlib/issues) - Report bugs or request features +- :material-book-open-variant: [Documentation](https://universal-pathlib.readthedocs.io/) - You're reading it! + +!!! tip "Before Opening an Issue" + Please check if your question has already been answered in the documentation or existing issues. + +--- + +## License :page_with_curl: + +Universal Pathlib is distributed under the [MIT license](https://github.com/fsspec/universal_pathlib/blob/main/LICENSE), making it free and open source software. Use it freely in your projects! + +--- + +
+ +**Ready to get started?** + +[Install Now](install.md){ .md-button .md-button--primary } + +
diff --git a/docs/install.md b/docs/install.md new file mode 100644 index 00000000..954176b8 --- /dev/null +++ b/docs/install.md @@ -0,0 +1,81 @@ + +# Installation :package: + +Getting started with `universal-pathlib` is easy! Choose your preferred package manager below and you'll be working with cloud storage in minutes. + +## Quick Install + +=== "uv" + + ```bash + uv add universal-pathlib + ``` + +=== "pip" + + ```bash + python -m pip install universal-pathlib + ``` + +=== "conda" + + ```bash + conda install -c conda-forge universal-pathlib + ``` + +That's it! You now have `universal-pathlib` installed. :tada: + +## Filesystem-Specific Dependencies + +While `universal-pathlib` comes with `fsspec` out of the box, **some filesystems require additional packages**. Don't worry—installing them is straightforward! + +For example, to work with **AWS S3**, you'll need to install `s3fs`: + +```bash +pip install s3fs +# or better yet, use fsspec extras: +pip install "fsspec[s3]" +``` + +Here are some common filesystem extras you might need: + +| Filesystem | Install Command | +|------------|----------------| +| **AWS S3** | `pip install "fsspec[s3]"` | +| **Google Cloud Storage** | `pip install "fsspec[gcs]"` | +| **Azure Blob Storage** | `pip install "fsspec[azure]"` | +| **HTTP/HTTPS** | `pip install "fsspec[http]"` | +| **GitHub** | `pip install "fsspec[github]"` | +| **SSH/SFTP** | `pip install "fsspec[ssh]"` | + +## Adding to Your Project + +When adding `universal-pathlib` to your project, specify the filesystem extras you need. Here's a `pyproject.toml` example for a project using **S3** and **HTTP**: + +```toml +[project] +name = "myproject" +requires-python = ">=3.9" +dependencies = [ + "universal_pathlib>=0.3.4", + "fsspec[s3,http]", # Add the filesystems you need +] +``` + +!!! tip "Complete List of Filesystem Extras" + + For a complete overview of all available filesystem extras and their dependencies, check out the [fsspec pyproject.toml file][fsspec-pyproject-toml]. It includes extras for: + + - Cloud storage (S3, GCS, Azure, etc.) + - Remote protocols (HTTP, FTP, SSH, etc.) + - Specialized systems (HDFS, WebDAV, SMB, etc.) + + [fsspec-pyproject-toml]: https://github.com/fsspec/filesystem_spec/blob/master/pyproject.toml#L26 + +--- + +
+ +**Ready to get started?** Learn about [Universal Pathlib Concepts](concepts/index.md) :rocket: + +
diff --git a/docs/migration.md b/docs/migration.md new file mode 100644 index 00000000..1e7569d9 --- /dev/null +++ b/docs/migration.md @@ -0,0 +1,259 @@ + +# Migration Guide + +This guide helps you migrate to newer versions of universal-pathlib. + +!!! warning + + Please open an issue if you run into issues when migrating to a newer UPath version + and this guide is missing information. + + +## Migrating to v0.3.0 + +Version `0.3.0` introduced a breaking change to fix a longstanding bug related to `os.PathLike` protocol compliance. This change affects how UPath instances work with standard library functions that expect local filesystem paths. + +### Background: PathLike Protocol and Local Filesystem Paths + +In Python, `os.PathLike` objects and `pathlib.Path` subclasses represent **local filesystem paths**. The standard library functions like `os.remove()`, `shutil.copy()`, and similar expect paths that point to the local filesystem. However, UPath implementations like `S3Path` or `MemoryPath` do not represent local filesystem paths and should not be treated as such. + +Prior to `v0.3.0`, all UPath instances incorrectly implemented `os.PathLike`, which could lead to runtime errors when non-local paths were passed to functions expecting local paths. Starting with `v0.3.0`, only local UPath implementations (`PosixUPath`, `WindowsUPath`, and `FilePath`) implement `os.PathLike`. + +### Migration Strategies + +If your code passes UPath instances to functions expecting `os.PathLike` objects, you have several options: + +**Option 1: Explicitly Request a Local Path** (Recommended) + +```python +import os +from upath import UPath + +# Explicitly specify the file:// protocol to get a FilePath instance +path = UPath(__file__, protocol="file") +assert isinstance(path, os.PathLike) # True + +# Now you can safely use it with os functions +os.remove(path) +``` + +**Option 2: Use UPath's Filesystem Operations** + +```python +from upath import UPath + +# Works for any UPath implementation, not just local paths +path = UPath("s3://bucket/file.txt") +path.unlink() # UPath's native unlink method +``` + +**Option 3: Use Type Checking with upath.types** + +For code that needs to work with different path types, use the type hints from `upath.types` to properly specify your requirements: + +```python +import os +from upath import UPath +from upath.types import ( + JoinablePathLike, + ReadablePathLike, + WritablePathLike, +) + +def read_only_local_file(path: os.PathLike) -> str: + """Read a file on the local filesystem.""" + with open(path) as f: + return f.read() + +def write_only_local_file(path: os.PathLike, content: str) -> None: + """Write to a file on the local filesystem.""" + with open(path, 'w') as f: + f.write(content) + +def read_any_file(path: ReadablePathLike) -> str: + """Read a file on any filesystem.""" + return UPath(path).read_text() + +def write_any_file(path: WritablePathLike, content: str) -> None: + """Write a file on any filesystem.""" + UPath(path).write_text(content) +``` + +### Example: Incorrect Code That Would Fail + +The following example shows code that would incorrectly work in `v0.2.x` but properly fail in `v0.3.0`: + +```python +import os +from upath import UPath + +# This creates a MemoryPath, which is not a local filesystem path +path = UPath("memory:///file.txt") + +# In v0.2.x this would incorrectly accept the path and fail at runtime +# In v0.3.0 this correctly fails at type-check time +os.remove(path) # TypeError: expected str, bytes or os.PathLike, not MemoryPath +``` + +### Working with Polars and Object Store + +When using UPath with [Polars](https://pola.rs/), be aware that Polars uses Rust's [object-store](https://docs.rs/object_store/) library instead of fsspec. This requires special handling to preserve storage options. + +!!! warning "Don't Rely on Implicit String Conversion" + Avoid passing UPath instances directly to functions that implicitly cast them to strings via `os.fspath()` or `os.path.expanduser()`. This loses storage options and can lead to authentication failures. + +**Problematic Pattern:** + +```python +import polars as pl +from upath import UPath + +# This loses storage_options when implicitly converted to string! +path = UPath('s3://bucket/file.parquet', anon=True) +df = pl.read_parquet(path) # anon=True is lost! +``` + +**Recommended Approaches:** + +**Option 1: Use fsspec/s3fs as the Backend** (via file handle) + +```python +import polars as pl +from upath import UPath + +path = UPath("s3://bucket/file.parquet", anon=True) + +# Open file handle with fsspec, preserving storage_options +df = pl.scan_parquet(path.open('rb')) +``` + +**Option 2: Use Polars' Native Rust Backend** (with object-store options) + +```python +import polars as pl +from upath import UPath + +path = UPath("s3://bucket/file.parquet", key="ACCESS_KEY", secret="SECRET_KEY") + +# Convert fsspec storage_options to object-store format +object_store_options = { + "aws_access_key_id": path.storage_options.get("key"), + "aws_secret_access_key": path.storage_options.get("secret"), + # Add other options as needed +} + +df = pl.scan_parquet(path.as_uri(), storage_options=object_store_options) +``` + +!!! info "Storage Options Mapping" + Polars uses [object-store configuration keys](https://docs.rs/object_store/latest/object_store/aws/enum.AmazonS3ConfigKey.html), which differ from fsspec's naming: + + | fsspec/s3fs | object-store | + |-------------|--------------| + | `key` | `aws_access_key_id` | + | `secret` | `aws_secret_access_key` | + | `endpoint_url` | `aws_endpoint` | + | `region_name` | `aws_region` | + +See also: [pola-rs/polars#24921](https://github.com/pola-rs/polars/issues/24921) + +### Extending UPath via `_protocol_dispatch=False` + +If you previously used `_protocol_dispatch=False` to enable extension of the UPath API, we now recommend subclassing `upath.extensions.ProxyUPath`. See the advanced usage documentation for examples. + +## Migrating to v0.2.0 + +### _FSSpecAccessor Subclasses with Custom Filesystem Access Methods + +If you implemented a custom accessor subclass, override the corresponding `UPath` methods in your subclass directly: + +```python +# OLD: v0.1.x +from upath.core import UPath, _FSSpecAccessor + +class MyAccessor(_FSSpecAccessor): + def exists(self, path, **kwargs): + # custom logic + pass + +class MyPath(UPath): + _default_accessor = MyAccessor + + +# NEW: v0.2.0+ +from upath import UPath + +class MyPath(UPath): + def exists(self, *, follow_symlinks=True): + # custom logic + pass +``` + +### _FSSpecAccessor Subclasses with Custom `__init__` Method + +If you implemented a custom `__init__` method for your accessor subclass to customize fsspec filesystem instantiation, use the new `_fs_factory` or `_parse_storage_options` classmethods: + +```python +# OLD: v0.1.x +import fsspec +from upath.core import UPath, _FSSpecAccessor + +class MyAccessor(_FSSpecAccessor): + def __init__(self, parsed_url, **kwargs): + # custom filesystem setup + super().__init__(parsed_url, **kwargs) + +class MyPath(UPath): + _default_accessor = MyAccessor + + +# NEW: v0.2.0+ +from upath import UPath + +class MyPath(UPath): + @classmethod + def _fs_factory(cls, protocol, storage_options): + # custom filesystem setup + return super()._fs_factory(protocol, storage_options) +``` + +### Access to `._accessor` + +The `_accessor` attribute and the `_FSSpecAccessor` class are deprecated. Use `UPath().fs` to access the underlying filesystem: + +```python +# OLD: v0.1.x +from upath import UPath + +class MyPath(UPath): + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + self._accessor.mkdir(self.path, **kwargs) + + +# NEW: v0.2.0+ +from upath import UPath + +class MyPath(UPath): + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + self.fs.mkdir(self.path, **kwargs) +``` + +### Private Attributes to Public API + +Move from deprecated private attributes to public API: + +| Deprecated | v0.2.0+ | +|:-----------|:--------| +| `UPath()._path` | `UPath().path` | +| `UPath()._kwargs` | `UPath().storage_options` | +| `UPath()._drv` | `UPath().drive` | +| `UPath()._root` | `UPath().root` | +| `UPath()._parts` | `UPath().parts` | + +### Access to `._url` + +The `._url` attribute will likely be deprecated once `UPath()` has support for URI fragments and query parameters through a public API. If you need this functionality, please open an issue. + +### Custom Path Flavours + +The `_URIFlavour` class was removed. The internal `FSSpecFlavour` in `upath._flavour` is experimental. If you need custom path flavour functionality, please open an issue to discuss maintainable solutions. diff --git a/docs/usage.md b/docs/usage.md new file mode 100644 index 00000000..90f6b0bf --- /dev/null +++ b/docs/usage.md @@ -0,0 +1,273 @@ + + +# Usage Guide + +Welcome! This guide will help you get started with Universal Pathlib. If you already know Python's `pathlib`, you'll feel right at home—`UPath` works the same way, but for any filesystem. + +## Getting Started + +First, import what you need: + +```python +from upath import UPath +``` + +That's it! You're ready to work with files anywhere. + +--- + +## Common Tasks + +### How do I work with local files? + +UPath behaves just like `pathlib.Path` for local files: + +```python +import pathlib +from tempfile import NamedTemporaryFile + +# Create a local path +tmp = NamedTemporaryFile() +local_path = UPath(tmp.name) + +# It's a regular pathlib object! +assert isinstance(local_path, (pathlib.PosixPath, pathlib.WindowsPath)) +``` + +Want to rely on fsspec for local filesystems too? Use a `file://` URI: + +```python +local_uri = local_path.absolute().as_uri() +# Result: 'file:///tmp/tmpXXXXXX' + +local_upath = UPath(local_uri) +# Now it uses fsspec backend +assert isinstance(local_upath, UPath) +``` + +### How do I connect to cloud storage or remote filesystems? + +Just use the appropriate URI scheme. UPath will automatically connect to the right filesystem: + +```python +# Amazon S3 +s3path = UPath("s3://my-bucket/data/file.txt") + +# Google Cloud Storage +gcs_path = UPath("gs://my-bucket/data.csv") + +# Azure Blob Storage +az_path = UPath("az://container/blob.parquet") + +# GitHub repositories +gh_path = UPath("github://fsspec:universal_pathlib@main/") +``` + +You can also pass connection options as keyword arguments: + +```python +# GitHub with explicit parameters +ghpath = UPath('github:/', org='fsspec', repo='universal_pathlib', sha='main') +``` + +Or use the `protocol` keyword argument to select the implementation: + +```python +# Using protocol kwarg for S3 +s3path = UPath('my-bucket/data/file.txt', protocol='s3') + +# Using protocol kwarg for Azure with configuration +azpath = UPath( + 'container/blob.parquet', + protocol='az', + account_name='myaccount', + account_key='mykey' +) +``` + +### How do I read a file? + +Same as regular pathlib—just call `read_text()` or `read_bytes()`: + +```python +# Read from GitHub +ghpath = UPath('github://fsspec:universal_pathlib@main/') +readme = ghpath / "README.md" +first_line = readme.read_text().splitlines()[0] +print(first_line) +``` + +For more control, use `open()`: + +```python +s3path = UPath("s3://spacenet-dataset/LICENSE.md", anon=True) +with s3path.open("rt", encoding="utf-8") as f: + print(f.read(22)) +``` + +### How do I list files in a directory? + +Use `iterdir()` to iterate through contents: + +```python +s3path = UPath("s3://spacenet-dataset", anon=True) + +for item in s3path.iterdir(): + if item.is_file(): + print(item) + break +``` + +### How do I work with path components? + +All the familiar `pathlib` attributes work: + +```python +ghpath = UPath('github://fsspec:universal_pathlib@main/') +readme = ghpath / "README.md" + +# Get different parts of the path +print(readme.name) # "README.md" +print(readme.stem) # "README" +print(readme.suffix) # ".md" +print(readme.parent) # The parent directory + +# Get the full path as a string +print(str(readme)) + +# Get just the path without the scheme +print(readme.path) +``` + +### How do I join paths? + +Use the `/` operator, just like pathlib: + +```python +base = UPath("s3://my-bucket") +data_dir = base / "data" / "processed" +csv_file = data_dir / "output.csv" +``` + +### How do I check if a file exists? + +Use `exists()`, `is_file()`, or `is_dir()`: + +```python +s3path = UPath("s3://spacenet-dataset/LICENSE.md") + +if s3path.exists(): + print("File exists!") + +if s3path.is_file(): + print("It's a file!") +``` + +### How do I search for files matching a pattern? + +Use `glob()` for pattern matching: + +```python +s3path = UPath("s3://spacenet-dataset") + +# Find all .TIF files in a specific directory +paris_data = s3path / "AOIs" / "AOI_3_Paris" +for tif_file in paris_data.glob("**.TIF"): + print(tif_file) + break +``` + +!!! note "Glob Syntax" + The glob syntax follows [fsspec conventions](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.glob), which may differ slightly from pathlib's glob. + +--- + +## Advanced Usage + +### How do I pass credentials or configuration? + +Pass them as keyword arguments when creating the path: + +```python +# S3 with custom endpoint and credentials +s3_path = UPath( + "s3://my-bucket/file.txt", + endpoint_url="https://s3.custom-domain.com", + key="ACCESS_KEY", + secret="SECRET_KEY" +) + +# Anonymous access to public buckets +public_s3 = UPath("s3://spacenet-dataset", anon=True) +``` + +### How do I access the underlying filesystem? + +UPath gives you access to the fsspec filesystem when you need lower-level control: + +```python +path = UPath("s3://my-bucket/file.txt") + +# Access the filesystem object +fs = path.fs + +# Get the path string for use with fsspec +path_str = path.path + +# Get storage options +options = path.storage_options + +# Use fsspec directly if needed +with fs.open(path_str, 'rb') as f: + data = f.read() +``` + +### How do I create a custom UPath for a new filesystem? + +Subclass `UPath` and register it: + +```python +from upath import UPath +from upath.registry import register_implementation + +class MyCustomPath(UPath): + def custom_method(self): + return f"Custom behavior for {self}" + +# Register for your protocol +register_implementation("myproto", MyCustomPath) + +# Now it works! +my_path = UPath("myproto://server/path") +``` + +--- + +## Supported Filesystems + +Universal Pathlib works with any [fsspec](https://filesystem-spec.readthedocs.io/) filesystem. Here are some popular ones: + +| Protocol | Filesystem | Install Package | +|----------|-----------|-----------------| +| `file://` | Local files | _(built-in)_ | +| `s3://` | Amazon S3 | `s3fs` | +| `gs://`, `gcs://` | Google Cloud Storage | `gcsfs` | +| `az://`, `abfs://` | Azure Blob Storage | `adlfs` | +| `github://` | GitHub | _(built-in)_ | +| `http://`, `https://` | HTTP(S) | _(built-in)_ | +| `ssh://`, `sftp://` | SSH/SFTP | `paramiko` | +| `hdfs://` | Hadoop HDFS | `pyarrow` | +| `smb://` | SMB/Samba | `smbprotocol` | +| `webdav://` | WebDAV | `webdav4` | + +You can see all available implementations programmatically: + +```python +from fsspec.registry import known_implementations + +for name, details in sorted(known_implementations.items()): + print(f"{name}: {details['class']}") +``` + +!!! tip "Custom Filesystems" + Built a custom fsspec filesystem? UPath will work with it automatically! diff --git a/docs/why.md b/docs/why.md new file mode 100644 index 00000000..db4f8555 --- /dev/null +++ b/docs/why.md @@ -0,0 +1,241 @@ +# Why Use Universal Pathlib? :sparkles: + +If you've ever worked with cloud storage or remote filesystems in Python, you've probably experienced the frustration of juggling different APIs. Universal Pathlib solves this problem elegantly by bringing the beloved `pathlib.Path` interface to *any* filesystem spec filesystem. + +--- + +## The Problem: Filesystem dependent APIs :broken_heart: + +Let's face it: working with files across different storage backends is messy. + +### Example: The Old Way + +```python +# Local files +from pathlib import Path +local_file = Path("data/results.csv") +with local_file.open('r') as f: + data = f.read() + +# S3 files +import boto3 +s3 = boto3.resource('s3') +obj = s3.Object('my-bucket', 'data/results.csv') +data = obj.get()['Body'].read().decode('utf-8') + +# Azure Blob Storage +from azure.storage.blob import BlobServiceClient +blob_client = BlobServiceClient.from_connection_string(conn_str) +container_client = blob_client.get_container_client('my-container') +blob_client = container_client.get_blob_client('data/results.csv') +data = blob_client.download_blob().readall().decode('utf-8') + +# Three different APIs, three different patterns 😫 +``` + +Each storage backend has its own: + +- :material-api: **Different API** - Learn a new interface for each service +- :material-puzzle: **Different patterns** - Different ways to read, write, and list files +- :material-code-braces: **Different imports** - Manage multiple dependencies +- :material-hammer-wrench: **Different configurations** - Each with unique setup requirements + +!!! danger "The Maintenance Nightmare" + Want to switch from S3 to GCS? Rewrite your code. Need to support multiple backends? Write wrapper functions. Testing? Mock each service differently. This doesn't scale! + +--- + +## The Solution: One API to Rule Them All :crown: + +Universal Pathlib provides a single, unified interface that works everywhere: + +```python +from upath import UPath + +# Local files +local_file = UPath("data/results.csv") + +# S3 files +s3_file = UPath("s3://my-bucket/data/results.csv") + +# Azure Blob Storage +azure_file = UPath("az://my-container/data/results.csv") + +# Same API everywhere! ✨ +for path in [local_file, s3_file, azure_file]: + with path.open('r') as f: + data = f.read() +``` + +!!! success "One API, Infinite Possibilities" + Write your code once, run it anywhere. Switch backends by changing a URL. Test locally, deploy to the cloud. It just works! :sparkles: + +--- + +## Key Benefits :trophy: + +### 1. Familiar and Pythonic :snake: + +If you know Python's `pathlib`, you already know Universal Pathlib! + +```python +from upath import UPath + +# All the familiar pathlib operations +path = UPath("s3://bucket/data/file.txt") + +print(path.name) # "file.txt" +print(path.stem) # "file" +print(path.suffix) # ".txt" +print(path.parent) # UPath("s3://bucket/data") + +# Path joining +output = path.parent / "processed" / "output.csv" + +# File operations +path.write_text("Hello!") +content = path.read_text() + +# Directory operations +for item in path.parent.iterdir(): + print(item) +``` + +!!! tip "Zero Learning Curve" + Your existing pathlib knowledge transfers directly. No new concepts to learn, no cognitive overhead! + +### 2. Write Once, Run Anywhere :earth_americas: + +Change storage backends without changing code: + +=== "Development (Local)" + + ```python + from upath import UPath + + def process_data(input_path: str, output_path: str): + data_file = UPath(input_path) + result_file = UPath(output_path) + + # Read, process, write + data = data_file.read_text() + processed = data.upper() + result_file.write_text(processed) + + # Local development + process_data("data/input.txt", "data/output.txt") + ``` + +=== "Production (S3)" + + ```python + from upath import UPath + + def process_data(input_path: str, output_path: str): + data_file = UPath(input_path) + result_file = UPath(output_path) + + # Same code! Just different paths + data = data_file.read_text() + processed = data.upper() + result_file.write_text(processed) + + # Production on S3 + process_data( + "s3://my-bucket/data/input.txt", + "s3://my-bucket/data/output.txt" + ) + ``` + +=== "Testing (Memory)" + + ```python + from upath import UPath + + def process_data(input_path: str, output_path: str): + data_file = UPath(input_path) + result_file = UPath(output_path) + + # Same code! No mocking needed + data = data_file.read_text() + processed = data.upper() + result_file.write_text(processed) + + # Fast tests with in-memory filesystem + process_data( + "memory://input.txt", + "memory://output.txt" + ) + ``` + +!!! success "Truly Portable Code" + Your business logic stays clean and your application does not have to + care about where the files live anymore. + +### 3. Type-Safe and IDE-Friendly :computer: + +Universal Pathlib includes type hints for excellent IDE support: + +```python +from upath import UPath +from pathlib import Path + +def process_file(path: UPath | Path) -> str: + # Your IDE knows about all methods! + if path.exists(): # ✓ Autocomplete + content = path.read_text() # ✓ Type checked + return content.upper() + return "" + +# Works with both! +local_result = process_file(UPath("file.txt")) +s3_result = process_file(UPath("s3://bucket/file.txt")) +``` + +!!! info "Editor Support" + Get autocomplete, type checking, and inline documentation in VS Code, PyCharm, and other modern Python IDEs. + +### 4. Extensively Tested :test_tube: + +Universal Pathlib runs a large subset of CPython's pathlib test suite: + +- :white_check_mark: **Compatibility tested** against standard library pathlib +- :white_check_mark: **Cross-version tested** on Python 3.9-3.14 +- :white_check_mark: **Integration tested** with real filesystems +- :white_check_mark: **Regression tested** for each release + +!!! quote "Extensively Tested" + When we say "pathlib-compatible," we mean it. + +### 5. Extensible and Future-Proof :rocket: + +Built on `fsspec`, the standard for Python filesystem abstractions: + +```python +# Works with many fsspec filesystems! +UPath("s3://...", anon=True) +UPath("gs://...", token='anon') +UPath("az://...") +UPath("https://...") +``` + +Need a custom filesystem? Implement it once with fsspec, and UPath works automatically! + +!!! tip "Ecosystem Benefits" + Leverage the entire fsspec ecosystem: caching, compression, callback hooks, and more! + +--- + +## Next Steps :footprints: + +Ready to give Universal Pathlib a try? + +1. **[Install Universal Pathlib](install.md)** - Get set up in minutes +2. **[Understand the concepts](concepts/index.md)** - Understand the concepts +3. **[Read the API docs](api/index.md)** - Learn about all the features + +
+ +[Install Now →](install.md){ .md-button .md-button--primary } + +
diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..daaa1dc4 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,135 @@ +site_name: Universal Pathlib +site_description: A universal pathlib implementation for Python +strict: true +# site_url: !ENV READTHEDOCS_CANONICAL_URL + +theme: + name: 'material' + logo: assets/logo-128x128-white.svg + favicon: 'assets/favicon.png' + palette: + - media: "(prefers-color-scheme: light)" + toggle: + icon: material/lightbulb-outline + name: "Switch to dark mode" + - media: "(prefers-color-scheme: dark)" + scheme: slate + toggle: + icon: material/lightbulb + name: "Switch to light mode" + features: + # - content.tabs.link + - content.code.annotate + - content.code.copy + - announce.dismiss + - navigation.tabs + +extra_css: + - css/extra.css + +repo_name: fsspec/universal_pathlib +repo_url: https://github.com/fsspec/universal_pathlib +edit_uri: edit/main/docs/ + +validation: + omitted_files: warn + absolute_links: warn + unrecognized_links: warn + +nav: + - Home: + - Introduction: index.md + - Why use Universal Pathlib: why.md + - Installation: install.md + - Contributing: contributing.md + - Changelog: changelog.md + - Concepts: + - Overview: concepts/index.md + - Filesystem Spec: concepts/fsspec.md + - Standard Library Pathlib: concepts/pathlib.md + - Universal Pathlib: concepts/upath.md + - Usage: + - Basic Usage: usage.md + - API Reference: + - Core: api/index.md + - Registry: api/registry.md + - Implementations: api/implementations.md + - Extensions: api/extensions.md + - Types: api/types.md + - Migration Guide: migration.md + +markdown_extensions: +- tables +- attr_list +- toc: + permalink: true + title: Page contents +- admonition +- pymdownx.details +- pymdownx.highlight: + pygments_lang_class: true +- pymdownx.extra +- pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg +- pymdownx.tasklist: + custom_checkbox: true +- pymdownx.tabbed: + alternate_style: true +- pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + +watch: + - docs/ + - upath/ + +plugins: +- search +- mkdocstrings: + handlers: + python: + inventories: + - https://docs.python.org/3/objects.inv + paths: [.] + options: + preload_modules: + - __future__ + - typing + - abc + - asyncio + - pathlib + - pathlib_abc + - fsspec + - upath.types._abc + - upath.types + - upath.registry + - upath.core + docstring_style: numpy + docstring_section_style: list + group_by_category: false + members_order: source + docstring_options: + ignore_init_summary: true + docstring_section_style: spacy + merge_init_into_class: true + show_source: true + show_root_heading: true + show_root_toc_entry: true + allow_inspection: true + separate_signature: true + show_signature: true + show_signature_annotations: true + show_signature_type_parameters: true + signature_crossrefs: true + show_symbol_type_heading: true +- exclude: + glob: + - _plugins/* + - __pycache__/* + - tests/* + - test_*.py +hooks: + - docs/_plugins/copy_changelog.py diff --git a/noxfile.py b/noxfile.py index 1a1952ce..044529cd 100644 --- a/noxfile.py +++ b/noxfile.py @@ -161,3 +161,17 @@ def generate_flavours(session): stdout=target, stderr=None, ) + + +@nox.session(name="docs-build", python=BASE_PYTHON) +def docs_build(session): + """Build the documentation in strict mode.""" + session.install("--group=docs", "-e", ".") + session.run("mkdocs", "build") + + +@nox.session(name="docs-serve", python=BASE_PYTHON) +def docs_serve(session): + """Serve the documentation with live reloading.""" + session.install("--group=docs", "-e", ".") + session.run("mkdocs", "serve", "--no-strict") diff --git a/pyproject.toml b/pyproject.toml index 4fe8d629..2d16950d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -209,3 +209,14 @@ ignore-words-list = " " [tool.bandit] exclude_dirs = ["tests"] skips = ["B101"] + +[dependency-groups] +docs = [ + "mkdocs>=1.6.1", + "click!=8.2.2,!=8.3.0", # https://github.com/mkdocs/mkdocs/issues/4032 + "mkdocs-material>=9.6.22", + "mkdocstrings[python]>=0.30.1", + "mkdocs-exclude>=1.0.2", + "pymdown-extensions>=10.7.0", + "ruff>=0.14.1", +] diff --git a/upath/core.py b/upath/core.py index 0e7b48ff..7f25bb21 100644 --- a/upath/core.py +++ b/upath/core.py @@ -1,3 +1,5 @@ +"""upath.core module: UPath base class implementation""" + from __future__ import annotations import posixpath @@ -101,11 +103,17 @@ def _buffering2blocksize(mode: str, buffering: int) -> int | None: def _raise_unsupported(cls_name: str, method: str) -> NoReturn: - "relative path does not support method(), because cls_name.cwd() is unsupported" raise NotImplementedError(f"{cls_name}.{method}() is unsupported") class _UPathMeta(ABCMeta): + """metaclass for UPath to customize instance creation + + There are two main reasons for this metaclass: + - support copying UPath instances via UPath(existing_upath) + - force calling __init__ on instance creation for instances of a non-subclass + """ + if sys.version_info < (3, 11): # pathlib 3.9 and 3.10 supported `Path[str]` but # did not return a GenericAlias but the class itself? @@ -129,11 +137,16 @@ def __call__(cls: type[_MT], *args: Any, **kwargs: Any) -> _MT: class _UPathMixin(metaclass=_UPathMeta): + """Mixin class for UPath to allow sharing some common functionality + between UPath and PosixUPath/WindowsUPath. + """ + __slots__ = () @property @abstractmethod def parser(self) -> UPathParser: + """The parser (flavour) for this UPath instance.""" raise NotImplementedError @property @@ -203,17 +216,67 @@ def _relative_base(self, value: str | None) -> None: @property def protocol(self) -> str: - """The fsspec protocol for the path.""" + """The fsspec protocol for the path. + + Note + ---- + Protocols are linked to upath and fsspec filesystems via the + `upath.registry` and `fsspec.registry` modules. They basically + represent the URI scheme used for the specific filesystem. + + Examples + -------- + >>> from upath import UPath + >>> p0 = UPath("s3://my-bucket/path/to/file.txt") + >>> p0.protocol + 's3' + >>> p1 = UPath("/foo/bar/baz.txt", protocol="memory") + >>> p1.protocol + 'memory' + + """ return self._protocol @property def storage_options(self) -> Mapping[str, Any]: - """The fsspec storage options for the path.""" + """The read-only fsspec storage options for the path. + + Note + ---- + Storage options are specific to each fsspec filesystem and + can include parameters such as authentication credentials, + connection settings, and other options that affect how the + filesystem interacts with the underlying storage. + + Examples + -------- + >>> from upath import UPath + >>> p = UPath("s3://my-bucket/path/to/file.txt", anon=True) + >>> p.storage_options['anon'] + True + + """ return MappingProxyType(self._storage_options) @property def fs(self) -> AbstractFileSystem: - """The cached fsspec filesystem instance for the path.""" + """The cached fsspec filesystem instance for the path. + + This is the underlying fsspec filesystem instance. It's + instantiated on first filesystem access and cached. Can + be used to access fsspec-specific functionality not exposed + by the UPath API. + + Examples + -------- + >>> from upath import UPath + >>> p = UPath("s3://my-bucket/path/to/file.txt") + >>> p.fs + + >>> p.fs.get_tags(p.path) + {'VersionId': 'null', 'ContentLength': 12345, ...} + + """ try: return self._fs_cached except AttributeError: @@ -224,7 +287,25 @@ def fs(self) -> AbstractFileSystem: @property def path(self) -> str: - """The path that a fsspec filesystem can use.""" + """The path used by fsspec filesystem. + + FSSpec filesystems usually handle paths stripped of protocol. + This property returns the path suitable for use with the + underlying fsspec filesystem. It guarantees that a filesystem's + strip_protocol method is applied correctly. + + Examples + -------- + >>> from upath import UPath + >>> p = UPath("memory:///foo/bar.txt") + >>> str(p) + 'memory:///foo/bar.txt' + >>> p.path + '/foo/bar.txt' + >>> p.fs.exists(p.path) + True + + """ if self._relative_base is not None: try: # For relative paths, we need to resolve to absolute path @@ -243,7 +324,20 @@ def path(self) -> str: return self._chain.active_path def joinuri(self, uri: JoinablePathLike) -> UPath: - """Join with urljoin behavior for UPath instances""" + """Join with urljoin behavior for UPath instances. + + Examples + -------- + >>> from upath import UPath + >>> p = UPath("https://example.com/dir/subdir/") + >>> p.joinuri("file.txt") + HTTPSPath('https://example.com/dir/subdir/file.txt') + >>> p.joinuri("/anotherdir/otherfile.txt") + HTTPSPath('https://example.com/anotherdir/otherfile.txt') + >>> p.joinuri("memory:///foo/bar.txt" + MemoryPath('memory:///foo/bar.txt') + + """ # short circuit if the new uri uses a different protocol other_protocol = get_upath_protocol(uri) if other_protocol and other_protocol != self._protocol: @@ -383,6 +477,28 @@ def __init__( chain_parser: FSSpecChainParser = DEFAULT_CHAIN_PARSER, **storage_options: Any, ) -> None: + """Initialize a UPath instance + + When instantiating a `UPath`, the detected or provided protocol determines + the `UPath` subclass that will be instantiated. The protocol is looked up + via the `get_upath_protocol` function, which loads the registered `UPath` + implementation from the registry. If no `UPath` implementation is found for + the detected protocol, but a registered `fsspec` filesystem exists for the + protocol, a default dynamically created `UPath` implementation will be used. + + Parameters + ---------- + *args : + The path (or uri) segments to construct the UPath from. The first + argument is used to detect the protocol if no protocol is provided. + protocol : + The protocol to use for the path. + chain_parser : + A chain parser instance for chained urlpaths. _(experimental)_ + **storage_options : + Additional storage options for the path. + + """ # todo: avoid duplicating this call from __new__ protocol = get_upath_protocol( @@ -459,6 +575,187 @@ def _url(self) -> SplitResult: class UPath(_UPathMixin, WritablePath, ReadablePath): + """Base class for pathlike paths backed by an fsspec filesystem. + + Note + ---- + The following attributes and methods are specific to UPath instances and are not + available on pathlib.Path instances. + + Attributes + ---------- + protocol : + The fsspec protocol for the path. + storage_options : + The fsspec storage options for the path. + path : + The path that a fsspec filesystem can use. + fs : + The cached fsspec filesystem instance for the path. + + Methods + ------- + joinuri(*parts) : + Join URI parts to this path. + + + Info + ---- + Below are pathlib attributes and methods available on UPath instances. + + Attributes + ---------- + drive : + The drive component of the path. + root : + The root component of the path. + anchor : + The concatenation of the drive and root. + parent : + The logical parent of the path. + parents : + An immutable sequence providing access to the logical ancestors of the path. + name : + The final path component, excluding the drive and root, if any. + suffix : + The file extension of the final component, if any. + suffixes : + A list of the path's file extensions. + stem : + The final path component, without its suffix. + info : + Filesystem information about the path. + parser : + The path parser instance for parsing path segments. + + Methods + ------- + __truediv__(key) : + Combine this path with the argument using the `/` operator. + __rtruediv__(key) : + Combine the argument with this path using the `/` operator. + as_posix() : + Return the string representation of the path with forward slashes. + is_absolute() : + Return True if the path is absolute. + is_relative_to(other) : + Return True if the path is relative to another path. + is_reserved() : + Return True if the path is reserved under Windows. + joinpath(*pathsegments) : + Combine this path with one or several arguments, and return a new path. + full_match(pattern, *, case_sensitive=None) : + Match this path against the provided glob-style pattern. + match(pattern, *, case_sensitive=None) : + Match this path against the provided glob-style pattern. + relative_to(other, walk_up=False) : + Return a version of this path relative to another path. + with_name(name) : + Return a new path with the name changed. + with_stem(stem) : + Return a new path with the stem changed. + with_suffix(suffix) : + Return a new path with the suffix changed. + with_segments(*pathsegments) : + Construct a new path object from any number of path-like objects. + from_uri(uri) : + Return a new path from the given URI. + as_uri() : + Return the path as a URI. + home() : + Return a new path pointing to the user's home directory. + expanduser() : + Return a new path with expanded `~` constructs. + cwd() : + Return a new path pointing to the current working directory. + absolute() : + Make the path absolute, without normalization or resolving symlinks. + resolve(strict=False) : + Make the path absolute, resolving any symlinks. + readlink() : + Return the path to which the symbolic link points. + stat(*, follow_symlinks=True) : + Return the result of the stat() system call on this path. + lstat() : + Like stat(), but if the path points to a symlink, return the symlink's + information. + exists(*, follow_symlinks=True) : + Return True if the path exists. + is_file(*, follow_symlinks=True) : + Return True if the path is a regular file. + is_dir(*, follow_symlinks=True) : + Return True if the path is a directory. + is_symlink() : + Return True if the path is a symbolic link. + is_junction() : + Return True if the path is a junction. + is_mount() : + Return True if the path is a mount point. + is_socket() : + Return True if the path is a socket. + is_fifo() : + Return True if the path is a FIFO. + is_block_device() : + Return True if the path is a block device. + is_char_device() : + Return True if the path is a character device. + samefile(other_path) : + Return True if this path points to the same file as other_path. + open(mode='r', buffering=-1, encoding=None, errors=None, newline=None) : + Open the file pointed to by the path. + read_text(encoding=None, errors=None, newline=None) : + Open the file in text mode, read it, and close the file. + read_bytes() : + Open the file in bytes mode, read it, and close the file. + write_text(data, encoding=None, errors=None, newline=None) : + Open the file in text mode, write to it, and close the file. + write_bytes(data) : + Open the file in bytes mode, write to it, and close the file. + iterdir() : + Yield path objects of the directory contents. + glob(pattern, *, case_sensitive=None) : + Iterate over this subtree and yield all existing files matching the + given pattern. + rglob(pattern, *, case_sensitive=None) : + Recursively yield all existing files matching the given pattern. + walk(top_down=True, on_error=None, follow_symlinks=False) : + Generate the file names in a directory tree by walking the tree. + touch(mode=0o666, exist_ok=True) : + Create this file with the given access mode, if it doesn't exist. + mkdir(mode=0o777, parents=False, exist_ok=False) : + Create a new directory at this given path. + symlink_to(target, target_is_directory=False) : + Make this path a symbolic link pointing to target. + hardlink_to(target) : + Make this path a hard link pointing to the same file as target. + copy(target, *, follow_symlinks=True, preserve_metadata=False) : + Copy the contents of this file to the target file. + copy_into(target_dir, *, follow_symlinks=True, preserve_metadata=False) : + Copy this file or directory into the target directory. + rename(target) : + Rename this path to the target path. + replace(target) : + Rename this path to the target path, overwriting if that path exists. + move(target) : + Move this file or directory tree to the target path. + move_into(target_dir) : + Move this file or directory into the target directory. + unlink(missing_ok=False) : + Remove this file or link. + rmdir() : + Remove this directory. + owner(*, follow_symlinks=True) : + Return the login name of the file owner. + group(*, follow_symlinks=True) : + Return the group name of the file gid. + chmod(mode, *, follow_symlinks=True) : + Change the permissions of the path. + lchmod(mode) : + Like chmod() but, if the path points to a symlink, modify the symlink's + permissions. + + """ + __slots__ = ( "_chain", "_chain_parser", @@ -625,6 +922,7 @@ def __new__( parser: UPathParser = LazyFlavourDescriptor() # type: ignore[assignment] def with_segments(self, *pathsegments: JoinablePathLike) -> Self: + """Construct a new path object from any number of path-like objects.""" # we change joinpath behavior if called from a relative path # this is not fully ideal, but currently the best way to move forward if is_relative := self._relative_base is not None: @@ -676,6 +974,19 @@ def __repr__(self) -> str: @property def parts(self) -> Sequence[str]: + """Provides sequence-like access to the filesystem path components. + + Examples + -------- + >>> from upath import UPath + >>> p = UPath("s3://my-bucket/path/to/file.txt") + >>> p.parts + ('my-bucket/', 'path', 'to', 'file.txt') + >>> p2 = UPath("/foo/bar/baz.txt", protocol="memory") + >>> p2.parts + ('/', 'foo', 'bar', 'baz.txt') + + """ # For relative paths, return parts of the relative path only if self._relative_base is not None: rel_str = str(self) @@ -719,12 +1030,23 @@ def with_name(self, name: str) -> Self: @property def anchor(self) -> str: + """The concatenation of the drive and root or an empty string.""" if self._relative_base is not None: return "" return self.drive + self.root @property def parent(self) -> Self: + """The logical parent of the path. + + Examples + -------- + >>> from upath import UPath + >>> p = UPath("s3://my-bucket/path/to/file.txt") + >>> p.parent + S3Path('s3://my-bucket/path/to') + + """ if self._relative_base is not None: if str(self) == ".": return self @@ -743,6 +1065,20 @@ def parent(self) -> Self: @property def parents(self) -> Sequence[Self]: + """A sequence providing access to the logical ancestors of the path. + + Examples + -------- + >>> from upath import UPath + >>> p = UPath("memory:///foo/bar/baz.txt") + >>> list(p.parents) + [ + MemoryPath('memory:///foo/bar'), + MemoryPath('memory:///foo'), + MemoryPath('memory:///'), + ] + + """ if self._relative_base is not None: parents = [] parent = self @@ -755,6 +1091,18 @@ def parents(self) -> Sequence[Self]: return super().parents def joinpath(self, *pathsegments: JoinablePathLike) -> Self: + """Combine this path with one or several arguments, and return a new path. + + For one argument, this is equivalent to using the `/` operator. + + Examples + -------- + >>> from upath import UPath + >>> p = UPath("s3://my-bucket/path/to") + >>> p.joinpath("file.txt") + S3Path('s3://my-bucket/path/to/file.txt') + + """ return self.with_segments(self.__vfspath__(), *pathsegments) def __truediv__(self, key: JoinablePathLike) -> Self: @@ -773,9 +1121,32 @@ def __rtruediv__(self, key: JoinablePathLike) -> Self: @property def info(self) -> PathInfo: + """ + A PathInfo object that exposes the file type and other file attributes + of this path. + + Returns + ------- + : UPathInfo + The UPathInfo object for this path. + """ return UPathInfo(self) def iterdir(self) -> Iterator[Self]: + """Yield path objects of the directory contents. + + Examples + -------- + >>> from upath import UPath + >>> p = UPath("memory:///foo/") + >>> p.joinpath("bar.txt").touch() + >>> p.joinpath("baz.txt").touch() + >>> for child in p.iterdir(): + ... print(child) + MemoryPath('memory:///foo/bar.txt') + MemoryPath('memory:///foo/baz.txt') + + """ sep = self.parser.sep base = self if self.parts[-1:] == ("",): @@ -809,6 +1180,9 @@ def copy(self, target: _WT, **kwargs: Any) -> _WT: ... def copy(self, target: SupportsPathLike | str, **kwargs: Any) -> Self: ... def copy(self, target: _WT | SupportsPathLike | str, **kwargs: Any) -> _WT | UPath: + """ + Recursively copy this file or directory tree to the given destination. + """ if not isinstance(target, UPath): return super().copy(self.with_segments(target), **kwargs) else: @@ -823,6 +1197,9 @@ def copy_into(self, target_dir: SupportsPathLike | str, **kwargs: Any) -> Self: def copy_into( self, target_dir: _WT | SupportsPathLike | str, **kwargs: Any ) -> _WT | UPath: + """ + Copy this file or directory tree into the given existing directory. + """ if not isinstance(target_dir, UPath): return super().copy_into(self.with_segments(target_dir), **kwargs) else: @@ -835,6 +1212,9 @@ def move(self, target: _WT, **kwargs: Any) -> _WT: ... def move(self, target: SupportsPathLike | str, **kwargs: Any) -> Self: ... def move(self, target: _WT | SupportsPathLike | str, **kwargs: Any) -> _WT | UPath: + """ + Recursively move this file or directory tree to the given destination. + """ target = self.copy(target, **kwargs) self.fs.rm(self.path, recursive=self.is_dir()) return target @@ -848,6 +1228,9 @@ def move_into(self, target_dir: SupportsPathLike | str, **kwargs: Any) -> Self: def move_into( self, target_dir: _WT | SupportsPathLike | str, **kwargs: Any ) -> _WT | UPath: + """ + Move this file or directory tree into the given existing directory. + """ name = self.name if not name: raise ValueError(f"{self!r} has an empty name") @@ -872,6 +1255,9 @@ def mkdir( parents: bool = False, exist_ok: bool = False, ) -> None: + """ + Create a new directory at this given path. + """ if parents and not exist_ok and self.exists(): raise FileExistsError(str(self)) try: @@ -974,6 +1360,20 @@ def stat( *, follow_symlinks: bool = True, ) -> UPathStatResult: + """ + Return the result of the stat() system call on this path, like + os.stat() does. + + Info + ---- + For fsspec filesystems follow_symlinks is currently ignored. + + Returns + ------- + : UPathStatResult + The upath stat result for this path, emulating `os.stat_result`. + + """ if not follow_symlinks: warnings.warn( f"{type(self).__name__}.stat(follow_symlinks=False):" @@ -990,18 +1390,41 @@ def chmod(self, mode: int, *, follow_symlinks: bool = True) -> None: _raise_unsupported(type(self).__name__, "chmod") def exists(self, *, follow_symlinks: bool = True) -> bool: + """ + Whether this path exists. + + Info + ---- + For fsspec filesystems follow_symlinks is currently ignored. + """ return self.fs.exists(self.path) def is_dir(self) -> bool: + """ + Whether this path is a directory. + """ return self.fs.isdir(self.path) def is_file(self) -> bool: + """ + Whether this path is a regular file. + """ return self.fs.isfile(self.path) def is_mount(self) -> bool: + """ + Check if this path is a mount point + + Info + ---- + For fsspec filesystems this is always False. + """ return False def is_symlink(self) -> bool: + """ + Whether this path is a symbolic link. + """ try: info = self.fs.info(self.path) if "islink" in info: @@ -1011,24 +1434,72 @@ def is_symlink(self) -> bool: return False def is_junction(self) -> bool: + """ + Whether this path is a junction. + + Info + ---- + For fsspec filesystems this is always False. + """ return False def is_block_device(self) -> bool: + """ + Whether this path is a block device. + + Info + ---- + For fsspec filesystems this is always False. + """ return False def is_char_device(self) -> bool: + """ + Whether this path is a character device. + + Info + ---- + For fsspec filesystems this is always False. + """ return False def is_fifo(self) -> bool: + """ + Whether this path is a FIFO (named pipe). + + Info + ---- + For fsspec filesystems this is always False. + """ return False def is_socket(self) -> bool: + """ + Whether this path is a socket. + + Info + ---- + For fsspec filesystems this is always False. + """ return False def is_reserved(self) -> bool: + """ + Whether this path is reserved under Windows. + + Info + ---- + For fsspec filesystems this is always False. + """ return False def expanduser(self) -> Self: + """Return a new path with expanded `~` constructs. + + Info + ---- + For fsspec filesystems this is currently a no-op. + """ return self def glob( @@ -1038,6 +1509,8 @@ def glob( case_sensitive: bool = UNSET_DEFAULT, recurse_symlinks: bool = UNSET_DEFAULT, ) -> Iterator[Self]: + """Iterate over this subtree and yield all existing files (of any + kind, including directories) matching the given relative pattern.""" if case_sensitive is not UNSET_DEFAULT: warnings.warn( "UPath.glob(): case_sensitive is currently ignored.", @@ -1066,6 +1539,10 @@ def rglob( case_sensitive: bool = UNSET_DEFAULT, recurse_symlinks: bool = UNSET_DEFAULT, ) -> Iterator[Self]: + """Recursively yield all existing files (of any kind, including + directories) matching the given relative pattern, anywhere in + this subtree. + """ if case_sensitive is not UNSET_DEFAULT: warnings.warn( "UPath.glob(): case_sensitive is currently ignored.", @@ -1111,11 +1588,18 @@ def group(self) -> str: _raise_unsupported(type(self).__name__, "group") def absolute(self) -> Self: + """Return an absolute version of this path + No normalization or symlink resolution is performed. + + Use resolve() to resolve symlinks and remove '..' segments. + """ if self._relative_base is not None: return self.cwd().joinpath(self.__vfspath__()) return self def is_absolute(self) -> bool: + """True if the path is absolute (has both a root and, if applicable, + a drive).""" if self._relative_base is not None: return False else: @@ -1177,6 +1661,10 @@ def __ge__(self, other: object) -> bool: return self.__vfspath__() >= other.__vfspath__() def resolve(self, strict: bool = False) -> Self: + """ + Make the path absolute, resolving all symlinks on the way and also + normalizing it. + """ if self._relative_base is not None: self = self.absolute() _parts = self.parts @@ -1197,6 +1685,7 @@ def resolve(self, strict: bool = False) -> Self: return self.with_segments(*_parts[:1], *resolved) def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None: + """Create this file with the given access mode, if it doesn't exist.""" exists = self.fs.exists(self.path) if exists and not exist_ok: raise FileExistsError(str(self)) @@ -1212,6 +1701,10 @@ def lchmod(self, mode: int) -> None: _raise_unsupported(type(self).__name__, "lchmod") def unlink(self, missing_ok: bool = False) -> None: + """ + Remove this file or link. + If the path is a directory, use rmdir() instead. + """ if not self.exists(): if not missing_ok: raise FileNotFoundError(str(self)) @@ -1219,6 +1712,19 @@ def unlink(self, missing_ok: bool = False) -> None: self.fs.rm(self.path, recursive=False) def rmdir(self, recursive: bool = True) -> None: # fixme: non-standard + """ + Remove this directory. + + Warning + ------- + This method is non-standard compared to pathlib.Path.rmdir(), + as it supports a `recursive` parameter to remove non-empty + directories and defaults to recursive deletion. + + This behavior is likely to change in future releases once + `.delete()` is introduced. + + """ if not self.is_dir(): raise NotADirectoryError(str(self)) if not recursive and next(self.iterdir()): # type: ignore[arg-type] @@ -1233,6 +1739,25 @@ def rename( maxdepth: int | None = UNSET_DEFAULT, **kwargs: Any, ) -> Self: + """ + Rename this file or directory to the given target. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + + Warning + ------- + This method is non-standard compared to pathlib.Path.rename(), + as it supports `recursive` and `maxdepth` parameters for + directory moves. This will be revisited in future releases. + + It's better to use `.move()` or `.move_into()` to avoid + running into future compatibility issues. + + """ target_protocol = get_upath_protocol(target) if target_protocol and target_protocol != self.protocol: raise ValueError( @@ -1272,16 +1797,38 @@ def rename( return self.with_segments(target_) def replace(self, target: WritablePathLike) -> Self: + """ + Rename this path to the target path, overwriting if that path exists. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + + Warning + ------- + This method is currently not implemented. + + """ _raise_unsupported(type(self).__name__, "replace") @property def drive(self) -> str: + """The drive prefix (letter or UNC path), if any. + + Info + ---- + On non-Windows systems, the drive is always an empty string. + On cloud storage systems, the drive is the bucket name or equivalent. + """ if self._relative_base is not None: return "" return self.parser.splitroot(str(self))[0] @property def root(self) -> str: + """The root of the path, if any.""" if self._relative_base is not None: return "" return self.parser.splitroot(str(self))[1] @@ -1308,6 +1855,7 @@ def from_uri(cls, uri: str, **storage_options: Any) -> Self: return cls(uri, **storage_options) def as_uri(self) -> str: + """Return the string representation of the path as a URI.""" if self._relative_base is not None: raise ValueError( f"relative path can't be expressed as a {self.protocol} URI" @@ -1315,6 +1863,7 @@ def as_uri(self) -> str: return str(self) def as_posix(self) -> str: + """Return the string representation of the path with POSIX-style separators.""" return str(self) def samefile(self, other_path) -> bool: @@ -1327,6 +1876,16 @@ def samefile(self, other_path) -> bool: @classmethod def cwd(cls) -> Self: + """ + Return a new UPath object representing the current working directory. + + Info + ---- + None of the fsspec filesystems support a global current working + directory, so this method only works for the base UPath class, + returning the local current working directory. + + """ if cls is UPath: # default behavior for UPath.cwd() is to return local cwd return get_upath_class("").cwd() # type: ignore[union-attr,return-value] @@ -1335,6 +1894,16 @@ def cwd(cls) -> Self: @classmethod def home(cls) -> Self: + """ + Return a new UPath object representing the user's home directory. + + Info + ---- + None of the fsspec filesystems support user home directories, + so this method only works for the base UPath class, returning the + local user's home directory. + + """ if cls is UPath: return get_upath_class("").home() # type: ignore[union-attr,return-value] else: @@ -1344,9 +1913,16 @@ def relative_to( # type: ignore[override] self, other: Self | str, /, - *_deprecated, + *_deprecated: Any, walk_up: bool = False, ) -> Self: + """Return the relative path to another path identified by the passed + arguments. If the operation is not possible (because this is not + related to the other path), raise ValueError. + + The *walk_up* parameter controls whether `..` may be used to resolve + the path. + """ if walk_up: raise NotImplementedError("walk_up=True is not implemented yet") @@ -1374,9 +1950,17 @@ def relative_to( # type: ignore[override] rel._relative_base = other.path return rel - def is_relative_to(self, other, /, *_deprecated) -> bool: # type: ignore[override] + def is_relative_to( + self, + other: Self | str, + /, + *_deprecated: Any, + ) -> bool: # type: ignore[override] + """Return True if the path is relative to another path identified.""" if isinstance(other, UPath) and self.storage_options != other.storage_options: return False + elif isinstance(other, str): + other = self.with_segments(other) return self == other or other in self.parents def hardlink_to(self, target: ReadablePathLike) -> None: diff --git a/upath/implementations/smb.py b/upath/implementations/smb.py index 7adaade1..53ea99a3 100644 --- a/upath/implementations/smb.py +++ b/upath/implementations/smb.py @@ -6,8 +6,6 @@ from typing import TYPE_CHECKING from typing import Any -from smbprotocol.exceptions import SMBOSError - from upath.core import UPath from upath.types import UNSET_DEFAULT from upath.types import JoinablePathLike @@ -60,6 +58,8 @@ def mkdir( exist_ok: bool = False, ) -> None: # smbclient does not support setting mode externally + from smbprotocol.exceptions import SMBOSError + if parents and not exist_ok and self.exists(): raise FileExistsError(str(self)) try: