Add upath.implementations.cloud.HfPath

README.md

@@ -91,6 +91,7 @@ For more examples, see the [example notebook here][example-notebook].
 - `az:`, `adl:`, `abfs:` and `abfss:` Azure Storage _(requires `adlfs`)_
 - `data:` RFC 2397 style data URLs _(requires `fsspec>=2023.12.2`)_
 - `github:` GitHub repository filesystem
+- `hf:` Hugging Face filesystem _(requires `huggingface_hub`)_
 - `http:` and `https:` HTTP(S)-based filesystem
 - `hdfs:` Hadoop distributed filesystem
 - `gs:` and `gcs:` Google Cloud Storage _(requires `gcsfs`)_

pyproject.toml

@@ -55,6 +55,7 @@ dev = [
     "s3fs >=2024.5.0",
     "gcsfs >=2024.5.0",
     "adlfs >=2024",
+    "huggingface_hub",
     "webdav4[fsspec]",
     # testing
     "moto[s3,server]",

typesafety/test_upath_signatures.yml

@@ -68,6 +68,8 @@
       cls: GCSPath
     - module: upath.implementations.cloud
       cls: AzurePath
+    - module: upath.implementations.cloud
+      cls: HfPath
     - module: upath.implementations.data
       cls: DataPath
     - module: upath.implementations.github
@@ -253,6 +255,8 @@
       cls: GCSPath
     - module: upath.implementations.cloud
       cls: AzurePath
+    - module: upath.implementations.cloud
+      cls: HfPath
     - module: upath.implementations.data
       cls: DataPath
     - module: upath.implementations.github
@@ -568,6 +572,8 @@
       cls: GCSPath
     - module: upath.implementations.cloud
       cls: AzurePath
+    - module: upath.implementations.cloud
+      cls: HfPath
     - module: upath.implementations.data
       cls: DataPath
     - module: upath.implementations.github
@@ -962,6 +968,8 @@
       cls: GCSPath
     - module: upath.implementations.cloud
       cls: AzurePath
+    - module: upath.implementations.cloud
+      cls: HfPath
     - module: upath.implementations.data
       cls: DataPath
     - module: upath.implementations.github

typesafety/test_upath_types.yml

@@ -31,6 +31,8 @@
       protocol: gcs
     - cls_fqn: upath.implementations.cloud.AzurePath
       protocol: abfs
+    - cls_fqn: upath.implementations.cloud.HfPath
+      protocol: hf
     - cls_fqn: upath.implementations.data.DataPath
       protocol: data
     - cls_fqn: upath.implementations.github.GitHubPath
@@ -106,6 +108,8 @@
       protocol: gcs
     - cls_fqn: upath.implementations.cloud.AzurePath
       protocol: abfs
+    - cls_fqn: upath.implementations.cloud.HfPath
+      protocol: hf
     - cls_fqn: upath.implementations.data.DataPath
       protocol: data
     - cls_fqn: upath.implementations.github.GitHubPath
@@ -173,6 +177,10 @@
       cls: AzurePath
       supported_example_name: account_name
       supported_example_value: '"myaccount"'
+    - module: upath.implementations.cloud
+      cls: HfPath
+      supported_example_name: token
+      supported_example_value: '"abcd1234"'
     - module: upath.implementations.data
       cls: DataPath
       supported_example_name: use_listings_cache
@@ -242,6 +250,10 @@
       cls: AzurePath
       supported_example_name: account_name
       unsupported_example_value: '123'
+    - module: upath.implementations.cloud
+      cls: HfPath
+      supported_example_name: endpoint
+      unsupported_example_value: '123'
     - module: upath.implementations.data
       cls: DataPath
       supported_example_name: use_listings_cache
@@ -302,6 +314,8 @@
       cls: S3Path
     - module: upath.implementations.cloud
       cls: AzurePath
+    - module: upath.implementations.cloud
+      cls: HfPath
     - module: upath.implementations.data
       cls: DataPath
     - module: upath.implementations.github
@@ -347,6 +361,9 @@
     - module: upath.implementations.cloud
       cls: AzurePath
       td: AzureStorageOptions
+    - module: upath.implementations.cloud
+      cls: HfPath
+      td: HfStorageOptions
     - module: upath.implementations.data
       cls: DataPath
       td: DataStorageOptions

upath/core.py

@@ -507,6 +507,13 @@ def __new__(
             **_: Any,
         ) -> _uimpl.cloud.AzurePath: ...
         @overload  # noqa: E301
+        def __new__(
+            cls,
+            *args: JoinablePathLike,
+            protocol: Literal["hf"],
+            **_: Any,
+        ) -> _uimpl.cloud.HfPath: ...
+        @overload  # noqa: E301
         def __new__(
             cls,
             *args: JoinablePathLike,

upath/implementations/cloud.py

@@ -23,13 +23,15 @@
     from upath._chain import FSSpecChainParser
     from upath.types.storage_options import AzureStorageOptions
     from upath.types.storage_options import GCSStorageOptions
+    from upath.types.storage_options import HfStorageOptions
     from upath.types.storage_options import S3StorageOptions
 
 __all__ = [
     "CloudPath",
     "GCSPath",
     "S3Path",
     "AzurePath",
+    "HfPath",
 ]
 
 
@@ -157,3 +159,18 @@ def __init__(
         )
         if not self.drive and len(self.parts) > 1:
             raise ValueError("non key-like path provided (bucket/container missing)")
+
+
+class HfPath(CloudPath):
+    __slots__ = ()
+
+    def __init__(
+        self,
+        *args: JoinablePathLike,
+        protocol: Literal["hf"] | None = None,
+        chain_parser: FSSpecChainParser = DEFAULT_CHAIN_PARSER,
+        **storage_options: Unpack[HfStorageOptions],
+    ) -> None:
+        super().__init__(
+            *args, protocol=protocol, chain_parser=chain_parser, **storage_options
+        )

upath/registry.py

@@ -55,6 +55,7 @@
     from upath.implementations.cached import SimpleCachePath as _SimpleCachePath
     from upath.implementations.cloud import AzurePath as _AzurePath
     from upath.implementations.cloud import GCSPath as _GCSPath
+    from upath.implementations.cloud import HfPath as _HfPath
     from upath.implementations.cloud import S3Path as _S3Path
     from upath.implementations.data import DataPath as _DataPath
     from upath.implementations.github import GitHubPath as _GitHubPath
@@ -95,6 +96,7 @@ class _Registry(MutableMapping[str, "type[upath.UPath]"]):
         "gcs": "upath.implementations.cloud.GCSPath",
         "gs": "upath.implementations.cloud.GCSPath",
         "hdfs": "upath.implementations.hdfs.HDFSPath",
+        "hf": "upath.implementations.cloud.HfPath",
         "http": "upath.implementations.http.HTTPPath",
         "https": "upath.implementations.http.HTTPPath",
         "memory": "upath.implementations.memory.MemoryPath",
@@ -228,6 +230,8 @@ def get_upath_class(protocol: Literal["github"]) -> type[_GitHubPath]: ...
     @overload
     def get_upath_class(protocol: Literal["hdfs"]) -> type[_HDFSPath]: ...
     @overload
+    def get_upath_class(protocol: Literal["hf"]) -> type[_HfPath]: ...
+    @overload
     def get_upath_class(protocol: Literal["http", "https"]) -> type[_HTTPPath]: ...
     @overload
     def get_upath_class(protocol: Literal["file", "local"]) -> type[_FilePath]: ...

upath/tests/conftest.py

@@ -553,3 +553,99 @@ def ssh_fixture(ssh_container, local_testdir, monkeypatch):
         )
     finally:
         fs.delete("/app/testdir", recursive=True)
+
+
+@pytest.fixture
+def hf_test_repo():
+    # "__username__" is an invalid username so we can use it for tests
+    return "__username__/test_repo"
+
+
+@pytest.fixture
+def mock_hf_api(pathlib_base, monkeypatch, hf_test_repo):  # noqa: C901
+    huggingface_hub = pytest.importorskip(
+        "huggingface_hub", reason="hf tests require huggingface_hub"
+    )
+    hf_file_system = pytest.importorskip(
+        "huggingface_hub.hf_file_system", reason="hf tests require huggingface_hub"
+    )
+
+    class MockedHfApi(huggingface_hub.HfApi):
+
+        def repo_info(self, repo_id, *args, repo_type=None, **kwargs):
+            if repo_id != hf_test_repo:
+                raise huggingface_hub.errors.RepositoryNotFoundError(repo_id)
+            elif repo_type is None or repo_type == "model":
+                return huggingface_hub.hf_api.ModelInfo(id=repo_id)
+            elif repo_type == "dataset":
+                return huggingface_hub.hf_api.DatasetInfo(id=repo_id)
+            elif repo_type == "space":
+                return huggingface_hub.hf_api.SpaceInfo(id=repo_id)
+            else:
+                raise ValueError("Unsupported repo type.")
+
+        def get_paths_info(self, repo_id, paths, *args, **kwargs):
+            if repo_id != hf_test_repo:
+                raise huggingface_hub.errors.RepositoryNotFoundError(repo_id)
+            paths_info = []
+            for path in paths:
+                if path:
+                    path = pathlib_base / path
+                    if path.is_file():
+                        paths_info.append(
+                            huggingface_hub.hf_api.RepoFile(
+                                path=path.relative_to(pathlib_base).as_posix(),
+                                blob_id="blob_id",
+                                size=path.stat().st_size,
+                            )
+                        )
+                    elif path.is_dir():
+                        paths_info.append(
+                            huggingface_hub.hf_api.RepoFolder(
+                                path=path.relative_to(pathlib_base).as_posix(),
+                                tree_id="tree_id",
+                            )
+                        )
+            return paths_info
+
+        def list_repo_tree(
+            self, repo_id, path_in_repo, *args, recursive=False, **kwargs
+        ):
+            if repo_id != hf_test_repo:
+                raise huggingface_hub.errors.RepositoryNotFoundError(repo_id)
+            pathlib_dir = pathlib_base / path_in_repo if path_in_repo else pathlib_base
+            for path in pathlib_dir.rglob("*") if recursive else pathlib_dir.glob("*"):
+                if path.is_file():
+                    yield huggingface_hub.hf_api.RepoFile(
+                        path=path.relative_to(pathlib_base).as_posix(),
+                        oid="oid",
+                        size=path.stat().st_size,
+                    )
+                else:
+                    yield huggingface_hub.hf_api.RepoFolder(
+                        path=path.relative_to(pathlib_base).as_posix(),
+                        oid="oid",
+                    )
+
+    hf_file_system.HfFileSystem.clear_instance_cache()
+    monkeypatch.setattr(hf_file_system, "HfApi", MockedHfApi)
+
+
+@pytest.fixture
+def mock_hf_filesystem_open(pathlib_base, monkeypatch):
+    hf_file_system = pytest.importorskip(
+        "huggingface_hub.hf_file_system", reason="hf tests require huggingface_hub"
+    )
+
+    def mocked_open(fs, path, mode="rb", *args, **kwargs):
+        resolved_path = fs.resolve_path(path)
+        return (pathlib_base / resolved_path.path_in_repo).open(mode)
+
+    monkeypatch.setattr(hf_file_system.HfFileSystem, "_open", mocked_open)
+
+
+@pytest.fixture
+def hf_fixture_with_readonly_mocked_hf_api(
+    hf_test_repo, mock_hf_api, mock_hf_filesystem_open
+):
+    return "hf://" + hf_test_repo

upath/tests/implementations/test_hf.py

@@ -0,0 +1,97 @@
+import pytest
+from fsspec import get_filesystem_class
+
+from upath import UPath
+from upath.implementations.cloud import HfPath
+
+from ..cases import BaseTests
+
+try:
+    get_filesystem_class("hf")
+except ImportError:
+    pytestmark = pytest.mark.skip
+
+
+def test_hfpath():
+    path = UPath("hf://HuggingFaceTB/SmolLM2-135M")
+    assert isinstance(path, HfPath)
+    assert path.exists()
+
+
+class TestUPathHttp(BaseTests):
+    @pytest.fixture(autouse=True, scope="function")
+    def path(self, hf_fixture_with_readonly_mocked_hf_api):
+        self.path = UPath(hf_fixture_with_readonly_mocked_hf_api)
+
+    @pytest.mark.skip
+    def test_mkdir(self):
+        pass
+
+    @pytest.mark.skip
+    def test_mkdir_exists_ok_false(self):
+        pass
+
+    @pytest.mark.skip
+    def test_mkdir_exists_ok_true(self):
+        pass
+
+    @pytest.mark.skip
+    def test_mkdir_parents_true_exists_ok_true(self):
+        pass
+
+    @pytest.mark.skip
+    def test_mkdir_parents_true_exists_ok_false(self):
+        pass
+
+    @pytest.mark.skip
+    def test_makedirs_exist_ok_true(self):
+        pass
+
+    @pytest.mark.skip
+    def test_makedirs_exist_ok_false(self):
+        pass
+
+    @pytest.mark.skip
+    def test_touch(self):
+        pass
+
+    @pytest.mark.skip
+    def test_touch_unlink(self):
+        pass
+
+    @pytest.mark.skip
+    def test_write_bytes(self, pathlib_base):
+        pass
+
+    @pytest.mark.skip
+    def test_write_text(self, pathlib_base):
+        pass
+
+    def test_fsspec_compat(self):
+        pass
+
+    def test_rename(self):
+        pass
+
+    def test_rename2(self):
+        pass
+
+    def test_move_local(self, tmp_path):
+        pass
+
+    def test_move_into_local(self, tmp_path):
+        pass
+
+    def test_move_memory(self, clear_fsspec_memory_cache):
+        pass
+
+    def test_move_into_memory(self, clear_fsspec_memory_cache):
+        pass
+
+    @pytest.mark.skip(reason="HfPath does not support listing repositories")
+    def test_iterdir(self, local_testdir):
+        pass
+
+    @pytest.mark.skip(reason="HfPath does not support listing repositories")
+    def test_iterdir2(self, local_testdir):
+        pass

upath/tests/test_registry.py

@@ -23,6 +23,7 @@
     "gcs",
     "gs",
     "hdfs",
+    "hf",
     "http",
     "https",
     "local",