From c22ae2e88524881c3ae15e440903842114f9b94c Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Wed, 3 Sep 2025 22:08:43 +0200 Subject: [PATCH] upath: update the flavours to newest versions --- dev/requirements.txt | 12 ++-- noxfile.py | 5 +- upath/_flavour_sources.py | 115 ++++++++++++++++++++++++++++---------- 3 files changed, 95 insertions(+), 37 deletions(-) diff --git a/dev/requirements.txt b/dev/requirements.txt index d9f3683d..5c0e910d 100644 --- a/dev/requirements.txt +++ b/dev/requirements.txt @@ -1,18 +1,18 @@ -fsspec[git,hdfs,dask,http,sftp,smb]==2025.3.2 +fsspec[git,hdfs,dask,http,sftp,smb]==2025.7.0 # these dependencies define their own filesystems adlfs==2024.12.0 boxfs==0.3.0 dropboxdrivefs==1.4.1 -gcsfs==2025.3.2 -s3fs==2025.3.2 +gcsfs==2025.7.0 +s3fs==2025.7.0 ocifs==1.3.2 webdav4[fsspec]==0.10.0 # gfrivefs @ git+https://github.com/fsspec/gdrivefs@master broken ... morefs[asynclocalfs]==0.2.2 -dvc==3.59.1 -huggingface_hub==0.30.2 +dvc==3.62.0 +huggingface_hub==0.34.4 lakefs-spec==0.11.3 -ossfs==2023.12.0 +ossfs==2025.5.0 fsspec-xrootd==0.5.1 wandbfs==0.0.2 diff --git a/noxfile.py b/noxfile.py index 92199985..d5aeb9b9 100644 --- a/noxfile.py +++ b/noxfile.py @@ -123,5 +123,8 @@ def generate_flavours(session): session.install("-r", "dev/requirements.txt") with open("upath/_flavour_sources.py", "w") as target: session.run( - "python", "dev/fsspec_inspector/generate_flavours.py", stdout=target + "python", + "dev/fsspec_inspector/generate_flavours.py", + stdout=target, + stderr=None, ) diff --git a/upath/_flavour_sources.py b/upath/_flavour_sources.py index b77c921d..cb59b87f 100644 --- a/upath/_flavour_sources.py +++ b/upath/_flavour_sources.py @@ -25,9 +25,10 @@ # - filecache # - simplecache # protocol import errors: -# - async_wrapper (None) -# - gdrive (Please install gdrivefs for access to Google Drive) +# - gdrive (Please install gdrive_fs for access to Google Drive) # - generic (GenericFileSystem: '_strip_protocol' not a classmethod) +# - pyscript (Install requests (cpython) or run in pyscript) +# - tos (Install tosfs to access ByteDance volcano engine Tinder Object Storage) # - tosfs (Install tosfs to access ByteDance volcano engine Tinder Object Storage) # from __future__ import annotations @@ -89,7 +90,7 @@ def __init_subclass__(cls: Any, **kwargs): class AbstractFileSystemFlavour(FileSystemFlavourBase): __orig_class__ = 'fsspec.spec.AbstractFileSystem' - __orig_version__ = '2025.3.2' + __orig_version__ = '2025.7.0' protocol: str | tuple[str, ...] = 'abstract' root_marker: Literal['', '/'] = '' sep: Literal['/'] = '/' @@ -136,6 +137,14 @@ def _parent(cls, path): return cls.root_marker +class AsyncFileSystemWrapperFlavour(AbstractFileSystemFlavour): + __orig_class__ = 'fsspec.implementations.asyn_wrapper.AsyncFileSystemWrapper' + __orig_version__ = '2025.7.0' + protocol = ('asyncwrapper', 'async_wrapper') + root_marker = '' + sep = '/' + + class AsyncLocalFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'morefs.asyn_local.AsyncLocalFileSystem' __orig_version__ = '0.2.2' @@ -314,7 +323,7 @@ def _strip_protocol(cls, path) -> str: class DaskWorkerFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'fsspec.implementations.dask.DaskWorkerFileSystem' - __orig_version__ = '2025.3.2' + __orig_version__ = '2025.7.0' protocol = ('dask',) root_marker = '' sep = '/' @@ -330,7 +339,7 @@ def _get_kwargs_from_urls(path): class DataFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'fsspec.implementations.data.DataFileSystem' - __orig_version__ = '2025.3.2' + __orig_version__ = '2025.7.0' protocol = ('data',) root_marker = '' sep = '/' @@ -338,7 +347,7 @@ class DataFileSystemFlavour(AbstractFileSystemFlavour): class DatabricksFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'fsspec.implementations.dbfs.DatabricksFileSystem' - __orig_version__ = '2025.3.2' + __orig_version__ = '2025.7.0' protocol = ('dbfs',) root_marker = '' sep = '/' @@ -371,7 +380,7 @@ class DropboxDriveFileSystemFlavour(AbstractFileSystemFlavour): class FTPFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'fsspec.implementations.ftp.FTPFileSystem' - __orig_version__ = '2025.3.2' + __orig_version__ = '2025.7.0' protocol = ('ftp',) root_marker = '/' sep = '/' @@ -390,7 +399,7 @@ def _get_kwargs_from_urls(urlpath): class GCSFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'gcsfs.core.GCSFileSystem' - __orig_version__ = '2025.3.2' + __orig_version__ = '2025.7.0' protocol = ('gs', 'gcs') root_marker = '' sep = '/' @@ -465,9 +474,57 @@ def _split_path(cls, path, version_aware=False): ) +class GistFileSystemFlavour(AbstractFileSystemFlavour): + __orig_class__ = 'fsspec.implementations.gist.GistFileSystem' + __orig_version__ = '2025.7.0' + protocol = ('gist',) + root_marker = '' + sep = '/' + + @classmethod + def _strip_protocol(cls, path): + """ + Remove 'gist://' from the path, if present. + """ + # The default infer_storage_options can handle gist://username:token@id/file + # or gist://id/file, but let's ensure we handle a normal usage too. + # We'll just strip the protocol prefix if it exists. + path = infer_storage_options(path).get("path", path) + return path.lstrip("/") + + @staticmethod + def _get_kwargs_from_urls(path): + """ + Parse 'gist://' style URLs into GistFileSystem constructor kwargs. + For example: + gist://:TOKEN@/file.txt + gist://username:TOKEN@/file.txt + """ + so = infer_storage_options(path) + out = {} + if "username" in so and so["username"]: + out["username"] = so["username"] + if "password" in so and so["password"]: + out["token"] = so["password"] + if "host" in so and so["host"]: + # We interpret 'host' as the gist ID + out["gist_id"] = so["host"] + + # Extract SHA and filename from path + if "path" in so and so["path"]: + path_parts = so["path"].rsplit("/", 2)[-2:] + if len(path_parts) == 2: + if path_parts[0]: # SHA present + out["sha"] = path_parts[0] + if path_parts[1]: # filename also present + out["filenames"] = [path_parts[1]] + + return out + + class GitFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'fsspec.implementations.git.GitFileSystem' - __orig_version__ = '2025.3.2' + __orig_version__ = '2025.7.0' protocol = ('git',) root_marker = '' sep = '/' @@ -483,8 +540,7 @@ def _strip_protocol(cls, path): @staticmethod def _get_kwargs_from_urls(path): - if path.startswith("git://"): - path = path[6:] + path = path.removeprefix("git://") out = {} if ":" in path: out["path"], path = path.split(":", 1) @@ -495,7 +551,7 @@ def _get_kwargs_from_urls(path): class GithubFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'fsspec.implementations.github.GithubFileSystem' - __orig_version__ = '2025.3.2' + __orig_version__ = '2025.7.0' protocol = ('github',) root_marker = '' sep = '/' @@ -520,7 +576,7 @@ def _get_kwargs_from_urls(path): class HTTPFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'fsspec.implementations.http.HTTPFileSystem' - __orig_version__ = '2025.3.2' + __orig_version__ = '2025.7.0' protocol = ('http', 'https') root_marker = '' sep = '/' @@ -541,7 +597,7 @@ def _parent(cls, path): class HadoopFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'fsspec.implementations.arrow.HadoopFileSystem' - __orig_version__ = '2025.3.2' + __orig_version__ = '2025.7.0' protocol = ('hdfs', 'arrow_hdfs') root_marker = '/' sep = '/' @@ -574,7 +630,7 @@ def _get_kwargs_from_urls(path): class HfFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'huggingface_hub.hf_file_system.HfFileSystem' - __orig_version__ = '0.30.2' + __orig_version__ = '0.34.4' protocol = ('hf',) root_marker = '' sep = '/' @@ -582,7 +638,7 @@ class HfFileSystemFlavour(AbstractFileSystemFlavour): class JupyterFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'fsspec.implementations.jupyter.JupyterFileSystem' - __orig_version__ = '2025.3.2' + __orig_version__ = '2025.7.0' protocol = ('jupyter', 'jlab') root_marker = '' sep = '/' @@ -608,7 +664,7 @@ def _strip_protocol(cls, path): class LibArchiveFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'fsspec.implementations.libarchive.LibArchiveFileSystem' - __orig_version__ = '2025.3.2' + __orig_version__ = '2025.7.0' protocol = ('libarchive',) root_marker = '' sep = '/' @@ -621,7 +677,7 @@ def _strip_protocol(cls, path): class LocalFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'fsspec.implementations.local.LocalFileSystem' - __orig_version__ = '2025.3.2' + __orig_version__ = '2025.7.0' protocol = ('file', 'local') root_marker = '/' sep = '/' @@ -699,7 +755,7 @@ def _strip_protocol(cls, path): class MemoryFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'fsspec.implementations.memory.MemoryFileSystem' - __orig_version__ = '2025.3.2' + __orig_version__ = '2025.7.0' protocol = ('memory',) root_marker = '/' sep = '/' @@ -712,8 +768,7 @@ def _strip_protocol(cls, path): else: path = stringify_path(path) - if path.startswith("memory://"): - path = path[len("memory://") :] + path = path.removeprefix("memory://") if "::" in path or "://" in path: return path.rstrip("/") path = path.lstrip("/").rstrip("/") @@ -750,7 +805,7 @@ def _parent(cls, path): class OSSFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'ossfs.core.OSSFileSystem' - __orig_version__ = '2023.12.0' + __orig_version__ = '2025.5.0' protocol = ('oss',) root_marker = '' sep = '/' @@ -798,7 +853,7 @@ class OverlayFileSystemFlavour(AbstractFileSystemFlavour): class ReferenceFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'fsspec.implementations.reference.ReferenceFileSystem' - __orig_version__ = '2025.3.2' + __orig_version__ = '2025.7.0' protocol = ('reference',) root_marker = '' sep = '/' @@ -806,7 +861,7 @@ class ReferenceFileSystemFlavour(AbstractFileSystemFlavour): class S3FileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 's3fs.core.S3FileSystem' - __orig_version__ = '2025.3.2' + __orig_version__ = '2025.7.0' protocol = ('s3', 's3a') root_marker = '' sep = '/' @@ -834,7 +889,7 @@ def _get_kwargs_from_urls(urlpath): class SFTPFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'fsspec.implementations.sftp.SFTPFileSystem' - __orig_version__ = '2025.3.2' + __orig_version__ = '2025.7.0' protocol = ('sftp', 'ssh') root_marker = '' sep = '/' @@ -853,7 +908,7 @@ def _get_kwargs_from_urls(urlpath): class SMBFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'fsspec.implementations.smb.SMBFileSystem' - __orig_version__ = '2025.3.2' + __orig_version__ = '2025.7.0' protocol = ('smb',) root_marker = '' sep = '/' @@ -873,7 +928,7 @@ def _get_kwargs_from_urls(path): class TarFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'fsspec.implementations.tar.TarFileSystem' - __orig_version__ = '2025.3.2' + __orig_version__ = '2025.7.0' protocol = ('tar',) root_marker = '' sep = '/' @@ -889,7 +944,7 @@ class WandbFSFlavour(AbstractFileSystemFlavour): class WebHDFSFlavour(AbstractFileSystemFlavour): __orig_class__ = 'fsspec.implementations.webhdfs.WebHDFS' - __orig_version__ = '2025.3.2' + __orig_version__ = '2025.7.0' protocol = ('webhdfs', 'webHDFS') root_marker = '' sep = '/' @@ -950,7 +1005,7 @@ def _get_kwargs_from_urls(u: str) -> dict[Any, Any]: class ZipFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'fsspec.implementations.zip.ZipFileSystem' - __orig_version__ = '2025.3.2' + __orig_version__ = '2025.7.0' protocol = ('zip',) root_marker = '' sep = '/' @@ -963,7 +1018,7 @@ def _strip_protocol(cls, path): class _DVCFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'dvc.fs.dvc._DVCFileSystem' - __orig_version__ = '3.59.1' + __orig_version__ = '3.62.0' protocol = ('dvc',) root_marker = '/' sep = '/'