fix: avoid trailing filesystem directory probes

dlt/destinations/impl/filesystem/filesystem.py

@@ -591,7 +591,7 @@ def dataset_path(self) -> str:
         """A path within a bucket to tables in a dataset
         NOTE: dataset_name changes if with_staging_dataset is active
         """
-        return self.pathlib.join(self.bucket_path, self.dataset_name, "")  # type: ignore[no-any-return]
+        return self.pathlib.join(self.bucket_path, self.dataset_name)  # type: ignore[no-any-return]
 
     @contextmanager
     def with_staging_dataset(self) -> Iterator["FilesystemClient"]:
@@ -972,12 +972,12 @@ def prepare_load_table(self, table_name: str) -> PreparedTableSchema:
     def get_table_dir(
         self, table_name: str, remote: bool = False, schema_name: Optional[str] = None
     ) -> str:
-        """Returns a directory containing table files, ending with separator.
+        """Returns a directory containing table files.
         Note that many tables can share the same table dir
         """
         # dlt tables do not respect layout (for now)
         table_prefix = self.get_table_prefix(table_name, schema_name=schema_name)
-        table_dir: str = self.pathlib.dirname(table_prefix) + self.pathlib.sep
+        table_dir: str = self.pathlib.dirname(table_prefix)
         if remote:
             table_dir = self.make_remote_url(table_dir)
         return table_dir

tests/load/filesystem/test_filesystem_client.py

@@ -137,22 +137,26 @@ def test_filesystem_follows_local_dir(location: str) -> None:
 @pytest.mark.parametrize(
     "layout", ("{table_name}/{load_id}.{file_id}.{ext}", "{table_name}.{load_id}.{file_id}.{ext}")
 )
-def test_trailing_separators(layout: str, with_gdrive_buckets_env: str) -> None:
+def test_filesystem_dirs_do_not_have_trailing_separators(
+    layout: str, with_gdrive_buckets_env: str
+) -> None:
     os.environ["DESTINATION__FILESYSTEM__LAYOUT"] = layout
     load = setup_loader("_data")
     client: FilesystemClient = load.get_destination_client(Schema("empty"))  # type: ignore[assignment]
-    # assert separators
-    assert client.dataset_path.endswith("_data/")
-    assert client.get_table_dir("_dlt_versions").endswith("_dlt_versions/")
-    assert client.get_table_dir("_dlt_versions", remote=True).endswith("_dlt_versions/")
+    # dataset and table dirs are queried directly with fsspec and must not end
+    # in a separator: some backends reject trailing-separator directory probes.
+    assert client.dataset_path.endswith("_data")
+    assert not client.dataset_path.endswith("/")
+    assert client.get_table_dir("_dlt_versions").endswith("_dlt_versions")
+    assert client.get_table_dir("_dlt_versions", remote=True).endswith("_dlt_versions")
     is_folder = layout.startswith("{table_name}/")
     if is_folder:
-        assert client.get_table_dir("letters").endswith("_data/letters/")
-        assert client.get_table_dir("letters", remote=True).endswith("_data/letters/")
+        assert client.get_table_dir("letters").endswith("_data/letters")
+        assert client.get_table_dir("letters", remote=True).endswith("_data/letters")
     else:
         # strip prefix
-        assert client.get_table_dir("letters").endswith("_data/")
-        assert client.get_table_dir("letters", remote=True).endswith("_data/")
+        assert client.get_table_dir("letters").endswith("_data")
+        assert client.get_table_dir("letters", remote=True).endswith("_data")
     if is_folder:
         assert client.get_table_prefix("letters").endswith("_data/letters/")
     else: