Skip to content

TypeError: Lzw.__init__() got an unexpected keyword argument 'name' when opening Icechunk #887

Description

@scottyhq

Following the docs here, I'm trying to save my virtual dataset to icechunk to open later https://virtualizarr.readthedocs.io/en/stable/usage.html#writing-to-an-icechunk-store

Bypassing virtualizarr / icechunk and opening a manifest_store directly works fine. But trying to open from icechunk leads to a big traceback ending with TypeError: Lzw.__init__() got an unexpected keyword argument 'name'

from obstore.store import GCSStore
from virtualizarr import open_virtual_dataset
from obspec_utils.registry import ObjectStoreRegistry
from virtual_tiff import VirtualTIFF
import xarray as xr
import icechunk

bucket = "solus100pub"
file = "cec7_0_cm_p.tif"
url = f"gs://{bucket}/{file}"
store = GCSStore(bucket=bucket, skip_signature=True)

# Working directly with a Manifest Store is fine: 
registry = ObjectStoreRegistry({f"gs://{bucket}": store})
parser = VirtualTIFF(ifd=4) # Use overview for testing
registry = ObjectStoreRegistry({f"gs://{bucket}": store}) # Not really sure what this does...
manifest_store = parser(url=url, registry=registry)
ds = xr.open_zarr(manifest_store, zarr_format=3, consolidated=False)
ds.load()

But saving to Icechunk and then opening leads to Error:

# Save
vds = open_virtual_dataset(
    url=url,
    registry=registry,
    parser=VirtualTIFF(ifd=4),
)
vds

config = icechunk.RepositoryConfig.default()
config.set_virtual_chunk_container(
    icechunk.VirtualChunkContainer(
        url_prefix=f"gs://{bucket}/",
        store=icechunk.gcs_store(),
    ),
)

local_path = '/tmp/test-icechunk'
storage = icechunk.local_filesystem_storage(local_path)
repo = icechunk.Repository.create(storage, config)
session = repo.writable_session("main")
vds.vz.to_icechunk(session.store)
session.commit("initial cec7 virtual dataset")


# Open
storage = icechunk.local_filesystem_storage(local_path)
upstream_credential = icechunk.containers_credentials({f'gs://{bucket}/': 
                                                       icechunk.gcs_credentials(anonymous=True)})
repo = icechunk.Repository.open(storage, authorize_virtual_chunk_access=upstream_credential)
session = repo.readonly_session("main")

ds = xr.open_zarr(session.store, zarr_format=3, consolidated=False)
ds.load()
Show traceback
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[6], [line 9](vscode-notebook-cell:?execution_count=6&line=9)
      6 session = repo.readonly_session("main")
      8 ds = xr.open_zarr(session.store, zarr_format=3, consolidated=False)
----> [9](vscode-notebook-cell:?execution_count=6&line=9) ds.load()

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/xarray/core/dataset.py:566, in Dataset.load(self, **kwargs)
    563         self.variables[k].data = data
    565 # load everything else sequentially
--> [566](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/xarray/core/dataset.py:566) [v.load() for k, v in self.variables.items() if k not in chunked_data]
    568 return self

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/xarray/core/variable.py:1011, in Variable.load(self, **kwargs)
    982 def load(self, **kwargs) -> Self:
    983     """Trigger loading data into memory and return this variable.
    984 
    985     Data will be computed and/or loaded from disk or a remote source.
   (...)   1009     Dataset.load
   1010     """
-> [1011](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/xarray/core/variable.py:1011)     self._data = to_duck_array(self._data, **kwargs)
   1012     return self

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/xarray/namedarray/pycompat.py:143, in to_duck_array(data, **kwargs)
    140     return loaded_data
    142 if isinstance(data, ExplicitlyIndexed | ImplicitToExplicitIndexingAdapter):
--> [143](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/xarray/namedarray/pycompat.py:143)     return data.get_duck_array()  # type: ignore[no-untyped-call, no-any-return]
    144 elif is_duck_array(data):
    145     return data

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/xarray/core/indexing.py:943, in MemoryCachedArray.get_duck_array(self)
    942 def get_duck_array(self):
--> [943](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/xarray/core/indexing.py:943)     duck_array = self.array.get_duck_array()
    944     # ensure the array object is cached in-memory
    945     self.array = as_indexable(duck_array)

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/xarray/core/indexing.py:897, in CopyOnWriteArray.get_duck_array(self)
    896 def get_duck_array(self):
--> [897](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/xarray/core/indexing.py:897)     return self.array.get_duck_array()

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/xarray/core/indexing.py:737, in LazilyIndexedArray.get_duck_array(self)
    734 from xarray.backends.common import BackendArray
    736 if isinstance(self.array, BackendArray):
--> [737](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/xarray/core/indexing.py:737)     array = self.array[self.key]
    738 else:
    739     array = apply_indexer(self.array, self.key)

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/xarray/backends/zarr.py:262, in ZarrArrayWrapper.__getitem__(self, key)
    260 elif isinstance(key, indexing.OuterIndexer):
    261     method = self._oindex
--> [262](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/xarray/backends/zarr.py:262) return indexing.explicit_indexing_adapter(
    263     key, array.shape, indexing.IndexingSupport.VECTORIZED, method
    264 )

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/xarray/core/indexing.py:1129, in explicit_indexing_adapter(key, shape, indexing_support, raw_indexing_method)
   1107 """Support explicit indexing by delegating to a raw indexing method.
   1108 
   1109 Outer and/or vectorized indexers are supported by indexing a second time
   (...)   1126 Indexing result, in the form of a duck numpy-array.
   1127 """
   1128 raw_key, numpy_indices = decompose_indexer(key, shape, indexing_support)
-> [1129](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/xarray/core/indexing.py:1129) result = raw_indexing_method(raw_key.tuple)
   1130 if numpy_indices.tuple:
   1131     # index the loaded duck array
   1132     indexable = as_indexable(result)

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/xarray/backends/zarr.py:225, in ZarrArrayWrapper._getitem(self, key)
    224 def _getitem(self, key):
--> [225](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/xarray/backends/zarr.py:225)     return self._array[key]

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/array.py:2868, in Array.__getitem__(self, selection)
   2866     return self.vindex[cast("CoordinateSelection | MaskSelection", selection)]
   2867 elif is_pure_orthogonal_indexing(pure_selection, self.ndim):
-> [2868](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/array.py:2868)     return self.get_orthogonal_selection(pure_selection, fields=fields)
   2869 else:
   2870     return self.get_basic_selection(cast("BasicSelection", pure_selection), fields=fields)

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/array.py:3339, in Array.get_orthogonal_selection(self, selection, out, fields, prototype)
   3337     prototype = default_buffer_prototype()
   3338 indexer = OrthogonalIndexer(selection, self.shape, self.metadata.chunk_grid)
-> [3339](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/array.py:3339) return sync(
   3340     self.async_array._get_selection(
   3341         indexer=indexer, out=out, fields=fields, prototype=prototype
   3342     )
   3343 )

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/sync.py:159, in sync(coro, loop, timeout)
    156 return_result = next(iter(finished)).result()
    158 if isinstance(return_result, BaseException):
--> [159](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/sync.py:159)     raise return_result
    160 else:
    161     return return_result

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/sync.py:119, in _runner(coro)
    114 """
    115 Await a coroutine and return the result of running it. If awaiting the coroutine raises an
    116 exception, the exception will be returned.
    117 """
    118 try:
--> [119](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/sync.py:119)     return await coro
    120 except Exception as ex:
    121     return ex

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/array.py:1565, in AsyncArray._get_selection(self, indexer, prototype, out, fields)
   1562         _config = replace(_config, order=self.order)
   1564     # reading chunks and decoding them
-> [1565](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/array.py:1565)     await self.codec_pipeline.read(
   1566         [
   1567             (
   1568                 self.store_path / self.metadata.encode_chunk_key(chunk_coords),
   1569                 self.metadata.get_chunk_spec(chunk_coords, _config, prototype=prototype),
   1570                 chunk_selection,
   1571                 out_selection,
   1572                 is_complete_chunk,
   1573             )
   1574             for chunk_coords, chunk_selection, out_selection, is_complete_chunk in indexer
   1575         ],
   1576         out_buffer,
   1577         drop_axes=indexer.drop_axes,
   1578     )
   1579 if isinstance(indexer, BasicIndexer) and indexer.shape == ():
   1580     return out_buffer.as_scalar()

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/codec_pipeline.py:473, in BatchedCodecPipeline.read(self, batch_info, out, drop_axes)
    467 async def read(
    468     self,
    469     batch_info: Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple, bool]],
    470     out: NDBuffer,
    471     drop_axes: tuple[int, ...] = (),
    472 ) -> None:
--> [473](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/codec_pipeline.py:473)     await concurrent_map(
    474         [
    475             (single_batch_info, out, drop_axes)
    476             for single_batch_info in batched(batch_info, self.batch_size)
    477         ],
    478         self.read_batch,
    479         config.get("async.concurrency"),
    480     )

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/common.py:[116](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/common.py:116), in concurrent_map(items, func, limit)
    113     async with sem:
    114         return await func(*item)
--> 116 return await asyncio.gather(*[asyncio.ensure_future(run(item)) for item in items])

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/common.py:[114](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/common.py:114), in concurrent_map.<locals>.run(item)
    112 async def run(item: tuple[Any]) -> V:
    113     async with sem:
--> 114         return await func(*item)

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/codec_pipeline.py:275, in BatchedCodecPipeline.read_batch(self, batch_info, out, drop_axes)
    269 else:
    270     chunk_bytes_batch = await concurrent_map(
    271         [(byte_getter, array_spec.prototype) for byte_getter, array_spec, *_ in batch_info],
    272         lambda byte_getter, prototype: byte_getter.get(prototype),
    273         config.get("async.concurrency"),
    274     )
--> [275](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/codec_pipeline.py:275)     chunk_array_batch = await self.decode_batch(
    276         [
    277             (chunk_bytes, chunk_spec)
    278             for chunk_bytes, (_, chunk_spec, *_) in zip(
    279                 chunk_bytes_batch, batch_info, strict=False
    280             )
    281         ],
    282     )
    283     for chunk_array, (_, chunk_spec, chunk_selection, out_selection, _) in zip(
    284         chunk_array_batch, batch_info, strict=False
    285     ):
    286         if chunk_array is not None:

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/codec_pipeline.py:190, in BatchedCodecPipeline.decode_batch(self, chunk_bytes_and_specs)
    183 (
    184     aa_codecs_with_spec,
    185     ab_codec_with_spec,
    186     bb_codecs_with_spec,
    187 ) = self._codecs_with_resolved_metadata_batched(chunk_specs)
    189 for bb_codec, chunk_spec_batch in bb_codecs_with_spec[::-1]:
--> [190](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/codec_pipeline.py:190)     chunk_bytes_batch = await bb_codec.decode(
    191         zip(chunk_bytes_batch, chunk_spec_batch, strict=False)
    192     )
    194 ab_codec, chunk_spec_batch = ab_codec_with_spec
    195 chunk_array_batch = await ab_codec.decode(
    196     zip(chunk_bytes_batch, chunk_spec_batch, strict=False)
    197 )

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/abc/codec.py:159, in BaseCodec.decode(self, chunks_and_specs)
    143 async def decode(
    144     self,
    145     chunks_and_specs: Iterable[tuple[CodecOutput | None, ArraySpec]],
    146 ) -> Iterable[CodecInput | None]:
    147     """Decodes a batch of chunks.
    148     Chunks can be None in which case they are ignored by the codec.
    149 
   (...)    157     Iterable[CodecInput | None]
    158     """
--> [159](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/abc/codec.py:159)     return await _batching_helper(self._decode_single, chunks_and_specs)

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/abc/codec.py:467, in _batching_helper(func, batch_info)
    463 async def _batching_helper(
    464     func: Callable[[CodecInput, ArraySpec], Awaitable[CodecOutput | None]],
    465     batch_info: Iterable[tuple[CodecInput | None, ArraySpec]],
    466 ) -> list[CodecOutput | None]:
--> [467](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/abc/codec.py:467)     return await concurrent_map(
    468         list(batch_info),
    469         _noop_for_none(func),
    470         config.get("async.concurrency"),
    471     )

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/common.py:116, in concurrent_map(items, func, limit)
    113     async with sem:
    114         return await func(*item)
--> [116](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/common.py:116) return await asyncio.gather(*[asyncio.ensure_future(run(item)) for item in items])

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/common.py:114, in concurrent_map.<locals>.run(item)
    112 async def run(item: tuple[Any]) -> V:
    113     async with sem:
--> [114](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/core/common.py:114)         return await func(*item)

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/abc/codec.py:480, in _noop_for_none.<locals>.wrap(chunk, chunk_spec)
    478 if chunk is None:
    479     return None
--> [480](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/zarr/abc/codec.py:480) return await func(chunk, chunk_spec)

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/virtual_tiff/imagecodecs.py:102, in _ImageCodecsBytesBytesCodec._decode_single(self, chunk_bytes, chunk_spec)
     97 async def _decode_single(
     98     self, chunk_bytes: Buffer, chunk_spec: ArraySpec
     99 ) -> Buffer:
    100     return await asyncio.to_thread(
    101         as_numpy_array_wrapper,
--> [102](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/virtual_tiff/imagecodecs.py:102)         self._codec.decode,
    103         chunk_bytes,
    104         chunk_spec.prototype,
    105     )

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/functools.py:1025, in cached_property.__get__(self, instance, owner)
   1023 val = cache.get(self.attrname, _NOT_FOUND)
   1024 if val is _NOT_FOUND:
-> [1025](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/functools.py:1025)     val = self.func(instance)
   1026     try:
   1027         cache[self.attrname] = val

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/virtual_tiff/imagecodecs.py:74, in _ImageCodecsCodec._codec(self)
     72 codec_config = self.codec_config["configuration"]
     73 codec_config["id"] = self.codec_config["name"]
---> [74](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/virtual_tiff/imagecodecs.py:74) return numcodecs.get_codec(codec_config)

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/numcodecs/registry.py:53, in get_codec(config)
     51     register_codec(cls, codec_id=codec_id)
     52 if cls:
---> [53](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/numcodecs/registry.py:53)     return cls.from_config(config)
     54 raise UnknownCodecError(f"{codec_id!r}")

File ~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/numcodecs/abc.py:106, in Codec.from_config(cls, config)
     98 """Instantiate codec from a configuration object."""
     99 # N.B., assume at this point the 'id' field has been removed from
    100 # the config object
    101 
   (...)    104 # by default, assume constructor accepts configuration parameters as
    105 # keyword arguments without any special decoding
--> [106](https://file+.vscode-resource.vscode-cdn.net/Users/scotthenderson/GitHub/cresst/solus-stac/~/GitHub/cresst/solus-stac/.pixi/envs/default/lib/python3.13/site-packages/numcodecs/abc.py:106) return cls(**config)

TypeError: Lzw.__init__() got an unexpected keyword argument 'name'

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions