From 68128d638dde290267de63ca807f28ba226779d0 Mon Sep 17 00:00:00 2001 From: Marcel Sayre <52462209+msayr@users.noreply.github.com> Date: Tue, 30 Jun 2026 13:48:43 +1000 Subject: [PATCH 01/11] Add Z alignment repair utility --- fix_z_align.py | 161 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 fix_z_align.py diff --git a/fix_z_align.py b/fix_z_align.py new file mode 100644 index 0000000..56f73ff --- /dev/null +++ b/fix_z_align.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Repair a bad Z-alignment boundary without rebuilding the whole stack. + +This utility reuses an existing ``align_stack_z.py`` JSON config, re-aligns a +small user-selected preview range, opens a Neuroglancer inspection view, and, on +CLI confirmation, propagates the repaired alignment from the first bad slice to +the end of that stack by rerunning Z alignment for the suffix only. +""" + +import argparse +import copy +import json +import logging +import os +from typing import Optional, Tuple + +from emalign.inspect_dataset import inspect_dataset +from emalign.io.progress import get_mongo_client, get_mongo_db +from emalign.io.store import open_store +from emalign.scripts.align_stack_z import align_stack_z + + +logging.basicConfig(level=logging.INFO) +LOGGER = logging.getLogger(__name__) + + +def _parse_slice_range(value: str) -> Tuple[int, int]: + """Parse ``N`` or ``N:M`` / ``N-M`` into an inclusive local slice range.""" + value = value.strip() + sep = ':' if ':' in value else '-' if '-' in value else None + if sep is None: + start = end = int(value) + else: + start_s, end_s = value.split(sep, maxsplit=1) + start, end = int(start_s), int(end_s) + if start < 0 or end < start: + raise argparse.ArgumentTypeError('Slice range must be N or START:END with 0 <= START <= END.') + return start, end + + +def _stack_bounds(config: dict) -> Tuple[int, int]: + """Return the local [min, max) bounds covered by the original stack config.""" + dataset = open_store(os.path.abspath(config['dataset_path']), mode='r') + default_min = int(dataset.domain.inclusive_min[0]) + default_max = int(dataset.domain.exclusive_max[0]) + return int(config.get('local_z_min', default_min) or default_min), int(config.get('local_z_max', default_max) or default_max) + + +def _global_z_for_local(config: dict, original_local_min: int, local_z: int) -> int: + """Map an input stack-local z index to the destination/global z index.""" + return int(config.get('z_offset', 0)) + int(local_z) - int(original_local_min) + + +def _delete_progress_suffix(db, dataset_name: str, start_local: int, start_global: int, *, include_mesh: bool) -> None: + """Remove cached progress docs that would otherwise cause suffix repair to be skipped.""" + collection = db[dataset_name] + result = collection.delete_many({ + '$or': [ + {'step_name': 'flow_z', 'local_slice': {'$gte': int(start_local)}}, + {'step_name': 'render_z', 'global_slice': {'$gte': int(start_global)}}, + ] + }) + LOGGER.info('Deleted %d cached flow/render MongoDB progress documents.', result.deleted_count) + if include_mesh: + mesh_result = collection.delete_many({'step_name': 'mesh_relax_z'}) + LOGGER.info('Deleted %d cached mesh MongoDB progress documents.', mesh_result.deleted_count) + + +def _make_repair_config(base_config: dict, original_local_min: int, local_start: int, local_stop_exclusive: int) -> dict: + """Build an align_stack_z config for a suffix/subrange starting at local_start.""" + config = copy.deepcopy(base_config) + config['local_z_min'] = int(local_start) + config['local_z_max'] = int(local_stop_exclusive) + config['z_offset'] = _global_z_for_local(base_config, original_local_min, local_start) + config['first_slice'] = config['z_offset'] - 1 + config['overwrite'] = True + config['wipe_progress_flag'] = False + return config + + +def _confirm(prompt: str) -> bool: + answer = input(f'{prompt} [y/N]: ').strip().lower() + return answer in {'y', 'yes'} + + +def repair_z_alignment(config_path: str, + slice_range: Tuple[int, int], + preview_after: int, + inspect_port: int, + skip_preview: bool = False, + project_name: Optional[str] = None) -> None: + with open(config_path, 'r') as f: + base_config = json.load(f) + + if project_name is not None: + base_config['project_name'] = project_name + if not base_config.get('project_name'): + base_config['project_name'] = os.path.basename(base_config['destination_path']).rstrip('.zarr') + + dataset_name = base_config['dataset_name'] + original_min, original_max = _stack_bounds(base_config) + repair_start, repair_end = slice_range + if repair_start < original_min or repair_end >= original_max: + raise ValueError(f'Repair range {repair_start}:{repair_end} is outside configured stack bounds [{original_min}, {original_max}).') + if repair_start == original_min and base_config.get('first_slice') is None and base_config.get('reference_path') is None: + raise ValueError('Cannot repair the first slice of a root stack because there is no previous aligned slice to anchor to.') + + preview_stop = min(original_max, max(repair_end + 1, repair_start + 1) + max(0, preview_after)) + start_global = _global_z_for_local(base_config, original_min, repair_start) + + client = get_mongo_client(base_config.get('mongodb_config_filepath')) + db = get_mongo_db(client, base_config['project_name']) + + if not skip_preview: + LOGGER.info('Realigning preview range local z [%d, %d) for %s.', repair_start, preview_stop, dataset_name) + _delete_progress_suffix(db, dataset_name, repair_start, start_global, include_mesh=True) + preview_config = _make_repair_config(base_config, original_min, repair_start, preview_stop) + align_stack_z(**preview_config) + + inspect_min = max(0, start_global - 2) + inspect_max = _global_z_for_local(base_config, original_min, preview_stop - 1) + 3 + LOGGER.info('Opening Neuroglancer for destination slices [%d, %d).', inspect_min, inspect_max) + inspect_dataset(base_config['destination_path'], + bounding_box=[inspect_min, inspect_max], + keep_missing=True, + bind_port=inspect_port) + + if not _confirm('Does the previewed Z alignment look correct and should the repair be propagated to the end of the stack?'): + LOGGER.warning('Repair was not propagated. Preview slices already written to the destination remain in place.') + return + + LOGGER.info('Propagating repaired alignment from local z %d through %d for %s.', repair_start, original_max, dataset_name) + _delete_progress_suffix(db, dataset_name, repair_start, start_global, include_mesh=True) + final_config = _make_repair_config(base_config, original_min, repair_start, original_max) + align_stack_z(**final_config) + + LOGGER.info('Repair complete. Opening final Neuroglancer inspection view.') + inspect_dataset(base_config['destination_path'], + bounding_box=[max(0, start_global - 2), min(start_global + 2 * preview_after + 5, start_global + (original_max - repair_start))], + keep_missing=True, + bind_port=inspect_port) + + +def main() -> None: + parser = argparse.ArgumentParser(description='Repair and propagate a problematic Z alignment in an existing EMalign stack.') + parser.add_argument('config_file', help='Existing align_stack_z JSON config for the stack to repair.') + parser.add_argument('--slice-range', required=True, type=_parse_slice_range, + help='Input stack-local bad slice or inclusive range to preview, e.g. "201" or "201:205".') + parser.add_argument('--preview-after', type=int, default=5, + help='Number of additional slices after --slice-range to include in the preview realignment.') + parser.add_argument('--port', type=int, default=55555, help='Neuroglancer bind port for inspection.') + parser.add_argument('--skip-preview', action='store_true', + help='Do not pause for Neuroglancer confirmation; immediately propagate to the end of the stack.') + parser.add_argument('--project-name', default=None, help='Override project_name from the config file.') + args = parser.parse_args() + + repair_z_alignment(args.config_file, args.slice_range, args.preview_after, args.port, args.skip_preview, args.project_name) + + +if __name__ == '__main__': + main() From 55c37f9587a5dfc51af72293e8c26104af6f8218 Mon Sep 17 00:00:00 2001 From: Marcel Sayre <52462209+msayr@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:42:04 +1000 Subject: [PATCH 02/11] Clarify fix_z_align config argument --- fix_z_align.py | 45 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/fix_z_align.py b/fix_z_align.py index 56f73ff..982305c 100644 --- a/fix_z_align.py +++ b/fix_z_align.py @@ -14,10 +14,6 @@ import os from typing import Optional, Tuple -from emalign.inspect_dataset import inspect_dataset -from emalign.io.progress import get_mongo_client, get_mongo_db -from emalign.io.store import open_store -from emalign.scripts.align_stack_z import align_stack_z logging.basicConfig(level=logging.INFO) @@ -40,6 +36,8 @@ def _parse_slice_range(value: str) -> Tuple[int, int]: def _stack_bounds(config: dict) -> Tuple[int, int]: """Return the local [min, max) bounds covered by the original stack config.""" + from emalign.io.store import open_store + dataset = open_store(os.path.abspath(config['dataset_path']), mode='r') default_min = int(dataset.domain.inclusive_min[0]) default_max = int(dataset.domain.exclusive_max[0]) @@ -89,6 +87,10 @@ def repair_z_alignment(config_path: str, inspect_port: int, skip_preview: bool = False, project_name: Optional[str] = None) -> None: + from emalign.inspect_dataset import inspect_dataset + from emalign.io.progress import get_mongo_client, get_mongo_db + from emalign.scripts.align_stack_z import align_stack_z + with open(config_path, 'r') as f: base_config = json.load(f) @@ -142,8 +144,32 @@ def repair_z_alignment(config_path: str, def main() -> None: - parser = argparse.ArgumentParser(description='Repair and propagate a problematic Z alignment in an existing EMalign stack.') - parser.add_argument('config_file', help='Existing align_stack_z JSON config for the stack to repair.') + parser = argparse.ArgumentParser( + description='Repair and propagate a problematic Z alignment in an existing EMalign stack.', + epilog=( + 'CONFIG_FILE must be the JSON file generated for align_stack_z (for example, ' + 'a stack config under your z-alignment config output directory), not the ' + 'destination zarr/final aligned stack path. You may pass it positionally ' + 'or with --config-file.' + ) + ) + parser.add_argument( + 'config_file', + nargs='?', + help=( + 'Path to the existing align_stack_z JSON config for the stack to repair. ' + 'This is not the destination_path/final aligned stack directory.' + ) + ) + parser.add_argument( + '--config-file', + '--config_file', + dest='config_file_option', + help=( + 'Path to the existing align_stack_z JSON config for the stack to repair. ' + 'Use this if you prefer a named flag instead of the positional CONFIG_FILE.' + ) + ) parser.add_argument('--slice-range', required=True, type=_parse_slice_range, help='Input stack-local bad slice or inclusive range to preview, e.g. "201" or "201:205".') parser.add_argument('--preview-after', type=int, default=5, @@ -153,8 +179,13 @@ def main() -> None: help='Do not pause for Neuroglancer confirmation; immediately propagate to the end of the stack.') parser.add_argument('--project-name', default=None, help='Override project_name from the config file.') args = parser.parse_args() + config_file = args.config_file_option or args.config_file + if config_file is None: + parser.error('CONFIG_FILE is required. Provide an align_stack_z JSON config positionally or with --config-file.') + if args.config_file_option and args.config_file and args.config_file_option != args.config_file: + parser.error('Provide CONFIG_FILE either positionally or with --config-file, not both with different values.') - repair_z_alignment(args.config_file, args.slice_range, args.preview_after, args.port, args.skip_preview, args.project_name) + repair_z_alignment(config_file, args.slice_range, args.preview_after, args.port, args.skip_preview, args.project_name) if __name__ == '__main__': From 511543573cf0bd59a28fdfa90e8d4bab4523f4a5 Mon Sep 17 00:00:00 2001 From: Marcel Sayre <52462209+msayr@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:42:08 +1000 Subject: [PATCH 03/11] Support global z ranges in fix_z_align --- fix_z_align.py | 44 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/fix_z_align.py b/fix_z_align.py index 982305c..0edca37 100644 --- a/fix_z_align.py +++ b/fix_z_align.py @@ -49,6 +49,28 @@ def _global_z_for_local(config: dict, original_local_min: int, local_z: int) -> return int(config.get('z_offset', 0)) + int(local_z) - int(original_local_min) +def _normalize_slice_range(config: dict, original_local_min: int, original_local_max: int, slice_range: Tuple[int, int]) -> Tuple[int, int, str]: + """Return a local inclusive range, accepting either local or destination/global z values.""" + start, end = slice_range + if original_local_min <= start <= end < original_local_max: + return start, end, 'local' + + global_min = _global_z_for_local(config, original_local_min, original_local_min) + global_max_exclusive = _global_z_for_local(config, original_local_min, original_local_max) + if global_min <= start <= end < global_max_exclusive: + return ( + original_local_min + start - global_min, + original_local_min + end - global_min, + 'global', + ) + + raise ValueError( + f'Repair range {start}:{end} is outside both local stack bounds ' + f'[{original_local_min}, {original_local_max}) and destination/global bounds ' + f'[{global_min}, {global_max_exclusive}).' + ) + + def _delete_progress_suffix(db, dataset_name: str, start_local: int, start_global: int, *, include_mesh: bool) -> None: """Remove cached progress docs that would otherwise cause suffix repair to be skipped.""" collection = db[dataset_name] @@ -101,14 +123,22 @@ def repair_z_alignment(config_path: str, dataset_name = base_config['dataset_name'] original_min, original_max = _stack_bounds(base_config) - repair_start, repair_end = slice_range - if repair_start < original_min or repair_end >= original_max: - raise ValueError(f'Repair range {repair_start}:{repair_end} is outside configured stack bounds [{original_min}, {original_max}).') - if repair_start == original_min and base_config.get('first_slice') is None and base_config.get('reference_path') is None: - raise ValueError('Cannot repair the first slice of a root stack because there is no previous aligned slice to anchor to.') + repair_start, repair_end, range_space = _normalize_slice_range(base_config, original_min, original_max, slice_range) + start_global = _global_z_for_local(base_config, original_min, repair_start) + LOGGER.info( + 'Interpreting requested slice range %d:%d as %s coordinates -> local z %d:%d, destination/global z %d:%d.', + slice_range[0], + slice_range[1], + range_space, + repair_start, + repair_end, + start_global, + _global_z_for_local(base_config, original_min, repair_end), + ) + if repair_start == original_min and base_config.get('first_slice') is None and base_config.get('reference_path') is None and start_global <= 0: + raise ValueError('Cannot repair the first slice of a root stack because there is no previous aligned destination slice to anchor to.') preview_stop = min(original_max, max(repair_end + 1, repair_start + 1) + max(0, preview_after)) - start_global = _global_z_for_local(base_config, original_min, repair_start) client = get_mongo_client(base_config.get('mongodb_config_filepath')) db = get_mongo_db(client, base_config['project_name']) @@ -171,7 +201,7 @@ def main() -> None: ) ) parser.add_argument('--slice-range', required=True, type=_parse_slice_range, - help='Input stack-local bad slice or inclusive range to preview, e.g. "201" or "201:205".') + help='Bad slice or inclusive range to preview. Values may be stack-local z (e.g. "0" or "0:5") or destination/global z (e.g. "530" or "530:531").') parser.add_argument('--preview-after', type=int, default=5, help='Number of additional slices after --slice-range to include in the preview realignment.') parser.add_argument('--port', type=int, default=55555, help='Neuroglancer bind port for inspection.') From adebf58474879ec1367bef9389b716e6cf0bae2d Mon Sep 17 00:00:00 2001 From: Marcel Sayre <52462209+msayr@users.noreply.github.com> Date: Wed, 1 Jul 2026 18:26:37 +1000 Subject: [PATCH 04/11] Use global slices in z alignment repair --- fix_z_align.py | 148 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 113 insertions(+), 35 deletions(-) diff --git a/fix_z_align.py b/fix_z_align.py index 0edca37..dae5657 100644 --- a/fix_z_align.py +++ b/fix_z_align.py @@ -1,17 +1,21 @@ #!/usr/bin/env python3 """Repair a bad Z-alignment boundary without rebuilding the whole stack. -This utility reuses an existing ``align_stack_z.py`` JSON config, re-aligns a -small user-selected preview range, opens a Neuroglancer inspection view, and, on -CLI confirmation, propagates the repaired alignment from the first bad slice to -the end of that stack by rerunning Z alignment for the suffix only. +This utility should usually be pointed at ``00_align_plan.json`` and a +destination/global slice number from the final aligned dataset. It selects the +matching stack config, re-aligns a small preview range, opens a Neuroglancer +inspection view, and, on CLI confirmation, propagates the repaired alignment +from the first bad slice to the end of that stack by rerunning Z alignment for +the suffix only. """ import argparse import copy import json +import inspect import logging import os +from glob import glob from typing import Optional, Tuple @@ -21,7 +25,7 @@ def _parse_slice_range(value: str) -> Tuple[int, int]: - """Parse ``N`` or ``N:M`` / ``N-M`` into an inclusive local slice range.""" + """Parse ``N`` or ``N:M`` / ``N-M`` into an inclusive global slice range.""" value = value.strip() sep = ':' if ':' in value else '-' if '-' in value else None if sep is None: @@ -36,12 +40,15 @@ def _parse_slice_range(value: str) -> Tuple[int, int]: def _stack_bounds(config: dict) -> Tuple[int, int]: """Return the local [min, max) bounds covered by the original stack config.""" + if config.get('local_z_min') is not None and config.get('local_z_max') is not None: + return int(config['local_z_min']), int(config['local_z_max']) + from emalign.io.store import open_store dataset = open_store(os.path.abspath(config['dataset_path']), mode='r') default_min = int(dataset.domain.inclusive_min[0]) default_max = int(dataset.domain.exclusive_max[0]) - return int(config.get('local_z_min', default_min) or default_min), int(config.get('local_z_max', default_max) or default_max) + return default_min, default_max def _global_z_for_local(config: dict, original_local_min: int, local_z: int) -> int: @@ -49,25 +56,88 @@ def _global_z_for_local(config: dict, original_local_min: int, local_z: int) -> return int(config.get('z_offset', 0)) + int(local_z) - int(original_local_min) -def _normalize_slice_range(config: dict, original_local_min: int, original_local_max: int, slice_range: Tuple[int, int]) -> Tuple[int, int, str]: - """Return a local inclusive range, accepting either local or destination/global z values.""" +def _global_bounds(config: dict, original_local_min: int, original_local_max: int) -> Tuple[int, int]: + """Return destination/global [min, max) bounds for a stack config.""" + return ( + _global_z_for_local(config, original_local_min, original_local_min), + _global_z_for_local(config, original_local_min, original_local_max), + ) + + +def _global_to_local(config: dict, original_local_min: int, global_z: int) -> int: + """Map a destination/global z index to an input stack-local z index.""" + return int(original_local_min) + int(global_z) - int(config.get('z_offset', 0)) + + +def _load_json(path: str) -> dict: + with open(path, 'r') as f: + return json.load(f) + + +def _load_dataset_configs(config_dir: str) -> dict: + configs = {} + for path in glob(os.path.join(config_dir, 'z_*.json')): + cfg = _load_json(path) + if 'dataset_name' in cfg: + cfg['_config_path'] = path + configs[cfg['dataset_name']] = cfg + if not configs: + raise FileNotFoundError(f'No z_*.json stack configs found in {config_dir}.') + return configs + + +def _is_align_plan(config: dict) -> bool: + return 'paths' in config and 'dataset_local_bounds' in config and 'dataset_path' not in config + + +def _config_for_global_range(config_path: str, slice_range: Tuple[int, int]) -> dict: + """Load the stack config containing an inclusive global slice range.""" + config = _load_json(config_path) + if not _is_align_plan(config): + return config + + config_dir = os.path.dirname(os.path.abspath(config_path)) + dataset_configs = _load_dataset_configs(config_dir) start, end = slice_range - if original_local_min <= start <= end < original_local_max: - return start, end, 'local' + matches = [] + for dataset_name, dataset_config in dataset_configs.items(): + local_min, local_max = _stack_bounds(dataset_config) + global_min, global_max = _global_bounds(dataset_config, local_min, local_max) + if global_min <= start <= end < global_max: + matches.append((global_min, dataset_name, dataset_config)) + + if not matches: + ranges = [] + for dataset_name, dataset_config in sorted(dataset_configs.items()): + local_min, local_max = _stack_bounds(dataset_config) + global_min, global_max = _global_bounds(dataset_config, local_min, local_max) + ranges.append(f'{dataset_name}=[{global_min}, {global_max})') + raise ValueError( + f'Global repair range {start}:{end} is not fully contained in one stack from {config_path}. ' + f'Available global ranges: {"; ".join(ranges)}' + ) + + matches.sort() + selected = matches[0][2] + LOGGER.info('Selected stack config %s from align plan for global range %d:%d.', selected.get('_config_path', selected['dataset_name']), start, end) + return selected - global_min = _global_z_for_local(config, original_local_min, original_local_min) - global_max_exclusive = _global_z_for_local(config, original_local_min, original_local_max) + +def _normalize_slice_range(config: dict, original_local_min: int, original_local_max: int, slice_range: Tuple[int, int]) -> Tuple[int, int]: + """Return a local inclusive range from an inclusive destination/global z range.""" + start, end = slice_range + global_min, global_max_exclusive = _global_bounds(config, original_local_min, original_local_max) if global_min <= start <= end < global_max_exclusive: return ( - original_local_min + start - global_min, - original_local_min + end - global_min, - 'global', + _global_to_local(config, original_local_min, start), + _global_to_local(config, original_local_min, end), ) raise ValueError( - f'Repair range {start}:{end} is outside both local stack bounds ' - f'[{original_local_min}, {original_local_max}) and destination/global bounds ' - f'[{global_min}, {global_max_exclusive}).' + f'Global repair range {start}:{end} is outside destination/global bounds ' + f'[{global_min}, {global_max_exclusive}) for stack {config.get("dataset_name", "")}. ' + 'Pass the global slice number from the final aligned dataset, or pass 00_align_plan.json ' + 'so the script can select the correct stack config automatically.' ) @@ -89,6 +159,7 @@ def _delete_progress_suffix(db, dataset_name: str, start_local: int, start_globa def _make_repair_config(base_config: dict, original_local_min: int, local_start: int, local_stop_exclusive: int) -> dict: """Build an align_stack_z config for a suffix/subrange starting at local_start.""" config = copy.deepcopy(base_config) + config.pop('_config_path', None) config['local_z_min'] = int(local_start) config['local_z_max'] = int(local_stop_exclusive) config['z_offset'] = _global_z_for_local(base_config, original_local_min, local_start) @@ -98,6 +169,16 @@ def _make_repair_config(base_config: dict, original_local_min: int, local_start: return config +def _call_align_stack_z(align_stack_z, config: dict) -> None: + """Call align_stack_z with only the keyword arguments accepted by this checkout.""" + params = inspect.signature(align_stack_z).parameters + relevant_args = {k: v for k, v in config.items() if k in params} + ignored = sorted(set(config) - set(relevant_args)) + if ignored: + LOGGER.debug('Ignoring config keys not accepted by align_stack_z: %s', ', '.join(ignored)) + align_stack_z(**relevant_args) + + def _confirm(prompt: str) -> bool: answer = input(f'{prompt} [y/N]: ').strip().lower() return answer in {'y', 'yes'} @@ -113,8 +194,7 @@ def repair_z_alignment(config_path: str, from emalign.io.progress import get_mongo_client, get_mongo_db from emalign.scripts.align_stack_z import align_stack_z - with open(config_path, 'r') as f: - base_config = json.load(f) + base_config = _config_for_global_range(config_path, slice_range) if project_name is not None: base_config['project_name'] = project_name @@ -123,17 +203,15 @@ def repair_z_alignment(config_path: str, dataset_name = base_config['dataset_name'] original_min, original_max = _stack_bounds(base_config) - repair_start, repair_end, range_space = _normalize_slice_range(base_config, original_min, original_max, slice_range) + repair_start, repair_end = _normalize_slice_range(base_config, original_min, original_max, slice_range) start_global = _global_z_for_local(base_config, original_min, repair_start) LOGGER.info( - 'Interpreting requested slice range %d:%d as %s coordinates -> local z %d:%d, destination/global z %d:%d.', + 'Interpreting requested global slice range %d:%d -> local z %d:%d in %s.', slice_range[0], slice_range[1], - range_space, repair_start, repair_end, - start_global, - _global_z_for_local(base_config, original_min, repair_end), + base_config['dataset_name'], ) if repair_start == original_min and base_config.get('first_slice') is None and base_config.get('reference_path') is None and start_global <= 0: raise ValueError('Cannot repair the first slice of a root stack because there is no previous aligned destination slice to anchor to.') @@ -147,7 +225,7 @@ def repair_z_alignment(config_path: str, LOGGER.info('Realigning preview range local z [%d, %d) for %s.', repair_start, preview_stop, dataset_name) _delete_progress_suffix(db, dataset_name, repair_start, start_global, include_mesh=True) preview_config = _make_repair_config(base_config, original_min, repair_start, preview_stop) - align_stack_z(**preview_config) + _call_align_stack_z(align_stack_z, preview_config) inspect_min = max(0, start_global - 2) inspect_max = _global_z_for_local(base_config, original_min, preview_stop - 1) + 3 @@ -164,7 +242,7 @@ def repair_z_alignment(config_path: str, LOGGER.info('Propagating repaired alignment from local z %d through %d for %s.', repair_start, original_max, dataset_name) _delete_progress_suffix(db, dataset_name, repair_start, start_global, include_mesh=True) final_config = _make_repair_config(base_config, original_min, repair_start, original_max) - align_stack_z(**final_config) + _call_align_stack_z(align_stack_z, final_config) LOGGER.info('Repair complete. Opening final Neuroglancer inspection view.') inspect_dataset(base_config['destination_path'], @@ -177,17 +255,17 @@ def main() -> None: parser = argparse.ArgumentParser( description='Repair and propagate a problematic Z alignment in an existing EMalign stack.', epilog=( - 'CONFIG_FILE must be the JSON file generated for align_stack_z (for example, ' - 'a stack config under your z-alignment config output directory), not the ' - 'destination zarr/final aligned stack path. You may pass it positionally ' - 'or with --config-file.' + 'CONFIG_FILE should usually be 00_align_plan.json from config/z_config. The script ' + 'uses global destination slice numbers only, selects the matching stack config, ' + 'and repairs from that global slice onward. A per-stack z_*.json config is still ' + 'accepted for advanced use, but --slice-range remains global.' ) ) parser.add_argument( 'config_file', nargs='?', help=( - 'Path to the existing align_stack_z JSON config for the stack to repair. ' + 'Path to 00_align_plan.json from config/z_config, or an advanced per-stack z_*.json config. ' 'This is not the destination_path/final aligned stack directory.' ) ) @@ -196,12 +274,12 @@ def main() -> None: '--config_file', dest='config_file_option', help=( - 'Path to the existing align_stack_z JSON config for the stack to repair. ' + 'Path to 00_align_plan.json from config/z_config, or an advanced per-stack z_*.json config. ' 'Use this if you prefer a named flag instead of the positional CONFIG_FILE.' ) ) parser.add_argument('--slice-range', required=True, type=_parse_slice_range, - help='Bad slice or inclusive range to preview. Values may be stack-local z (e.g. "0" or "0:5") or destination/global z (e.g. "530" or "530:531").') + help='Bad destination/global slice or inclusive global range to preview, e.g. "531" or "531:536". Local stack slice numbers are not accepted.') parser.add_argument('--preview-after', type=int, default=5, help='Number of additional slices after --slice-range to include in the preview realignment.') parser.add_argument('--port', type=int, default=55555, help='Neuroglancer bind port for inspection.') @@ -211,7 +289,7 @@ def main() -> None: args = parser.parse_args() config_file = args.config_file_option or args.config_file if config_file is None: - parser.error('CONFIG_FILE is required. Provide an align_stack_z JSON config positionally or with --config-file.') + parser.error('CONFIG_FILE is required. Provide 00_align_plan.json, or an advanced per-stack z_*.json config, positionally or with --config-file.') if args.config_file_option and args.config_file and args.config_file_option != args.config_file: parser.error('Provide CONFIG_FILE either positionally or with --config-file, not both with different values.') From 6c8c1d8301b0a28fb41ea5c588ac6d81ad5fab2c Mon Sep 17 00:00:00 2001 From: Marcel Sayre <52462209+msayr@users.noreply.github.com> Date: Wed, 1 Jul 2026 20:49:17 +1000 Subject: [PATCH 05/11] Force rerun and restore rejected z previews --- fix_z_align.py | 144 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 127 insertions(+), 17 deletions(-) diff --git a/fix_z_align.py b/fix_z_align.py index dae5657..33fb13e 100644 --- a/fix_z_align.py +++ b/fix_z_align.py @@ -11,12 +11,12 @@ import argparse import copy -import json import inspect +import json import logging import os from glob import glob -from typing import Optional, Tuple +from typing import List, Optional, Tuple @@ -141,19 +141,34 @@ def _normalize_slice_range(config: dict, original_local_min: int, original_local ) -def _delete_progress_suffix(db, dataset_name: str, start_local: int, start_global: int, *, include_mesh: bool) -> None: +def _progress_delete_filter(start_local: int, start_global: int, *, include_mesh: bool) -> dict: + filters = [ + {'step_name': 'flow_z', 'local_slice': {'$gte': int(start_local)}}, + {'step_name': 'render_z', 'local_slice': {'$gte': int(start_local)}}, + {'step_name': 'render_z', 'global_slice': {'$gte': int(start_global)}}, + ] + if include_mesh: + filters.append({'step_name': 'mesh_relax_z'}) + return {'$or': filters} + + +def _delete_progress_suffix(db, dataset_name: str, start_local: int, start_global: int, *, include_mesh: bool) -> List[dict]: """Remove cached progress docs that would otherwise cause suffix repair to be skipped.""" collection = db[dataset_name] - result = collection.delete_many({ - '$or': [ - {'step_name': 'flow_z', 'local_slice': {'$gte': int(start_local)}}, - {'step_name': 'render_z', 'global_slice': {'$gte': int(start_global)}}, - ] - }) - LOGGER.info('Deleted %d cached flow/render MongoDB progress documents.', result.deleted_count) - if include_mesh: - mesh_result = collection.delete_many({'step_name': 'mesh_relax_z'}) - LOGGER.info('Deleted %d cached mesh MongoDB progress documents.', mesh_result.deleted_count) + doc_filter = _progress_delete_filter(start_local, start_global, include_mesh=include_mesh) + deleted_docs = list(collection.find(doc_filter)) + result = collection.delete_many(doc_filter) + LOGGER.info('Deleted %d cached MongoDB progress documents so the requested Z range is recomputed.', result.deleted_count) + return deleted_docs + + +def _restore_progress_suffix(db, dataset_name: str, start_local: int, start_global: int, docs: List[dict], *, include_mesh: bool) -> None: + """Restore progress docs saved before a preview realignment.""" + collection = db[dataset_name] + collection.delete_many(_progress_delete_filter(start_local, start_global, include_mesh=include_mesh)) + if docs: + collection.insert_many(docs) + LOGGER.info('Restored %d MongoDB progress documents from before the preview.', len(docs)) def _make_repair_config(base_config: dict, original_local_min: int, local_start: int, local_stop_exclusive: int) -> dict: @@ -179,6 +194,89 @@ def _call_align_stack_z(align_stack_z, config: dict) -> None: align_stack_z(**relevant_args) +def _force_stack_unprocessed(config: dict) -> dict: + """Temporarily clear the source z_aligned flag so align_stack_z cannot skip.""" + from emalign.io.store import get_store_attributes, set_store_attributes + + attrs = get_store_attributes(config['dataset_path']) + updated = copy.deepcopy(attrs) + updated['z_aligned'] = False + set_store_attributes(config['dataset_path'], updated) + return attrs + + +def _restore_stack_attrs(config: dict, attrs: dict) -> None: + from emalign.io.store import set_store_attributes + + set_store_attributes(config['dataset_path'], attrs) + + +def _run_forced_align_stack_z(align_stack_z, config: dict) -> dict: + """Run align_stack_z after clearing the metadata flag that would skip it.""" + original_attrs = _force_stack_unprocessed(config) + _call_align_stack_z(align_stack_z, config) + return original_attrs + + +def _downsampled_destination_path(destination_path: str, save_downsampled: float) -> str: + output_path, project_name_from_path = destination_path.rsplit('/', maxsplit=1) + return os.path.join(output_path, f'{save_downsampled}x_' + project_name_from_path) + + +def _snapshot_store_range(path: str, start_global: int, stop_global: int, dtype, *, allow_missing: bool = False) -> Optional[dict]: + from emalign.io.store import open_store + + store = open_store(path, mode='r+', dtype=dtype, allow_missing=allow_missing) + if store is None: + return None + return { + 'path': path, + 'dtype': dtype, + 'start': int(start_global), + 'stop': int(stop_global), + 'data': store[start_global:stop_global].read().result(), + } + + +def _restore_store_range(snapshot: Optional[dict]) -> None: + if snapshot is None: + return + + from emalign.io.store import open_store + + store = open_store(snapshot['path'], mode='r+', dtype=snapshot['dtype']) + store[snapshot['start']:snapshot['stop']].write(snapshot['data']).result() + + +def _snapshot_preview_outputs(config: dict, start_global: int, stop_global: int) -> List[Optional[dict]]: + """Capture destination data that preview alignment may overwrite.""" + import tensorstore as ts + + snapshots = [ + _snapshot_store_range(config['destination_path'], start_global, stop_global, ts.uint8), + _snapshot_store_range(config['destination_path'] + '_mask', start_global, stop_global, ts.bool), + ] + + save_downsampled = config.get('save_downsampled', 1) + if save_downsampled != 1: + snapshots.append( + _snapshot_store_range( + _downsampled_destination_path(config['destination_path'], save_downsampled), + start_global, + stop_global, + ts.uint8, + allow_missing=True, + ) + ) + + return snapshots + + +def _restore_preview_outputs(snapshots: List[Optional[dict]]) -> None: + for snapshot in snapshots: + _restore_store_range(snapshot) + + def _confirm(prompt: str) -> bool: answer = input(f'{prompt} [y/N]: ').strip().lower() return answer in {'y', 'yes'} @@ -223,9 +321,18 @@ def repair_z_alignment(config_path: str, if not skip_preview: LOGGER.info('Realigning preview range local z [%d, %d) for %s.', repair_start, preview_stop, dataset_name) - _delete_progress_suffix(db, dataset_name, repair_start, start_global, include_mesh=True) preview_config = _make_repair_config(base_config, original_min, repair_start, preview_stop) - _call_align_stack_z(align_stack_z, preview_config) + preview_stop_global = _global_z_for_local(base_config, original_min, preview_stop - 1) + 1 + preview_output_snapshots = _snapshot_preview_outputs(base_config, start_global, preview_stop_global) + preview_progress_docs = _delete_progress_suffix(db, dataset_name, repair_start, start_global, include_mesh=True) + preview_attrs = _force_stack_unprocessed(preview_config) + try: + _call_align_stack_z(align_stack_z, preview_config) + except Exception: + _restore_preview_outputs(preview_output_snapshots) + _restore_progress_suffix(db, dataset_name, repair_start, start_global, preview_progress_docs, include_mesh=True) + _restore_stack_attrs(preview_config, preview_attrs) + raise inspect_min = max(0, start_global - 2) inspect_max = _global_z_for_local(base_config, original_min, preview_stop - 1) + 3 @@ -236,13 +343,16 @@ def repair_z_alignment(config_path: str, bind_port=inspect_port) if not _confirm('Does the previewed Z alignment look correct and should the repair be propagated to the end of the stack?'): - LOGGER.warning('Repair was not propagated. Preview slices already written to the destination remain in place.') + LOGGER.warning('Repair was not propagated. Restoring destination slices, MongoDB progress, and source stack metadata from before the preview.') + _restore_preview_outputs(preview_output_snapshots) + _restore_progress_suffix(db, dataset_name, repair_start, start_global, preview_progress_docs, include_mesh=True) + _restore_stack_attrs(preview_config, preview_attrs) return LOGGER.info('Propagating repaired alignment from local z %d through %d for %s.', repair_start, original_max, dataset_name) _delete_progress_suffix(db, dataset_name, repair_start, start_global, include_mesh=True) final_config = _make_repair_config(base_config, original_min, repair_start, original_max) - _call_align_stack_z(align_stack_z, final_config) + _run_forced_align_stack_z(align_stack_z, final_config) LOGGER.info('Repair complete. Opening final Neuroglancer inspection view.') inspect_dataset(base_config['destination_path'], From 4c13470d2a214da4c751c23bcb6ee621138afacd Mon Sep 17 00:00:00 2001 From: Marcel Sayre <52462209+msayr@users.noreply.github.com> Date: Wed, 1 Jul 2026 21:33:03 +1000 Subject: [PATCH 06/11] Restore preview stores after resize --- fix_z_align.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fix_z_align.py b/fix_z_align.py index 33fb13e..2892f9f 100644 --- a/fix_z_align.py +++ b/fix_z_align.py @@ -234,6 +234,8 @@ def _snapshot_store_range(path: str, start_global: int, stop_global: int, dtype, 'dtype': dtype, 'start': int(start_global), 'stop': int(stop_global), + 'inclusive_min': [int(v) for v in store.domain.inclusive_min], + 'exclusive_max': [int(v) for v in store.domain.exclusive_max], 'data': store[start_global:stop_global].read().result(), } @@ -245,6 +247,10 @@ def _restore_store_range(snapshot: Optional[dict]) -> None: from emalign.io.store import open_store store = open_store(snapshot['path'], mode='r+', dtype=snapshot['dtype']) + store = store.resize( + inclusive_min=snapshot['inclusive_min'], + exclusive_max=snapshot['exclusive_max'], + ).result() store[snapshot['start']:snapshot['stop']].write(snapshot['data']).result() From fa2ecb1ba7c275441797a4682b1220c2295cd015 Mon Sep 17 00:00:00 2001 From: Marcel Sayre <52462209+msayr@users.noreply.github.com> Date: Wed, 1 Jul 2026 22:34:59 +1000 Subject: [PATCH 07/11] Support anchored z boundary repair ranges --- fix_z_align.py | 41 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/fix_z_align.py b/fix_z_align.py index 2892f9f..b855e8b 100644 --- a/fix_z_align.py +++ b/fix_z_align.py @@ -91,7 +91,7 @@ def _is_align_plan(config: dict) -> bool: def _config_for_global_range(config_path: str, slice_range: Tuple[int, int]) -> dict: - """Load the stack config containing an inclusive global slice range.""" + """Load the stack config containing, or anchored by, an inclusive global slice range.""" config = _load_json(config_path) if not _is_align_plan(config): return config @@ -107,14 +107,46 @@ def _config_for_global_range(config_path: str, slice_range: Tuple[int, int]) -> matches.append((global_min, dataset_name, dataset_config)) if not matches: + boundary_matches = [] + for dataset_name, dataset_config in dataset_configs.items(): + first_slice = dataset_config.get('first_slice') + if first_slice is None: + continue + first_slice = int(first_slice) + if start == first_slice and end >= start: + local_min, local_max = _stack_bounds(dataset_config) + repair_end = min(local_max - 1, local_min + max(0, end - start - 1)) + boundary_config = copy.deepcopy(dataset_config) + boundary_config['_boundary_anchor_global'] = first_slice + boundary_config['_repair_local_override'] = (local_min, repair_end) + boundary_matches.append((first_slice, dataset_name, boundary_config)) + + if boundary_matches: + boundary_matches.sort() + selected = boundary_matches[0][2] + LOGGER.info( + 'Selected stack config %s from align plan because requested range %d:%d starts at its previous-stack anchor slice %d.', + selected.get('_config_path', selected['dataset_name']), + start, + end, + selected['_boundary_anchor_global'], + ) + return selected + ranges = [] + anchors = [] for dataset_name, dataset_config in sorted(dataset_configs.items()): local_min, local_max = _stack_bounds(dataset_config) global_min, global_max = _global_bounds(dataset_config, local_min, local_max) ranges.append(f'{dataset_name}=[{global_min}, {global_max})') + if dataset_config.get('first_slice') is not None: + anchors.append(f'{dataset_name} anchors to previous global slice {int(dataset_config["first_slice"])}') raise ValueError( f'Global repair range {start}:{end} is not fully contained in one stack from {config_path}. ' - f'Available global ranges: {"; ".join(ranges)}' + f'Available global ranges: {"; ".join(ranges)}. ' + f'Available between-stack anchors: {"; ".join(anchors) if anchors else "none"}. ' + 'For a boundary repair, pass PREVIOUS_SLICE:FIRST_BAD_SLICE, where PREVIOUS_SLICE ' + 'matches the successor stack first_slice anchor.' ) matches.sort() @@ -125,6 +157,9 @@ def _config_for_global_range(config_path: str, slice_range: Tuple[int, int]) -> def _normalize_slice_range(config: dict, original_local_min: int, original_local_max: int, slice_range: Tuple[int, int]) -> Tuple[int, int]: """Return a local inclusive range from an inclusive destination/global z range.""" + if '_repair_local_override' in config: + return tuple(config['_repair_local_override']) + start, end = slice_range global_min, global_max_exclusive = _global_bounds(config, original_local_min, original_local_max) if global_min <= start <= end < global_max_exclusive: @@ -175,6 +210,8 @@ def _make_repair_config(base_config: dict, original_local_min: int, local_start: """Build an align_stack_z config for a suffix/subrange starting at local_start.""" config = copy.deepcopy(base_config) config.pop('_config_path', None) + config.pop('_boundary_anchor_global', None) + config.pop('_repair_local_override', None) config['local_z_min'] = int(local_start) config['local_z_max'] = int(local_stop_exclusive) config['z_offset'] = _global_z_for_local(base_config, original_local_min, local_start) From 7c6cf63dfe671e1a98c2c23431fd82f9b27b08f2 Mon Sep 17 00:00:00 2001 From: Marcel Sayre <52462209+msayr@users.noreply.github.com> Date: Wed, 1 Jul 2026 22:42:33 +1000 Subject: [PATCH 08/11] Clamp preview snapshots to existing bounds --- fix_z_align.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/fix_z_align.py b/fix_z_align.py index b855e8b..550d28e 100644 --- a/fix_z_align.py +++ b/fix_z_align.py @@ -266,14 +266,20 @@ def _snapshot_store_range(path: str, start_global: int, stop_global: int, dtype, store = open_store(path, mode='r+', dtype=dtype, allow_missing=allow_missing) if store is None: return None + inclusive_min = [int(v) for v in store.domain.inclusive_min] + exclusive_max = [int(v) for v in store.domain.exclusive_max] + read_start = max(int(start_global), inclusive_min[0]) + read_stop = min(int(stop_global), exclusive_max[0]) return { 'path': path, 'dtype': dtype, 'start': int(start_global), 'stop': int(stop_global), - 'inclusive_min': [int(v) for v in store.domain.inclusive_min], - 'exclusive_max': [int(v) for v in store.domain.exclusive_max], - 'data': store[start_global:stop_global].read().result(), + 'read_start': read_start, + 'read_stop': read_stop, + 'inclusive_min': inclusive_min, + 'exclusive_max': exclusive_max, + 'data': store[read_start:read_stop].read().result() if read_start < read_stop else None, } @@ -288,7 +294,8 @@ def _restore_store_range(snapshot: Optional[dict]) -> None: inclusive_min=snapshot['inclusive_min'], exclusive_max=snapshot['exclusive_max'], ).result() - store[snapshot['start']:snapshot['stop']].write(snapshot['data']).result() + if snapshot['data'] is not None: + store[snapshot['read_start']:snapshot['read_stop']].write(snapshot['data']).result() def _snapshot_preview_outputs(config: dict, start_global: int, stop_global: int) -> List[Optional[dict]]: From 183a8159a60bfd03c997ee8baec6c066f7e9155a Mon Sep 17 00:00:00 2001 From: Marcel Sayre <52462209+msayr@users.noreply.github.com> Date: Wed, 1 Jul 2026 22:50:28 +1000 Subject: [PATCH 09/11] Preserve boundary anchors during z repair --- fix_z_align.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fix_z_align.py b/fix_z_align.py index 550d28e..d85ee8d 100644 --- a/fix_z_align.py +++ b/fix_z_align.py @@ -209,13 +209,14 @@ def _restore_progress_suffix(db, dataset_name: str, start_local: int, start_glob def _make_repair_config(base_config: dict, original_local_min: int, local_start: int, local_stop_exclusive: int) -> dict: """Build an align_stack_z config for a suffix/subrange starting at local_start.""" config = copy.deepcopy(base_config) + boundary_anchor_global = config.get('_boundary_anchor_global') config.pop('_config_path', None) config.pop('_boundary_anchor_global', None) config.pop('_repair_local_override', None) config['local_z_min'] = int(local_start) config['local_z_max'] = int(local_stop_exclusive) config['z_offset'] = _global_z_for_local(base_config, original_local_min, local_start) - config['first_slice'] = config['z_offset'] - 1 + config['first_slice'] = int(boundary_anchor_global) if boundary_anchor_global is not None else config['z_offset'] - 1 config['overwrite'] = True config['wipe_progress_flag'] = False return config @@ -291,7 +292,6 @@ def _restore_store_range(snapshot: Optional[dict]) -> None: store = open_store(snapshot['path'], mode='r+', dtype=snapshot['dtype']) store = store.resize( - inclusive_min=snapshot['inclusive_min'], exclusive_max=snapshot['exclusive_max'], ).result() if snapshot['data'] is not None: From 3aa915b8bde2556adb214c04475a83e4ea02fd9f Mon Sep 17 00:00:00 2001 From: Marcel Sayre <52462209+msayr@users.noreply.github.com> Date: Wed, 1 Jul 2026 22:54:54 +1000 Subject: [PATCH 10/11] Limit preview realignment to requested slices --- fix_z_align.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fix_z_align.py b/fix_z_align.py index d85ee8d..41903bf 100644 --- a/fix_z_align.py +++ b/fix_z_align.py @@ -364,13 +364,13 @@ def repair_z_alignment(config_path: str, if repair_start == original_min and base_config.get('first_slice') is None and base_config.get('reference_path') is None and start_global <= 0: raise ValueError('Cannot repair the first slice of a root stack because there is no previous aligned destination slice to anchor to.') - preview_stop = min(original_max, max(repair_end + 1, repair_start + 1) + max(0, preview_after)) + preview_stop = min(original_max, repair_end + 1) client = get_mongo_client(base_config.get('mongodb_config_filepath')) db = get_mongo_db(client, base_config['project_name']) if not skip_preview: - LOGGER.info('Realigning preview range local z [%d, %d) for %s.', repair_start, preview_stop, dataset_name) + LOGGER.info('Realigning requested local z range [%d, %d) for %s.', repair_start, preview_stop, dataset_name) preview_config = _make_repair_config(base_config, original_min, repair_start, preview_stop) preview_stop_global = _global_z_for_local(base_config, original_min, preview_stop - 1) + 1 preview_output_snapshots = _snapshot_preview_outputs(base_config, start_global, preview_stop_global) @@ -385,7 +385,7 @@ def repair_z_alignment(config_path: str, raise inspect_min = max(0, start_global - 2) - inspect_max = _global_z_for_local(base_config, original_min, preview_stop - 1) + 3 + inspect_max = _global_z_for_local(base_config, original_min, preview_stop - 1) + max(3, preview_after + 1) LOGGER.info('Opening Neuroglancer for destination slices [%d, %d).', inspect_min, inspect_max) inspect_dataset(base_config['destination_path'], bounding_box=[inspect_min, inspect_max], @@ -441,7 +441,7 @@ def main() -> None: parser.add_argument('--slice-range', required=True, type=_parse_slice_range, help='Bad destination/global slice or inclusive global range to preview, e.g. "531" or "531:536". Local stack slice numbers are not accepted.') parser.add_argument('--preview-after', type=int, default=5, - help='Number of additional slices after --slice-range to include in the preview realignment.') + help='Number of additional destination/global slices after --slice-range to include in the Neuroglancer preview only. These context slices are not realigned during preview.') parser.add_argument('--port', type=int, default=55555, help='Neuroglancer bind port for inspection.') parser.add_argument('--skip-preview', action='store_true', help='Do not pause for Neuroglancer confirmation; immediately propagate to the end of the stack.') From 264a25fe6aaa311f4a0825545f82a2415e6775a7 Mon Sep 17 00:00:00 2001 From: Marcel Sayre <52462209+msayr@users.noreply.github.com> Date: Thu, 2 Jul 2026 00:33:59 +1000 Subject: [PATCH 11/11] Write boundary repairs after anchor slice --- fix_z_align.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/fix_z_align.py b/fix_z_align.py index 41903bf..120959e 100644 --- a/fix_z_align.py +++ b/fix_z_align.py @@ -69,6 +69,13 @@ def _global_to_local(config: dict, original_local_min: int, global_z: int) -> in return int(original_local_min) + int(global_z) - int(config.get('z_offset', 0)) +def _repair_global_z_for_local(config: dict, original_local_min: int, local_z: int) -> int: + """Map local z to the destination/global z that the repair should write.""" + if config.get('_boundary_anchor_global') is not None: + return int(config['_boundary_anchor_global']) + 1 + int(local_z) - int(original_local_min) + return _global_z_for_local(config, original_local_min, local_z) + + def _load_json(path: str) -> dict: with open(path, 'r') as f: return json.load(f) @@ -215,7 +222,7 @@ def _make_repair_config(base_config: dict, original_local_min: int, local_start: config.pop('_repair_local_override', None) config['local_z_min'] = int(local_start) config['local_z_max'] = int(local_stop_exclusive) - config['z_offset'] = _global_z_for_local(base_config, original_local_min, local_start) + config['z_offset'] = _repair_global_z_for_local(base_config, original_local_min, local_start) config['first_slice'] = int(boundary_anchor_global) if boundary_anchor_global is not None else config['z_offset'] - 1 config['overwrite'] = True config['wipe_progress_flag'] = False @@ -352,7 +359,7 @@ def repair_z_alignment(config_path: str, dataset_name = base_config['dataset_name'] original_min, original_max = _stack_bounds(base_config) repair_start, repair_end = _normalize_slice_range(base_config, original_min, original_max, slice_range) - start_global = _global_z_for_local(base_config, original_min, repair_start) + start_global = _repair_global_z_for_local(base_config, original_min, repair_start) LOGGER.info( 'Interpreting requested global slice range %d:%d -> local z %d:%d in %s.', slice_range[0], @@ -372,7 +379,7 @@ def repair_z_alignment(config_path: str, if not skip_preview: LOGGER.info('Realigning requested local z range [%d, %d) for %s.', repair_start, preview_stop, dataset_name) preview_config = _make_repair_config(base_config, original_min, repair_start, preview_stop) - preview_stop_global = _global_z_for_local(base_config, original_min, preview_stop - 1) + 1 + preview_stop_global = _repair_global_z_for_local(base_config, original_min, preview_stop - 1) + 1 preview_output_snapshots = _snapshot_preview_outputs(base_config, start_global, preview_stop_global) preview_progress_docs = _delete_progress_suffix(db, dataset_name, repair_start, start_global, include_mesh=True) preview_attrs = _force_stack_unprocessed(preview_config) @@ -385,7 +392,7 @@ def repair_z_alignment(config_path: str, raise inspect_min = max(0, start_global - 2) - inspect_max = _global_z_for_local(base_config, original_min, preview_stop - 1) + max(3, preview_after + 1) + inspect_max = _repair_global_z_for_local(base_config, original_min, preview_stop - 1) + max(3, preview_after + 1) LOGGER.info('Opening Neuroglancer for destination slices [%d, %d).', inspect_min, inspect_max) inspect_dataset(base_config['destination_path'], bounding_box=[inspect_min, inspect_max],