From 09c0e292dba6987ca9e9c0e7b602a1f2cc8ef6d8 Mon Sep 17 00:00:00 2001 From: archis Date: Tue, 16 Jun 2026 09:03:24 -0700 Subject: [PATCH 1/2] Refactor: break the 3 module couplings blocking a clean forward/inverse split MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prerequisite, behavior-preserving moves toward #105. These break the only dependency cycles/wrong-direction edges in the package, so the forward kernel can later be isolated (as extras or a separate repo) without further surgery. - Merge `utils/data_handling` + `utils/process` into one `tsadar/data/` package. They were mutually recursive (data_handling->process and process->data_handling); the split was artificial since preprocessing *is* data handling. Now the cycle is intra-package. - Move `postprocess.py` into `tsadar/inverse/`. It is post-*fit* processing and imported `inverse.loss_function`/`inverse.loops` while `inverse.fitter` imported it back — a true `inverse <-> process` cycle. Now intra-`inverse`. - Move the interactive runner `calc_vs_data.py` from `forward/` to `inverse/`. It imported `inverse.fitter`/`inverse.loss_function`, making `forward` depend on `inverse` (wrong direction). `forward` no longer imports `inverse`. No behavior change: pure file moves + import-path updates (incl. tests). Verified statically — all intra-tsadar imports resolve, every module compiles, and forward->inverse / data->inverse / data->forward edges are gone; `core` kernel remains free of inverse/forward/data/plotting deps. Co-Authored-By: Claude Opus 4.8 (1M context) --- test_angular_2v.py | 2 +- test_arts2v_random.py | 2 +- tests/test_forward/test_1d.py | 2 +- tests/test_forward/test_angular_1v.py | 2 +- tests/test_forward/test_angular_2v.py | 2 +- tests/test_inverse/test_1d_random.py | 2 +- tests/test_inverse/test_angular_1v.py | 2 +- tests/test_inverse/test_angular_2v.py | 2 +- tests/test_inverse/test_feature_detector.py | 2 +- tsadar/__init__.py | 2 +- tsadar/{utils/data_handling => data}/__init__.py | 0 tsadar/{utils/data_handling => data}/calibration.py | 0 tsadar/{utils/process => data}/correct_throughput.py | 0 tsadar/{utils/data_handling => data}/data_visualizer.py | 2 +- tsadar/{utils/process => data}/evaluate_background.py | 2 +- tsadar/{utils/process => data}/feature_detector.py | 0 tsadar/{utils/data_handling => data}/lam_parse.py | 0 tsadar/{utils/process => data}/lineouts.py | 2 +- tsadar/{utils/data_handling => data}/load_ts_data.py | 2 +- tsadar/{utils/process => data}/prepare.py | 8 ++++---- tsadar/{utils/process => data}/warpcorr.py | 0 tsadar/forward/calc_series.py | 2 +- tsadar/{forward => inverse}/calc_vs_data.py | 6 +++--- tsadar/inverse/fitter.py | 3 ++- tsadar/{utils/process => inverse}/postprocess.py | 4 ++-- tsadar/runner.py | 2 +- tsadar/utils/process/__init__.py | 0 27 files changed, 27 insertions(+), 26 deletions(-) rename tsadar/{utils/data_handling => data}/__init__.py (100%) rename tsadar/{utils/data_handling => data}/calibration.py (100%) rename tsadar/{utils/process => data}/correct_throughput.py (100%) rename tsadar/{utils/data_handling => data}/data_visualizer.py (99%) rename tsadar/{utils/process => data}/evaluate_background.py (99%) rename tsadar/{utils/process => data}/feature_detector.py (100%) rename tsadar/{utils/data_handling => data}/lam_parse.py (100%) rename tsadar/{utils/process => data}/lineouts.py (99%) rename tsadar/{utils/data_handling => data}/load_ts_data.py (98%) rename tsadar/{utils/process => data}/prepare.py (97%) rename tsadar/{utils/process => data}/warpcorr.py (100%) rename tsadar/{forward => inverse}/calc_vs_data.py (97%) rename tsadar/{utils/process => inverse}/postprocess.py (99%) delete mode 100644 tsadar/utils/process/__init__.py diff --git a/test_angular_2v.py b/test_angular_2v.py index e3548499..39f91e58 100644 --- a/test_angular_2v.py +++ b/test_angular_2v.py @@ -12,7 +12,7 @@ from tsadar.utils import misc from tsadar.core.thomson_diagnostic import ThomsonScatteringDiagnostic from tsadar.core.modules.ts_params import ThomsonParams -from tsadar.utils.data_handling.calibration import get_scattering_angles, get_calibrations +from tsadar.data.calibration import get_scattering_angles, get_calibrations def test_arts2d_forward_pass(): diff --git a/test_arts2v_random.py b/test_arts2v_random.py index 98c602bf..445781c1 100644 --- a/test_arts2v_random.py +++ b/test_arts2v_random.py @@ -17,7 +17,7 @@ from tsadar.utils import misc from tsadar.core.thomson_diagnostic import ThomsonScatteringDiagnostic from tsadar.core.modules.ts_params import ThomsonParams, get_filter_spec -from tsadar.utils.data_handling.calibration import get_scattering_angles, get_calibrations +from tsadar.data.calibration import get_scattering_angles, get_calibrations def _dump_ts_params(td: str, dist_type: str, ts_params: ThomsonParams, prefix: str = ""): diff --git a/tests/test_forward/test_1d.py b/tests/test_forward/test_1d.py index b7736d90..84f43716 100644 --- a/tests/test_forward/test_1d.py +++ b/tests/test_forward/test_1d.py @@ -11,7 +11,7 @@ from tsadar.utils import misc from tsadar.core.thomson_diagnostic import ThomsonScatteringDiagnostic from tsadar.core.modules.ts_params import ThomsonParams -from tsadar.utils.data_handling.calibration import get_scattering_angles +from tsadar.data.calibration import get_scattering_angles def test_1d_forward_pass(): diff --git a/tests/test_forward/test_angular_1v.py b/tests/test_forward/test_angular_1v.py index 012118fb..f3ddffbb 100644 --- a/tests/test_forward/test_angular_1v.py +++ b/tests/test_forward/test_angular_1v.py @@ -11,7 +11,7 @@ from tsadar.core.thomson_diagnostic import ThomsonScatteringDiagnostic from tsadar.core.modules.ts_params import ThomsonParams -from tsadar.utils.data_handling.calibration import get_scattering_angles, get_calibrations +from tsadar.data.calibration import get_scattering_angles, get_calibrations def test_arts1d_forward_pass(): diff --git a/tests/test_forward/test_angular_2v.py b/tests/test_forward/test_angular_2v.py index 731febba..ca81df04 100644 --- a/tests/test_forward/test_angular_2v.py +++ b/tests/test_forward/test_angular_2v.py @@ -12,7 +12,7 @@ from tsadar.utils import misc from tsadar.core.thomson_diagnostic import ThomsonScatteringDiagnostic from tsadar.core.modules.ts_params import ThomsonParams -from tsadar.utils.data_handling.calibration import get_scattering_angles, get_calibrations +from tsadar.data.calibration import get_scattering_angles, get_calibrations def test_arts2d_forward_pass(): diff --git a/tests/test_inverse/test_1d_random.py b/tests/test_inverse/test_1d_random.py index d293645b..56ddc26d 100644 --- a/tests/test_inverse/test_1d_random.py +++ b/tests/test_inverse/test_1d_random.py @@ -15,7 +15,7 @@ from tsadar.utils import misc from tsadar.core.thomson_diagnostic import ThomsonScatteringDiagnostic from tsadar.core.modules.ts_params import ThomsonParams, get_filter_spec -from tsadar.utils.data_handling.calibration import get_scattering_angles +from tsadar.data.calibration import get_scattering_angles def _perturb_params_(rng, params): diff --git a/tests/test_inverse/test_angular_1v.py b/tests/test_inverse/test_angular_1v.py index 7d0b4188..ec191197 100644 --- a/tests/test_inverse/test_angular_1v.py +++ b/tests/test_inverse/test_angular_1v.py @@ -20,7 +20,7 @@ from tsadar.utils import misc from tsadar.core.thomson_diagnostic import ThomsonScatteringDiagnostic from tsadar.core.modules.ts_params import ThomsonParams, get_filter_spec -from tsadar.utils.data_handling.calibration import get_scattering_angles, get_calibrations +from tsadar.data.calibration import get_scattering_angles, get_calibrations from tsadar.inverse.loops import label def _dump_ts_params(td: str, ts_params: ThomsonParams, prefix: str = ""): diff --git a/tests/test_inverse/test_angular_2v.py b/tests/test_inverse/test_angular_2v.py index 8498fcd8..69a6f630 100644 --- a/tests/test_inverse/test_angular_2v.py +++ b/tests/test_inverse/test_angular_2v.py @@ -17,7 +17,7 @@ from tsadar.utils import misc from tsadar.core.thomson_diagnostic import ThomsonScatteringDiagnostic from tsadar.core.modules.ts_params import ThomsonParams, get_filter_spec -from tsadar.utils.data_handling.calibration import get_scattering_angles, get_calibrations +from tsadar.data.calibration import get_scattering_angles, get_calibrations def _dump_ts_params(td: str, dist_type: str, ts_params: ThomsonParams, prefix: str = ""): diff --git a/tests/test_inverse/test_feature_detector.py b/tests/test_inverse/test_feature_detector.py index b86bf3e5..f15ea750 100644 --- a/tests/test_inverse/test_feature_detector.py +++ b/tests/test_inverse/test_feature_detector.py @@ -1,5 +1,5 @@ from jax import config -from tsadar.utils.process import prepare +from tsadar.data import prepare config.update("jax_enable_x64", True) diff --git a/tsadar/__init__.py b/tsadar/__init__.py index 4293bbe7..714a2774 100644 --- a/tsadar/__init__.py +++ b/tsadar/__init__.py @@ -1,3 +1,3 @@ from .runner import run_for_app from .core import ThomsonScatteringDiagnostic, ThomsonParams -from .utils.data_handling.calibration import get_scattering_angles +from .data.calibration import get_scattering_angles diff --git a/tsadar/utils/data_handling/__init__.py b/tsadar/data/__init__.py similarity index 100% rename from tsadar/utils/data_handling/__init__.py rename to tsadar/data/__init__.py diff --git a/tsadar/utils/data_handling/calibration.py b/tsadar/data/calibration.py similarity index 100% rename from tsadar/utils/data_handling/calibration.py rename to tsadar/data/calibration.py diff --git a/tsadar/utils/process/correct_throughput.py b/tsadar/data/correct_throughput.py similarity index 100% rename from tsadar/utils/process/correct_throughput.py rename to tsadar/data/correct_throughput.py diff --git a/tsadar/utils/data_handling/data_visualizer.py b/tsadar/data/data_visualizer.py similarity index 99% rename from tsadar/utils/data_handling/data_visualizer.py rename to tsadar/data/data_visualizer.py index 4ea5d84b..fd0df016 100644 --- a/tsadar/utils/data_handling/data_visualizer.py +++ b/tsadar/data/data_visualizer.py @@ -7,7 +7,7 @@ from matplotlib.patches import Rectangle import tempfile, mlflow, os -from tsadar.utils.process.lineouts import compute_lineout_pixel_indices +from .lineouts import compute_lineout_pixel_indices def launch_data_visualizer(elecData, ionData, all_data, all_axes, config): diff --git a/tsadar/utils/process/evaluate_background.py b/tsadar/data/evaluate_background.py similarity index 99% rename from tsadar/utils/process/evaluate_background.py rename to tsadar/data/evaluate_background.py index ef32ab6c..4562aaf9 100644 --- a/tsadar/utils/process/evaluate_background.py +++ b/tsadar/data/evaluate_background.py @@ -6,7 +6,7 @@ from scipy.signal import convolve2d as conv2 -from ..data_handling.load_ts_data import loadData +from .load_ts_data import loadData from .correct_throughput import correctThroughput diff --git a/tsadar/utils/process/feature_detector.py b/tsadar/data/feature_detector.py similarity index 100% rename from tsadar/utils/process/feature_detector.py rename to tsadar/data/feature_detector.py diff --git a/tsadar/utils/data_handling/lam_parse.py b/tsadar/data/lam_parse.py similarity index 100% rename from tsadar/utils/data_handling/lam_parse.py rename to tsadar/data/lam_parse.py diff --git a/tsadar/utils/process/lineouts.py b/tsadar/data/lineouts.py similarity index 99% rename from tsadar/utils/process/lineouts.py rename to tsadar/data/lineouts.py index a672bdf0..b78de4ae 100644 --- a/tsadar/utils/process/lineouts.py +++ b/tsadar/data/lineouts.py @@ -3,7 +3,7 @@ from collections import defaultdict import numpy as np -from tsadar.utils.process.evaluate_background import get_lineout_bg +from .evaluate_background import get_lineout_bg def compute_lineout_pixel_indices(config, axisxE, axisxI, shift_zero, IAWtime, type_name="lineouts"): diff --git a/tsadar/utils/data_handling/load_ts_data.py b/tsadar/data/load_ts_data.py similarity index 98% rename from tsadar/utils/data_handling/load_ts_data.py rename to tsadar/data/load_ts_data.py index 2189e783..34a8f0cd 100644 --- a/tsadar/utils/data_handling/load_ts_data.py +++ b/tsadar/data/load_ts_data.py @@ -3,7 +3,7 @@ import os import numpy as np from scipy.signal import find_peaks -from tsadar.utils.process.warpcorr import perform_warp_correction +from .warpcorr import perform_warp_correction BASE_FILES_PATH = os.path.join(os.path.dirname(__file__), "..", "..", "external") diff --git a/tsadar/utils/process/prepare.py b/tsadar/data/prepare.py similarity index 97% rename from tsadar/utils/process/prepare.py rename to tsadar/data/prepare.py index 3a6d6791..1cfa244c 100644 --- a/tsadar/utils/process/prepare.py +++ b/tsadar/data/prepare.py @@ -4,12 +4,12 @@ import os from .evaluate_background import get_shot_bg -from ..data_handling.load_ts_data import loadData +from .load_ts_data import loadData from .correct_throughput import correctThroughput -from ..data_handling.calibration import get_calibrations, get_scattering_angles +from .calibration import get_calibrations, get_scattering_angles from .lineouts import get_lineouts -from ..data_handling.data_visualizer import launch_data_visualizer -from tsadar.utils.process.feature_detector import first_guess +from .data_visualizer import launch_data_visualizer +from .feature_detector import first_guess def prepare_data(config: Dict, shotNum: int) -> Dict: diff --git a/tsadar/utils/process/warpcorr.py b/tsadar/data/warpcorr.py similarity index 100% rename from tsadar/utils/process/warpcorr.py rename to tsadar/data/warpcorr.py diff --git a/tsadar/forward/calc_series.py b/tsadar/forward/calc_series.py index f14bee08..c038ba1a 100644 --- a/tsadar/forward/calc_series.py +++ b/tsadar/forward/calc_series.py @@ -13,7 +13,7 @@ from ..utils.plotting import plotters from ..core.thomson_diagnostic import ThomsonScatteringDiagnostic from ..core.modules.ts_params import ThomsonParams -from ..utils.data_handling.calibration import get_scattering_angles, get_calibrations +from ..data.calibration import get_scattering_angles, get_calibrations def forward_pass(config): diff --git a/tsadar/forward/calc_vs_data.py b/tsadar/inverse/calc_vs_data.py similarity index 97% rename from tsadar/forward/calc_vs_data.py rename to tsadar/inverse/calc_vs_data.py index d0140d24..50a683e2 100644 --- a/tsadar/forward/calc_vs_data.py +++ b/tsadar/inverse/calc_vs_data.py @@ -5,9 +5,9 @@ from ..core.thomson_diagnostic import ThomsonScatteringDiagnostic from ..core.modules.ts_params import ThomsonParams -from ..inverse.fitter import _validate_inputs_, load_data_for_fitting -from ..utils.data_handling.calibration import get_calibrations -from ..inverse.loss_function import LossFunction +from .fitter import _validate_inputs_, load_data_for_fitting +from ..data.calibration import get_calibrations +from .loss_function import LossFunction from plotly.subplots import make_subplots import plotly.graph_objects as go diff --git a/tsadar/inverse/fitter.py b/tsadar/inverse/fitter.py index 3793aaa7..210063e0 100644 --- a/tsadar/inverse/fitter.py +++ b/tsadar/inverse/fitter.py @@ -7,7 +7,8 @@ from tsadar.inverse.loops import multirun_angular_optax, one_d_loop -from ..utils.process import prepare, postprocess +from ..data import prepare +from . import postprocess def _validate_inputs_(config: Dict) -> Dict: diff --git a/tsadar/utils/process/postprocess.py b/tsadar/inverse/postprocess.py similarity index 99% rename from tsadar/utils/process/postprocess.py rename to tsadar/inverse/postprocess.py index df8836aa..669ec2cd 100644 --- a/tsadar/utils/process/postprocess.py +++ b/tsadar/inverse/postprocess.py @@ -9,9 +9,9 @@ from equinox import filter_jit from tsadar.utils.plotting import plotters -from tsadar.inverse.loss_function import LossFunction +from .loss_function import LossFunction from tsadar.core.modules.ts_params import IonParams -from tsadar.inverse.loops import one_d_loop +from .loops import one_d_loop from tsadar.core.thomson_diagnostic import ThomsonScatteringDiagnostic diff --git a/tsadar/runner.py b/tsadar/runner.py index 69b0bb0a..23050e59 100644 --- a/tsadar/runner.py +++ b/tsadar/runner.py @@ -8,7 +8,7 @@ from .inverse import fitter from .forward import calc_series from .utils import misc -from .forward import calc_vs_data +from .inverse import calc_vs_data if "BASE_TEMPDIR" in os.environ: BASE_TEMPDIR = os.environ["BASE_TEMPDIR"] diff --git a/tsadar/utils/process/__init__.py b/tsadar/utils/process/__init__.py deleted file mode 100644 index e69de29b..00000000 From f48e9573733393b9b8a71c4d980a84d90ff0e048 Mon Sep 17 00:00:00 2001 From: archis Date: Wed, 17 Jun 2026 15:48:02 -0700 Subject: [PATCH 2/2] Fix external/ asset paths broken by the data package move The moved files compute `BASE_FILES_PATH` from `__file__` relative to the package's `external/` asset dir (calibration .mat files, sample data). They lived at `tsadar/utils/{data_handling,process}/` (depth 3) where `../../external` reached `tsadar/external`; after moving to `tsadar/data/` (depth 2) the same `../../external` overshot to repo-root `external/`, which doesn't exist. Drop one `..` so all four resolve back to `tsadar/external/` from the new location. Fixes the CI FileNotFoundError on test_arts1d_forward_pass and test_data. Co-Authored-By: Claude Opus 4.8 (1M context) --- tsadar/data/calibration.py | 2 +- tsadar/data/correct_throughput.py | 2 +- tsadar/data/load_ts_data.py | 2 +- tsadar/data/warpcorr.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tsadar/data/calibration.py b/tsadar/data/calibration.py index 602c38db..2796304d 100644 --- a/tsadar/data/calibration.py +++ b/tsadar/data/calibration.py @@ -3,7 +3,7 @@ import scipy.io as sio import os -BASE_FILES_PATH = os.path.join(os.path.dirname(__file__), "..", "..", "external") +BASE_FILES_PATH = os.path.join(os.path.dirname(__file__), "..", "external") def sa_lookup(beam): diff --git a/tsadar/data/correct_throughput.py b/tsadar/data/correct_throughput.py index 370a7503..e18e216d 100644 --- a/tsadar/data/correct_throughput.py +++ b/tsadar/data/correct_throughput.py @@ -7,7 +7,7 @@ import os import pandas as pd -BASE_FILES_PATH = os.path.join(os.path.dirname(__file__), "..", "..", "external") +BASE_FILES_PATH = os.path.join(os.path.dirname(__file__), "..", "external") def correctThroughput(data, tstype, axisy, shotNum): diff --git a/tsadar/data/load_ts_data.py b/tsadar/data/load_ts_data.py index 34a8f0cd..eaaa8f42 100644 --- a/tsadar/data/load_ts_data.py +++ b/tsadar/data/load_ts_data.py @@ -5,7 +5,7 @@ from scipy.signal import find_peaks from .warpcorr import perform_warp_correction -BASE_FILES_PATH = os.path.join(os.path.dirname(__file__), "..", "..", "external") +BASE_FILES_PATH = os.path.join(os.path.dirname(__file__), "..", "external") def _import_pyhdf(): diff --git a/tsadar/data/warpcorr.py b/tsadar/data/warpcorr.py index 225fe783..a78a739c 100644 --- a/tsadar/data/warpcorr.py +++ b/tsadar/data/warpcorr.py @@ -3,7 +3,7 @@ import math, os from os.path import join, exists -BASE_FILES_PATH = os.path.join(os.path.dirname(__file__), "..", "..", "external") +BASE_FILES_PATH = os.path.join(os.path.dirname(__file__), "..", "external") def perform_warp_correction(warpedData, instrument="EPW", sweepSpeed=5, flatField=True):