diff --git a/.circleci/config.yml b/.circleci/config.yml index 522b1f0..cd9b6f8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,27 +8,73 @@ orbs: commands: install-miniconda: - description: Install Miniconda on machine executors (macOS only; Windows images ship with Miniconda) + description: Ensure conda is on PATH (install if missing on Windows/macOS) steps: - run: name: Install Miniconda command: | set -euo pipefail + + conda_present_at() { + local root="$1" + [ -n "${root}" ] && { [ -f "${root}/Scripts/conda.exe" ] || [ -f "${root}/bin/conda" ]; } + } + if command -v conda >/dev/null 2>&1; then echo "Conda already available at $(command -v conda)" + conda config --set quiet true conda --version exit 0 fi - MINICONDA_DIR="${HOME}/miniconda3" - if [ -x "${MINICONDA_DIR}/Scripts/conda.exe" ] || [ -x "${MINICONDA_DIR}/bin/conda" ]; then - echo "Miniconda already present at ${MINICONDA_DIR}" - else - case "$(uname -s)" in - MINGW*|MSYS*|CYGWIN*) - echo "install-miniconda: unexpected Windows path without preinstalled conda" - exit 1 - ;; - Darwin) + + MINICONDA_DIR="" + case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) + for candidate in \ + "${CONDA:-}" \ + "/c/Miniconda" \ + "/c/Miniconda3" \ + "/c/ProgramData/miniconda3" \ + "/c/ProgramData/Miniconda3" \ + "${HOME}/miniconda3" \ + "${HOME}/Miniconda3" \ + "${HOME}/appdata/Local/miniconda3" \ + "${HOME}/appdata/Local/Continuum/miniconda3"; do + if conda_present_at "${candidate}"; then + MINICONDA_DIR="${candidate}" + break + fi + done + + if [ -z "${MINICONDA_DIR}" ]; then + conda_exe="$(cmd.exe //c "where conda 2>nul" 2>/dev/null | head -n1 | tr -d '\r' || true)" + if [ -n "${conda_exe}" ]; then + if command -v cygpath >/dev/null 2>&1; then + conda_exe="$(cygpath -u "${conda_exe}")" + else + conda_exe="$(echo "${conda_exe}" | sed 's|\\|/|g' | sed 's|^\([A-Za-z]\):|/\L\1|')" + fi + MINICONDA_DIR="$(cd "$(dirname "${conda_exe}")/.." && pwd)" + fi + fi + + if [ -z "${MINICONDA_DIR}" ]; then + MINICONDA_DIR="${HOME}/miniconda3" + fi + + if ! conda_present_at "${MINICONDA_DIR}"; then + echo "Installing Miniconda to %USERPROFILE%\\miniconda3 via cmd.exe..." + curl -fsSL -o miniconda-installer.exe \ + "https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe" + # /D= must be last and must not be quoted (NSIS installer requirement). + cmd.exe //c "miniconda-installer.exe /InstallationType=JustMe /RegisterPython=0 /S /D=%USERPROFILE%\\miniconda3" + rm -f miniconda-installer.exe + MINICONDA_DIR="${HOME}/miniconda3" + fi + ;; + Darwin) + MINICONDA_DIR="${HOME}/miniconda3" + if ! conda_present_at "${MINICONDA_DIR}"; then ARCH="$(uname -m)" if [ "${ARCH}" = "arm64" ]; then INSTALLER_URL="https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh" @@ -38,13 +84,20 @@ commands: curl -fsSL -o miniconda-installer.sh "${INSTALLER_URL}" bash miniconda-installer.sh -b -p "${MINICONDA_DIR}" rm -f miniconda-installer.sh - ;; - *) - echo "install-miniconda: unsupported OS $(uname -s)" - exit 1 - ;; - esac + fi + ;; + *) + echo "install-miniconda: unsupported OS $(uname -s)" + exit 1 + ;; + esac + + if ! conda_present_at "${MINICONDA_DIR}"; then + echo "install-miniconda: conda not found under ${MINICONDA_DIR}" + exit 1 fi + echo "Using Miniconda at ${MINICONDA_DIR}" + { echo "export MINICONDA_DIR=${MINICONDA_DIR}" case "$(uname -s)" in @@ -58,6 +111,7 @@ commands: } >> "${BASH_ENV}" # shellcheck source=/dev/null source "${BASH_ENV}" + conda config --set quiet true conda --version run-conda-test-suite: @@ -75,8 +129,9 @@ commands: [ -f "${BASH_ENV}" ] && source "${BASH_ENV}" eval "$(conda shell.bash hook)" - conda update -y conda - conda create -n cheminf python=3.12 -y + conda config --set quiet true + conda update -y -q conda + conda create -n cheminf python=3.11 -y -q conda activate cheminf python -m pip install --upgrade pip @@ -93,6 +148,7 @@ commands: Linux) CONDA_PKGS+=( libgfortran=14.2.0 + libgomp libxkbcommon mscorefonts fonts-conda-forge @@ -101,8 +157,20 @@ commands: Darwin) CONDA_PKGS+=(libgfortran=14.2.0) ;; + MINGW*|MSYS*|CYGWIN*) + # xTB on win-64 is built with MinGW; match Linux/macOS Fortran/OpenMP runtimes. + CONDA_PKGS+=( + libgfortran=14.2.0 + llvm-openmp + ) + ;; esac - conda install -y -c conda-forge "${CONDA_PKGS[@]}" + if ! conda install -y -q -c conda-forge "${CONDA_PKGS[@]}" >conda-install.log 2>&1; then + echo "conda install failed; last 200 lines:" + tail -n 200 conda-install.log + exit 1 + fi + echo "conda install finished ($(wc -c < conda-install.log) bytes of log)" case "$(uname -s)" in Linux) @@ -120,6 +188,76 @@ commands: pip install . pip install pytest pytest-cov pytest-qt + # xtb needs OpenMP/Fortran runtimes from the conda env (verify before AQME tests). + case "$(uname -s)" in + Linux) + export LD_LIBRARY_PATH="$CONDA_PREFIX/lib:${LD_LIBRARY_PATH:-}" + ;; + Darwin) + export DYLD_FALLBACK_LIBRARY_PATH="$CONDA_PREFIX/lib:${DYLD_FALLBACK_LIBRARY_PATH:-}" + ;; + MINGW*|MSYS*|CYGWIN*) + export PATH="$CONDA_PREFIX/Library/bin:$CONDA_PREFIX/Scripts:$PATH" + ;; + esac + command -v xtb + xtb --version + python -c "import importlib.metadata as m; import aqme; print('aqme', m.version('aqme'))" + # Windows-only preflight: Linux/macOS AQME tests already pass in CI. + case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) + conda list | grep -E '^(xtb|aqme|libgfortran|libgomp|llvm-openmp)\s' || true + echo "=== Windows AQME/xTB preflight ===" + PREFLIGHT_DIR="$(mktemp -d)" + printf '%s\n' \ + '3' \ + 'preflight' \ + 'C 0.000000 0.000000 0.000000' \ + 'H 1.089000 0.000000 0.000000' \ + 'H -0.363000 1.028000 0.000000' \ + > "${PREFLIGHT_DIR}/preflight.xyz" + set +e + xtb_out="$(xtb "${PREFLIGHT_DIR}/preflight.xyz" --gfn 2 --chrg 0 --uhf 0 2>&1)" + xtb_rc=$? + set -e + echo "${xtb_out}" | tail -n 40 + if [ "${xtb_rc}" -ne 0 ]; then + echo "xtb preflight failed (exit ${xtb_rc})" + exit 1 + fi + if ! echo "${xtb_out}" | grep -qiE 'total energy|:: total energy'; then + echo "xtb preflight: no energy line in output" + exit 1 + fi + + printf 'code_name,SMILES\npreflight,C\n' > "${PREFLIGHT_DIR}/AQME_indiv.csv" + ( + cd "${PREFLIGHT_DIR}" + python -u -m aqme --qdescp \ + --input AQME_indiv.csv \ + --program xtb \ + --csv_name AQME_indiv.csv \ + --nprocs 1 \ + --sample 1 \ + --robert + ) + if [ ! -f "${PREFLIGHT_DIR}/AQME-ROBERT_interpret_AQME_indiv.csv" ]; then + echo "AQME preflight: expected CSV not created" + ls -la "${PREFLIGHT_DIR}" || true + for log in "${PREFLIGHT_DIR}/QDESCP/QDESCP_data.dat" \ + "${PREFLIGHT_DIR}/CSEARCH/CSEARCH_data.dat"; do + if [ -f "${log}" ]; then + echo "--- tail ${log} ---" + tail -n 80 "${log}" + fi + done + exit 1 + fi + echo "Windows AQME/xTB preflight passed" + rm -rf "${PREFLIGHT_DIR}" + ;; + esac + set +e python -m pytest \ @@ -279,6 +417,7 @@ jobs: shell: bash.exe steps: - checkout + - install-miniconda - run-conda-test-suite: upload_coverage: false diff --git a/.gitignore b/.gitignore index 05f3e66..bbe3cc2 100644 --- a/.gitignore +++ b/.gitignore @@ -131,3 +131,14 @@ dmypy.json # Pyre type checker .pyre/ + +# ROBERT module run outputs (local working directory artifacts) +CURATE/ +GENERATE/ +VERIFY/ +PREDICT/ +AQME/ +EVALUATE/ +ROBERT_report.pdf +report_debug.txt +report.css diff --git a/README.md b/README.md index 3c5aad4..10866f3 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Full documentation with installation instructions, technical details and example Don't miss out the latest hands-on tutorials from our [YouTube channel](https://www.youtube.com/channel/UCHRqI8N61bYxWV9BjbUI4Xw)! ## Recommended installation -1. (Only once) Create new conda environment: `conda create -n robert python=3.10` +1. (Only once) Create new conda environment: `conda create -n robert python=3.12` 2. Activate conda environment: `conda activate robert` 3. Install ROBERT using pip: `pip install robert` 4. Install RDKit (only if you plan to use easyROB): `pip install rdkit` diff --git a/build_easyrob/config_files/backend_env.yaml b/build_easyrob/config_files/backend_env.yaml index add7aed..1d06a71 100644 --- a/build_easyrob/config_files/backend_env.yaml +++ b/build_easyrob/config_files/backend_env.yaml @@ -13,5 +13,5 @@ dependencies: - mscorefonts - pip - pip: - - robert==2.0.2 + - robert==2.2.0 - aqme==2.0.0 \ No newline at end of file diff --git a/docs/Examples/full_workflow/CSV/Robert_example.csv b/docs/Examples/full_workflow/CSV/Robert_example.csv index 3537dc7..0ca3ce6 100644 --- a/docs/Examples/full_workflow/CSV/Robert_example.csv +++ b/docs/Examples/full_workflow/CSV/Robert_example.csv @@ -1,6 +1,38 @@ Name,Target_values,x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11 1,1.854766065,12,110.9270401,70.8240401,Csub-H,89.87553406,49.77253406,1,0,0,0,1 2,2.034511341,11.7,110.6553116,70.25231158,Csub-Csub,78.65235138,55.53135138,1,0,0,0,1 -..., , , , , , , , , , , , +3,0.860667495,-23.05,112.0635071,36.91050708,H-O,88.85021973,13.69721973,0,0,1,1,1 +4,1.026447198,-16.24,112.101181,43.75818103,Csub-O,69.10329437,18.04229437,0,1,0,1,1 +5,0.952469022,-31.94,111.8824539,27.83945392,H-O,88.58656311,4.54356311,1,0,1,1,2 +6,0.308776518,-68.04,113.0647278,-7.078272217,H-O,89.46292114,-30.68007886,0,0,2,2,2 +7,0.982261986,-34.71,110.7689209,23.9559209,Csub-Csub,79.076828,9.545828003,1,0,1,1,2 +8,1.129601283,-34.94,111.5555725,24.51257251,Csub-Csub,78.75331879,8.992318787,1,0,1,1,2 +9,1.047746118,-34.48,110.5679703,23.98497028,Csub-Csub,78.03665161,8.735651611,1,0,1,1,2 +10,0.515551116,-62.34,110.4608917,-3.982108276,Csub-O,58.43679047,-38.72420953,0,1,1,2,2 +11,0.515551116,-62.34,113.9221497,-0.520850342,Csub-O,69.55827332,-27.60272668,0,1,1,2,2 +12,0.44305174,-62.34,112.8844147,-1.558585327,Csub-O,62.99021912,-34.17078088,0,1,1,2,2 +13,2.221390241,0.29,110.5127563,58.69975635,Csub-Csub,77.87947083,43.34847083,2,0,0,0,2 +14,2.138192104,-0.78,110.3923798,57.50937976,Csub-Csub,78.41550446,42.81450446,2,0,0,0,2 +15,1.883806797,4.54,110.719635,63.15663501,Csub-H,89.83821106,42.27521106,2,0,0,0,2 +16,2.091025544,4.12,111.3189087,63.33590869,Csub-H,89.78244019,41.79944019,2,0,0,0,2 +17,1.902644865,4.29,110.7536316,62.94063159,Csub-H,89.6808548,41.8678548,2,0,0,0,2 +18,2.312908191,-0.46,110.959137,58.39613696,Csub-Csub,78.64178467,43.36078467,2,0,0,0,2 +19,1.16234335,-30.93,110.8476944,27.8146944,Csub-Csub,76.10786438,10.35686438,1,0,0,0,1 +20,1.989080521,-32.41,111.2339783,26.72097827,Csub-H,82.79759216,5.392952454,2,0,0,0,2 +21,2.801329141,-5.9,110.3707199,52.36771991,Csub-Csub,78.09907532,37.37807532,2,0,0,0,2 +22,2.391560251,-5.9,110.9408264,52.93782642,Csub-Csub,78.66916656,37.94816656,2,0,0,0,2 +23,2.011592734,-5.9,110.4220352,52.41903522,Csub-Csub,78.39561462,37.67461462,2,0,0,0,2 +24,0.998633019,-27.6,110.4682465,30.76524646,Csub-O,66.3848114,3.963811401,1,1,0,1,2 +25,1.031375085,-27.6,113.392807,33.68980701,Csub-O,69.00031281,6.579312805,1,1,0,1,2 +26,1.006818535,-27.6,112.2061539,32.50315387,Csub-O,67.16915131,4.748151306,1,1,0,1,2 +27,1.694401925,-29.7,110.9802704,29.17727039,Csub-Csub,77.10182953,51.03182953,1,0,0,0,1 +28,0.515838419,-65.68,111.9355087,-5.847491272,H-O,78.45265198,-39.33034802,0,0,2,2,2 +29,0.916777853,-24.3,111.7115936,35.30859363,Csub-O,70.3069458,17.6069458,0,1,0,1,1 +30,0.837770564,-71.32,111.7983551,-11.6246449,H-O,76.89291382,-46.53008618,1,0,2,2,3 +31,0.942297022,-30.67,111.0540771,28.28107715,H-O,86.16723633,3.394236328,1,0,1,1,2 +32,0.965415203,-29.94,110.8601837,28.81718372,H-O,86.97593689,4.93293689,1,0,1,1,2 +33,0.39762592,-101.6,111.0383148,-42.66468518,Csub-O,55.76971817,-71.26043939,0,2,1,3,3 +34,1.178714383,-101.6,111.0308151,-42.67218488,H-O,84.12934875,-69.57365125,2,0,1,1,3 +35,0.410599034,-101.6,111.6276932,-42.07530682,Csub-O,59.93067169,-76.49032831,0,2,1,3,3 36,0.321084552,-101.6,110.7593079,-42.94369214,Csub-O,59.81459808,-76.60640192,0,2,1,3,3 37,0.329517076,-101.6,115.2292938,-38.47370618,Csub-O,70.45233154,-65.96866846,0,2,1,3,3 diff --git a/docs/Examples/full_workflow/full_workflow.rst b/docs/Examples/full_workflow/full_workflow.rst index ad86f22..70375f3 100644 --- a/docs/Examples/full_workflow/full_workflow.rst +++ b/docs/Examples/full_workflow/full_workflow.rst @@ -49,11 +49,19 @@ Executing the job Execution time and versions +++++++++++++++++++++++++++ -Time: ~3.5 min +Time: ~10 min -System: 8 processors (11th Gen Intel(R) Core(TM) i7-1165G7 @ 2.80GHz) using 16.0 GB RAM memory +System: 8 processors (x86_64 Linux) using 16.0 GB RAM memory -ROBERT version: 2.0.1 +ROBERT version: 2.2.0 + +numpy version: 2.3.4 + +scikit-learn version: 1.7.2 + +xgboost version: 2.1.4 + +bayesian-optimization version: 3.1.0 scikit-learn-intelex version: 2025.2.0 diff --git a/docs/Examples/full_workflow/full_workflow_test.rst b/docs/Examples/full_workflow/full_workflow_test.rst index e62cd45..cd678eb 100644 --- a/docs/Examples/full_workflow/full_workflow_test.rst +++ b/docs/Examples/full_workflow/full_workflow_test.rst @@ -49,11 +49,19 @@ Executing the job Execution time and versions +++++++++++++++++++++++++++ -Time: ~10 sec +Time: ~1.5 min -System: 8 processors (11th Gen Intel(R) Core(TM) i7-1165G7 @ 2.80GHz) using 16.0 GB RAM memory +System: 8 processors (x86_64 Linux) using 16.0 GB RAM memory -ROBERT version: 2.0.1 +ROBERT version: 2.2.0 + +numpy version: 2.3.4 + +scikit-learn version: 1.7.2 + +xgboost version: 2.1.4 + +bayesian-optimization version: 3.1.0 scikit-learn-intelex version: 2025.2.0 diff --git a/docs/Examples/full_workflow/smiles_vaskas.rst b/docs/Examples/full_workflow/smiles_vaskas.rst index 1b284f9..9e30833 100644 --- a/docs/Examples/full_workflow/smiles_vaskas.rst +++ b/docs/Examples/full_workflow/smiles_vaskas.rst @@ -148,15 +148,23 @@ Execution time and versions Time: ~3 min -System: 4 processors (Intel Xeon Ice Lake 8352Y) using 8.0 GB RAM memory +System: 8 processors (x86_64 Linux) using 16.0 GB RAM memory -ROBERT version: 1.2.0 +ROBERT version: 2.2.0 -scikit-learn-intelex version: 2024.5.0 +numpy version: 2.3.4 -AQME version: 1.6.1 +scikit-learn version: 1.7.2 -xTB version: 6.6.1 +xgboost version: 2.1.4 + +bayesian-optimization version: 3.1.0 + +scikit-learn-intelex version: 2025.2.0 + +AQME version: 2.0.0 + +xTB version: 6.7.1 Results +++++++ diff --git a/docs/Examples/full_workflow/smiles_workflow.rst b/docs/Examples/full_workflow/smiles_workflow.rst index 76a0599..8f4db2c 100644 --- a/docs/Examples/full_workflow/smiles_workflow.rst +++ b/docs/Examples/full_workflow/smiles_workflow.rst @@ -112,15 +112,23 @@ Execution time and versions Time: ~1.5 min -System: 4 processors (Intel Xeon Ice Lake 8352Y) using 8.0 GB RAM memory +System: 8 processors (x86_64 Linux) using 16.0 GB RAM memory -ROBERT version: 1.2.0 +ROBERT version: 2.2.0 -scikit-learn-intelex version: 2024.5.0 +numpy version: 2.3.4 -AQME version: 1.6.1 +scikit-learn version: 1.7.2 -xTB version: 6.6.1 +xgboost version: 2.1.4 + +bayesian-optimization version: 3.1.0 + +scikit-learn-intelex version: 2025.2.0 + +AQME version: 2.0.0 + +xTB version: 6.7.1 Results +++++++ diff --git a/docs/Misc/versions.rst b/docs/Misc/versions.rst index 2a6755c..b13154b 100644 --- a/docs/Misc/versions.rst +++ b/docs/Misc/versions.rst @@ -12,6 +12,10 @@ Version 2.2.0 [`url `__] - ``plot_verbosity`` and ``predict_diagnostics`` control diagnostic figure generation - Matplotlib threading workaround consolidated in ``_mpl_plot_context``; figures closed after saving - VERIFY metrics plot handles degenerate axis limits when all test scores are equal + - Added xgboost 2.1.4 as an install-time dependency for opt-in ``XGB`` screening + - PDF reproducibility section now records installed ML stack versions at run time + - Test extras: pytest, pytest-cov, pytest-qt (``requirements-test.txt`` and ``setup.py`` extras) + - Dependency stack aligned with v2.0.3/v2.1.0 pins: numpy 2.x, scikit-learn 1.7.2, hyperopt replaced by bayesian-optimization 3.1.0, rdkit 2025.9.1, Python minimum 3.11 Version 2.1.1 [`url `__] - Adding RMSE values for each fold to calculate t- and Wilconxon tests @@ -33,6 +37,11 @@ Version 2.1.0 [`url `__] - Default split is 'RND' for classification problems - The sklearn-intelex accelerator was removed +Version 2.0.3 [`url `__] + - Major dependency stack update in ``setup.py``: numpy 2.3.4, pandas 2.3.3, scikit-learn 1.7.2, numba 0.62.1, shap 0.49.1, rdkit 2025.9.1, PySide6 6.9.2, weasyprint 63.1 + - Replaced hyperopt with bayesian-optimization 3.1.0 + - Python minimum raised to 3.11 + Version 2.0.2 [`url `__] - Fixed bug in MAC and Linux OS from the GENERATE - Updating AQME version to 1.7.3 diff --git a/docs/README.rst b/docs/README.rst index 502d814..7b61a3f 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -374,25 +374,30 @@ Requirements Python and Python libraries +++++++++++++++++++++++++++ -*These libraries are installed during the initial conda-forge installation.* +*These libraries are installed during the initial conda-forge or pip installation.* -* Python >= 3.6 -* matplotlib-base +* Python >= 3.11 (3.11 or 3.12 recommended) +* matplotlib * pandas * numpy * progress -* pyyaml +* PyYAML * seaborn * scipy * scikit-learn * xgboost (dependency for optional ``XGB`` model screening; not part of the default model list) -* hyperopt +* bayesian-optimization * numba * shap -* glib +* rdkit * weasyprint -* gtk3 -* pango +* psutil +* requests +* PySide6 (easyROB GUI) +* PyMuPDF, pdfplumber, openpyxl (report and GUI extras) +* glib, gtk3, pango (PDF report generation via conda-forge) + +Key pinned versions in ``setup.py`` (v2.2.0): numpy 2.3.4, pandas 2.3.3, scikit-learn 1.7.2, xgboost 2.1.4, bayesian-optimization 3.1.0, rdkit 2025.9.1. .. requirements-end diff --git a/docs/Report/repro.rst b/docs/Report/repro.rst index 896a7a8..e24531d 100644 --- a/docs/Report/repro.rst +++ b/docs/Report/repro.rst @@ -8,7 +8,7 @@ Overview ROBERT aims to improve the reproducibility and transparency of chemical machine learning protocols, addressing two long-standing concerns within the scientific community when publishing in peer-reviewed journals. Several authors have emphasized the critical necessity of establishing and enforcing standards in this regard since a significant proportion of publications are challenging or even impossible to replicate (see references below). -The generated ROBERT_report.pdf files include a **reproducibility section** that instructs authors on which files they should upload as supporting information. This section also provides other researchers with the precise programs, versions, and commands required to replicate the results. Additionally, there is a **transparency section** containing comprehensive information about the machine learning models used and other relevant details regarding ROBERT protocols. +The generated ROBERT_report.pdf files include a **reproducibility section** that instructs authors on which files they should upload as supporting information. This section also provides other researchers with the precise programs, versions, and commands required to replicate the results. At run time, ROBERT records the installed versions of core Python dependencies (for example numpy, scikit-learn, and xgboost) alongside ROBERT, AQME, and xTB when applicable. Additionally, there is a **transparency section** containing comprehensive information about the machine learning models used and other relevant details regarding ROBERT protocols. * `Best practices in ML for chemistry `__ * `Artificial intelligence faces reproducibility crisis `__ diff --git a/docs/_static/ROBERT_report.pdf b/docs/_static/ROBERT_report.pdf index 248f873..adaa90a 100644 Binary files a/docs/_static/ROBERT_report.pdf and b/docs/_static/ROBERT_report.pdf differ diff --git a/docs/_static/Robert_example_test_NN_No_PFI.csv b/docs/_static/Robert_example_test_NN_No_PFI.csv index fa275c0..cd8351f 100644 --- a/docs/_static/Robert_example_test_NN_No_PFI.csv +++ b/docs/_static/Robert_example_test_NN_No_PFI.csv @@ -1,10 +1,10 @@ -Name,x2,x7,x8,x9,x10,x11,Csub-Csub,Csub-H,H-O,Target_values_pred,Target_values_pred_sd -38,110.9270401,1,0,0,0,1,False,True,False,1.753831004793896,0.2 -39,110.6553116,1,0,0,0,1,True,False,False,1.6616478853668877,0.19 -40,112.0635071,0,0,1,1,1,False,False,True,0.8159464846213476,0.09 -41,112.101181,0,1,0,1,1,False,False,False,0.8924375393989473,0.06 -42,111.0308151,2,0,1,1,3,False,False,True,1.3215452666479903,0.26 -43,111.6276932,0,2,1,3,3,False,False,False,0.3664838773548268,0.04 -44,110.7593079,0,2,1,3,3,False,False,False,0.3725433113911357,0.05 -45,115.2292938,0,2,1,3,3,False,False,False,0.35194973888994213,0.04 -46,110.7536316,2,0,0,0,2,False,True,False,1.9516702590941255,0.07 +Name,Csub-Csub,Csub-H,H-O,x10,x11,x2,x5,x7,x8,x9,Target_values_pred,Target_values_pred_sd,Target_values_pred_conformal_hw +38,False,True,False,0,1,110.9270401,89.87553406,1,0,0,1.8446003115453957,0.05695460974052216,0.28563811342591683 +39,True,False,False,0,1,110.6553116,78.65235138,1,0,0,1.9783042852042876,0.21957174869137297,0.28563811342591683 +40,False,False,True,1,1,112.0635071,88.85021973,0,0,1,0.8474099319836889,0.11643334548089342,0.28563811342591683 +41,False,False,False,1,1,112.101181,69.10329437,0,1,0,0.9055940198803172,0.08889719442686804,0.28563811342591683 +42,False,False,True,1,3,111.0308151,84.12934875,2,0,1,1.1873694866030764,0.33795435801976403,0.28563811342591683 +43,False,False,False,3,3,111.6276932,59.93067169,0,2,1,0.34732667926164495,0.04076839771200385,0.28563811342591683 +44,False,False,False,3,3,110.7593079,59.81459808,0,2,1,0.36561300278058884,0.051573745400225655,0.28563811342591683 +45,False,False,False,3,3,115.2292938,70.45233154,0,2,1,0.33608944871517776,0.03424179942124084,0.28563811342591683 +46,False,True,False,0,2,110.7536316,89.6808548,2,0,0,1.9423663707555074,0.03836997209453564,0.28563811342591683 diff --git a/docs/_static/Robert_example_test_NN_PFI.csv b/docs/_static/Robert_example_test_NN_PFI.csv index 604efd1..ca8a20c 100644 --- a/docs/_static/Robert_example_test_NN_PFI.csv +++ b/docs/_static/Robert_example_test_NN_PFI.csv @@ -1,10 +1,10 @@ -Name,x7,x10,x9,Target_values_pred,Target_values_pred_sd -38,1,0,0,1.688580043637469,0.11 -39,1,0,0,1.688580043637469,0.11 -40,0,1,1,0.81434497731962,0.05 -41,0,1,0,0.8725792106127912,0.05 -42,2,1,1,1.5057304860621061,0.11 -43,0,3,1,0.34861059412849515,0.01 -44,0,3,1,0.34861059412849515,0.01 -45,0,3,1,0.34861059412849515,0.01 -46,2,0,0,2.17565631918241,0.06 +Name,x5,x9,x7,Csub-H,Csub-Csub,x10,Target_values_pred,Target_values_pred_sd,Target_values_pred_conformal_hw +38,89.87553406,0,1,True,False,0,1.8499401382712373,0.14564498480510163,0.039293051943657176 +39,78.65235138,0,1,False,True,0,1.917828392322571,0.18463613302714438,0.039293051943657176 +40,88.85021973,1,0,False,False,1,0.8577495753792,0.06814042046701735,0.039293051943657176 +41,69.10329437,0,0,False,False,1,0.9018240078633354,0.06969800102526243,0.039293051943657176 +42,84.12934875,1,2,False,False,1,1.0771879144257182,0.16058500493020844,0.039293051943657176 +43,59.93067169,1,0,False,False,3,0.33743298067439653,0.02292407360492822,0.039293051943657176 +44,59.81459808,1,0,False,False,3,0.33743298067439653,0.022985804002561953,0.039293051943657176 +45,70.45233154,1,0,False,False,3,0.33743298067439653,0.02202919295780639,0.039293051943657176 +46,89.6808548,0,2,True,False,0,1.9409455764777754,0.03949647479654763,0.039293051943657176 diff --git a/environment/env.yaml b/environment/env.yaml index be4c2e7..7460802 100644 --- a/environment/env.yaml +++ b/environment/env.yaml @@ -11,14 +11,19 @@ dependencies: - pango - mscorefonts - libgfortran=14.2.0 + - libgomp - pip - pip: - aqme==2.0.0 - - robert==2.1.0 + - robert==2.2.0 + - numpy==2.3.4 + - pandas==2.3.3 + - scikit-learn==1.7.2 - xgboost==2.1.4 + - bayesian-optimization==3.1.0 - PySide6==6.9.2 - PySide6-Addons==6.9.2 - PySide6-Essentials==6.9.2 - psutil - requests==2.32.5 - - scikit-learn-intelex==2025.2.0 \ No newline at end of file + - scikit-learn-intelex==2025.2.0 diff --git a/pyproject.toml b/pyproject.toml index a330a0a..dbd6993 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ extend-exclude = [ ] [tool.ruff.lint] -per-file-ignores = { "robert/gui_easyrob/easyrob_launcher.py" = ["E402"], "tests/test_easyrob.py" = ["E402"] } +per-file-ignores = { "robert/gui_easyrob/easyrob_launcher.py" = ["E402"], "robert/gui_easyrob/utils/utils_gui.py" = ["F401"], "tests/test_easyrob.py" = ["E402"] } [tool.ruff.format] quote-style = "double" diff --git a/pytest.ini b/pytest.ini index 77f7860..1114863 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,3 +2,7 @@ testpaths = tests python_files = test_*.py qt_api = pyside6 +filterwarnings = + error + ignore::pyparsing.warnings.PyparsingDeprecationWarning + ignore::sklearn.exceptions.ConvergenceWarning diff --git a/robert/aqme.py b/robert/aqme.py index ed4338e..88f405a 100644 --- a/robert/aqme.py +++ b/robert/aqme.py @@ -27,6 +27,7 @@ import time import shutil import sys +import shlex from pathlib import Path import pandas as pd from robert.utils import load_variables, finish_print, load_database @@ -45,6 +46,51 @@ ] +def _expected_robert_csv(descp_lvl, aqme_indv_name): + return f"AQME-ROBERT_{descp_lvl}_{aqme_indv_name}.csv" + + +def _append_aqme_job_logs(log, tail=4000): + for log_path in ( + Path("QDESCP/QDESCP_data.dat"), + Path("CSEARCH/CSEARCH_data.dat"), + ): + if log_path.is_file(): + log.write( + f"\n--- tail {log_path} ---\n" + + log_path.read_text(encoding="utf-8", errors="replace")[-tail:] + ) + + +def _append_aqme_runtime_diagnostics(log): + """Log xTB/PATH context to simplify CI triage when AQME subprocess fails.""" + import shutil + + xtb_exe = shutil.which("xtb") + if xtb_exe: + try: + version = subprocess.run( + [xtb_exe, "--version"], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + version_text = (version.stdout or version.stderr or "").strip() + if len(version_text) > 500: + version_text = version_text[:500] + "..." + log.write(f" xtb executable: {xtb_exe}\n xtb --version: {version_text}") + except (OSError, subprocess.TimeoutExpired) as exc: + log.write(f" xtb executable: {xtb_exe} (version check failed: {exc})") + else: + log.write(" xtb executable: not found on PATH") + + path_preview = os.environ.get("PATH", "") + if len(path_preview) > 800: + path_preview = path_preview[:800] + "..." + log.write(f" PATH: {path_preview}") + + class aqme: """ Class containing all the functions from the AQME module. @@ -112,7 +158,7 @@ def run_csearch_qdescp(self, csv_target, aqme_test=False): f"\nx WARNING! The names provided in the CSV contain * (i.e. {name_csv_indiv}). Please, remove all the * characters." ) self.args.log.finalize() - sys.exit() + sys.exit(1) # find if there is more than one SMILES column in the CSV file for column in csv_df.columns: @@ -142,7 +188,7 @@ def run_csearch_qdescp(self, csv_target, aqme_test=False): # run AQME-QDESCP to generate descriptors cmd_qdescp = [ - "python", + sys.executable, "-u", "-m", "aqme", @@ -155,22 +201,25 @@ def run_csearch_qdescp(self, csv_target, aqme_test=False): f"{aqme_indv_name}.csv", "--nprocs", f"{self.args.nprocs}", + "--sample", + "3", "--robert", ] - _ = self.run_aqme(cmd_qdescp, self.args.qdescp_keywords) + expected_csv = _expected_robert_csv(self.args.descp_lvl, aqme_indv_name) + aqme_success = self.run_aqme( + cmd_qdescp, + self.args.qdescp_keywords, + expected_csv=expected_csv, + ) + if not aqme_success: + self._fail_aqme_job( + "x ROBERT stopped because AQME did not create descriptor " + "CSV output. Please, check the previous AQME warnings." + ) if smi_suffix is not None: # Change column names by adding suffix - try: - df_temp = pd.read_csv( - f"AQME-ROBERT_{self.args.descp_lvl}_{aqme_indv_name}.csv", - encoding="utf-8", - ) - except FileNotFoundError: - self.args.log.write( - "x WARNING! ROBERT stopped due to a problem with the AQME job. Please, check the previous AQME warnings." - ) - sys.exit() + df_temp = pd.read_csv(expected_csv, encoding="utf-8") df_temp.columns = [ f"{col}_{smi_suffix}" if col not in ["code_name", "SMILES"] and col not in aqme_args @@ -252,7 +301,9 @@ def run_csearch_qdescp(self, csv_target, aqme_test=False): self.args.log.write( "\nx The initial AQME descriptor protocol did not create any CSV output!" ) - sys.exit() + _append_aqme_job_logs(self.args.log) + self.args.log.finalize() + sys.exit(1) # remove atomic properties if no SMARTS patterns were selected in qdescp, # and drop AQME argument columns from CSV inputs (single read/write) @@ -267,16 +318,84 @@ def run_csearch_qdescp(self, csv_target, aqme_test=False): # this returns stores options just in case csv_test is included return self - def run_aqme(self, command, extra_keywords): + def _fail_aqme_job(self, message): + self.args.log.write(f"\n{message}") + _append_aqme_job_logs(self.args.log) + self.args.log.finalize() + sys.exit(1) + + def run_aqme(self, command, extra_keywords, *, expected_csv=None): """ Function that runs the AQME jobs """ if extra_keywords != "": - for keyword in extra_keywords.split(): - command.append(keyword) - - subprocess.run(command) + split_args = shlex.split(extra_keywords, posix=(os.name != "nt")) + command.extend(split_args) + + env = os.environ.copy() + py_bin = os.path.dirname(sys.executable) + if os.name == "nt": + win_paths = [ + py_bin, + os.path.join(sys.prefix, "Library", "bin"), + os.path.join(sys.prefix, "Scripts"), + ] + else: + win_paths = [py_bin] + current_path = env.get("PATH", "") + path_entries = current_path.split(os.pathsep) if current_path else [] + for path_dir in win_paths: + if os.path.isdir(path_dir) and path_dir not in path_entries: + path_entries.insert(0, path_dir) + env["PATH"] = os.pathsep.join(path_entries) + + lib = os.path.join(sys.prefix, "lib") + if os.path.isdir(lib): + if sys.platform == "darwin": + var_name = "DYLD_FALLBACK_LIBRARY_PATH" + else: + var_name = "LD_LIBRARY_PATH" + previous = env.get(var_name, "") + entries = previous.split(os.pathsep) if previous else [] + if lib not in entries: + env[var_name] = lib + (os.pathsep + previous if previous else "") + + # Avoid pipe deadlock: nested AQME/CSEARCH can be very verbose on stdout. + result = subprocess.run( + command, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + stderr_tail = (result.stderr or "")[-4000:] + missing_csv = expected_csv is not None and not os.path.isfile(expected_csv) + if result.returncode != 0 or missing_csv: + if result.returncode != 0: + self.args.log.write( + f"\nx AQME subprocess failed with exit code {result.returncode}." + ) + if missing_csv: + self.args.log.write( + f"\nx Expected AQME output not found: {expected_csv}" + ) + self.args.log.write(f" Command: {' '.join(command)}") + if stderr_tail: + self.args.log.write(f" stderr (tail):\n{stderr_tail}") + if ( + "full_level_boltz" in stderr_tail + and "TypeError" in stderr_tail + and "NoneType" in stderr_tail + ): + self.args.log.write( + " x AQME failed while computing Boltzmann properties (None energies). " + "This usually indicates an AQME-side qdescp issue for one or more structures." + ) + _append_aqme_runtime_diagnostics(self.args.log) + _append_aqme_job_logs(self.args.log) + return False + return True def init_aqme(self): """ @@ -289,7 +408,8 @@ def init_aqme(self): self.args.log.write( "x AQME is not installed (required for the --aqme option)! The program is typically installed within 2-5 minutes (https://aqme.readthedocs.io, see the Installation section)" ) - sys.exit() + self.args.log.finalize() + sys.exit(1) def filter_atom_prop_and_aqme_args(aqme_db, csv_df, *, strip_atom_lists): @@ -346,3 +466,6 @@ def move_aqme(): else: os.remove(f"AQME/{file}") shutil.move(file, f"AQME/{file}") + for dat_file in ["AQME/CSEARCH_data.dat", "AQME/QDESCP_data.dat"]: + if not os.path.exists(dat_file): + Path(dat_file).write_text("", encoding="utf-8") diff --git a/robert/generate_utils.py b/robert/generate_utils.py index 005289b..bd6c0b2 100644 --- a/robert/generate_utils.py +++ b/robert/generate_utils.py @@ -5,7 +5,6 @@ import os import shutil import pandas as pd -import numpy as np import glob import json from robert.utils import ( @@ -210,24 +209,27 @@ def detect_best(folder): """ # detect files - file_list = glob.glob(f"{folder}/*.csv") - errors = [] + file_list = sorted(glob.glob(f"{folder}/*.csv")) + candidates = [] for file in file_list: - if "_db" not in file: - results_model = pd.read_csv(f"{file}", encoding="utf-8") - training_error = results_model[ - f"combined_{results_model['error_type'][0]}" - ][0] - errors.append(training_error) - else: - errors.append(np.nan) + if file.endswith("_db.csv"): + continue + results_model = pd.read_csv(file, encoding="utf-8") + training_error = float( + results_model[f"combined_{results_model['error_type'][0]}"][0] + ) + candidates.append((training_error, os.path.basename(file), file)) + if len(candidates) == 0: + raise UnboundLocalError # detect best result and copy files to the Best_model folder if results_model["error_type"][0].lower() in ["mae", "rmse"]: - min_idx = errors.index(np.nanmin(errors)) + _, _, best_name = min(candidates, key=lambda item: (item[0], item[1])) else: - min_idx = errors.index(np.nanmax(errors)) - best_name = file_list[min_idx] - best_db = f"{os.path.dirname(file_list[min_idx])}/{os.path.basename(file_list[min_idx]).split('.csv')[0]}_db.csv" + _, _, best_name = max(candidates, key=lambda item: (item[0], item[1])) + best_db = ( + f"{os.path.dirname(best_name)}/" + f"{os.path.basename(best_name).split('.csv')[0]}_db.csv" + ) shutil.copyfile(f"{best_name}", f"{best_name}".replace("Raw_data", "Best_model")) shutil.copyfile(f"{best_db}", f"{best_db}".replace("Raw_data", "Best_model")) diff --git a/robert/gui_easyrob/main/window.py b/robert/gui_easyrob/main/window.py index b92111e..58b87ca 100644 --- a/robert/gui_easyrob/main/window.py +++ b/robert/gui_easyrob/main/window.py @@ -30,17 +30,12 @@ # ------------------------------------------------------------ # Import resolution (local vs installed package) # ------------------------------------------------------------ -# Try local imports first (portable mode). If they fail, -# fall back to the installed package structure. +# Try installed package imports first (test/package mode). If they fail, +# fall back to local imports for portable execution from source. import webbrowser try: - from version import SOFTWARE_VERSIONS - from utils import utils_gui, molssi_utils - from tabs import predictions, aqme, advanced_options, molssi, results, images - -except ImportError: from robert.gui_easyrob.version import SOFTWARE_VERSIONS from robert.gui_easyrob.utils import utils_gui, molssi_utils from robert.gui_easyrob.tabs import ( @@ -51,6 +46,13 @@ results, images, ) +except ModuleNotFoundError as exc: + if exc.name and exc.name.startswith("robert"): + from version import SOFTWARE_VERSIONS + from utils import utils_gui, molssi_utils + from tabs import predictions, aqme, advanced_options, molssi, results, images + else: + raise # ------------------------------------------------------------ diff --git a/robert/gui_easyrob/tabs/predictions.py b/robert/gui_easyrob/tabs/predictions.py index 3afe41a..4adbeae 100644 --- a/robert/gui_easyrob/tabs/predictions.py +++ b/robert/gui_easyrob/tabs/predictions.py @@ -25,28 +25,28 @@ """ # ------------------------------------------------------------ -# Import resolution (local vs installed package) +# Qt imports # ------------------------------------------------------------ -try: - from utils.utils_gui import ( - QFrame, - QHBoxLayout, - QHeaderView, - QLabel, - QMenu, - QMessageBox, - QSize, - QSizePolicy, - QTabWidget, - QTableView, - QThreadPool, - QVBoxLayout, - QWidget, - Qt, - Signal, - ) +from PySide6.QtCore import QSize, Qt, Signal, QThreadPool +from PySide6.QtWidgets import ( + QFrame, + QHBoxLayout, + QHeaderView, + QLabel, + QMenu, + QMessageBox, + QSizePolicy, + QTabWidget, + QTableView, + QVBoxLayout, + QWidget, +) - from utils.predictions_utils import ( +# ------------------------------------------------------------ +# Import resolution (installed package vs local) +# ------------------------------------------------------------ +try: + from robert.gui_easyrob.utils.predictions_utils import ( LoadCsvTask, NumericSortProxy, PandasTableModel, @@ -60,27 +60,8 @@ find_prediction_csvs, get_robert_report_path, ) - except ImportError: - from robert.gui_easyrob.utils.utils_gui import ( - QFrame, - QHBoxLayout, - QHeaderView, - QLabel, - QMenu, - QMessageBox, - QSize, - QSizePolicy, - QTabWidget, - QTableView, - QThreadPool, - QVBoxLayout, - QWidget, - Qt, - Signal, - ) - - from robert.gui_easyrob.utils.predictions_utils import ( + from utils.predictions_utils import ( LoadCsvTask, NumericSortProxy, PandasTableModel, diff --git a/robert/gui_easyrob/utils/utils_gui.py b/robert/gui_easyrob/utils/utils_gui.py index 155f8b2..b9549ce 100644 --- a/robert/gui_easyrob/utils/utils_gui.py +++ b/robert/gui_easyrob/utils/utils_gui.py @@ -35,6 +35,10 @@ import subprocess import sys import threading +import glob +import re +import shutil +from io import BytesIO from pathlib import Path from importlib.resources import as_file, files @@ -43,6 +47,12 @@ # ------------------------------------------------------------ import pandas as pd import psutil +import fitz +import rdkit +from rdkit import Chem +from rdkit.Chem import Draw, MolsFromCDXMLFile, rdDepictor +from rdkit.Chem.Draw import rdMolDraw2D +from rdkit.Chem.rdmolops import GetMolFrags from ansi2html import Ansi2HTMLConverter @@ -51,24 +61,66 @@ # Qt (PySide6) # ------------------------------------------------------------ from PySide6.QtCore import ( + QObject, + QRunnable, QThread, + QThreadPool, + QTimer, + QUrl, + QEventLoop, + QSize, Qt, Signal, + Slot, ) from PySide6.QtGui import ( + QDesktopServices, + QIcon, + QImage, + QMouseEvent, + QPixmap, QWheelEvent, ) from PySide6.QtWidgets import ( + QApplication, + QCheckBox, QComboBox, + QDialog, QFileDialog, + QFormLayout, QFrame, + QGridLayout, + QGroupBox, + QHBoxLayout, + QInputDialog, QLabel, + QLineEdit, + QListWidget, + QMainWindow, + QMenu, + QMessageBox, + QProgressBar, QPushButton, + QScrollArea, + QSizePolicy, + QSlider, + QStackedWidget, + QStatusBar, + QStyle, + QTabWidget, + QTableWidget, + QTableWidgetItem, + QTableView, + QTextEdit, + QToolButton, QVBoxLayout, + QWidget, ) +from PySide6.QtWebEngineCore import QWebEngineDownloadRequest +from PySide6.QtWebEngineWidgets import QWebEngineView class DropLabel(QFrame): @@ -246,7 +298,9 @@ def read_stderr(): reset_html = self.ansi_converter.convert("\033[0m", full=False) self.output_received.emit(reset_html) - self.process = None + if self.process is not None: + self._close_process_streams(self.process) + self.process = None self.process_finished.emit(-1 if self._stop_requested else exit_code) except Exception as exc: import traceback @@ -258,6 +312,17 @@ def stop(self): """Stop the subprocess and emit a stop signal.""" self.request_stop.emit() + @staticmethod + def _close_process_streams(process): + """Close subprocess pipes to avoid ResourceWarning on GC.""" + for stream in (process.stdout, process.stderr): + if stream is None: + continue + try: + stream.close() + except Exception: + pass + def _handle_stop(self): """Handle the stop signal and terminate the subprocess.""" self._stop_requested = True diff --git a/robert/report.py b/robert/report.py index e995653..494e72a 100644 --- a/robert/report.py +++ b/robert/report.py @@ -41,6 +41,7 @@ adv_sorted_cv, get_col_text, repro_info, + get_repro_ml_stack_versions, make_report, css_content, format_lines, @@ -86,7 +87,7 @@ def __init__(self, **kwargs): print( "\nx The REPORT module requires some libraries that are missing, the PDF with the summary of the results has not been created. Try installing the libraries with 'conda install -y -c conda-forge glib gtk3 pango mscorefonts'" ) - sys.exit() + sys.exit(1) finally: if platform.system() == "Windows": os.dup2(old_stderr, 2) @@ -174,22 +175,34 @@ def __init__(self, **kwargs): # Suppress fontconfig warnings from WeasyPrint on Windows # These warnings come from the C library level, so we need to redirect at OS level - if platform.system() == "Windows": - import tempfile - - # Create a temporary file to redirect stderr - temp_stderr = tempfile.TemporaryFile(mode="w+") - old_stderr = os.dup(2) # Duplicate stderr file descriptor - os.dup2(temp_stderr.fileno(), 2) # Redirect stderr to temp file - - try: + try: + if platform.system() == "Windows": + import tempfile + + # Create a temporary file to redirect stderr + temp_stderr = tempfile.TemporaryFile(mode="w+") + old_stderr = os.dup(2) # Duplicate stderr file descriptor + os.dup2(temp_stderr.fileno(), 2) # Redirect stderr to temp file + + try: + _ = make_report(report_html, HTML) + finally: + os.dup2(old_stderr, 2) # Restore stderr + os.close(old_stderr) + temp_stderr.close() + else: _ = make_report(report_html, HTML) - finally: - os.dup2(old_stderr, 2) # Restore stderr - os.close(old_stderr) - temp_stderr.close() - else: - _ = make_report(report_html, HTML) + except Exception as exc: + print(f"\nx ROBERT_report.pdf could not be created: {exc}") + sys.exit(1) + + pdf_path = Path(os.getcwd()) / "ROBERT_report.pdf" + if not pdf_path.is_file(): + print( + "\nx ROBERT_report.pdf was not written to the working directory " + "(WeasyPrint may have failed silently)." + ) + sys.exit(1) # Remove report.css file os.remove("report.css") @@ -238,7 +251,8 @@ def print_score(self, dat_files, pred_type, eval_only, spacing_PFI): data_score = calc_score(dat_files, suffix, pred_type, data_score) # initial two-column ROBERT score summary - score_info = f"""{spacing}

""" + score_idx = max(0, min(int(data_score[f"robert_score_{suffix}"]), 10)) + score_info = f"""{spacing}

""" columns_score.append( get_col_score(score_info, data_score, suffix, spacing, eval_only) ) @@ -1017,20 +1031,37 @@ def get_repro(self, eval_only): repro_dat += f"""{reduced_line}{space}- External test set ({self.args.csv_test})

""" if aqme_workflow: - try: - path_aqme = Path(f"{os.getcwd()}/AQME/CSEARCH_data.dat") - datfile = open(path_aqme, "r", errors="replace") - outlines = datfile.readlines() - aqme_version = outlines[0].split()[2] - datfile.close() - find_aqme = True - except Exception: - find_aqme = False - aqme_version = "0.0" # dummy number - if ( - int(aqme_version.split(".")[0]) in [0, 1] - and int(aqme_version.split(".")[1]) < 6 + find_aqme = False + aqme_version = "0.0" # dummy number + for path_aqme in ( + Path(f"{os.getcwd()}/AQME/AQME_data.dat"), + Path(f"{os.getcwd()}/AQME/CSEARCH/CSEARCH_data.dat"), + Path(f"{os.getcwd()}/AQME/CSEARCH_data.dat"), ): + try: + with open( + path_aqme, "r", encoding="utf-8", errors="replace" + ) as datfile: + outlines = datfile.readlines() + if not outlines: + continue + parts = outlines[0].split() + if len(parts) >= 3 and parts[0].upper() == "AQME": + aqme_version = parts[2] + elif len(parts) >= 2: + aqme_version = parts[1].lstrip("v") + find_aqme = True + break + except Exception: + continue + try: + version_parts = aqme_version.split(".") + aqme_outdated = ( + int(version_parts[0]) in [0, 1] and int(version_parts[1]) < 6 + ) + except (ValueError, IndexError): + aqme_outdated = False + if aqme_outdated: aqme_updated = False repro_dat += f"""{reduced_line}{space}Warning! This workflow might not be exactly reproducible, update to AQME v1.6.0+ (pip install aqme --upgrade)

""" repro_dat += f"""{reduced_line}{space}To obtain the same results, download the descriptor database (AQME-ROBERT_{self.args.csv_name}) and run:

""" @@ -1119,6 +1150,9 @@ def get_repro(self, eval_only): if find_crest: repro_dat += f"""{reduced_line}{space}- Adjust CREST version: conda install -c conda-forge crest={crest_version})

""" + for display_name, pkg_name, pkg_version in get_repro_ml_stack_versions(): + repro_dat += f"""{reduced_line}{space}- {display_name}: pip install {pkg_name}=={pkg_version}

""" + character_line = "" if self.args.csv_test != "": character_line += "s" @@ -1386,15 +1420,32 @@ def print_img( # keep the ordering (No_PFI in the left, PFI in the right of the PDF) results_images = revert_list(results_images) + results_images = [Path(p).resolve().as_posix() for p in results_images] - # add the graphs width = 100 - - pair_list = f'

' - if not eval_only: - pair_list += f"{(' ') * 22}" - pair_list += f'

' - - html_png = f"{pair_list}" + img_style = ( + "margin: 0; width: 270px; height: {height}px; " + "object-fit: cover; object-position: 0 100%;" + ).format(height=height) + base_style = f"width: {width}%; margin-bottom: -2px; margin-top: {margin_top}px" + + if not results_images: + html_png = ( + f'

Plot not available for {file_name}.

' + ) + elif len(results_images) == 1 or eval_only: + html_png = ( + f'

' + f'' + f"

" + ) + else: + html_png = ( + f'

' + f'' + f"{(' ') * 22}" + f'' + f"

" + ) return html_png diff --git a/robert/report_utils.py b/robert/report_utils.py index 0cc08c1..fa6a297 100644 --- a/robert/report_utils.py +++ b/robert/report_utils.py @@ -1218,6 +1218,46 @@ def calc_penalty_r2(r2_val): return penalty_r2 +REPRO_PACKAGES = [ + ("numpy", "numpy"), + ("pandas", "pandas"), + ("scipy", "scipy"), + ("scikit-learn", "scikit-learn"), + ("xgboost", "xgboost"), + ("bayesian-optimization", "bayesian-optimization"), + ("numba", "numba"), + ("shap", "shap"), + ("rdkit", "rdkit"), + ("PyYAML", "PyYAML"), + ("matplotlib", "matplotlib"), + ("weasyprint", "weasyprint"), +] + +OPTIONAL_REPRO_PACKAGES = [ + ("scikit-learn-intelex", "scikit-learn-intelex"), +] + + +def get_repro_ml_stack_versions(): + """ + Installed versions of core ML/report dependencies for the PDF repro section. + """ + from importlib.metadata import PackageNotFoundError, version as importlib_version + + versions = [] + for display_name, pkg_name in REPRO_PACKAGES: + try: + versions.append((display_name, pkg_name, importlib_version(pkg_name))) + except PackageNotFoundError: + continue + for display_name, pkg_name in OPTIONAL_REPRO_PACKAGES: + try: + versions.append((display_name, pkg_name, importlib_version(pkg_name))) + except PackageNotFoundError: + continue + return versions + + def repro_info(modules): """ Retrieves variables used in the Reproducibility section @@ -1272,7 +1312,7 @@ def make_report(report_html, HTML): print( '\nx ROBERT_report.pdf is open! Please, close the PDF file and run ROBERT again with --report (i.e., "python -m robert --report").' ) - sys.exit() + sys.exit(1) pdf = make_pdf(report_html, HTML, css_files) _ = Path(outfile).write_bytes(pdf) diff --git a/robert/utils.py b/robert/utils.py index 716a388..d673355 100644 --- a/robert/utils.py +++ b/robert/utils.py @@ -28,7 +28,6 @@ from matplotlib.legend_handler import HandlerPatch from matplotlib.ticker import FormatStrFormatter from scipy import stats -from importlib.resources import files # sklearnex was deactivated in ROBERT v2.1 because it only accelerated RF # try: @@ -537,7 +536,8 @@ def load_variables(kwargs, robert_module): ) elif robert_module.upper() == "REPORT": - self.path_icons = files("robert").joinpath("report") + # Filesystem path so WeasyPrint can resolve file:/// assets in CI/wheels. + self.path_icons = (Path(__file__).resolve().parent / "report").as_posix() # sklearnex was deactivated in ROBERT v2.1 because it only accelerated RF # using or not the intelex accelerator might affect the results @@ -1978,7 +1978,7 @@ def generate_lhs_points(pbounds, n_points, random_state=None): Returns: List of dictionaries with parameter values """ - np.random.seed(random_state) + rng = np.random.default_rng(random_state) param_names = list(pbounds.keys()) n_params = len(param_names) @@ -1989,9 +1989,9 @@ def generate_lhs_points(pbounds, n_points, random_state=None): for i in range(n_params): # Create intervals and sample within each intervals = np.linspace(0, 1, n_points + 1) - samples[:, i] = np.random.uniform(intervals[:-1], intervals[1:]) + samples[:, i] = rng.uniform(intervals[:-1], intervals[1:]) # Shuffle to break correlation between dimensions - np.random.shuffle(samples[:, i]) + rng.shuffle(samples[:, i]) # Scale samples to actual parameter bounds initial_points = [] @@ -3588,12 +3588,14 @@ def shap_analysis(self, Xy_data, model_data, path_n_suffix, fitted_model=None): height_shap = 1.2 + min(shap_show) / 4 # explainer = shap.TreeExplainer(loaded_model) # in case the standard version doesn't work + plot_rng = np.random.default_rng(model_data["seed"]) _ = shap.summary_plot( shap_values, Xy_data["X_train_scaled"], max_display=self.args.shap_show, show=False, plot_size=[7.45, height_shap], + rng=plot_rng, ) # set title diff --git a/tests/platform_goldens.py b/tests/platform_goldens.py new file mode 100644 index 0000000..98e9d47 --- /dev/null +++ b/tests/platform_goldens.py @@ -0,0 +1,173 @@ +""" +Platform-specific integration-test goldens (Linux vs Windows CI). + +Values are ROBERT log metrics formatted with :.2. Re-baseline both tables when +sklearn/BO stack changes (.circleci/config.yml conda env). +""" + +from __future__ import annotations + +import math +import re +import sys + +# Tight tolerance: guards parse noise only, not cross-platform drift (separate goldens). +_METRIC_REL_TOL = 0.01 +_METRIC_ABS_TOL = 0.005 + + +def platform_key() -> str: + """``linux`` or ``win32`` (CircleCI Windows uses win32).""" + return "win32" if sys.platform == "win32" else "linux" + + +def log_line_metric_close( + line: str, + prefix: str, + expected: float, + *, + rel_tol: float = _METRIC_REL_TOL, + abs_tol: float = _METRIC_ABS_TOL, +) -> bool: + """True if the first numeric token after ``prefix`` matches ``expected``.""" + idx = line.find(prefix) + if idx == -1: + return False + rest = line[idx + len(prefix) :].lstrip() + m = re.match(r"([-+]?(?:\d*\.\d+|\d+)(?:[eE][-+]?\d+)?)", rest) + if not m: + return False + return math.isclose(float(m.group(1)), expected, rel_tol=rel_tol, abs_tol=abs_tol) + + +# VERIFY standard regression (PFI model section): (CV RMSE, +15% threshold, +30% threshold) +VERIFY_STANDARD_RMSE = { + "linux": (0.29, 0.33, 0.37), + "win32": (0.27, 0.31, 0.35), +} + +# VERIFY standard regression: y_shuffle flawed-model RMSE (:.2 in log) +VERIFY_STANDARD_Y_SHUFFLE_RMSE = { + "linux": 1.0, + "win32": 0.97, +} + +# VERIFY standard regression: sorted 5-fold CV summary line (exact log substring) +VERIFY_STANDARD_SORTED_CV_LINE = { + "linux": ( + "- Sorted 5-fold CV : R2 = [0.0, 0.48, 0.24, 0.07, 0.17], " + "MAE = [0.17, 0.25, 0.09, 0.37, 0.45], " + "RMSE = [0.22, 0.27, 0.11, 0.45, 0.51]" + ), + "win32": ( + "- Sorted 5-fold CV : R2 = [0.25, 0.47, 0.19, 0.07, 0.24], " + "MAE = [0.21, 0.22, 0.09, 0.35, 0.44], " + "RMSE = [0.24, 0.24, 0.1, 0.42, 0.49]" + ), +} + +# GENERATE standard job: log prefix -> expected combined RMSE +GENERATE_STANDARD_RMSE = { + "linux": { + "o Best combined RMSE (target) found in BO for RF (no PFI filter):": 0.64, + "o Combined RMSE for RF (with PFI filter):": 0.77, + "o Best combined RMSE (target) found in BO for GB (no PFI filter):": 0.47, + "o Combined RMSE for GB (with PFI filter):": 0.41, + "o Best combined RMSE (target) found in BO for NN (no PFI filter):": 0.38, + "o Combined RMSE for NN (with PFI filter):": 0.37, + "o Combined RMSE for MVL (no BO needed) (no PFI filter):": 0.47, + "o Combined RMSE for MVL (with PFI filter):": 0.47, + }, + "win32": { + "o Best combined RMSE (target) found in BO for RF (no PFI filter):": 0.64, + "o Combined RMSE for RF (with PFI filter):": 0.77, + "o Best combined RMSE (target) found in BO for GB (no PFI filter):": 0.47, + "o Combined RMSE for GB (with PFI filter):": 0.41, + "o Best combined RMSE (target) found in BO for NN (no PFI filter):": 0.35, + "o Combined RMSE for NN (with PFI filter):": 0.38, + "o Combined RMSE for MVL (no BO needed) (no PFI filter):": 0.47, + "o Combined RMSE for MVL (with PFI filter):": 0.47, + }, +} + + +def assert_verify_standard_y_shuffle_line(line: str) -> None: + """Assert the y_shuffle flawed-model line for standard VERIFY tests.""" + expected = VERIFY_STANDARD_Y_SHUFFLE_RMSE[platform_key()] + assert "o y_shuffle: PASSED" in line + assert f"RMSE = {expected:.2}" in line, ( + f"y_shuffle RMSE line {line!r} vs golden {expected:.2} ({platform_key()})" + ) + + +def assert_verify_standard_rmse_line(line: str) -> None: + """Assert the Original RMSE summary line for standard VERIFY tests.""" + assert "Original RMSE (10x 5-fold CV)" in line + cv_rmse, t15, t30 = VERIFY_STANDARD_RMSE[platform_key()] + pattern = ( + r"Original RMSE \(10x 5-fold CV\)\s+" + r"([-+]?(?:\d*\.\d+|\d+))\s+\+\s+15%\s+&\s+30%\s+threshold\s+=\s+" + r"([-+]?(?:\d*\.\d+|\d+))\s+&\s+([-+]?(?:\d*\.\d+|\d+))" + ) + m = re.search(pattern, line) + assert m, f"Could not parse VERIFY RMSE line: {line!r}" + actual_cv, actual_t15, actual_t30 = (float(m.group(i)) for i in range(1, 4)) + assert math.isclose( + actual_cv, cv_rmse, rel_tol=_METRIC_REL_TOL, abs_tol=_METRIC_ABS_TOL + ), f"CV RMSE {actual_cv} vs golden {cv_rmse} ({platform_key()})" + assert math.isclose( + actual_t15, t15, rel_tol=_METRIC_REL_TOL, abs_tol=_METRIC_ABS_TOL + ), f"15% threshold {actual_t15} vs golden {t15} ({platform_key()})" + assert math.isclose( + actual_t30, t30, rel_tol=_METRIC_REL_TOL, abs_tol=_METRIC_ABS_TOL + ), f"30% threshold {actual_t30} vs golden {t30} ({platform_key()})" + + +def assert_verify_standard_sorted_cv_line(line: str) -> None: + """Assert the sorted 5-fold CV summary line for standard VERIFY tests.""" + expected = VERIFY_STANDARD_SORTED_CV_LINE[platform_key()] + assert expected in line, ( + f"sorted CV line {line!r} vs golden {expected!r} ({platform_key()})" + ) + + +def count_generate_standard_rmse_matches(outlines: list[str]) -> int: + """Count how many standard-job BO RMSE log lines match platform goldens.""" + goldens = GENERATE_STANDARD_RMSE[platform_key()] + count = 0 + for line in outlines: + for prefix, expected in goldens.items(): + if log_line_metric_close(line, prefix, expected): + count += 1 + break + return count + + +def assert_generate_standard_rmse_matches(outlines: list[str]) -> None: + """Assert all eight standard-job BO RMSE log lines match platform goldens.""" + goldens = GENERATE_STANDARD_RMSE[platform_key()] + expected_count = len(goldens) + actual_count = count_generate_standard_rmse_matches(outlines) + if actual_count == expected_count: + return + mismatches = [] + for prefix, golden in goldens.items(): + for line in outlines: + if prefix not in line: + continue + idx = line.find(prefix) + rest = line[idx + len(prefix) :].lstrip() + m = re.match(r"([-+]?(?:\d*\.\d+|\d+)(?:[eE][-+]?\d+)?)", rest) + if not m: + mismatches.append(f"{prefix}: could not parse {line!r}") + break + actual = float(m.group(1)) + if not log_line_metric_close(line, prefix, golden): + mismatches.append(f"{prefix}: log={actual:.2f} golden={golden:.2f}") + break + else: + mismatches.append(f"{prefix}: line not found in GENERATE_data.dat") + raise AssertionError( + f"GENERATE standard RMSE matches {actual_count}/{expected_count} " + f"({platform_key()}): " + "; ".join(mismatches) + ) diff --git a/tests/test_2generate.py b/tests/test_2generate.py index deb847f..9ef240c 100644 --- a/tests/test_2generate.py +++ b/tests/test_2generate.py @@ -9,14 +9,18 @@ import os import sys import glob -import math -import re import pytest import shutil import subprocess import pandas as pd from pathlib import Path from robert.generate import generate +from tests.platform_goldens import ( + GENERATE_STANDARD_RMSE, + assert_generate_standard_rmse_matches, + log_line_metric_close, + platform_key, +) # saves the working directory (os.path.join keeps paths valid on Windows and Linux) path_main = os.getcwd() @@ -35,6 +39,82 @@ def _read_best_model_pair(best_dir): ) +def _model_tag(test_job): + """ML model name for single-model GENERATE test jobs.""" + return { + "reduced_gp": "GP", + "reduced_adab": "ADAB", + "reduced_xgb": "XGB", + "reduced_vr": "VR", + "reduced_vr_clas": "VR", + "reduced_clas": "RF", + }.get(test_job, "RF") + + +def _read_model_pair(generate_dir, folder, model, *, pfi=False): + """Load params and *_db.csv for a specific model under Raw_data.""" + suffix = f"{model}_PFI" if pfi else model + params_path = os.path.join(generate_dir, "Raw_data", folder, f"{suffix}.csv") + db_path = os.path.join(generate_dir, "Raw_data", folder, f"{suffix}_db.csv") + assert os.path.isfile(params_path), params_path + assert os.path.isfile(db_path), db_path + return pd.read_csv(params_path, encoding="utf-8"), pd.read_csv( + db_path, encoding="utf-8" + ) + + +# Expected descriptors per model for the standard multi-model job +_STANDARD_MODEL_DESCS = { + "No_PFI": { + "RF": ["x10", "x2", "x5", "x7", "x9"], + "GB": [ + "Csub-Csub", + "Csub-H", + "H-O", + "x10", + "x11", + "x2", + "x5", + "x7", + "x8", + "ynoise", + ], + "NN": [ + "Csub-Csub", + "Csub-H", + "H-O", + "x10", + "x11", + "x2", + "x5", + "x7", + "x8", + "x9", + "ynoise", + ], + "MVL": [ + "Csub-Csub", + "Csub-H", + "H-O", + "x10", + "x11", + "x2", + "x5", + "x7", + "x8", + "x9", + "ynoise", + ], + }, + "PFI": { + "RF": ["x10", "x7"], + "GB": ["x10", "x7"], + "NN": ["x7", "x10", "x9", "x5", "x2", "x8"], + "MVL": ["x7", "Csub-Csub", "x9", "x10", "H-O", "Csub-H"], + }, +} + + def _parse_x_descriptors(raw): """Parse X_descriptors from GENERATE CSV (JSON or Python literal list).""" text = raw if isinstance(raw, str) else str(raw) @@ -44,22 +124,6 @@ def _parse_x_descriptors(raw): return ast.literal_eval(text) -def _log_line_metric_close(line, prefix, expected, *, rel_tol=0.05, abs_tol=0.02): - """ - True if line contains prefix and the numeric token immediately after prefix - matches expected within tolerance (robust to OS/library float drift and :.2 - log formatting). - """ - idx = line.find(prefix) - if idx == -1: - return False - rest = line[idx + len(prefix) :].lstrip() - m = re.match(r"([-+]?(?:\d*\.\d+|\d+)(?:[eE][-+]?\d+)?)", rest) - if not m: - return False - return math.isclose(float(m.group(1)), expected, rel_tol=rel_tol, abs_tol=abs_tol) - - # GENERATE tests @pytest.mark.parametrize( "test_job", @@ -204,16 +268,20 @@ def test_GENERATE(test_job): ): finding_line += 0.5 # it appears two times, in PFI and no PFI # this elif adds 4 points to the standard test (0.5*2(PFI and no PFI)*4(models) - elif _log_line_metric_close( + elif log_line_metric_close( line, "o Best combined RMSE (target) found in BO for RF (no PFI filter):", - 0.55, + 0.45, + rel_tol=0.05, + abs_tol=0.02, ): reproducibility += 1 - elif _log_line_metric_close( + elif log_line_metric_close( line, "o Combined RMSE for RF (with PFI filter):", - 0.57, + 0.41, + rel_tol=0.05, + abs_tol=0.02, ): reproducibility += 1 # lines only for standard @@ -225,55 +293,11 @@ def test_GENERATE(test_job): finding_line += 1 elif "- 4/4 - ML model: MVL" in line: finding_line += 1 - elif _log_line_metric_close( - line, - "o Best combined RMSE (target) found in BO for RF (no PFI filter):", - 0.49, + elif any( + log_line_metric_close(line, prefix, expected) + for prefix, expected in GENERATE_STANDARD_RMSE[platform_key()].items() ): reproducibility += 1 - elif _log_line_metric_close( - line, - "o Combined RMSE for RF (with PFI filter):", - 0.5, - ): - reproducibility += 1 - elif _log_line_metric_close( - line, - "o Best combined RMSE (target) found in BO for GB (no PFI filter):", - 0.41, - ): - reproducibility += 1 - elif _log_line_metric_close( - line, - "o Combined RMSE for GB (with PFI filter):", - 0.38, - ): - reproducibility += 1 - elif _log_line_metric_close( - line, - "o Best combined RMSE (target) found in BO for NN (no PFI filter):", - 0.33, - ): - reproducibility += 1 - elif _log_line_metric_close( - line, - "o Combined RMSE for NN (with PFI filter):", - 0.41, - ): - reproducibility += 1 - elif _log_line_metric_close( - line, - "o Combined RMSE for MVL (no BO needed) (no PFI filter):", - 0.47, - ): - reproducibility += 1 - elif _log_line_metric_close( - line, - "o Combined RMSE for MVL (with PFI filter):", - 0.47, - ): - reproducibility += 1 - # lines only for elif "1. 50% = RMSE from a 5x repeated 10-fold CV (interpoplation)" in line: finding_changed_kfold += 1 @@ -288,7 +312,8 @@ def test_GENERATE(test_job): assert reproducibility == 0 if test_job == "standard": assert finding_line == 11 - assert reproducibility == 8 + assert_generate_standard_rmse_matches(outlines) + assert reproducibility == len(GENERATE_STANDARD_RMSE[platform_key()]) if test_job == "reduced_kfold": assert finding_changed_kfold == 1 @@ -309,10 +334,11 @@ def test_GENERATE(test_job): ) assert expected_amount == len(csv_amount) - params_best, db_best = _read_best_model_pair( - os.path.join(path_generate, "Best_model", folder) - ) - if test_job in [ + _read_best_model_pair(os.path.join(path_generate, "Best_model", folder)) + + if test_job == "standard": + model_descs = _STANDARD_MODEL_DESCS[folder] + elif test_job in [ "reduced", "reduced_cmd", "reduced_PFImax", @@ -320,8 +346,8 @@ def test_GENERATE(test_job): "reduced_adab", "reduced_xgb", "reduced_clas", - "standard", ]: + model = _model_tag(test_job) if folder == "No_PFI": if test_job in ["reduced", "reduced_cmd", "reduced_PFImax"]: desc_list = ["x10", "x2", "x5", "x7", "x9"] @@ -365,20 +391,6 @@ def test_GENERATE(test_job): "x8", "x9", ] - elif test_job == "standard": - desc_list = [ - "Csub-Csub", - "Csub-H", - "H-O", - "x10", - "x11", - "x2", - "x5", - "x7", - "x8", - "x9", - "ynoise", - ] elif folder == "PFI": if test_job in ["reduced", "reduced_cmd"]: desc_list = ["x7", "x10"] @@ -387,15 +399,21 @@ def test_GENERATE(test_job): elif test_job == "reduced_gp": desc_list = ["x5", "Csub-Csub", "x7", "x10", "x8", "Csub-H"] elif test_job == "reduced_xgb": - desc_list = ["x10", "x7"] + desc_list = ["x10", "x5", "x7"] elif test_job == "reduced_adab": desc_list = ["x10", "x9"] elif test_job == "reduced_clas": desc_list = ["x6"] - elif test_job == "standard": - desc_list = ["x10", "x7"] - if test_job in ["reduced", "reduced_cmd"]: - # check set splits + model_descs = {model: desc_list} + else: + model_descs = {} + + for model, desc_list in model_descs.items(): + params_best, db_best = _read_model_pair( + path_generate, folder, model, pfi=(folder == "PFI") + ) + + if test_job in ["reduced", "reduced_cmd"] and model == "RF": expected_sets = [ "Training", "Training", @@ -405,7 +423,6 @@ def test_GENERATE(test_job): ] for i, expected_set in enumerate(expected_sets): assert db_best["Set"][i] == expected_set - # check whether the values are sorted (for reproducibility) expected_rows = [ 0.308776518, 0.321084552, @@ -416,7 +433,6 @@ def test_GENERATE(test_job): assert list( db_best["Target_values"][: len(expected_rows)] ) == pytest.approx(expected_rows, rel=1e-5, abs=1e-8) - # check whether the columns are sorted (for reproducibility) expected_cols = ["x10", "x2", "x5", "x7", "x9"] for i, expected_col in enumerate(expected_cols): assert db_best.columns[i] == expected_col @@ -449,7 +465,7 @@ def test_GENERATE(test_job): if test_job == "reduced_clas": assert params_best["split"][0].upper() == "RND" - else: + elif test_job != "standard": assert params_best["split"][0].upper() == "EVEN" # check that the heatmap plots were generated diff --git a/tests/test_3verify.py b/tests/test_3verify.py index 96828dd..192bf5f 100644 --- a/tests/test_3verify.py +++ b/tests/test_3verify.py @@ -16,6 +16,11 @@ clas_generate_layout, restore_regression_generate_layout, ) +from tests.platform_goldens import ( + assert_verify_standard_rmse_line, + assert_verify_standard_sorted_cv_line, + assert_verify_standard_y_shuffle_line, +) # VERIFY tests @@ -66,7 +71,7 @@ def _run_verify(test_job, repo_root, path_verify): results_line = True if test_job == "clas": assert ( - "Original MCC (10x 5-fold CV) 0.63 - 15% & 30% threshold = 0.53 & 0.44" + "Original MCC (10x 5-fold CV) 0.67 - 15% & 30% threshold = 0.57 & 0.47" in outlines[i + 1] ) assert ( @@ -74,29 +79,23 @@ def _run_verify(test_job, repo_root, path_verify): in outlines[i + 2] ) assert ( - "o y_shuffle: PASSED, MCC = 0.042, lower than thresholds" + "o y_shuffle: PASSED, MCC = -0.014, lower than thresholds" in outlines[i + 3] ) assert ( - "o onehot: PASSED, MCC = -0.034, lower than thresholds" + "o onehot: PASSED, MCC = 0.0, lower than thresholds" in outlines[i + 4] ) assert ( - "- Sorted CV : Accuracy = [0.83, 0.83, 1.0, 0.67, 0.6], F1 score = [0.86, 0.86, 1.0, 0.67, 0.5], MCC = [0.71, 0.71, 1.0, 0.5, 0.41]" + "- Sorted CV : Accuracy = [0.67, 0.83, 1.0, 0.67, 0.6], F1 score = [0.75, 0.86, 1.0, 0.67, 0.5], MCC = [0.45, 0.71, 1.0, 0.5, 0.41]" in outlines[i + 5] ) - elif test_job == "standard": - assert ( - "Original RMSE (10x 5-fold CV) 0.24 + 15% & 30% threshold = 0.28 & 0.31" - in outlines[i + 1] - ) + elif test_job in ["standard", "standard_cmd"]: + assert_verify_standard_rmse_line(outlines[i + 1]) assert "o y_mean: PASSED, RMSE = 0.7" in outlines[i + 2] - assert "o y_shuffle: PASSED, RMSE = 0.84" in outlines[i + 3] - assert "- onehot: UNCLEAR, RMSE = 0.3" in outlines[i + 4] - assert ( - "- Sorted 5-fold CV : R2 = [0.0, 0.54, 0.0, 0.42, 0.2], MAE = [0.31, 0.15, 0.04, 0.36, 0.46], RMSE = [0.32, 0.2, 0.05, 0.43, 0.51]" - in outlines[i + 5] - ) + assert_verify_standard_y_shuffle_line(outlines[i + 3]) + assert "x onehot: FAILED, RMSE = 0.3" in outlines[i + 4] + assert_verify_standard_sorted_cv_line(outlines[i + 5]) break assert results_line diff --git a/tests/test_5aqme_n_full.py b/tests/test_5aqme_n_full.py index 1c1905b..f4c040e 100644 --- a/tests/test_5aqme_n_full.py +++ b/tests/test_5aqme_n_full.py @@ -11,6 +11,7 @@ import shutil import subprocess import pandas as pd +from pathlib import Path # saves the working directory path_main = os.getcwd() @@ -26,6 +27,59 @@ def _aqme_installed() -> bool: return False +def _aqme_log_hints() -> str: + log_hints = [] + for log_path in ( + Path(path_main) / "AQME" / "AQME_data.dat", + Path(path_main) / "QDESCP" / "QDESCP_data.dat", + Path(path_main) / "CSEARCH" / "CSEARCH_data.dat", + Path(path_main) / "GENERATE" / "GENERATE_data.dat", + Path(path_main) / "CURATE" / "CURATE_data.dat", + ): + if log_path.is_file(): + log_hints.append( + f"--- tail {log_path} ---\n" + + log_path.read_text(encoding="utf-8", errors="replace")[-4000:] + ) + return "\n".join(log_hints) if log_hints else "(no ROBERT .dat logs found)" + + +def _aqme_runtime_context() -> str: + """Collect xTB/AQME artifact hints when a subprocess-based AQME test fails.""" + lines = [] + xtb_exe = shutil.which("xtb") + lines.append(f"xtb on PATH: {xtb_exe or '(not found)'}") + if xtb_exe: + try: + version = subprocess.run( + [xtb_exe, "--version"], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + version_text = (version.stdout or version.stderr or "").strip() + if len(version_text) > 400: + version_text = version_text[:400] + "..." + lines.append(f"xtb --version: {version_text}") + except (OSError, subprocess.TimeoutExpired) as exc: + lines.append(f"xtb --version check failed: {exc}") + + for pattern in ( + "AQME-ROBERT_*.csv", + "AQME_indiv*.csv", + "QDESCP/**/*.json", + "CSEARCH/*.sdf", + ): + matches = sorted(glob.glob(f"{path_main}/{pattern}", recursive=True)) + if matches: + lines.append(f"{pattern}: {', '.join(matches[:8])}") + if len(matches) > 8: + lines.append(f" ... and {len(matches) - 8} more") + + return "\n".join(lines) if lines else "(no AQME runtime context)" + + # AQME and full workflow tests @pytest.mark.parametrize( "test_job", @@ -51,6 +105,8 @@ def test_AQME(test_job): "PREDICT", "VERIFY", "AQME", + "CSEARCH", + "QDESCP", ] for folder in folders: if os.path.exists(f"{path_main}/{folder}"): @@ -64,9 +120,12 @@ def test_AQME(test_job): "Robert_example.csv", "solubility.csv", "solubility_solvent.csv", + "AQME_indiv.csv", ]: if os.path.exists(f"{path_main}/{file}"): os.remove(f"{path_main}/{file}") + for file in glob.glob(f"{path_main}/AQME_indiv_*.csv"): + os.remove(file) # runs the program with the different tests if test_job in ["full_workflow", "full_workflow_test"]: @@ -132,11 +191,14 @@ def test_AQME(test_job): if test_job in ["full_clas", "full_clas_test"]: cmd_robert = cmd_robert + ["--type", "clas"] + if test_job in ("aqme", "2smiles_columns"): + cmd_robert = cmd_robert + ["--nprocs", "1"] + if test_job == "aqme": cmd_robert = cmd_robert + [ "--aqme", "--qdescp_keywords", - "--qdescp_atoms ['C'] --qdescp_acc 5 --qdescp_opt normal", + '--qdescp_atoms ["C"] --qdescp_acc 5 --qdescp_opt normal', "--alpha", "0.5", ] @@ -144,11 +206,53 @@ def test_AQME(test_job): if test_job == "2smiles_columns": cmd_robert = cmd_robert + ["--aqme", "--alpha", "0.5"] - subprocess.run(cmd_robert) + # Logger.print() writes to stdout; capturing it can fill the pipe buffer on + # long AQME workflows and block ROBERT before REPORT finishes on CI. + env = os.environ.copy() + py_bin = os.path.dirname(sys.executable) + path_entries = env.get("PATH", "").split(os.pathsep) if env.get("PATH") else [] + if sys.platform == "win32": + for path_dir in ( + py_bin, + os.path.join(sys.prefix, "Library", "bin"), + os.path.join(sys.prefix, "Scripts"), + ): + if os.path.isdir(path_dir) and path_dir not in path_entries: + path_entries.insert(0, path_dir) + env["PATH"] = os.pathsep.join(path_entries) + else: + lib = os.path.join(sys.prefix, "lib") + if os.path.isdir(lib): + prev = env.get("LD_LIBRARY_PATH", "") + if lib not in prev.split(os.pathsep): + env["LD_LIBRARY_PATH"] = lib + (os.pathsep + prev if prev else "") + + completed = subprocess.run( + cmd_robert, + cwd=path_main, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + stderr_tail = (completed.stderr or "")[-8000:] + if completed.returncode != 0: + runtime_ctx = "" + if test_job in ("aqme", "2smiles_columns"): + runtime_ctx = f"\n{_aqme_runtime_context()}\n" + pytest.fail( + f"ROBERT subprocess failed (exit {completed.returncode}):\n" + f"stderr:\n{stderr_tail}{runtime_ctx}{_aqme_log_hints()}" + ) # check that all the plots, CSV and DAT files are created - # find ROBERT_report.pdf - assert os.path.exists(f"{path_main}/ROBERT_report.pdf") + pdf_path = Path(path_main) / "ROBERT_report.pdf" + debug_path = Path(path_main) / "report_debug.txt" + assert pdf_path.is_file(), ( + "ROBERT_report.pdf missing after ROBERT exited 0. " + f"report_debug.txt exists={debug_path.is_file()}. " + f"stderr tail:\n{stderr_tail[-4000:]}" + ) # CURATE folder if ( @@ -360,7 +464,7 @@ def test_AQME(test_job): if test_job == "full_workflow": # model summary, robert score, predict graphs and model metrics assert robert_score[0] == "5" - assert robert_score[1] == "6" + assert robert_score[1] == "7" assert ml_model_count == 2 assert partition_count == 2 assert points_desc[0] == "30:5" @@ -375,8 +479,8 @@ def test_AQME(test_job): # advanced analysis, predictive ability section 2 assert "1 / 2" in pred_ability[0] assert "report/score_w_2_1.jpg" in pred_ability[0] - assert "1 / 2" in pred_ability[1] - assert "report/score_w_2_1.jpg" in pred_ability[1] + assert "2 / 2" in pred_ability[1] + assert "report/score_w_2_2.jpg" in pred_ability[1] # advanced analysis, predictive ability of external test set section 3a assert "2 / 2" in pred_test_ability[0] assert "report/score_w_2_2.jpg" in pred_test_ability[0] @@ -408,11 +512,11 @@ def test_AQME(test_job): elif test_job == "full_clas": # model summary, robert score, predict graphs and model metrics - assert robert_score[0] == "4" - assert robert_score[1] == "6" + assert robert_score[0] == "6" + assert robert_score[1] == "3" assert ml_model_count == 2 assert points_desc[0] == "29:6" - assert points_desc[1] == "29:4" + assert points_desc[1] == "29:2" # advanced analysis, flawed models section 1 assert "-2 / 0" in flawed_models[0] assert "-2 / 0" in flawed_models[1] @@ -420,21 +524,21 @@ def test_AQME(test_job): # advanced analysis, predictive ability section 2 assert "2 / 3" in pred_ability[0] assert "report/score_w_3_2.jpg" in pred_ability[0] - assert "2 / 3" in pred_ability[1] - assert "report/score_w_3_2.jpg" in pred_ability[1] + assert "1 / 3" in pred_ability[1] + assert "report/score_w_3_1.jpg" in pred_ability[1] # advanced analysis, predictive ability of external test set section 3a assert "3 / 3" in pred_test_ability[0] assert "report/score_w_3_3.jpg" in pred_test_ability[0] - assert "3 / 3" in pred_test_ability[1] - assert "report/score_w_3_3.jpg" in pred_test_ability[1] + assert "1 / 3" in pred_test_ability[1] + assert "report/score_w_3_1.jpg" in pred_test_ability[1] # advanced analysis, predictive ability of CV vs test section 3b assert "1 / 2" in cv_vs_test_models[0] assert "report/score_w_2_1.jpg" in cv_vs_test_models[0] assert "2 / 2" in cv_vs_test_models[1] assert "report/score_w_2_2.jpg" in cv_vs_test_models[1] - # advanced analysis, extrapolation section 3d - assert "0 / 2" in extrapol_ability_clas[0] - assert "report/score_w_2_0.jpg" in extrapol_ability_clas[0] + # advanced analysis, consistency (sorted CV) section 3c + assert "2 / 2" in extrapol_ability_clas[0] + assert "report/score_w_2_2.jpg" in extrapol_ability_clas[0] assert "1 / 2" in extrapol_ability_clas[1] assert "report/score_w_2_1.jpg" in extrapol_ability_clas[1] # y distribution and Pearson images diff --git a/tests/test_report_utils.py b/tests/test_report_utils.py new file mode 100644 index 0000000..74989d6 --- /dev/null +++ b/tests/test_report_utils.py @@ -0,0 +1,16 @@ +from importlib.metadata import PackageNotFoundError +from unittest.mock import patch + +from robert.report_utils import get_repro_ml_stack_versions + + +def test_get_repro_ml_stack_versions_skips_missing_packages(): + def fake_version(pkg): + if pkg == "numpy": + return "2.3.4" + raise PackageNotFoundError(pkg) + + with patch("importlib.metadata.version", side_effect=fake_version): + versions = get_repro_ml_stack_versions() + + assert versions == [("numpy", "numpy", "2.3.4")]