diff --git a/.gitignore b/.gitignore index 39fd2f1..5c472ef 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ outputs/checkpoints/ *.pt *.pth *.onnx +*.zip __pycache__/ .venv/ diff --git a/MODEL_CARD.md b/MODEL_CARD.md index 6917f5f..275cfc6 100644 --- a/MODEL_CARD.md +++ b/MODEL_CARD.md @@ -64,7 +64,16 @@ The model has not been clinically validated and should not be used in real medic ## Dataset -The project expects a chest X-ray dataset organized into train, validation, and test folders: +This project uses the public [Chest X-Ray Images (Pneumonia)](https://www.kaggle.com/datasets/paultimothymooney/chest-xray-pneumonia/data) dataset available on Kaggle. + +The dataset contains chest X-ray images categorized into two classes: + +* `NORMAL` +* `PNEUMONIA` + +The dataset is organized into training, validation, and test folders. It is not included in this repository. Users must download it separately from Kaggle and follow the dataset’s licensing and usage terms. + +Expected local structure: ```text data/ @@ -80,8 +89,6 @@ data/ └── PNEUMONIA/ ``` -The dataset is not included in this repository. Users must download the dataset separately and ensure that they comply with the dataset license and usage terms. - --- ## Training Procedure @@ -129,35 +136,32 @@ The model is evaluated on the test set using: --- -## Current Smoke-Test Results +## Current GPU Baseline Results -The following results are from a local CPU smoke-test run using `ResNet18`. +The following results are from the `v1.1.0` GPU-trained baseline using `ResNet50`. | Metric | Value | | ----------- | ----: | -| Accuracy | 0.840 | -| ROC-AUC | 0.941 | -| PR-AUC | 0.962 | -| Brier Score | 0.124 | +| Accuracy | 0.902 | +| ROC-AUC | 0.971 | +| PR-AUC | 0.975 | +| Brier Score | 0.078 | +| Test Loss | 0.457 | Confusion matrix: ```text -[[145, 89], - [ 11, 379]] +[[176, 58], + [ 3, 387]] ``` -These results are useful for verifying that the full pipeline works, but they should not be interpreted as final model performance. +The model was trained and evaluated using the project’s GPU configuration: -A stronger final evaluation should include: +```text +configs/gpu_resnet50_colab.yaml +``` -* full GPU training -* larger number of epochs -* repeated runs with multiple random seeds -* confidence intervals -* external validation data -* subgroup analysis if metadata is available -* calibration analysis +These results are useful as a stronger baseline than the initial CPU smoke test, but they should not be interpreted as clinical validation. --- diff --git a/README.md b/README.md index 2d67702..2d0d89c 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,24 @@ MedicalImageClassifier/ ## Dataset -This project expects the chest X-ray dataset to be organized as follows: +This project uses the public [Chest X-Ray Images (Pneumonia)](https://www.kaggle.com/datasets/paultimothymooney/chest-xray-pneumonia/data) dataset available on Kaggle. + +Dataset reference: + +* **Dataset:** Chest X-Ray Images (Pneumonia) +* **Kaggle uploader:** Paul Mooney +* **Task:** Binary chest X-ray classification +* **Classes:** `NORMAL`, `PNEUMONIA` +* **Image format:** JPEG chest X-ray images +* **Expected structure:** train, validation, and test folders + +The dataset is not included in this repository. Users should download it separately from Kaggle and place it under: + +```text +data/chest_xray/ +``` + +Expected folder structure: ```text data/ @@ -120,13 +137,7 @@ data/ └── PNEUMONIA/ ``` -The dataset is not included in this repository. Users should download the chest X-ray pneumonia dataset separately and place it under: - -```text -data/chest_xray/ -``` - -The dataset folder is ignored by Git to avoid uploading large files and to respect dataset licensing. +The dataset is excluded from GitHub to avoid uploading large files and to respect the dataset’s original distribution and licensing terms. --- @@ -277,21 +288,22 @@ The model is evaluated using several metrics: ## Current Results -The following results are from a local CPU smoke-test run using `ResNet18`. -They demonstrate that the full training and evaluation pipeline works. +The following results are from the `v1.1.0` GPU-trained baseline using `ResNet50`. -| Model | Accuracy | ROC-AUC | PR-AUC | Brier Score | -| ------------------ | -------: | ------: | -----: | ----------: | -| ResNet18 smoke run | 0.840 | 0.941 | 0.962 | 0.124 | +| Model | Accuracy | ROC-AUC | PR-AUC | Brier Score | +| --------------------- | -------: | ------: | -----: | ----------: | +| ResNet50 GPU baseline | 0.902 | 0.971 | 0.975 | 0.078 | -Confusion matrix from the smoke-test run: +Confusion matrix from the test set: ```text -[[145, 89], - [ 11, 379]] +[[176, 58], + [ 3, 387]] ``` -These results should not be interpreted as final clinical performance. A full benchmark should be run with a stronger configuration, repeated seeds, and a GPU-trained model. +This baseline improves on the initial CPU smoke-test run and demonstrates that the full training, evaluation, explainability, and reporting workflow can run on a GPU-backed environment. + +These results should still not be interpreted as clinical performance. A clinically meaningful evaluation would require external validation, repeated runs, subgroup analysis where metadata is available, expert review, and prospective validation. --- diff --git a/configs/gpu_resnet50_colab.yaml b/configs/gpu_resnet50_colab.yaml new file mode 100644 index 0000000..fcd2133 --- /dev/null +++ b/configs/gpu_resnet50_colab.yaml @@ -0,0 +1,43 @@ +seed: 42 + +paths: + data_dir: data/chest_xray + outputs_dir: outputs + ckpt_dir: outputs/checkpoints + figures_dir: outputs/figures + logs_dir: outputs/logs + metrics_dir: outputs/metrics + +training: + batch_size: 32 + num_workers: 2 + epochs: 10 + lr: 0.0001 + weight_decay: 0.0001 + warmup_epochs: 1 + mixed_precision: true + early_stopping_patience: 3 + class_weighted_loss: true + use_focal_loss: false + scheduler: cosine + +model: + arch: resnet50 + pretrained: true + num_classes: 2 + +augment: + image_size: 224 + hflip: 0.5 + rotation: 7 + brightness: 0.1 + contrast: 0.1 + +loss: + name: weighted_cross_entropy + +metrics: + - accuracy + - roc_auc + - pr_auc + - brier_score \ No newline at end of file diff --git a/outputs/figures/confusion_matrix.png b/outputs/figures/confusion_matrix.png index d1adea6..5f1233c 100644 Binary files a/outputs/figures/confusion_matrix.png and b/outputs/figures/confusion_matrix.png differ diff --git a/outputs/figures/gradcam_false_negative.png b/outputs/figures/gradcam_false_negative.png index fc5933f..08d99bd 100644 Binary files a/outputs/figures/gradcam_false_negative.png and b/outputs/figures/gradcam_false_negative.png differ diff --git a/outputs/figures/gradcam_false_positive.png b/outputs/figures/gradcam_false_positive.png index d879b92..66b282e 100644 Binary files a/outputs/figures/gradcam_false_positive.png and b/outputs/figures/gradcam_false_positive.png differ diff --git a/outputs/figures/gradcam_true_negative.png b/outputs/figures/gradcam_true_negative.png index a42833c..0f1e9e8 100644 Binary files a/outputs/figures/gradcam_true_negative.png and b/outputs/figures/gradcam_true_negative.png differ diff --git a/outputs/figures/gradcam_true_positive.png b/outputs/figures/gradcam_true_positive.png index f0840d1..8ca5822 100644 Binary files a/outputs/figures/gradcam_true_positive.png and b/outputs/figures/gradcam_true_positive.png differ diff --git a/outputs/figures/pr_test.png b/outputs/figures/pr_test.png index 491e3fc..f92861d 100644 Binary files a/outputs/figures/pr_test.png and b/outputs/figures/pr_test.png differ diff --git a/outputs/figures/roc_test.png b/outputs/figures/roc_test.png index 9dc3537..c1aca85 100644 Binary files a/outputs/figures/roc_test.png and b/outputs/figures/roc_test.png differ diff --git a/outputs/metrics/test_metrics.json b/outputs/metrics/test_metrics.json index 89672dc..bddd2dd 100644 --- a/outputs/metrics/test_metrics.json +++ b/outputs/metrics/test_metrics.json @@ -1,19 +1,19 @@ { - "accuracy": 0.8397435897435898, - "auc": 0.9405873328950254, - "pr_auc": 0.9620353256109125, - "brier": 0.12366715347033523, + "accuracy": 0.9022435897435898, + "auc": 0.9713894367740522, + "pr_auc": 0.9754793528981164, + "brier": 0.07833218747240564, "confusion_matrix": [ [ - 145, - 89 + 176, + 58 ], [ - 11, - 379 + 3, + 387 ] ], - "test_loss": 0.41938387189377235, - "model": "resnet18", + "test_loss": 0.4571893001672997, + "model": "resnet50", "threshold": 0.5 } \ No newline at end of file diff --git a/outputs/reports/ml_audit_report.md b/outputs/reports/ml_audit_report.md index b0c3c81..cc3d805 100644 --- a/outputs/reports/ml_audit_report.md +++ b/outputs/reports/ml_audit_report.md @@ -1,6 +1,6 @@ # Multi-Agent ML Audit Report -Generated: 2026-07-01T17:25:33 +Generated: 2026-07-02T09:06:57 This report is generated by a lightweight multi-agent audit workflow. The agents do not make clinical decisions. They check engineering, evaluation, documentation, and explainability artifacts. @@ -47,22 +47,22 @@ No warnings detected. "metrics_path": "outputs/metrics/test_metrics.json", "exists": true, "metrics": { - "accuracy": 0.8397435897435898, - "auc": 0.9405873328950254, - "pr_auc": 0.9620353256109125, - "brier": 0.12366715347033523, + "accuracy": 0.9022435897435898, + "auc": 0.9713894367740522, + "pr_auc": 0.9754793528981164, + "brier": 0.07833218747240564, "confusion_matrix": [ [ - 145, - 89 + 176, + 58 ], [ - 11, - 379 + 3, + 387 ] ], - "test_loss": 0.41938387189377235, - "model": "resnet18", + "test_loss": 0.4571893001672997, + "model": "resnet50", "threshold": 0.5 }, "warnings": []