diff --git a/.github/workflows/azure-deploy.yml b/.github/workflows/azure-deploy.yml new file mode 100644 index 00000000..ad75a864 --- /dev/null +++ b/.github/workflows/azure-deploy.yml @@ -0,0 +1,82 @@ +name: Deploy to Azure App Service + +on: + push: + branches: + - master + - release-* + workflow_dispatch: #Allow manual triggers + +permissions: + contents: read + +jobs: + build-and-test: + name: Build and Test + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: recursive # Include cdisc-json-validation submodule + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: '3.13' + cache: 'pip' + cache-dependency-path: | + pyproject.toml + requirements.txt + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + + - name: Run tests + run: | + pytest -q tests --disable-warnings --tb=short + env: + CDISC_CONCEPTS_JSON: '[]' # Bypass CDISC API for tests + + - name: Create deployment package + run: | + # Nothing special needed - Azure will handle pip install + echo "Build successful" + + deploy: + name: Deploy to Azure + needs: build-and-test + runs-on: ubuntu-latest + environment: production # Optional: requires manual approval + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Azure login + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + - name: Stop Azure Web App + uses: azure/cli@v1 + with: + azcliversion: 2.36.0 + inlineScript: | + az webapp stop --name ${{ secrets.AZURE_WEBAPP_NAME }} --resource-group cdisc-soa-workbench-rg + + - name: Deploy to Azure Web App + uses: azure/webapps-deploy@v2 + with: + app-name: ${{ secrets.AZURE_WEBAPP_NAME }} + publish-profile: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE }} + package: . + + - name: Verify deployment + run: | + echo "Deployment complete!" + echo "App URL: https://${{ secrets.AZURE_WEBAPP_NAME }}.azurewebsites.net" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 7837009d..45905424 100644 --- a/.gitignore +++ b/.gitignore @@ -49,8 +49,16 @@ Thumbs.db # SQLite / local DBs *.db *.db-shm +*.db-shm.* *.db-wal +*.db-wal.* *.sqlite +*.db.backup.* + +# Azure deployment +*.publish-settings +*.PublishSettings +.azure/ # Environment variables / secrets (add if created) .env @@ -97,6 +105,7 @@ docs/~* files/~* output/*.xlsx output/*.svg +output/*.json SOA Workbench Wishlist.docx NCT01750580_limited.json CLAUDE.md diff --git a/docs/DEPLOYMENT_AZURE.md b/docs/DEPLOYMENT_AZURE.md new file mode 100644 index 00000000..74a5b2a1 --- /dev/null +++ b/docs/DEPLOYMENT_AZURE.md @@ -0,0 +1,891 @@ +# Azure Deployment Plan for SoA Workbench + +## Context + +The SoA Workbench is a FastAPI web application for clinical trial Schedule of Activities management. It currently runs locally using SQLite for persistence and serves HTML UI via Jinja2 templates. The application needs to be deployed to Azure for production use with: + +- **Persistent data storage** (SQLite database must survive deployments) +- **Environment configuration** (CDISC API keys, database paths) +- **Static asset serving** (CSS, images, help files - 3.5 MB total) +- **External API access** (CDISC Library API for biomedical concepts) + +Current state: No Docker configuration exists. Application runs via `uvicorn` on port 8000. Dependencies managed via pip/requirements.txt. + +## Quick Start: Deploying with Existing Data + +If you have an existing local database with data you want to preserve: + +1. **First**: Follow Azure resource creation steps (sections 1-9) +2. **Then**: Upload your local database to Azure Files (see Phase 5: Database Migration) +3. **Finally**: Deploy the application code via GitHub Actions + +Your local `soa_builder_web.db` will be uploaded to Azure Files and automatically used by the deployed application. + +## Recommended Approach: Azure App Service (Web App for Linux) + +**Why Azure App Service:** +- Native Python support (no Dockerfile needed initially) +- Built-in persistent storage via Azure Files +- Automatic HTTPS/SSL +- Easy environment variable configuration +- Integrated with GitHub Actions (existing CI pipeline) +- Cost-effective for single-instance applications +- Built-in logging and monitoring + +**Why NOT Azure Container Apps or AKS:** +- Container Apps: Requires containerization (extra complexity) +- AKS: Significant operational overhead for a single web app +- Both are overkill for this application's scale + +## Azure Resource Creation Steps (Manual via Portal) + +**Prerequisites:** +- Azure subscription with Contributor access +- Azure Portal access (portal.azure.com) + +**Step-by-step resource creation:** + +### 1. Create Resource Group +- Navigate to: Home → Resource groups → Create +- Subscription: Select your subscription +- Resource group name: `rg-soa-workbench-prod` +- Region: `East US` (or your preferred region) +- Click: Review + create → Create + +### 2. Create Storage Account (for SQLite persistence) +- Navigate to: Resource group → Add → Storage account +- Basics: + - Storage account name: `stwbdata` (must be globally unique, lowercase, no hyphens) + - Region: Same as resource group + - Performance: Standard + - Redundancy: Locally-redundant storage (LRS) +- Advanced: Leave defaults +- Create → Wait for deployment + +- After creation: + - Go to: Storage account → File shares → + File share + - Name: `soa-workbench-data` + - Tier: Transaction optimized + - Create + +### 3. Create App Service Plan +- Navigate to: Resource group → Add → App Service Plan +- Basics: + - Name: `plan-soa-workbench` + - Operating System: Linux + - Region: Same as resource group + - Pricing tier: B1 (Basic - $13/month) or P1v2 (Production - $78/month) +- Create + +### 4. Create Web App (App Service) +- Navigate to: Resource group → Add → Web App +- Basics: + - Name: `app-soa-workbench` (must be globally unique - becomes app-soa-workbench.azurewebsites.net) + - Publish: Code + - Runtime stack: Python 3.13 + - Operating System: Linux + - Region: Same as resource group + - App Service Plan: Select `plan-soa-workbench` created above +- Deployment: Enable GitHub Actions (configure later) +- Networking: Leave defaults (public access) +- Monitoring: Enable Application Insights (optional but recommended) +- Create → Wait for deployment + +### 5. Configure Storage Mount in Web App +- Navigate to: Web App → Settings → Configuration → Path mappings +- Click: + New Azure Storage Mount + - Name: `data` + - Configuration options: Advanced edit + - Storage accounts: Select `stwbdata` + - Storage type: Azure Files + - Share name: `soa-workbench-data` + - Mount path: `/mnt/data` +- Save → Restart app + +### 6. Create Key Vault (for secrets) +- Navigate to: Resource group → Add → Key Vault +- Basics: + - Key vault name: `kv-soa-workbench` + - Region: Same as resource group + - Pricing tier: Standard +- Access configuration: + - Permission model: Azure role-based access control +- Create + +- After creation: + - Go to: Key Vault → Secrets → + Generate/Import + - Create two secrets: + 1. Name: `CDISC-API-KEY`, Value: Your CDISC API key + 2. Name: `CDISC-SUBSCRIPTION-KEY`, Value: Your CDISC subscription key + +### 7. Configure App Service Managed Identity +**What this does:** Allows your Web App to securely access Key Vault secrets without storing credentials. + +**Step 7a: Enable Managed Identity on Web App** +- Navigate to: Web App → Settings → Identity +- System assigned tab: + - Status = **On** → Save + - Wait for confirmation message + - Copy the **Object (principal) ID** (looks like: `12345678-1234-1234-1234-123456789abc`) + - Note: This ID uniquely identifies your web app to Azure services + +**Step 7b: Grant Key Vault Access to Web App** +- Navigate to: Key Vault (`kv-soa-workbench`) → Access control (IAM) +- Click: **+ Add** → **Add role assignment** +- Role tab: + - Search for: `Key Vault Secrets User` + - Select it → Click: **Next** +- Members tab: + - Assign access to: **Managed identity** + - Click: **+ Select members** + - In the side panel: + - Managed identity: Select **App Service** from dropdown + - Select: Your web app (`app-soa-workbench`) - it will show the same Object ID from Step 7a + - Click: **Select** (closes side panel) + - Click: **Next** +- Conditions tab: + - Leave default (no conditions needed) → Click: **Next** +- Review + assign tab: + - Review settings → Click: **Review + assign** + - Wait for "Role assignment added" confirmation + +**Verify access:** +- Navigate back to: Key Vault → Access control (IAM) → Role assignments +- Filter by: Key Vault Secrets User role +- Should see: `app-soa-workbench` listed with type "Managed Identity" + +### 8. Configure Web App Settings +- Navigate to: Web App → Settings → Environment variables → Application settings +- Add the following (+ New application setting): + ``` + SOA_BUILDER_DB = /mnt/data/soa_builder_web.db + CDISC_API_KEY = @Microsoft.KeyVault(SecretUri=https://kv-soa-workbench.vault.azure.net/secrets/CDISC-API-KEY/) + CDISC_SUBSCRIPTION_KEY = @Microsoft.KeyVault(SecretUri=https://kv-soa-workbench.vault.azure.net/secrets/CDISC-SUBSCRIPTION-KEY/) + PYTHONUNBUFFERED = 1 + SCM_DO_BUILD_DURING_DEPLOYMENT = true + ``` +- Save + +- Navigate to: Web App → Settings → Configuration → General settings + - Stack settings: + - Stack: Python + - Major version: Python 3.13 + - Minor version: (auto-select latest) + - Startup Command: `bash startup.sh` + - Platform settings: + - Always On: On (prevents cold starts - requires Basic tier or higher) + - HTTP version: 2.0 + - Save → Restart + +### 9. Configure Deployment Center +- Navigate to: Web App → Deployment → Deployment Center +- Source: GitHub +- Organization: Your GitHub username +- Repository: `soa-workbench` +- Branch: `master` +- Workflow option: Use existing workflow (we'll create `.github/workflows/azure-deploy.yml`) +- Save + +- Download publish profile: + - Click: Download publish profile → Save the file + - Content of this file will be added to GitHub secrets + +## Implementation Steps + +### Phase 1: Create Deployment Assets + +**1.1 Create startup script for Azure App Service:** + +File: `startup.sh` (project root) +```bash +#!/bin/bash +# Ensure database directory exists +mkdir -p /mnt/data + +# Run database migrations (handled by app lifespan) +# Start gunicorn with uvicorn worker +gunicorn soa_builder.web.app:app \ + --bind 0.0.0.0:8000 \ + --workers 1 \ + --worker-class uvicorn.workers.UvicornWorker \ + --timeout 120 \ + --access-logfile - \ + --error-logfile - +``` + +This script will be configured in Azure App Service as the startup command. + +**1.2 Create GitHub Actions deployment workflow:** + +File: `.github/workflows/azure-deploy.yml` + +Key workflow components: +- **Trigger:** Automatic deployment when PRs are merged to `master` branch + - Deployment is triggered by GitHub merge (not local push) + - Works with PR-based workflow: develop on feature branches → create PR → merge on GitHub → auto-deploy + - Also triggers on `release-*` branches for releases + +- **Build job:** + - Checkout code with submodules (`--recurse-submodules` for cdisc-json-validation) + - Set up Python 3.13 + - Install dependencies: `pip install -e ".[dev]"` + - Run tests: `pytest -q tests --disable-warnings` + - Fail deployment if tests fail + +- **Deploy job:** + - Use `azure/webapps-deploy@v2` action + - Authenticate with publish profile (stored in GitHub secrets) + - Deploy package to Azure App Service + - No need to build artifacts - Azure will handle pip install + +- **GitHub Secrets required:** + - `AZURE_WEBAPP_NAME`: Name of the Azure App Service + - `AZURE_WEBAPP_PUBLISH_PROFILE`: Download from Azure Portal (Deployment Center → Download publish profile) + +- **Environment:** Create a "production" environment in GitHub for manual approvals (optional but recommended) + +**1.3 Update `.gitignore`:** +- Add `.azure/` directory exceptions for deployment configs +- Ensure `soa_builder_web.db` is NOT committed + +### Phase 2: Security Configuration + +**2.1 Key Vault Setup:**** +- Create Key Vault: `kv-soa-workbench` +- Store secrets: + - `CDISC-API-KEY` + - `CDISC-SUBSCRIPTION-KEY` +- Grant App Service managed identity access to Key Vault + +**2.2 App Service Authentication (optional):** +- Enable Azure AD authentication if needed +- Configure allowed users/groups + +**2.3 Network Security:** +- Enable App Service diagnostic logs +- Configure Azure Monitor alerts for errors +- Set up Application Insights for performance monitoring + +### Phase 3: Deployment Pipeline + +**3.1 Configure GitHub Repository Settings:** + +**Step 3.1a: Add Repository Secrets** + +Navigate to: GitHub repository → **Settings** tab → **Secrets and variables** → **Actions** + +Click: **New repository secret** and add each of the following: + +1. **AZURE_WEBAPP_NAME** + - Name: `AZURE_WEBAPP_NAME` + - Value: `app-soa-workbench` (or your actual web app name) + - Click: **Add secret** + +2. **AZURE_WEBAPP_PUBLISH_PROFILE** + - Name: `AZURE_WEBAPP_PUBLISH_PROFILE` + - Value: Open the `.PublishSettings` file you downloaded from Azure Portal + - Copy the **entire XML content** (starts with ``, ends with ``) + - Paste it into the Value field + - Click: **Add secret** + +**Verify secrets added:** +- You should see both secrets listed (values are hidden) +- If you need to update, click the secret name → **Update secret** + +**Step 3.1b: Create Production Environment (Optional but Recommended)** + +This adds a manual approval step before deployment to production. + +1. Navigate to: GitHub repository → **Settings** tab → **Environments** +2. Click: **New environment** +3. Name: `production` (must be lowercase, match workflow file) +4. Click: **Configure environment** + +**Configure protection rules:** + +5. **Required reviewers** + - Check: ✅ **Required reviewers** + - Click: Search field and select yourself (and/or team members) + - This means: deployment pauses until someone approves + +6. **Deployment branches** (optional) + - Click: **Deployment branches** dropdown → **Selected branches** + - Click: **Add deployment branch rule** + - Enter: `master` + - This means: only master branch can deploy to production + +7. **Wait timer** (optional) + - Uncheck (not needed unless you want a mandatory delay) + +8. Click: **Save protection rules** at the top + +**What this does:** +- When PR is merged, workflow runs tests +- After tests pass, deployment **pauses** +- GitHub shows: "Review pending deployments" +- You (or approved reviewer) click **Review deployments** → Select `production` → **Approve and deploy** +- Deployment continues to Azure + +**To skip approval (not recommended):** +- Either don't create the environment +- Or comment out `environment: production` in `.github/workflows/azure-deploy.yml` + +**Step 3.1c: Enable Actions (if needed)** + +If this is your first GitHub Actions workflow: +1. Navigate to: GitHub repository → **Actions** tab +2. If you see "Get started with GitHub Actions": Click **I understand my workflows** +3. Or if asked to enable: Click **Enable Actions** + +**3.2 Development and Deployment Workflow:** + +**Standard workflow (recommended):** +1. **Develop locally** on feature branch (e.g., `feature/new-endpoint`) +2. **Push feature branch** to GitHub: `git push origin feature/new-endpoint` +3. **Create Pull Request** on GitHub to merge into `master` +4. **Review and merge PR** on GitHub (no local push to master needed) +5. **Automatic deployment** triggers when PR is merged to `master` +6. **Monitor deployment** in GitHub Actions tab + +**Key points:** +- ✅ Never push to `master` from local - only merge PRs on GitHub +- ✅ Tests run automatically before deployment (must pass) +- ✅ Deployment happens automatically after PR merge +- ✅ Use "production" environment for manual approval gate (optional) + +**3.3 Deployment Steps (automatic after PR merge):** +1. GitHub detects push to `master` (from PR merge) +2. GitHub Actions workflow starts +3. Checkout code with submodules +4. Set up Python 3.13 +5. Install dependencies (`pip install -e ".[dev]"`) +6. Run tests (`pytest`) - deployment fails if tests fail +7. Deploy to Azure App Service +8. Verify deployment (health check) + +### Phase 4: Database Migration (Upload Existing Local Database) + +**4.1 Backup Local Database:** +Before uploading, create a backup of your local database: +```bash +# Create backup with timestamp +cp soa_builder_web.db soa_builder_web.db.backup.$(date +%Y%m%d_%H%M%S) + +# Also backup WAL and SHM files if they exist +cp soa_builder_web.db-wal soa_builder_web.db-wal.backup.$(date +%Y%m%d_%H%M%S) 2>/dev/null || true +cp soa_builder_web.db-shm soa_builder_web.db-shm.backup.$(date +%Y%m%d_%H%M%S) 2>/dev/null || true +``` + +**4.2 Upload Database to Azure Files:** + +**Quick Method: Using Upload Script (Recommended)** +```bash +# Edit the script first to set your storage account name +# Open scripts/upload_database_to_azure.sh and update: +# STORAGE_ACCOUNT="stwbdata" + +# Run the upload script +./scripts/upload_database_to_azure.sh +``` + +The script will: +- Create a timestamped backup of your local database +- Verify Azure CLI is installed and logged in +- Upload the database to Azure Files +- Verify the upload was successful + +**Option A: Using Azure Portal (GUI method)** +1. Navigate to: Storage Account (`stwbdata`) → File shares → `soa-workbench-data` +2. Click: Upload +3. Select your local `soa_builder_web.db` file +4. Click: Upload +5. Verify file appears in the file share + +**Option B: Using Azure CLI (recommended for automation)** +```bash +# Install Azure CLI if needed +# brew install azure-cli # macOS +# or download from https://aka.ms/installazurecli + +# Login to Azure +az login + +# Set variables (replace with your actual values) +RESOURCE_GROUP="rg-soa-workbench-prod" +STORAGE_ACCOUNT="stwbdata" +FILE_SHARE="soa-workbench-data" +LOCAL_DB="soa_builder_web.db" + +# Get storage account key +STORAGE_KEY=$(az storage account keys list \ + --resource-group $RESOURCE_GROUP \ + --account-name $STORAGE_ACCOUNT \ + --query '[0].value' \ + --output tsv) + +# Upload database file +az storage file upload \ + --account-name $STORAGE_ACCOUNT \ + --account-key $STORAGE_KEY \ + --share-name $FILE_SHARE \ + --source $LOCAL_DB \ + --path soa_builder_web.db + +# Verify upload +az storage file list \ + --account-name $STORAGE_ACCOUNT \ + --account-key $STORAGE_KEY \ + --share-name $FILE_SHARE \ + --output table +``` + +**Option C: Using Azure Storage Explorer (GUI tool)** +1. Download Azure Storage Explorer: https://azure.microsoft.com/features/storage-explorer/ +2. Sign in with your Azure account +3. Navigate to: Storage Accounts → `stwbdata` → File Shares → `soa-workbench-data` +4. Click: Upload → Upload Files +5. Select `soa_builder_web.db` from your local directory +6. Click: Upload + +**4.3 Verify Database Upload:** +After upload, check the file is accessible: +- Navigate to: Web App → Development Tools → SSH +- Click: Go +- Run command: `ls -lh /mnt/data/soa_builder_web.db` +- Should show file size and timestamp + +**4.4 Database Initialization:** +- If starting fresh (no upload): First deployment will create database automatically +- If uploaded existing database: Migrations will run on startup (via app.py lifespan) to ensure schema is up-to-date +- Verify database exists at `/mnt/data/soa_builder_web.db` + +**⚠️ Important Notes:** +- **WAL mode files**: SQLite creates `-wal` and `-shm` files in WAL mode. These will be recreated automatically by Azure; you only need to upload the main `.db` file. +- **Database locking**: Ensure your local app is stopped before uploading the database to avoid corruption. +- **Schema migrations**: The app will automatically run any pending migrations on startup, so your local database will be updated to match the latest schema. + +**4.5 Smoke Testing:** +- Access application URL: `https://app-soa-workbench.azurewebsites.net` +- Verify homepage loads +- Create a test SoA +- Test CDISC API integration (biomedical concepts page) + +**5.3 Monitoring Setup:** +- Configure Log Analytics workspace +- Set up alerts for: + - HTTP 5xx errors + - Response time > 5s + - CPU/Memory > 80% + +## Deployment Architecture + +**Azure App Service with Python 3.13 runtime:** +``` +┌─────────────────────────────────────┐ +│ GitHub Actions (CI/CD) │ +│ - Run tests │ +│ - Deploy on push to master │ +└──────────────┬──────────────────────┘ + │ + ▼ +┌─────────────────────────────────────┐ +│ Azure App Service (Linux) │ +│ - Python 3.13 runtime │ +│ - Gunicorn + Uvicorn workers │ +│ - Port 8000 (mapped to 443) │ +│ - Environment variables from │ +│ Key Vault │ +└──────────────┬──────────────────────┘ + │ + ▼ +┌─────────────────────────────────────┐ +│ Azure Files (Persistent Storage) │ +│ - Mounted at /mnt/data │ +│ - SQLite database file │ +│ - Survives deployments │ +└─────────────────────────────────────┘ +``` + +## Critical Files to Create + +### 1. `startup.sh` (project root) +**Purpose:** Azure App Service startup script for running the application + +**Content:** +```bash +#!/bin/bash +set -e + +echo "Starting SoA Workbench deployment..." + +# Ensure database directory exists +mkdir -p /mnt/data + +# Display Python version for debugging +python --version + +# Display environment (sanitized) +echo "Database path: $SOA_BUILDER_DB" +echo "Mount check: $(ls -la /mnt/data 2>&1 || echo 'Mount not available')" + +# Start gunicorn with uvicorn worker +echo "Starting gunicorn..." +gunicorn soa_builder.web.app:app \ + --bind 0.0.0.0:8000 \ + --workers 1 \ + --worker-class uvicorn.workers.UvicornWorker \ + --timeout 120 \ + --access-logfile - \ + --error-logfile - \ + --log-level info +``` + +### 2. `.github/workflows/azure-deploy.yml` +**Purpose:** Automated deployment pipeline from GitHub to Azure + +**Content:** +```yaml +name: Deploy to Azure App Service + +on: + push: + branches: + - master # Triggers when PRs are merged to master + - release-* # Also triggers for release branches + workflow_dispatch: # Allow manual triggers from GitHub UI if needed + +permissions: + contents: read + +jobs: + build-and-test: + name: Build and Test + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: recursive # Include cdisc-json-validation submodule + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: '3.13' + cache: 'pip' + cache-dependency-path: | + pyproject.toml + requirements.txt + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + + - name: Run tests + run: | + pytest -q tests --disable-warnings --tb=short + env: + CDISC_CONCEPTS_JSON: '[]' # Bypass CDISC API for tests + + - name: Create deployment package + run: | + # Nothing special needed - Azure will handle pip install + echo "Build successful" + + deploy: + name: Deploy to Azure + needs: build-and-test + runs-on: ubuntu-latest + environment: production # Optional: requires manual approval + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Deploy to Azure Web App + uses: azure/webapps-deploy@v2 + with: + app-name: ${{ secrets.AZURE_WEBAPP_NAME }} + publish-profile: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE }} + package: . + + - name: Verify deployment + run: | + echo "Deployment complete!" + echo "App URL: https://${{ secrets.AZURE_WEBAPP_NAME }}.azurewebsites.net" +``` + +**Required GitHub Secrets:** +- `AZURE_WEBAPP_NAME`: The name of your Azure Web App (e.g., `app-soa-workbench`) +- `AZURE_WEBAPP_PUBLISH_PROFILE`: Contents of the publish profile downloaded from Azure Portal + +### 3. `.gitignore` updates +**Add these lines:** +```gitignore +# Azure deployment +*.publish-settings +*.PublishSettings +.azure/ + +# Production database (never commit) +soa_builder_web.db +soa_builder_web.db-wal +soa_builder_web.db-shm + +# Local environment +.env +``` + +### 4. `requirements.txt` update +**Add gunicorn dependency:** +``` +# ... existing dependencies ... +gunicorn>=21.0.0 +``` + +## Files to Modify/Create + +### Files to Create: +1. `.github/workflows/azure-deploy.yml` - Deployment pipeline +2. `startup.sh` - Custom startup script for App Service +3. `docs/DEPLOYMENT_AZURE.md` - This deployment runbook + +### Files to Update: +1. `.gitignore` - Add Azure-specific ignores (`*.publish-settings`, `.azure/`) +2. `requirements.txt` - Add `gunicorn>=21.0.0` for production WSGI server +3. `README.md` - Add "Deployment to Azure" section with link to this doc +4. `pyproject.toml` - Ensure Python >=3.9 compatibility statement is accurate + +### Files NOT to Change: +- `src/soa_builder/web/app.py` - Already configured correctly +- `src/soa_builder/web/db.py` - Environment variable handling works as-is +- `pyproject.toml` - Dependencies are correct + +## Verification Steps + +**Pre-deployment checks:** +1. Run `pytest` - all tests must pass +2. Test locally with production-like config: + ```bash + export SOA_BUILDER_DB=/tmp/test.db + export CDISC_API_KEY= + soa-builder-web + ``` +3. Verify static files load correctly +4. Test CDISC API integration + +**Post-deployment verification:** +1. Access Azure app URL +2. **If uploaded existing database:** Verify your existing SoAs and data are visible +3. **If fresh database:** Create a new SoA (tests database writes) +4. View biomedical concepts (tests CDISC API) +5. Export USDM JSON (tests complex operations) +6. Check Azure logs for errors +7. Verify database persistence (redeploy and check data survives) + +## Cost Estimate (Azure) + +**Monthly costs (approximate):** +- App Service Plan (B1): ~$13/month +- Azure Files (10 GB): ~$2/month +- Bandwidth: ~$5/month +- **Total: ~$20/month** (dev/test) + +**Production tier:** +- App Service Plan (P1v2): ~$78/month +- Azure Files (10 GB): ~$2/month +- Application Insights: ~$10/month +- **Total: ~$90/month** + +## Risk Mitigation + +**SQLite limitations on Azure:** +- Single-writer limitation (App Service single instance OK) +- No write scaling (consider Azure SQL if multi-instance needed) +- Backup strategy: Azure Files snapshots or manual exports + +**Environment variables:** +- Use Azure Key Vault references, not plain text +- Never commit .env files with real keys + +**Deployment failures:** +- Always run tests before deploy +- Use deployment slots for zero-downtime (P1v2 tier) +- Keep previous version for quick rollback + +## Deployment Workflow Summary + +### Initial Setup (One-time) +1. ✅ Create Azure resources (sections 1-9 above) +2. ✅ Upload your local database to Azure Files (Phase 5) +3. ✅ Create `startup.sh` and `.github/workflows/azure-deploy.yml` +4. ✅ Configure GitHub secrets (AZURE_WEBAPP_NAME, AZURE_WEBAPP_PUBLISH_PROFILE) +5. ✅ Push deployment files to GitHub + +### Ongoing Development & Deployment + +**Your complete workflow:** + +1. **Local development:** + - Create feature branch: `git checkout -b feature/my-change` + - Make changes and test locally: `pytest && soa-builder-web` + - Commit changes: `git commit -m "Add new feature"` + - Push branch: `git push origin feature/my-change` + +2. **Create Pull Request on GitHub:** + - Go to GitHub repository + - Click: **Pull requests** → **New pull request** + - Base: `master` ← Compare: `feature/my-change` + - Click: **Create pull request** + - Add description of changes + - (Optional) Request review from team member + +3. **Merge and Deploy:** + - Review the changes (or wait for peer review) + - Click: **Merge pull request** → **Confirm merge** + - **GitHub Actions automatically triggers deployment** + +4. **Monitor Deployment:** + - Go to: GitHub → **Actions** tab + - Click on the latest workflow run (should be running) + - Watch progress: + - ✅ "Build and Test" job (runs pytest) + - ⏸️ "Deploy to Azure" job (waits for approval if environment configured) + - ✅ Deployment completes + +5. **Approve Deployment (if production environment configured):** + - During "Deploy to Azure" job, you'll see: **Review pending deployments** + - Click: **Review deployments** + - Select: ✅ `production` + - Click: **Approve and deploy** + - Deployment continues to Azure + +6. **Verify Deployment:** + - Wait for workflow to complete (green checkmark) + - Visit: `https://app-soa-workbench.azurewebsites.net` + - Verify your changes are live + - Check Azure logs if issues + +### Quick Reference Commands + +**Local development:** +```bash +# Start working on new feature +git checkout -b feature/my-feature + +# Test locally +pytest +soa-builder-web + +# Push to GitHub (does NOT deploy) +git add . +git commit -m "Implement feature" +git push origin feature/my-feature +``` + +**Deploy to Azure:** +```bash +# No local commands needed! +# 1. Merge PR on GitHub +# 2. GitHub Actions automatically runs tests +# 3. If production environment configured: +# - Go to GitHub → Actions tab +# - Click on the running workflow +# - Click "Review deployments" button +# - Select "production" → Click "Approve and deploy" +# 4. If no environment: deployment happens automatically after tests pass +``` + +**Monitor deployment:** +```bash +# 1. Go to GitHub → Actions tab +# 2. Click on the latest workflow run +# 3. Watch "Build and Test" and "Deploy to Azure" jobs +# 4. Green checkmark = success +# 5. Red X = failure (click to see logs) +``` + +**Manual deployment (workflow_dispatch):** +```bash +# Use when you need to redeploy without a new commit +# 1. Go to GitHub → Actions tab +# 2. Click "Deploy to Azure App Service" workflow (left sidebar) +# 3. Click "Run workflow" button (right side) +# 4. Select branch: master +# 5. Click green "Run workflow" button +``` + +## Database Backup and Restore + +### Creating Backups + +**Automated backup via Azure Files snapshots:** +1. Navigate to: Storage Account → File shares → `soa-workbench-data` +2. Click: Snapshots → + Snapshot +3. Add description: "Manual backup before deployment" +4. Click: Create + +**Schedule automated snapshots:** +- Set up Azure Backup for File Shares +- Navigate to: File share → Backup +- Configure backup policy (daily/weekly) + +**Manual backup via download:** +```bash +# Using Azure CLI +az storage file download \ + --account-name $STORAGE_ACCOUNT \ + --account-key $STORAGE_KEY \ + --share-name $FILE_SHARE \ + --path soa_builder_web.db \ + --dest ./backups/soa_builder_web.db.$(date +%Y%m%d_%H%M%S) +``` + +### Restoring from Backup + +**Restore from Azure Files snapshot:** +1. Navigate to: File share → Snapshots +2. Select snapshot to restore +3. Find `soa_builder_web.db` file +4. Click: Restore +5. Choose: Overwrite original file +6. Restart Web App + +**Restore from local backup:** +- Follow the database upload steps in Phase 5 +- Upload your backup file +- Restart Web App + +## Troubleshooting + +### Application won't start +- Check Azure logs: Web App → Monitoring → Log stream +- Verify startup command is correct: `bash startup.sh` +- Ensure Python version matches (3.13) +- Check environment variables are set + +### Database connection errors +- Verify Azure Files mount is configured correctly +- Check `/mnt/data` is accessible (view logs) +- Ensure `SOA_BUILDER_DB` points to `/mnt/data/soa_builder_web.db` + +### CDISC API failures +- Verify Key Vault secrets are accessible +- Check managed identity has Key Vault Secrets User role +- Test API keys are valid + +### Performance issues +- Enable "Always On" to prevent cold starts +- Consider upgrading to P1v2 tier +- Check Application Insights for bottlenecks + +## Next Steps + +1. Follow Azure Resource Creation steps above +2. Create deployment files (startup.sh, azure-deploy.yml) +3. Configure GitHub secrets +4. Test deployment to Azure +5. Set up monitoring and alerts diff --git a/requirements.txt b/requirements.txt index 4e4762b6..42d82573 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ +# Install the soa_builder package itself (required for Azure deployment) +-e . + annotated-doc==0.0.4 annotated-types==0.7.0 anyio==4.12.1 @@ -28,3 +31,4 @@ typing_extensions==4.15.0 urllib3==2.6.3 usdm==0.66.0 uvicorn==0.38.0 +gunicorn>=21.0.0 \ No newline at end of file diff --git a/scripts/upload_database_to_azure.sh b/scripts/upload_database_to_azure.sh new file mode 100755 index 00000000..f9659584 --- /dev/null +++ b/scripts/upload_database_to_azure.sh @@ -0,0 +1,117 @@ +#!/bin/bash +# Upload local SQLite database to Azure Files +# This script uploads your local soa_builder_web.db to Azure for deployment + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Configuration - UPDATE THESE VALUES +RESOURCE_GROUP="rg-soa-workbench-prod" +STORAGE_ACCOUNT="soaworkbenchsa" # Replace with your storage account suffix +FILE_SHARE="soa-workbench-data" +LOCAL_DB="soa_builder_web.db" + +echo -e "${GREEN}=== Azure Database Upload Script ===${NC}" +echo "" + +# Check if local database exists +if [ ! -f "$LOCAL_DB" ]; then + echo -e "${RED}Error: Local database file '$LOCAL_DB' not found${NC}" + echo "Please run this script from the project root directory where soa_builder_web.db is located" + exit 1 +fi + +# Check if Azure CLI is installed +if ! command -v az &> /dev/null; then + echo -e "${RED}Error: Azure CLI is not installed${NC}" + echo "Please install Azure CLI first:" + echo " macOS: brew install azure-cli" + echo " Linux: curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash" + echo " Windows: Download from https://aka.ms/installazurecli" + exit 1 +fi + +# Create backup before upload +echo -e "${YELLOW}Creating backup of local database...${NC}" +BACKUP_FILE="$LOCAL_DB.backup.$(date +%Y%m%d_%H%M%S)" +cp "$LOCAL_DB" "$BACKUP_FILE" +echo -e "${GREEN}✓ Backup created: $BACKUP_FILE${NC}" +echo "" + +# Check if logged in to Azure +echo "Checking Azure login status..." +if ! az account show &> /dev/null; then + echo -e "${YELLOW}Not logged in to Azure. Opening login...${NC}" + az login +fi + +SUBSCRIPTION=$(az account show --query name -o tsv) +echo -e "${GREEN}✓ Logged in to Azure subscription: $SUBSCRIPTION${NC}" +echo "" + +# Get storage account key +echo "Retrieving storage account key..." +STORAGE_KEY=$(az storage account keys list \ + --resource-group "$RESOURCE_GROUP" \ + --account-name "$STORAGE_ACCOUNT" \ + --query '[0].value' \ + --output tsv 2>&1) + +if [ $? -ne 0 ]; then + echo -e "${RED}Error: Failed to retrieve storage account key${NC}" + echo "Please verify:" + echo " 1. Resource group '$RESOURCE_GROUP' exists" + echo " 2. Storage account '$STORAGE_ACCOUNT' exists" + echo " 3. You have permission to access the storage account" + exit 1 +fi + +echo -e "${GREEN}✓ Storage account key retrieved${NC}" +echo "" + +# Get database file size +DB_SIZE=$(du -h "$LOCAL_DB" | cut -f1) +echo "Database file size: $DB_SIZE" +echo "" + +# Upload database +echo -e "${YELLOW}Uploading database to Azure Files...${NC}" +echo "This may take a few moments depending on file size..." + +az storage file upload \ + --account-name "$STORAGE_ACCOUNT" \ + --account-key "$STORAGE_KEY" \ + --share-name "$FILE_SHARE" \ + --source "$LOCAL_DB" \ + --path soa_builder_web.db \ + --no-progress + +if [ $? -ne 0 ]; then + echo -e "${RED}Error: Failed to upload database${NC}" + exit 1 +fi + +echo -e "${GREEN}✓ Database uploaded successfully!${NC}" +echo "" + +# Verify upload +echo "Verifying upload..." +az storage file list \ + --account-name "$STORAGE_ACCOUNT" \ + --account-key "$STORAGE_KEY" \ + --share-name "$FILE_SHARE" \ + --output table | grep soa_builder_web.db + +echo "" +echo -e "${GREEN}=== Upload Complete ===${NC}" +echo "" +echo "Next steps:" +echo " 1. Restart your Azure Web App to use the new database" +echo " 2. Verify data is accessible at your app URL" +echo "" +echo "Backup location: $BACKUP_FILE" diff --git a/src/soa_builder.egg-info/PKG-INFO b/src/soa_builder.egg-info/PKG-INFO index 74ac6d31..8cb6caeb 100644 --- a/src/soa_builder.egg-info/PKG-INFO +++ b/src/soa_builder.egg-info/PKG-INFO @@ -15,191 +15,55 @@ Requires-Dist: openpyxl>=3.1.0 Requires-Dist: reportlab>=4.0.0 Requires-Dist: requests>=2.31.0 Requires-Dist: python-dotenv>=1.0.0 +Requires-Dist: jinja2>=3.1.0 +Requires-Dist: python-multipart>=0.0.9 Provides-Extra: dev Requires-Dist: pytest>=7.0.0; extra == "dev" +Requires-Dist: pytest-cov>=4.0.0; extra == "dev" Requires-Dist: ruff>=0.5.0; extra == "dev" Requires-Dist: black>=24.0.0; extra == "dev" Requires-Dist: detect-secrets>=1.4.0; extra == "dev" +Requires-Dist: httpx>=0.27.0; extra == "dev" Dynamic: license-file -# SoA Builder (Normalization, Expansion & Validation) - -This workspace provides a Python package `soa_builder` with a CLI and APIs to: - -1. Normalize a wide Schedule of Activities (SoA) matrix into relational tables. -2. Expand repeating schedule rules into projected calendar instances. -3. Validate imaging (and future) activity intervals. - -Legacy standalone scripts (`normalize_soa.py`, `validate_soa.py`) remain for reference; new work should use the CLI. - -## Source -Input format: first column `Activity`, subsequent columns are visit/timepoint headers. Cells contain markers like `X`, `Optional`, `If indicated`, or repeating patterns (`Every 2 cycles`, `q12w`). - -## Output Artifacts -Running the script produces (in `--out-dir`): -- `visits.csv` — One row per visit/timepoint with parsed window info, inferred category, repeat pattern. -- `activities.csv` — Unique activities (one per original row). -- `visit_activities.csv` — Junction table mapping activities to visits with status and flags. -- `activity_categories.csv` — Heuristic classification of each activity (labs, imaging, dosing, admin, etc.). -- `schedule_rules.csv` — Extracted repeating schedule logic from headers and cells (e.g., `q12w`, `Every 2 cycles`). -- Optional: SQLite database (`--sqlite path`) containing all tables. - -### visits.csv Columns -- `visit_id`: Sequential numeric id. -- `raw_header`: Original header text. -- `visit_name`: Header stripped of parenthetical codes. -- `visit_code`: Code extracted from parentheses (e.g., `C1D1`, `EOT`). -- `sequence_index`: Positional order. -- `window_lower` / `window_upper`: Parsed day offsets if available. -- `repeat_pattern`: Detected repeating pattern (e.g., `every 2 cycles`). -- `category`: Heuristic classification (screening, baseline, treatment, follow_up, eot). - -### activities.csv Columns -- `activity_id`: Sequential id. -- `activity_name`: Name from first column. - -### visit_activities.csv Columns -- `id`: Junction id. -- `visit_id`: FK to visits. -- `activity_id`: FK to activities. -- `status`: Raw cell content. -- `required_flag`: 1 if cell starts with `X`. -- `conditional_flag`: 1 if cell contains `Optional` or `If indicated`. - -### activity_categories.csv Columns -- `activity_id`: FK to activities. -- `category`: Assigned heuristic category label. - -### schedule_rules.csv Columns -- `rule_id`: Unique rule id. -- `pattern`: Normalized repeating pattern token (e.g., `q12w`). -- `description`: Human readable description of pattern source. -- `source_type`: `header` or `cell` origin. -- `activity_id`: Populated if pattern came from a cell (else null). -- `visit_id`: Populated if pattern came from a header. -- `raw_text`: Original text fragment containing the pattern. +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -## Installation +# SoA Workbench -Recommended: editable install for development. -```bash -python3 -m venv .venv -source .venv/bin/activate -pip install -e .[dev] -``` +This workspace provides a Python package `soa_builder` with APIs to create a Schedule of Activites for Clinical Studies. -This installs the console script `soa-builder`. -Example: -```bash -soa-builder normalize --input files/SoA_breast_cancer.csv --out-dir normalized -soa-builder expand --normalized-dir normalized --start-date 2025-01-01 --json-out normalized/schedule_instances.json -soa-builder validate --normalized-dir normalized -``` -## CLI Usage +## Cloning the repository +This project now includes a submodule for USDM JSON validation with the USDM_API_v4.0.0.json schema. -The CLI exposes three subcommands: `normalize`, `expand`, `validate`. +In order to clone the repository with the new submodule, use the command: -### Normalize ```bash -soa-builder normalize --input files/SoA_breast_cancer.csv --out-dir normalized --sqlite normalized/soa.db +> git clone --recurse-submodules https://github.com/pendingintent/soa-workbench.git ``` -Outputs written to `normalized/` (CSV and optional SQLite). -### Expand Schedule Rules +Once the repository has been cloned locally, in order to ensure the submodule is up-to-date, use the commands: ```bash -soa-builder expand --normalized-dir normalized --start-date 2025-01-01 \ - --cycle-length-days 21 --num-cycles 8 --followup-weeks 104 \ - --json-out normalized/schedule_instances.json +> cd cdisc-json-validation +> git pull +# or use the command for updating all registered submodules +> git submodule update --remote ``` -Options: -- `--filter-pattern PATTERN` (repeatable) to limit patterns (e.g. `--filter-pattern q12w`) -- `--cycle-lengths 21,21,28` for heterogeneous cycle lengths -- `--horizon-days DAYS` override default calculated horizon -- `--max-occurrences N` cap per-rule expansions -### Validate Imaging Intervals -```bash -soa-builder validate --normalized-dir normalized --expected-interval-weeks 6 --tolerance-days 4 -``` -Exit code non-zero indicates deviations; listed per interval. +This will ensure the submodule is always up-to-date. -## Python API -```python -from soa_builder import normalize_soa, expand_schedule_rules, validate_imaging_schedule -summary = normalize_oa('files/SoA_breast_cancer.csv', 'normalized') -# Load rules/visits then expand (see cli implementation for loaders) -``` -## Development & Testing -Run unit tests: +## Installation +Recommended: editable install for development. ```bash -pytest -``` - -## Roadmap -- Additional validators (PK sampling, PRO schedule completeness) -- Console script entry point publication via `pyproject.toml` -- Enriched rule grammar (e.g. conditional frequency changes) -- SDTM domain mapping utilities - - Web application for interactive SoA authoring (FastAPI + HTMX) extended with biomedical concept browsing and stable activity UIDs - -## Assumptions & Heuristics -- All non-first header columns are considered visits. -- Windows parsed from patterns like `(-28 to -1d)`, `(±7d)`, `30±7d`. -- Repeat patterns detected: `every 2 cycles`, `q12w`, `q3w`, `every 12 weeks`. -- Additional conditional text retained in `status`. - -## Extending -- Refine category taxonomy with controlled terminology (CDISC) -- Richer recurrence parsing (e.g., bi-weekly then monthly transitions) -- Endpoint linkage & CRF mapping tables -- Additional validators (PK sampling alignment, PRO schedule completeness) - - Web UI (React or HTMX) atop FastAPI backend for matrix editing - -## Web Application (Experimental) -An initial FastAPI backend (`soa_builder.web.app`) allows creating an SoA interactively via REST: - -> Full, continuously updated endpoint reference (including Elements, freezes, audits, JSON CRUD and UI helpers) lives in `README_endpoints.md`. Consult that file for detailed request/response examples, curl snippets, and future enhancement notes. - -Endpoints: -- POST /soa {"name": "Breast Cancer Phase 2"} -- POST /soa/{id}/visits {"name": "C1D1", "raw_header": "Cycle 1 Day 1 (C1D1)"} -- POST /soa/{id}/activities {"name": "Hematology"} -- POST /soa/{id}/cells {"visit_id": 1, "activity_id": 1, "status": "X"} -- GET /soa/{id}/matrix -> JSON matrix -- GET /soa/{id}/normalized -> Runs normalization pipeline; returns summary - - DELETE /soa/{id}/visits/{visit_id} -> Remove a visit and all its cells; remaining visits reindexed - - DELETE /soa/{id}/activities/{activity_id} -> Remove an activity and all its cells; remaining activities reindexed - - POST /soa/{id}/activities/bulk {"names": ["Hematology", "Chemistry", "ECG"]} -> create multiple activities (skips duplicates & blanks) - - POST /soa/{id}/matrix/import -> Ingest wide matrix JSON body - - GET /soa/{id}/export/xlsx -> Download current matrix as Excel workbook (sheet: SoA) - - GET /soa/{id}/export/pdf -> Download current matrix as PDF table - -### Wide Matrix Import Format -`POST /soa/{id}/matrix/import` -```jsonc -{ - "visits": [ - {"name": "C1D1", "raw_header": "Cycle 1 Day 1 (C1D1)"}, - {"name": "C1D8"}, - {"name": "C1D15"} - ], - "activities": [ - {"name": "Hematology", "statuses": ["X", "X", "O"]}, - {"name": "Chemistry", "statuses": ["", "X", ""]}, - {"name": "ECG", "statuses": ["O", "", "O"]} - ], - "reset": true -} +> python3 -m venv .venv +> source .venv/bin/activate +> pip install -r requirements.txt +> pre-commit install +> pre-commit run --all-files ``` -Rules: -- `statuses` array length must equal number of `visits`. -- Blank / empty status strings are ignored (no cell row created). -- When `reset` is true existing visits, activities, and cells for the SoA are cleared first. -- All inserts preserve provided order for indexing. -Run server: +## Start web server ```bash soa-builder-web # starts uvicorn on 0.0.0.0:8000 with reload ``` @@ -208,26 +72,17 @@ Or manually: ```bash uvicorn soa_builder.web.app:app --reload --port 8000 ``` - -After populating data, retrieve normalized artifacts: -```bash -curl http://localhost:8000/soa/1/normalized -``` - HTML UI: - Open http://localhost:8000/ in a browser. -- Add visits and activities; click cells to toggle status (blank -> X -> blank). 'O' values are not surfaced in the UI; clearing removes the cell row. -- Use "Generate Normalized Summary" link to produce artifacts. - - Use export buttons (to be added) or hit endpoints directly for XLSX/PDF output. - - Delete a visit or activity using the ✕ button next to its name (confirmation dialog). Deletion cascades to associated cells and automatically reorders remaining items. - - (Upcoming) Bulk add activities and matrix import could be surfaced via a textarea or JSON upload panel. - - View biomedical concepts via the "Concepts" navigation link (`GET /ui/concepts`): renders a table of concept codes, titles and API links (cached; force refresh per study using `POST /ui/soa/{id}/concepts_refresh`). - -Activity Identifiers: -- Each activity now has a stable `activity_uid` (format `Activity_` unique within a study) maintained during reorder using a two-phase temporary renaming to avoid uniqueness collisions. -- Unique index `(soa_id, activity_uid)` enforces stability for exports, snapshots and audit trails. - -Biomedical Concepts API Access: +- Create a new Schedule of Activities for a study or access an existing one. + - When a study is chosen, additional navigation links are available in the navigation menu that are unique to the Study context. + - More options and parameters for configuring the USDM classes are available through these navigation links. +- Add Scheduled Activity instances (columns) and activities (rows) to create an SoA matrix on the edit page for a Study; click cells to toggle status (blank -> X -> blank). 'O' values are not surfaced in the UI; clearing removes the cell row. + - Use export buttons (to be added) for XLSX output of the Matrix. +- View avialable biomedical concepts via the "Biomedical Concepts" navigation link to render a table of concept codes, titles and API links (cached; force refresh available). +- View available data set specializations via the "SDTM Dataset Specializations" navigation link to render a table of specializations and API links to view associated concepts (cached; force refresh available). + +CDISC Library API Access: - The concepts list and detail pages call the CDISC Library API. - Set one (or both) of: `CDISC_SUBSCRIPTION_KEY`, `CDISC_API_KEY`. - The server will send all of these headers when possible: @@ -237,11 +92,42 @@ Biomedical Concepts API Access: - If only one key is defined it is reused across header variants. - Directly opening the API URL in the browser will 401 because the browser does not attach the required headers; use the internal detail page or an API client (curl/Postman) with the headers above. -Notes: -- HTMX is loaded via CDN; no build step required. -- For production, configure a persistent DB path via SOA_BUILDER_DB env variable. +## Development & Testing +Run unit tests: +```bash +pytest +``` + +### Test database +- Tests run against a separate SQLite file to avoid touching your local/prod data. +- Default path: `soa_builder_web_tests.db` in the repo root. Override with env var `SOA_BUILDER_DB`. +- A pytest session fixture removes any stale test DB/WAL/SHM files at start to prevent I/O errors. +- Manually clear the test DB before a run if needed: +```bash +rm -f soa_builder_web_tests.db soa_builder_web_tests.db-wal soa_builder_web_tests.db-shm +``` + +> **Full API Documentation**: See `README_endpoints.md` for complete endpoint reference with curl examples, request/response schemas, and usage patterns. +> +> **Endpoint Catalog**: See `docs/api_endpoints.csv` for sortable/filterable list of all 165+ endpoints. + +## USDM Export +Export USDM-compliant JSON for integration with external systems: +```bash +# Use the USDM generator scripts directly +python -m usdm.generate_usdm 1 -o study_usdm.json +python -m usdm.generate_activities 1 -o activities.json +python -m usdm.generate_encounters 1 -o encounters.json +python -m usdm.generate_study_epochs 1 -o epochs.json +# See src/usdm/ for all generator scripts +``` + +--- + +## Architecture Notes +- **Database**: SQLite with WAL mode (production) or DELETE mode (tests) +- **Test Isolation**: Tests use `soa_builder_web_tests.db` (set via `SOA_BUILDER_DB` env var) +- **Production Config**: Set `SOA_BUILDER_DB` environment variable for persistent DB path +- **USDM Generators**: Python scripts in `src/usdm/` transform database state → USDM JSON artifacts -Artifacts stored under `normalized/soa_{id}/`. -## License -Internal use; extend as needed. diff --git a/src/soa_builder.egg-info/SOURCES.txt b/src/soa_builder.egg-info/SOURCES.txt index b6615e46..92c060fc 100644 --- a/src/soa_builder.egg-info/SOURCES.txt +++ b/src/soa_builder.egg-info/SOURCES.txt @@ -1,6 +1,9 @@ LICENSE README.md pyproject.toml +src/sdtm/generate_ta.py +src/sdtm/generate_te.py +src/sdtm/generate_tv.py src/soa_builder/__init__.py src/soa_builder/cli.py src/soa_builder/normalization.py @@ -18,23 +21,89 @@ src/soa_builder/web/db.py src/soa_builder/web/initialize_database.py src/soa_builder/web/migrate_database.py src/soa_builder/web/schemas.py +src/soa_builder/web/utils.py +src/soa_builder/web/routers/_freeze_helpers.py src/soa_builder/web/routers/activities.py src/soa_builder/web/routers/arms.py +src/soa_builder/web/routers/audits.py +src/soa_builder/web/routers/bc_surrogates.py +src/soa_builder/web/routers/cdash_terminology.py +src/soa_builder/web/routers/cells.py +src/soa_builder/web/routers/concept_groups.py +src/soa_builder/web/routers/condition_assignments.py +src/soa_builder/web/routers/ddf_controlled_terminology.py +src/soa_builder/web/routers/decision_instances.py +src/soa_builder/web/routers/define_xml_terminology.py src/soa_builder/web/routers/elements.py +src/soa_builder/web/routers/endpoints.py src/soa_builder/web/routers/epochs.py +src/soa_builder/web/routers/footnotes.py src/soa_builder/web/routers/freezes.py +src/soa_builder/web/routers/instances.py +src/soa_builder/web/routers/objectives.py +src/soa_builder/web/routers/protocol_controlled_terminology.py src/soa_builder/web/routers/rollback.py +src/soa_builder/web/routers/rules.py +src/soa_builder/web/routers/schedule_timelines.py +src/soa_builder/web/routers/sdtm_terminology.py +src/soa_builder/web/routers/tdd.py +src/soa_builder/web/routers/timings.py +src/soa_builder/web/routers/usdm_json.py src/soa_builder/web/routers/visits.py +src/usdm/generate_activities.py +src/usdm/generate_arms.py +src/usdm/generate_bc_surrogates.py +src/usdm/generate_biomedical_concept_properties.py +src/usdm/generate_biomedical_concepts.py +src/usdm/generate_elements.py +src/usdm/generate_encounters.py +src/usdm/generate_extension_attributes.py +src/usdm/generate_schedule_timelines.py +src/usdm/generate_scheduled_activity_instances.py +src/usdm/generate_scheduled_decision_instances.py +src/usdm/generate_study_cells.py +src/usdm/generate_study_epochs.py +src/usdm/generate_study_timings.py +src/usdm/generate_usdm.py +src/usdm/usdm_utils.py tests/test_bulk_import.py -tests/test_cell_clear.py -tests/test_deletion.py +tests/test_categories_cache.py +tests/test_categories_ui_force.py +tests/test_code_uid_generation.py +tests/test_concept_categories.py +tests/test_concept_category_force_refresh.py +tests/test_concepts_by_category_ui_force.py +tests/test_element_audit_endpoint.py tests/test_element_id_generation.py -tests/test_exports.py +tests/test_element_id_monotonic.py +tests/test_epoch_reorder_audit_api.py +tests/test_epoch_type_options.py tests/test_fetch_sdtm_specializations.py -tests/test_terminology_date.py -tests/test_ui_add_element.py -tests/test_ui_export_buttons.py -tests/test_ui_set_visit_epoch.py -tests/test_ui_toggle.py -tests/test_ui_visit_create.py -tests/test_web_api.py \ No newline at end of file +tests/test_generate_biomedical_concept_properties.py +tests/test_generate_extension_attributes.py +tests/test_instances_audit.py +tests/test_routers_activities.py +tests/test_routers_arms.py +tests/test_routers_audits.py +tests/test_routers_bc_surrogates.py +tests/test_routers_decision_instances.py +tests/test_routers_elements.py +tests/test_routers_endpoints.py +tests/test_routers_epochs.py +tests/test_routers_freezes.py +tests/test_routers_instances.py +tests/test_routers_objectives.py +tests/test_routers_rollback.py +tests/test_routers_rules.py +tests/test_routers_schedule_timelines.py +tests/test_routers_study_timing.py +tests/test_routers_tdd.py +tests/test_routers_timings.py +tests/test_routers_usdm_json.py +tests/test_routers_visits.py +tests/test_study_cell_uid_reuse.py +tests/test_study_cell_uid_reuse_later.py +tests/test_timing_audit.py +tests/test_timing_audit_endpoint.py +tests/test_timings.py +tests/test_ui_add_element.py \ No newline at end of file diff --git a/src/soa_builder.egg-info/requires.txt b/src/soa_builder.egg-info/requires.txt index af89bf93..94caa9eb 100644 --- a/src/soa_builder.egg-info/requires.txt +++ b/src/soa_builder.egg-info/requires.txt @@ -6,9 +6,13 @@ openpyxl>=3.1.0 reportlab>=4.0.0 requests>=2.31.0 python-dotenv>=1.0.0 +jinja2>=3.1.0 +python-multipart>=0.0.9 [dev] pytest>=7.0.0 +pytest-cov>=4.0.0 ruff>=0.5.0 black>=24.0.0 detect-secrets>=1.4.0 +httpx>=0.27.0 diff --git a/src/soa_builder.egg-info/top_level.txt b/src/soa_builder.egg-info/top_level.txt index 880175c2..962264c1 100644 --- a/src/soa_builder.egg-info/top_level.txt +++ b/src/soa_builder.egg-info/top_level.txt @@ -1 +1,3 @@ +sdtm soa_builder +usdm diff --git a/src/soa_builder/web/app.py b/src/soa_builder/web/app.py index 01ad3440..11d1ed92 100644 --- a/src/soa_builder/web/app.py +++ b/src/soa_builder/web/app.py @@ -81,14 +81,25 @@ _migrate_activity_surrogate_add_concept_group_uid, _migrate_add_activity_concept_dss_table, _migrate_activity_concept_dss_add_display, + _migrate_activity_concept_dss_add_extension_attribute_uid, _migrate_drop_protocol_terminology_tables, _migrate_drop_ddf_terminology_tables, + _migrate_add_objective_table, + _migrate_add_objective_audit_table, + _migrate_add_endpoint_table, + _migrate_add_endpoint_audit_table, ) from .routers import activities as activities_router from .routers import arms as arms_router from .routers import elements as elements_router from .routers import epochs as epochs_router from .routers import freezes as freezes_router +from .routers._freeze_helpers import ( + _diff_freezes_limited, + _get_freeze, + _list_freezes, + _list_rollback_audit, +) from .routers import rollback as rollback_router from .routers import visits as visits_router from .routers import audits as audits_router @@ -113,6 +124,8 @@ from .routers import ( ddf_controlled_terminology as ddf_controlled_terminology_router, ) +from .routers import objectives as objectives_router +from .routers import endpoints as endpoints_router from .audit import _record_element_audit @@ -246,6 +259,11 @@ def _configure_logging(): _migrate_drop_ddf_terminology_tables() _migrate_add_activity_concept_dss_table() _migrate_activity_concept_dss_add_display() +_migrate_activity_concept_dss_add_extension_attribute_uid() +_migrate_add_objective_table() +_migrate_add_objective_audit_table() +_migrate_add_endpoint_table() +_migrate_add_endpoint_audit_table() # Include routers @@ -278,6 +296,10 @@ def _configure_logging(): app.include_router(define_xml_terminology_router.router) app.include_router(protocol_controlled_terminology_router.router) app.include_router(ddf_controlled_terminology_router.router) +app.include_router(objectives_router.router) +app.include_router(objectives_router.ui_router) +app.include_router(endpoints_router.router) +app.include_router(endpoints_router.ui_router) def _record_visit_audit( @@ -361,561 +383,6 @@ def reorder_visits_api(soa_id: int, order: List[int]): ''' -def _list_freezes(soa_id: int): - conn = _connect() - cur = conn.cursor() - cur.execute( - "SELECT id, version_label, created_at FROM soa_freeze WHERE soa_id=? ORDER BY id DESC", - (soa_id,), - ) - rows = [dict(id=r[0], version_label=r[1], created_at=r[2]) for r in cur.fetchall()] - conn.close() - return rows - - -def _get_freeze(soa_id: int, freeze_id: int): - conn = _connect() - cur = conn.cursor() - cur.execute( - "SELECT id, version_label, created_at, snapshot_json FROM soa_freeze WHERE id=? AND soa_id=?", - (freeze_id, soa_id), - ) - row = cur.fetchone() - conn.close() - if not row: - return None - try: - snap = json.loads(row[3]) - except Exception: - snap = {"error": "Corrupt snapshot"} - return { - "id": row[0], - "version_label": row[1], - "created_at": row[2], - "snapshot": snap, - } - - -def _create_freeze(soa_id: int, version_label: Optional[str]): - if not soa_exists(soa_id): - raise HTTPException(404, "SOA not found") - # Auto version label if not provided - conn = _connect() - cur = conn.cursor() - cur.execute("SELECT version_label FROM soa_freeze WHERE soa_id=?", (soa_id,)) - existing_labels = {r[0] for r in cur.fetchall()} - if not version_label or not version_label.strip(): - # Find next available vN - n = 1 - while f"v{n}" in existing_labels: - n += 1 - version_label = f"v{n}" - else: - version_label = version_label.strip() - if version_label in existing_labels: - raise HTTPException(400, "Version label already exists for this SOA") - # Gather snapshot data - cur.execute( - "SELECT name, created_at, study_id, study_label, study_description FROM soa WHERE id=?", - (soa_id,), - ) - row = cur.fetchone() - soa_name = row[0] if row else f"SOA {soa_id}" - study_id_val = row[2] if row else None - study_label_val = row[3] if row else None - study_description_val = row[4] if row else None - visits, activities, cells = _fetch_matrix(soa_id) - # Epochs snapshot (ordered) - conn2 = _connect() - cur2 = conn2.cursor() - cur2.execute( - "SELECT id,name,order_index,epoch_seq,epoch_label,epoch_description FROM epoch WHERE soa_id=? ORDER BY order_index", - (soa_id,), - ) - epochs = [ - dict( - id=r[0], - name=r[1], - order_index=r[2], - epoch_seq=r[3], - epoch_label=r[4], - epoch_description=r[5], - ) - for r in cur2.fetchall() - ] - conn2.close() - # Elements snapshot (ordered) - conn_el = _connect() - cur_el = conn_el.cursor() - cur_el.execute( - "SELECT id,name,label,description,testrl,teenrl,order_index FROM element WHERE soa_id=? ORDER BY order_index", - (soa_id,), - ) - elements = [ - dict( - id=r[0], - name=r[1], - label=r[2], - description=r[3], - testrl=r[4], - teenrl=r[5], - order_index=r[6], - ) - for r in cur_el.fetchall() - ] - conn_el.close() - # Concept mapping - activity_ids = [a["id"] for a in activities] - concepts_map = {} - if activity_ids: - placeholders = ",".join("?" for _ in activity_ids) - has_uid = _table_has_columns(cur, "activity_concept", ("concept_uid",)) - if _table_has_columns(cur, "activity_concept", ("soa_id",)): - if has_uid: - cur.execute( - f"SELECT activity_id, concept_code, concept_title, concept_uid FROM activity_concept WHERE soa_id=? AND activity_id IN ({placeholders})", - [soa_id] + activity_ids, - ) - else: - cur.execute( - f"SELECT activity_id, concept_code, concept_title, NULL as concept_uid FROM activity_concept WHERE soa_id=? AND activity_id IN ({placeholders})", - [soa_id] + activity_ids, - ) - else: - if has_uid: - cur.execute( - f"SELECT activity_id, concept_code, concept_title, concept_uid FROM activity_concept WHERE activity_id IN ({placeholders})", - activity_ids, - ) - else: - cur.execute( - f"SELECT activity_id, concept_code, concept_title, NULL as concept_uid FROM activity_concept WHERE activity_id IN ({placeholders})", - activity_ids, - ) - for aid, code, title, cuid in cur.fetchall(): - entry = {"code": code, "title": title} - if cuid: - entry["uid"] = cuid - concepts_map.setdefault(aid, []).append(entry) - snapshot = { - "soa_id": soa_id, - "soa_name": soa_name, - "study_id": study_id_val, - "study_label": study_label_val, - "study_description": study_description_val, - "version_label": version_label, - "frozen_at": datetime.now(timezone.utc).isoformat(), - "epochs": epochs, - "elements": elements, - "visits": visits, - "activities": activities, - "cells": cells, - "activity_concepts": concepts_map, - } - snap_json = json.dumps(snapshot) - cur.execute( - "INSERT INTO soa_freeze (soa_id, version_label, created_at, snapshot_json) VALUES (?,?,?,?)", - (soa_id, version_label, datetime.now(timezone.utc).isoformat(), snap_json), - ) - fid = cur.lastrowid - conn.commit() - conn.close() - return fid, version_label - - -def _diff_freezes(soa_id: int, left_id: int, right_id: int): - return _diff_freezes_limited(soa_id, left_id, right_id, limit=None) - - -def _diff_freezes_limited( - soa_id: int, left_id: int, right_id: int, limit: Optional[int] -): - left = _get_freeze(soa_id, left_id) - right = _get_freeze(soa_id, right_id) - if not left or not right: - raise HTTPException(404, "Freeze not found") - l_snap = left["snapshot"] - r_snap = right["snapshot"] - # Visits - l_vis = { - str(v["id"]): v - for v in l_snap.get("visits", []) - if isinstance(v, dict) and "id" in v - } - r_vis = { - str(v["id"]): v - for v in r_snap.get("visits", []) - if isinstance(v, dict) and "id" in v - } - visits_added_all = [r_vis[k] for k in r_vis.keys() - l_vis.keys()] - visits_removed_all = [l_vis[k] for k in l_vis.keys() - r_vis.keys()] - # Activities - l_act = { - str(a["id"]): a - for a in l_snap.get("activities", []) - if isinstance(a, dict) and "id" in a - } - r_act = { - str(a["id"]): a - for a in r_snap.get("activities", []) - if isinstance(a, dict) and "id" in a - } - acts_added_all = [r_act[k] for k in r_act.keys() - l_act.keys()] - acts_removed_all = [l_act[k] for k in l_act.keys() - r_act.keys()] - # Cells (status changes). Newer snapshots key by instance_id; older ones used visit_id. - - def _cell_key(cell: dict) -> Optional[tuple[str, int, int]]: - if not isinstance(cell, dict): - return None - activity_id = cell.get("activity_id") - if activity_id is None: - return None - if cell.get("instance_id") is not None: - return ("instance", int(cell["instance_id"]), int(activity_id)) - if cell.get("visit_id") is not None: - return ("visit", int(cell["visit_id"]), int(activity_id)) - return None - - def _normalize_cell(cell: dict) -> dict: - axis_type = ( - "instance" - if cell.get("instance_id") is not None - else "visit" - if cell.get("visit_id") is not None - else None - ) - axis_id = None - if axis_type == "instance": - axis_id = cell.get("instance_id") - elif axis_type == "visit": - axis_id = cell.get("visit_id") - return { - "axis_type": axis_type, - "axis_id": axis_id, - "instance_id": cell.get("instance_id"), - "visit_id": cell.get("visit_id"), - "activity_id": cell.get("activity_id"), - "status": cell.get("status"), - } - - def _build_cell_map(snapshot_cells: list[dict]) -> dict: - mapped = {} - for raw in snapshot_cells or []: - key = _cell_key(raw) - if not key: - continue - mapped[key] = _normalize_cell(raw) - return mapped - - l_cells = _build_cell_map(l_snap.get("cells", [])) - r_cells = _build_cell_map(r_snap.get("cells", [])) - cells_added_all = [r_cells[k] for k in r_cells.keys() - l_cells.keys()] - cells_removed_all = [l_cells[k] for k in l_cells.keys() - r_cells.keys()] - cells_changed_all = [] - for k in r_cells.keys() & l_cells.keys(): - if r_cells[k].get("status") != l_cells[k].get("status"): - cells_changed_all.append( - { - "axis_type": l_cells[k].get("axis_type"), - "axis_id": l_cells[k].get("axis_id"), - "visit_id": l_cells[k].get("visit_id"), - "instance_id": l_cells[k].get("instance_id"), - "activity_id": l_cells[k].get("activity_id"), - "old_status": l_cells[k].get("status"), - "new_status": r_cells[k].get("status"), - } - ) - # Concepts per activity with title change detection - l_concepts_map = l_snap.get("activity_concepts", {}) or {} - r_concepts_map = r_snap.get("activity_concepts", {}) or {} - concept_changes_all = [] - all_aids = set(map(str, l_concepts_map.keys())) | set( - map(str, r_concepts_map.keys()) - ) - - def _get_concept_list(m, key): - # Support snapshots where JSON serialization converted int keys to strings - if key in m: - return m[key] or [] - if key.isdigit() and int(key) in m: - return m[int(key)] or [] - return [] - - for aid in all_aids: - la = _get_concept_list(l_concepts_map, aid) - ra = _get_concept_list(r_concepts_map, aid) - l_set = {c["code"] for c in la if isinstance(c, dict)} - r_set = {c["code"] for c in ra if isinstance(c, dict)} - added = sorted(list(r_set - l_set)) - removed = sorted(list(l_set - r_set)) - title_changes = [] - for code in sorted(list(l_set & r_set)): - l_title = next((c["title"] for c in la if c.get("code") == code), None) - r_title = next((c["title"] for c in ra if c.get("code") == code), None) - if l_title is not None and r_title is not None and l_title != r_title: - title_changes.append( - {"code": code, "old_title": l_title, "new_title": r_title} - ) - if added or removed or title_changes: - concept_changes_all.append( - { - "activity_id": aid, - "added": added, - "removed": removed, - "title_changes": title_changes, - } - ) - - # Apply limit truncation if provided and >0 - def _truncate(lst): - if limit and limit > 0 and len(lst) > limit: - return lst[:limit], True - return lst, False - - visits_added, visits_added_trunc = _truncate(visits_added_all) - visits_removed, visits_removed_trunc = _truncate(visits_removed_all) - acts_added, acts_added_trunc = _truncate(acts_added_all) - acts_removed, acts_removed_trunc = _truncate(acts_removed_all) - cells_added, cells_added_trunc = _truncate(cells_added_all) - cells_removed, cells_removed_trunc = _truncate(cells_removed_all) - cells_changed, cells_changed_trunc = _truncate(cells_changed_all) - concept_changes, concept_changes_trunc = _truncate(concept_changes_all) - meta = { - "limit": limit, - "visits": { - "added_total": len(visits_added_all), - "removed_total": len(visits_removed_all), - "added_truncated": visits_added_trunc, - "removed_truncated": visits_removed_trunc, - }, - "activities": { - "added_total": len(acts_added_all), - "removed_total": len(acts_removed_all), - "added_truncated": acts_added_trunc, - "removed_truncated": acts_removed_trunc, - }, - "cells": { - "added_total": len(cells_added_all), - "removed_total": len(cells_removed_all), - "changed_total": len(cells_changed_all), - "added_truncated": cells_added_trunc, - "removed_truncated": cells_removed_trunc, - "changed_truncated": cells_changed_trunc, - }, - "concepts": { - "changes_total": len(concept_changes_all), - "changes_truncated": concept_changes_trunc, - }, - } - return { - "left": { - "id": left["id"], - "label": left["version_label"], - "created_at": left["created_at"], - }, - "right": { - "id": right["id"], - "label": right["version_label"], - "created_at": right["created_at"], - }, - "visits": {"added": visits_added, "removed": visits_removed}, - "activities": {"added": acts_added, "removed": acts_removed}, - "cells": { - "added": cells_added, - "removed": cells_removed, - "changed": cells_changed, - }, - "concepts": concept_changes, - "meta": meta, - } - - -def _rollback_freeze(soa_id: int, freeze_id: int) -> dict: - freeze = _get_freeze(soa_id, freeze_id) - if not freeze: - raise HTTPException(404, "Freeze not found") - snap = freeze["snapshot"] - if snap.get("soa_id") != soa_id: - raise HTTPException(400, "Snapshot SoA mismatch") - visits = snap.get("visits", []) - activities = snap.get("activities", []) - cells = snap.get("cells", []) - elements = snap.get("elements", []) - concepts_map = snap.get("activity_concepts", {}) or {} - conn = _connect() - cur = conn.cursor() - # Clear existing - # Order matters: delete cells, then concepts (while activity rows still exist), then activities, then visits. - cur.execute("DELETE FROM matrix_cells WHERE soa_id=?", (soa_id,)) - cur.execute( - "DELETE FROM activity_concept WHERE activity_id IN (SELECT id FROM activity WHERE soa_id=? )", - (soa_id,), - ) - cur.execute("DELETE FROM biomedical_concept WHERE soa_id=?", (soa_id,)) - cur.execute("DELETE FROM alias_code WHERE soa_id=?", (soa_id,)) - cur.execute("DELETE FROM code WHERE soa_id=?", (soa_id,)) - cur.execute("DELETE FROM code_association WHERE soa_id=?", (soa_id,)) - cur.execute("DELETE FROM activity WHERE soa_id=?", (soa_id,)) - cur.execute("DELETE FROM visit WHERE soa_id=?", (soa_id,)) - cur.execute("DELETE FROM element WHERE soa_id=?", (soa_id,)) - # Reinsert visits mapping old id->new id - visit_id_map = {} - for v in sorted(visits, key=lambda x: x.get("order_index", 0)): - cur.execute( - "INSERT INTO visit (soa_id,name,label,order_index) VALUES (?,?,?,?)", - ( - soa_id, - v.get("name"), - v.get("label") or None, - v.get("order_index"), - ), - ) - new_id = cur.lastrowid - visit_id_map[v.get("id")] = new_id - # Reinsert activities mapping old id->new id - activity_id_map = {} - for a in sorted(activities, key=lambda x: x.get("order_index", 0)): - cur.execute( - "INSERT INTO activity (soa_id,name,order_index) VALUES (?,?,?)", - (soa_id, a.get("name"), a.get("order_index")), - ) - new_id = cur.lastrowid - activity_id_map[a.get("id")] = new_id - # Reinsert cells - inserted_cells = 0 - for c in cells: - old_vid = c.get("visit_id") - old_aid = c.get("activity_id") - status = c.get("status", "").strip() - if status == "": - continue - vid = visit_id_map.get(old_vid) - aid = activity_id_map.get(old_aid) - if vid and aid: - cur.execute( - "INSERT INTO matrix_cells (soa_id, visit_id, activity_id, status) VALUES (?,?,?,?)", - (soa_id, vid, aid, status), - ) - inserted_cells += 1 - # Reinsert concepts - # Reinsert elements - elements_restored = 0 - for el in sorted(elements, key=lambda x: x.get("order_index", 0)): - cur.execute( - "INSERT INTO element (soa_id,name,label,description,testrl,teenrl,order_index,created_at) VALUES (?,?,?,?,?,?,?,?)", - ( - soa_id, - el.get("name"), - el.get("label"), - el.get("description"), - el.get("testrl"), - el.get("teenrl"), - el.get("order_index"), - datetime.now(timezone.utc).isoformat(), - ), - ) - elements_restored += 1 - inserted_concepts = 0 - for old_aid, concept_list in concepts_map.items(): - new_aid = activity_id_map.get(int(old_aid)) - if not new_aid: - continue - # Fetch activity_uid for the new activity id - cur.execute("SELECT activity_uid FROM activity WHERE id=?", (new_aid,)) - row_uid = cur.fetchone() - new_activity_uid = row_uid[0] if row_uid else None - ac_has_soa = _table_has_columns(cur, "activity_concept", ("soa_id",)) - ac_has_actuid = _table_has_columns(cur, "activity_concept", ("activity_uid",)) - ac_has_conceptuid = _table_has_columns( - cur, "activity_concept", ("concept_uid",) - ) - for c in concept_list: - code = c.get("code") - title = c.get("title") or code - if not code: - continue - # Insert concept mapping; include soa_id if column exists - concept_uid = ( - _get_next_concept_uid(cur, soa_id) if ac_has_conceptuid else None - ) - if ac_has_soa and ac_has_actuid: - if ac_has_conceptuid: - cur.execute( - "INSERT INTO activity_concept (soa_id, activity_id, activity_uid, concept_uid, concept_code, concept_title) VALUES (?,?,?,?,?,?)", - (soa_id, new_aid, new_activity_uid, concept_uid, code, title), - ) - else: - cur.execute( - "INSERT INTO activity_concept (soa_id, activity_id, activity_uid, concept_code, concept_title) VALUES (?,?,?,?,?)", - (soa_id, new_aid, new_activity_uid, code, title), - ) - elif ac_has_actuid: - if ac_has_conceptuid: - cur.execute( - "INSERT INTO activity_concept (activity_id, activity_uid, concept_uid, concept_code, concept_title) VALUES (?,?,?,?,?)", - (new_aid, new_activity_uid, concept_uid, code, title), - ) - else: - cur.execute( - "INSERT INTO activity_concept (activity_id, activity_uid, concept_code, concept_title) VALUES (?,?,?,?)", - (new_aid, new_activity_uid, code, title), - ) - elif ac_has_soa: - if ac_has_conceptuid: - cur.execute( - "INSERT INTO activity_concept (soa_id, activity_id, concept_uid, concept_code, concept_title) VALUES (?,?,?,?,?)", - (soa_id, new_aid, concept_uid, code, title), - ) - else: - cur.execute( - "INSERT INTO activity_concept (soa_id, activity_id, concept_code, concept_title) VALUES (?,?,?,?)", - (soa_id, new_aid, code, title), - ) - else: - if ac_has_conceptuid: - cur.execute( - "INSERT INTO activity_concept (activity_id, concept_uid, concept_code, concept_title) VALUES (?,?,?,?)", - (new_aid, concept_uid, code, title), - ) - else: - cur.execute( - "INSERT INTO activity_concept (activity_id, concept_code, concept_title) VALUES (?,?,?)", - (new_aid, code, title), - ) - _upsert_biomedical_concept(cur, soa_id, concept_uid, title, code) - inserted_concepts += 1 - conn.commit() - conn.close() - return { - "rollback_freeze_id": freeze_id, - "visits_restored": len(visits), - "activities_restored": len(activities), - "cells_restored": inserted_cells, - "concept_mappings_restored": inserted_concepts, - "elements_restored": elements_restored, - } - - -def _record_rollback_audit(soa_id: int, freeze_id: int, stats: dict): - conn = _connect() - cur = conn.cursor() - cur.execute( - "INSERT INTO rollback_audit (soa_id, freeze_id, performed_at, visits_restored, activities_restored, cells_restored, concepts_restored, elements_restored) VALUES (?,?,?,?,?,?,?,?)", - ( - soa_id, - freeze_id, - datetime.now(timezone.utc).isoformat(), - stats.get("visits_restored"), - stats.get("activities_restored"), - stats.get("cells_restored"), - stats.get("concept_mappings_restored"), - stats.get("elements_restored"), - ), - ) - conn.commit() - conn.close() - - def _record_reorder_audit( soa_id: int, entity_type: str, old_order: list[int], new_order: list[int] ): @@ -999,48 +466,6 @@ def _fetch_arms_for_edit(soa_id: int) -> list[dict]: return [] -def _list_rollback_audit(soa_id: int) -> list[dict]: - conn = _connect() - cur = conn.cursor() - cur.execute( - "SELECT id, freeze_id, performed_at, visits_restored, activities_restored, cells_restored, concepts_restored FROM rollback_audit WHERE soa_id=? ORDER BY id DESC", - (soa_id,), - ) - rows = [ - { - "id": r[0], - "freeze_id": r[1], - "performed_at": r[2], - "visits_restored": r[3], - "activities_restored": r[4], - "cells_restored": r[5], - "concepts_restored": r[6], - } - for r in cur.fetchall() - ] - conn.close() - return rows - - -def _rollback_preview(soa_id: int, freeze_id: int) -> dict: - freeze = _get_freeze(soa_id, freeze_id) - if not freeze: - raise HTTPException(404, "Freeze not found") - snap = freeze["snapshot"] - visits = snap.get("visits", []) - activities = snap.get("activities", []) - cells = [c for c in snap.get("cells", []) if c.get("status", "").strip() != ""] - concepts_map = snap.get("activity_concepts", {}) or {} - return { - "freeze_id": freeze_id, - "version_label": freeze.get("version_label"), - "visits_to_restore": len(visits), - "activities_to_restore": len(activities), - "cells_to_restore": len(cells), - "concept_mappings_to_restore": sum(len(v) for v in concepts_map.values()), - } - - def _fetch_matrix(soa_id: int): conn = _connect() cur = conn.cursor() @@ -4327,8 +3752,6 @@ def ui_edit(request: Request, soa_id: int): last_fetch_relative = f"{secs // 60}m ago" else: last_fetch_relative = f"{secs // 3600}h ago" - freeze_list = _list_freezes(soa_id) - last_frozen_at = freeze_list[0]["created_at"] if freeze_list else None # Study metadata for edit form conn_meta = _connect() cur_meta = conn_meta.cursor() @@ -4563,6 +3986,25 @@ def ui_edit(request: Request, soa_id: int): instances_by_timeline[timeline_key] = [] instances_by_timeline[timeline_key].append(inst) + # Activities per timeline: an activity is shown in timeline T's matrix + # if any matrix_cells row connects it to an instance whose + # member_of_timeline == T. The instance->timeline link (set on the + # study_timing page) is the authoritative criterion. + instance_timeline = { + inst["id"]: (inst.get("member_of_timeline") or "unassigned") + for inst in instances + } + activity_ids_by_timeline: dict = {tl: set() for tl in instances_by_timeline.keys()} + for c in cells: + tl = instance_timeline.get(c["instance_id"]) + if tl is None or tl not in activity_ids_by_timeline: + continue + activity_ids_by_timeline[tl].add(c["activity_id"]) + activities_by_timeline: dict = { + tl: [a for a in activities_page if a["id"] in ids] + for tl, ids in activity_ids_by_timeline.items() + } + # Determine default timeline (main_timeline or first available) default_timeline = None for tl in timelines: @@ -4604,6 +4046,69 @@ def ui_edit(request: Request, soa_id: int): schedule_timelines_options = get_schedule_timeline(soa_id) instance_options = get_scheduled_activity_instance(soa_id) + # Objectives + Endpoints with DDF level decode lookups + c188725_map = _get_ddf_ct_codelist_map("C188725") + c188726_map = _get_ddf_ct_codelist_map("C188726") + objective_level_options = sorted({v for v in c188725_map.values() if v}) + endpoint_level_options = sorted({v for v in c188726_map.values() if v}) + conn_obj = _connect() + cur_obj = conn_obj.cursor() + cur_obj.execute( + "SELECT code_uid, code FROM code_association " + "WHERE soa_id=? AND codelist_code IN ('C188725','C188726')", + (soa_id,), + ) + level_code_to_sv: dict = {} + for code_uid, code_val in cur_obj.fetchall(): + level_code_to_sv[code_uid] = code_val or "" + cur_obj.execute( + "SELECT id,objective_uid,name,label,description,text," + "level_code_uid,order_index " + "FROM objective WHERE soa_id=? ORDER BY order_index, id", + (soa_id,), + ) + objectives = [ + { + "id": r[0], + "objective_uid": r[1], + "name": r[2], + "label": r[3], + "description": r[4], + "text": r[5], + "level_code_uid": r[6], + "level": level_code_to_sv.get(r[6], ""), + "order_index": r[7], + } + for r in cur_obj.fetchall() + ] + cur_obj.execute( + "SELECT id,endpoint_uid,objective_uid,name,label,description," + "text,purpose,level_code_uid,order_index " + "FROM endpoint WHERE soa_id=? ORDER BY order_index, id", + (soa_id,), + ) + endpoints_by_objective: dict = {} + orphan_endpoints: list = [] + for r in cur_obj.fetchall(): + ep = { + "id": r[0], + "endpoint_uid": r[1], + "objective_uid": r[2], + "name": r[3], + "label": r[4], + "description": r[5], + "text": r[6], + "purpose": r[7], + "level_code_uid": r[8], + "level": level_code_to_sv.get(r[8], ""), + "order_index": r[9], + } + if ep["objective_uid"]: + endpoints_by_objective.setdefault(ep["objective_uid"], []).append(ep) + else: + orphan_endpoints.append(ep) + conn_obj.close() + return templates.TemplateResponse( request, "edit.html", @@ -4627,9 +4132,6 @@ def ui_edit(request: Request, soa_id: int): "concepts_diag": concepts_diag, "concepts_last_fetch_iso": last_fetch_iso, "concepts_last_fetch_relative": last_fetch_relative, - "freezes": freeze_list, - "freeze_count": len(freeze_list), - "last_frozen_at": last_frozen_at, **study_meta, "protocol_terminology_C174222": protocol_terminology_C174222, "ddf_terminology_C188727": ddf_terminology_C188727, @@ -4641,9 +4143,15 @@ def ui_edit(request: Request, soa_id: int): "timings": timings, "timelines": timelines, "instances_by_timeline": instances_by_timeline, + "activities_by_timeline": activities_by_timeline, "default_timeline": default_timeline, "footnotes": footnotes, "superscript_map": superscript_map, + "objectives": objectives, + "endpoints_by_objective": endpoints_by_objective, + "orphan_endpoints": orphan_endpoints, + "objective_level_options": objective_level_options, + "endpoint_level_options": endpoint_level_options, }, ) diff --git a/src/soa_builder/web/audit.py b/src/soa_builder/web/audit.py index a45a7db1..70bea7ac 100644 --- a/src/soa_builder/web/audit.py +++ b/src/soa_builder/web/audit.py @@ -419,3 +419,57 @@ def _record_footnote_audit( conn.close() except Exception as e: logger.warning("Failed recording footnote audit: %s", e) + + +def _record_objective_audit( + soa_id: int, + action: str, + objective_id: Optional[int], + before: Optional[Dict[str, Any]] = None, + after: Optional[Dict[str, Any]] = None, +): + try: + conn = _connect() + cur = conn.cursor() + cur.execute( + "INSERT INTO objective_audit (soa_id, objective_id, action, before_json, after_json, performed_at) VALUES (?,?,?,?,?,?)", + ( + soa_id, + objective_id, + action, + json.dumps(before) if before else None, + json.dumps(after) if after else None, + datetime.now(timezone.utc).isoformat(), + ), + ) + conn.commit() + conn.close() + except Exception as e: + logger.warning("Failed recording objective audit: %s", e) + + +def _record_endpoint_audit( + soa_id: int, + action: str, + endpoint_id: Optional[int], + before: Optional[Dict[str, Any]] = None, + after: Optional[Dict[str, Any]] = None, +): + try: + conn = _connect() + cur = conn.cursor() + cur.execute( + "INSERT INTO endpoint_audit (soa_id, endpoint_id, action, before_json, after_json, performed_at) VALUES (?,?,?,?,?,?)", + ( + soa_id, + endpoint_id, + action, + json.dumps(before) if before else None, + json.dumps(after) if after else None, + datetime.now(timezone.utc).isoformat(), + ), + ) + conn.commit() + conn.close() + except Exception as e: + logger.warning("Failed recording endpoint audit: %s", e) diff --git a/src/soa_builder/web/migrate_database.py b/src/soa_builder/web/migrate_database.py index d9299498..844597d3 100644 --- a/src/soa_builder/web/migrate_database.py +++ b/src/soa_builder/web/migrate_database.py @@ -1717,6 +1717,32 @@ def _migrate_activity_concept_dss_add_display(): logger.warning("_migrate_activity_concept_dss_add_display failed: %s", e) +def _migrate_activity_concept_dss_add_extension_attribute_uid(): + """Add extension_attribute_uid column to activity_concept_dss. + + Stores the immutable ExtensionAttribute_N identifier for each DSS + assignment so USDM exports produce stable IDs across runs. + """ + try: + conn = _connect() + cur = conn.cursor() + cur.execute("PRAGMA table_info(activity_concept_dss)") + cols = {r[1] for r in cur.fetchall()} + if "extension_attribute_uid" not in cols: + cur.execute( + "ALTER TABLE activity_concept_dss" + " ADD COLUMN extension_attribute_uid TEXT" + ) + conn.commit() + logger.info("Added extension_attribute_uid column to activity_concept_dss") + conn.close() + except Exception as e: + logger.warning( + "_migrate_activity_concept_dss_add_extension_attribute_uid failed: %s", + e, + ) + + def _migrate_drop_protocol_terminology_tables(): """Drop the legacy protocol_terminology and protocol_terminology_audit tables. @@ -1750,3 +1776,103 @@ def _migrate_drop_ddf_terminology_tables(): conn.close() except Exception as e: logger.warning("_migrate_drop_ddf_terminology_tables failed: %s", e) + + +def _migrate_add_objective_table(): + """Create the objective table for USDM study objectives.""" + try: + conn = _connect() + cur = conn.cursor() + cur.execute( + """CREATE TABLE IF NOT EXISTS objective ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + soa_id INTEGER NOT NULL, + objective_uid TEXT NOT NULL, + name TEXT NOT NULL, + label TEXT, + description TEXT, + text TEXT, + level_code_uid TEXT, + order_index INTEGER, + UNIQUE(soa_id, objective_uid) + )""" + ) + conn.commit() + conn.close() + logger.info("_migrate_add_objective_table created objective table") + except Exception as e: + logger.warning("_migrate_add_objective_table failed: %s", e) + + +def _migrate_add_objective_audit_table(): + """Create objective_audit table for tracking objective mutations.""" + try: + conn = _connect() + cur = conn.cursor() + cur.execute( + """CREATE TABLE IF NOT EXISTS objective_audit ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + soa_id INTEGER NOT NULL, + objective_id INTEGER, + action TEXT NOT NULL, + before_json TEXT, + after_json TEXT, + performed_at TEXT NOT NULL + )""" + ) + conn.commit() + conn.close() + logger.info("_migrate_add_objective_audit_table created objective_audit table") + except Exception as e: + logger.warning("_migrate_add_objective_audit_table failed: %s", e) + + +def _migrate_add_endpoint_table(): + """Create the endpoint table for USDM study endpoints.""" + try: + conn = _connect() + cur = conn.cursor() + cur.execute( + """CREATE TABLE IF NOT EXISTS endpoint ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + soa_id INTEGER NOT NULL, + endpoint_uid TEXT NOT NULL, + objective_uid TEXT, + name TEXT NOT NULL, + label TEXT, + description TEXT, + text TEXT, + purpose TEXT, + level_code_uid TEXT, + order_index INTEGER, + UNIQUE(soa_id, endpoint_uid) + )""" + ) + conn.commit() + conn.close() + logger.info("_migrate_add_endpoint_table created endpoint table") + except Exception as e: + logger.warning("_migrate_add_endpoint_table failed: %s", e) + + +def _migrate_add_endpoint_audit_table(): + """Create endpoint_audit table for tracking endpoint mutations.""" + try: + conn = _connect() + cur = conn.cursor() + cur.execute( + """CREATE TABLE IF NOT EXISTS endpoint_audit ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + soa_id INTEGER NOT NULL, + endpoint_id INTEGER, + action TEXT NOT NULL, + before_json TEXT, + after_json TEXT, + performed_at TEXT NOT NULL + )""" + ) + conn.commit() + conn.close() + logger.info("_migrate_add_endpoint_audit_table created endpoint_audit table") + except Exception as e: + logger.warning("_migrate_add_endpoint_audit_table failed: %s", e) diff --git a/src/soa_builder/web/routers/_freeze_helpers.py b/src/soa_builder/web/routers/_freeze_helpers.py new file mode 100644 index 00000000..11dc3a48 --- /dev/null +++ b/src/soa_builder/web/routers/_freeze_helpers.py @@ -0,0 +1,664 @@ +"""Freeze/rollback helpers for SOA snapshots. + +Extracted from app.py so the freeze router can import them directly +without the previous lazy-import circularity workaround. +""" + +import json +import logging +from datetime import datetime, timezone +from typing import Optional + +from fastapi import HTTPException + +from ..db import _connect +from ..utils import ( + soa_exists, + get_next_concept_uid as _get_next_concept_uid, + table_has_columns as _table_has_columns, +) + +logger = logging.getLogger("soa_builder.web.routers._freeze_helpers") + + +def _list_freezes(soa_id: int): + conn = _connect() + cur = conn.cursor() + cur.execute( + "SELECT id, version_label, created_at FROM soa_freeze" + " WHERE soa_id=? ORDER BY id DESC", + (soa_id,), + ) + rows = [dict(id=r[0], version_label=r[1], created_at=r[2]) for r in cur.fetchall()] + conn.close() + return rows + + +def _get_freeze(soa_id: int, freeze_id: int): + conn = _connect() + cur = conn.cursor() + cur.execute( + "SELECT id, version_label, created_at, snapshot_json" + " FROM soa_freeze WHERE id=? AND soa_id=?", + (freeze_id, soa_id), + ) + row = cur.fetchone() + conn.close() + if not row: + return None + try: + snap = json.loads(row[3]) + except Exception: + snap = {"error": "Corrupt snapshot"} + return { + "id": row[0], + "version_label": row[1], + "created_at": row[2], + "snapshot": snap, + } + + +def _create_freeze(soa_id: int, version_label: Optional[str]): + # Lazy imports to avoid circular dependency with app.py + from ..app import _fetch_matrix + + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + conn = _connect() + cur = conn.cursor() + cur.execute("SELECT version_label FROM soa_freeze WHERE soa_id=?", (soa_id,)) + existing_labels = {r[0] for r in cur.fetchall()} + if not version_label or not version_label.strip(): + n = 1 + while f"v{n}" in existing_labels: + n += 1 + version_label = f"v{n}" + else: + version_label = version_label.strip() + if version_label in existing_labels: + raise HTTPException(400, "Version label already exists for this SOA") + cur.execute( + "SELECT name, created_at, study_id, study_label, study_description" + " FROM soa WHERE id=?", + (soa_id,), + ) + row = cur.fetchone() + soa_name = row[0] if row else f"SOA {soa_id}" + study_id_val = row[2] if row else None + study_label_val = row[3] if row else None + study_description_val = row[4] if row else None + visits, activities, cells = _fetch_matrix(soa_id) + conn2 = _connect() + cur2 = conn2.cursor() + cur2.execute( + "SELECT id,name,order_index,epoch_seq,epoch_label,epoch_description" + " FROM epoch WHERE soa_id=? ORDER BY order_index", + (soa_id,), + ) + epochs = [ + dict( + id=r[0], + name=r[1], + order_index=r[2], + epoch_seq=r[3], + epoch_label=r[4], + epoch_description=r[5], + ) + for r in cur2.fetchall() + ] + conn2.close() + conn_el = _connect() + cur_el = conn_el.cursor() + cur_el.execute( + "SELECT id,name,label,description,testrl,teenrl,order_index" + " FROM element WHERE soa_id=? ORDER BY order_index", + (soa_id,), + ) + elements = [ + dict( + id=r[0], + name=r[1], + label=r[2], + description=r[3], + testrl=r[4], + teenrl=r[5], + order_index=r[6], + ) + for r in cur_el.fetchall() + ] + conn_el.close() + activity_ids = [a["id"] for a in activities] + concepts_map: dict = {} + if activity_ids: + placeholders = ",".join("?" for _ in activity_ids) + has_uid = _table_has_columns(cur, "activity_concept", ("concept_uid",)) + if _table_has_columns(cur, "activity_concept", ("soa_id",)): + if has_uid: + cur.execute( + f"SELECT activity_id, concept_code, concept_title, concept_uid" + f" FROM activity_concept WHERE soa_id=? AND activity_id IN ({placeholders})", + [soa_id] + activity_ids, + ) + else: + cur.execute( + f"SELECT activity_id, concept_code, concept_title, NULL as concept_uid" + f" FROM activity_concept WHERE soa_id=? AND activity_id IN ({placeholders})", + [soa_id] + activity_ids, + ) + else: + if has_uid: + cur.execute( + f"SELECT activity_id, concept_code, concept_title, concept_uid" + f" FROM activity_concept WHERE activity_id IN ({placeholders})", + activity_ids, + ) + else: + cur.execute( + f"SELECT activity_id, concept_code, concept_title, NULL as concept_uid" + f" FROM activity_concept WHERE activity_id IN ({placeholders})", + activity_ids, + ) + for aid, code, title, cuid in cur.fetchall(): + entry = {"code": code, "title": title} + if cuid: + entry["uid"] = cuid + concepts_map.setdefault(aid, []).append(entry) + snapshot = { + "soa_id": soa_id, + "soa_name": soa_name, + "study_id": study_id_val, + "study_label": study_label_val, + "study_description": study_description_val, + "version_label": version_label, + "frozen_at": datetime.now(timezone.utc).isoformat(), + "epochs": epochs, + "elements": elements, + "visits": visits, + "activities": activities, + "cells": cells, + "activity_concepts": concepts_map, + } + snap_json = json.dumps(snapshot) + cur.execute( + "INSERT INTO soa_freeze (soa_id, version_label, created_at, snapshot_json)" + " VALUES (?,?,?,?)", + ( + soa_id, + version_label, + datetime.now(timezone.utc).isoformat(), + snap_json, + ), + ) + fid = cur.lastrowid + conn.commit() + conn.close() + return fid, version_label + + +def _diff_freezes(soa_id: int, left_id: int, right_id: int): + return _diff_freezes_limited(soa_id, left_id, right_id, limit=None) + + +def _diff_freezes_limited( + soa_id: int, left_id: int, right_id: int, limit: Optional[int] +): + left = _get_freeze(soa_id, left_id) + right = _get_freeze(soa_id, right_id) + if not left or not right: + raise HTTPException(404, "Freeze not found") + l_snap = left["snapshot"] + r_snap = right["snapshot"] + l_vis = { + str(v["id"]): v + for v in l_snap.get("visits", []) + if isinstance(v, dict) and "id" in v + } + r_vis = { + str(v["id"]): v + for v in r_snap.get("visits", []) + if isinstance(v, dict) and "id" in v + } + visits_added_all = [r_vis[k] for k in r_vis.keys() - l_vis.keys()] + visits_removed_all = [l_vis[k] for k in l_vis.keys() - r_vis.keys()] + l_act = { + str(a["id"]): a + for a in l_snap.get("activities", []) + if isinstance(a, dict) and "id" in a + } + r_act = { + str(a["id"]): a + for a in r_snap.get("activities", []) + if isinstance(a, dict) and "id" in a + } + acts_added_all = [r_act[k] for k in r_act.keys() - l_act.keys()] + acts_removed_all = [l_act[k] for k in l_act.keys() - r_act.keys()] + + def _cell_key(cell: dict): + if not isinstance(cell, dict): + return None + activity_id = cell.get("activity_id") + if activity_id is None: + return None + if cell.get("instance_id") is not None: + return ("instance", int(cell["instance_id"]), int(activity_id)) + if cell.get("visit_id") is not None: + return ("visit", int(cell["visit_id"]), int(activity_id)) + return None + + def _normalize_cell(cell: dict) -> dict: + axis_type = ( + "instance" + if cell.get("instance_id") is not None + else "visit" + if cell.get("visit_id") is not None + else None + ) + axis_id = None + if axis_type == "instance": + axis_id = cell.get("instance_id") + elif axis_type == "visit": + axis_id = cell.get("visit_id") + return { + "axis_type": axis_type, + "axis_id": axis_id, + "instance_id": cell.get("instance_id"), + "visit_id": cell.get("visit_id"), + "activity_id": cell.get("activity_id"), + "status": cell.get("status"), + } + + def _build_cell_map(snapshot_cells): + mapped = {} + for raw in snapshot_cells or []: + key = _cell_key(raw) + if not key: + continue + mapped[key] = _normalize_cell(raw) + return mapped + + l_cells = _build_cell_map(l_snap.get("cells", [])) + r_cells = _build_cell_map(r_snap.get("cells", [])) + cells_added_all = [r_cells[k] for k in r_cells.keys() - l_cells.keys()] + cells_removed_all = [l_cells[k] for k in l_cells.keys() - r_cells.keys()] + cells_changed_all = [] + for k in r_cells.keys() & l_cells.keys(): + if r_cells[k].get("status") != l_cells[k].get("status"): + cells_changed_all.append( + { + "axis_type": l_cells[k].get("axis_type"), + "axis_id": l_cells[k].get("axis_id"), + "visit_id": l_cells[k].get("visit_id"), + "instance_id": l_cells[k].get("instance_id"), + "activity_id": l_cells[k].get("activity_id"), + "old_status": l_cells[k].get("status"), + "new_status": r_cells[k].get("status"), + } + ) + l_concepts_map = l_snap.get("activity_concepts", {}) or {} + r_concepts_map = r_snap.get("activity_concepts", {}) or {} + concept_changes_all = [] + all_aids = set(map(str, l_concepts_map.keys())) | set( + map(str, r_concepts_map.keys()) + ) + + def _get_concept_list(m, key): + if key in m: + return m[key] or [] + if key.isdigit() and int(key) in m: + return m[int(key)] or [] + return [] + + for aid in all_aids: + la = _get_concept_list(l_concepts_map, aid) + ra = _get_concept_list(r_concepts_map, aid) + l_set = {c["code"] for c in la if isinstance(c, dict)} + r_set = {c["code"] for c in ra if isinstance(c, dict)} + added = sorted(list(r_set - l_set)) + removed = sorted(list(l_set - r_set)) + title_changes = [] + for code in sorted(list(l_set & r_set)): + l_title = next((c["title"] for c in la if c.get("code") == code), None) + r_title = next((c["title"] for c in ra if c.get("code") == code), None) + if l_title is not None and r_title is not None and l_title != r_title: + title_changes.append( + {"code": code, "old_title": l_title, "new_title": r_title} + ) + if added or removed or title_changes: + concept_changes_all.append( + { + "activity_id": aid, + "added": added, + "removed": removed, + "title_changes": title_changes, + } + ) + + def _truncate(lst): + if limit and limit > 0 and len(lst) > limit: + return lst[:limit], True + return lst, False + + visits_added, visits_added_trunc = _truncate(visits_added_all) + visits_removed, visits_removed_trunc = _truncate(visits_removed_all) + acts_added, acts_added_trunc = _truncate(acts_added_all) + acts_removed, acts_removed_trunc = _truncate(acts_removed_all) + cells_added, cells_added_trunc = _truncate(cells_added_all) + cells_removed, cells_removed_trunc = _truncate(cells_removed_all) + cells_changed, cells_changed_trunc = _truncate(cells_changed_all) + concept_changes, concept_changes_trunc = _truncate(concept_changes_all) + meta = { + "limit": limit, + "visits": { + "added_total": len(visits_added_all), + "removed_total": len(visits_removed_all), + "added_truncated": visits_added_trunc, + "removed_truncated": visits_removed_trunc, + }, + "activities": { + "added_total": len(acts_added_all), + "removed_total": len(acts_removed_all), + "added_truncated": acts_added_trunc, + "removed_truncated": acts_removed_trunc, + }, + "cells": { + "added_total": len(cells_added_all), + "removed_total": len(cells_removed_all), + "changed_total": len(cells_changed_all), + "added_truncated": cells_added_trunc, + "removed_truncated": cells_removed_trunc, + "changed_truncated": cells_changed_trunc, + }, + "concepts": { + "changes_total": len(concept_changes_all), + "changes_truncated": concept_changes_trunc, + }, + } + return { + "left": { + "id": left["id"], + "label": left["version_label"], + "created_at": left["created_at"], + }, + "right": { + "id": right["id"], + "label": right["version_label"], + "created_at": right["created_at"], + }, + "visits": {"added": visits_added, "removed": visits_removed}, + "activities": {"added": acts_added, "removed": acts_removed}, + "cells": { + "added": cells_added, + "removed": cells_removed, + "changed": cells_changed, + }, + "concepts": concept_changes, + "meta": meta, + } + + +def _rollback_freeze(soa_id: int, freeze_id: int) -> dict: + # Lazy import to avoid circular dependency with app.py + from ..app import _upsert_biomedical_concept + + freeze = _get_freeze(soa_id, freeze_id) + if not freeze: + raise HTTPException(404, "Freeze not found") + snap = freeze["snapshot"] + if snap.get("soa_id") != soa_id: + raise HTTPException(400, "Snapshot SoA mismatch") + visits = snap.get("visits", []) + activities = snap.get("activities", []) + cells = snap.get("cells", []) + elements = snap.get("elements", []) + concepts_map = snap.get("activity_concepts", {}) or {} + conn = _connect() + cur = conn.cursor() + cur.execute("DELETE FROM matrix_cells WHERE soa_id=?", (soa_id,)) + cur.execute( + "DELETE FROM activity_concept WHERE activity_id IN" + " (SELECT id FROM activity WHERE soa_id=? )", + (soa_id,), + ) + cur.execute("DELETE FROM biomedical_concept WHERE soa_id=?", (soa_id,)) + cur.execute("DELETE FROM alias_code WHERE soa_id=?", (soa_id,)) + cur.execute("DELETE FROM code WHERE soa_id=?", (soa_id,)) + cur.execute("DELETE FROM code_association WHERE soa_id=?", (soa_id,)) + cur.execute("DELETE FROM activity WHERE soa_id=?", (soa_id,)) + cur.execute("DELETE FROM visit WHERE soa_id=?", (soa_id,)) + cur.execute("DELETE FROM element WHERE soa_id=?", (soa_id,)) + visit_id_map = {} + for v in sorted(visits, key=lambda x: x.get("order_index", 0)): + cur.execute( + "INSERT INTO visit (soa_id,name,label,order_index) VALUES (?,?,?,?)", + ( + soa_id, + v.get("name"), + v.get("label") or None, + v.get("order_index"), + ), + ) + new_id = cur.lastrowid + visit_id_map[v.get("id")] = new_id + activity_id_map = {} + for a in sorted(activities, key=lambda x: x.get("order_index", 0)): + cur.execute( + "INSERT INTO activity (soa_id,name,order_index) VALUES (?,?,?)", + (soa_id, a.get("name"), a.get("order_index")), + ) + new_id = cur.lastrowid + activity_id_map[a.get("id")] = new_id + inserted_cells = 0 + for c in cells: + old_vid = c.get("visit_id") + old_aid = c.get("activity_id") + status = c.get("status", "").strip() + if status == "": + continue + vid = visit_id_map.get(old_vid) + aid = activity_id_map.get(old_aid) + if vid and aid: + cur.execute( + "INSERT INTO matrix_cells (soa_id, visit_id, activity_id, status)" + " VALUES (?,?,?,?)", + (soa_id, vid, aid, status), + ) + inserted_cells += 1 + elements_restored = 0 + for el in sorted(elements, key=lambda x: x.get("order_index", 0)): + cur.execute( + "INSERT INTO element" + " (soa_id,name,label,description,testrl,teenrl,order_index,created_at)" + " VALUES (?,?,?,?,?,?,?,?)", + ( + soa_id, + el.get("name"), + el.get("label"), + el.get("description"), + el.get("testrl"), + el.get("teenrl"), + el.get("order_index"), + datetime.now(timezone.utc).isoformat(), + ), + ) + elements_restored += 1 + inserted_concepts = 0 + for old_aid, concept_list in concepts_map.items(): + new_aid = activity_id_map.get(int(old_aid)) + if not new_aid: + continue + cur.execute("SELECT activity_uid FROM activity WHERE id=?", (new_aid,)) + row_uid = cur.fetchone() + new_activity_uid = row_uid[0] if row_uid else None + ac_has_soa = _table_has_columns(cur, "activity_concept", ("soa_id",)) + ac_has_actuid = _table_has_columns(cur, "activity_concept", ("activity_uid",)) + ac_has_conceptuid = _table_has_columns( + cur, "activity_concept", ("concept_uid",) + ) + for c in concept_list: + code = c.get("code") + title = c.get("title") or code + if not code: + continue + concept_uid = ( + _get_next_concept_uid(cur, soa_id) if ac_has_conceptuid else None + ) + if ac_has_soa and ac_has_actuid: + if ac_has_conceptuid: + cur.execute( + "INSERT INTO activity_concept" + " (soa_id, activity_id, activity_uid, concept_uid," + " concept_code, concept_title) VALUES (?,?,?,?,?,?)", + ( + soa_id, + new_aid, + new_activity_uid, + concept_uid, + code, + title, + ), + ) + else: + cur.execute( + "INSERT INTO activity_concept" + " (soa_id, activity_id, activity_uid, concept_code," + " concept_title) VALUES (?,?,?,?,?)", + (soa_id, new_aid, new_activity_uid, code, title), + ) + elif ac_has_actuid: + if ac_has_conceptuid: + cur.execute( + "INSERT INTO activity_concept" + " (activity_id, activity_uid, concept_uid, concept_code," + " concept_title) VALUES (?,?,?,?,?)", + (new_aid, new_activity_uid, concept_uid, code, title), + ) + else: + cur.execute( + "INSERT INTO activity_concept" + " (activity_id, activity_uid, concept_code, concept_title)" + " VALUES (?,?,?,?)", + (new_aid, new_activity_uid, code, title), + ) + elif ac_has_soa: + if ac_has_conceptuid: + cur.execute( + "INSERT INTO activity_concept" + " (soa_id, activity_id, concept_uid, concept_code," + " concept_title) VALUES (?,?,?,?,?)", + (soa_id, new_aid, concept_uid, code, title), + ) + else: + cur.execute( + "INSERT INTO activity_concept" + " (soa_id, activity_id, concept_code, concept_title)" + " VALUES (?,?,?,?)", + (soa_id, new_aid, code, title), + ) + else: + if ac_has_conceptuid: + cur.execute( + "INSERT INTO activity_concept" + " (activity_id, concept_uid, concept_code, concept_title)" + " VALUES (?,?,?,?)", + (new_aid, concept_uid, code, title), + ) + else: + cur.execute( + "INSERT INTO activity_concept" + " (activity_id, concept_code, concept_title)" + " VALUES (?,?,?)", + (new_aid, code, title), + ) + _upsert_biomedical_concept(cur, soa_id, concept_uid, title, code) + inserted_concepts += 1 + conn.commit() + conn.close() + return { + "rollback_freeze_id": freeze_id, + "visits_restored": len(visits), + "activities_restored": len(activities), + "cells_restored": inserted_cells, + "concept_mappings_restored": inserted_concepts, + "elements_restored": elements_restored, + } + + +def _record_rollback_audit(soa_id: int, freeze_id: int, stats: dict): + conn = _connect() + cur = conn.cursor() + cur.execute( + "INSERT INTO rollback_audit" + " (soa_id, freeze_id, performed_at, visits_restored," + " activities_restored, cells_restored, concepts_restored," + " elements_restored) VALUES (?,?,?,?,?,?,?,?)", + ( + soa_id, + freeze_id, + datetime.now(timezone.utc).isoformat(), + stats.get("visits_restored"), + stats.get("activities_restored"), + stats.get("cells_restored"), + stats.get("concept_mappings_restored"), + stats.get("elements_restored"), + ), + ) + conn.commit() + conn.close() + + +def _list_rollback_audit(soa_id: int) -> list: + conn = _connect() + cur = conn.cursor() + cur.execute( + "SELECT id, freeze_id, performed_at, visits_restored," + " activities_restored, cells_restored, concepts_restored" + " FROM rollback_audit WHERE soa_id=? ORDER BY id DESC", + (soa_id,), + ) + rows = [ + { + "id": r[0], + "freeze_id": r[1], + "performed_at": r[2], + "visits_restored": r[3], + "activities_restored": r[4], + "cells_restored": r[5], + "concepts_restored": r[6], + } + for r in cur.fetchall() + ] + conn.close() + return rows + + +def _delete_freeze(soa_id: int, freeze_id: int) -> bool: + """Delete a freeze row. Returns True if a row was deleted, False otherwise.""" + conn = _connect() + cur = conn.cursor() + cur.execute( + "DELETE FROM soa_freeze WHERE id=? AND soa_id=?", + (freeze_id, soa_id), + ) + deleted = cur.rowcount > 0 + conn.commit() + conn.close() + return deleted + + +def _rollback_preview(soa_id: int, freeze_id: int) -> dict: + freeze = _get_freeze(soa_id, freeze_id) + if not freeze: + raise HTTPException(404, "Freeze not found") + snap = freeze["snapshot"] + visits = snap.get("visits", []) + activities = snap.get("activities", []) + cells = [c for c in snap.get("cells", []) if c.get("status", "").strip() != ""] + concepts_map = snap.get("activity_concepts", {}) or {} + return { + "freeze_id": freeze_id, + "version_label": freeze.get("version_label"), + "visits_to_restore": len(visits), + "activities_to_restore": len(activities), + "cells_to_restore": len(cells), + "concept_mappings_to_restore": sum(len(v) for v in concepts_map.values()), + } diff --git a/src/soa_builder/web/routers/endpoints.py b/src/soa_builder/web/routers/endpoints.py new file mode 100644 index 00000000..46d3eb24 --- /dev/null +++ b/src/soa_builder/web/routers/endpoints.py @@ -0,0 +1,417 @@ +import json +import logging + +from fastapi import APIRouter, Form, HTTPException, Request +from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse + +from ..audit import _record_endpoint_audit +from ..db import _connect +from ..schemas import EndpointCreate, EndpointUpdate +from ..utils import ( + get_latest_ddf_ct_href, + get_next_code_uid, + soa_exists, +) + +router = APIRouter(prefix="/soa/{soa_id}") +ui_router = APIRouter() +logger = logging.getLogger("soa_builder.web.routers.endpoints") + +_ENDPOINT_LEVEL_CODELIST = "C188726" + + +def _next_endpoint_uid(cur, soa_id: int) -> str: + """Return next Endpoint_N UID, never reusing deleted UIDs.""" + max_n = 0 + cur.execute( + "SELECT endpoint_uid FROM endpoint WHERE soa_id=? " + "AND endpoint_uid LIKE 'Endpoint_%'", + (soa_id,), + ) + for (uid,) in cur.fetchall(): + if isinstance(uid, str) and uid.startswith("Endpoint_"): + try: + n = int(uid.split("_")[-1]) + if n > max_n: + max_n = n + except (ValueError, IndexError): + pass + cur.execute( + "SELECT before_json, after_json FROM endpoint_audit WHERE soa_id=?", + (soa_id,), + ) + for before_raw, after_raw in cur.fetchall(): + for raw in (before_raw, after_raw): + if not raw: + continue + try: + uid = json.loads(raw).get("endpoint_uid", "") + if isinstance(uid, str) and uid.startswith("Endpoint_"): + n = int(uid.split("_")[-1]) + if n > max_n: + max_n = n + except Exception: + pass + return f"Endpoint_{max_n + 1}" + + +def _row_to_dict(row) -> dict: + keys = [ + "id", + "soa_id", + "endpoint_uid", + "objective_uid", + "name", + "label", + "description", + "text", + "purpose", + "level_code_uid", + "order_index", + ] + return dict(zip(keys, row)) + + +def _objective_exists(cur, soa_id: int, objective_uid: str) -> bool: + cur.execute( + "SELECT 1 FROM objective WHERE soa_id=? AND objective_uid=?", + (soa_id, objective_uid), + ) + return cur.fetchone() is not None + + +def _insert_level_code(cur, soa_id: int, submission_value: str) -> str: + code_uid = get_next_code_uid(cur, soa_id) + slug = get_latest_ddf_ct_href() or "" + codelist_table = f"/mdr/ct/packages/{slug}" if slug else "/mdr/ct/packages" + cur.execute( + "INSERT INTO code_association " + "(soa_id, code_uid, codelist_table, codelist_code, code) " + "VALUES (?,?,?,?,?)", + ( + soa_id, + code_uid, + codelist_table, + _ENDPOINT_LEVEL_CODELIST, + submission_value, + ), + ) + return code_uid + + +def _delete_level_code(cur, soa_id: int, code_uid: str | None) -> None: + if not code_uid: + return + cur.execute( + "DELETE FROM code_association WHERE soa_id=? AND code_uid=?", + (soa_id, code_uid), + ) + + +# --------------------------------------------------------------------------- +# JSON API endpoints +# --------------------------------------------------------------------------- + + +@router.get("/endpoints", response_class=JSONResponse) +def list_endpoints(soa_id: int): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + conn = _connect() + cur = conn.cursor() + cur.execute( + "SELECT id,soa_id,endpoint_uid,objective_uid,name,label," + "description,text,purpose,level_code_uid,order_index " + "FROM endpoint WHERE soa_id=? ORDER BY order_index, id", + (soa_id,), + ) + rows = [_row_to_dict(r) for r in cur.fetchall()] + conn.close() + return JSONResponse(rows) + + +@router.post("/endpoints", response_class=JSONResponse) +def create_endpoint(soa_id: int, body: EndpointCreate): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + name = (body.name or "").strip() + level = (body.level or "").strip() + objective_uid = (body.objective_uid or "").strip() + if not name: + raise HTTPException(400, "Endpoint name required") + if not level: + raise HTTPException(400, "Endpoint level required") + if not objective_uid: + raise HTTPException(400, "Parent objective_uid required") + + conn = _connect() + cur = conn.cursor() + if not _objective_exists(cur, soa_id, objective_uid): + conn.close() + raise HTTPException(400, f"Objective {objective_uid!r} not found for this SOA") + + cur.execute( + "SELECT COALESCE(MAX(order_index),0) FROM endpoint WHERE soa_id=?", + (soa_id,), + ) + next_ord = (cur.fetchone() or [0])[0] + 1 + endpoint_uid = _next_endpoint_uid(cur, soa_id) + level_code_uid = _insert_level_code(cur, soa_id, level) + + label = (body.label or "").strip() or None + description = (body.description or "").strip() or None + text = (body.text or "").strip() or None + purpose = (body.purpose or "").strip() or None + + cur.execute( + "INSERT INTO endpoint " + "(soa_id,endpoint_uid,objective_uid,name,label,description," + "text,purpose,level_code_uid,order_index) " + "VALUES (?,?,?,?,?,?,?,?,?,?)", + ( + soa_id, + endpoint_uid, + objective_uid, + name, + label, + description, + text, + purpose, + level_code_uid, + next_ord, + ), + ) + endpoint_id = cur.lastrowid + conn.commit() + conn.close() + + after = { + "id": endpoint_id, + "endpoint_uid": endpoint_uid, + "objective_uid": objective_uid, + "name": name, + "label": label, + "description": description, + "text": text, + "purpose": purpose, + "level_code_uid": level_code_uid, + "level": level, + "order_index": next_ord, + } + _record_endpoint_audit(soa_id, "create", endpoint_id, before=None, after=after) + return JSONResponse(after, status_code=201) + + +@router.patch("/endpoints/{endpoint_id}", response_class=JSONResponse) +def update_endpoint(soa_id: int, endpoint_id: int, body: EndpointUpdate): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + conn = _connect() + cur = conn.cursor() + cur.execute( + "SELECT id,soa_id,endpoint_uid,objective_uid,name,label," + "description,text,purpose,level_code_uid,order_index " + "FROM endpoint WHERE id=? AND soa_id=?", + (endpoint_id, soa_id), + ) + row = cur.fetchone() + if not row: + conn.close() + raise HTTPException(404, "Endpoint not found") + before = _row_to_dict(row) + + new_objective_uid = before["objective_uid"] + if body.objective_uid is not None: + candidate = body.objective_uid.strip() or None + if candidate is not None and not _objective_exists(cur, soa_id, candidate): + conn.close() + raise HTTPException(400, f"Objective {candidate!r} not found for this SOA") + new_objective_uid = candidate + + new_name = body.name if body.name is not None else before["name"] + new_label = body.label if body.label is not None else before["label"] + new_desc = ( + body.description if body.description is not None else before["description"] + ) + new_text = body.text if body.text is not None else before["text"] + new_purpose = body.purpose if body.purpose is not None else before["purpose"] + + new_level_code_uid = before["level_code_uid"] + if body.level is not None: + new_level = body.level.strip() + if not new_level: + conn.close() + raise HTTPException(400, "Endpoint level cannot be empty") + if before["level_code_uid"]: + cur.execute( + "UPDATE code_association SET code=? WHERE soa_id=? AND code_uid=?", + (new_level, soa_id, before["level_code_uid"]), + ) + else: + new_level_code_uid = _insert_level_code(cur, soa_id, new_level) + + cur.execute( + "UPDATE endpoint SET objective_uid=?, name=?, label=?, " + "description=?, text=?, purpose=?, level_code_uid=? " + "WHERE id=? AND soa_id=?", + ( + new_objective_uid, + new_name, + (new_label or None) if new_label is not None else None, + (new_desc or None) if new_desc is not None else None, + (new_text or None) if new_text is not None else None, + (new_purpose or None) if new_purpose is not None else None, + new_level_code_uid, + endpoint_id, + soa_id, + ), + ) + conn.commit() + conn.close() + + after = { + **before, + "objective_uid": new_objective_uid, + "name": new_name, + "label": new_label, + "description": new_desc, + "text": new_text, + "purpose": new_purpose, + "level_code_uid": new_level_code_uid, + } + _record_endpoint_audit(soa_id, "update", endpoint_id, before=before, after=after) + return JSONResponse(after) + + +@router.delete("/endpoints/{endpoint_id}", response_class=JSONResponse) +def delete_endpoint(soa_id: int, endpoint_id: int): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + conn = _connect() + cur = conn.cursor() + cur.execute( + "SELECT id,soa_id,endpoint_uid,objective_uid,name,label," + "description,text,purpose,level_code_uid,order_index " + "FROM endpoint WHERE id=? AND soa_id=?", + (endpoint_id, soa_id), + ) + row = cur.fetchone() + if not row: + conn.close() + raise HTTPException(404, "Endpoint not found") + before = _row_to_dict(row) + + _delete_level_code(cur, soa_id, before["level_code_uid"]) + cur.execute( + "DELETE FROM endpoint WHERE id=? AND soa_id=?", + (endpoint_id, soa_id), + ) + + # Reindex remaining endpoints + cur.execute( + "SELECT id FROM endpoint WHERE soa_id=? ORDER BY order_index, id", + (soa_id,), + ) + remaining = [r[0] for r in cur.fetchall()] + for idx, eid in enumerate(remaining, start=1): + cur.execute("UPDATE endpoint SET order_index=? WHERE id=?", (idx, eid)) + conn.commit() + conn.close() + + _record_endpoint_audit(soa_id, "delete", endpoint_id, before=before, after=None) + return JSONResponse({"deleted": endpoint_id}) + + +# --------------------------------------------------------------------------- +# UI form endpoints +# --------------------------------------------------------------------------- + + +@ui_router.post("/ui/soa/{soa_id}/endpoints/create", response_class=HTMLResponse) +def ui_create_endpoint( + request: Request, + soa_id: int, + name: str = Form(...), + level: str = Form(...), + objective_uid: str = Form(...), + label: str | None = Form(None), + description: str | None = Form(None), + text: str | None = Form(None), + purpose: str | None = Form(None), +): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + create_endpoint( + soa_id, + EndpointCreate( + name=name, + level=level, + objective_uid=objective_uid, + label=label, + description=description, + text=text, + purpose=purpose, + ), + ) + safe_soa_id = int(soa_id) + redirect_url = f"/ui/soa/{safe_soa_id}/edit" + if request.headers.get("HX-Request") == "true": + return HTMLResponse("", headers={"HX-Redirect": redirect_url}) + return RedirectResponse(redirect_url, status_code=303) + + +@ui_router.post( + "/ui/soa/{soa_id}/endpoints/{endpoint_id}/update", + response_class=HTMLResponse, +) +def ui_update_endpoint( + request: Request, + soa_id: int, + endpoint_id: int, + name: str | None = Form(None), + level: str | None = Form(None), + objective_uid: str | None = Form(None), + label: str | None = Form(None), + description: str | None = Form(None), + text: str | None = Form(None), + purpose: str | None = Form(None), +): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + update_endpoint( + soa_id, + endpoint_id, + EndpointUpdate( + name=name, + level=level, + objective_uid=objective_uid, + label=label, + description=description, + text=text, + purpose=purpose, + ), + ) + safe_soa_id = int(soa_id) + redirect_url = f"/ui/soa/{safe_soa_id}/edit" + if request.headers.get("HX-Request") == "true": + return HTMLResponse("", headers={"HX-Redirect": redirect_url}) + return RedirectResponse(redirect_url, status_code=303) + + +@ui_router.post( + "/ui/soa/{soa_id}/endpoints/{endpoint_id}/delete", + response_class=HTMLResponse, +) +def ui_delete_endpoint( + request: Request, + soa_id: int, + endpoint_id: int, +): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + delete_endpoint(soa_id, endpoint_id) + safe_soa_id = int(soa_id) + redirect_url = f"/ui/soa/{safe_soa_id}/edit" + if request.headers.get("HX-Request") == "true": + return HTMLResponse("", headers={"HX-Redirect": redirect_url}) + return RedirectResponse(redirect_url, status_code=303) diff --git a/src/soa_builder/web/routers/freezes.py b/src/soa_builder/web/routers/freezes.py index 62704520..96e3613a 100644 --- a/src/soa_builder/web/routers/freezes.py +++ b/src/soa_builder/web/routers/freezes.py @@ -3,10 +3,21 @@ import os from fastapi import APIRouter, Form, HTTPException, Request -from fastapi.responses import HTMLResponse, JSONResponse +from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse from fastapi.templating import Jinja2Templates + from ..db import _connect from ..utils import soa_exists +from ._freeze_helpers import ( + _create_freeze, + _delete_freeze, + _diff_freezes_limited, + _get_freeze, + _list_freezes, + _record_rollback_audit, + _rollback_freeze, + _rollback_preview, +) TEMPLATES_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "templates") templates = Jinja2Templates(directory=TEMPLATES_DIR) @@ -15,10 +26,20 @@ logger = logging.getLogger("soa_builder.web.routers.freezes") -# Removed local _soa_exists; using shared utils.soa_exists - - -# Dynamic helper imports inside endpoint bodies avoid circular import at module load. +@router.get( + "/ui/soa/{soa_id}/freezes", + response_class=HTMLResponse, + name="ui_list_freezes", +) +def ui_list_freezes(request: Request, soa_id: int): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + freezes = _list_freezes(soa_id) + return templates.TemplateResponse( + request, + "freezes.html", + {"soa_id": soa_id, "freezes": freezes}, + ) @router.post("/ui/soa/{soa_id}/freeze", response_class=HTMLResponse) @@ -26,20 +47,21 @@ def ui_freeze_soa(request: Request, soa_id: int, version_label: str = Form("")): if not soa_exists(soa_id): raise HTTPException(404, "SOA not found") try: - from ..app import _create_freeze # type: ignore - - _fid, _vlabel = _create_freeze(soa_id, version_label or None) + _create_freeze(soa_id, version_label or None) except HTTPException as he: if request.headers.get("HX-Request") == "true": return HTMLResponse( - f"
Error: {he.detail}
" + f"
" + f"Error: {he.detail}
" ) return HTMLResponse( - f"" + f"
" + f"Error: {he.detail}
", + headers={"Refresh": f"2; url=/ui/soa/{soa_id}/freezes"}, ) if request.headers.get("HX-Request") == "true": - return HTMLResponse("", headers={"HX-Redirect": f"/ui/soa/{soa_id}/edit"}) - return HTMLResponse(f"") + return HTMLResponse("", headers={"HX-Redirect": f"/ui/soa/{soa_id}/freezes"}) + return RedirectResponse(url=f"/ui/soa/{soa_id}/freezes", status_code=303) @router.get("/soa/{soa_id}/freeze/{freeze_id}") @@ -71,8 +93,6 @@ def get_freeze(soa_id: int, freeze_id: int): @router.get("/ui/soa/{soa_id}/freeze/{freeze_id}/view", response_class=HTMLResponse) def ui_freeze_view(request: Request, soa_id: int, freeze_id: int): - from ..app import _get_freeze # type: ignore - freeze = _get_freeze(soa_id, freeze_id) if not freeze: raise HTTPException(404, "Freeze not found") @@ -85,8 +105,6 @@ def ui_freeze_view(request: Request, soa_id: int, freeze_id: int): @router.get("/ui/soa/{soa_id}/freeze/diff", response_class=HTMLResponse) def ui_freeze_diff(request: Request, soa_id: int, left: int, right: int, full: int = 0): - from ..app import _diff_freezes_limited # type: ignore - limit = None if full == 1 else 50 diff = _diff_freezes_limited(soa_id, left, right, limit=limit) return templates.TemplateResponse( @@ -100,8 +118,6 @@ def ui_freeze_diff(request: Request, soa_id: int, left: int, right: int, full: i "/ui/soa/{soa_id}/freeze/{freeze_id}/rollback", response_class=HTMLResponse ) def ui_freeze_rollback(request: Request, soa_id: int, freeze_id: int): - from ..app import _record_rollback_audit, _rollback_freeze # type: ignore - result = _rollback_freeze(soa_id, freeze_id) _record_rollback_audit( soa_id, @@ -115,16 +131,15 @@ def ui_freeze_rollback(request: Request, soa_id: int, freeze_id: int): }, ) if request.headers.get("HX-Request") == "true": - return HTMLResponse("", headers={"HX-Redirect": f"/ui/soa/{soa_id}/edit"}) - return HTMLResponse(f"") + return HTMLResponse("", headers={"HX-Redirect": f"/ui/soa/{soa_id}/freezes"}) + return RedirectResponse(url=f"/ui/soa/{soa_id}/freezes", status_code=303) @router.get( - "/ui/soa/{soa_id}/freeze/{freeze_id}/rollback_preview", response_class=HTMLResponse + "/ui/soa/{soa_id}/freeze/{freeze_id}/rollback_preview", + response_class=HTMLResponse, ) def ui_freeze_rollback_preview(request: Request, soa_id: int, freeze_id: int): - from ..app import _get_freeze, _rollback_preview # type: ignore - preview = _rollback_preview(soa_id, freeze_id) freeze = _get_freeze(soa_id, freeze_id) return templates.TemplateResponse( @@ -139,10 +154,19 @@ def ui_freeze_rollback_preview(request: Request, soa_id: int, freeze_id: int): ) +@router.post("/ui/soa/{soa_id}/freeze/{freeze_id}/delete", response_class=HTMLResponse) +def ui_freeze_delete(request: Request, soa_id: int, freeze_id: int): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + if not _delete_freeze(soa_id, freeze_id): + raise HTTPException(404, "Freeze not found") + if request.headers.get("HX-Request") == "true": + return HTMLResponse("", headers={"HX-Redirect": f"/ui/soa/{soa_id}/freezes"}) + return HTMLResponse(f"") + + @router.get("/soa/{soa_id}/freeze/diff.json") def get_freeze_diff_json(soa_id: int, left: int, right: int, full: int = 0): - from ..app import _diff_freezes_limited # type: ignore - limit = None if full == 1 else 1000 diff = _diff_freezes_limited(soa_id, left, right, limit=limit) return JSONResponse(diff) diff --git a/src/soa_builder/web/routers/objectives.py b/src/soa_builder/web/routers/objectives.py new file mode 100644 index 00000000..73381d3c --- /dev/null +++ b/src/soa_builder/web/routers/objectives.py @@ -0,0 +1,408 @@ +import json +import logging + +from fastapi import APIRouter, Form, HTTPException, Request +from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse + +from ..audit import _record_endpoint_audit, _record_objective_audit +from ..db import _connect +from ..schemas import ObjectiveCreate, ObjectiveUpdate +from ..utils import ( + get_latest_ddf_ct_href, + get_next_code_uid, + soa_exists, +) + +router = APIRouter(prefix="/soa/{soa_id}") +ui_router = APIRouter() +logger = logging.getLogger("soa_builder.web.routers.objectives") + +_OBJECTIVE_LEVEL_CODELIST = "C188725" + + +def _next_objective_uid(cur, soa_id: int) -> str: + """Return next Objective_N UID, never reusing deleted UIDs.""" + max_n = 0 + cur.execute( + "SELECT objective_uid FROM objective WHERE soa_id=? " + "AND objective_uid LIKE 'Objective_%'", + (soa_id,), + ) + for (uid,) in cur.fetchall(): + if isinstance(uid, str) and uid.startswith("Objective_"): + try: + n = int(uid.split("_")[-1]) + if n > max_n: + max_n = n + except (ValueError, IndexError): + pass + cur.execute( + "SELECT before_json, after_json FROM objective_audit WHERE soa_id=?", + (soa_id,), + ) + for before_raw, after_raw in cur.fetchall(): + for raw in (before_raw, after_raw): + if not raw: + continue + try: + uid = json.loads(raw).get("objective_uid", "") + if isinstance(uid, str) and uid.startswith("Objective_"): + n = int(uid.split("_")[-1]) + if n > max_n: + max_n = n + except Exception: + pass + return f"Objective_{max_n + 1}" + + +def _row_to_dict(row) -> dict: + keys = [ + "id", + "soa_id", + "objective_uid", + "name", + "label", + "description", + "text", + "level_code_uid", + "order_index", + ] + return dict(zip(keys, row)) + + +def _insert_level_code(cur, soa_id: int, submission_value: str) -> str: + """Insert a code_association row for the objective level and return + the generated Code_N UID.""" + code_uid = get_next_code_uid(cur, soa_id) + slug = get_latest_ddf_ct_href() or "" + codelist_table = f"/mdr/ct/packages/{slug}" if slug else "/mdr/ct/packages" + cur.execute( + "INSERT INTO code_association " + "(soa_id, code_uid, codelist_table, codelist_code, code) " + "VALUES (?,?,?,?,?)", + ( + soa_id, + code_uid, + codelist_table, + _OBJECTIVE_LEVEL_CODELIST, + submission_value, + ), + ) + return code_uid + + +def _delete_level_code(cur, soa_id: int, code_uid: str | None) -> None: + if not code_uid: + return + cur.execute( + "DELETE FROM code_association WHERE soa_id=? AND code_uid=?", + (soa_id, code_uid), + ) + + +# --------------------------------------------------------------------------- +# JSON API endpoints +# --------------------------------------------------------------------------- + + +@router.get("/objectives", response_class=JSONResponse) +def list_objectives(soa_id: int): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + conn = _connect() + cur = conn.cursor() + cur.execute( + "SELECT id,soa_id,objective_uid,name,label,description,text," + "level_code_uid,order_index " + "FROM objective WHERE soa_id=? ORDER BY order_index, id", + (soa_id,), + ) + rows = [_row_to_dict(r) for r in cur.fetchall()] + conn.close() + return JSONResponse(rows) + + +@router.post("/objectives", response_class=JSONResponse) +def create_objective(soa_id: int, body: ObjectiveCreate): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + name = (body.name or "").strip() + level = (body.level or "").strip() + if not name: + raise HTTPException(400, "Objective name required") + if not level: + raise HTTPException(400, "Objective level required") + + conn = _connect() + cur = conn.cursor() + cur.execute( + "SELECT COALESCE(MAX(order_index),0) FROM objective WHERE soa_id=?", + (soa_id,), + ) + next_ord = (cur.fetchone() or [0])[0] + 1 + objective_uid = _next_objective_uid(cur, soa_id) + level_code_uid = _insert_level_code(cur, soa_id, level) + + label = (body.label or "").strip() or None + description = (body.description or "").strip() or None + text = (body.text or "").strip() or None + + cur.execute( + "INSERT INTO objective " + "(soa_id,objective_uid,name,label,description,text," + "level_code_uid,order_index) VALUES (?,?,?,?,?,?,?,?)", + ( + soa_id, + objective_uid, + name, + label, + description, + text, + level_code_uid, + next_ord, + ), + ) + objective_id = cur.lastrowid + conn.commit() + conn.close() + + after = { + "id": objective_id, + "objective_uid": objective_uid, + "name": name, + "label": label, + "description": description, + "text": text, + "level_code_uid": level_code_uid, + "level": level, + "order_index": next_ord, + } + _record_objective_audit(soa_id, "create", objective_id, before=None, after=after) + return JSONResponse(after, status_code=201) + + +@router.patch("/objectives/{objective_id}", response_class=JSONResponse) +def update_objective(soa_id: int, objective_id: int, body: ObjectiveUpdate): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + conn = _connect() + cur = conn.cursor() + cur.execute( + "SELECT id,soa_id,objective_uid,name,label,description,text," + "level_code_uid,order_index " + "FROM objective WHERE id=? AND soa_id=?", + (objective_id, soa_id), + ) + row = cur.fetchone() + if not row: + conn.close() + raise HTTPException(404, "Objective not found") + before = _row_to_dict(row) + + new_name = body.name if body.name is not None else before["name"] + new_label = body.label if body.label is not None else before["label"] + new_desc = ( + body.description if body.description is not None else before["description"] + ) + new_text = body.text if body.text is not None else before["text"] + + new_level_code_uid = before["level_code_uid"] + if body.level is not None: + new_level = body.level.strip() + if not new_level: + conn.close() + raise HTTPException(400, "Objective level cannot be empty") + if before["level_code_uid"]: + # Update the submission value in the existing Code_N row. + cur.execute( + "UPDATE code_association SET code=? WHERE soa_id=? AND code_uid=?", + (new_level, soa_id, before["level_code_uid"]), + ) + else: + new_level_code_uid = _insert_level_code(cur, soa_id, new_level) + + cur.execute( + "UPDATE objective SET name=?, label=?, description=?, text=?, " + "level_code_uid=? WHERE id=? AND soa_id=?", + ( + new_name, + (new_label or None) if new_label is not None else None, + (new_desc or None) if new_desc is not None else None, + (new_text or None) if new_text is not None else None, + new_level_code_uid, + objective_id, + soa_id, + ), + ) + conn.commit() + conn.close() + + after = { + **before, + "name": new_name, + "label": new_label, + "description": new_desc, + "text": new_text, + "level_code_uid": new_level_code_uid, + } + _record_objective_audit(soa_id, "update", objective_id, before=before, after=after) + return JSONResponse(after) + + +@router.delete("/objectives/{objective_id}", response_class=JSONResponse) +def delete_objective(soa_id: int, objective_id: int): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + conn = _connect() + cur = conn.cursor() + cur.execute( + "SELECT id,soa_id,objective_uid,name,label,description,text," + "level_code_uid,order_index " + "FROM objective WHERE id=? AND soa_id=?", + (objective_id, soa_id), + ) + row = cur.fetchone() + if not row: + conn.close() + raise HTTPException(404, "Objective not found") + before = _row_to_dict(row) + + # Orphan child endpoints: set objective_uid to NULL + orphaned: list[dict] = [] + cur.execute( + "SELECT id,endpoint_uid,objective_uid FROM endpoint " + "WHERE soa_id=? AND objective_uid=?", + (soa_id, before["objective_uid"]), + ) + for ep_id, ep_uid, ep_parent in cur.fetchall(): + orphaned.append( + { + "id": ep_id, + "endpoint_uid": ep_uid, + "objective_uid_before": ep_parent, + } + ) + if orphaned: + cur.execute( + "UPDATE endpoint SET objective_uid=NULL WHERE soa_id=? AND objective_uid=?", + (soa_id, before["objective_uid"]), + ) + + _delete_level_code(cur, soa_id, before["level_code_uid"]) + cur.execute( + "DELETE FROM objective WHERE id=? AND soa_id=?", + (objective_id, soa_id), + ) + + # Reindex remaining objectives + cur.execute( + "SELECT id FROM objective WHERE soa_id=? ORDER BY order_index, id", + (soa_id,), + ) + remaining = [r[0] for r in cur.fetchall()] + for idx, oid in enumerate(remaining, start=1): + cur.execute("UPDATE objective SET order_index=? WHERE id=?", (idx, oid)) + conn.commit() + conn.close() + + _record_objective_audit(soa_id, "delete", objective_id, before=before, after=None) + for entry in orphaned: + _record_endpoint_audit( + soa_id, + "update", + entry["id"], + before={ + "endpoint_uid": entry["endpoint_uid"], + "objective_uid": entry["objective_uid_before"], + }, + after={ + "endpoint_uid": entry["endpoint_uid"], + "objective_uid": None, + "orphaned_by_objective_delete": before["objective_uid"], + }, + ) + return JSONResponse({"deleted": objective_id, "orphaned_endpoints": len(orphaned)}) + + +# --------------------------------------------------------------------------- +# UI form endpoints +# --------------------------------------------------------------------------- + + +@ui_router.post("/ui/soa/{soa_id}/objectives/create", response_class=HTMLResponse) +def ui_create_objective( + request: Request, + soa_id: int, + name: str = Form(...), + level: str = Form(...), + label: str | None = Form(None), + description: str | None = Form(None), + text: str | None = Form(None), +): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + create_objective( + soa_id, + ObjectiveCreate( + name=name, + level=level, + label=label, + description=description, + text=text, + ), + ) + redirect_url = f"/ui/soa/{soa_id}/edit" + if request.headers.get("HX-Request") == "true": + return HTMLResponse("", headers={"HX-Redirect": redirect_url}) + return RedirectResponse(redirect_url, status_code=303) + + +@ui_router.post( + "/ui/soa/{soa_id}/objectives/{objective_id}/update", + response_class=HTMLResponse, +) +def ui_update_objective( + request: Request, + soa_id: int, + objective_id: int, + name: str | None = Form(None), + level: str | None = Form(None), + label: str | None = Form(None), + description: str | None = Form(None), + text: str | None = Form(None), +): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + update_objective( + soa_id, + objective_id, + ObjectiveUpdate( + name=name, + level=level, + label=label, + description=description, + text=text, + ), + ) + redirect_url = f"/ui/soa/{soa_id}/edit" + if request.headers.get("HX-Request") == "true": + return HTMLResponse("", headers={"HX-Redirect": redirect_url}) + return RedirectResponse(redirect_url, status_code=303) + + +@ui_router.post( + "/ui/soa/{soa_id}/objectives/{objective_id}/delete", + response_class=HTMLResponse, +) +def ui_delete_objective( + request: Request, + soa_id: int, + objective_id: int, +): + if not soa_exists(soa_id): + raise HTTPException(404, "SOA not found") + delete_objective(soa_id, objective_id) + redirect_url = f"/ui/soa/{soa_id}/edit" + if request.headers.get("HX-Request") == "true": + return HTMLResponse("", headers={"HX-Redirect": redirect_url}) + return RedirectResponse(redirect_url, status_code=303) diff --git a/src/soa_builder/web/schemas.py b/src/soa_builder/web/schemas.py index 75002d09..276acdb7 100644 --- a/src/soa_builder/web/schemas.py +++ b/src/soa_builder/web/schemas.py @@ -267,6 +267,42 @@ class ConceptsUpdate(BaseModel): concept_codes: List[str] +class ObjectiveCreate(BaseModel): + name: str + level: str + label: Optional[str] = None + description: Optional[str] = None + text: Optional[str] = None + + +class ObjectiveUpdate(BaseModel): + name: Optional[str] = None + label: Optional[str] = None + description: Optional[str] = None + text: Optional[str] = None + level: Optional[str] = None + + +class EndpointCreate(BaseModel): + name: str + level: str + objective_uid: str + label: Optional[str] = None + description: Optional[str] = None + text: Optional[str] = None + purpose: Optional[str] = None + + +class EndpointUpdate(BaseModel): + name: Optional[str] = None + objective_uid: Optional[str] = None + label: Optional[str] = None + description: Optional[str] = None + text: Optional[str] = None + purpose: Optional[str] = None + level: Optional[str] = None + + class FreezeCreate(BaseModel): version_label: Optional[str] = None diff --git a/src/soa_builder/web/templates/_objectives_section.html b/src/soa_builder/web/templates/_objectives_section.html new file mode 100644 index 00000000..af7e12f9 --- /dev/null +++ b/src/soa_builder/web/templates/_objectives_section.html @@ -0,0 +1,209 @@ + + +
+ Objectives ({{ objectives|length }}) · Endpoints ({{ endpoints_by_objective.values()|map('length')|sum + orphan_endpoints|length }}) + + {% if objectives|length == 0 and orphan_endpoints|length == 0 %} +

No objectives defined yet.

+ {% endif %} + + {% for obj in objectives %} +
+
+ {{ obj.objective_uid }} + — {{ obj.name }} + {% if obj.level %}[{{ obj.level }}]{% endif %} + {% if obj.label %} «{{ obj.label }}»{% endif %} +
+ {% if obj.description %}
{{ obj.description }}
{% endif %} + {% if obj.text %}
{{ obj.text }}
{% endif %} +
+
+ + + + + + +
+
+ +
+
+ + {% set obj_endpoints = endpoints_by_objective.get(obj.objective_uid, []) %} + {% if obj_endpoints %} +
    + {% for ep in obj_endpoints %} +
  • + {{ ep.endpoint_uid }} — {{ ep.name }} + {% if ep.level %}[{{ ep.level }}]{% endif %} + {% if ep.label %} «{{ ep.label }}»{% endif %} + {% if ep.purpose %}
    Purpose: {{ ep.purpose }}
    {% endif %} + {% if ep.text %}
    {{ ep.text }}
    {% endif %} + +
    + + + + + + + + +
    +
    + +
    +     +
    • + {% endfor %} +
+ {% endif %} +
+ {% endfor %} + + {% if orphan_endpoints %} +
+
Unassigned Endpoints ({{ orphan_endpoints|length }})
+
    + {% for ep in orphan_endpoints %} +
  • + {{ ep.endpoint_uid }} — {{ ep.name }} + {% if ep.level %}[{{ ep.level }}]{% endif %} + +
    + + + +
    +
    + +
    +     +
    • + {% endfor %} +
+
+ {% endif %} + +
+ Create Objective +
+ + + + + + +
+
+ +
+ Create Endpoint + {% if objectives|length == 0 %} +

Create an objective first — endpoints must be assigned to a parent objective.

+ {% else %} +
+ + + + + + + + +
+ {% endif %} +
+
diff --git a/src/soa_builder/web/templates/base.html b/src/soa_builder/web/templates/base.html index 9e54966a..df116e73 100644 --- a/src/soa_builder/web/templates/base.html +++ b/src/soa_builder/web/templates/base.html @@ -26,6 +26,7 @@
  • Transition Rules
  • Generate USDM JSON
  • Generate Trial Design Domains
    • +
  • Frozen Versions
    • diff --git a/src/soa_builder/web/templates/edit.html b/src/soa_builder/web/templates/edit.html index 78b7f525..91d51bb1 100644 --- a/src/soa_builder/web/templates/edit.html +++ b/src/soa_builder/web/templates/edit.html @@ -22,49 +22,6 @@

    Editing SoA for {% if study_label %}{{ study_label }}{% else %}{{ study_id } -
    -
    -
    - - -
    -
    -
    - Count: {{ freeze_count }} - {% if last_frozen_at %} • Last: {{ last_frozen_at }}{% endif %} -
    -
    - {% if freezes %} -
    - Versions: - {% for f in freezes %} -
    - {{ f.version_label }} - -
    - {% endfor %} -
    -
    - - - Rollback Audit XLSX - Reorder Audit XLSX - Reorder CSV -
    -
    - Diff: - - - -
    - {% else %} -
    No versions frozen yet.
    - {% endif %} -
    @@ -125,6 +82,9 @@

    Editing SoA for {% if study_label %}{{ study_label }}{% else %}{{ study_id }

    + +{% include '_objectives_section.html' %} +