From cffaa39f31864f67d0a6f6543ca37d90021dc960 Mon Sep 17 00:00:00 2001 From: Darren <3921919+pendingintent@users.noreply.github.com> Date: Wed, 8 Apr 2026 13:19:07 -0400 Subject: [PATCH 01/36] Added NCIt and LOINC API explorer feature --- README.md | 37 +-- app.py | 2 + files/~$BC Examples.xlsx | Bin 0 -> 165 bytes migrations/README | 1 + migrations/alembic.ini | 50 ++++ migrations/env.py | 113 ++++++++ migrations/script.py.mako | 24 ++ ...add_ncit_metadata_to_biomedical_concept.py | 32 +++ ...dd_loinc_metadata_to_biomedical_concept.py | 32 +++ models/bc.py | 8 +- routes/bc.py | 48 ++++ routes/dashboard.py | 10 +- routes/loinc.py | 22 ++ routes/ncit.py | 11 + services/cdisc_api.py | 29 +- services/loinc_api.py | 49 ++++ services/ncit_api.py | 15 +- static/css/custom.css | 29 ++ static/js/main.js | 254 +++++++++++++++++- templates/bc_detail.html | 105 ++++++-- templates/library_bc_detail.html | 56 ++-- tests/test_bc_routes.py | 133 ++++++++- tests/test_loinc.py | 184 +++++++++++++ tests/test_ncit.py | 150 +++++++++++ 24 files changed, 1332 insertions(+), 62 deletions(-) create mode 100644 files/~$BC Examples.xlsx create mode 100644 migrations/README create mode 100644 migrations/alembic.ini create mode 100644 migrations/env.py create mode 100644 migrations/script.py.mako create mode 100644 migrations/versions/b9ee22a174fe_add_ncit_metadata_to_biomedical_concept.py create mode 100644 migrations/versions/f27a606163b0_add_loinc_metadata_to_biomedical_concept.py create mode 100644 routes/loinc.py create mode 100644 services/loinc_api.py create mode 100644 tests/test_loinc.py create mode 100644 tests/test_ncit.py diff --git a/README.md b/README.md index fbf4e9b..41794a2 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,8 @@ The application is configured entirely through environment variables. | `CDISC_API_KEY` | Yes | _(empty)_ | API key for the CDISC Library. All CDISC API calls will fail without it. | | `SECRET_KEY` | No | `dev-secret-key-change-in-prod` | Flask session secret. Set a strong value in production. | | `DATABASE_URL` | No | `sqlite:///cdisc_curation.db` | SQLAlchemy database URI. Defaults to a local SQLite file. | +| `LOINC_USER` | No | _(empty)_ | Optional Basic Auth username for the NLM Clinical Tables API. If set, `LOINC_PASSWORD` must also be set. | +| `LOINC_PASSWORD` | No | _(empty)_ | Optional Basic Auth password for the NLM Clinical Tables API. If set, `LOINC_USER` must also be set. | Set environment variables before running the app: @@ -102,9 +104,9 @@ The sidebar navigation exposes seven screens, accessible at these URL prefixes: | Screen | URL | What it does | |--------|-----|-------------| -| Dashboard | `/` | KPI cards (total BCs, pending review, published), governance pipeline chart, recent submissions table | +| Dashboard | `/` | KPI cards (total BCs, pending review, published), governance pipeline chart with concurrent CDISC API fetches (ThreadPoolExecutor), recent submissions table | | Ingestion | `/ingestion` | Upload XLSX, CSV, or JSON files; AI field mapper assigns confidence scores; approve or reject rows to the database | -| BCs | `/bc` | Browse, create, edit, and delete Biomedical Concepts including all CDISC fields and Data Element Concept sub-records | +| BCs | `/bc` | Browse, create, edit, and delete Biomedical Concepts; LOINC code entry with live search and automatic metadata population from NLM Clinical Tables API (LONG_COMMON_NAME, SHORTNAME, COMPONENT, units, etc.); NCIt concept selection with full metadata display (definitions, parents, semantic type); Data Element Concept sub-records | | NCIT Mapping | `/ncit` | Search the NCI Thesaurus, resolve low-confidence mappings, and confirm NCIt codes for each BC | | Specializations | `/specializations` | View and generate SDTM/CDASH dataset specializations and CRF variable mappings | | Governance | `/governance` | 4-stage Kanban board (Provisional > SME Review > CDISC Approval > Published) with advance and reject actions | @@ -121,21 +123,23 @@ cdisc-concept-curation/ ├── extensions.py # db + migrate instances (avoids circular imports) ├── requirements.txt ├── models/ -│ ├── bc.py # BiomedicalConcept, DataElementConcept +│ ├── bc.py # BiomedicalConcept (loinc_metadata and ncit_metadata store API responses as JSON), DataElementConcept │ ├── specialization.py # DatasetSpecialization │ ├── governance.py # GovernanceRecord │ └── audit.py # AuditLog -├── routes/ # 7 Flask blueprints -│ ├── dashboard.py -│ ├── ingestion.py -│ ├── bc.py -│ ├── ncit.py -│ ├── specializations.py -│ ├── governance.py -│ └── audit.py +├── routes/ # 8 Flask blueprints +│ ├── dashboard.py # Concurrent CDISC API fetches (ThreadPoolExecutor), KPI cards +│ ├── ingestion.py # File upload and AI field mapper +│ ├── bc.py # Create, edit, detail views with LOINC and NCIt API integration +│ ├── ncit.py # GET /ncit/search and GET /ncit/concept/ JSON endpoints with full metadata +│ ├── loinc.py # GET /loinc/search JSON API endpoint +│ ├── specializations.py # Dataset specializations and CRF mappings +│ ├── governance.py # Kanban board and status workflows +│ └── audit.py # Immutable change log with filters ├── services/ -│ ├── cdisc_api.py # CDISC Library API client -│ ├── ncit_api.py # NCI EVS REST API client (no key required) +│ ├── cdisc_api.py # CDISC Library API client with stale-while-refresh caching (5-min fresh TTL, 1-hour stale fallback) +│ ├── ncit_api.py # NCI EVS REST API client (full concept detail with definitions, parents, semantic type) +│ ├── loinc_api.py # NLM Clinical Tables API client (optional Basic Auth, metadata caching) │ ├── ingestion.py # File parser and AI field mapper │ └── export.py # XLSX, JSON, ODM-XML export ├── templates/ @@ -153,5 +157,8 @@ cdisc-concept-curation/ ## External APIs -- **CDISC Library** — `https://api.library.cdisc.org/api/cosmos/v2` — requires `CDISC_API_KEY` -- **NCI EVS REST API** — `https://api-evsrest.nci.nih.gov/api/v1` — no key required +The platform integrates with three external APIs to provide rich concept metadata: + +- **CDISC Library** (`https://api.library.cdisc.org/api/cosmos/v2`) — Requires `CDISC_API_KEY`. Used in Dashboard and BC Library detail views. Implements stale-while-refresh caching to gracefully handle transient failures. +- **NCI EVS REST API** (`https://api-evsrest.nci.nih.gov/api/v1`) — No authentication required. Returns NCIt concept definitions, parent concepts, and semantic types. Integrated into BC detail views via `/ncit/concept/` endpoint. +- **NLM Clinical Tables API (LOINC)** (`https://clinicaltables.nlm.nih.gov/api/loinc_items/v3/search`) — Optional Basic Auth via `LOINC_USER` / `LOINC_PASSWORD`. Returns LOINC metadata including LONG_COMMON_NAME, SHORTNAME, COMPONENT, METHOD_TYP, units, datatype, and copyright notices. Integrated into BC detail views via `/loinc/search` endpoint. diff --git a/app.py b/app.py index 3dfdb15..fc82897 100644 --- a/app.py +++ b/app.py @@ -14,6 +14,7 @@ def create_app(config_class=Config): from routes.ingestion import bp as ingestion_bp from routes.bc import bp as bc_bp from routes.ncit import bp as ncit_bp + from routes.loinc import bp as loinc_bp from routes.specializations import bp as specializations_bp from routes.governance import bp as governance_bp from routes.audit import bp as audit_bp @@ -22,6 +23,7 @@ def create_app(config_class=Config): app.register_blueprint(ingestion_bp, url_prefix='/ingestion') app.register_blueprint(bc_bp, url_prefix='/bc') app.register_blueprint(ncit_bp, url_prefix='/ncit') + app.register_blueprint(loinc_bp, url_prefix='/loinc') app.register_blueprint(specializations_bp, url_prefix='/specializations') app.register_blueprint(governance_bp, url_prefix='/governance') app.register_blueprint(audit_bp, url_prefix='/audit') diff --git a/files/~$BC Examples.xlsx b/files/~$BC Examples.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..4e0617bff91f7e88227a4fa989507e04b69f3a09 GIT binary patch literal 165 zcmd;gNh~T#%~SBrFG|fx%u7)q4)8O$FeEY*0bwdb9)kjdFGD_=3 + return current_app.extensions['migrate'].db.engine + + +def get_engine_url(): + try: + return get_engine().url.render_as_string(hide_password=False).replace( + '%', '%%') + except AttributeError: + return str(get_engine().url).replace('%', '%%') + + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +config.set_main_option('sqlalchemy.url', get_engine_url()) +target_db = current_app.extensions['migrate'].db + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def get_metadata(): + if hasattr(target_db, 'metadatas'): + return target_db.metadatas[None] + return target_db.metadata + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=get_metadata(), literal_binds=True + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + # this callback is used to prevent an auto-migration from being generated + # when there are no changes to the schema + # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html + def process_revision_directives(context, revision, directives): + if getattr(config.cmd_opts, 'autogenerate', False): + script = directives[0] + if script.upgrade_ops.is_empty(): + directives[:] = [] + logger.info('No changes in schema detected.') + + conf_args = current_app.extensions['migrate'].configure_args + if conf_args.get("process_revision_directives") is None: + conf_args["process_revision_directives"] = process_revision_directives + + connectable = get_engine() + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=get_metadata(), + **conf_args + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 0000000..2c01563 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/b9ee22a174fe_add_ncit_metadata_to_biomedical_concept.py b/migrations/versions/b9ee22a174fe_add_ncit_metadata_to_biomedical_concept.py new file mode 100644 index 0000000..da2c60f --- /dev/null +++ b/migrations/versions/b9ee22a174fe_add_ncit_metadata_to_biomedical_concept.py @@ -0,0 +1,32 @@ +"""add ncit_metadata to biomedical_concept + +Revision ID: b9ee22a174fe +Revises: f27a606163b0 +Create Date: 2026-04-08 13:12:22.755530 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'b9ee22a174fe' +down_revision = 'f27a606163b0' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('biomedical_concepts', schema=None) as batch_op: + batch_op.add_column(sa.Column('ncit_metadata', sa.Text(), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('biomedical_concepts', schema=None) as batch_op: + batch_op.drop_column('ncit_metadata') + + # ### end Alembic commands ### diff --git a/migrations/versions/f27a606163b0_add_loinc_metadata_to_biomedical_concept.py b/migrations/versions/f27a606163b0_add_loinc_metadata_to_biomedical_concept.py new file mode 100644 index 0000000..8594140 --- /dev/null +++ b/migrations/versions/f27a606163b0_add_loinc_metadata_to_biomedical_concept.py @@ -0,0 +1,32 @@ +"""add loinc_metadata to biomedical_concept + +Revision ID: f27a606163b0 +Revises: +Create Date: 2026-04-08 12:28:59.039371 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'f27a606163b0' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('biomedical_concepts', schema=None) as batch_op: + batch_op.add_column(sa.Column('loinc_metadata', sa.Text(), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('biomedical_concepts', schema=None) as batch_op: + batch_op.drop_column('loinc_metadata') + + # ### end Alembic commands ### diff --git a/models/bc.py b/models/bc.py index 8cc4f02..8934786 100644 --- a/models/bc.py +++ b/models/bc.py @@ -13,8 +13,10 @@ class BiomedicalConcept(db.Model): synonyms = db.Column(db.Text) result_scales = db.Column(db.String(255)) # e.g. "Quantitative; Ordinal" system = db.Column(db.String(255)) # e.g. http://loinc.org/ - system_name = db.Column(db.String(100)) # e.g. LOINC - code = db.Column(db.String(50)) # code in external system + system_name = db.Column(db.String(100)) # LOINC LONG_COMMON_NAME + code = db.Column(db.String(50)) # LOINC_NUM + loinc_metadata = db.Column(db.Text) # JSON blob of all LOINC ef fields + ncit_metadata = db.Column(db.Text) # JSON blob of NCIt concept detail package_date = db.Column(db.String(20)) status = db.Column(db.String(50), default='provisional') # provisional/sme_review/cdisc_approval/published submitter = db.Column(db.String(100)) @@ -40,6 +42,8 @@ def to_dict(self): 'system': self.system, 'system_name': self.system_name, 'code': self.code, + 'loinc_metadata': self.loinc_metadata, + 'ncit_metadata': self.ncit_metadata, 'package_date': self.package_date, 'status': self.status, 'submitter': self.submitter, diff --git a/routes/bc.py b/routes/bc.py index a232a9a..828af73 100644 --- a/routes/bc.py +++ b/routes/bc.py @@ -1,9 +1,12 @@ +import json from flask import Blueprint, render_template, request, redirect, url_for, flash, Response from models.bc import BiomedicalConcept, DataElementConcept from models.audit import AuditLog from extensions import db from services.export import export_json, export_xlsx, export_odm_xml from services.cdisc_api import CDISCApiClient +from services.loinc_api import LoincApiClient +from services.ncit_api import NCItApiClient from datetime import datetime bp = Blueprint('bc', __name__) @@ -43,6 +46,8 @@ def new_bc(): bc=bc, decs=[], is_new=True, + loinc_data={}, + ncit_data={}, page_title='New Biomedical Concept', ) @@ -80,9 +85,24 @@ def library_detail(concept_id): if 'error' in bc: flash(f'Could not load concept {concept_id}: {bc["error"]}', 'danger') return redirect(url_for('dashboard.index')) + + # Look for a LOINC coding entry in the CDISC API response + loinc_code = None + for c in bc.get('coding', []): + if (c.get('systemName') or '').upper() == 'LOINC' and c.get('code'): + loinc_code = c['code'] + break + + loinc_data = {} + if loinc_code: + results = LoincApiClient().search(loinc_code, size=1) + if results and not results[0].get('error'): + loinc_data = results[0] + return render_template( 'library_bc_detail.html', bc=bc, + loinc_data=loinc_data, page_title=bc.get('shortName') or bc.get('name') or concept_id, ) @@ -96,11 +116,35 @@ def detail(bc_id): .order_by(DataElementConcept.sort_order) .all() ) + loinc_data = {} + if bc.code: + results = LoincApiClient().search(bc.code, size=1) + if results and not results[0].get('error'): + loinc_data = results[0] + elif bc.loinc_metadata: + try: + loinc_data = json.loads(bc.loinc_metadata) + except (ValueError, TypeError): + pass + + ncit_data = {} + if bc.ncit_code: + result = NCItApiClient().get_concept(bc.ncit_code) + if not result.get('error'): + ncit_data = result + elif bc.ncit_metadata: + try: + ncit_data = json.loads(bc.ncit_metadata) + except (ValueError, TypeError): + pass + return render_template( 'bc_detail.html', bc=bc, decs=decs, is_new=False, + loinc_data=loinc_data, + ncit_data=ncit_data, page_title=bc.short_name, ) @@ -126,6 +170,8 @@ def create(): system=request.form.get('system', ''), system_name=request.form.get('system_name', ''), code=request.form.get('code', ''), + loinc_metadata=request.form.get('loinc_metadata', '') or None, + ncit_metadata=request.form.get('ncit_metadata', '') or None, package_date=request.form.get('package_date', ''), status='provisional', submitter=request.form.get('submitter', 'unknown'), @@ -159,6 +205,8 @@ def edit(bc_id): bc.system = request.form.get('system', bc.system) bc.system_name = request.form.get('system_name', bc.system_name) bc.code = request.form.get('code', bc.code) + bc.loinc_metadata = request.form.get('loinc_metadata', '') or bc.loinc_metadata + bc.ncit_metadata = request.form.get('ncit_metadata', '') or bc.ncit_metadata bc.package_date = request.form.get('package_date', bc.package_date) bc.updated_at = datetime.utcnow() log = AuditLog( diff --git a/routes/dashboard.py b/routes/dashboard.py index dc81c33..bb37997 100644 --- a/routes/dashboard.py +++ b/routes/dashboard.py @@ -1,3 +1,4 @@ +from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timedelta from flask import Blueprint, render_template from models.bc import BiomedicalConcept @@ -9,10 +10,13 @@ @bp.route("/") def index(): - # --- CDISC Library API data --- + # --- CDISC Library API data (fetched concurrently) --- client = CDISCApiClient() - api_bcs = client.get_biomedical_concepts() - api_specs = client.get_dataset_specializations() + with ThreadPoolExecutor(max_workers=2) as pool: + fut_bcs = pool.submit(client.get_biomedical_concepts) + fut_specs = pool.submit(client.get_dataset_specializations) + api_bcs = fut_bcs.result() + api_specs = fut_specs.result() api_bc_error = ( api_bcs[0].get("error") if api_bcs and "error" in api_bcs[0] else None diff --git a/routes/loinc.py b/routes/loinc.py new file mode 100644 index 0000000..8eb58bf --- /dev/null +++ b/routes/loinc.py @@ -0,0 +1,22 @@ +from flask import Blueprint, jsonify, request +from services.loinc_api import LoincApiClient + +bp = Blueprint('loinc', __name__) + + +@bp.route('/search') +def search(): + term = request.args.get('term', '').strip() + is_ajax = ( + request.headers.get('X-Requested-With') == 'XMLHttpRequest' + or request.args.get('format') == 'json' + or 'application/json' in request.headers.get('Accept', '') + ) + if not term: + if is_ajax: + return jsonify([]) + return jsonify([]) + + client = LoincApiClient() + results = client.search(term, size=10) + return jsonify(results) diff --git a/routes/ncit.py b/routes/ncit.py index c663658..97b4431 100644 --- a/routes/ncit.py +++ b/routes/ncit.py @@ -63,6 +63,17 @@ def search_ncit(): ) +@bp.route('/concept/') +def concept_detail(ncit_code): + """Return full NCIt concept details as JSON.""" + client = NCItApiClient() + result = client.get_concept(ncit_code) + if 'error' in result: + from flask import abort + abort(404) + return jsonify(result) + + @bp.route('/resolve/', methods=['POST']) def resolve(bc_id): bc = BiomedicalConcept.query.get_or_404(bc_id) diff --git a/services/cdisc_api.py b/services/cdisc_api.py index 975415a..da35780 100644 --- a/services/cdisc_api.py +++ b/services/cdisc_api.py @@ -4,18 +4,33 @@ import requests from flask import current_app -# Simple in-memory cache: {(base_url, api_key_digest, endpoint): (timestamp, data)} +# In-memory cache: {key: (timestamp, data)} +# Entries are never evicted — stale data is served while a refresh is attempted, +# so a timeout never blocks the request with an empty response. _cache = {} -_CACHE_TTL = 300 # 5 minutes +_CACHE_TTL = 300 # serve fresh data for 5 minutes +_CACHE_STALE_TTL = 3600 # serve stale data for up to 1 hour while refresh fails def _cached(cache_key, fn): + """Return cached data if fresh. If stale, attempt a refresh but fall back + to the stale entry rather than propagating an error or blocking indefinitely.""" now = time.time() - if cache_key in _cache and now - _cache[cache_key][0] < _CACHE_TTL: - return _cache[cache_key][1] - data = fn() - _cache[cache_key] = (now, data) - return data + entry = _cache.get(cache_key) + + if entry and now - entry[0] < _CACHE_TTL: + return entry[1] # fresh — serve immediately + + # Attempt a refresh + try: + data = fn() + _cache[cache_key] = (now, data) + return data + except Exception: + if entry and now - entry[0] < _CACHE_STALE_TTL: + # Serve stale rather than an error + return entry[1] + raise # genuinely no data at all — let caller handle class CDISCApiClient: diff --git a/services/loinc_api.py b/services/loinc_api.py new file mode 100644 index 0000000..23785c0 --- /dev/null +++ b/services/loinc_api.py @@ -0,0 +1,49 @@ +import os +import requests + +LOINC_EF_FIELDS = ( + 'LOINC_NUM,SHORTNAME,LONG_COMMON_NAME,RELATEDNAMES2,PROPERTY,' + 'METHOD_TYP,AnswerLists,units,datatype,isCopyrighted,' + 'containsCopyrighted,CONSUMER_NAME,COMPONENT,' + 'EXTERNAL_COPYRIGHT_NOTICE,EXTERNAL_COPYRIGHT_LINK' +) + + +class LoincApiClient: + BASE_URL = 'https://clinicaltables.nlm.nih.gov/api/loinc_items/v3/search' + + def _auth(self): + user = os.environ.get('LOINC_USER') + password = os.environ.get('LOINC_PASSWORD') + if user and password: + return (user, password) + return None + + def search(self, term, size=10): + """Search LOINC by code or name. Returns a list of dicts with all LOINC fields. + + Uses the ef (extra fields) parameter so all field values are returned. + Response format: [total, [codes], {field: [values, ...]}, display_data] + """ + try: + response = requests.get( + self.BASE_URL, + params={'ef': LOINC_EF_FIELDS, 'terms': term, 'maxList': size}, + auth=self._auth(), + timeout=15, + ) + response.raise_for_status() + data = response.json() + # response[1] = list of internal codes (length = number of results) + # response[2] = dict of {field_name: [value_per_result, ...]} + codes = data[1] if len(data) > 1 and data[1] else [] + extra = data[2] if len(data) > 2 and data[2] else {} + results = [] + for i in range(len(codes)): + item = {} + for field, values in extra.items(): + item[field] = values[i] if values and i < len(values) else None + results.append(item) + return results + except Exception as e: + return [{'error': str(e)}] diff --git a/services/ncit_api.py b/services/ncit_api.py index e477599..3380c2b 100644 --- a/services/ncit_api.py +++ b/services/ncit_api.py @@ -33,21 +33,32 @@ def search_concept(self, term, size=10): return [{'error': str(e)}] def get_concept(self, ncit_code): - """Fetch full concept details including synonyms.""" + """Fetch full concept details including synonyms, definitions, parents, and semantic type.""" try: result = self._get(f'/concept/ncit/{ncit_code}', params={'include': 'full'}) return { 'code': result.get('code'), 'name': result.get('name'), + 'preferred_name': result.get('name'), 'definition': next( (d.get('definition') for d in result.get('definitions', []) if d.get('source') == 'NCI'), '' ), + 'definitions': [ + {'definition': d.get('definition'), 'source': d.get('source')} + for d in result.get('definitions', []) + ], 'synonyms': [ s.get('name') for s in result.get('synonyms', []) if s.get('termType') in ('SY', 'AB', 'PT') ], - 'preferred_name': result.get('name'), + 'parents': [ + {'code': p.get('code'), 'name': p.get('name')} + for p in result.get('parents', []) + ], + 'semantic_type': [ + st.get('name') for st in result.get('semanticType', []) + ], } except Exception as e: return {'error': str(e)} diff --git a/static/css/custom.css b/static/css/custom.css index 6e6fb89..234eec8 100644 --- a/static/css/custom.css +++ b/static/css/custom.css @@ -863,6 +863,35 @@ code, .mono { margin-bottom: 8px; } +/* ── LOINC metadata read-only display grid ── */ +.loinc-meta-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 6px 16px; + border-top: 1px solid var(--border-light); + padding-top: 10px; +} + +.loinc-meta-item { + display: flex; + flex-direction: column; + font-size: 11px; + gap: 1px; +} + +.loinc-meta-label { + color: var(--text-secondary); + font-weight: 500; + text-transform: uppercase; + letter-spacing: 0.03em; + font-size: 10px; +} + +.loinc-meta-value { + color: var(--text-primary); + word-break: break-word; +} + /* ── Conflict resolution cards ── */ .conflict-card { border: 1px solid var(--border-mid); diff --git a/static/js/main.js b/static/js/main.js index ee93bbe..b306634 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -164,7 +164,7 @@ // Wire up "Use this concept" buttons container.querySelectorAll('.use-ncit-btn').forEach(function (btn) { - btn.addEventListener('click', function () { + btn.addEventListener('click', async function () { const code = btn.dataset.code; if (ncitCodeInput) ncitCodeInput.value = code; // Also fill short_name if empty @@ -175,10 +175,69 @@ // Hide the panel after selection const panel = document.getElementById('ncit-results-panel'); if (panel) panel.classList.add('d-none'); + + // Fetch full concept detail and render the NCIt metadata grid + try { + const url = new URL(`/ncit/concept/${encodeURIComponent(code)}`, window.location.origin); + const response = await fetch(url.toString(), { headers: { 'Accept': 'application/json' } }); + if (response.ok) { + const data = await response.json(); + renderNcitMetaDisplay(data); + const metaInput = document.getElementById('ncit_metadata'); + if (metaInput) metaInput.value = JSON.stringify(data); + } + } catch (err) { + console.error('NCIt concept detail fetch error:', err); + } }); }); } + /** + * Populates #ncit-meta-display grid with fields from a selected NCIt concept. + * Fields: code, preferred_name, synonyms, definitions, parents, semantic_type. + */ + function renderNcitMetaDisplay(item) { + var display = document.getElementById('ncit-meta-display'); + if (!display) return; + + var fields = [ + { key: 'code', label: 'Code' }, + { key: 'preferred_name',label: 'Preferred Name' }, + { key: 'synonyms', label: 'Source Synonyms' }, + { key: 'definitions', label: 'Definitions' }, + { key: 'parents', label: 'Parent Concepts' }, + { key: 'semantic_type', label: 'Semantic Type' }, + ]; + + var html = fields.filter(function (f) { + var v = item[f.key]; + return v && (!Array.isArray(v) || v.length > 0); + }).map(function (f) { + var v = item[f.key]; + var text; + if (f.key === 'definitions') { + text = v.map(function (d) { return d.definition || ''; }).filter(Boolean).join('; '); + } else if (f.key === 'parents') { + text = v.map(function (p) { return `${p.name} (${p.code})`; }).join('; '); + } else if (Array.isArray(v)) { + text = v.join('; '); + } else { + text = String(v); + } + return `
+ ${escapeHtml(f.label)} + ${escapeHtml(text)} +
`; + }).join(''); + + // Replace inner content but keep the section heading + var heading = display.querySelector('.form-section-title'); + display.innerHTML = (heading ? heading.outerHTML : '

NCIt

') + + '
' + html + '
'; + display.classList.toggle('d-none', !html); + } + /* ───────────────────────────────────────────── DEC table: add / delete rows dynamically ───────────────────────────────────────────── */ @@ -485,6 +544,198 @@ }); } + /* ───────────────────────────────────────────── + LOINC lookup + On click of #loinc-lookup-btn, fetches + /loinc/search?term= and + renders results into #loinc-results-panel. + ───────────────────────────────────────────── */ + function initLoincLookup() { + const lookupBtn = document.getElementById('loinc-lookup-btn'); + const codeInput = document.getElementById('loinc_code'); + const nameInput = document.getElementById('loinc_name'); + const resultsPanel = document.getElementById('loinc-results-panel'); + const resultsContainer = document.getElementById('loinc-results-container'); + + if (!lookupBtn || !codeInput || !resultsPanel) return; + + async function doLoincSearch(term) { + if (!term) return; + lookupBtn.disabled = true; + lookupBtn.textContent = 'Searching…'; + try { + const url = new URL('/loinc/search', window.location.origin); + url.searchParams.set('term', term); + const response = await fetch(url.toString(), { headers: { 'Accept': 'application/json' } }); + if (!response.ok) throw new Error('LOINC search failed: ' + response.status); + const data = await response.json(); + renderLoincResults(data, resultsContainer); + resultsPanel.classList.remove('d-none'); + } catch (err) { + if (resultsContainer) { + resultsContainer.innerHTML = '

Error fetching LOINC results. Please try again.

'; + } + resultsPanel.classList.remove('d-none'); + console.error('LOINC lookup error:', err); + } finally { + lookupBtn.disabled = false; + lookupBtn.textContent = 'Search LOINC'; + } + } + + // Button searches using whichever field has a value (code takes priority) + lookupBtn.addEventListener('click', function () { + var term = codeInput.value.trim() || (nameInput && nameInput.value.trim()); + if (!term) { codeInput.focus(); return; } + doLoincSearch(term); + }); + + // Enter key on code field triggers search + codeInput.addEventListener('keydown', function (e) { + if (e.key === 'Enter') { e.preventDefault(); doLoincSearch(codeInput.value.trim()); } + }); + + // Debounced autocomplete on the Long Common Name field + if (nameInput) { + var nameDebounceTimer; + nameInput.addEventListener('input', function () { + clearTimeout(nameDebounceTimer); + var term = nameInput.value.trim(); + if (!term) { resultsPanel.classList.add('d-none'); return; } + nameDebounceTimer = setTimeout(function () { doLoincSearch(term); }, 400); + }); + nameInput.addEventListener('keydown', function (e) { + if (e.key === 'Enter') { + e.preventDefault(); + clearTimeout(nameDebounceTimer); + doLoincSearch(nameInput.value.trim()); + } + }); + } + } + + // LOINC field labels for display in results and saved metadata grid. + var LOINC_FIELD_LABELS = { + LOINC_NUM: 'LOINC Code', + SHORTNAME: 'Short Name', + LONG_COMMON_NAME: 'Long Common Name', + COMPONENT: 'Component', + PROPERTY: 'Property', + METHOD_TYP: 'Method Type', + units: 'Units', + datatype: 'Data Type', + CONSUMER_NAME: 'Consumer Name', + RELATEDNAMES2: 'Related Names', + AnswerLists: 'Answer Lists', + isCopyrighted: 'Is Copyrighted', + containsCopyrighted: 'Contains Copyrighted', + EXTERNAL_COPYRIGHT_NOTICE: 'Copyright Notice', + EXTERNAL_COPYRIGHT_LINK: 'Copyright Link', + }; + + /** + * Renders LOINC search results into the container element. + * Expects data: Array of objects with LOINC ef fields + * (LOINC_NUM, SHORTNAME, LONG_COMMON_NAME, PROPERTY, …) + */ + function renderLoincResults(data, container) { + if (!container) return; + + if (!data || data.length === 0) { + container.innerHTML = '

No results found.

'; + return; + } + + if (data[0] && data[0].error) { + container.innerHTML = `

Error: ${escapeHtml(data[0].error)}

`; + return; + } + + container.innerHTML = data.map(function (item, idx) { + const safeCode = escapeHtml(item.LOINC_NUM || ''); + const safeName = escapeHtml(item.LONG_COMMON_NAME || ''); + + // Build metadata rows for all non-empty fields except LOINC_NUM + // (shown in the header row; LONG_COMMON_NAME is included in the grid) + var metaRows = Object.keys(LOINC_FIELD_LABELS) + .filter(function (k) { return k !== 'LOINC_NUM' && item[k]; }) + .map(function (k) { + return `
+ ${escapeHtml(LOINC_FIELD_LABELS[k])} + ${escapeHtml(String(item[k]))} +
`; + }).join(''); + + return ` +
+
+ ${safeCode} + +
+
${safeName}
+ ${metaRows ? `
${metaRows}
` : ''} +
+ `; + }).join(''); + + // Store the raw result objects on the container so the "use" handler can access them + container._loincResults = data; + + // Wire up "Use this code" buttons + container.querySelectorAll('.use-loinc-btn').forEach(function (btn) { + btn.addEventListener('click', function () { + var idx = parseInt(btn.dataset.idx, 10); + var item = container._loincResults && container._loincResults[idx]; + if (!item) return; + + var codeInput = document.getElementById('loinc_code'); + var nameInput = document.getElementById('loinc_name'); + var metaInput = document.getElementById('loinc_metadata'); + + if (codeInput) codeInput.value = item.LOINC_NUM || ''; + if (nameInput) nameInput.value = item.LONG_COMMON_NAME || ''; + if (metaInput) metaInput.value = JSON.stringify(item); + + renderLoincMetaDisplay(item); + + var panel = document.getElementById('loinc-results-panel'); + if (panel) panel.classList.add('d-none'); + }); + }); + } + + /** + * Populates the #loinc-meta-display grid with fields from a selected LOINC item. + * Called immediately after the user clicks "Use this code" so they see the + * properties without needing to save and reload the page. + */ + function renderLoincMetaDisplay(item) { + var display = document.getElementById('loinc-meta-display'); + if (!display) return; + + var displayFields = [ + 'LONG_COMMON_NAME', 'SHORTNAME', 'COMPONENT', 'PROPERTY', 'METHOD_TYP', 'units', 'datatype', + 'CONSUMER_NAME', 'RELATEDNAMES2', 'AnswerLists', 'isCopyrighted', + 'containsCopyrighted', 'EXTERNAL_COPYRIGHT_NOTICE', 'EXTERNAL_COPYRIGHT_LINK', + ]; + + var html = displayFields + .filter(function (k) { return item[k]; }) + .map(function (k) { + return `
+ ${escapeHtml(LOINC_FIELD_LABELS[k] || k)} + ${escapeHtml(String(item[k]))} +
`; + }).join(''); + + display.innerHTML = html; + display.classList.toggle('d-none', !html); + } + /* ───────────────────────────────────────────── Init all modules on DOM ready ───────────────────────────────────────────── */ @@ -492,6 +743,7 @@ initFlashDismiss(); initFileUpload(); initNcitLookup(); + initLoincLookup(); initDecTable(); initKanban(); initAuditDiff(); diff --git a/templates/bc_detail.html b/templates/bc_detail.html index 5c2b8c0..022a08d 100644 --- a/templates/bc_detail.html +++ b/templates/bc_detail.html @@ -110,6 +110,45 @@

Identification

+ + +
+

NCIt

+ {% if ncit_data %} + {% set ncit_fields = [ + ('code', 'Code'), + ('preferred_name','Preferred Name'), + ('synonyms', 'Source Synonyms'), + ('definitions', 'Definitions'), + ('parents', 'Parent Concepts'), + ('semantic_type', 'Semantic Type'), + ] %} +
+ {% for key, label in ncit_fields %} + {% set val = ncit_data.get(key) %} + {% if val %} +
+ {{ label }} + + {% if val is iterable and val is not string %} + {% if key == 'definitions' %} + {% for d in val %}{{ d.definition }}{% if not loop.last %}; {% endif %}{% endfor %} + {% elif key == 'parents' %} + {% for p in val %}{{ p.name }} ({{ p.code }}){% if not loop.last %}; {% endif %}{% endfor %} + {% else %} + {{ val | join('; ') }} + {% endif %} + {% else %} + {{ val }} + {% endif %} + +
+ {% endif %} + {% endfor %} +
+ {% endif %} +
+

Classification

@@ -136,28 +175,62 @@

Classification

- +
-

External Coding

-
+

LOINC

+ + +
- - + +
-
- - + + + placeholder="e.g. Hemoglobin A1c/Hemoglobin.total in Blood">
-
- - +
+ +
+
+ + {# Saved LOINC metadata fields (read-only display) — always present so JS can update it #} + {% set loinc = loinc_data %} +
+ {% if loinc %} + {% set loinc_fields = [ + ('LONG_COMMON_NAME', 'Long Common Name'), + ('SHORTNAME', 'Short Name'), + ('COMPONENT', 'Component'), + ('PROPERTY', 'Property'), + ('METHOD_TYP', 'Method Type'), + ('units', 'Units'), + ('datatype', 'Data Type'), + ('CONSUMER_NAME', 'Consumer Name'), + ('RELATEDNAMES2', 'Related Names'), + ('AnswerLists', 'Answer Lists'), + ('isCopyrighted', 'Is Copyrighted'), + ('containsCopyrighted', 'Contains Copyrighted'), + ('EXTERNAL_COPYRIGHT_NOTICE', 'Copyright Notice'), + ('EXTERNAL_COPYRIGHT_LINK', 'Copyright Link'), + ] %} + {% for key, label in loinc_fields %} + {% if loinc.get(key) %} +
+ {{ label }} + {{ loinc[key] }}
+ {% endif %} + {% endfor %} + {% endif %} +
+ +
+
diff --git a/templates/library_bc_detail.html b/templates/library_bc_detail.html index 9108639..893b8f8 100644 --- a/templates/library_bc_detail.html +++ b/templates/library_bc_detail.html @@ -53,28 +53,54 @@

Definition

{{ bc.definition | default('No definition available.') }}
- - {% if bc.coding %} + + {% set loinc_coding = bc.coding | selectattr('systemName', 'equalto', 'LOINC') | list if bc.coding else [] %}
-

External Coding

- {% for c in bc.coding %} -
+

LOINC

+ {% if loinc_coding %} + {% set lc = loinc_coding[0] %} +
- -
{{ c.system | default('—') }}
+ +
{{ lc.code | default('—') }}
-
- -
{{ c.systemName | default('—') }}
+
+ +
{{ loinc_data.get('LONG_COMMON_NAME') or lc.system | default('—') }}
-
- -
{{ c.code | default('—') }}
+
+ {% if loinc_data %} + {% set loinc_fields = [ + ('LONG_COMMON_NAME', 'Long Common Name'), + ('SHORTNAME', 'Short Name'), + ('COMPONENT', 'Component'), + ('PROPERTY', 'Property'), + ('METHOD_TYP', 'Method Type'), + ('units', 'Units'), + ('datatype', 'Data Type'), + ('CONSUMER_NAME', 'Consumer Name'), + ('RELATEDNAMES2', 'Related Names'), + ('AnswerLists', 'Answer Lists'), + ('isCopyrighted', 'Is Copyrighted'), + ('containsCopyrighted', 'Contains Copyrighted'), + ('EXTERNAL_COPYRIGHT_NOTICE', 'Copyright Notice'), + ('EXTERNAL_COPYRIGHT_LINK', 'Copyright Link'), + ] %} +
+ {% for key, label in loinc_fields %} + {% if loinc_data.get(key) %} +
+ {{ label }} + {{ loinc_data[key] }}
+ {% endif %} + {% endfor %}
- {% endfor %} + {% endif %} + {% else %} +

No LOINC code assigned in the CDISC Library for this concept.

+ {% endif %}
- {% endif %}
diff --git a/tests/test_bc_routes.py b/tests/test_bc_routes.py index 56b2c9a..2ffa575 100644 --- a/tests/test_bc_routes.py +++ b/tests/test_bc_routes.py @@ -1,5 +1,6 @@ """Tests for routes/bc.py — CRUD, export, submission.""" import pytest +from unittest.mock import patch from models.bc import BiomedicalConcept, DataElementConcept from models.audit import AuditLog from extensions import db @@ -96,13 +97,71 @@ def test_create_with_decs(self, client, app): class TestBcDetail: def test_existing_bc_returns_200(self, client, sample_bc): - r = client.get('/bc/C12345') + with patch('routes.bc.LoincApiClient') as MockLoinc: + MockLoinc.return_value.search.return_value = [] + r = client.get('/bc/C12345') assert r.status_code == 200 def test_missing_bc_returns_404(self, client): r = client.get('/bc/DOESNOTEXIST') assert r.status_code == 404 + def test_loinc_api_called_when_code_set(self, client, app): + with app.app_context(): + bc = BiomedicalConcept( + bc_id='C99901', + short_name='HbA1c', + status='provisional', + submitter='tester', + code='4548-4', + ) + db.session.add(bc) + db.session.commit() + + loinc_result = {'LOINC_NUM': '4548-4', 'LONG_COMMON_NAME': 'Hemoglobin A1c/Hemoglobin.total in Blood', 'SHORTNAME': 'HbA1c MFr Bld'} + with patch('routes.bc.LoincApiClient') as MockLoinc: + MockLoinc.return_value.search.return_value = [loinc_result] + r = client.get('/bc/C99901') + + assert r.status_code == 200 + MockLoinc.return_value.search.assert_called_once_with('4548-4', size=1) + assert b'HbA1c MFr Bld' in r.data + + def test_loinc_api_not_called_when_no_code(self, client, app): + with app.app_context(): + bc = BiomedicalConcept( + bc_id='C99902', + short_name='No LOINC', + status='provisional', + submitter='tester', + ) + db.session.add(bc) + db.session.commit() + + with patch('routes.bc.LoincApiClient') as MockLoinc: + r = client.get('/bc/C99902') + + assert r.status_code == 200 + MockLoinc.return_value.search.assert_not_called() + + def test_loinc_api_error_does_not_break_page(self, client, app): + with app.app_context(): + bc = BiomedicalConcept( + bc_id='C99903', + short_name='LOINC Error BC', + status='provisional', + submitter='tester', + code='4548-4', + ) + db.session.add(bc) + db.session.commit() + + with patch('routes.bc.LoincApiClient') as MockLoinc: + MockLoinc.return_value.search.return_value = [{'error': 'timeout'}] + r = client.get('/bc/C99903') + + assert r.status_code == 200 + # --------------------------------------------------------------------------- # POST /bc//edit @@ -185,3 +244,75 @@ def test_odm_xml_export(self, client, sample_bc): r = client.get('/bc/export?format=odm') assert r.status_code == 200 assert 'xml' in r.content_type + + +# --------------------------------------------------------------------------- +# GET /bc/library/ +# --------------------------------------------------------------------------- + +LIBRARY_BC_NO_LOINC = { + 'conceptId': 'C147905', + 'shortName': 'Diastolic Blood Pressure', + 'definition': 'The minimum pressure in the arteries.', + 'coding': [], + 'dataElementConcepts': [], +} + +LIBRARY_BC_WITH_LOINC = { + 'conceptId': 'C64849', + 'shortName': 'HbA1c Percent', + 'definition': 'A test measuring HbA1c.', + 'coding': [{'system': 'http://loinc.org/', 'systemName': 'LOINC', 'code': '4548-4'}], + 'dataElementConcepts': [], +} + +LOINC_RESULT = { + 'LOINC_NUM': '4548-4', + 'LONG_COMMON_NAME': 'Hemoglobin A1c/Hemoglobin.total in Blood', + 'SHORTNAME': 'HbA1c MFr Bld', + 'PROPERTY': 'MFr', + 'units': '%', +} + + +class TestLibraryDetail: + def test_renders_page_for_valid_concept(self, client): + with patch('routes.bc.CDISCApiClient') as MockCDISC, \ + patch('routes.bc.LoincApiClient') as MockLoinc: + MockCDISC.return_value.get_bc.return_value = LIBRARY_BC_NO_LOINC + MockLoinc.return_value.search.return_value = [] + r = client.get('/bc/library/C147905') + assert r.status_code == 200 + assert b'Diastolic Blood Pressure' in r.data + + def test_redirects_on_api_error(self, client): + with patch('routes.bc.CDISCApiClient') as MockCDISC: + MockCDISC.return_value.get_bc.return_value = {'error': 'Not found'} + r = client.get('/bc/library/CXXX', follow_redirects=False) + assert r.status_code == 302 + + def test_loinc_api_called_when_loinc_coding_present(self, client): + with patch('routes.bc.CDISCApiClient') as MockCDISC, \ + patch('routes.bc.LoincApiClient') as MockLoinc: + MockCDISC.return_value.get_bc.return_value = LIBRARY_BC_WITH_LOINC + MockLoinc.return_value.search.return_value = [LOINC_RESULT] + r = client.get('/bc/library/C64849') + assert r.status_code == 200 + MockLoinc.return_value.search.assert_called_once_with('4548-4', size=1) + assert b'HbA1c MFr Bld' in r.data + + def test_loinc_api_not_called_when_no_loinc_coding(self, client): + with patch('routes.bc.CDISCApiClient') as MockCDISC, \ + patch('routes.bc.LoincApiClient') as MockLoinc: + MockCDISC.return_value.get_bc.return_value = LIBRARY_BC_NO_LOINC + r = client.get('/bc/library/C147905') + assert r.status_code == 200 + MockLoinc.return_value.search.assert_not_called() + + def test_loinc_api_error_does_not_break_page(self, client): + with patch('routes.bc.CDISCApiClient') as MockCDISC, \ + patch('routes.bc.LoincApiClient') as MockLoinc: + MockCDISC.return_value.get_bc.return_value = LIBRARY_BC_WITH_LOINC + MockLoinc.return_value.search.return_value = [{'error': 'timeout'}] + r = client.get('/bc/library/C64849') + assert r.status_code == 200 diff --git a/tests/test_loinc.py b/tests/test_loinc.py new file mode 100644 index 0000000..7248ca0 --- /dev/null +++ b/tests/test_loinc.py @@ -0,0 +1,184 @@ +"""Tests for services/loinc_api.py and routes/loinc.py.""" +import json +from unittest.mock import MagicMock, patch + +import pytest + +from services.loinc_api import LoincApiClient, LOINC_EF_FIELDS + + +# --------------------------------------------------------------------------- +# Sample NLM response using ef parameter +# response format: [total, [internal_codes], {field: [values...]}, display_data] +# --------------------------------------------------------------------------- + +NLM_EF_RESPONSE = [ + 2, + ['4548-4', '17856-6'], + { + 'LOINC_NUM': ['4548-4', '17856-6'], + 'SHORTNAME': ['HbA1c MFr Bld', 'HbA1c MFr Bld HPLC'], + 'LONG_COMMON_NAME': ['Hemoglobin A1c/Hemoglobin.total in Blood', + 'Hemoglobin A1c/Hemoglobin.total in Blood by HPLC'], + 'RELATEDNAMES2': ['Glycated Hb', 'Glycohemoglobin'], + 'PROPERTY': ['MFr', 'MFr'], + 'METHOD_TYP': [None, 'HPLC'], + 'AnswerLists': [None, None], + 'units': ['%', '%'], + 'datatype': ['NM', 'NM'], + 'isCopyrighted': ['N', 'N'], + 'containsCopyrighted': ['N', 'N'], + 'CONSUMER_NAME': ['Hemoglobin A1c', 'Hemoglobin A1c by HPLC'], + 'COMPONENT': ['Hemoglobin A1c', 'Hemoglobin A1c'], + 'EXTERNAL_COPYRIGHT_NOTICE': [None, None], + 'EXTERNAL_COPYRIGHT_LINK': [None, None], + }, + None, +] + + +# --------------------------------------------------------------------------- +# LoincApiClient.search() +# --------------------------------------------------------------------------- + +class TestLoincApiClientSearch: + def _mock_response(self, data, status=200): + mock = MagicMock() + mock.status_code = status + mock.json.return_value = data + mock.raise_for_status = MagicMock() + return mock + + def test_returns_normalized_list(self): + with patch('services.loinc_api.requests.get') as mock_get: + mock_get.return_value = self._mock_response(NLM_EF_RESPONSE) + results = LoincApiClient().search('hba1c') + + assert len(results) == 2 + assert results[0]['LOINC_NUM'] == '4548-4' + assert results[0]['LONG_COMMON_NAME'] == 'Hemoglobin A1c/Hemoglobin.total in Blood' + assert results[0]['SHORTNAME'] == 'HbA1c MFr Bld' + assert results[0]['units'] == '%' + assert results[0]['datatype'] == 'NM' + assert results[0]['PROPERTY'] == 'MFr' + assert results[0]['METHOD_TYP'] is None + assert results[1]['LOINC_NUM'] == '17856-6' + assert results[1]['METHOD_TYP'] == 'HPLC' + + def test_all_ef_fields_present_in_result(self): + with patch('services.loinc_api.requests.get') as mock_get: + mock_get.return_value = self._mock_response(NLM_EF_RESPONSE) + results = LoincApiClient().search('hba1c') + + expected_fields = [ + 'LOINC_NUM', 'SHORTNAME', 'LONG_COMMON_NAME', 'RELATEDNAMES2', + 'PROPERTY', 'METHOD_TYP', 'AnswerLists', 'units', 'datatype', + 'isCopyrighted', 'containsCopyrighted', 'CONSUMER_NAME', 'COMPONENT', + 'EXTERNAL_COPYRIGHT_NOTICE', 'EXTERNAL_COPYRIGHT_LINK', + ] + for field in expected_fields: + assert field in results[0], f"Missing field: {field}" + + def test_uses_ef_parameter(self): + with patch('services.loinc_api.requests.get') as mock_get: + mock_get.return_value = self._mock_response(NLM_EF_RESPONSE) + LoincApiClient().search('glucose', size=5) + + call_kwargs = mock_get.call_args[1] + params = call_kwargs['params'] + assert 'ef' in params + assert 'df' not in params + assert 'LOINC_NUM' in params['ef'] + assert 'LONG_COMMON_NAME' in params['ef'] + assert params['terms'] == 'glucose' + assert params['maxList'] == 5 + + def test_ef_fields_constant_contains_all_required_fields(self): + required = [ + 'LOINC_NUM', 'SHORTNAME', 'LONG_COMMON_NAME', 'RELATEDNAMES2', + 'PROPERTY', 'METHOD_TYP', 'AnswerLists', 'units', 'datatype', + 'isCopyrighted', 'containsCopyrighted', 'CONSUMER_NAME', 'COMPONENT', + 'EXTERNAL_COPYRIGHT_NOTICE', 'EXTERNAL_COPYRIGHT_LINK', + ] + for field in required: + assert field in LOINC_EF_FIELDS, f"Missing from LOINC_EF_FIELDS: {field}" + + def test_uses_basic_auth_when_env_vars_set(self, monkeypatch): + monkeypatch.setenv('LOINC_USER', 'myuser') + monkeypatch.setenv('LOINC_PASSWORD', 'mypass') + with patch('services.loinc_api.requests.get') as mock_get: + mock_get.return_value = self._mock_response([0, [], {}, None]) + LoincApiClient().search('test') + + auth = mock_get.call_args[1].get('auth') + assert auth == ('myuser', 'mypass') + + def test_no_auth_when_env_vars_missing(self, monkeypatch): + monkeypatch.delenv('LOINC_USER', raising=False) + monkeypatch.delenv('LOINC_PASSWORD', raising=False) + with patch('services.loinc_api.requests.get') as mock_get: + mock_get.return_value = self._mock_response([0, [], {}, None]) + LoincApiClient().search('test') + + auth = mock_get.call_args[1].get('auth') + assert auth is None + + def test_empty_results(self): + with patch('services.loinc_api.requests.get') as mock_get: + mock_get.return_value = self._mock_response([0, [], {}, None]) + results = LoincApiClient().search('zzznomatch') + + assert results == [] + + def test_missing_codes_array_returns_empty(self): + with patch('services.loinc_api.requests.get') as mock_get: + mock_get.return_value = self._mock_response([0]) + results = LoincApiClient().search('test') + + assert results == [] + + def test_network_error_returns_error_entry(self): + with patch('services.loinc_api.requests.get', side_effect=Exception('timeout')): + results = LoincApiClient().search('hba1c') + + assert len(results) == 1 + assert 'error' in results[0] + assert 'timeout' in results[0]['error'] + + +# --------------------------------------------------------------------------- +# GET /loinc/search route +# --------------------------------------------------------------------------- + +class TestLoincSearchRoute: + def test_returns_json_for_ajax(self, client): + with patch('routes.loinc.LoincApiClient') as MockClient: + MockClient.return_value.search.return_value = [ + {'LOINC_NUM': '4548-4', 'LONG_COMMON_NAME': 'Hemoglobin A1c/Hemoglobin.total in Blood'} + ] + r = client.get('/loinc/search?term=hba1c', headers={'Accept': 'application/json'}) + + assert r.status_code == 200 + data = json.loads(r.data) + assert isinstance(data, list) + assert data[0]['LOINC_NUM'] == '4548-4' + + def test_empty_term_returns_empty_list(self, client): + r = client.get('/loinc/search', headers={'Accept': 'application/json'}) + assert r.status_code == 200 + assert json.loads(r.data) == [] + + def test_calls_client_with_term(self, client): + with patch('routes.loinc.LoincApiClient') as MockClient: + MockClient.return_value.search.return_value = [] + client.get('/loinc/search?term=glucose', headers={'Accept': 'application/json'}) + + MockClient.return_value.search.assert_called_once_with('glucose', size=10) + + def test_format_json_param_triggers_json_response(self, client): + with patch('routes.loinc.LoincApiClient') as MockClient: + MockClient.return_value.search.return_value = [] + r = client.get('/loinc/search?term=hba1c&format=json') + + assert r.status_code == 200 + assert r.content_type.startswith('application/json') diff --git a/tests/test_ncit.py b/tests/test_ncit.py new file mode 100644 index 0000000..4739601 --- /dev/null +++ b/tests/test_ncit.py @@ -0,0 +1,150 @@ +"""Tests for NCIt service extensions and the /ncit/concept/ route.""" +import json +from unittest.mock import MagicMock, patch + +import pytest + +from services.ncit_api import NCItApiClient + + +# --------------------------------------------------------------------------- +# Sample EVS full-concept response +# --------------------------------------------------------------------------- + +EVS_FULL_CONCEPT = { + 'code': 'C64849', + 'name': 'Hemoglobin A1c Measurement', + 'definitions': [ + {'definition': 'A quantitative measurement of HbA1c.', 'source': 'NCI'}, + {'definition': 'Other source def.', 'source': 'OTHER'}, + ], + 'synonyms': [ + {'name': 'HbA1c', 'termType': 'SY', 'source': 'NCI'}, + {'name': 'Glycated Hemoglobin', 'termType': 'SY', 'source': 'CDISC'}, + {'name': 'A1C', 'termType': 'AB', 'source': 'NCI'}, + {'name': 'Hemoglobin A1c Measurement', 'termType': 'PT', 'source': 'NCI'}, + {'name': 'Internal Code', 'termType': 'CODE', 'source': 'NCI'}, + ], + 'parents': [ + {'code': 'C17721', 'name': 'Laboratory Test'}, + {'code': 'C45398', 'name': 'Glucose Measurement'}, + ], + 'semanticType': [ + {'name': 'Laboratory Procedure'}, + ], +} + + +# --------------------------------------------------------------------------- +# NCItApiClient.get_concept() — extended fields +# --------------------------------------------------------------------------- + +class TestNcitGetConceptExtended: + def _mock_get(self, data): + mock = MagicMock() + mock.json.return_value = data + mock.raise_for_status = MagicMock() + return mock + + def test_returns_parents(self): + with patch('services.ncit_api.requests.get') as mock_get: + mock_get.return_value = self._mock_get(EVS_FULL_CONCEPT) + result = NCItApiClient().get_concept('C64849') + + assert result['parents'] == [ + {'code': 'C17721', 'name': 'Laboratory Test'}, + {'code': 'C45398', 'name': 'Glucose Measurement'}, + ] + + def test_returns_semantic_type(self): + with patch('services.ncit_api.requests.get') as mock_get: + mock_get.return_value = self._mock_get(EVS_FULL_CONCEPT) + result = NCItApiClient().get_concept('C64849') + + assert result['semantic_type'] == ['Laboratory Procedure'] + + def test_returns_source_synonyms(self): + with patch('services.ncit_api.requests.get') as mock_get: + mock_get.return_value = self._mock_get(EVS_FULL_CONCEPT) + result = NCItApiClient().get_concept('C64849') + + # SY, AB, PT terms — CODE excluded + assert 'HbA1c' in result['synonyms'] + assert 'A1C' in result['synonyms'] + assert 'Hemoglobin A1c Measurement' in result['synonyms'] + assert 'Internal Code' not in result['synonyms'] + + def test_returns_all_definitions(self): + with patch('services.ncit_api.requests.get') as mock_get: + mock_get.return_value = self._mock_get(EVS_FULL_CONCEPT) + result = NCItApiClient().get_concept('C64849') + + assert 'definitions' in result + assert isinstance(result['definitions'], list) + assert any(d['definition'] == 'A quantitative measurement of HbA1c.' for d in result['definitions']) + + def test_empty_parents_returns_empty_list(self): + data = dict(EVS_FULL_CONCEPT, parents=[]) + with patch('services.ncit_api.requests.get') as mock_get: + mock_get.return_value = self._mock_get(data) + result = NCItApiClient().get_concept('C64849') + + assert result['parents'] == [] + + def test_missing_semantic_type_returns_empty_list(self): + data = {k: v for k, v in EVS_FULL_CONCEPT.items() if k != 'semanticType'} + with patch('services.ncit_api.requests.get') as mock_get: + mock_get.return_value = self._mock_get(data) + result = NCItApiClient().get_concept('C64849') + + assert result['semantic_type'] == [] + + def test_error_returns_error_dict(self): + with patch('services.ncit_api.requests.get', side_effect=Exception('timeout')): + result = NCItApiClient().get_concept('C64849') + + assert 'error' in result + + +# --------------------------------------------------------------------------- +# GET /ncit/concept/ route +# --------------------------------------------------------------------------- + +CONCEPT_RESULT = { + 'code': 'C64849', + 'name': 'Hemoglobin A1c Measurement', + 'preferred_name': 'Hemoglobin A1c Measurement', + 'definition': 'A quantitative measurement of HbA1c.', + 'definitions': [{'definition': 'A quantitative measurement of HbA1c.', 'source': 'NCI'}], + 'synonyms': ['HbA1c', 'A1C'], + 'parents': [{'code': 'C17721', 'name': 'Laboratory Test'}], + 'semantic_type': ['Laboratory Procedure'], +} + + +class TestNcitConceptRoute: + def test_returns_json(self, client): + with patch('routes.ncit.NCItApiClient') as MockClient: + MockClient.return_value.get_concept.return_value = CONCEPT_RESULT + r = client.get('/ncit/concept/C64849', headers={'Accept': 'application/json'}) + + assert r.status_code == 200 + data = json.loads(r.data) + assert data['code'] == 'C64849' + assert data['preferred_name'] == 'Hemoglobin A1c Measurement' + assert 'parents' in data + assert 'semantic_type' in data + + def test_calls_get_concept_with_code(self, client): + with patch('routes.ncit.NCItApiClient') as MockClient: + MockClient.return_value.get_concept.return_value = CONCEPT_RESULT + client.get('/ncit/concept/C64849') + + MockClient.return_value.get_concept.assert_called_once_with('C64849') + + def test_error_from_service_returns_500(self, client): + with patch('routes.ncit.NCItApiClient') as MockClient: + MockClient.return_value.get_concept.return_value = {'error': 'Not found'} + r = client.get('/ncit/concept/CXXXXX') + + assert r.status_code == 404 From d0e3cf512b3d5493652130c32cbf9aee96a95127 Mon Sep 17 00:00:00 2001 From: Darren <3921919+pendingintent@users.noreply.github.com> Date: Tue, 14 Apr 2026 11:17:20 -0400 Subject: [PATCH 02/36] Performance improvements for bc details --- README.md | 10 ++-- routes/bc.py | 53 ++++++++++++++---- routes/dashboard.py | 33 +++-------- services/ncit_api.py | 17 +++++- templates/bc_detail.html | 118 ++++++++++++++++++++++++++++++++++++++- 5 files changed, 187 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 41794a2..2a2dabf 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ The sidebar navigation exposes seven screens, accessible at these URL prefixes: |--------|-----|-------------| | Dashboard | `/` | KPI cards (total BCs, pending review, published), governance pipeline chart with concurrent CDISC API fetches (ThreadPoolExecutor), recent submissions table | | Ingestion | `/ingestion` | Upload XLSX, CSV, or JSON files; AI field mapper assigns confidence scores; approve or reject rows to the database | -| BCs | `/bc` | Browse, create, edit, and delete Biomedical Concepts; LOINC code entry with live search and automatic metadata population from NLM Clinical Tables API (LONG_COMMON_NAME, SHORTNAME, COMPONENT, units, etc.); NCIt concept selection with full metadata display (definitions, parents, semantic type); Data Element Concept sub-records | +| BCs | `/bc` | Browse, create, edit, and delete Biomedical Concepts; LOINC code entry with live search and automatic metadata population from NLM Clinical Tables API (LONG_COMMON_NAME, SHORTNAME, COMPONENT, units, etc.); NCIt concept selection with full metadata display (definitions, parents, semantic type); concurrent LOINC and NCIt API fetches with stored metadata prioritization; Data Element Concept sub-records | | NCIT Mapping | `/ncit` | Search the NCI Thesaurus, resolve low-confidence mappings, and confirm NCIt codes for each BC | | Specializations | `/specializations` | View and generate SDTM/CDASH dataset specializations and CRF variable mappings | | Governance | `/governance` | 4-stage Kanban board (Provisional > SME Review > CDISC Approval > Published) with advance and reject actions | @@ -130,7 +130,7 @@ cdisc-concept-curation/ ├── routes/ # 8 Flask blueprints │ ├── dashboard.py # Concurrent CDISC API fetches (ThreadPoolExecutor), KPI cards │ ├── ingestion.py # File upload and AI field mapper -│ ├── bc.py # Create, edit, detail views with LOINC and NCIt API integration +│ ├── bc.py # Create, edit, detail views; concurrent LOINC and NCIt API fetches (ThreadPoolExecutor) with stored metadata prioritization │ ├── ncit.py # GET /ncit/search and GET /ncit/concept/ JSON endpoints with full metadata │ ├── loinc.py # GET /loinc/search JSON API endpoint │ ├── specializations.py # Dataset specializations and CRF mappings @@ -138,7 +138,7 @@ cdisc-concept-curation/ │ └── audit.py # Immutable change log with filters ├── services/ │ ├── cdisc_api.py # CDISC Library API client with stale-while-refresh caching (5-min fresh TTL, 1-hour stale fallback) -│ ├── ncit_api.py # NCI EVS REST API client (full concept detail with definitions, parents, semantic type) +│ ├── ncit_api.py # NCI EVS REST API client with in-memory caching (5-min fresh TTL, 1-hour stale fallback); full concept detail with definitions, parents, semantic type │ ├── loinc_api.py # NLM Clinical Tables API client (optional Basic Auth, metadata caching) │ ├── ingestion.py # File parser and AI field mapper │ └── export.py # XLSX, JSON, ODM-XML export @@ -159,6 +159,6 @@ cdisc-concept-curation/ The platform integrates with three external APIs to provide rich concept metadata: -- **CDISC Library** (`https://api.library.cdisc.org/api/cosmos/v2`) — Requires `CDISC_API_KEY`. Used in Dashboard and BC Library detail views. Implements stale-while-refresh caching to gracefully handle transient failures. -- **NCI EVS REST API** (`https://api-evsrest.nci.nih.gov/api/v1`) — No authentication required. Returns NCIt concept definitions, parent concepts, and semantic types. Integrated into BC detail views via `/ncit/concept/` endpoint. +- **CDISC Library** (`https://api.library.cdisc.org/api/cosmos/v2`) — Requires `CDISC_API_KEY`. Used in Dashboard and BC Library detail views. Implements stale-while-refresh caching (5-min fresh TTL, 1-hour stale fallback) to gracefully handle transient failures. +- **NCI EVS REST API** (`https://api-evsrest.nci.nih.gov/api/v1`) — No authentication required. Returns NCIt concept definitions, parent concepts, and semantic types. Integrated into BC detail views via `/ncit/concept/` endpoint. Implements in-memory caching (5-min fresh TTL, 1-hour stale fallback) to serve cached data rapidly and degrade gracefully when the service is unavailable. - **NLM Clinical Tables API (LOINC)** (`https://clinicaltables.nlm.nih.gov/api/loinc_items/v3/search`) — Optional Basic Auth via `LOINC_USER` / `LOINC_PASSWORD`. Returns LOINC metadata including LONG_COMMON_NAME, SHORTNAME, COMPONENT, METHOD_TYP, units, datatype, and copyright notices. Integrated into BC detail views via `/loinc/search` endpoint. diff --git a/routes/bc.py b/routes/bc.py index 828af73..8140456 100644 --- a/routes/bc.py +++ b/routes/bc.py @@ -1,4 +1,5 @@ import json +from concurrent.futures import ThreadPoolExecutor from flask import Blueprint, render_template, request, redirect, url_for, flash, Response from models.bc import BiomedicalConcept, DataElementConcept from models.audit import AuditLog @@ -117,22 +118,14 @@ def detail(bc_id): .all() ) loinc_data = {} - if bc.code: - results = LoincApiClient().search(bc.code, size=1) - if results and not results[0].get('error'): - loinc_data = results[0] - elif bc.loinc_metadata: + if bc.loinc_metadata: try: loinc_data = json.loads(bc.loinc_metadata) except (ValueError, TypeError): pass ncit_data = {} - if bc.ncit_code: - result = NCItApiClient().get_concept(bc.ncit_code) - if not result.get('error'): - ncit_data = result - elif bc.ncit_metadata: + if bc.ncit_metadata: try: ncit_data = json.loads(bc.ncit_metadata) except (ValueError, TypeError): @@ -145,10 +138,50 @@ def detail(bc_id): is_new=False, loinc_data=loinc_data, ncit_data=ncit_data, + needs_loinc_fetch=not loinc_data and bool(bc.code), + needs_ncit_fetch=not ncit_data and bool(bc.ncit_code), page_title=bc.short_name, ) +@bp.route('//fetch-metadata') +def fetch_metadata(bc_id): + """Fetch LOINC and NCIt data concurrently for a BC that has no stored metadata yet. + Saves results to the DB so subsequent visits use the fast stored-metadata path.""" + from flask import jsonify + bc = BiomedicalConcept.query.get_or_404(bc_id) + + def _fetch_loinc(): + results = LoincApiClient().search(bc.code, size=1) + return results[0] if results and not results[0].get('error') else {} + + def _fetch_ncit(): + result = NCItApiClient().get_concept(bc.ncit_code) + return result if not result.get('error') else {} + + loinc_data = {} + ncit_data = {} + with ThreadPoolExecutor(max_workers=2) as ex: + loinc_future = ex.submit(_fetch_loinc) if bc.code else None + ncit_future = ex.submit(_fetch_ncit) if bc.ncit_code else None + if loinc_future: + loinc_data = loinc_future.result() + if ncit_future: + ncit_data = ncit_future.result() + + changed = False + if loinc_data and not bc.loinc_metadata: + bc.loinc_metadata = json.dumps(loinc_data) + changed = True + if ncit_data and not bc.ncit_metadata: + bc.ncit_metadata = json.dumps(ncit_data) + changed = True + if changed: + db.session.commit() + + return jsonify(loinc=loinc_data, ncit=ncit_data) + + @bp.route('/', methods=['POST']) def create(): bc_id = request.form.get('bc_id', '').strip() diff --git a/routes/dashboard.py b/routes/dashboard.py index bb37997..6939ecf 100644 --- a/routes/dashboard.py +++ b/routes/dashboard.py @@ -13,35 +13,22 @@ def index(): # --- CDISC Library API data (fetched concurrently) --- client = CDISCApiClient() with ThreadPoolExecutor(max_workers=2) as pool: - fut_bcs = pool.submit(client.get_biomedical_concepts) + fut_bcs = pool.submit(client.get_biomedical_concepts) fut_specs = pool.submit(client.get_dataset_specializations) - api_bcs = fut_bcs.result() + api_bcs = fut_bcs.result() api_specs = fut_specs.result() - api_bc_error = ( - api_bcs[0].get("error") if api_bcs and "error" in api_bcs[0] else None - ) - api_spec_error = ( - api_specs[0].get("error") if api_specs and "error" in api_specs[0] else None - ) + api_bc_error = api_bcs[0].get("error") if api_bcs and "error" in api_bcs[0] else None + api_spec_error = api_specs[0].get("error") if api_specs and "error" in api_specs[0] else None api_bc_count = len(api_bcs) if not api_bc_error else 0 api_spec_count = len(api_specs) if not api_spec_error else 0 # --- Local DB stats --- local_total_bcs = BiomedicalConcept.query.count() - local_pending = BiomedicalConcept.query.filter( - BiomedicalConcept.status.in_(["provisional", "sme_review", "cdisc_approval"]) - ).count() - recent_additions = BiomedicalConcept.query.filter( - BiomedicalConcept.created_at >= datetime.utcnow() - timedelta(days=7) - ).count() + local_pending = BiomedicalConcept.query.filter(BiomedicalConcept.status.in_(["provisional", "sme_review", "cdisc_approval"])).count() + recent_additions = BiomedicalConcept.query.filter(BiomedicalConcept.created_at >= datetime.utcnow() - timedelta(days=7)).count() - governance_items = ( - BiomedicalConcept.query.filter(BiomedicalConcept.status != "published") - .order_by(BiomedicalConcept.updated_at.desc()) - .limit(10) - .all() - ) + governance_items = BiomedicalConcept.query.filter(BiomedicalConcept.status != "published").order_by(BiomedicalConcept.updated_at.desc()).limit(10).all() recent_audits = AuditLog.query.order_by(AuditLog.timestamp.desc()).limit(10).all() stats = { @@ -61,11 +48,7 @@ def index(): api_spec_error=api_spec_error, api_bc_count=api_bc_count, api_spec_count=api_spec_count, - recent_submissions=BiomedicalConcept.query.order_by( - BiomedicalConcept.created_at.desc() - ) - .limit(10) - .all(), + recent_submissions=BiomedicalConcept.query.order_by(BiomedicalConcept.created_at.desc()).limit(10).all(), governance_items=governance_items, recent_audits=recent_audits, page_title="Dashboard", diff --git a/services/ncit_api.py b/services/ncit_api.py index 3380c2b..e95dcce 100644 --- a/services/ncit_api.py +++ b/services/ncit_api.py @@ -1,5 +1,10 @@ +import time import requests +_ncit_cache = {} +_NCIT_TTL = 300 # serve fresh data for 5 minutes +_NCIT_STALE_TTL = 3600 # serve stale data for up to 1 hour while refresh fails + class NCItApiClient: BASE_URL = 'https://api-evsrest.nci.nih.gov/api/v1' @@ -34,9 +39,15 @@ def search_concept(self, term, size=10): def get_concept(self, ncit_code): """Fetch full concept details including synonyms, definitions, parents, and semantic type.""" + cache_key = ('concept', ncit_code) + now = time.time() + entry = _ncit_cache.get(cache_key) + if entry and now - entry[0] < _NCIT_TTL: + return entry[1] + try: result = self._get(f'/concept/ncit/{ncit_code}', params={'include': 'full'}) - return { + data = { 'code': result.get('code'), 'name': result.get('name'), 'preferred_name': result.get('name'), @@ -60,7 +71,11 @@ def get_concept(self, ncit_code): st.get('name') for st in result.get('semanticType', []) ], } + _ncit_cache[cache_key] = (now, data) + return data except Exception as e: + if entry and now - entry[0] < _NCIT_STALE_TTL: + return entry[1] return {'error': str(e)} def get_preferred_name(self, ncit_code): diff --git a/templates/bc_detail.html b/templates/bc_detail.html index 022a08d..419bc17 100644 --- a/templates/bc_detail.html +++ b/templates/bc_detail.html @@ -112,8 +112,15 @@

Identification

-
-

NCIt

+
+

+ NCIt + {% if needs_ncit_fetch %} + + Loading… + + {% endif %} +

{% if ncit_data %} {% set ncit_fields = [ ('code', 'Code'), @@ -177,7 +184,14 @@

Classification

-

LOINC

+

+ LOINC + {% if needs_loinc_fetch %} + + Loading… + + {% endif %} +

@@ -359,3 +373,101 @@

Data Element Concepts (DECs)

{% endblock %} + +{% if needs_loinc_fetch or needs_ncit_fetch %} +{% block extra_js %} + +{% endblock %} +{% endif %} From 09c1d8e6add4d260af137576db8612927e33f30c Mon Sep 17 00:00:00 2001 From: Darren <3921919+pendingintent@users.noreply.github.com> Date: Tue, 14 Apr 2026 11:32:21 -0400 Subject: [PATCH 03/36] Changed headers/titles in HTML pages --- requirements.txt | 2 ++ templates/governance.html | 15 +++++---------- templates/library_bc_detail.html | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/requirements.txt b/requirements.txt index 168c07a..aa8e25d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,3 +31,5 @@ urllib3==2.6.3 Werkzeug==3.1.7 pytest==8.3.5 pytest-flask==1.3.0 +pre-commit==4.2.0 +black==26.3.1 diff --git a/templates/governance.html b/templates/governance.html index 8af0866..91c2f7b 100644 --- a/templates/governance.html +++ b/templates/governance.html @@ -27,7 +27,7 @@

Governance Workflow Board

{{ columns.published | length if columns else 0 }}
-
Published
+
Ready to Publish
@@ -62,12 +62,7 @@

Governance Workflow Board

aria-label="Advance {{ bc.short_name }} to SME Review"> Advance - +
{% endfor %} @@ -165,7 +160,7 @@

Governance Workflow Board

- Published + Ready to Publish {{ columns.published | length if columns else 0 }} @@ -181,11 +176,11 @@

Governance Workflow Board

{% if bc.ncit_code %}{{ bc.ncit_code }}{% endif %} {% if bc.package_date %} · Published {{ bc.package_date }}{% endif %}
- Published + Ready to Publish
{% endfor %} {% else %} -
No published BCs yet.
+
No BCs ready for publication yet.
{% endif %}
diff --git a/templates/library_bc_detail.html b/templates/library_bc_detail.html index 9108639..237d2cc 100644 --- a/templates/library_bc_detail.html +++ b/templates/library_bc_detail.html @@ -88,7 +88,7 @@

Metadata

{% endif %} {% if bc.href %}
- +
{{ bc.href }}
{% endif %} From 5ceb91ac2bcb87a5b144cbff69c2a9e8aa4757e8 Mon Sep 17 00:00:00 2001 From: Darren <3921919+pendingintent@users.noreply.github.com> Date: Tue, 14 Apr 2026 11:33:55 -0400 Subject: [PATCH 04/36] Configured pre-commit with flake8 and black --- .pre-commit-config.yaml | 10 ++ README.md | 19 +++ app.py | 16 +-- config.py | 11 +- models/audit.py | 8 +- models/bc.py | 46 ++++---- models/governance.py | 6 +- models/ingestion.py | 12 +- models/specialization.py | 6 +- routes/audit.py | 28 +++-- routes/bc.py | 197 +++++++++++++++----------------- routes/dashboard.py | 29 +---- routes/governance.py | 73 ++++++------ routes/ingestion.py | 164 +++++++++++++------------- routes/ncit.py | 52 ++++----- routes/specializations.py | 65 +++++------ services/export.py | 87 ++++++++------ services/ingestion.py | 156 +++++++++++++------------ services/ncit_api.py | 42 +++---- tests/conftest.py | 24 ++-- tests/test_audit_routes.py | 41 +++---- tests/test_bc_routes.py | 113 +++++++++--------- tests/test_governance_routes.py | 78 ++++++------- tests/test_ingestion_routes.py | 123 ++++++++++---------- tests/test_ingestion_service.py | 141 ++++++++++++----------- tests/test_models.py | 61 +++++----- 26 files changed, 810 insertions(+), 798 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..d71bf48 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,10 @@ +repos: + - repo: https://github.com/psf/black + rev: 26.3.1 + hooks: + - id: black + + - repo: https://github.com/PyCQA/flake8 + rev: 7.3.0 + hooks: + - id: flake8 diff --git a/README.md b/README.md index fbf4e9b..1d6c603 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,13 @@ source .venv/bin/activate # macOS / Linux # 3. Install dependencies pip install -r requirements.txt + +# 4. Install git hooks for code quality +pre-commit install ``` +Pre-commit hooks will now run automatically before each `git commit`, enforcing code formatting (black) and linting (flake8). + ### Dependencies installed | Package | Version | Purpose | @@ -40,11 +45,16 @@ pip install -r requirements.txt | lxml | 5.2.2 | ODM-XML export | | pytest | 8.3.5 | Unit and integration tests | | pytest-flask | 1.3.0 | Flask test client fixture | +| pre-commit | 4.2.0 | Git hook framework for code quality checks | +| black | 26.3.1 | Python code formatter (via pre-commit) | +| flake8 | 7.3.0 | Python linter (via pre-commit) | --- ## Configuration +### Environment Variables + The application is configured entirely through environment variables. | Variable | Required | Default | Description | @@ -60,6 +70,15 @@ export CDISC_API_KEY=your_cdisc_api_key_here export SECRET_KEY=a-strong-random-secret # recommended for non-dev use ``` +### Code Quality + +Code style and linting are enforced automatically via [pre-commit](https://pre-commit.com) hooks (configured in [`.pre-commit-config.yaml`](.pre-commit-config.yaml)): + +- **black** (26.3.1) — enforces consistent Python formatting with a line length of 200 characters (configured in `pyproject.toml`) +- **flake8** (7.3.0) — enforces PEP8 linting rules (configured in [`.flake8`](.flake8)) + +These hooks run automatically before each `git commit`. If black reformats any files, the commit is blocked and you must `git add` the reformatted files and retry the commit. + --- ## Running the App diff --git a/app.py b/app.py index 3dfdb15..149440f 100644 --- a/app.py +++ b/app.py @@ -18,13 +18,13 @@ def create_app(config_class=Config): from routes.governance import bp as governance_bp from routes.audit import bp as audit_bp - app.register_blueprint(dashboard_bp, url_prefix='/') - app.register_blueprint(ingestion_bp, url_prefix='/ingestion') - app.register_blueprint(bc_bp, url_prefix='/bc') - app.register_blueprint(ncit_bp, url_prefix='/ncit') - app.register_blueprint(specializations_bp, url_prefix='/specializations') - app.register_blueprint(governance_bp, url_prefix='/governance') - app.register_blueprint(audit_bp, url_prefix='/audit') + app.register_blueprint(dashboard_bp, url_prefix="/") + app.register_blueprint(ingestion_bp, url_prefix="/ingestion") + app.register_blueprint(bc_bp, url_prefix="/bc") + app.register_blueprint(ncit_bp, url_prefix="/ncit") + app.register_blueprint(specializations_bp, url_prefix="/specializations") + app.register_blueprint(governance_bp, url_prefix="/governance") + app.register_blueprint(audit_bp, url_prefix="/audit") with app.app_context(): db.create_all() @@ -32,6 +32,6 @@ def create_app(config_class=Config): return app -if __name__ == '__main__': +if __name__ == "__main__": app = create_app() app.run(debug=True) diff --git a/config.py b/config.py index 3a00811..eac9c0a 100644 --- a/config.py +++ b/config.py @@ -1,10 +1,11 @@ import os + class Config: - SECRET_KEY = os.environ.get('SECRET_KEY', 'dev-secret-key-change-in-prod') - SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URL', 'sqlite:///cdisc_curation.db') + SECRET_KEY = os.environ.get("SECRET_KEY", "dev-secret-key-change-in-prod") + SQLALCHEMY_DATABASE_URI = os.environ.get("DATABASE_URL", "sqlite:///cdisc_curation.db") SQLALCHEMY_TRACK_MODIFICATIONS = False - CDISC_API_KEY = os.environ.get('CDISC_API_KEY', '') - CDISC_API_BASE_URL = 'https://api.library.cdisc.org/api/cosmos/v2' - NCIT_API_BASE_URL = 'https://api-evsrest.nci.nih.gov/api/v1' + CDISC_API_KEY = os.environ.get("CDISC_API_KEY", "") + CDISC_API_BASE_URL = "https://api.library.cdisc.org/api/cosmos/v2" + NCIT_API_BASE_URL = "https://api-evsrest.nci.nih.gov/api/v1" MAX_CONTENT_LENGTH = 16 * 1024 * 1024 # 16MB upload limit diff --git a/models/audit.py b/models/audit.py index 9c65c74..a561d6d 100644 --- a/models/audit.py +++ b/models/audit.py @@ -4,14 +4,14 @@ class AuditLog(db.Model): - __tablename__ = 'audit_logs' + __tablename__ = "audit_logs" id = db.Column(db.Integer, primary_key=True) entity_type = db.Column(db.String(50)) # BiomedicalConcept, GovernanceRecord, etc. entity_id = db.Column(db.String(100)) action = db.Column(db.String(100)) # created, updated, status_changed, deleted - actor = db.Column(db.String(100), default='system') - _before_state = db.Column('before_state', db.Text) - _after_state = db.Column('after_state', db.Text) + actor = db.Column(db.String(100), default="system") + _before_state = db.Column("before_state", db.Text) + _after_state = db.Column("after_state", db.Text) timestamp = db.Column(db.DateTime, default=datetime.utcnow) @property diff --git a/models/bc.py b/models/bc.py index 8cc4f02..328f020 100644 --- a/models/bc.py +++ b/models/bc.py @@ -3,12 +3,12 @@ class BiomedicalConcept(db.Model): - __tablename__ = 'biomedical_concepts' + __tablename__ = "biomedical_concepts" bc_id = db.Column(db.String(50), primary_key=True) # NCIt C-code e.g. C49237 short_name = db.Column(db.String(255), nullable=False) definition = db.Column(db.Text) ncit_code = db.Column(db.String(50)) - parent_bc_id = db.Column(db.String(50), db.ForeignKey('biomedical_concepts.bc_id'), nullable=True) + parent_bc_id = db.Column(db.String(50), db.ForeignKey("biomedical_concepts.bc_id"), nullable=True) bc_categories = db.Column(db.String(500)) # semicolon-separated synonyms = db.Column(db.Text) result_scales = db.Column(db.String(255)) # e.g. "Quantitative; Ordinal" @@ -16,41 +16,41 @@ class BiomedicalConcept(db.Model): system_name = db.Column(db.String(100)) # e.g. LOINC code = db.Column(db.String(50)) # code in external system package_date = db.Column(db.String(20)) - status = db.Column(db.String(50), default='provisional') # provisional/sme_review/cdisc_approval/published + status = db.Column(db.String(50), default="provisional") # provisional/sme_review/cdisc_approval/published submitter = db.Column(db.String(100)) created_at = db.Column(db.DateTime, default=datetime.utcnow) updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) history_of_change = db.Column(db.Text) - source = db.Column(db.String(50), default='local') # 'local' or 'cdisc_api' + source = db.Column(db.String(50), default="local") # 'local' or 'cdisc_api' - children = db.relationship('BiomedicalConcept', backref=db.backref('parent', remote_side='BiomedicalConcept.bc_id'), lazy='dynamic') - decs = db.relationship('DataElementConcept', backref='bc', lazy='dynamic', cascade='all, delete-orphan') - specializations = db.relationship('DatasetSpecialization', backref='bc', lazy='dynamic', cascade='all, delete-orphan') + children = db.relationship("BiomedicalConcept", backref=db.backref("parent", remote_side="BiomedicalConcept.bc_id"), lazy="dynamic") + decs = db.relationship("DataElementConcept", backref="bc", lazy="dynamic", cascade="all, delete-orphan") + specializations = db.relationship("DatasetSpecialization", backref="bc", lazy="dynamic", cascade="all, delete-orphan") def to_dict(self): return { - 'bc_id': self.bc_id, - 'short_name': self.short_name, - 'definition': self.definition, - 'ncit_code': self.ncit_code, - 'parent_bc_id': self.parent_bc_id, - 'bc_categories': self.bc_categories, - 'synonyms': self.synonyms, - 'result_scales': self.result_scales, - 'system': self.system, - 'system_name': self.system_name, - 'code': self.code, - 'package_date': self.package_date, - 'status': self.status, - 'submitter': self.submitter, + "bc_id": self.bc_id, + "short_name": self.short_name, + "definition": self.definition, + "ncit_code": self.ncit_code, + "parent_bc_id": self.parent_bc_id, + "bc_categories": self.bc_categories, + "synonyms": self.synonyms, + "result_scales": self.result_scales, + "system": self.system, + "system_name": self.system_name, + "code": self.code, + "package_date": self.package_date, + "status": self.status, + "submitter": self.submitter, } class DataElementConcept(db.Model): - __tablename__ = 'data_element_concepts' + __tablename__ = "data_element_concepts" id = db.Column(db.Integer, primary_key=True) dec_id = db.Column(db.String(50), nullable=False) - bc_id = db.Column(db.String(50), db.ForeignKey('biomedical_concepts.bc_id'), nullable=False) + bc_id = db.Column(db.String(50), db.ForeignKey("biomedical_concepts.bc_id"), nullable=False) ncit_dec_code = db.Column(db.String(50)) dec_label = db.Column(db.String(255)) data_type = db.Column(db.String(50)) # string, decimal, boolean, datetime diff --git a/models/governance.py b/models/governance.py index 6141ad3..85549cb 100644 --- a/models/governance.py +++ b/models/governance.py @@ -3,13 +3,13 @@ class GovernanceRecord(db.Model): - __tablename__ = 'governance_records' + __tablename__ = "governance_records" id = db.Column(db.Integer, primary_key=True) - bc_id = db.Column(db.String(50), db.ForeignKey('biomedical_concepts.bc_id'), nullable=False) + bc_id = db.Column(db.String(50), db.ForeignKey("biomedical_concepts.bc_id"), nullable=False) stage = db.Column(db.Integer, default=0) # 0=Scoping, 1=Development, 2=Draft, 3a=Internal Review, 3b=Public Review, 3c=Publication, 4=Maintenance action = db.Column(db.String(100)) # submitted, advanced, rejected, approved, published actor = db.Column(db.String(100)) comment = db.Column(db.Text) created_at = db.Column(db.DateTime, default=datetime.utcnow) - bc = db.relationship('BiomedicalConcept', backref=db.backref('governance_records', lazy='dynamic')) + bc = db.relationship("BiomedicalConcept", backref=db.backref("governance_records", lazy="dynamic")) diff --git a/models/ingestion.py b/models/ingestion.py index 5bf0861..17d31fa 100644 --- a/models/ingestion.py +++ b/models/ingestion.py @@ -4,17 +4,17 @@ class IngestionRecord(db.Model): - __tablename__ = 'ingestion_records' + __tablename__ = "ingestion_records" id = db.Column(db.Integer, primary_key=True) session_key = db.Column(db.String(64), index=True) source_file = db.Column(db.String(255)) source_sheet = db.Column(db.String(100)) - _mapped = db.Column('mapped', db.Text) - _confidences = db.Column('confidences', db.Text) - _errors = db.Column('errors', db.Text) - _decs = db.Column('decs', db.Text) + _mapped = db.Column("mapped", db.Text) + _confidences = db.Column("confidences", db.Text) + _errors = db.Column("errors", db.Text) + _decs = db.Column("decs", db.Text) duplicate = db.Column(db.Boolean, default=False) - status = db.Column(db.String(20), default='pending') # pending / approved / rejected + status = db.Column(db.String(20), default="pending") # pending / approved / rejected created_at = db.Column(db.DateTime, default=datetime.utcnow) @property diff --git a/models/specialization.py b/models/specialization.py index e6978fc..f07f17e 100644 --- a/models/specialization.py +++ b/models/specialization.py @@ -3,12 +3,12 @@ class DatasetSpecialization(db.Model): - __tablename__ = 'dataset_specializations' + __tablename__ = "dataset_specializations" vlm_group_id = db.Column(db.String(100), primary_key=True) - bc_id = db.Column(db.String(50), db.ForeignKey('biomedical_concepts.bc_id'), nullable=False) + bc_id = db.Column(db.String(50), db.ForeignKey("biomedical_concepts.bc_id"), nullable=False) domain = db.Column(db.String(20)) # SDTM or CDASH short_name = db.Column(db.String(255)) - _variables = db.Column('variables', db.Text, default='[]') + _variables = db.Column("variables", db.Text, default="[]") created_at = db.Column(db.DateTime) @property diff --git a/routes/audit.py b/routes/audit.py index eb511fc..07bb0cf 100644 --- a/routes/audit.py +++ b/routes/audit.py @@ -1,35 +1,33 @@ from flask import Blueprint, render_template, request from models.audit import AuditLog -bp = Blueprint('audit', __name__) +bp = Blueprint("audit", __name__) -@bp.route('/') +@bp.route("/") def index(): - page = request.args.get('page', 1, type=int) - entity_type = request.args.get('entity_type', '') - action = request.args.get('action', '') - actor = request.args.get('actor', '') - date_from = request.args.get('date_from', '') - date_to = request.args.get('date_to', '') + page = request.args.get("page", 1, type=int) + entity_type = request.args.get("entity_type", "") + action = request.args.get("action", "") + actor = request.args.get("actor", "") + date_from = request.args.get("date_from", "") + date_to = request.args.get("date_to", "") query = AuditLog.query if entity_type: query = query.filter_by(entity_type=entity_type) if action: - query = query.filter(AuditLog.action.ilike(f'%{action}%')) + query = query.filter(AuditLog.action.ilike(f"%{action}%")) if actor: - query = query.filter(AuditLog.actor.ilike(f'%{actor}%')) + query = query.filter(AuditLog.actor.ilike(f"%{actor}%")) if date_from: query = query.filter(AuditLog.timestamp >= date_from) if date_to: query = query.filter(AuditLog.timestamp <= date_to) - logs = query.order_by(AuditLog.timestamp.desc()).paginate( - page=page, per_page=50, error_out=False - ) + logs = query.order_by(AuditLog.timestamp.desc()).paginate(page=page, per_page=50, error_out=False) return render_template( - 'audit.html', + "audit.html", audit_logs=logs, pagination=logs, entity_type=entity_type, @@ -37,5 +35,5 @@ def index(): actor=actor, date_from=date_from, date_to=date_to, - page_title='Audit Trail', + page_title="Audit Trail", ) diff --git a/routes/bc.py b/routes/bc.py index a232a9a..2b604a9 100644 --- a/routes/bc.py +++ b/routes/bc.py @@ -6,98 +6,87 @@ from services.cdisc_api import CDISCApiClient from datetime import datetime -bp = Blueprint('bc', __name__) +bp = Blueprint("bc", __name__) -@bp.route('/') +@bp.route("/") def index(): - q = request.args.get('q', '') - status = request.args.get('status', '') - page = request.args.get('page', 1, type=int) + q = request.args.get("q", "") + status = request.args.get("status", "") + page = request.args.get("page", 1, type=int) query = BiomedicalConcept.query if q: - query = query.filter( - BiomedicalConcept.short_name.ilike(f'%{q}%') | - BiomedicalConcept.bc_id.ilike(f'%{q}%') | - BiomedicalConcept.ncit_code.ilike(f'%{q}%') - ) + query = query.filter(BiomedicalConcept.short_name.ilike(f"%{q}%") | BiomedicalConcept.bc_id.ilike(f"%{q}%") | BiomedicalConcept.ncit_code.ilike(f"%{q}%")) if status: query = query.filter_by(status=status) - bcs = query.order_by(BiomedicalConcept.updated_at.desc()).paginate( - page=page, per_page=25, error_out=False - ) + bcs = query.order_by(BiomedicalConcept.updated_at.desc()).paginate(page=page, per_page=25, error_out=False) return render_template( - 'bc_list.html', + "bc_list.html", bcs=bcs, q=q, status=status, - page_title='Biomedical Concepts', + page_title="Biomedical Concepts", ) -@bp.route('/new') +@bp.route("/new") def new_bc(): bc = BiomedicalConcept() return render_template( - 'bc_detail.html', + "bc_detail.html", bc=bc, decs=[], is_new=True, - page_title='New Biomedical Concept', + page_title="New Biomedical Concept", ) -@bp.route('/export') +@bp.route("/export") def export(): - fmt = request.args.get('format', 'json') + fmt = request.args.get("format", "json") bcs = [bc.to_dict() for bc in BiomedicalConcept.query.all()] - if fmt == 'xlsx': + if fmt == "xlsx": buf = export_xlsx(bcs) return Response( buf, - mimetype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - headers={'Content-Disposition': 'attachment; filename=bcs.xlsx'}, + mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + headers={"Content-Disposition": "attachment; filename=bcs.xlsx"}, ) - elif fmt == 'odm': + elif fmt == "odm": xml = export_odm_xml(bcs) return Response( xml, - mimetype='application/xml', - headers={'Content-Disposition': 'attachment; filename=bcs_odm.xml'}, + mimetype="application/xml", + headers={"Content-Disposition": "attachment; filename=bcs_odm.xml"}, ) else: return Response( export_json(bcs), - mimetype='application/json', - headers={'Content-Disposition': 'attachment; filename=bcs.json'}, + mimetype="application/json", + headers={"Content-Disposition": "attachment; filename=bcs.json"}, ) -@bp.route('/library/') +@bp.route("/library/") def library_detail(concept_id): client = CDISCApiClient() bc = client.get_bc(concept_id) - if 'error' in bc: - flash(f'Could not load concept {concept_id}: {bc["error"]}', 'danger') - return redirect(url_for('dashboard.index')) + if "error" in bc: + flash(f'Could not load concept {concept_id}: {bc["error"]}', "danger") + return redirect(url_for("dashboard.index")) return render_template( - 'library_bc_detail.html', + "library_bc_detail.html", bc=bc, - page_title=bc.get('shortName') or bc.get('name') or concept_id, + page_title=bc.get("shortName") or bc.get("name") or concept_id, ) -@bp.route('/') +@bp.route("/") def detail(bc_id): bc = BiomedicalConcept.query.get_or_404(bc_id) - decs = ( - DataElementConcept.query - .filter_by(bc_id=bc_id) - .order_by(DataElementConcept.sort_order) - .all() - ) + decs = DataElementConcept.query.filter_by(bc_id=bc_id).order_by(DataElementConcept.sort_order).all() return render_template( - 'bc_detail.html', + "bc_detail.html", bc=bc, decs=decs, is_new=False, @@ -105,112 +94,112 @@ def detail(bc_id): ) -@bp.route('/', methods=['POST']) +@bp.route("/", methods=["POST"]) def create(): - bc_id = request.form.get('bc_id', '').strip() + bc_id = request.form.get("bc_id", "").strip() if not bc_id: - flash('BC ID is required', 'danger') - return redirect(url_for('bc.new_bc')) + flash("BC ID is required", "danger") + return redirect(url_for("bc.new_bc")) if BiomedicalConcept.query.get(bc_id): - flash(f'BC {bc_id} already exists', 'danger') - return redirect(url_for('bc.new_bc')) + flash(f"BC {bc_id} already exists", "danger") + return redirect(url_for("bc.new_bc")) bc = BiomedicalConcept( bc_id=bc_id, - short_name=request.form.get('short_name', ''), - definition=request.form.get('definition', ''), - ncit_code=request.form.get('ncit_code', ''), - parent_bc_id=request.form.get('parent_bc_id') or None, - bc_categories=request.form.get('bc_categories', ''), - synonyms=request.form.get('synonyms', ''), - result_scales=request.form.get('result_scales', ''), - system=request.form.get('system', ''), - system_name=request.form.get('system_name', ''), - code=request.form.get('code', ''), - package_date=request.form.get('package_date', ''), - status='provisional', - submitter=request.form.get('submitter', 'unknown'), + short_name=request.form.get("short_name", ""), + definition=request.form.get("definition", ""), + ncit_code=request.form.get("ncit_code", ""), + parent_bc_id=request.form.get("parent_bc_id") or None, + bc_categories=request.form.get("bc_categories", ""), + synonyms=request.form.get("synonyms", ""), + result_scales=request.form.get("result_scales", ""), + system=request.form.get("system", ""), + system_name=request.form.get("system_name", ""), + code=request.form.get("code", ""), + package_date=request.form.get("package_date", ""), + status="provisional", + submitter=request.form.get("submitter", "unknown"), ) db.session.add(bc) log = AuditLog( - entity_type='BiomedicalConcept', + entity_type="BiomedicalConcept", entity_id=bc_id, - action='created', + action="created", actor=bc.submitter, after_state=bc.to_dict(), ) db.session.add(log) db.session.commit() _save_decs(bc_id, request.form) - flash(f'BC {bc_id} created', 'success') - return redirect(url_for('bc.detail', bc_id=bc_id)) + flash(f"BC {bc_id} created", "success") + return redirect(url_for("bc.detail", bc_id=bc_id)) -@bp.route('//edit', methods=['POST']) +@bp.route("//edit", methods=["POST"]) def edit(bc_id): bc = BiomedicalConcept.query.get_or_404(bc_id) before = bc.to_dict() - bc.short_name = request.form.get('short_name', bc.short_name) - bc.definition = request.form.get('definition', bc.definition) - bc.ncit_code = request.form.get('ncit_code', bc.ncit_code) - bc.parent_bc_id = request.form.get('parent_bc_id') or bc.parent_bc_id - bc.bc_categories = request.form.get('bc_categories', bc.bc_categories) - bc.synonyms = request.form.get('synonyms', bc.synonyms) - bc.result_scales = request.form.get('result_scales', bc.result_scales) - bc.system = request.form.get('system', bc.system) - bc.system_name = request.form.get('system_name', bc.system_name) - bc.code = request.form.get('code', bc.code) - bc.package_date = request.form.get('package_date', bc.package_date) + bc.short_name = request.form.get("short_name", bc.short_name) + bc.definition = request.form.get("definition", bc.definition) + bc.ncit_code = request.form.get("ncit_code", bc.ncit_code) + bc.parent_bc_id = request.form.get("parent_bc_id") or bc.parent_bc_id + bc.bc_categories = request.form.get("bc_categories", bc.bc_categories) + bc.synonyms = request.form.get("synonyms", bc.synonyms) + bc.result_scales = request.form.get("result_scales", bc.result_scales) + bc.system = request.form.get("system", bc.system) + bc.system_name = request.form.get("system_name", bc.system_name) + bc.code = request.form.get("code", bc.code) + bc.package_date = request.form.get("package_date", bc.package_date) bc.updated_at = datetime.utcnow() log = AuditLog( - entity_type='BiomedicalConcept', + entity_type="BiomedicalConcept", entity_id=bc_id, - action='updated', - actor='user', + action="updated", + actor="user", before_state=before, after_state=bc.to_dict(), ) db.session.add(log) db.session.commit() _save_decs(bc_id, request.form) - flash(f'BC {bc_id} updated', 'success') - return redirect(url_for('bc.detail', bc_id=bc_id)) + flash(f"BC {bc_id} updated", "success") + return redirect(url_for("bc.detail", bc_id=bc_id)) -@bp.route('//submit', methods=['POST']) +@bp.route("//submit", methods=["POST"]) def submit_for_review(bc_id): bc = BiomedicalConcept.query.get_or_404(bc_id) before = bc.to_dict() - bc.status = 'sme_review' + bc.status = "sme_review" bc.updated_at = datetime.utcnow() log = AuditLog( - entity_type='BiomedicalConcept', + entity_type="BiomedicalConcept", entity_id=bc_id, - action='submitted_for_review', - actor='user', + action="submitted_for_review", + actor="user", before_state=before, after_state=bc.to_dict(), ) db.session.add(log) db.session.commit() - flash(f'BC {bc_id} submitted for SME review', 'success') - return redirect(url_for('bc.detail', bc_id=bc_id)) + flash(f"BC {bc_id} submitted for SME review", "success") + return redirect(url_for("bc.detail", bc_id=bc_id)) -@bp.route('//delete', methods=['POST']) +@bp.route("//delete", methods=["POST"]) def delete(bc_id): bc = BiomedicalConcept.query.get_or_404(bc_id) log = AuditLog( - entity_type='BiomedicalConcept', + entity_type="BiomedicalConcept", entity_id=bc_id, - action='deleted', - actor='user', + action="deleted", + actor="user", before_state=bc.to_dict(), ) db.session.add(log) db.session.delete(bc) db.session.commit() - flash(f'BC {bc_id} deleted', 'success') - return redirect(url_for('bc.index')) + flash(f"BC {bc_id} deleted", "success") + return redirect(url_for("bc.index")) def _save_decs(bc_id, form): @@ -220,11 +209,11 @@ def _save_decs(bc_id, form): dec_example_set[], dec_id[], dec_ncit_code[]. Any existing DECs for the BC are replaced on every call so that deletions are honoured. """ - labels = form.getlist('dec_label[]') - dtypes = form.getlist('dec_data_type[]') - examples = form.getlist('dec_example_set[]') - dec_ids = form.getlist('dec_id[]') - ncit_codes = form.getlist('dec_ncit_code[]') + labels = form.getlist("dec_label[]") + dtypes = form.getlist("dec_data_type[]") + examples = form.getlist("dec_example_set[]") + dec_ids = form.getlist("dec_id[]") + ncit_codes = form.getlist("dec_ncit_code[]") if not labels: return DataElementConcept.query.filter_by(bc_id=bc_id).delete() @@ -232,12 +221,12 @@ def _save_decs(bc_id, form): if not label.strip(): continue dec = DataElementConcept( - dec_id=dec_ids[i] if i < len(dec_ids) and dec_ids[i] else f'{bc_id}.DEC.{i + 1}', + dec_id=dec_ids[i] if i < len(dec_ids) and dec_ids[i] else f"{bc_id}.DEC.{i + 1}", bc_id=bc_id, - ncit_dec_code=ncit_codes[i] if i < len(ncit_codes) else '', + ncit_dec_code=ncit_codes[i] if i < len(ncit_codes) else "", dec_label=label.strip(), - data_type=dtypes[i] if i < len(dtypes) else 'string', - example_set=examples[i] if i < len(examples) else '', + data_type=dtypes[i] if i < len(dtypes) else "string", + example_set=examples[i] if i < len(examples) else "", sort_order=i, ) db.session.add(dec) diff --git a/routes/dashboard.py b/routes/dashboard.py index dc81c33..71a579c 100644 --- a/routes/dashboard.py +++ b/routes/dashboard.py @@ -14,30 +14,17 @@ def index(): api_bcs = client.get_biomedical_concepts() api_specs = client.get_dataset_specializations() - api_bc_error = ( - api_bcs[0].get("error") if api_bcs and "error" in api_bcs[0] else None - ) - api_spec_error = ( - api_specs[0].get("error") if api_specs and "error" in api_specs[0] else None - ) + api_bc_error = api_bcs[0].get("error") if api_bcs and "error" in api_bcs[0] else None + api_spec_error = api_specs[0].get("error") if api_specs and "error" in api_specs[0] else None api_bc_count = len(api_bcs) if not api_bc_error else 0 api_spec_count = len(api_specs) if not api_spec_error else 0 # --- Local DB stats --- local_total_bcs = BiomedicalConcept.query.count() - local_pending = BiomedicalConcept.query.filter( - BiomedicalConcept.status.in_(["provisional", "sme_review", "cdisc_approval"]) - ).count() - recent_additions = BiomedicalConcept.query.filter( - BiomedicalConcept.created_at >= datetime.utcnow() - timedelta(days=7) - ).count() + local_pending = BiomedicalConcept.query.filter(BiomedicalConcept.status.in_(["provisional", "sme_review", "cdisc_approval"])).count() + recent_additions = BiomedicalConcept.query.filter(BiomedicalConcept.created_at >= datetime.utcnow() - timedelta(days=7)).count() - governance_items = ( - BiomedicalConcept.query.filter(BiomedicalConcept.status != "published") - .order_by(BiomedicalConcept.updated_at.desc()) - .limit(10) - .all() - ) + governance_items = BiomedicalConcept.query.filter(BiomedicalConcept.status != "published").order_by(BiomedicalConcept.updated_at.desc()).limit(10).all() recent_audits = AuditLog.query.order_by(AuditLog.timestamp.desc()).limit(10).all() stats = { @@ -57,11 +44,7 @@ def index(): api_spec_error=api_spec_error, api_bc_count=api_bc_count, api_spec_count=api_spec_count, - recent_submissions=BiomedicalConcept.query.order_by( - BiomedicalConcept.created_at.desc() - ) - .limit(10) - .all(), + recent_submissions=BiomedicalConcept.query.order_by(BiomedicalConcept.created_at.desc()).limit(10).all(), governance_items=governance_items, recent_audits=recent_audits, page_title="Dashboard", diff --git a/routes/governance.py b/routes/governance.py index 3732f95..1e94286 100644 --- a/routes/governance.py +++ b/routes/governance.py @@ -5,30 +5,25 @@ from extensions import db from datetime import datetime -bp = Blueprint('governance', __name__) +bp = Blueprint("governance", __name__) -STATUS_ORDER = ['provisional', 'sme_review', 'cdisc_approval', 'published'] +STATUS_ORDER = ["provisional", "sme_review", "cdisc_approval", "published"] -@bp.route('/board') +@bp.route("/board") def board(): bcs_by_status = {} for status in STATUS_ORDER: - bcs_by_status[status] = ( - BiomedicalConcept.query - .filter_by(status=status) - .order_by(BiomedicalConcept.updated_at.desc()) - .all() - ) + bcs_by_status[status] = BiomedicalConcept.query.filter_by(status=status).order_by(BiomedicalConcept.updated_at.desc()).all() return render_template( - 'governance.html', + "governance.html", columns=bcs_by_status, status_order=STATUS_ORDER, - page_title='Governance Board', + page_title="Governance Board", ) -@bp.route('/advance/', methods=['POST']) +@bp.route("/advance/", methods=["POST"]) def advance(bc_id): bc = BiomedicalConcept.query.get_or_404(bc_id) before_status = bc.status @@ -39,54 +34,54 @@ def advance(bc_id): rec = GovernanceRecord( bc_id=bc_id, stage=current_idx + 1, - action='advanced', - actor='user', - comment=request.form.get('comment', ''), + action="advanced", + actor="user", + comment=request.form.get("comment", ""), ) log = AuditLog( - entity_type='BiomedicalConcept', + entity_type="BiomedicalConcept", entity_id=bc_id, - action='status_changed', - actor='user', - before_state={'status': before_status}, - after_state={'status': bc.status}, + action="status_changed", + actor="user", + before_state={"status": before_status}, + after_state={"status": bc.status}, ) db.session.add(rec) db.session.add(log) db.session.commit() - if request.headers.get('X-Requested-With') == 'XMLHttpRequest': - return jsonify({'status': bc.status, 'bc_id': bc_id}) - flash(f'{bc.short_name} advanced to {bc.status}', 'success') + if request.headers.get("X-Requested-With") == "XMLHttpRequest": + return jsonify({"status": bc.status, "bc_id": bc_id}) + flash(f"{bc.short_name} advanced to {bc.status}", "success") else: - flash(f'{bc.short_name} is already published', 'info') - return redirect(url_for('governance.board')) + flash(f"{bc.short_name} is already published", "info") + return redirect(url_for("governance.board")) -@bp.route('/reject/', methods=['POST']) +@bp.route("/reject/", methods=["POST"]) def reject_bc(bc_id): bc = BiomedicalConcept.query.get_or_404(bc_id) before_status = bc.status - bc.status = 'provisional' + bc.status = "provisional" bc.updated_at = datetime.utcnow() rec = GovernanceRecord( bc_id=bc_id, stage=0, - action='rejected', - actor='user', - comment=request.form.get('comment', ''), + action="rejected", + actor="user", + comment=request.form.get("comment", ""), ) log = AuditLog( - entity_type='BiomedicalConcept', + entity_type="BiomedicalConcept", entity_id=bc_id, - action='rejected', - actor='user', - before_state={'status': before_status}, - after_state={'status': 'provisional'}, + action="rejected", + actor="user", + before_state={"status": before_status}, + after_state={"status": "provisional"}, ) db.session.add(rec) db.session.add(log) db.session.commit() - if request.headers.get('X-Requested-With') == 'XMLHttpRequest': - return jsonify({'status': 'provisional', 'bc_id': bc_id}) - flash(f'{bc.short_name} rejected and returned to provisional', 'warning') - return redirect(url_for('governance.board')) + if request.headers.get("X-Requested-With") == "XMLHttpRequest": + return jsonify({"status": "provisional", "bc_id": bc_id}) + flash(f"{bc.short_name} rejected and returned to provisional", "warning") + return redirect(url_for("governance.board")) diff --git a/routes/ingestion.py b/routes/ingestion.py index eeb735c..bb2e376 100644 --- a/routes/ingestion.py +++ b/routes/ingestion.py @@ -1,7 +1,12 @@ import uuid from flask import ( - Blueprint, render_template, request, - redirect, url_for, flash, session, + Blueprint, + render_template, + request, + redirect, + url_for, + flash, + session, ) from models.bc import BiomedicalConcept, DataElementConcept from models.ingestion import IngestionRecord @@ -9,86 +14,78 @@ from extensions import db from services.ingestion import parse_xlsx, parse_csv, parse_json, deduplicate -bp = Blueprint('ingestion', __name__) +bp = Blueprint("ingestion", __name__) -ALLOWED_EXTENSIONS = {'xlsx', 'csv', 'json'} +ALLOWED_EXTENSIONS = {"xlsx", "csv", "json"} def _allowed_file(filename): - ext = filename.rsplit('.', 1)[1].lower() if '.' in filename else '' + ext = filename.rsplit(".", 1)[1].lower() if "." in filename else "" return ext in ALLOWED_EXTENSIONS def _get_session_key(): - if 'ingestion_key' not in session: - session['ingestion_key'] = uuid.uuid4().hex - return session['ingestion_key'] + if "ingestion_key" not in session: + session["ingestion_key"] = uuid.uuid4().hex + return session["ingestion_key"] def _bc_from_mapped(bc_id, mapped): return BiomedicalConcept( bc_id=bc_id, - short_name=mapped.get('short_name', ''), - definition=mapped.get('definition', ''), - ncit_code=mapped.get('ncit_code', ''), - parent_bc_id=mapped.get('parent_bc_id') or None, - bc_categories=mapped.get('bc_categories', ''), - synonyms=mapped.get('synonyms', ''), - result_scales=mapped.get('result_scales', ''), - system=mapped.get('system', ''), - system_name=mapped.get('system_name', ''), - code=mapped.get('code', ''), - package_date=mapped.get('package_date', ''), - status='provisional', - source='ingestion', + short_name=mapped.get("short_name", ""), + definition=mapped.get("definition", ""), + ncit_code=mapped.get("ncit_code", ""), + parent_bc_id=mapped.get("parent_bc_id") or None, + bc_categories=mapped.get("bc_categories", ""), + synonyms=mapped.get("synonyms", ""), + result_scales=mapped.get("result_scales", ""), + system=mapped.get("system", ""), + system_name=mapped.get("system_name", ""), + code=mapped.get("code", ""), + package_date=mapped.get("package_date", ""), + status="provisional", + source="ingestion", ) def _create_decs(bc_id, decs): for i, d in enumerate(decs): dec = DataElementConcept( - dec_id=d.get('dec_id') or f'{bc_id}.DEC.{i + 1}', + dec_id=d.get("dec_id") or f"{bc_id}.DEC.{i + 1}", bc_id=bc_id, - ncit_dec_code=d.get('ncit_dec_code', ''), - dec_label=d.get('dec_label', ''), - data_type=d.get('data_type', 'string'), - example_set=d.get('example_set', ''), + ncit_dec_code=d.get("ncit_dec_code", ""), + dec_label=d.get("dec_label", ""), + data_type=d.get("data_type", "string"), + example_set=d.get("example_set", ""), sort_order=i, ) db.session.add(dec) -@bp.route('/') +@bp.route("/") def index(): - key = session.get('ingestion_key') - queue = ( - IngestionRecord.query - .filter_by(session_key=key, status='pending') - .all() - ) if key else [] - return render_template('ingestion.html', queue=queue, page_title='Ingestion') + key = session.get("ingestion_key") + queue = (IngestionRecord.query.filter_by(session_key=key, status="pending").all()) if key else [] + return render_template("ingestion.html", queue=queue, page_title="Ingestion") -@bp.route('/upload', methods=['POST']) +@bp.route("/upload", methods=["POST"]) def upload(): - if 'file' not in request.files: - flash('No file selected', 'danger') - return redirect(url_for('ingestion.index')) - f = request.files['file'] + if "file" not in request.files: + flash("No file selected", "danger") + return redirect(url_for("ingestion.index")) + f = request.files["file"] if not f.filename or not _allowed_file(f.filename): - flash('Please upload an XLSX, CSV, or JSON file', 'danger') - return redirect(url_for('ingestion.index')) + flash("Please upload an XLSX, CSV, or JSON file", "danger") + return redirect(url_for("ingestion.index")) - ext = f.filename.rsplit('.', 1)[1].lower() - existing_ids = { - bc.bc_id - for bc in BiomedicalConcept.query - .with_entities(BiomedicalConcept.bc_id).all() - } + ext = f.filename.rsplit(".", 1)[1].lower() + existing_ids = {bc.bc_id for bc in BiomedicalConcept.query.with_entities(BiomedicalConcept.bc_id).all()} - if ext == 'xlsx': + if ext == "xlsx": records = parse_xlsx(f) - elif ext == 'csv': + elif ext == "csv": records = parse_csv(f) else: records = parse_json(f) @@ -96,87 +93,80 @@ def upload(): records = deduplicate(records, existing_ids) key = _get_session_key() - IngestionRecord.query.filter_by(session_key=key, status='pending').delete() + IngestionRecord.query.filter_by(session_key=key, status="pending").delete() for rec in records: ir = IngestionRecord( session_key=key, source_file=f.filename, - source_sheet=rec.get('source_sheet', ''), - duplicate=rec.get('duplicate', False), + source_sheet=rec.get("source_sheet", ""), + duplicate=rec.get("duplicate", False), ) - ir.mapped = rec.get('mapped', {}) - ir.confidences = rec.get('confidences', {}) - ir.errors = rec.get('errors', []) - ir.decs = rec.get('decs', []) + ir.mapped = rec.get("mapped", {}) + ir.confidences = rec.get("confidences", {}) + ir.errors = rec.get("errors", []) + ir.decs = rec.get("decs", []) db.session.add(ir) db.session.commit() - flash(f'Parsed {len(records)} records from {f.filename}', 'success') - return redirect(url_for('ingestion.index')) + flash(f"Parsed {len(records)} records from {f.filename}", "success") + return redirect(url_for("ingestion.index")) -@bp.route('/approve/', methods=['POST']) +@bp.route("/approve/", methods=["POST"]) def approve(record_id): ir = IngestionRecord.query.get_or_404(record_id) mapped = ir.mapped - bc_id = mapped.get('bc_id') or mapped.get('ncit_code', f'IMPORT_{record_id}') + bc_id = mapped.get("bc_id") or mapped.get("ncit_code", f"IMPORT_{record_id}") if not BiomedicalConcept.query.get(bc_id): bc = _bc_from_mapped(bc_id, mapped) db.session.add(bc) _create_decs(bc_id, ir.decs) log = AuditLog( - entity_type='BiomedicalConcept', + entity_type="BiomedicalConcept", entity_id=bc_id, - action='created_via_ingestion', - actor='system', + action="created_via_ingestion", + actor="system", after_state=mapped, ) db.session.add(log) - flash(f'BC {bc_id} added to library', 'success') + flash(f"BC {bc_id} added to library", "success") else: - flash(f'BC {bc_id} already exists', 'warning') - ir.status = 'approved' + flash(f"BC {bc_id} already exists", "warning") + ir.status = "approved" db.session.commit() - return redirect(url_for('ingestion.index')) + return redirect(url_for("ingestion.index")) -@bp.route('/reject/', methods=['POST']) +@bp.route("/reject/", methods=["POST"]) def reject(record_id): ir = IngestionRecord.query.get_or_404(record_id) - ir.status = 'rejected' + ir.status = "rejected" db.session.commit() - return redirect(url_for('ingestion.index')) + return redirect(url_for("ingestion.index")) -@bp.route('/approve_all', methods=['POST']) +@bp.route("/approve_all", methods=["POST"]) def approve_all(): - key = session.get('ingestion_key') + key = session.get("ingestion_key") if not key: - return redirect(url_for('ingestion.index')) - pending = ( - IngestionRecord.query - .filter_by(session_key=key, status='pending') - .all() - ) + return redirect(url_for("ingestion.index")) + pending = IngestionRecord.query.filter_by(session_key=key, status="pending").all() added = 0 for ir in pending: if ir.errors or ir.duplicate: - ir.status = 'rejected' + ir.status = "rejected" continue mapped = ir.mapped - bc_id = ( - mapped.get('bc_id') - or mapped.get('ncit_code', f'IMPORT_{ir.id}') - ) + bc_id = mapped.get("bc_id") or mapped.get("ncit_code", f"IMPORT_{ir.id}") if not BiomedicalConcept.query.get(bc_id): bc = _bc_from_mapped(bc_id, mapped) db.session.add(bc) _create_decs(bc_id, ir.decs) - ir.status = 'approved' + ir.status = "approved" added += 1 else: - ir.status = 'approved' + ir.status = "approved" db.session.commit() - flash(f'Approved {added} BCs', 'success') - return redirect(url_for('ingestion.index')) + flash(f"Approved {added} BCs", "success") + return redirect(url_for("ingestion.index")) diff --git a/routes/ncit.py b/routes/ncit.py index c663658..aeb04b6 100644 --- a/routes/ncit.py +++ b/routes/ncit.py @@ -3,46 +3,40 @@ from extensions import db from services.ncit_api import NCItApiClient -bp = Blueprint('ncit', __name__) +bp = Blueprint("ncit", __name__) -@bp.route('/') +@bp.route("/") def index(): - return redirect(url_for('ncit.mapping')) + return redirect(url_for("ncit.mapping")) -@bp.route('/mapping') +@bp.route("/mapping") def mapping(): # Surface BCs that have no NCIt code yet — these need manual resolution - unresolved = BiomedicalConcept.query.filter( - (BiomedicalConcept.ncit_code == None) | (BiomedicalConcept.ncit_code == '') - ).limit(50).all() + unresolved = BiomedicalConcept.query.filter((BiomedicalConcept.ncit_code == None) | (BiomedicalConcept.ncit_code == "")).limit(50).all() return render_template( - 'ncit_mapping.html', + "ncit_mapping.html", unresolved=unresolved, results=[], - search_term='', - page_title='NCIt Mapping', + search_term="", + page_title="NCIt Mapping", ) -@bp.route('/search') +@bp.route("/search") def search_ncit(): - term = request.args.get('term', '').strip() - is_ajax = ( - request.headers.get('X-Requested-With') == 'XMLHttpRequest' - or request.args.get('format') == 'json' - or 'application/json' in request.headers.get('Accept', '') - ) + term = request.args.get("term", "").strip() + is_ajax = request.headers.get("X-Requested-With") == "XMLHttpRequest" or request.args.get("format") == "json" or "application/json" in request.headers.get("Accept", "") if not term: if is_ajax: return jsonify([]) return render_template( - 'ncit_mapping.html', + "ncit_mapping.html", results=[], - search_term='', + search_term="", unresolved=[], - page_title='NCIt Mapping', + page_title="NCIt Mapping", ) client = NCItApiClient() @@ -51,27 +45,25 @@ def search_ncit(): if is_ajax: return jsonify(results) - unresolved = BiomedicalConcept.query.filter( - (BiomedicalConcept.ncit_code == None) | (BiomedicalConcept.ncit_code == '') - ).limit(50).all() + unresolved = BiomedicalConcept.query.filter((BiomedicalConcept.ncit_code == None) | (BiomedicalConcept.ncit_code == "")).limit(50).all() return render_template( - 'ncit_mapping.html', + "ncit_mapping.html", results=results, search_term=term, unresolved=unresolved, - page_title='NCIt Mapping', + page_title="NCIt Mapping", ) -@bp.route('/resolve/', methods=['POST']) +@bp.route("/resolve/", methods=["POST"]) def resolve(bc_id): bc = BiomedicalConcept.query.get_or_404(bc_id) - ncit_code = request.form.get('ncit_code', '').strip() + ncit_code = request.form.get("ncit_code", "").strip() if ncit_code: bc.ncit_code = ncit_code # Promote temporary IMPORT_ IDs to their resolved NCIt code - if not bc.bc_id or bc.bc_id.startswith('IMPORT_'): + if not bc.bc_id or bc.bc_id.startswith("IMPORT_"): bc.bc_id = ncit_code db.session.commit() - flash(f'NCIt mapping updated for {bc.short_name}', 'success') - return redirect(url_for('ncit.mapping')) + flash(f"NCIt mapping updated for {bc.short_name}", "success") + return redirect(url_for("ncit.mapping")) diff --git a/routes/specializations.py b/routes/specializations.py index 4122e0f..af90ad7 100644 --- a/routes/specializations.py +++ b/routes/specializations.py @@ -4,84 +4,81 @@ from extensions import db from services.cdisc_api import CDISCApiClient -bp = Blueprint('specializations', __name__) +bp = Blueprint("specializations", __name__) -@bp.route('/') +@bp.route("/") def index(): specs = DatasetSpecialization.query.all() bcs = BiomedicalConcept.query.order_by(BiomedicalConcept.short_name).all() return render_template( - 'specializations.html', + "specializations.html", specs=specs, bcs=bcs, - page_title='Specializations', + page_title="Specializations", ) -@bp.route('/library/') +@bp.route("/library/") def library_detail(spec_path): client = CDISCApiClient() - spec = client.get_specialization('/' + spec_path) - if 'error' in spec: - flash(f'Could not load specialization: {spec["error"]}', 'danger') - return redirect(url_for('dashboard.index')) + spec = client.get_specialization("/" + spec_path) + if "error" in spec: + flash(f'Could not load specialization: {spec["error"]}', "danger") + return redirect(url_for("dashboard.index")) return render_template( - 'library_spec_detail.html', + "library_spec_detail.html", spec=spec, - page_title=spec.get('shortName') or spec.get('datasetSpecializationId') or spec_path.split('/')[-1], + page_title=spec.get("shortName") or spec.get("datasetSpecializationId") or spec_path.split("/")[-1], ) -@bp.route('/') +@bp.route("/") def detail(vlm_group_id): spec = DatasetSpecialization.query.get_or_404(vlm_group_id) specs = DatasetSpecialization.query.all() bcs = BiomedicalConcept.query.order_by(BiomedicalConcept.short_name).all() return render_template( - 'specializations.html', + "specializations.html", specs=specs, bcs=bcs, edit_spec=spec, - page_title='Specializations', + page_title="Specializations", ) -@bp.route('/', methods=['POST']) +@bp.route("/", methods=["POST"]) def create(): - vlm_group_id = request.form.get('vlm_group_id', '').strip() - bc_id = request.form.get('bc_id', '').strip() + vlm_group_id = request.form.get("vlm_group_id", "").strip() + bc_id = request.form.get("bc_id", "").strip() if not vlm_group_id or not bc_id: - flash('VLM Group ID and BC are required', 'danger') - return redirect(url_for('specializations.index')) + flash("VLM Group ID and BC are required", "danger") + return redirect(url_for("specializations.index")) spec = DatasetSpecialization( vlm_group_id=vlm_group_id, bc_id=bc_id, - domain=request.form.get('domain', 'SDTM'), - short_name=request.form.get('short_name', ''), + domain=request.form.get("domain", "SDTM"), + short_name=request.form.get("short_name", ""), ) spec.variables = [] db.session.add(spec) db.session.commit() - flash(f'Specialization {vlm_group_id} created', 'success') - return redirect(url_for('specializations.index')) + flash(f"Specialization {vlm_group_id} created", "success") + return redirect(url_for("specializations.index")) -@bp.route('/generate/', methods=['POST']) +@bp.route("/generate/", methods=["POST"]) def generate(bc_id): """Generate a specialization from DEC templates for a BC.""" bc = BiomedicalConcept.query.get_or_404(bc_id) decs = DataElementConcept.query.filter_by(bc_id=bc_id).all() - domain = request.form.get('domain', 'SDTM') - vlm_group_id = f'{bc_id}.{domain}' + domain = request.form.get("domain", "SDTM") + vlm_group_id = f"{bc_id}.{domain}" existing = DatasetSpecialization.query.get(vlm_group_id) if existing: - flash(f'Specialization {vlm_group_id} already exists', 'warning') - return redirect(url_for('specializations.index')) - variables = [ - {'name': d.dec_label, 'data_type': d.data_type, 'required': d.required} - for d in decs - ] + flash(f"Specialization {vlm_group_id} already exists", "warning") + return redirect(url_for("specializations.index")) + variables = [{"name": d.dec_label, "data_type": d.data_type, "required": d.required} for d in decs] spec = DatasetSpecialization( vlm_group_id=vlm_group_id, bc_id=bc_id, @@ -91,5 +88,5 @@ def generate(bc_id): spec.variables = variables db.session.add(spec) db.session.commit() - flash(f'Specialization {vlm_group_id} generated', 'success') - return redirect(url_for('specializations.index')) + flash(f"Specialization {vlm_group_id} generated", "success") + return redirect(url_for("specializations.index")) diff --git a/services/export.py b/services/export.py index fbb02f0..cee7036 100644 --- a/services/export.py +++ b/services/export.py @@ -15,9 +15,19 @@ BC_EXPORT_FIELDS = [ - 'bc_id', 'short_name', 'definition', 'ncit_code', 'parent_bc_id', - 'bc_categories', 'synonyms', 'result_scales', 'system', 'system_name', - 'code', 'package_date', 'status', + "bc_id", + "short_name", + "definition", + "ncit_code", + "parent_bc_id", + "bc_categories", + "synonyms", + "result_scales", + "system", + "system_name", + "code", + "package_date", + "status", ] @@ -32,21 +42,21 @@ def export_xlsx(bc_list): raise ImportError("openpyxl is required for XLSX export") wb = openpyxl.Workbook() ws = wb.active - ws.title = 'Biomedical Concepts' + ws.title = "Biomedical Concepts" header_font = Font(bold=True) - header_fill = PatternFill('solid', fgColor='003366') - header_font_white = Font(bold=True, color='FFFFFF') + header_fill = PatternFill("solid", fgColor="003366") + header_font_white = Font(bold=True, color="FFFFFF") for col_idx, field in enumerate(BC_EXPORT_FIELDS, start=1): - cell = ws.cell(row=1, column=col_idx, value=field.replace('_', ' ').title()) + cell = ws.cell(row=1, column=col_idx, value=field.replace("_", " ").title()) cell.font = header_font_white cell.fill = header_fill - cell.alignment = Alignment(horizontal='center') + cell.alignment = Alignment(horizontal="center") for row_idx, bc in enumerate(bc_list, start=2): for col_idx, field in enumerate(BC_EXPORT_FIELDS, start=1): - ws.cell(row=row_idx, column=col_idx, value=bc.get(field, '')) + ws.cell(row=row_idx, column=col_idx, value=bc.get(field, "")) buf = io.BytesIO() wb.save(buf) @@ -57,29 +67,40 @@ def export_xlsx(bc_list): def export_odm_xml(bc_list): """Export BCs as ODM-XML string.""" if etree is None: - return '' - - root = etree.Element('ODM', attrib={ - 'xmlns': 'http://www.cdisc.org/ns/odm/v1.3', - 'FileType': 'Snapshot', - 'FileOID': f'CDISC.BC.Export.{datetime.utcnow().strftime("%Y%m%d%H%M%S")}', - 'CreationDateTime': datetime.utcnow().isoformat(), - }) + return "" + + root = etree.Element( + "ODM", + attrib={ + "xmlns": "http://www.cdisc.org/ns/odm/v1.3", + "FileType": "Snapshot", + "FileOID": f'CDISC.BC.Export.{datetime.utcnow().strftime("%Y%m%d%H%M%S")}', + "CreationDateTime": datetime.utcnow().isoformat(), + }, + ) for bc in bc_list: - item_def = etree.SubElement(root, 'ItemDef', attrib={ - 'OID': bc.get('bc_id', 'UNKNOWN'), - 'Name': bc.get('short_name', ''), - 'DataType': 'text', - }) - desc = etree.SubElement(item_def, 'Description') - XML_NS = 'http://www.w3.org/XML/1998/namespace' - translated = etree.SubElement(desc, 'TranslatedText', attrib={f'{{{XML_NS}}}lang': 'en'}) - translated.text = bc.get('definition', '') - if bc.get('ncit_code'): - etree.SubElement(item_def, 'Alias', attrib={ - 'Context': 'nci:ExtCodeID', - 'Name': bc.get('ncit_code', ''), - }) - - return etree.tostring(root, pretty_print=True, xml_declaration=True, encoding='UTF-8').decode() + item_def = etree.SubElement( + root, + "ItemDef", + attrib={ + "OID": bc.get("bc_id", "UNKNOWN"), + "Name": bc.get("short_name", ""), + "DataType": "text", + }, + ) + desc = etree.SubElement(item_def, "Description") + XML_NS = "http://www.w3.org/XML/1998/namespace" + translated = etree.SubElement(desc, "TranslatedText", attrib={f"{{{XML_NS}}}lang": "en"}) + translated.text = bc.get("definition", "") + if bc.get("ncit_code"): + etree.SubElement( + item_def, + "Alias", + attrib={ + "Context": "nci:ExtCodeID", + "Name": bc.get("ncit_code", ""), + }, + ) + + return etree.tostring(root, pretty_print=True, xml_declaration=True, encoding="UTF-8").decode() diff --git a/services/ingestion.py b/services/ingestion.py index 276f4b7..3e5c674 100644 --- a/services/ingestion.py +++ b/services/ingestion.py @@ -3,26 +3,25 @@ import pandas as pd from difflib import SequenceMatcher - # Canonical BC field names and known aliases for fuzzy field mapping FIELD_MAP = { - 'bc_id': ['bc_id', 'bcid', 'concept_id', 'id', 'identifier'], - 'short_name': ['short_name', 'shortname', 'name', 'bc_name', 'concept_name', 'title'], - 'definition': ['definition', 'def', 'description', 'desc'], - 'ncit_code': ['ncit_code', 'ncit', 'nci_code', 'c_code', 'ccode'], - 'parent_bc_id': ['parent_bc_id', 'parent_id', 'parent', 'parentid'], - 'bc_categories': ['bc_categories', 'categories', 'category', 'class', 'domain'], - 'synonyms': ['synonyms', 'synonym', 'aliases', 'alt_names'], - 'result_scales': ['result_scales', 'scale', 'scales', 'result_scale', 'data_scale'], - 'system': ['system', 'coding_system', 'system_url', 'system_uri'], - 'system_name': ['system_name', 'systemname', 'coding_system_name'], - 'code': ['code', 'external_code', 'loinc', 'snomed'], - 'package_date': ['package_date', 'date', 'release_date'], - 'dec_id': ['dec_id', 'decid', 'data_element_id'], - 'ncit_dec_code': ['ncit_dec_code', 'dec_ncit', 'dec_code'], - 'dec_label': ['dec_label', 'label', 'dec_name', 'element_label'], - 'data_type': ['data_type', 'datatype', 'type', 'value_type'], - 'example_set': ['example_set', 'examples', 'example_values', 'values'], + "bc_id": ["bc_id", "bcid", "concept_id", "id", "identifier"], + "short_name": ["short_name", "shortname", "name", "bc_name", "concept_name", "title"], + "definition": ["definition", "def", "description", "desc"], + "ncit_code": ["ncit_code", "ncit", "nci_code", "c_code", "ccode"], + "parent_bc_id": ["parent_bc_id", "parent_id", "parent", "parentid"], + "bc_categories": ["bc_categories", "categories", "category", "class", "domain"], + "synonyms": ["synonyms", "synonym", "aliases", "alt_names"], + "result_scales": ["result_scales", "scale", "scales", "result_scale", "data_scale"], + "system": ["system", "coding_system", "system_url", "system_uri"], + "system_name": ["system_name", "systemname", "coding_system_name"], + "code": ["code", "external_code", "loinc", "snomed"], + "package_date": ["package_date", "date", "release_date"], + "dec_id": ["dec_id", "decid", "data_element_id"], + "ncit_dec_code": ["ncit_dec_code", "dec_ncit", "dec_code"], + "dec_label": ["dec_label", "label", "dec_name", "element_label"], + "data_type": ["data_type", "datatype", "type", "value_type"], + "example_set": ["example_set", "examples", "example_values", "values"], } @@ -32,7 +31,7 @@ def _similarity(a, b): def _match_field(col_name): """Return (canonical_field, confidence) for a column name.""" - col_lower = col_name.lower().replace(' ', '_').replace('-', '_') + col_lower = col_name.lower().replace(" ", "_").replace("-", "_") best_field, best_score = None, 0.0 for canonical, aliases in FIELD_MAP.items(): for alias in aliases: @@ -51,11 +50,11 @@ def map_fields(raw_dict): mapped = {} confidences = {} for col, value in raw_dict.items(): - if value is None or (isinstance(value, float) and str(value) == 'nan'): + if value is None or (isinstance(value, float) and str(value) == "nan"): continue field, score = _match_field(str(col)) if field and score > 0.5: - mapped[field] = str(value).strip() if value is not None else '' + mapped[field] = str(value).strip() if value is not None else "" confidences[field] = score return mapped, confidences @@ -66,16 +65,16 @@ def validate_bc(bc_dict): Returns list of validation error strings. """ errors = [] - if not bc_dict.get('short_name'): - errors.append('short_name is required') - if not bc_dict.get('definition'): - errors.append('definition is required') - if not bc_dict.get('bc_id') and not bc_dict.get('ncit_code'): - errors.append('Either bc_id (NCIt C-code) or ncit_code is required') + if not bc_dict.get("short_name"): + errors.append("short_name is required") + if not bc_dict.get("definition"): + errors.append("definition is required") + if not bc_dict.get("bc_id") and not bc_dict.get("ncit_code"): + errors.append("Either bc_id (NCIt C-code) or ncit_code is required") # NCIt code format check - ncit = bc_dict.get('ncit_code') or bc_dict.get('bc_id', '') - if ncit and not ncit.upper().startswith('C'): - errors.append(f'NCIt code should start with C (got: {ncit})') + ncit = bc_dict.get("ncit_code") or bc_dict.get("bc_id", "") + if ncit and not ncit.upper().startswith("C"): + errors.append(f"NCIt code should start with C (got: {ncit})") return errors @@ -87,46 +86,51 @@ def _group_by_bc(rows, sheet=None): Returns a list of merged dicts ready for IngestionRecord creation. """ from collections import OrderedDict + groups = OrderedDict() for mapped, confs in rows: - bc_id = mapped.get('bc_id') or mapped.get('ncit_code', '') + bc_id = mapped.get("bc_id") or mapped.get("ncit_code", "") if not bc_id: continue if bc_id not in groups: - groups[bc_id] = {'mapped': {}, 'confidences': {}, 'decs': [], 'source_sheet': sheet} + groups[bc_id] = {"mapped": {}, "confidences": {}, "decs": [], "source_sheet": sheet} g = groups[bc_id] - if mapped.get('definition') and not g['mapped'].get('definition'): + if mapped.get("definition") and not g["mapped"].get("definition"): # Absorb BC-level fields from this row for k, v in mapped.items(): - if k not in ('dec_id', 'ncit_dec_code', 'dec_label', 'data_type', 'example_set'): - g['mapped'][k] = v - g['confidences'].update(confs) - if mapped.get('dec_id') or mapped.get('dec_label'): - g['decs'].append({ - 'dec_id': mapped.get('dec_id', ''), - 'ncit_dec_code': mapped.get('ncit_dec_code', ''), - 'dec_label': mapped.get('dec_label', ''), - 'data_type': mapped.get('data_type', 'string'), - 'example_set': mapped.get('example_set', ''), - }) + if k not in ("dec_id", "ncit_dec_code", "dec_label", "data_type", "example_set"): + g["mapped"][k] = v + g["confidences"].update(confs) + if mapped.get("dec_id") or mapped.get("dec_label"): + g["decs"].append( + { + "dec_id": mapped.get("dec_id", ""), + "ncit_dec_code": mapped.get("ncit_dec_code", ""), + "dec_label": mapped.get("dec_label", ""), + "data_type": mapped.get("data_type", "string"), + "example_set": mapped.get("example_set", ""), + } + ) # If still no definition absorbed, keep the mapped fields - if not g['mapped']: - g['mapped'].update(mapped) - g['confidences'].update(confs) + if not g["mapped"]: + g["mapped"].update(mapped) + g["confidences"].update(confs) results = [] for bc_id, g in groups.items(): - mapped = g['mapped'] - if not mapped.get('bc_id') and bc_id: - mapped['bc_id'] = bc_id + mapped = g["mapped"] + if not mapped.get("bc_id") and bc_id: + mapped["bc_id"] = bc_id errors = validate_bc(mapped) - results.append({ - 'mapped': mapped, - 'confidences': g['confidences'], - 'decs': g['decs'], - 'errors': errors, - 'source_sheet': g['source_sheet'], - }) + results.append( + { + "mapped": mapped, + "confidences": g["confidences"], + "decs": g["decs"], + "errors": errors, + "source_sheet": g["source_sheet"], + } + ) return results @@ -148,7 +152,7 @@ def parse_xlsx(file_obj): rows.append((mapped, confs)) results.extend(_group_by_bc(rows, sheet=sheet)) except Exception as e: - results.append({'error': str(e), 'mapped': {}, 'confidences': {}, 'decs': [], 'errors': [str(e)]}) + results.append({"error": str(e), "mapped": {}, "confidences": {}, "decs": [], "errors": [str(e)]}) return results @@ -163,14 +167,16 @@ def parse_csv(file_obj): if not mapped: continue errors = validate_bc(mapped) - results.append({ - 'raw': {k: str(v) for k, v in raw.items() if v is not None}, - 'mapped': mapped, - 'confidences': confs, - 'errors': errors, - }) + results.append( + { + "raw": {k: str(v) for k, v in raw.items() if v is not None}, + "mapped": mapped, + "confidences": confs, + "errors": errors, + } + ) except Exception as e: - results.append({'error': str(e), 'raw': {}, 'mapped': {}, 'confidences': {}, 'errors': [str(e)]}) + results.append({"error": str(e), "raw": {}, "mapped": {}, "confidences": {}, "errors": [str(e)]}) return results @@ -186,14 +192,16 @@ def parse_json(file_obj): if not mapped: continue errors = validate_bc(mapped) - results.append({ - 'raw': {k: str(v) for k, v in item.items()}, - 'mapped': mapped, - 'confidences': confs, - 'errors': errors, - }) + results.append( + { + "raw": {k: str(v) for k, v in item.items()}, + "mapped": mapped, + "confidences": confs, + "errors": errors, + } + ) except Exception as e: - results.append({'error': str(e), 'raw': {}, 'mapped': {}, 'confidences': {}, 'errors': [str(e)]}) + results.append({"error": str(e), "raw": {}, "mapped": {}, "confidences": {}, "errors": [str(e)]}) return results @@ -204,6 +212,6 @@ def deduplicate(parsed_records, existing_ids): Returns same list with 'duplicate': True/False added. """ for rec in parsed_records: - bc_id = rec.get('mapped', {}).get('bc_id') or rec.get('mapped', {}).get('ncit_code', '') - rec['duplicate'] = bc_id.upper() in {e.upper() for e in existing_ids} + bc_id = rec.get("mapped", {}).get("bc_id") or rec.get("mapped", {}).get("ncit_code", "") + rec["duplicate"] = bc_id.upper() in {e.upper() for e in existing_ids} return parsed_records diff --git a/services/ncit_api.py b/services/ncit_api.py index e477599..1dfcdc9 100644 --- a/services/ncit_api.py +++ b/services/ncit_api.py @@ -2,7 +2,7 @@ class NCItApiClient: - BASE_URL = 'https://api-evsrest.nci.nih.gov/api/v1' + BASE_URL = "https://api-evsrest.nci.nih.gov/api/v1" def _get(self, path, params=None): url = f"{self.BASE_URL}{path}" @@ -13,46 +13,34 @@ def _get(self, path, params=None): def search_concept(self, term, size=10): """Search NCIt for concepts matching term. Returns list of matches.""" try: - results = self._get( - '/concept/ncit/search', - params={'term': term, 'type': 'contains', 'include': 'minimal', 'pageSize': size} - ) - concepts = results.get('concepts', []) + results = self._get("/concept/ncit/search", params={"term": term, "type": "contains", "include": "minimal", "pageSize": size}) + concepts = results.get("concepts", []) return [ { - 'code': c.get('code'), - 'name': c.get('name'), - 'definition': next( - (d.get('definition') for d in c.get('definitions', []) if d.get('source') == 'NCI'), - c.get('name') - ), + "code": c.get("code"), + "name": c.get("name"), + "definition": next((d.get("definition") for d in c.get("definitions", []) if d.get("source") == "NCI"), c.get("name")), } for c in concepts ] except Exception as e: - return [{'error': str(e)}] + return [{"error": str(e)}] def get_concept(self, ncit_code): """Fetch full concept details including synonyms.""" try: - result = self._get(f'/concept/ncit/{ncit_code}', params={'include': 'full'}) + result = self._get(f"/concept/ncit/{ncit_code}", params={"include": "full"}) return { - 'code': result.get('code'), - 'name': result.get('name'), - 'definition': next( - (d.get('definition') for d in result.get('definitions', []) if d.get('source') == 'NCI'), - '' - ), - 'synonyms': [ - s.get('name') for s in result.get('synonyms', []) - if s.get('termType') in ('SY', 'AB', 'PT') - ], - 'preferred_name': result.get('name'), + "code": result.get("code"), + "name": result.get("name"), + "definition": next((d.get("definition") for d in result.get("definitions", []) if d.get("source") == "NCI"), ""), + "synonyms": [s.get("name") for s in result.get("synonyms", []) if s.get("termType") in ("SY", "AB", "PT")], + "preferred_name": result.get("name"), } except Exception as e: - return {'error': str(e)} + return {"error": str(e)} def get_preferred_name(self, ncit_code): """Return just the preferred name for an NCIt code.""" concept = self.get_concept(ncit_code) - return concept.get('preferred_name') or concept.get('name', '') + return concept.get("preferred_name") or concept.get("name", "") diff --git a/tests/conftest.py b/tests/conftest.py index 3b8746c..dfe64d7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,16 +9,16 @@ class TestConfig: TESTING = True - SQLALCHEMY_DATABASE_URI = 'sqlite:///:memory:' - SECRET_KEY = 'test-secret-key' - CDISC_API_KEY = '' - CDISC_API_BASE_URL = 'https://api.library.cdisc.org/api/cosmos/v2' - NCIT_API_BASE_URL = 'https://api-evsrest.nci.nih.gov/api/v1' + SQLALCHEMY_DATABASE_URI = "sqlite:///:memory:" + SECRET_KEY = "test-secret-key" + CDISC_API_KEY = "" + CDISC_API_BASE_URL = "https://api.library.cdisc.org/api/cosmos/v2" + NCIT_API_BASE_URL = "https://api-evsrest.nci.nih.gov/api/v1" MAX_CONTENT_LENGTH = 16 * 1024 * 1024 WTF_CSRF_ENABLED = False -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def app(): app = create_app(TestConfig) return app @@ -44,12 +44,12 @@ def sample_bc(app): """A minimal BiomedicalConcept persisted to the test DB.""" with app.app_context(): bc = BiomedicalConcept( - bc_id='C12345', - short_name='Test Concept', - definition='A test BC definition.', - ncit_code='C12345', - status='provisional', - submitter='tester', + bc_id="C12345", + short_name="Test Concept", + definition="A test BC definition.", + ncit_code="C12345", + status="provisional", + submitter="tester", ) _db.session.add(bc) _db.session.commit() diff --git a/tests/test_audit_routes.py b/tests/test_audit_routes.py index 053e792..8e2b649 100644 --- a/tests/test_audit_routes.py +++ b/tests/test_audit_routes.py @@ -1,10 +1,11 @@ """Tests for routes/audit.py — log listing and filtering.""" + import pytest from models.audit import AuditLog from extensions import db -def _add_log(app, entity_type='BiomedicalConcept', entity_id='C001', action='created', actor='alice'): +def _add_log(app, entity_type="BiomedicalConcept", entity_id="C001", action="created", actor="alice"): with app.app_context(): log = AuditLog(entity_type=entity_type, entity_id=entity_id, action=action, actor=actor) db.session.add(log) @@ -13,37 +14,37 @@ def _add_log(app, entity_type='BiomedicalConcept', entity_id='C001', action='cre class TestAuditIndex: def test_returns_200_empty(self, client): - r = client.get('/audit/') + r = client.get("/audit/") assert r.status_code == 200 def test_shows_log_entries(self, client, app): _add_log(app) - r = client.get('/audit/') + r = client.get("/audit/") assert r.status_code == 200 - assert b'C001' in r.data + assert b"C001" in r.data def test_filter_by_entity_type(self, client, app): - _add_log(app, entity_type='BiomedicalConcept', entity_id='C001') - _add_log(app, entity_type='GovernanceRecord', entity_id='G001') - r = client.get('/audit/?entity_type=BiomedicalConcept') - assert b'C001' in r.data - assert b'G001' not in r.data + _add_log(app, entity_type="BiomedicalConcept", entity_id="C001") + _add_log(app, entity_type="GovernanceRecord", entity_id="G001") + r = client.get("/audit/?entity_type=BiomedicalConcept") + assert b"C001" in r.data + assert b"G001" not in r.data def test_filter_by_action(self, client, app): - _add_log(app, entity_id='C001', action='created') - _add_log(app, entity_id='C002', action='deleted') - r = client.get('/audit/?action=deleted') - assert b'C002' in r.data - assert b'created' not in r.data or b'C001' not in r.data + _add_log(app, entity_id="C001", action="created") + _add_log(app, entity_id="C002", action="deleted") + r = client.get("/audit/?action=deleted") + assert b"C002" in r.data + assert b"created" not in r.data or b"C001" not in r.data def test_filter_by_actor(self, client, app): - _add_log(app, entity_id='C001', actor='alice') - _add_log(app, entity_id='C002', actor='bob') - r = client.get('/audit/?actor=alice') - assert b'alice' in r.data - assert b'bob' not in r.data + _add_log(app, entity_id="C001", actor="alice") + _add_log(app, entity_id="C002", actor="bob") + r = client.get("/audit/?actor=alice") + assert b"alice" in r.data + assert b"bob" not in r.data def test_pagination_param_accepted(self, client, app): _add_log(app) - r = client.get('/audit/?page=1') + r = client.get("/audit/?page=1") assert r.status_code == 200 diff --git a/tests/test_bc_routes.py b/tests/test_bc_routes.py index 56b2c9a..13a7db7 100644 --- a/tests/test_bc_routes.py +++ b/tests/test_bc_routes.py @@ -1,4 +1,5 @@ """Tests for routes/bc.py — CRUD, export, submission.""" + import pytest from models.bc import BiomedicalConcept, DataElementConcept from models.audit import AuditLog @@ -7,11 +8,11 @@ def _bc_form(**kwargs): defaults = { - 'bc_id': 'C00001', - 'short_name': 'Test Concept', - 'definition': 'A definition.', - 'ncit_code': 'C00001', - 'submitter': 'tester', + "bc_id": "C00001", + "short_name": "Test Concept", + "definition": "A definition.", + "ncit_code": "C00001", + "submitter": "tester", } defaults.update(kwargs) return defaults @@ -21,22 +22,23 @@ def _bc_form(**kwargs): # GET /bc/ # --------------------------------------------------------------------------- + class TestBcIndex: def test_returns_200(self, client): - r = client.get('/bc/') + r = client.get("/bc/") assert r.status_code == 200 def test_search_by_name(self, client, app, sample_bc): - r = client.get('/bc/?q=Test') + r = client.get("/bc/?q=Test") assert r.status_code == 200 - assert b'Test Concept' in r.data + assert b"Test Concept" in r.data def test_search_no_match(self, client): - r = client.get('/bc/?q=zzznomatch') + r = client.get("/bc/?q=zzznomatch") assert r.status_code == 200 def test_filter_by_status(self, client, sample_bc): - r = client.get('/bc/?status=provisional') + r = client.get("/bc/?status=provisional") assert r.status_code == 200 @@ -44,9 +46,10 @@ def test_filter_by_status(self, client, sample_bc): # GET /bc/new # --------------------------------------------------------------------------- + class TestNewBc: def test_returns_200(self, client): - r = client.get('/bc/new') + r = client.get("/bc/new") assert r.status_code == 200 @@ -54,53 +57,55 @@ def test_returns_200(self, client): # POST /bc/ (create) # --------------------------------------------------------------------------- + class TestCreateBc: def test_creates_bc_and_redirects(self, client, app): - r = client.post('/bc/', data=_bc_form(), follow_redirects=False) + r = client.post("/bc/", data=_bc_form(), follow_redirects=False) assert r.status_code == 302 with app.app_context(): - assert BiomedicalConcept.query.get('C00001') is not None + assert BiomedicalConcept.query.get("C00001") is not None def test_missing_bc_id_redirects_with_error(self, client): - r = client.post('/bc/', data=_bc_form(bc_id=''), follow_redirects=True) - assert b'required' in r.data.lower() or r.status_code in (200, 302) + r = client.post("/bc/", data=_bc_form(bc_id=""), follow_redirects=True) + assert b"required" in r.data.lower() or r.status_code in (200, 302) def test_duplicate_bc_id_rejected(self, client, sample_bc): # First creation (sample_bc fixture did it already) - r = client.post('/bc/', data=_bc_form(bc_id='C12345'), follow_redirects=True) - assert b'already exists' in r.data or r.status_code in (200, 302) + r = client.post("/bc/", data=_bc_form(bc_id="C12345"), follow_redirects=True) + assert b"already exists" in r.data or r.status_code in (200, 302) def test_create_writes_audit_log(self, client, app): - client.post('/bc/', data=_bc_form()) + client.post("/bc/", data=_bc_form()) with app.app_context(): - log = AuditLog.query.filter_by(entity_id='C00001', action='created').first() + log = AuditLog.query.filter_by(entity_id="C00001", action="created").first() assert log is not None def test_create_with_decs(self, client, app): data = _bc_form() - data['dec_label[]'] = ['Systolic', 'Diastolic'] - data['dec_data_type[]'] = ['decimal', 'decimal'] - data['dec_example_set[]'] = ['120', '80'] - data['dec_id[]'] = ['', ''] - data['dec_ncit_code[]'] = ['', ''] - client.post('/bc/', data=data) + data["dec_label[]"] = ["Systolic", "Diastolic"] + data["dec_data_type[]"] = ["decimal", "decimal"] + data["dec_example_set[]"] = ["120", "80"] + data["dec_id[]"] = ["", ""] + data["dec_ncit_code[]"] = ["", ""] + client.post("/bc/", data=data) with app.app_context(): - decs = DataElementConcept.query.filter_by(bc_id='C00001').all() + decs = DataElementConcept.query.filter_by(bc_id="C00001").all() assert len(decs) == 2 - assert decs[0].dec_label == 'Systolic' + assert decs[0].dec_label == "Systolic" # --------------------------------------------------------------------------- # GET /bc/ # --------------------------------------------------------------------------- + class TestBcDetail: def test_existing_bc_returns_200(self, client, sample_bc): - r = client.get('/bc/C12345') + r = client.get("/bc/C12345") assert r.status_code == 200 def test_missing_bc_returns_404(self, client): - r = client.get('/bc/DOESNOTEXIST') + r = client.get("/bc/DOESNOTEXIST") assert r.status_code == 404 @@ -108,22 +113,23 @@ def test_missing_bc_returns_404(self, client): # POST /bc//edit # --------------------------------------------------------------------------- + class TestEditBc: def test_updates_short_name(self, client, app, sample_bc): - client.post('/bc/C12345/edit', data={'short_name': 'Updated Name'}) + client.post("/bc/C12345/edit", data={"short_name": "Updated Name"}) with app.app_context(): - bc = BiomedicalConcept.query.get('C12345') - assert bc.short_name == 'Updated Name' + bc = BiomedicalConcept.query.get("C12345") + assert bc.short_name == "Updated Name" def test_edit_writes_audit_log(self, client, app, sample_bc): - client.post('/bc/C12345/edit', data={'short_name': 'Updated Name'}) + client.post("/bc/C12345/edit", data={"short_name": "Updated Name"}) with app.app_context(): - log = AuditLog.query.filter_by(entity_id='C12345', action='updated').first() + log = AuditLog.query.filter_by(entity_id="C12345", action="updated").first() assert log is not None - assert log.before_state['short_name'] == 'Test Concept' + assert log.before_state["short_name"] == "Test Concept" def test_nonexistent_bc_returns_404(self, client): - r = client.post('/bc/NOPE/edit', data={'short_name': 'X'}) + r = client.post("/bc/NOPE/edit", data={"short_name": "X"}) assert r.status_code == 404 @@ -131,17 +137,18 @@ def test_nonexistent_bc_returns_404(self, client): # POST /bc//submit # --------------------------------------------------------------------------- + class TestSubmitForReview: def test_advances_status_to_sme_review(self, client, app, sample_bc): - client.post('/bc/C12345/submit') + client.post("/bc/C12345/submit") with app.app_context(): - bc = BiomedicalConcept.query.get('C12345') - assert bc.status == 'sme_review' + bc = BiomedicalConcept.query.get("C12345") + assert bc.status == "sme_review" def test_submit_writes_audit_log(self, client, app, sample_bc): - client.post('/bc/C12345/submit') + client.post("/bc/C12345/submit") with app.app_context(): - log = AuditLog.query.filter_by(entity_id='C12345', action='submitted_for_review').first() + log = AuditLog.query.filter_by(entity_id="C12345", action="submitted_for_review").first() assert log is not None @@ -149,20 +156,21 @@ def test_submit_writes_audit_log(self, client, app, sample_bc): # POST /bc//delete # --------------------------------------------------------------------------- + class TestDeleteBc: def test_deletes_bc(self, client, app, sample_bc): - client.post('/bc/C12345/delete') + client.post("/bc/C12345/delete") with app.app_context(): - assert BiomedicalConcept.query.get('C12345') is None + assert BiomedicalConcept.query.get("C12345") is None def test_delete_writes_audit_log(self, client, app, sample_bc): - client.post('/bc/C12345/delete') + client.post("/bc/C12345/delete") with app.app_context(): - log = AuditLog.query.filter_by(entity_id='C12345', action='deleted').first() + log = AuditLog.query.filter_by(entity_id="C12345", action="deleted").first() assert log is not None def test_nonexistent_bc_returns_404(self, client): - r = client.post('/bc/NOPE/delete') + r = client.post("/bc/NOPE/delete") assert r.status_code == 404 @@ -170,18 +178,19 @@ def test_nonexistent_bc_returns_404(self, client): # GET /bc/export # --------------------------------------------------------------------------- + class TestExport: def test_json_export(self, client, sample_bc): - r = client.get('/bc/export?format=json') + r = client.get("/bc/export?format=json") assert r.status_code == 200 - assert r.content_type == 'application/json' + assert r.content_type == "application/json" def test_xlsx_export(self, client, sample_bc): - r = client.get('/bc/export?format=xlsx') + r = client.get("/bc/export?format=xlsx") assert r.status_code == 200 - assert 'spreadsheetml' in r.content_type + assert "spreadsheetml" in r.content_type def test_odm_xml_export(self, client, sample_bc): - r = client.get('/bc/export?format=odm') + r = client.get("/bc/export?format=odm") assert r.status_code == 200 - assert 'xml' in r.content_type + assert "xml" in r.content_type diff --git a/tests/test_governance_routes.py b/tests/test_governance_routes.py index beba67a..5eb0ea9 100644 --- a/tests/test_governance_routes.py +++ b/tests/test_governance_routes.py @@ -1,106 +1,106 @@ """Tests for routes/governance.py — Kanban advance and reject.""" + import pytest from models.bc import BiomedicalConcept from models.governance import GovernanceRecord from models.audit import AuditLog from extensions import db - -STATUS_ORDER = ['provisional', 'sme_review', 'cdisc_approval', 'published'] +STATUS_ORDER = ["provisional", "sme_review", "cdisc_approval", "published"] class TestGovernanceBoard: def test_board_returns_200(self, client): - r = client.get('/governance/board') + r = client.get("/governance/board") assert r.status_code == 200 class TestAdvance: def test_advances_provisional_to_sme_review(self, client, app, sample_bc): - client.post('/governance/advance/C12345') + client.post("/governance/advance/C12345") with app.app_context(): - bc = BiomedicalConcept.query.get('C12345') - assert bc.status == 'sme_review' + bc = BiomedicalConcept.query.get("C12345") + assert bc.status == "sme_review" def test_advance_through_all_stages(self, client, app, sample_bc): - for expected in ['sme_review', 'cdisc_approval', 'published']: - client.post('/governance/advance/C12345') + for expected in ["sme_review", "cdisc_approval", "published"]: + client.post("/governance/advance/C12345") with app.app_context(): - bc = BiomedicalConcept.query.get('C12345') - assert bc.status == 'published' + bc = BiomedicalConcept.query.get("C12345") + assert bc.status == "published" def test_already_published_stays_published(self, client, app, sample_bc): # Advance to published for _ in range(3): - client.post('/governance/advance/C12345') + client.post("/governance/advance/C12345") # Extra advance should not error or change status - r = client.post('/governance/advance/C12345', follow_redirects=True) + r = client.post("/governance/advance/C12345", follow_redirects=True) assert r.status_code == 200 with app.app_context(): - bc = BiomedicalConcept.query.get('C12345') - assert bc.status == 'published' + bc = BiomedicalConcept.query.get("C12345") + assert bc.status == "published" def test_advance_creates_governance_record(self, client, app, sample_bc): - client.post('/governance/advance/C12345') + client.post("/governance/advance/C12345") with app.app_context(): - rec = GovernanceRecord.query.filter_by(bc_id='C12345', action='advanced').first() + rec = GovernanceRecord.query.filter_by(bc_id="C12345", action="advanced").first() assert rec is not None def test_advance_writes_audit_log(self, client, app, sample_bc): - client.post('/governance/advance/C12345') + client.post("/governance/advance/C12345") with app.app_context(): - log = AuditLog.query.filter_by(entity_id='C12345', action='status_changed').first() + log = AuditLog.query.filter_by(entity_id="C12345", action="status_changed").first() assert log is not None - assert log.before_state == {'status': 'provisional'} - assert log.after_state == {'status': 'sme_review'} + assert log.before_state == {"status": "provisional"} + assert log.after_state == {"status": "sme_review"} def test_advance_nonexistent_bc_returns_404(self, client): - r = client.post('/governance/advance/NOPE') + r = client.post("/governance/advance/NOPE") assert r.status_code == 404 def test_advance_ajax_returns_json(self, client, sample_bc): r = client.post( - '/governance/advance/C12345', - headers={'X-Requested-With': 'XMLHttpRequest'}, + "/governance/advance/C12345", + headers={"X-Requested-With": "XMLHttpRequest"}, ) assert r.status_code == 200 data = r.get_json() - assert data['status'] == 'sme_review' - assert data['bc_id'] == 'C12345' + assert data["status"] == "sme_review" + assert data["bc_id"] == "C12345" class TestReject: def test_reject_returns_to_provisional(self, client, app, sample_bc): # First advance to sme_review, then reject - client.post('/governance/advance/C12345') - client.post('/governance/reject/C12345') + client.post("/governance/advance/C12345") + client.post("/governance/reject/C12345") with app.app_context(): - bc = BiomedicalConcept.query.get('C12345') - assert bc.status == 'provisional' + bc = BiomedicalConcept.query.get("C12345") + assert bc.status == "provisional" def test_reject_creates_governance_record(self, client, app, sample_bc): - client.post('/governance/reject/C12345') + client.post("/governance/reject/C12345") with app.app_context(): - rec = GovernanceRecord.query.filter_by(bc_id='C12345', action='rejected').first() + rec = GovernanceRecord.query.filter_by(bc_id="C12345", action="rejected").first() assert rec is not None def test_reject_writes_audit_log(self, client, app, sample_bc): - client.post('/governance/advance/C12345') # move to sme_review - client.post('/governance/reject/C12345') + client.post("/governance/advance/C12345") # move to sme_review + client.post("/governance/reject/C12345") with app.app_context(): - log = AuditLog.query.filter_by(entity_id='C12345', action='rejected').first() + log = AuditLog.query.filter_by(entity_id="C12345", action="rejected").first() assert log is not None - assert log.after_state == {'status': 'provisional'} + assert log.after_state == {"status": "provisional"} def test_reject_ajax_returns_json(self, client, sample_bc): r = client.post( - '/governance/reject/C12345', - headers={'X-Requested-With': 'XMLHttpRequest'}, + "/governance/reject/C12345", + headers={"X-Requested-With": "XMLHttpRequest"}, ) assert r.status_code == 200 data = r.get_json() - assert data['status'] == 'provisional' + assert data["status"] == "provisional" def test_reject_nonexistent_bc_returns_404(self, client): - r = client.post('/governance/reject/NOPE') + r = client.post("/governance/reject/NOPE") assert r.status_code == 404 diff --git a/tests/test_ingestion_routes.py b/tests/test_ingestion_routes.py index 18d615f..1ba19bf 100644 --- a/tests/test_ingestion_routes.py +++ b/tests/test_ingestion_routes.py @@ -1,4 +1,5 @@ """Tests for routes/ingestion.py — upload, approve, reject.""" + import io import json import csv @@ -14,27 +15,27 @@ def _csv_file(rows): writer = csv.DictWriter(buf, fieldnames=rows[0].keys()) writer.writeheader() writer.writerows(rows) - return (io.BytesIO(buf.getvalue().encode()), 'test.csv') + return (io.BytesIO(buf.getvalue().encode()), "test.csv") def _json_file(data): - return (io.BytesIO(json.dumps(data).encode()), 'test.json') + return (io.BytesIO(json.dumps(data).encode()), "test.json") class TestIngestionIndex: def test_returns_200(self, client): - r = client.get('/ingestion/') + r = client.get("/ingestion/") assert r.status_code == 200 class TestUpload: def test_upload_csv_creates_ingestion_records(self, client, app): - rows = [{'bc_id': 'C001', 'short_name': 'HR', 'definition': 'Heart Rate'}] + rows = [{"bc_id": "C001", "short_name": "HR", "definition": "Heart Rate"}] file_obj, filename = _csv_file(rows) r = client.post( - '/ingestion/upload', - data={'file': (file_obj, filename)}, - content_type='multipart/form-data', + "/ingestion/upload", + data={"file": (file_obj, filename)}, + content_type="multipart/form-data", follow_redirects=True, ) assert r.status_code == 200 @@ -42,12 +43,12 @@ def test_upload_csv_creates_ingestion_records(self, client, app): assert IngestionRecord.query.count() > 0 def test_upload_json_creates_ingestion_records(self, client, app): - data = [{'bc_id': 'C002', 'short_name': 'BP', 'definition': 'Blood Pressure'}] + data = [{"bc_id": "C002", "short_name": "BP", "definition": "Blood Pressure"}] file_obj, filename = _json_file(data) r = client.post( - '/ingestion/upload', - data={'file': (file_obj, filename)}, - content_type='multipart/form-data', + "/ingestion/upload", + data={"file": (file_obj, filename)}, + content_type="multipart/form-data", follow_redirects=True, ) assert r.status_code == 200 @@ -55,25 +56,25 @@ def test_upload_json_creates_ingestion_records(self, client, app): assert IngestionRecord.query.count() > 0 def test_no_file_redirects_with_error(self, client): - r = client.post('/ingestion/upload', data={}, follow_redirects=True) - assert b'No file' in r.data or r.status_code in (200, 302) + r = client.post("/ingestion/upload", data={}, follow_redirects=True) + assert b"No file" in r.data or r.status_code in (200, 302) def test_wrong_extension_rejected(self, client): r = client.post( - '/ingestion/upload', - data={'file': (io.BytesIO(b'data'), 'test.txt')}, - content_type='multipart/form-data', + "/ingestion/upload", + data={"file": (io.BytesIO(b"data"), "test.txt")}, + content_type="multipart/form-data", follow_redirects=True, ) - assert b'XLSX' in r.data or b'xlsx' in r.data or r.status_code in (200, 302) + assert b"XLSX" in r.data or b"xlsx" in r.data or r.status_code in (200, 302) def test_duplicate_bc_flagged(self, client, app, sample_bc): - rows = [{'bc_id': 'C12345', 'short_name': 'Test Concept', 'definition': 'A definition.'}] + rows = [{"bc_id": "C12345", "short_name": "Test Concept", "definition": "A definition."}] file_obj, filename = _csv_file(rows) client.post( - '/ingestion/upload', - data={'file': (file_obj, filename)}, - content_type='multipart/form-data', + "/ingestion/upload", + data={"file": (file_obj, filename)}, + content_type="multipart/form-data", ) with app.app_context(): ir = IngestionRecord.query.first() @@ -82,89 +83,89 @@ def test_duplicate_bc_flagged(self, client, app, sample_bc): class TestApprove: - def _upload_and_get_record_id(self, client, app, bc_id='C001'): - rows = [{'bc_id': bc_id, 'short_name': 'HR', 'definition': 'Heart Rate'}] + def _upload_and_get_record_id(self, client, app, bc_id="C001"): + rows = [{"bc_id": bc_id, "short_name": "HR", "definition": "Heart Rate"}] file_obj, filename = _csv_file(rows) client.post( - '/ingestion/upload', - data={'file': (file_obj, filename)}, - content_type='multipart/form-data', + "/ingestion/upload", + data={"file": (file_obj, filename)}, + content_type="multipart/form-data", ) with app.app_context(): - ir = IngestionRecord.query.filter_by(status='pending').first() + ir = IngestionRecord.query.filter_by(status="pending").first() return ir.id if ir else None def test_approve_creates_bc(self, client, app): record_id = self._upload_and_get_record_id(client, app) assert record_id is not None - client.post(f'/ingestion/approve/{record_id}') + client.post(f"/ingestion/approve/{record_id}") with app.app_context(): - assert BiomedicalConcept.query.get('C001') is not None + assert BiomedicalConcept.query.get("C001") is not None def test_approve_sets_status_approved(self, client, app): record_id = self._upload_and_get_record_id(client, app) - client.post(f'/ingestion/approve/{record_id}') + client.post(f"/ingestion/approve/{record_id}") with app.app_context(): ir = IngestionRecord.query.get(record_id) - assert ir.status == 'approved' + assert ir.status == "approved" def test_approve_nonexistent_record_returns_404(self, client): - r = client.post('/ingestion/approve/99999') + r = client.post("/ingestion/approve/99999") assert r.status_code == 404 def test_approve_already_existing_bc_does_not_duplicate(self, client, app, sample_bc): - rows = [{'bc_id': 'C12345', 'short_name': 'Test Concept', 'definition': 'A definition.'}] + rows = [{"bc_id": "C12345", "short_name": "Test Concept", "definition": "A definition."}] file_obj, filename = _csv_file(rows) client.post( - '/ingestion/upload', - data={'file': (file_obj, filename)}, - content_type='multipart/form-data', + "/ingestion/upload", + data={"file": (file_obj, filename)}, + content_type="multipart/form-data", ) with app.app_context(): - ir = IngestionRecord.query.filter_by(status='pending').first() + ir = IngestionRecord.query.filter_by(status="pending").first() if ir: - client.post(f'/ingestion/approve/{ir.id}') + client.post(f"/ingestion/approve/{ir.id}") with app.app_context(): # Should still be only one BC with that id - assert BiomedicalConcept.query.filter_by(bc_id='C12345').count() == 1 + assert BiomedicalConcept.query.filter_by(bc_id="C12345").count() == 1 class TestReject: def test_reject_sets_status_rejected(self, client, app): - rows = [{'bc_id': 'C001', 'short_name': 'HR', 'definition': 'Heart Rate'}] + rows = [{"bc_id": "C001", "short_name": "HR", "definition": "Heart Rate"}] file_obj, filename = _csv_file(rows) client.post( - '/ingestion/upload', - data={'file': (file_obj, filename)}, - content_type='multipart/form-data', + "/ingestion/upload", + data={"file": (file_obj, filename)}, + content_type="multipart/form-data", ) with app.app_context(): - ir = IngestionRecord.query.filter_by(status='pending').first() + ir = IngestionRecord.query.filter_by(status="pending").first() record_id = ir.id - client.post(f'/ingestion/reject/{record_id}') + client.post(f"/ingestion/reject/{record_id}") with app.app_context(): ir = IngestionRecord.query.get(record_id) - assert ir.status == 'rejected' + assert ir.status == "rejected" def test_reject_does_not_create_bc(self, client, app): - rows = [{'bc_id': 'C001', 'short_name': 'HR', 'definition': 'Heart Rate'}] + rows = [{"bc_id": "C001", "short_name": "HR", "definition": "Heart Rate"}] file_obj, filename = _csv_file(rows) client.post( - '/ingestion/upload', - data={'file': (file_obj, filename)}, - content_type='multipart/form-data', + "/ingestion/upload", + data={"file": (file_obj, filename)}, + content_type="multipart/form-data", ) with app.app_context(): - ir = IngestionRecord.query.filter_by(status='pending').first() + ir = IngestionRecord.query.filter_by(status="pending").first() record_id = ir.id - client.post(f'/ingestion/reject/{record_id}') + client.post(f"/ingestion/reject/{record_id}") with app.app_context(): - assert BiomedicalConcept.query.get('C001') is None + assert BiomedicalConcept.query.get("C001") is None def test_reject_nonexistent_record_returns_404(self, client): - r = client.post('/ingestion/reject/99999') + r = client.post("/ingestion/reject/99999") assert r.status_code == 404 @@ -172,15 +173,15 @@ class TestApproveAll: def test_approve_all_skips_records_with_errors(self, client, app): """Records missing required fields (errors list non-empty) should be rejected, not added.""" # Upload a record missing short_name (will have validation errors) - rows = [{'bc_id': 'C001', 'definition': 'Missing short name'}] + rows = [{"bc_id": "C001", "definition": "Missing short name"}] file_obj, filename = _csv_file(rows) with client.session_transaction() as sess: - sess['ingestion_key'] = 'testkey' + sess["ingestion_key"] = "testkey" client.post( - '/ingestion/upload', - data={'file': (file_obj, filename)}, - content_type='multipart/form-data', + "/ingestion/upload", + data={"file": (file_obj, filename)}, + content_type="multipart/form-data", ) - client.post('/ingestion/approve_all') + client.post("/ingestion/approve_all") with app.app_context(): - assert BiomedicalConcept.query.get('C001') is None + assert BiomedicalConcept.query.get("C001") is None diff --git a/tests/test_ingestion_service.py b/tests/test_ingestion_service.py index 1728cb1..e666974 100644 --- a/tests/test_ingestion_service.py +++ b/tests/test_ingestion_service.py @@ -1,4 +1,5 @@ """Tests for services/ingestion.py — field mapping, validation, parsing, deduplication.""" + import io import json import pytest @@ -13,23 +14,23 @@ _group_by_bc, ) - # --------------------------------------------------------------------------- # _similarity # --------------------------------------------------------------------------- + class TestSimilarity: def test_identical_strings(self): - assert _similarity('bc_id', 'bc_id') == 1.0 + assert _similarity("bc_id", "bc_id") == 1.0 def test_case_insensitive(self): - assert _similarity('BC_ID', 'bc_id') == 1.0 + assert _similarity("BC_ID", "bc_id") == 1.0 def test_completely_different(self): - assert _similarity('bc_id', 'zzzzzzz') < 0.5 + assert _similarity("bc_id", "zzzzzzz") < 0.5 def test_partial_match(self): - score = _similarity('short_name', 'shortname') + score = _similarity("short_name", "shortname") assert score > 0.8 @@ -37,29 +38,30 @@ def test_partial_match(self): # _match_field # --------------------------------------------------------------------------- + class TestMatchField: def test_exact_alias_match(self): - field, score = _match_field('bc_id') - assert field == 'bc_id' + field, score = _match_field("bc_id") + assert field == "bc_id" assert score == 1.0 def test_known_alias(self): - field, score = _match_field('concept_id') - assert field == 'bc_id' + field, score = _match_field("concept_id") + assert field == "bc_id" assert score > 0.8 def test_definition_alias(self): - field, score = _match_field('description') - assert field == 'definition' + field, score = _match_field("description") + assert field == "definition" assert score > 0.7 def test_spaces_normalised(self): - field, score = _match_field('short name') - assert field == 'short_name' + field, score = _match_field("short name") + assert field == "short_name" assert score > 0.8 def test_unknown_column_returns_low_score(self): - _, score = _match_field('xyzzy_random_col_9999') + _, score = _match_field("xyzzy_random_col_9999") assert score < 0.5 @@ -67,65 +69,67 @@ def test_unknown_column_returns_low_score(self): # map_fields # --------------------------------------------------------------------------- + class TestMapFields: def test_canonical_columns_mapped_at_full_confidence(self): - mapped, confs = map_fields({'bc_id': 'C001', 'short_name': 'Foo', 'definition': 'Bar'}) - assert mapped['bc_id'] == 'C001' - assert confs['bc_id'] == 1.0 + mapped, confs = map_fields({"bc_id": "C001", "short_name": "Foo", "definition": "Bar"}) + assert mapped["bc_id"] == "C001" + assert confs["bc_id"] == 1.0 def test_none_values_skipped(self): - mapped, _ = map_fields({'bc_id': 'C001', 'short_name': None}) - assert 'short_name' not in mapped + mapped, _ = map_fields({"bc_id": "C001", "short_name": None}) + assert "short_name" not in mapped def test_nan_values_skipped(self): - mapped, _ = map_fields({'bc_id': 'C001', 'definition': float('nan')}) - assert 'definition' not in mapped + mapped, _ = map_fields({"bc_id": "C001", "definition": float("nan")}) + assert "definition" not in mapped def test_low_confidence_columns_excluded(self): - mapped, _ = map_fields({'xyzzy_totally_unknown': 'value'}) + mapped, _ = map_fields({"xyzzy_totally_unknown": "value"}) assert mapped == {} def test_values_stripped(self): - mapped, _ = map_fields({'bc_id': ' C001 '}) - assert mapped['bc_id'] == 'C001' + mapped, _ = map_fields({"bc_id": " C001 "}) + assert mapped["bc_id"] == "C001" # --------------------------------------------------------------------------- # validate_bc # --------------------------------------------------------------------------- + class TestValidateBc: def _valid(self): - return {'bc_id': 'C001', 'short_name': 'Heart Rate', 'definition': 'Rate of the heart.'} + return {"bc_id": "C001", "short_name": "Heart Rate", "definition": "Rate of the heart."} def test_valid_record_has_no_errors(self): assert validate_bc(self._valid()) == [] def test_missing_short_name(self): d = self._valid() - del d['short_name'] + del d["short_name"] errors = validate_bc(d) - assert any('short_name' in e for e in errors) + assert any("short_name" in e for e in errors) def test_missing_definition(self): d = self._valid() - del d['definition'] + del d["definition"] errors = validate_bc(d) - assert any('definition' in e for e in errors) + assert any("definition" in e for e in errors) def test_missing_both_ids(self): - errors = validate_bc({'short_name': 'X', 'definition': 'Y'}) - assert any('bc_id' in e or 'ncit_code' in e for e in errors) + errors = validate_bc({"short_name": "X", "definition": "Y"}) + assert any("bc_id" in e or "ncit_code" in e for e in errors) def test_invalid_ncit_format(self): d = self._valid() - d['ncit_code'] = 'BADCODE' + d["ncit_code"] = "BADCODE" errors = validate_bc(d) - assert any('NCIt' in e for e in errors) + assert any("NCIt" in e for e in errors) def test_ncit_starting_with_c_accepted(self): d = self._valid() - d['ncit_code'] = 'C99999' + d["ncit_code"] = "C99999" assert validate_bc(d) == [] @@ -133,40 +137,43 @@ def test_ncit_starting_with_c_accepted(self): # deduplicate # --------------------------------------------------------------------------- + class TestDeduplicate: def _record(self, bc_id): - return {'mapped': {'bc_id': bc_id}, 'confidences': {}, 'errors': [], 'decs': []} + return {"mapped": {"bc_id": bc_id}, "confidences": {}, "errors": [], "decs": []} def test_marks_existing_ids_as_duplicate(self): - records = [self._record('C001'), self._record('C002')] - result = deduplicate(records, {'C001'}) - assert result[0]['duplicate'] is True - assert result[1]['duplicate'] is False + records = [self._record("C001"), self._record("C002")] + result = deduplicate(records, {"C001"}) + assert result[0]["duplicate"] is True + assert result[1]["duplicate"] is False def test_case_insensitive_comparison(self): - records = [self._record('c001')] - result = deduplicate(records, {'C001'}) - assert result[0]['duplicate'] is True + records = [self._record("c001")] + result = deduplicate(records, {"C001"}) + assert result[0]["duplicate"] is True def test_empty_existing_ids(self): - records = [self._record('C001')] + records = [self._record("C001")] result = deduplicate(records, set()) - assert result[0]['duplicate'] is False + assert result[0]["duplicate"] is False def test_falls_back_to_ncit_code(self): - record = {'mapped': {'ncit_code': 'C999'}, 'confidences': {}, 'errors': [], 'decs': []} - result = deduplicate([record], {'C999'}) - assert result[0]['duplicate'] is True + record = {"mapped": {"ncit_code": "C999"}, "confidences": {}, "errors": [], "decs": []} + result = deduplicate([record], {"C999"}) + assert result[0]["duplicate"] is True # --------------------------------------------------------------------------- # parse_csv # --------------------------------------------------------------------------- + class TestParseCsv: def _make_csv(self, rows): """Return a BytesIO CSV from a list of dicts.""" import csv + buf = io.StringIO() if rows: writer = csv.DictWriter(buf, fieldnames=rows[0].keys()) @@ -175,76 +182,78 @@ def _make_csv(self, rows): return io.BytesIO(buf.getvalue().encode()) def test_valid_row_parsed(self): - f = self._make_csv([{'bc_id': 'C001', 'short_name': 'HR', 'definition': 'Heart Rate'}]) + f = self._make_csv([{"bc_id": "C001", "short_name": "HR", "definition": "Heart Rate"}]) records = parse_csv(f) assert len(records) == 1 - assert records[0]['mapped']['bc_id'] == 'C001' + assert records[0]["mapped"]["bc_id"] == "C001" def test_invalid_file_returns_error_record(self): - records = parse_csv(io.BytesIO(b'not,valid\ncsv content')) + records = parse_csv(io.BytesIO(b"not,valid\ncsv content")) # Should parse without exception; may produce records or errors assert isinstance(records, list) def test_missing_definition_produces_validation_error(self): - f = self._make_csv([{'bc_id': 'C001', 'short_name': 'HR'}]) + f = self._make_csv([{"bc_id": "C001", "short_name": "HR"}]) records = parse_csv(f) - assert any(records) and any('definition' in e for r in records for e in r.get('errors', [])) + assert any(records) and any("definition" in e for r in records for e in r.get("errors", [])) # --------------------------------------------------------------------------- # parse_json # --------------------------------------------------------------------------- + class TestParseJson: def test_array_of_objects(self): - data = [{'bc_id': 'C001', 'short_name': 'HR', 'definition': 'Heart Rate'}] + data = [{"bc_id": "C001", "short_name": "HR", "definition": "Heart Rate"}] f = io.BytesIO(json.dumps(data).encode()) records = parse_json(f) assert len(records) == 1 - assert records[0]['mapped']['bc_id'] == 'C001' + assert records[0]["mapped"]["bc_id"] == "C001" def test_single_object_wrapped(self): - data = {'bc_id': 'C001', 'short_name': 'HR', 'definition': 'Heart Rate'} + data = {"bc_id": "C001", "short_name": "HR", "definition": "Heart Rate"} f = io.BytesIO(json.dumps(data).encode()) records = parse_json(f) assert len(records) == 1 def test_invalid_json_returns_error(self): - records = parse_json(io.BytesIO(b'not json at all')) + records = parse_json(io.BytesIO(b"not json at all")) assert len(records) == 1 - assert records[0].get('error') or records[0].get('errors') + assert records[0].get("error") or records[0].get("errors") # --------------------------------------------------------------------------- # _group_by_bc # --------------------------------------------------------------------------- + class TestGroupByBc: def test_single_row_becomes_one_record(self): - rows = [({'bc_id': 'C001', 'short_name': 'HR', 'definition': 'Heart Rate'}, {'bc_id': 1.0})] + rows = [({"bc_id": "C001", "short_name": "HR", "definition": "Heart Rate"}, {"bc_id": 1.0})] result = _group_by_bc(rows) assert len(result) == 1 - assert result[0]['mapped']['bc_id'] == 'C001' + assert result[0]["mapped"]["bc_id"] == "C001" def test_dec_sub_rows_grouped_under_parent(self): rows = [ - ({'bc_id': 'C001', 'short_name': 'HR', 'definition': 'Heart Rate'}, {}), - ({'bc_id': 'C001', 'dec_id': 'C001.DEC.1', 'dec_label': 'Value'}, {}), + ({"bc_id": "C001", "short_name": "HR", "definition": "Heart Rate"}, {}), + ({"bc_id": "C001", "dec_id": "C001.DEC.1", "dec_label": "Value"}, {}), ] result = _group_by_bc(rows) assert len(result) == 1 - assert len(result[0]['decs']) == 1 - assert result[0]['decs'][0]['dec_label'] == 'Value' + assert len(result[0]["decs"]) == 1 + assert result[0]["decs"][0]["dec_label"] == "Value" def test_multiple_bcs_produce_separate_records(self): rows = [ - ({'bc_id': 'C001', 'short_name': 'HR', 'definition': 'Heart Rate'}, {}), - ({'bc_id': 'C002', 'short_name': 'BP', 'definition': 'Blood Pressure'}, {}), + ({"bc_id": "C001", "short_name": "HR", "definition": "Heart Rate"}, {}), + ({"bc_id": "C002", "short_name": "BP", "definition": "Blood Pressure"}, {}), ] result = _group_by_bc(rows) assert len(result) == 2 def test_row_without_bc_id_skipped(self): - rows = [({'short_name': 'Orphan', 'definition': 'No ID'}, {})] + rows = [({"short_name": "Orphan", "definition": "No ID"}, {})] result = _group_by_bc(rows) assert result == [] diff --git a/tests/test_models.py b/tests/test_models.py index bbffee1..cdd0416 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,4 +1,5 @@ """Tests for model property serialization and helper methods.""" + import pytest from models.audit import AuditLog from models.ingestion import IngestionRecord @@ -10,14 +11,14 @@ class TestAuditLogJsonProperties: def test_before_state_round_trips(self, app): with app.app_context(): log = AuditLog() - log.before_state = {'status': 'provisional', 'bc_id': 'C001'} - assert log.before_state == {'status': 'provisional', 'bc_id': 'C001'} + log.before_state = {"status": "provisional", "bc_id": "C001"} + assert log.before_state == {"status": "provisional", "bc_id": "C001"} def test_after_state_round_trips(self, app): with app.app_context(): log = AuditLog() - log.after_state = {'status': 'sme_review'} - assert log.after_state == {'status': 'sme_review'} + log.after_state = {"status": "sme_review"} + assert log.after_state == {"status": "sme_review"} def test_none_before_state_returns_none(self, app): with app.app_context(): @@ -32,42 +33,42 @@ def test_none_after_state_returns_none(self, app): def test_persisted_log_retrieves_state(self, app): with app.app_context(): log = AuditLog( - entity_type='BiomedicalConcept', - entity_id='C001', - action='created', - actor='tester', + entity_type="BiomedicalConcept", + entity_id="C001", + action="created", + actor="tester", ) - log.after_state = {'bc_id': 'C001', 'short_name': 'Test'} + log.after_state = {"bc_id": "C001", "short_name": "Test"} db.session.add(log) db.session.commit() fetched = AuditLog.query.first() - assert fetched.after_state['bc_id'] == 'C001' + assert fetched.after_state["bc_id"] == "C001" class TestIngestionRecordProperties: def test_mapped_round_trips(self, app): with app.app_context(): ir = IngestionRecord() - ir.mapped = {'bc_id': 'C001', 'short_name': 'HR'} - assert ir.mapped == {'bc_id': 'C001', 'short_name': 'HR'} + ir.mapped = {"bc_id": "C001", "short_name": "HR"} + assert ir.mapped == {"bc_id": "C001", "short_name": "HR"} def test_confidences_round_trips(self, app): with app.app_context(): ir = IngestionRecord() - ir.confidences = {'bc_id': 1.0, 'short_name': 0.9} - assert ir.confidences == {'bc_id': 1.0, 'short_name': 0.9} + ir.confidences = {"bc_id": 1.0, "short_name": 0.9} + assert ir.confidences == {"bc_id": 1.0, "short_name": 0.9} def test_errors_round_trips(self, app): with app.app_context(): ir = IngestionRecord() - ir.errors = ['short_name is required'] - assert ir.errors == ['short_name is required'] + ir.errors = ["short_name is required"] + assert ir.errors == ["short_name is required"] def test_decs_round_trips(self, app): with app.app_context(): ir = IngestionRecord() - ir.decs = [{'dec_id': 'C001.DEC.1', 'dec_label': 'Value'}] - assert ir.decs[0]['dec_label'] == 'Value' + ir.decs = [{"dec_id": "C001.DEC.1", "dec_label": "Value"}] + assert ir.decs[0]["dec_label"] == "Value" def test_empty_mapped_returns_empty_dict(self, app): with app.app_context(): @@ -77,7 +78,7 @@ def test_empty_mapped_returns_empty_dict(self, app): def test_avg_confidence_computed_correctly(self, app): with app.app_context(): ir = IngestionRecord() - ir.confidences = {'bc_id': 1.0, 'short_name': 0.8, 'definition': 0.6} + ir.confidences = {"bc_id": 1.0, "short_name": 0.8, "definition": 0.6} assert ir.avg_confidence == round((1.0 + 0.8 + 0.6) / 3 * 100) def test_avg_confidence_empty_confidences(self, app): @@ -91,26 +92,26 @@ class TestBiomedicalConceptToDict: def test_to_dict_contains_required_keys(self, app): with app.app_context(): bc = BiomedicalConcept( - bc_id='C001', - short_name='Heart Rate', - definition='Rate of the heart.', - ncit_code='C001', - status='provisional', + bc_id="C001", + short_name="Heart Rate", + definition="Rate of the heart.", + ncit_code="C001", + status="provisional", ) d = bc.to_dict() - for key in ('bc_id', 'short_name', 'definition', 'ncit_code', 'status'): + for key in ("bc_id", "short_name", "definition", "ncit_code", "status"): assert key in d def test_to_dict_values_match(self, app): with app.app_context(): - bc = BiomedicalConcept(bc_id='C002', short_name='BP', definition='Blood Pressure', ncit_code='C002') + bc = BiomedicalConcept(bc_id="C002", short_name="BP", definition="Blood Pressure", ncit_code="C002") d = bc.to_dict() - assert d['bc_id'] == 'C002' - assert d['short_name'] == 'BP' + assert d["bc_id"] == "C002" + assert d["short_name"] == "BP" def test_default_status_is_provisional(self, app): with app.app_context(): - bc = BiomedicalConcept(bc_id='C003', short_name='X', definition='Y') + bc = BiomedicalConcept(bc_id="C003", short_name="X", definition="Y") db.session.add(bc) db.session.commit() - assert bc.status == 'provisional' + assert bc.status == "provisional" From b79a61079859f2622a21362258daa9bb7db30a1a Mon Sep 17 00:00:00 2001 From: Darren <3921919+pendingintent@users.noreply.github.com> Date: Tue, 14 Apr 2026 12:11:56 -0400 Subject: [PATCH 05/36] Fixed bc deletion --- routes/bc.py | 6 +++++ templates/bc_list.html | 55 +++++++++++++++++++++++++++++++++--------- 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/routes/bc.py b/routes/bc.py index d8c0eb3..d79d99d 100644 --- a/routes/bc.py +++ b/routes/bc.py @@ -3,6 +3,7 @@ from flask import Blueprint, render_template, request, redirect, url_for, flash, Response from models.bc import BiomedicalConcept, DataElementConcept from models.audit import AuditLog +from models.governance import GovernanceRecord from extensions import db from services.export import export_json, export_xlsx, export_odm_xml from services.cdisc_api import CDISCApiClient @@ -270,6 +271,11 @@ def submit_for_review(bc_id): @bp.route("//delete", methods=["POST"]) def delete(bc_id): bc = BiomedicalConcept.query.get_or_404(bc_id) + # Nullify self-referential parent FK on child BCs; without this SQLAlchemy + # raises CircularDependencyError when flushing the delete. + BiomedicalConcept.query.filter_by(parent_bc_id=bc_id).update({"parent_bc_id": None}, synchronize_session="fetch") + # GovernanceRecord.bc_id is NOT NULL with no ORM cascade, so delete explicitly. + GovernanceRecord.query.filter_by(bc_id=bc_id).delete(synchronize_session="fetch") log = AuditLog( entity_type="BiomedicalConcept", entity_id=bc_id, diff --git a/templates/bc_list.html b/templates/bc_list.html index f102a68..09b2bdf 100644 --- a/templates/bc_list.html +++ b/templates/bc_list.html @@ -132,17 +132,15 @@

Biomedical Concepts

aria-label="View/Edit {{ bc.short_name }}"> Edit -
- {{ form.hidden_tag() if form else '' }} - - -
+
@@ -202,4 +200,39 @@

Biomedical Concepts

{% endif %} + + + +{% endblock %} + +{% block extra_js %} + {% endblock %} From a098586f21dee2708e337e1791f9d1e09e07ea14 Mon Sep 17 00:00:00 2001 From: Darren <3921919+pendingintent@users.noreply.github.com> Date: Tue, 14 Apr 2026 13:38:28 -0400 Subject: [PATCH 06/36] Added bc selection to specializations; Fixed specialization search; added pagination to dashboard --- routes/dashboard.py | 4 +- routes/specializations.py | 36 ++++++++++++-- services/cdisc_api.py | 9 ++-- templates/bc_detail.html | 4 +- templates/dashboard.html | 86 ++++++++++++++++++++++++++-------- templates/specializations.html | 68 +++++++++++++++++++++------ 6 files changed, 162 insertions(+), 45 deletions(-) diff --git a/routes/dashboard.py b/routes/dashboard.py index 6939ecf..8c916cc 100644 --- a/routes/dashboard.py +++ b/routes/dashboard.py @@ -42,8 +42,8 @@ def index(): return render_template( "dashboard.html", stats=stats, - api_bcs=api_bcs[:50] if not api_bc_error else [], - api_specs=api_specs[:50] if not api_spec_error else [], + api_bcs=sorted(api_bcs, key=lambda x: (x.get("title") or "").lower()) if not api_bc_error else [], + api_specs=sorted(api_specs, key=lambda x: (x.get("title") or "").lower()) if not api_spec_error else [], api_bc_error=api_bc_error, api_spec_error=api_spec_error, api_bc_count=api_bc_count, diff --git a/routes/specializations.py b/routes/specializations.py index af90ad7..30a5cc0 100644 --- a/routes/specializations.py +++ b/routes/specializations.py @@ -1,4 +1,4 @@ -from flask import Blueprint, render_template, request, redirect, url_for, flash +from flask import Blueprint, render_template, request, redirect, url_for, flash, jsonify from models.bc import BiomedicalConcept, DataElementConcept from models.specialization import DatasetSpecialization from extensions import db @@ -7,14 +7,27 @@ bp = Blueprint("specializations", __name__) +def _get_bc_options(): + """Return (library_bcs, local_bcs) for the BC selector. + + library_bcs — list of dicts with bc_id/short_name from the CDISC Library (cached). + local_bcs — BiomedicalConcept ORM objects from the local governance pipeline. + """ + links = CDISCApiClient().get_biomedical_concepts() + library_bcs = [{"bc_id": lnk["href"].rstrip("/").split("/")[-1], "short_name": lnk.get("title", "")} for lnk in links if "href" in lnk and "error" not in lnk] + local_bcs = BiomedicalConcept.query.order_by(BiomedicalConcept.short_name).all() + return library_bcs, local_bcs + + @bp.route("/") def index(): specs = DatasetSpecialization.query.all() - bcs = BiomedicalConcept.query.order_by(BiomedicalConcept.short_name).all() + library_bcs, local_bcs = _get_bc_options() return render_template( "specializations.html", specs=specs, - bcs=bcs, + library_bcs=library_bcs, + local_bcs=local_bcs, page_title="Specializations", ) @@ -37,11 +50,12 @@ def library_detail(spec_path): def detail(vlm_group_id): spec = DatasetSpecialization.query.get_or_404(vlm_group_id) specs = DatasetSpecialization.query.all() - bcs = BiomedicalConcept.query.order_by(BiomedicalConcept.short_name).all() + library_bcs, local_bcs = _get_bc_options() return render_template( "specializations.html", specs=specs, - bcs=bcs, + library_bcs=library_bcs, + local_bcs=local_bcs, edit_spec=spec, page_title="Specializations", ) @@ -67,6 +81,18 @@ def create(): return redirect(url_for("specializations.index")) +@bp.route("/generate-from-dec", methods=["POST"]) +def generate_from_dec(): + """Return DEC-derived variable rows as JSON for the specialization form.""" + data = request.get_json(silent=True) or {} + bc_id = data.get("bc_id", "").strip() + if not bc_id: + return jsonify({"error": "bc_id required"}), 400 + decs = DataElementConcept.query.filter_by(bc_id=bc_id).all() + variables = [{"name": d.dec_label or "", "label": d.dec_label or "", "data_type": d.data_type or "string", "required": bool(d.required)} for d in decs] + return jsonify({"variables": variables}) + + @bp.route("/generate/", methods=["POST"]) def generate(bc_id): """Generate a specialization from DEC templates for a BC.""" diff --git a/services/cdisc_api.py b/services/cdisc_api.py index da35780..a90ec17 100644 --- a/services/cdisc_api.py +++ b/services/cdisc_api.py @@ -8,7 +8,7 @@ # Entries are never evicted — stale data is served while a refresh is attempted, # so a timeout never blocks the request with an empty response. _cache = {} -_CACHE_TTL = 300 # serve fresh data for 5 minutes +_CACHE_TTL = 300 # serve fresh data for 5 minutes _CACHE_STALE_TTL = 3600 # serve stale data for up to 1 hour while refresh fails @@ -102,8 +102,11 @@ def get_dataset_specializations(self): def _fetch(): try: data = self._get("/mdr/specializations/datasetspecializations") - sdtm = data.get("_links", {}).get("datasetSpecializations", {}).get("sdtm", []) - return sdtm + links = data.get("_links", {}).get("datasetSpecializations", []) + # API returns either a flat list or a domain-keyed dict (sdtm/cdash/…) + if isinstance(links, list): + return links + return [item for v in links.values() if isinstance(v, list) for item in v] except Exception as e: return [{"error": str(e)}] diff --git a/templates/bc_detail.html b/templates/bc_detail.html index 419bc17..09028c0 100644 --- a/templates/bc_detail.html +++ b/templates/bc_detail.html @@ -374,8 +374,8 @@

Data Element Concepts (DECs)

{% endblock %} -{% if needs_loinc_fetch or needs_ncit_fetch %} {% block extra_js %} +{% if bc.bc_id and (needs_loinc_fetch or needs_ncit_fetch) %} -{% endblock %} {% endif %} +{% endblock %} diff --git a/templates/dashboard.html b/templates/dashboard.html index 4964918..b06c72b 100644 --- a/templates/dashboard.html +++ b/templates/dashboard.html @@ -206,9 +206,17 @@

{% endif %} - {% if api_bc_count > 50 %} -

Showing 50 of {{ api_bc_count }} BCs.

- {% endif %} +
+ +
+ + +
+

@@ -256,9 +264,17 @@

{% endif %} - {% if api_spec_count > 50 %} -

Showing 50 of {{ api_spec_count }} specializations.

- {% endif %} +
+ +
+ + +
+
@@ -324,21 +340,53 @@

Quick Actions

{% endblock %} diff --git a/templates/specializations.html b/templates/specializations.html index ff8ebc4..c6f8daf 100644 --- a/templates/specializations.html +++ b/templates/specializations.html @@ -21,7 +21,7 @@

Dataset Specializations

- {% if editing_spec %}Edit Specialization{% else %}New Specialization{% endif %} + {% if edit_spec %}Edit Specialization{% else %}New Specialization{% endif %}

@@ -30,19 +30,43 @@

novalidate> {{ form.hidden_tag() if form else '' }} +
+
+ + +
+
+
+
+
@@ -53,12 +77,12 @@

+ {% if not edit_spec or edit_spec.domain == 'SDTM' %}checked{% endif %}>
+ {% if edit_spec and edit_spec.domain == 'CDASH' %}checked{% endif %}>
@@ -68,7 +92,7 @@

@@ -86,9 +110,9 @@

Variables

- - {% if editing_spec and editing_spec.variables %} - {% for var in editing_spec.variables %} + + {% if edit_spec and edit_spec.variables %} + {% for var in edit_spec.variables %} All Specializations From fa9a0d11532743cd9e9e73433b4d534f1d25c036 Mon Sep 17 00:00:00 2001 From: Darren <3921919+pendingintent@users.noreply.github.com> Date: Wed, 15 Apr 2026 14:07:26 -0400 Subject: [PATCH 09/36] Fixed LOINC search functionality --- app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app.py b/app.py index 7e08e32..cc46a8a 100644 --- a/app.py +++ b/app.py @@ -23,6 +23,7 @@ def create_app(config_class=Config): app.register_blueprint(ingestion_bp, url_prefix="/ingestion") app.register_blueprint(bc_bp, url_prefix="/bc") app.register_blueprint(ncit_bp, url_prefix="/ncit") + app.register_blueprint(loinc_bp, url_prefix="/loinc") app.register_blueprint(specializations_bp, url_prefix="/specializations") app.register_blueprint(governance_bp, url_prefix="/governance") app.register_blueprint(audit_bp, url_prefix="/audit") From dfeb97fa52fbe92c2dc54ffc2810fa82bc2795dd Mon Sep 17 00:00:00 2001 From: Darren <3921919+pendingintent@users.noreply.github.com> Date: Wed, 15 Apr 2026 15:14:59 -0400 Subject: [PATCH 10/36] Auto-search for assigned LOINC code in BC --- README.md | 4 +- .../a1c3e5f7b9d2_rename_code_to_loinc_code.py | 26 +++++ models/bc.py | 6 +- routes/bc.py | 15 ++- templates/bc_detail.html | 104 ++++++++++++------ 5 files changed, 115 insertions(+), 40 deletions(-) create mode 100644 migrations/versions/a1c3e5f7b9d2_rename_code_to_loinc_code.py diff --git a/README.md b/README.md index 3b23a2a..691b6c2 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ The sidebar navigation exposes seven screens, accessible at these URL prefixes: |--------|-----|-------------| | Dashboard | `/` | KPI cards (total BCs, pending review, published), governance pipeline chart with concurrent CDISC API fetches (ThreadPoolExecutor), recent submissions table | | Ingestion | `/ingestion` | Upload XLSX, CSV, or JSON files; AI field mapper assigns confidence scores; approve or reject rows to the database | -| BCs | `/bc` | Browse, create, edit, and delete Biomedical Concepts; LOINC code entry with live search and automatic metadata population from NLM Clinical Tables API (LONG_COMMON_NAME, SHORTNAME, COMPONENT, units, etc.); NCIt concept selection with live search and one-click integration — click "Use this concept" to fetch full metadata asynchronously (preferred name, synonyms, description, parent concepts, child concepts, semantic type, and NCIt Browser link); all available definitions displayed with source attribution as `[SOURCE] definition text` in the References section; query parameters `/bc/new?ncit_code=...&ncit_name=...&ncit_definition=...` pre-populate BC fields on page load; `parent_bc_id` auto-filled from first parent concept's code; Data Element Concept sub-records | +| BCs | `/bc` | Browse, create, edit, and delete Biomedical Concepts; LOINC code entry with asynchronous metadata fetch on demand — click "Search LOINC" to fetch full LOINC metadata from NLM Clinical Tables API (LONG_COMMON_NAME, SHORTNAME, COMPONENT, METHOD_TYP, units, datatype, copyright notices, etc.), auto-populate Long Common Name, and store in hidden metadata field; NCIt concept selection with live search and one-click integration — click "Use this concept" to fetch full metadata asynchronously (preferred name, synonyms, description, parent concepts, child concepts, semantic type, and NCIt Browser link); all available definitions displayed with source attribution as `[SOURCE] definition text` in the References section; query parameters `/bc/new?ncit_code=...&ncit_name=...&ncit_definition=...` pre-populate BC fields on page load; `parent_bc_id` auto-filled from first parent concept's code; Data Element Concept sub-records | | NCIT Mapping | `/ncit` | Search the NCI Thesaurus, resolve low-confidence mappings, and confirm NCIt codes for each BC | | Specializations | `/specializations` | View and generate SDTM/CDASH dataset specializations and CRF variable mappings | | Governance | `/governance` | 4-stage Kanban board (Provisional > SME Review > CDISC Approval > Published) with advance and reject actions | @@ -180,4 +180,4 @@ The platform integrates with three external APIs to provide rich concept metadat - **CDISC Library** (`https://api.library.cdisc.org/api/cosmos/v2`) — Requires `CDISC_API_KEY`. Used in Dashboard and BC Library detail views. Implements stale-while-refresh caching (5-min fresh TTL, 1-hour stale fallback) to gracefully handle transient failures. - **NCI EVS REST API** (`https://api-evsrest.nci.nih.gov/api/v1`) — No authentication required. Returns NCIt concept definitions with source attribution, parent and child concepts, semantic types, and browser links. Search requests use `include=summary` for richer metadata. Integrated into BC detail views via `/bc/fetch_metadata` endpoint with on-demand asynchronous full-concept fetch (no LOINC concurrent fetches). Prioritizes definitions by source using `_pick_definition()` helper: CDISC > NCI > first available. Displays all available definitions with source attribution as `[SOURCE] definition text` in the References section. Displays parent and child concepts with codes. Includes direct links to the NCIt Browser (via `https://ncithesaurus.nci.nih.gov/ncitbrowser/ConceptReport.jsp?dictionary=NCI_Thesaurus&code=`) for each concept. Auto-fills `parent_bc_id` from first parent concept code. Implements in-memory caching (5-min fresh TTL, 1-hour stale fallback) to serve cached data rapidly and degrade gracefully when the service is unavailable. -- **NLM Clinical Tables API (LOINC)** (`https://clinicaltables.nlm.nih.gov/api/loinc_items/v3/search`) — Optional Basic Auth via `LOINC_USER` / `LOINC_PASSWORD`. Returns LOINC metadata including LONG_COMMON_NAME, SHORTNAME, COMPONENT, METHOD_TYP, units, datatype, and copyright notices. Integrated into BC detail views via `/loinc/search` endpoint. +- **NLM Clinical Tables API (LOINC)** (`https://clinicaltables.nlm.nih.gov/api/loinc_items/v3/search`) — Optional Basic Auth via `LOINC_USER` / `LOINC_PASSWORD`. Returns LOINC metadata including LONG_COMMON_NAME, SHORTNAME, COMPONENT, METHOD_TYP, units, datatype, and copyright notices. Integrated into BC detail views via `/bc/fetch_metadata` endpoint with on-demand asynchronous metadata fetch — when a LOINC code is set and no cached metadata exists, the fetch triggers automatically on page load; metadata is cached in the database and formatted in a grid display (Long Common Name, Short Name, Component, Property, Method Type, Units, Data Type, Consumer Name, Related Names, Answer Lists, copyright status, and links). diff --git a/migrations/versions/a1c3e5f7b9d2_rename_code_to_loinc_code.py b/migrations/versions/a1c3e5f7b9d2_rename_code_to_loinc_code.py new file mode 100644 index 0000000..b8f24c9 --- /dev/null +++ b/migrations/versions/a1c3e5f7b9d2_rename_code_to_loinc_code.py @@ -0,0 +1,26 @@ +"""rename code to loinc_code in biomedical_concepts + +Revision ID: a1c3e5f7b9d2 +Revises: b9ee22a174fe +Create Date: 2026-04-15 14:00:00.000000 + +""" + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = "a1c3e5f7b9d2" +down_revision = "b9ee22a174fe" +branch_labels = None +depends_on = None + + +def upgrade(): + with op.batch_alter_table("biomedical_concepts", schema=None) as batch_op: + batch_op.alter_column("code", new_column_name="loinc_code", existing_type=sa.String(50), existing_nullable=True) + + +def downgrade(): + with op.batch_alter_table("biomedical_concepts", schema=None) as batch_op: + batch_op.alter_column("loinc_code", new_column_name="code", existing_type=sa.String(50), existing_nullable=True) diff --git a/models/bc.py b/models/bc.py index 4eca5c3..91c116f 100644 --- a/models/bc.py +++ b/models/bc.py @@ -13,8 +13,8 @@ class BiomedicalConcept(db.Model): synonyms = db.Column(db.Text) result_scales = db.Column(db.String(255)) # e.g. "Quantitative; Ordinal" system = db.Column(db.String(255)) # e.g. http://loinc.org/ - system_name = db.Column(db.String(100)) # LOINC LONG_COMMON_NAME - code = db.Column(db.String(50)) # LOINC_NUM + system_name = db.Column(db.String(100)) # coding system name e.g. LOINC + loinc_code = db.Column(db.String(50)) # LOINC_NUM loinc_metadata = db.Column(db.Text) # JSON blob of all LOINC ef fields ncit_metadata = db.Column(db.Text) # JSON blob of NCIt concept detail package_date = db.Column(db.String(20)) @@ -41,7 +41,7 @@ def to_dict(self): "result_scales": self.result_scales, "system": self.system, "system_name": self.system_name, - "code": self.code, + "loinc_code": self.loinc_code, "package_date": self.package_date, "status": self.status, "submitter": self.submitter, diff --git a/routes/bc.py b/routes/bc.py index 33cb0e1..6d5ade6 100644 --- a/routes/bc.py +++ b/routes/bc.py @@ -134,6 +134,7 @@ def detail(bc_id): loinc_data=loinc_data, ncit_data=ncit_data, needs_ncit_fetch=not ncit_data and bool(bc.ncit_code), + needs_loinc_fetch=not bc.loinc_metadata and bool(bc.loinc_code), page_title=bc.short_name, ) @@ -158,7 +159,15 @@ def _fetch_ncit(): bc.ncit_metadata = json.dumps(ncit_data) db.session.commit() - return jsonify(ncit=ncit_data) + loinc_data = {} + if bc.loinc_code and not bc.loinc_metadata: + results = LoincApiClient().search(bc.loinc_code, size=1) + if results and not results[0].get("error"): + loinc_data = results[0] + bc.loinc_metadata = json.dumps(loinc_data) + db.session.commit() + + return jsonify(ncit=ncit_data, loinc=loinc_data) @bp.route("/", methods=["POST"]) @@ -181,7 +190,7 @@ def create(): result_scales=request.form.get("result_scales", ""), system=request.form.get("system", ""), system_name=request.form.get("system_name", ""), - code=request.form.get("code", ""), + loinc_code=request.form.get("loinc_code", ""), loinc_metadata=request.form.get("loinc_metadata", "") or None, ncit_metadata=request.form.get("ncit_metadata", "") or None, package_date=request.form.get("package_date", ""), @@ -216,7 +225,7 @@ def edit(bc_id): bc.result_scales = request.form.get("result_scales", bc.result_scales) bc.system = request.form.get("system", bc.system) bc.system_name = request.form.get("system_name", bc.system_name) - bc.code = request.form.get("code", bc.code) + bc.loinc_code = request.form.get("loinc_code", bc.loinc_code) bc.loinc_metadata = request.form.get("loinc_metadata", "") or bc.loinc_metadata bc.ncit_metadata = request.form.get("ncit_metadata", "") or bc.ncit_metadata bc.package_date = request.form.get("package_date", bc.package_date) diff --git a/templates/bc_detail.html b/templates/bc_detail.html index fc975d7..4e6f874 100644 --- a/templates/bc_detail.html +++ b/templates/bc_detail.html @@ -210,15 +210,16 @@

-
- +
@@ -388,9 +389,12 @@

Data Element Concepts (DECs)

{% endblock %} {% block extra_js %} -{% if bc.bc_id and needs_ncit_fetch %} +{% if bc.bc_id and (needs_ncit_fetch or needs_loinc_fetch) %} From 9e22373ea48b94cc71363471fad30cda6c011966 Mon Sep 17 00:00:00 2001 From: Darren <3921919+pendingintent@users.noreply.github.com> Date: Wed, 15 Apr 2026 16:13:51 -0400 Subject: [PATCH 11/36] Fixed failing unit tests --- CLAUDE.md | 9 +- models/bc.py | 1 + results.txt | 264 ++++++++++++++++++++++++++++++++++++++++ routes/bc.py | 4 + tests/test_bc_routes.py | 4 +- tests/test_ncit.py | 138 +++++++++++---------- 6 files changed, 351 insertions(+), 69 deletions(-) create mode 100644 results.txt diff --git a/CLAUDE.md b/CLAUDE.md index 23604b1..ae3707e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,7 +31,7 @@ black --check . Line length is set to 200 (black) / 999 (flake8). Flake8 ignores F401, F841, E711 — see `.flake8` for full ignore list. -There is no test suite. +All tests are defined in the tests directory. ## Architecture @@ -41,6 +41,7 @@ There is no test suite. - `models/` — SQLAlchemy ORM models - `templates/` — Jinja2 + Bootstrap 5 - `extensions.py` — Shared `db` and `migrate` instances (avoids circular imports — always import from here) +- `tests/` - Unit tests **7 blueprints** registered in `app.py`: @@ -81,3 +82,9 @@ All configuration is in `config.py` via environment variables: CDISC API base: `https://api.library.cdisc.org/api/cosmos/v2` NCIt API base: `https://api-evsrest.nci.nih.gov/api/v1` + + +## Conventions +- Always follow test driven development principles. +- When updating Python code, ensure the associated unit tests are updated accordingly. +- Execute the appropriate unit tests after updating Python code. \ No newline at end of file diff --git a/models/bc.py b/models/bc.py index 91c116f..7e15423 100644 --- a/models/bc.py +++ b/models/bc.py @@ -15,6 +15,7 @@ class BiomedicalConcept(db.Model): system = db.Column(db.String(255)) # e.g. http://loinc.org/ system_name = db.Column(db.String(100)) # coding system name e.g. LOINC loinc_code = db.Column(db.String(50)) # LOINC_NUM + code = db.Column(db.String(50)) # generic coding value from ingestion loinc_metadata = db.Column(db.Text) # JSON blob of all LOINC ef fields ncit_metadata = db.Column(db.Text) # JSON blob of NCIt concept detail package_date = db.Column(db.String(20)) diff --git a/results.txt b/results.txt new file mode 100644 index 0000000..9125db1 --- /dev/null +++ b/results.txt @@ -0,0 +1,264 @@ +============================= test session starts ============================== +platform darwin -- Python 3.14.0, pytest-8.3.5, pluggy-1.6.0 -- /Users/dmoreland/projects/cdisc-concept-curation/.venv/bin/python3.14 +cachedir: .pytest_cache +rootdir: /Users/dmoreland/projects/cdisc-concept-curation +configfile: pyproject.toml +plugins: flask-1.3.0 +collecting ... collected 136 items + +tests/test_audit_routes.py::TestAuditIndex::test_returns_200_empty PASSED [ 0%] +tests/test_audit_routes.py::TestAuditIndex::test_shows_log_entries PASSED [ 1%] +tests/test_audit_routes.py::TestAuditIndex::test_filter_by_entity_type PASSED [ 2%] +tests/test_audit_routes.py::TestAuditIndex::test_filter_by_action PASSED [ 2%] +tests/test_audit_routes.py::TestAuditIndex::test_filter_by_actor PASSED [ 3%] +tests/test_audit_routes.py::TestAuditIndex::test_pagination_param_accepted PASSED [ 4%] +tests/test_bc_routes.py::TestBcIndex::test_returns_200 PASSED [ 5%] +tests/test_bc_routes.py::TestBcIndex::test_search_by_name PASSED [ 5%] +tests/test_bc_routes.py::TestBcIndex::test_search_no_match PASSED [ 6%] +tests/test_bc_routes.py::TestBcIndex::test_filter_by_status PASSED [ 7%] +tests/test_bc_routes.py::TestNewBc::test_returns_200 PASSED [ 8%] +tests/test_bc_routes.py::TestCreateBc::test_creates_bc_and_redirects PASSED [ 8%] +tests/test_bc_routes.py::TestCreateBc::test_missing_bc_id_redirects_with_error PASSED [ 9%] +tests/test_bc_routes.py::TestCreateBc::test_duplicate_bc_id_rejected PASSED [ 10%] +tests/test_bc_routes.py::TestCreateBc::test_create_writes_audit_log PASSED [ 11%] +tests/test_bc_routes.py::TestCreateBc::test_create_with_decs PASSED [ 11%] +tests/test_bc_routes.py::TestBcDetail::test_existing_bc_returns_200 PASSED [ 12%] +tests/test_bc_routes.py::TestBcDetail::test_missing_bc_returns_404 PASSED [ 13%] +tests/test_bc_routes.py::TestBcDetail::test_loinc_api_called_when_code_set PASSED [ 13%] +tests/test_bc_routes.py::TestBcDetail::test_loinc_api_not_called_when_no_code PASSED [ 14%] +tests/test_bc_routes.py::TestBcDetail::test_loinc_api_error_does_not_break_page PASSED [ 15%] +tests/test_bc_routes.py::TestEditBc::test_updates_short_name PASSED [ 16%] +tests/test_bc_routes.py::TestEditBc::test_edit_writes_audit_log PASSED [ 16%] +tests/test_bc_routes.py::TestEditBc::test_nonexistent_bc_returns_404 PASSED [ 17%] +tests/test_bc_routes.py::TestSubmitForReview::test_advances_status_to_sme_review PASSED [ 18%] +tests/test_bc_routes.py::TestSubmitForReview::test_submit_writes_audit_log PASSED [ 19%] +tests/test_bc_routes.py::TestDeleteBc::test_deletes_bc PASSED [ 19%] +tests/test_bc_routes.py::TestDeleteBc::test_delete_writes_audit_log PASSED [ 20%] +tests/test_bc_routes.py::TestDeleteBc::test_nonexistent_bc_returns_404 PASSED [ 21%] +tests/test_bc_routes.py::TestExport::test_json_export PASSED [ 22%] +tests/test_bc_routes.py::TestExport::test_xlsx_export PASSED [ 22%] +tests/test_bc_routes.py::TestExport::test_odm_xml_export PASSED [ 23%] +tests/test_bc_routes.py::TestLibraryDetail::test_renders_page_for_valid_concept PASSED [ 24%] +tests/test_bc_routes.py::TestLibraryDetail::test_redirects_on_api_error PASSED [ 25%] +tests/test_bc_routes.py::TestLibraryDetail::test_loinc_api_called_when_loinc_coding_present PASSED [ 25%] +tests/test_bc_routes.py::TestLibraryDetail::test_loinc_api_not_called_when_no_loinc_coding PASSED [ 26%] +tests/test_bc_routes.py::TestLibraryDetail::test_loinc_api_error_does_not_break_page PASSED [ 27%] +tests/test_governance_routes.py::TestGovernanceBoard::test_board_returns_200 PASSED [ 27%] +tests/test_governance_routes.py::TestAdvance::test_advances_provisional_to_sme_review PASSED [ 28%] +tests/test_governance_routes.py::TestAdvance::test_advance_through_all_stages PASSED [ 29%] +tests/test_governance_routes.py::TestAdvance::test_already_published_stays_published PASSED [ 30%] +tests/test_governance_routes.py::TestAdvance::test_advance_creates_governance_record PASSED [ 30%] +tests/test_governance_routes.py::TestAdvance::test_advance_writes_audit_log PASSED [ 31%] +tests/test_governance_routes.py::TestAdvance::test_advance_nonexistent_bc_returns_404 PASSED [ 32%] +tests/test_governance_routes.py::TestAdvance::test_advance_ajax_returns_json PASSED [ 33%] +tests/test_governance_routes.py::TestReject::test_reject_returns_to_provisional PASSED [ 33%] +tests/test_governance_routes.py::TestReject::test_reject_creates_governance_record PASSED [ 34%] +tests/test_governance_routes.py::TestReject::test_reject_writes_audit_log PASSED [ 35%] +tests/test_governance_routes.py::TestReject::test_reject_ajax_returns_json PASSED [ 36%] +tests/test_governance_routes.py::TestReject::test_reject_nonexistent_bc_returns_404 PASSED [ 36%] +tests/test_ingestion_routes.py::TestIngestionIndex::test_returns_200 PASSED [ 37%] +tests/test_ingestion_routes.py::TestUpload::test_upload_csv_creates_ingestion_records PASSED [ 38%] +tests/test_ingestion_routes.py::TestUpload::test_upload_json_creates_ingestion_records PASSED [ 38%] +tests/test_ingestion_routes.py::TestUpload::test_no_file_redirects_with_error PASSED [ 39%] +tests/test_ingestion_routes.py::TestUpload::test_wrong_extension_rejected PASSED [ 40%] +tests/test_ingestion_routes.py::TestUpload::test_duplicate_bc_flagged PASSED [ 41%] +tests/test_ingestion_routes.py::TestApprove::test_approve_creates_bc PASSED [ 41%] +tests/test_ingestion_routes.py::TestApprove::test_approve_sets_status_approved PASSED [ 42%] +tests/test_ingestion_routes.py::TestApprove::test_approve_nonexistent_record_returns_404 PASSED [ 43%] +tests/test_ingestion_routes.py::TestApprove::test_approve_already_existing_bc_does_not_duplicate PASSED [ 44%] +tests/test_ingestion_routes.py::TestReject::test_reject_sets_status_rejected PASSED [ 44%] +tests/test_ingestion_routes.py::TestReject::test_reject_does_not_create_bc PASSED [ 45%] +tests/test_ingestion_routes.py::TestReject::test_reject_nonexistent_record_returns_404 PASSED [ 46%] +tests/test_ingestion_routes.py::TestApproveAll::test_approve_all_skips_records_with_errors PASSED [ 47%] +tests/test_ingestion_service.py::TestSimilarity::test_identical_strings PASSED [ 47%] +tests/test_ingestion_service.py::TestSimilarity::test_case_insensitive PASSED [ 48%] +tests/test_ingestion_service.py::TestSimilarity::test_completely_different PASSED [ 49%] +tests/test_ingestion_service.py::TestSimilarity::test_partial_match PASSED [ 50%] +tests/test_ingestion_service.py::TestMatchField::test_exact_alias_match PASSED [ 50%] +tests/test_ingestion_service.py::TestMatchField::test_known_alias PASSED [ 51%] +tests/test_ingestion_service.py::TestMatchField::test_definition_alias PASSED [ 52%] +tests/test_ingestion_service.py::TestMatchField::test_spaces_normalised PASSED [ 52%] +tests/test_ingestion_service.py::TestMatchField::test_unknown_column_returns_low_score PASSED [ 53%] +tests/test_ingestion_service.py::TestMapFields::test_canonical_columns_mapped_at_full_confidence PASSED [ 54%] +tests/test_ingestion_service.py::TestMapFields::test_none_values_skipped PASSED [ 55%] +tests/test_ingestion_service.py::TestMapFields::test_nan_values_skipped PASSED [ 55%] +tests/test_ingestion_service.py::TestMapFields::test_low_confidence_columns_excluded PASSED [ 56%] +tests/test_ingestion_service.py::TestMapFields::test_values_stripped PASSED [ 57%] +tests/test_ingestion_service.py::TestValidateBc::test_valid_record_has_no_errors PASSED [ 58%] +tests/test_ingestion_service.py::TestValidateBc::test_missing_short_name PASSED [ 58%] +tests/test_ingestion_service.py::TestValidateBc::test_missing_definition PASSED [ 59%] +tests/test_ingestion_service.py::TestValidateBc::test_missing_both_ids PASSED [ 60%] +tests/test_ingestion_service.py::TestValidateBc::test_invalid_ncit_format PASSED [ 61%] +tests/test_ingestion_service.py::TestValidateBc::test_ncit_starting_with_c_accepted PASSED [ 61%] +tests/test_ingestion_service.py::TestDeduplicate::test_marks_existing_ids_as_duplicate PASSED [ 62%] +tests/test_ingestion_service.py::TestDeduplicate::test_case_insensitive_comparison PASSED [ 63%] +tests/test_ingestion_service.py::TestDeduplicate::test_empty_existing_ids PASSED [ 63%] +tests/test_ingestion_service.py::TestDeduplicate::test_falls_back_to_ncit_code PASSED [ 64%] +tests/test_ingestion_service.py::TestParseCsv::test_valid_row_parsed PASSED [ 65%] +tests/test_ingestion_service.py::TestParseCsv::test_invalid_file_returns_error_record PASSED [ 66%] +tests/test_ingestion_service.py::TestParseCsv::test_missing_definition_produces_validation_error PASSED [ 66%] +tests/test_ingestion_service.py::TestParseJson::test_array_of_objects PASSED [ 67%] +tests/test_ingestion_service.py::TestParseJson::test_single_object_wrapped PASSED [ 68%] +tests/test_ingestion_service.py::TestParseJson::test_invalid_json_returns_error PASSED [ 69%] +tests/test_ingestion_service.py::TestGroupByBc::test_single_row_becomes_one_record PASSED [ 69%] +tests/test_ingestion_service.py::TestGroupByBc::test_dec_sub_rows_grouped_under_parent PASSED [ 70%] +tests/test_ingestion_service.py::TestGroupByBc::test_multiple_bcs_produce_separate_records PASSED [ 71%] +tests/test_ingestion_service.py::TestGroupByBc::test_row_without_bc_id_skipped PASSED [ 72%] +tests/test_loinc.py::TestLoincApiClientSearch::test_returns_normalized_list PASSED [ 72%] +tests/test_loinc.py::TestLoincApiClientSearch::test_all_ef_fields_present_in_result PASSED [ 73%] +tests/test_loinc.py::TestLoincApiClientSearch::test_uses_ef_parameter PASSED [ 74%] +tests/test_loinc.py::TestLoincApiClientSearch::test_ef_fields_constant_contains_all_required_fields PASSED [ 75%] +tests/test_loinc.py::TestLoincApiClientSearch::test_uses_basic_auth_when_env_vars_set PASSED [ 75%] +tests/test_loinc.py::TestLoincApiClientSearch::test_no_auth_when_env_vars_missing PASSED [ 76%] +tests/test_loinc.py::TestLoincApiClientSearch::test_empty_results PASSED [ 77%] +tests/test_loinc.py::TestLoincApiClientSearch::test_missing_codes_array_returns_empty PASSED [ 77%] +tests/test_loinc.py::TestLoincApiClientSearch::test_network_error_returns_error_entry PASSED [ 78%] +tests/test_loinc.py::TestLoincSearchRoute::test_returns_json_for_ajax PASSED [ 79%] +tests/test_loinc.py::TestLoincSearchRoute::test_empty_term_returns_empty_list PASSED [ 80%] +tests/test_loinc.py::TestLoincSearchRoute::test_calls_client_with_term PASSED [ 80%] +tests/test_loinc.py::TestLoincSearchRoute::test_format_json_param_triggers_json_response PASSED [ 81%] +tests/test_models.py::TestAuditLogJsonProperties::test_before_state_round_trips PASSED [ 82%] +tests/test_models.py::TestAuditLogJsonProperties::test_after_state_round_trips PASSED [ 83%] +tests/test_models.py::TestAuditLogJsonProperties::test_none_before_state_returns_none PASSED [ 83%] +tests/test_models.py::TestAuditLogJsonProperties::test_none_after_state_returns_none PASSED [ 84%] +tests/test_models.py::TestAuditLogJsonProperties::test_persisted_log_retrieves_state PASSED [ 85%] +tests/test_models.py::TestIngestionRecordProperties::test_mapped_round_trips PASSED [ 86%] +tests/test_models.py::TestIngestionRecordProperties::test_confidences_round_trips PASSED [ 86%] +tests/test_models.py::TestIngestionRecordProperties::test_errors_round_trips PASSED [ 87%] +tests/test_models.py::TestIngestionRecordProperties::test_decs_round_trips PASSED [ 88%] +tests/test_models.py::TestIngestionRecordProperties::test_empty_mapped_returns_empty_dict PASSED [ 88%] +tests/test_models.py::TestIngestionRecordProperties::test_avg_confidence_computed_correctly PASSED [ 89%] +tests/test_models.py::TestIngestionRecordProperties::test_avg_confidence_empty_confidences PASSED [ 90%] +tests/test_models.py::TestBiomedicalConceptToDict::test_to_dict_contains_required_keys PASSED [ 91%] +tests/test_models.py::TestBiomedicalConceptToDict::test_to_dict_values_match PASSED [ 91%] +tests/test_models.py::TestBiomedicalConceptToDict::test_default_status_is_provisional PASSED [ 92%] +tests/test_ncit.py::TestNcitGetConceptExtended::test_returns_parents PASSED [ 93%] +tests/test_ncit.py::TestNcitGetConceptExtended::test_returns_semantic_type PASSED [ 94%] +tests/test_ncit.py::TestNcitGetConceptExtended::test_returns_source_synonyms PASSED [ 94%] +tests/test_ncit.py::TestNcitGetConceptExtended::test_returns_all_definitions PASSED [ 95%] +tests/test_ncit.py::TestNcitGetConceptExtended::test_empty_parents_returns_empty_list PASSED [ 96%] +tests/test_ncit.py::TestNcitGetConceptExtended::test_missing_semantic_type_returns_empty_list PASSED [ 97%] +tests/test_ncit.py::TestNcitGetConceptExtended::test_error_returns_error_dict PASSED [ 97%] +tests/test_ncit.py::TestNcitConceptRoute::test_returns_json PASSED [ 98%] +tests/test_ncit.py::TestNcitConceptRoute::test_calls_get_concept_with_code PASSED [ 99%] +tests/test_ncit.py::TestNcitConceptRoute::test_error_from_service_returns_500 PASSED [100%] + +=============================== warnings summary =============================== +tests/test_audit_routes.py: 8 warnings +tests/test_bc_routes.py: 49 warnings +tests/test_governance_routes.py: 52 warnings +tests/test_ingestion_routes.py: 19 warnings +tests/test_models.py: 3 warnings + /Users/dmoreland/projects/cdisc-concept-curation/.venv/lib/python3.14/site-packages/sqlalchemy/sql/schema.py:3624: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + return util.wrap_callable(lambda ctx: fn(), fn) # type: ignore + +tests/test_bc_routes.py::TestCreateBc::test_creates_bc_and_redirects +tests/test_bc_routes.py::TestCreateBc::test_duplicate_bc_id_rejected +tests/test_bc_routes.py::TestCreateBc::test_create_writes_audit_log +tests/test_bc_routes.py::TestCreateBc::test_create_with_decs + /Users/dmoreland/projects/cdisc-concept-curation/routes/bc.py:183: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) + if BiomedicalConcept.query.get(bc_id): + +tests/test_bc_routes.py::TestCreateBc::test_creates_bc_and_redirects + /Users/dmoreland/projects/cdisc-concept-curation/tests/test_bc_routes.py:67: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) + assert BiomedicalConcept.query.get("C00001") is not None + +tests/test_bc_routes.py: 13 warnings +tests/test_governance_routes.py: 19 warnings +tests/test_ingestion_routes.py: 7 warnings + /Users/dmoreland/projects/cdisc-concept-curation/.venv/lib/python3.14/site-packages/flask_sqlalchemy/query.py:30: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) + rv = self.get(ident) + +tests/test_bc_routes.py::TestEditBc::test_updates_short_name +tests/test_bc_routes.py::TestEditBc::test_edit_writes_audit_log + /Users/dmoreland/projects/cdisc-concept-curation/routes/bc.py:236: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + bc.updated_at = datetime.utcnow() + +tests/test_bc_routes.py::TestEditBc::test_updates_short_name + /Users/dmoreland/projects/cdisc-concept-curation/tests/test_bc_routes.py:180: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) + bc = BiomedicalConcept.query.get("C12345") + +tests/test_bc_routes.py::TestSubmitForReview::test_advances_status_to_sme_review +tests/test_bc_routes.py::TestSubmitForReview::test_submit_writes_audit_log + /Users/dmoreland/projects/cdisc-concept-curation/routes/bc.py:257: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + bc.updated_at = datetime.utcnow() + +tests/test_bc_routes.py::TestSubmitForReview::test_advances_status_to_sme_review + /Users/dmoreland/projects/cdisc-concept-curation/tests/test_bc_routes.py:204: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) + bc = BiomedicalConcept.query.get("C12345") + +tests/test_bc_routes.py::TestDeleteBc::test_deletes_bc + /Users/dmoreland/projects/cdisc-concept-curation/tests/test_bc_routes.py:223: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) + assert BiomedicalConcept.query.get("C12345") is None + +tests/test_bc_routes.py::TestExport::test_xlsx_export + /Users/dmoreland/projects/cdisc-concept-curation/.venv/lib/python3.14/site-packages/openpyxl/packaging/core.py:99: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + now = datetime.datetime.utcnow() + +tests/test_bc_routes.py::TestExport::test_xlsx_export + /Users/dmoreland/projects/cdisc-concept-curation/.venv/lib/python3.14/site-packages/openpyxl/writer/excel.py:292: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + workbook.properties.modified = datetime.datetime.utcnow() + +tests/test_bc_routes.py::TestExport::test_odm_xml_export + /Users/dmoreland/projects/cdisc-concept-curation/services/export.py:77: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + "FileOID": f'CDISC.BC.Export.{datetime.utcnow().strftime("%Y%m%d%H%M%S")}', + +tests/test_bc_routes.py::TestExport::test_odm_xml_export + /Users/dmoreland/projects/cdisc-concept-curation/services/export.py:78: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + "CreationDateTime": datetime.utcnow().isoformat(), + +tests/test_governance_routes.py: 12 warnings + /Users/dmoreland/projects/cdisc-concept-curation/routes/governance.py:33: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + bc.updated_at = datetime.utcnow() + +tests/test_governance_routes.py::TestAdvance::test_advances_provisional_to_sme_review + /Users/dmoreland/projects/cdisc-concept-curation/tests/test_governance_routes.py:22: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) + bc = BiomedicalConcept.query.get("C12345") + +tests/test_governance_routes.py::TestAdvance::test_advance_through_all_stages + /Users/dmoreland/projects/cdisc-concept-curation/tests/test_governance_routes.py:29: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) + bc = BiomedicalConcept.query.get("C12345") + +tests/test_governance_routes.py::TestAdvance::test_already_published_stays_published + /Users/dmoreland/projects/cdisc-concept-curation/tests/test_governance_routes.py:40: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) + bc = BiomedicalConcept.query.get("C12345") + +tests/test_governance_routes.py::TestReject::test_reject_returns_to_provisional +tests/test_governance_routes.py::TestReject::test_reject_creates_governance_record +tests/test_governance_routes.py::TestReject::test_reject_writes_audit_log +tests/test_governance_routes.py::TestReject::test_reject_ajax_returns_json + /Users/dmoreland/projects/cdisc-concept-curation/routes/governance.py:65: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + bc.updated_at = datetime.utcnow() + +tests/test_governance_routes.py::TestReject::test_reject_returns_to_provisional + /Users/dmoreland/projects/cdisc-concept-curation/tests/test_governance_routes.py:78: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) + bc = BiomedicalConcept.query.get("C12345") + +tests/test_ingestion_routes.py::TestApprove::test_approve_creates_bc +tests/test_ingestion_routes.py::TestApprove::test_approve_sets_status_approved +tests/test_ingestion_routes.py::TestApprove::test_approve_already_existing_bc_does_not_duplicate + /Users/dmoreland/projects/cdisc-concept-curation/routes/ingestion.py:121: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) + if not BiomedicalConcept.query.get(bc_id): + +tests/test_ingestion_routes.py::TestApprove::test_approve_creates_bc + /Users/dmoreland/projects/cdisc-concept-curation/tests/test_ingestion_routes.py:103: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) + assert BiomedicalConcept.query.get("C001") is not None + +tests/test_ingestion_routes.py::TestApprove::test_approve_sets_status_approved + /Users/dmoreland/projects/cdisc-concept-curation/tests/test_ingestion_routes.py:109: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) + ir = IngestionRecord.query.get(record_id) + +tests/test_ingestion_routes.py::TestReject::test_reject_sets_status_rejected + /Users/dmoreland/projects/cdisc-concept-curation/tests/test_ingestion_routes.py:148: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) + ir = IngestionRecord.query.get(record_id) + +tests/test_ingestion_routes.py::TestReject::test_reject_does_not_create_bc + /Users/dmoreland/projects/cdisc-concept-curation/tests/test_ingestion_routes.py:165: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) + assert BiomedicalConcept.query.get("C001") is None + +tests/test_ingestion_routes.py::TestApproveAll::test_approve_all_skips_records_with_errors + /Users/dmoreland/projects/cdisc-concept-curation/tests/test_ingestion_routes.py:187: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) + assert BiomedicalConcept.query.get("C001") is None + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +====================== 136 passed, 214 warnings in 0.87s ======================= diff --git a/routes/bc.py b/routes/bc.py index 6d5ade6..0517a41 100644 --- a/routes/bc.py +++ b/routes/bc.py @@ -118,6 +118,10 @@ def detail(bc_id): loinc_data = json.loads(bc.loinc_metadata) except (ValueError, TypeError): pass + elif bc.loinc_code: + results = LoincApiClient().search(bc.loinc_code, size=1) + if results and not results[0].get("error"): + loinc_data = results[0] ncit_data = {} if bc.ncit_metadata: diff --git a/tests/test_bc_routes.py b/tests/test_bc_routes.py index a57f22d..89ea193 100644 --- a/tests/test_bc_routes.py +++ b/tests/test_bc_routes.py @@ -118,7 +118,7 @@ def test_loinc_api_called_when_code_set(self, client, app): short_name="HbA1c", status="provisional", submitter="tester", - code="4548-4", + loinc_code="4548-4", ) db.session.add(bc) db.session.commit() @@ -156,7 +156,7 @@ def test_loinc_api_error_does_not_break_page(self, client, app): short_name="LOINC Error BC", status="provisional", submitter="tester", - code="4548-4", + loinc_code="4548-4", ) db.session.add(bc) db.session.commit() diff --git a/tests/test_ncit.py b/tests/test_ncit.py index 4739601..8479a36 100644 --- a/tests/test_ncit.py +++ b/tests/test_ncit.py @@ -1,4 +1,5 @@ """Tests for NCIt service extensions and the /ncit/concept/ route.""" + import json from unittest.mock import MagicMock, patch @@ -6,31 +7,30 @@ from services.ncit_api import NCItApiClient - # --------------------------------------------------------------------------- # Sample EVS full-concept response # --------------------------------------------------------------------------- EVS_FULL_CONCEPT = { - 'code': 'C64849', - 'name': 'Hemoglobin A1c Measurement', - 'definitions': [ - {'definition': 'A quantitative measurement of HbA1c.', 'source': 'NCI'}, - {'definition': 'Other source def.', 'source': 'OTHER'}, + "code": "C64849", + "name": "Hemoglobin A1c Measurement", + "definitions": [ + {"definition": "A quantitative measurement of HbA1c.", "source": "NCI"}, + {"definition": "Other source def.", "source": "OTHER"}, ], - 'synonyms': [ - {'name': 'HbA1c', 'termType': 'SY', 'source': 'NCI'}, - {'name': 'Glycated Hemoglobin', 'termType': 'SY', 'source': 'CDISC'}, - {'name': 'A1C', 'termType': 'AB', 'source': 'NCI'}, - {'name': 'Hemoglobin A1c Measurement', 'termType': 'PT', 'source': 'NCI'}, - {'name': 'Internal Code', 'termType': 'CODE', 'source': 'NCI'}, + "synonyms": [ + {"name": "HbA1c", "termType": "SY", "source": "NCI"}, + {"name": "Glycated Hemoglobin", "termType": "SY", "source": "CDISC"}, + {"name": "A1C", "termType": "AB", "source": "NCI"}, + {"name": "Hemoglobin A1c Measurement", "termType": "PT", "source": "NCI"}, + {"name": "Internal Code", "termType": "CODE", "source": "NCI"}, ], - 'parents': [ - {'code': 'C17721', 'name': 'Laboratory Test'}, - {'code': 'C45398', 'name': 'Glucose Measurement'}, + "parents": [ + {"code": "C17721", "name": "Laboratory Test"}, + {"code": "C45398", "name": "Glucose Measurement"}, ], - 'semanticType': [ - {'name': 'Laboratory Procedure'}, + "semanticType": [ + {"name": "Laboratory Procedure"}, ], } @@ -39,7 +39,13 @@ # NCItApiClient.get_concept() — extended fields # --------------------------------------------------------------------------- + class TestNcitGetConceptExtended: + def setup_method(self): + import services.ncit_api + + services.ncit_api._ncit_cache.clear() + def _mock_get(self, data): mock = MagicMock() mock.json.return_value = data @@ -47,63 +53,63 @@ def _mock_get(self, data): return mock def test_returns_parents(self): - with patch('services.ncit_api.requests.get') as mock_get: + with patch("services.ncit_api.requests.get") as mock_get: mock_get.return_value = self._mock_get(EVS_FULL_CONCEPT) - result = NCItApiClient().get_concept('C64849') + result = NCItApiClient().get_concept("C64849") - assert result['parents'] == [ - {'code': 'C17721', 'name': 'Laboratory Test'}, - {'code': 'C45398', 'name': 'Glucose Measurement'}, + assert result["parents"] == [ + {"code": "C17721", "name": "Laboratory Test"}, + {"code": "C45398", "name": "Glucose Measurement"}, ] def test_returns_semantic_type(self): - with patch('services.ncit_api.requests.get') as mock_get: + with patch("services.ncit_api.requests.get") as mock_get: mock_get.return_value = self._mock_get(EVS_FULL_CONCEPT) - result = NCItApiClient().get_concept('C64849') + result = NCItApiClient().get_concept("C64849") - assert result['semantic_type'] == ['Laboratory Procedure'] + assert result["semantic_type"] == ["Laboratory Procedure"] def test_returns_source_synonyms(self): - with patch('services.ncit_api.requests.get') as mock_get: + with patch("services.ncit_api.requests.get") as mock_get: mock_get.return_value = self._mock_get(EVS_FULL_CONCEPT) - result = NCItApiClient().get_concept('C64849') + result = NCItApiClient().get_concept("C64849") # SY, AB, PT terms — CODE excluded - assert 'HbA1c' in result['synonyms'] - assert 'A1C' in result['synonyms'] - assert 'Hemoglobin A1c Measurement' in result['synonyms'] - assert 'Internal Code' not in result['synonyms'] + assert "HbA1c" in result["synonyms"] + assert "A1C" in result["synonyms"] + assert "Hemoglobin A1c Measurement" in result["synonyms"] + assert "Internal Code" not in result["synonyms"] def test_returns_all_definitions(self): - with patch('services.ncit_api.requests.get') as mock_get: + with patch("services.ncit_api.requests.get") as mock_get: mock_get.return_value = self._mock_get(EVS_FULL_CONCEPT) - result = NCItApiClient().get_concept('C64849') + result = NCItApiClient().get_concept("C64849") - assert 'definitions' in result - assert isinstance(result['definitions'], list) - assert any(d['definition'] == 'A quantitative measurement of HbA1c.' for d in result['definitions']) + assert "definitions" in result + assert isinstance(result["definitions"], list) + assert any(d["definition"] == "A quantitative measurement of HbA1c." for d in result["definitions"]) def test_empty_parents_returns_empty_list(self): data = dict(EVS_FULL_CONCEPT, parents=[]) - with patch('services.ncit_api.requests.get') as mock_get: + with patch("services.ncit_api.requests.get") as mock_get: mock_get.return_value = self._mock_get(data) - result = NCItApiClient().get_concept('C64849') + result = NCItApiClient().get_concept("C64849") - assert result['parents'] == [] + assert result["parents"] == [] def test_missing_semantic_type_returns_empty_list(self): - data = {k: v for k, v in EVS_FULL_CONCEPT.items() if k != 'semanticType'} - with patch('services.ncit_api.requests.get') as mock_get: + data = {k: v for k, v in EVS_FULL_CONCEPT.items() if k != "semanticType"} + with patch("services.ncit_api.requests.get") as mock_get: mock_get.return_value = self._mock_get(data) - result = NCItApiClient().get_concept('C64849') + result = NCItApiClient().get_concept("C64849") - assert result['semantic_type'] == [] + assert result["semantic_type"] == [] def test_error_returns_error_dict(self): - with patch('services.ncit_api.requests.get', side_effect=Exception('timeout')): - result = NCItApiClient().get_concept('C64849') + with patch("services.ncit_api.requests.get", side_effect=Exception("timeout")): + result = NCItApiClient().get_concept("C64849") - assert 'error' in result + assert "error" in result # --------------------------------------------------------------------------- @@ -111,40 +117,40 @@ def test_error_returns_error_dict(self): # --------------------------------------------------------------------------- CONCEPT_RESULT = { - 'code': 'C64849', - 'name': 'Hemoglobin A1c Measurement', - 'preferred_name': 'Hemoglobin A1c Measurement', - 'definition': 'A quantitative measurement of HbA1c.', - 'definitions': [{'definition': 'A quantitative measurement of HbA1c.', 'source': 'NCI'}], - 'synonyms': ['HbA1c', 'A1C'], - 'parents': [{'code': 'C17721', 'name': 'Laboratory Test'}], - 'semantic_type': ['Laboratory Procedure'], + "code": "C64849", + "name": "Hemoglobin A1c Measurement", + "preferred_name": "Hemoglobin A1c Measurement", + "definition": "A quantitative measurement of HbA1c.", + "definitions": [{"definition": "A quantitative measurement of HbA1c.", "source": "NCI"}], + "synonyms": ["HbA1c", "A1C"], + "parents": [{"code": "C17721", "name": "Laboratory Test"}], + "semantic_type": ["Laboratory Procedure"], } class TestNcitConceptRoute: def test_returns_json(self, client): - with patch('routes.ncit.NCItApiClient') as MockClient: + with patch("routes.ncit.NCItApiClient") as MockClient: MockClient.return_value.get_concept.return_value = CONCEPT_RESULT - r = client.get('/ncit/concept/C64849', headers={'Accept': 'application/json'}) + r = client.get("/ncit/concept/C64849", headers={"Accept": "application/json"}) assert r.status_code == 200 data = json.loads(r.data) - assert data['code'] == 'C64849' - assert data['preferred_name'] == 'Hemoglobin A1c Measurement' - assert 'parents' in data - assert 'semantic_type' in data + assert data["code"] == "C64849" + assert data["preferred_name"] == "Hemoglobin A1c Measurement" + assert "parents" in data + assert "semantic_type" in data def test_calls_get_concept_with_code(self, client): - with patch('routes.ncit.NCItApiClient') as MockClient: + with patch("routes.ncit.NCItApiClient") as MockClient: MockClient.return_value.get_concept.return_value = CONCEPT_RESULT - client.get('/ncit/concept/C64849') + client.get("/ncit/concept/C64849") - MockClient.return_value.get_concept.assert_called_once_with('C64849') + MockClient.return_value.get_concept.assert_called_once_with("C64849") def test_error_from_service_returns_500(self, client): - with patch('routes.ncit.NCItApiClient') as MockClient: - MockClient.return_value.get_concept.return_value = {'error': 'Not found'} - r = client.get('/ncit/concept/CXXXXX') + with patch("routes.ncit.NCItApiClient") as MockClient: + MockClient.return_value.get_concept.return_value = {"error": "Not found"} + r = client.get("/ncit/concept/CXXXXX") assert r.status_code == 404 From 9dc53b8c0f37dd6ef9430bf9ad46bae6f2346494 Mon Sep 17 00:00:00 2001 From: Darren <3921919+pendingintent@users.noreply.github.com> Date: Wed, 15 Apr 2026 16:40:00 -0400 Subject: [PATCH 12/36] Addressed pytest warnings --- models/audit.py | 4 ++-- models/bc.py | 6 +++--- models/governance.py | 4 ++-- models/ingestion.py | 4 ++-- routes/bc.py | 18 +++++++++--------- routes/dashboard.py | 4 ++-- routes/governance.py | 10 +++++----- routes/ingestion.py | 8 ++++---- routes/ncit.py | 2 +- routes/specializations.py | 6 +++--- services/export.py | 6 +++--- tests/test_bc_routes.py | 8 ++++---- tests/test_governance_routes.py | 8 ++++---- tests/test_ingestion_routes.py | 10 +++++----- 14 files changed, 49 insertions(+), 49 deletions(-) diff --git a/models/audit.py b/models/audit.py index a561d6d..75a058f 100644 --- a/models/audit.py +++ b/models/audit.py @@ -1,5 +1,5 @@ from extensions import db -from datetime import datetime +from datetime import datetime, timezone import json @@ -12,7 +12,7 @@ class AuditLog(db.Model): actor = db.Column(db.String(100), default="system") _before_state = db.Column("before_state", db.Text) _after_state = db.Column("after_state", db.Text) - timestamp = db.Column(db.DateTime, default=datetime.utcnow) + timestamp = db.Column(db.DateTime, default=lambda: datetime.now(timezone.utc)) @property def before_state(self): diff --git a/models/bc.py b/models/bc.py index 7e15423..8177442 100644 --- a/models/bc.py +++ b/models/bc.py @@ -1,5 +1,5 @@ from extensions import db -from datetime import datetime +from datetime import datetime, timezone class BiomedicalConcept(db.Model): @@ -21,8 +21,8 @@ class BiomedicalConcept(db.Model): package_date = db.Column(db.String(20)) status = db.Column(db.String(50), default="provisional") # provisional/sme_review/cdisc_approval/published submitter = db.Column(db.String(100)) - created_at = db.Column(db.DateTime, default=datetime.utcnow) - updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + created_at = db.Column(db.DateTime, default=lambda: datetime.now(timezone.utc)) + updated_at = db.Column(db.DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) history_of_change = db.Column(db.Text) source = db.Column(db.String(50), default="local") # 'local' or 'cdisc_api' diff --git a/models/governance.py b/models/governance.py index 85549cb..1c90051 100644 --- a/models/governance.py +++ b/models/governance.py @@ -1,5 +1,5 @@ from extensions import db -from datetime import datetime +from datetime import datetime, timezone class GovernanceRecord(db.Model): @@ -10,6 +10,6 @@ class GovernanceRecord(db.Model): action = db.Column(db.String(100)) # submitted, advanced, rejected, approved, published actor = db.Column(db.String(100)) comment = db.Column(db.Text) - created_at = db.Column(db.DateTime, default=datetime.utcnow) + created_at = db.Column(db.DateTime, default=lambda: datetime.now(timezone.utc)) bc = db.relationship("BiomedicalConcept", backref=db.backref("governance_records", lazy="dynamic")) diff --git a/models/ingestion.py b/models/ingestion.py index 17d31fa..f64c130 100644 --- a/models/ingestion.py +++ b/models/ingestion.py @@ -1,5 +1,5 @@ from extensions import db -from datetime import datetime +from datetime import datetime, timezone import json @@ -15,7 +15,7 @@ class IngestionRecord(db.Model): _decs = db.Column("decs", db.Text) duplicate = db.Column(db.Boolean, default=False) status = db.Column(db.String(20), default="pending") # pending / approved / rejected - created_at = db.Column(db.DateTime, default=datetime.utcnow) + created_at = db.Column(db.DateTime, default=lambda: datetime.now(timezone.utc)) @property def mapped(self): diff --git a/routes/bc.py b/routes/bc.py index 0517a41..9ad6e17 100644 --- a/routes/bc.py +++ b/routes/bc.py @@ -8,7 +8,7 @@ from services.cdisc_api import CDISCApiClient from services.loinc_api import LoincApiClient from services.ncit_api import NCItApiClient -from datetime import datetime +from datetime import datetime, timezone bp = Blueprint("bc", __name__) @@ -110,7 +110,7 @@ def library_detail(concept_id): @bp.route("/") def detail(bc_id): - bc = BiomedicalConcept.query.get_or_404(bc_id) + bc = db.get_or_404(BiomedicalConcept, bc_id) decs = DataElementConcept.query.filter_by(bc_id=bc_id).order_by(DataElementConcept.sort_order).all() loinc_data = {} if bc.loinc_metadata: @@ -149,7 +149,7 @@ def fetch_metadata(bc_id): Saves results to the DB so subsequent visits use the fast stored-metadata path.""" from flask import jsonify - bc = BiomedicalConcept.query.get_or_404(bc_id) + bc = db.get_or_404(BiomedicalConcept, bc_id) def _fetch_ncit(): result = NCItApiClient().get_concept(bc.ncit_code) @@ -180,7 +180,7 @@ def create(): if not bc_id: flash("BC ID is required", "danger") return redirect(url_for("bc.new_bc")) - if BiomedicalConcept.query.get(bc_id): + if db.session.get(BiomedicalConcept, bc_id): flash(f"BC {bc_id} already exists", "danger") return redirect(url_for("bc.new_bc")) bc = BiomedicalConcept( @@ -218,7 +218,7 @@ def create(): @bp.route("//edit", methods=["POST"]) def edit(bc_id): - bc = BiomedicalConcept.query.get_or_404(bc_id) + bc = db.get_or_404(BiomedicalConcept, bc_id) before = bc.to_dict() bc.short_name = request.form.get("short_name", bc.short_name) bc.definition = request.form.get("definition", bc.definition) @@ -233,7 +233,7 @@ def edit(bc_id): bc.loinc_metadata = request.form.get("loinc_metadata", "") or bc.loinc_metadata bc.ncit_metadata = request.form.get("ncit_metadata", "") or bc.ncit_metadata bc.package_date = request.form.get("package_date", bc.package_date) - bc.updated_at = datetime.utcnow() + bc.updated_at = datetime.now(timezone.utc) log = AuditLog( entity_type="BiomedicalConcept", entity_id=bc_id, @@ -251,10 +251,10 @@ def edit(bc_id): @bp.route("//submit", methods=["POST"]) def submit_for_review(bc_id): - bc = BiomedicalConcept.query.get_or_404(bc_id) + bc = db.get_or_404(BiomedicalConcept, bc_id) before = bc.to_dict() bc.status = "sme_review" - bc.updated_at = datetime.utcnow() + bc.updated_at = datetime.now(timezone.utc) log = AuditLog( entity_type="BiomedicalConcept", entity_id=bc_id, @@ -271,7 +271,7 @@ def submit_for_review(bc_id): @bp.route("//delete", methods=["POST"]) def delete(bc_id): - bc = BiomedicalConcept.query.get_or_404(bc_id) + bc = db.get_or_404(BiomedicalConcept, bc_id) # Nullify self-referential parent FK on child BCs; without this SQLAlchemy # raises CircularDependencyError when flushing the delete. BiomedicalConcept.query.filter_by(parent_bc_id=bc_id).update({"parent_bc_id": None}, synchronize_session="fetch") diff --git a/routes/dashboard.py b/routes/dashboard.py index 8c916cc..b032e76 100644 --- a/routes/dashboard.py +++ b/routes/dashboard.py @@ -1,5 +1,5 @@ from concurrent.futures import ThreadPoolExecutor, as_completed -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from flask import Blueprint, render_template from models.bc import BiomedicalConcept from models.audit import AuditLog @@ -26,7 +26,7 @@ def index(): # --- Local DB stats --- local_total_bcs = BiomedicalConcept.query.count() local_pending = BiomedicalConcept.query.filter(BiomedicalConcept.status.in_(["provisional", "sme_review", "cdisc_approval"])).count() - recent_additions = BiomedicalConcept.query.filter(BiomedicalConcept.created_at >= datetime.utcnow() - timedelta(days=7)).count() + recent_additions = BiomedicalConcept.query.filter(BiomedicalConcept.created_at >= datetime.now(timezone.utc) - timedelta(days=7)).count() governance_items = BiomedicalConcept.query.filter(BiomedicalConcept.status != "published").order_by(BiomedicalConcept.updated_at.desc()).limit(10).all() recent_audits = AuditLog.query.order_by(AuditLog.timestamp.desc()).limit(10).all() diff --git a/routes/governance.py b/routes/governance.py index 1e94286..62050cd 100644 --- a/routes/governance.py +++ b/routes/governance.py @@ -3,7 +3,7 @@ from models.governance import GovernanceRecord from models.audit import AuditLog from extensions import db -from datetime import datetime +from datetime import datetime, timezone bp = Blueprint("governance", __name__) @@ -25,12 +25,12 @@ def board(): @bp.route("/advance/", methods=["POST"]) def advance(bc_id): - bc = BiomedicalConcept.query.get_or_404(bc_id) + bc = db.get_or_404(BiomedicalConcept, bc_id) before_status = bc.status current_idx = STATUS_ORDER.index(bc.status) if bc.status in STATUS_ORDER else 0 if current_idx < len(STATUS_ORDER) - 1: bc.status = STATUS_ORDER[current_idx + 1] - bc.updated_at = datetime.utcnow() + bc.updated_at = datetime.now(timezone.utc) rec = GovernanceRecord( bc_id=bc_id, stage=current_idx + 1, @@ -59,10 +59,10 @@ def advance(bc_id): @bp.route("/reject/", methods=["POST"]) def reject_bc(bc_id): - bc = BiomedicalConcept.query.get_or_404(bc_id) + bc = db.get_or_404(BiomedicalConcept, bc_id) before_status = bc.status bc.status = "provisional" - bc.updated_at = datetime.utcnow() + bc.updated_at = datetime.now(timezone.utc) rec = GovernanceRecord( bc_id=bc_id, stage=0, diff --git a/routes/ingestion.py b/routes/ingestion.py index bb2e376..e6c4bad 100644 --- a/routes/ingestion.py +++ b/routes/ingestion.py @@ -115,10 +115,10 @@ def upload(): @bp.route("/approve/", methods=["POST"]) def approve(record_id): - ir = IngestionRecord.query.get_or_404(record_id) + ir = db.get_or_404(IngestionRecord, record_id) mapped = ir.mapped bc_id = mapped.get("bc_id") or mapped.get("ncit_code", f"IMPORT_{record_id}") - if not BiomedicalConcept.query.get(bc_id): + if not db.session.get(BiomedicalConcept, bc_id): bc = _bc_from_mapped(bc_id, mapped) db.session.add(bc) _create_decs(bc_id, ir.decs) @@ -140,7 +140,7 @@ def approve(record_id): @bp.route("/reject/", methods=["POST"]) def reject(record_id): - ir = IngestionRecord.query.get_or_404(record_id) + ir = db.get_or_404(IngestionRecord, record_id) ir.status = "rejected" db.session.commit() return redirect(url_for("ingestion.index")) @@ -159,7 +159,7 @@ def approve_all(): continue mapped = ir.mapped bc_id = mapped.get("bc_id") or mapped.get("ncit_code", f"IMPORT_{ir.id}") - if not BiomedicalConcept.query.get(bc_id): + if not db.session.get(BiomedicalConcept, bc_id): bc = _bc_from_mapped(bc_id, mapped) db.session.add(bc) _create_decs(bc_id, ir.decs) diff --git a/routes/ncit.py b/routes/ncit.py index e554af2..23545bf 100644 --- a/routes/ncit.py +++ b/routes/ncit.py @@ -69,7 +69,7 @@ def concept_detail(ncit_code): @bp.route("/resolve/", methods=["POST"]) def resolve(bc_id): - bc = BiomedicalConcept.query.get_or_404(bc_id) + bc = db.get_or_404(BiomedicalConcept, bc_id) ncit_code = request.form.get("ncit_code", "").strip() if ncit_code: bc.ncit_code = ncit_code diff --git a/routes/specializations.py b/routes/specializations.py index 30a5cc0..7465bb5 100644 --- a/routes/specializations.py +++ b/routes/specializations.py @@ -48,7 +48,7 @@ def library_detail(spec_path): @bp.route("/") def detail(vlm_group_id): - spec = DatasetSpecialization.query.get_or_404(vlm_group_id) + spec = db.get_or_404(DatasetSpecialization, vlm_group_id) specs = DatasetSpecialization.query.all() library_bcs, local_bcs = _get_bc_options() return render_template( @@ -96,11 +96,11 @@ def generate_from_dec(): @bp.route("/generate/", methods=["POST"]) def generate(bc_id): """Generate a specialization from DEC templates for a BC.""" - bc = BiomedicalConcept.query.get_or_404(bc_id) + bc = db.get_or_404(BiomedicalConcept, bc_id) decs = DataElementConcept.query.filter_by(bc_id=bc_id).all() domain = request.form.get("domain", "SDTM") vlm_group_id = f"{bc_id}.{domain}" - existing = DatasetSpecialization.query.get(vlm_group_id) + existing = db.session.get(DatasetSpecialization, vlm_group_id) if existing: flash(f"Specialization {vlm_group_id} already exists", "warning") return redirect(url_for("specializations.index")) diff --git a/services/export.py b/services/export.py index cee7036..d8ac54f 100644 --- a/services/export.py +++ b/services/export.py @@ -1,6 +1,6 @@ import io import json -from datetime import datetime +from datetime import datetime, timezone try: import openpyxl @@ -74,8 +74,8 @@ def export_odm_xml(bc_list): attrib={ "xmlns": "http://www.cdisc.org/ns/odm/v1.3", "FileType": "Snapshot", - "FileOID": f'CDISC.BC.Export.{datetime.utcnow().strftime("%Y%m%d%H%M%S")}', - "CreationDateTime": datetime.utcnow().isoformat(), + "FileOID": f'CDISC.BC.Export.{datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S")}', + "CreationDateTime": datetime.now(timezone.utc).isoformat(), }, ) diff --git a/tests/test_bc_routes.py b/tests/test_bc_routes.py index 89ea193..c2dd415 100644 --- a/tests/test_bc_routes.py +++ b/tests/test_bc_routes.py @@ -64,7 +64,7 @@ def test_creates_bc_and_redirects(self, client, app): r = client.post("/bc/", data=_bc_form(), follow_redirects=False) assert r.status_code == 302 with app.app_context(): - assert BiomedicalConcept.query.get("C00001") is not None + assert db.session.get(BiomedicalConcept, "C00001") is not None def test_missing_bc_id_redirects_with_error(self, client): r = client.post("/bc/", data=_bc_form(bc_id=""), follow_redirects=True) @@ -177,7 +177,7 @@ class TestEditBc: def test_updates_short_name(self, client, app, sample_bc): client.post("/bc/C12345/edit", data={"short_name": "Updated Name"}) with app.app_context(): - bc = BiomedicalConcept.query.get("C12345") + bc = db.session.get(BiomedicalConcept, "C12345") assert bc.short_name == "Updated Name" def test_edit_writes_audit_log(self, client, app, sample_bc): @@ -201,7 +201,7 @@ class TestSubmitForReview: def test_advances_status_to_sme_review(self, client, app, sample_bc): client.post("/bc/C12345/submit") with app.app_context(): - bc = BiomedicalConcept.query.get("C12345") + bc = db.session.get(BiomedicalConcept, "C12345") assert bc.status == "sme_review" def test_submit_writes_audit_log(self, client, app, sample_bc): @@ -220,7 +220,7 @@ class TestDeleteBc: def test_deletes_bc(self, client, app, sample_bc): client.post("/bc/C12345/delete") with app.app_context(): - assert BiomedicalConcept.query.get("C12345") is None + assert db.session.get(BiomedicalConcept, "C12345") is None def test_delete_writes_audit_log(self, client, app, sample_bc): client.post("/bc/C12345/delete") diff --git a/tests/test_governance_routes.py b/tests/test_governance_routes.py index 5eb0ea9..263c2ee 100644 --- a/tests/test_governance_routes.py +++ b/tests/test_governance_routes.py @@ -19,14 +19,14 @@ class TestAdvance: def test_advances_provisional_to_sme_review(self, client, app, sample_bc): client.post("/governance/advance/C12345") with app.app_context(): - bc = BiomedicalConcept.query.get("C12345") + bc = db.session.get(BiomedicalConcept, "C12345") assert bc.status == "sme_review" def test_advance_through_all_stages(self, client, app, sample_bc): for expected in ["sme_review", "cdisc_approval", "published"]: client.post("/governance/advance/C12345") with app.app_context(): - bc = BiomedicalConcept.query.get("C12345") + bc = db.session.get(BiomedicalConcept, "C12345") assert bc.status == "published" def test_already_published_stays_published(self, client, app, sample_bc): @@ -37,7 +37,7 @@ def test_already_published_stays_published(self, client, app, sample_bc): r = client.post("/governance/advance/C12345", follow_redirects=True) assert r.status_code == 200 with app.app_context(): - bc = BiomedicalConcept.query.get("C12345") + bc = db.session.get(BiomedicalConcept, "C12345") assert bc.status == "published" def test_advance_creates_governance_record(self, client, app, sample_bc): @@ -75,7 +75,7 @@ def test_reject_returns_to_provisional(self, client, app, sample_bc): client.post("/governance/advance/C12345") client.post("/governance/reject/C12345") with app.app_context(): - bc = BiomedicalConcept.query.get("C12345") + bc = db.session.get(BiomedicalConcept, "C12345") assert bc.status == "provisional" def test_reject_creates_governance_record(self, client, app, sample_bc): diff --git a/tests/test_ingestion_routes.py b/tests/test_ingestion_routes.py index 1ba19bf..eeae74d 100644 --- a/tests/test_ingestion_routes.py +++ b/tests/test_ingestion_routes.py @@ -100,13 +100,13 @@ def test_approve_creates_bc(self, client, app): assert record_id is not None client.post(f"/ingestion/approve/{record_id}") with app.app_context(): - assert BiomedicalConcept.query.get("C001") is not None + assert db.session.get(BiomedicalConcept, "C001") is not None def test_approve_sets_status_approved(self, client, app): record_id = self._upload_and_get_record_id(client, app) client.post(f"/ingestion/approve/{record_id}") with app.app_context(): - ir = IngestionRecord.query.get(record_id) + ir = db.session.get(IngestionRecord, record_id) assert ir.status == "approved" def test_approve_nonexistent_record_returns_404(self, client): @@ -145,7 +145,7 @@ def test_reject_sets_status_rejected(self, client, app): client.post(f"/ingestion/reject/{record_id}") with app.app_context(): - ir = IngestionRecord.query.get(record_id) + ir = db.session.get(IngestionRecord, record_id) assert ir.status == "rejected" def test_reject_does_not_create_bc(self, client, app): @@ -162,7 +162,7 @@ def test_reject_does_not_create_bc(self, client, app): client.post(f"/ingestion/reject/{record_id}") with app.app_context(): - assert BiomedicalConcept.query.get("C001") is None + assert db.session.get(BiomedicalConcept, "C001") is None def test_reject_nonexistent_record_returns_404(self, client): r = client.post("/ingestion/reject/99999") @@ -184,4 +184,4 @@ def test_approve_all_skips_records_with_errors(self, client, app): ) client.post("/ingestion/approve_all") with app.app_context(): - assert BiomedicalConcept.query.get("C001") is None + assert db.session.get(BiomedicalConcept, "C001") is None From 6fc4cec326a868130f85d0001959915e2e228cbf Mon Sep 17 00:00:00 2001 From: Darren <3921919+pendingintent@users.noreply.github.com> Date: Thu, 16 Apr 2026 09:06:40 -0400 Subject: [PATCH 13/36] Added pytest to pre-commit and ci.yml --- .github/workflows/ci.yml | 21 +++++++++++++++++++++ .pre-commit-config.yaml | 9 +++++++++ 2 files changed, 30 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..fdbebad --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,21 @@ +name: CI + +on: + push: + branches: ["**"] + pull_request: + branches: ["**"] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: "pip" + - name: Install dependencies + run: pip install -r requirements.txt + - name: Run tests + run: pytest --tb=short diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d71bf48..486f9a9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,3 +8,12 @@ repos: rev: 7.3.0 hooks: - id: flake8 + + - repo: local + hooks: + - id: pytest + name: pytest + entry: pytest + language: system + pass_filenames: false + always_run: true From f030f5b3a1148be930436dabfa51344297fa95b0 Mon Sep 17 00:00:00 2001 From: Darren <3921919+pendingintent@users.noreply.github.com> Date: Thu, 16 Apr 2026 09:30:12 -0400 Subject: [PATCH 14/36] Added alembic migration file to readd column --- ...6f8a0b1_add_code_to_biomedical_concepts.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 migrations/versions/c2d4e6f8a0b1_add_code_to_biomedical_concepts.py diff --git a/migrations/versions/c2d4e6f8a0b1_add_code_to_biomedical_concepts.py b/migrations/versions/c2d4e6f8a0b1_add_code_to_biomedical_concepts.py new file mode 100644 index 0000000..615801a --- /dev/null +++ b/migrations/versions/c2d4e6f8a0b1_add_code_to_biomedical_concepts.py @@ -0,0 +1,26 @@ +"""add code column to biomedical_concepts + +Revision ID: c2d4e6f8a0b1 +Revises: a1c3e5f7b9d2 +Create Date: 2026-04-16 09:15:00.000000 + +""" + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = "c2d4e6f8a0b1" +down_revision = "a1c3e5f7b9d2" +branch_labels = None +depends_on = None + + +def upgrade(): + with op.batch_alter_table("biomedical_concepts", schema=None) as batch_op: + batch_op.add_column(sa.Column("code", sa.String(50), nullable=True)) + + +def downgrade(): + with op.batch_alter_table("biomedical_concepts", schema=None) as batch_op: + batch_op.drop_column("code") From 0f86c2b165b15bff8915a33e0fb4fa631877bfbd Mon Sep 17 00:00:00 2001 From: Darren <3921919+pendingintent@users.noreply.github.com> Date: Thu, 16 Apr 2026 12:56:25 -0400 Subject: [PATCH 15/36] Fixed NCIt auto fetch --- README.md | 4 +- routes/bc.py | 61 ++++++++++++-- templates/bc_detail.html | 45 ++++++++--- tests/test_bc_routes.py | 171 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 262 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 691b6c2..055ddcf 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ The sidebar navigation exposes seven screens, accessible at these URL prefixes: |--------|-----|-------------| | Dashboard | `/` | KPI cards (total BCs, pending review, published), governance pipeline chart with concurrent CDISC API fetches (ThreadPoolExecutor), recent submissions table | | Ingestion | `/ingestion` | Upload XLSX, CSV, or JSON files; AI field mapper assigns confidence scores; approve or reject rows to the database | -| BCs | `/bc` | Browse, create, edit, and delete Biomedical Concepts; LOINC code entry with asynchronous metadata fetch on demand — click "Search LOINC" to fetch full LOINC metadata from NLM Clinical Tables API (LONG_COMMON_NAME, SHORTNAME, COMPONENT, METHOD_TYP, units, datatype, copyright notices, etc.), auto-populate Long Common Name, and store in hidden metadata field; NCIt concept selection with live search and one-click integration — click "Use this concept" to fetch full metadata asynchronously (preferred name, synonyms, description, parent concepts, child concepts, semantic type, and NCIt Browser link); all available definitions displayed with source attribution as `[SOURCE] definition text` in the References section; query parameters `/bc/new?ncit_code=...&ncit_name=...&ncit_definition=...` pre-populate BC fields on page load; `parent_bc_id` auto-filled from first parent concept's code; Data Element Concept sub-records | +| BCs | `/bc` | Browse, create, edit, and delete Biomedical Concepts; LOINC code entry with asynchronous metadata fetch — fetch triggers automatically on page load if a LOINC code is set and no cached metadata exists, stores result in database for fast future loads; click "Search LOINC" button to manually trigger fresh fetch; click "Clear" button to remove LOINC code, metadata, and spinner; NCIt concept selection with live search and one-click integration — click "Use this concept" to fetch full metadata asynchronously (preferred name, synonyms, description, parent concepts, child concepts, semantic type, and NCIt Browser link); click "Clear" button to remove NCIt code, metadata, and parent BC ID; all available definitions displayed with source attribution as `[SOURCE] definition text` in the References section; query parameters `/bc/new?ncit_code=...&ncit_name=...&ncit_definition=...` pre-populate BC fields on page load; `parent_bc_id` auto-filled from first parent concept's code; form inputs use Jinja2 `or ''` pattern to prevent rendering Python `None` as literal string `"None"` in HTML attributes; Data Element Concept sub-records | | NCIT Mapping | `/ncit` | Search the NCI Thesaurus, resolve low-confidence mappings, and confirm NCIt codes for each BC | | Specializations | `/specializations` | View and generate SDTM/CDASH dataset specializations and CRF variable mappings | | Governance | `/governance` | 4-stage Kanban board (Provisional > SME Review > CDISC Approval > Published) with advance and reject actions | @@ -149,7 +149,7 @@ cdisc-concept-curation/ ├── routes/ # 8 Flask blueprints │ ├── dashboard.py # Concurrent CDISC API fetches (ThreadPoolExecutor), KPI cards │ ├── ingestion.py # File upload and AI field mapper -│ ├── bc.py # Create, edit, detail views; `/bc/new` accepts query parameters (`ncit_code`, `ncit_name`, `ncit_definition`) to pre-populate BC fields on page load; NCIt metadata fetch on demand; auto-fills `parent_bc_id` from first parent concept code +│ ├── bc.py # Create, edit, detail views; `/bc/new` accepts query parameters (`ncit_code`, `ncit_name`, `ncit_definition`) to pre-populate BC fields on page load; `/bc//clear-ncit` and `/bc//clear-loinc` endpoints to remove codes and metadata; NCIt metadata fetch on demand; auto-fills `parent_bc_id` from first parent concept code; LOINC metadata auto-fetched and saved on detail page load │ ├── ncit.py # GET /ncit/search and GET /ncit/concept/ JSON endpoints with full metadata, children, and NCIt Browser links │ ├── loinc.py # GET /loinc/search JSON API endpoint │ ├── specializations.py # Dataset specializations and CRF mappings diff --git a/routes/bc.py b/routes/bc.py index 9ad6e17..7e5de9a 100644 --- a/routes/bc.py +++ b/routes/bc.py @@ -122,6 +122,8 @@ def detail(bc_id): results = LoincApiClient().search(bc.loinc_code, size=1) if results and not results[0].get("error"): loinc_data = results[0] + bc.loinc_metadata = json.dumps(loinc_data) + db.session.commit() ncit_data = {} if bc.ncit_metadata: @@ -138,7 +140,7 @@ def detail(bc_id): loinc_data=loinc_data, ncit_data=ncit_data, needs_ncit_fetch=not ncit_data and bool(bc.ncit_code), - needs_loinc_fetch=not bc.loinc_metadata and bool(bc.loinc_code), + needs_loinc_fetch=not loinc_data and bool(bc.loinc_code), page_title=bc.short_name, ) @@ -222,16 +224,18 @@ def edit(bc_id): before = bc.to_dict() bc.short_name = request.form.get("short_name", bc.short_name) bc.definition = request.form.get("definition", bc.definition) - bc.ncit_code = request.form.get("ncit_code", bc.ncit_code) - bc.parent_bc_id = request.form.get("parent_bc_id") or bc.parent_bc_id + new_ncit_code = (request.form.get("ncit_code", "") or "").strip() or None + bc.ncit_code = new_ncit_code + bc.ncit_metadata = (request.form.get("ncit_metadata", "") or bc.ncit_metadata) if new_ncit_code else None + bc.parent_bc_id = (request.form.get("parent_bc_id", "") or "").strip() or None bc.bc_categories = request.form.get("bc_categories", bc.bc_categories) bc.synonyms = request.form.get("synonyms", bc.synonyms) bc.result_scales = request.form.get("result_scales", bc.result_scales) bc.system = request.form.get("system", bc.system) bc.system_name = request.form.get("system_name", bc.system_name) - bc.loinc_code = request.form.get("loinc_code", bc.loinc_code) - bc.loinc_metadata = request.form.get("loinc_metadata", "") or bc.loinc_metadata - bc.ncit_metadata = request.form.get("ncit_metadata", "") or bc.ncit_metadata + new_loinc_code = (request.form.get("loinc_code", "") or "").strip() or None + bc.loinc_code = new_loinc_code + bc.loinc_metadata = (request.form.get("loinc_metadata", "") or bc.loinc_metadata) if new_loinc_code else None bc.package_date = request.form.get("package_date", bc.package_date) bc.updated_at = datetime.now(timezone.utc) log = AuditLog( @@ -249,6 +253,51 @@ def edit(bc_id): return redirect(url_for("bc.detail", bc_id=bc_id)) +@bp.route("//clear-ncit", methods=["POST"]) +def clear_ncit(bc_id): + bc = db.get_or_404(BiomedicalConcept, bc_id) + before = bc.to_dict() + bc.ncit_code = None + bc.ncit_metadata = None + bc.parent_bc_id = None + bc.updated_at = datetime.now(timezone.utc) + db.session.add( + AuditLog( + entity_type="BiomedicalConcept", + entity_id=bc_id, + action="ncit_cleared", + actor="user", + before_state=before, + after_state=bc.to_dict(), + ) + ) + db.session.commit() + flash(f"NCIt code cleared from {bc_id}", "success") + return redirect(url_for("bc.detail", bc_id=bc_id)) + + +@bp.route("//clear-loinc", methods=["POST"]) +def clear_loinc(bc_id): + bc = db.get_or_404(BiomedicalConcept, bc_id) + before = bc.to_dict() + bc.loinc_code = None + bc.loinc_metadata = None + bc.updated_at = datetime.now(timezone.utc) + db.session.add( + AuditLog( + entity_type="BiomedicalConcept", + entity_id=bc_id, + action="loinc_cleared", + actor="user", + before_state=before, + after_state=bc.to_dict(), + ) + ) + db.session.commit() + flash(f"LOINC code cleared from {bc_id}", "success") + return redirect(url_for("bc.detail", bc_id=bc_id)) + + @bp.route("//submit", methods=["POST"]) def submit_for_review(bc_id): bc = db.get_or_404(BiomedicalConcept, bc_id) diff --git a/templates/bc_detail.html b/templates/bc_detail.html index 4e6f874..8e86bf0 100644 --- a/templates/bc_detail.html +++ b/templates/bc_detail.html @@ -61,14 +61,14 @@

Identification

@@ -93,12 +93,19 @@

+ {% if bc and bc.ncit_code %} + + {% endif %}

@@ -168,14 +175,13 @@

{% endif %}

-

Classification

Separate multiple categories with semicolons.
@@ -183,13 +189,13 @@

Classification

@@ -223,6 +229,13 @@

+ {% if bc and bc.loinc_code %} + + {% endif %}
@@ -261,14 +274,13 @@

-

Definition

+ placeholder="Provide a precise, unambiguous definition aligned with NCI preferred term definition…">{{ bc.definition or '' }}
@@ -278,12 +290,12 @@

Metadata

+ value="{{ bc.package_date or '' }}">
@@ -386,6 +398,17 @@

Data Element Concepts (DECs)

+{% if bc and bc.bc_id %} +{% if bc.ncit_code %} +
+{% endif %} +{% if bc.loinc_code %} +
+{% endif %} +{% endif %} + {% endblock %} {% block extra_js %} diff --git a/tests/test_bc_routes.py b/tests/test_bc_routes.py index c2dd415..1c99ddf 100644 --- a/tests/test_bc_routes.py +++ b/tests/test_bc_routes.py @@ -167,6 +167,48 @@ def test_loinc_api_error_does_not_break_page(self, client, app): assert r.status_code == 200 + def test_loinc_spinner_not_shown_when_loinc_fetched_server_side(self, client, app): + with app.app_context(): + bc = BiomedicalConcept( + bc_id="C99904", + short_name="LOINC Spinner BC", + status="provisional", + submitter="tester", + loinc_code="4548-4", + ) + db.session.add(bc) + db.session.commit() + + loinc_result = {"LOINC_NUM": "4548-4", "LONG_COMMON_NAME": "Hemoglobin A1c/Hemoglobin.total in Blood"} + with patch("routes.bc.LoincApiClient") as MockLoinc: + MockLoinc.return_value.search.return_value = [loinc_result] + r = client.get("/bc/C99904") + + assert r.status_code == 200 + assert b"loinc-loading-indicator" not in r.data + + def test_loinc_metadata_saved_when_fetched_in_detail(self, client, app): + with app.app_context(): + bc = BiomedicalConcept( + bc_id="C99905", + short_name="LOINC Save BC", + status="provisional", + submitter="tester", + loinc_code="4548-4", + ) + db.session.add(bc) + db.session.commit() + + loinc_result = {"LOINC_NUM": "4548-4", "LONG_COMMON_NAME": "Hemoglobin A1c/Hemoglobin.total in Blood"} + with patch("routes.bc.LoincApiClient") as MockLoinc: + MockLoinc.return_value.search.return_value = [loinc_result] + client.get("/bc/C99905") + + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C99905") + assert bc.loinc_metadata is not None + assert "4548-4" in bc.loinc_metadata + # --------------------------------------------------------------------------- # POST /bc//edit @@ -191,6 +233,135 @@ def test_nonexistent_bc_returns_404(self, client): r = client.post("/bc/NOPE/edit", data={"short_name": "X"}) assert r.status_code == 404 + def test_edit_clears_ncit_code_when_submitted_empty(self, client, app, sample_bc): + client.post("/bc/C12345/edit", data={"ncit_code": ""}) + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C12345") + assert bc.ncit_code is None + + def test_edit_clears_ncit_metadata_when_ncit_code_cleared(self, client, app, sample_bc): + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C12345") + bc.ncit_metadata = '{"preferred_name": "Test"}' + db.session.commit() + client.post("/bc/C12345/edit", data={"ncit_code": "", "ncit_metadata": '{"preferred_name": "Test"}'}) + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C12345") + assert bc.ncit_metadata is None + + def test_edit_clears_parent_bc_id_when_submitted_empty(self, client, app, sample_bc): + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C12345") + bc.parent_bc_id = "C99999" + db.session.commit() + client.post("/bc/C12345/edit", data={"parent_bc_id": ""}) + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C12345") + assert bc.parent_bc_id is None + + def test_edit_clears_loinc_code_and_metadata_when_submitted_empty(self, client, app, sample_bc): + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C12345") + bc.loinc_code = "4548-4" + bc.loinc_metadata = '{"LONG_COMMON_NAME": "HbA1c"}' + db.session.commit() + client.post("/bc/C12345/edit", data={"loinc_code": "", "loinc_metadata": '{"LONG_COMMON_NAME": "HbA1c"}'}) + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C12345") + assert bc.loinc_code is None + assert bc.loinc_metadata is None + + def test_edit_strips_whitespace_from_ncit_code(self, client, app, sample_bc): + client.post("/bc/C12345/edit", data={"ncit_code": " "}) + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C12345") + assert bc.ncit_code is None + + def test_detail_renders_empty_ncit_code_when_none(self, client, app, sample_bc): + """Regression: Jinja2 renders Python None as 'None' in HTML attributes. + When bc.ncit_code is None the input must have value='' not value='None', + otherwise the browser re-submits 'None' causing a spurious NCIt fetch spinner.""" + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C12345") + bc.ncit_code = None + db.session.commit() + with patch("routes.bc.LoincApiClient") as MockLoinc: + MockLoinc.return_value.search.return_value = [] + r = client.get("/bc/C12345") + assert b'value="None"' not in r.data + assert b'name="ncit_code"' in r.data + + +# --------------------------------------------------------------------------- +# POST /bc//clear-ncit +# --------------------------------------------------------------------------- + + +class TestClearNcitCode: + def test_clears_ncit_code_and_metadata(self, client, app, sample_bc): + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C12345") + bc.ncit_metadata = '{"preferred_name": "Test"}' + db.session.commit() + client.post("/bc/C12345/clear-ncit", follow_redirects=False) + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C12345") + assert bc.ncit_code is None + assert bc.ncit_metadata is None + + def test_clear_ncit_writes_audit_log(self, client, app, sample_bc): + client.post("/bc/C12345/clear-ncit") + with app.app_context(): + log = AuditLog.query.filter_by(entity_id="C12345", action="ncit_cleared").first() + assert log is not None + + def test_clear_ncit_nonexistent_bc_returns_404(self, client): + r = client.post("/bc/NOTREAL/clear-ncit") + assert r.status_code == 404 + + def test_clear_ncit_also_clears_parent_bc_id(self, client, app, sample_bc): + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C12345") + bc.parent_bc_id = "C99999" + db.session.commit() + client.post("/bc/C12345/clear-ncit", follow_redirects=False) + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C12345") + assert bc.parent_bc_id is None + + +# --------------------------------------------------------------------------- +# POST /bc//clear-loinc +# --------------------------------------------------------------------------- + + +class TestClearLoincCode: + def test_clears_loinc_code_and_metadata(self, client, app, sample_bc): + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C12345") + bc.loinc_code = "4548-4" + bc.loinc_metadata = '{"LONG_COMMON_NAME": "HbA1c"}' + db.session.commit() + client.post("/bc/C12345/clear-loinc", follow_redirects=False) + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C12345") + assert bc.loinc_code is None + assert bc.loinc_metadata is None + + def test_clear_loinc_writes_audit_log(self, client, app, sample_bc): + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C12345") + bc.loinc_code = "4548-4" + db.session.commit() + client.post("/bc/C12345/clear-loinc") + with app.app_context(): + log = AuditLog.query.filter_by(entity_id="C12345", action="loinc_cleared").first() + assert log is not None + + def test_clear_loinc_nonexistent_bc_returns_404(self, client): + r = client.post("/bc/NOTREAL/clear-loinc") + assert r.status_code == 404 + # --------------------------------------------------------------------------- # POST /bc//submit From 9ef270bf0733d76a3cdd3d611e0f51fa095b4557 Mon Sep 17 00:00:00 2001 From: Darren <3921919+pendingintent@users.noreply.github.com> Date: Thu, 16 Apr 2026 14:41:21 -0400 Subject: [PATCH 16/36] Added export for BCs ready for publication --- README.md | 6 +- routes/bc.py | 10 ++- routes/governance.py | 20 ++++- services/export.py | 74 +++++++++++++++++++ templates/governance.html | 35 +++++++++ tests/test_bc_routes.py | 53 ++++++++++++++ tests/test_governance_routes.py | 126 ++++++++++++++++++++++++++++++++ 7 files changed, 316 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 055ddcf..7b46cb1 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ The sidebar navigation exposes seven screens, accessible at these URL prefixes: | BCs | `/bc` | Browse, create, edit, and delete Biomedical Concepts; LOINC code entry with asynchronous metadata fetch — fetch triggers automatically on page load if a LOINC code is set and no cached metadata exists, stores result in database for fast future loads; click "Search LOINC" button to manually trigger fresh fetch; click "Clear" button to remove LOINC code, metadata, and spinner; NCIt concept selection with live search and one-click integration — click "Use this concept" to fetch full metadata asynchronously (preferred name, synonyms, description, parent concepts, child concepts, semantic type, and NCIt Browser link); click "Clear" button to remove NCIt code, metadata, and parent BC ID; all available definitions displayed with source attribution as `[SOURCE] definition text` in the References section; query parameters `/bc/new?ncit_code=...&ncit_name=...&ncit_definition=...` pre-populate BC fields on page load; `parent_bc_id` auto-filled from first parent concept's code; form inputs use Jinja2 `or ''` pattern to prevent rendering Python `None` as literal string `"None"` in HTML attributes; Data Element Concept sub-records | | NCIT Mapping | `/ncit` | Search the NCI Thesaurus, resolve low-confidence mappings, and confirm NCIt codes for each BC | | Specializations | `/specializations` | View and generate SDTM/CDASH dataset specializations and CRF variable mappings | -| Governance | `/governance` | 4-stage Kanban board (Provisional > SME Review > CDISC Approval > Published) with advance and reject actions | +| Governance | `/governance` | 4-stage Kanban board (Provisional > SME Review > CDISC Approval > Published) with advance and reject actions; export published BCs as XLSX (BC_LB worksheet format with 18 columns) via "Export Published BCs" button | | Audit Trail | `/audit` | Immutable log of every create, update, and status change with before/after state, filterable by entity, action, actor, and date | --- @@ -153,14 +153,14 @@ cdisc-concept-curation/ │ ├── ncit.py # GET /ncit/search and GET /ncit/concept/ JSON endpoints with full metadata, children, and NCIt Browser links │ ├── loinc.py # GET /loinc/search JSON API endpoint │ ├── specializations.py # Dataset specializations and CRF mappings -│ ├── governance.py # Kanban board and status workflows +│ ├── governance.py # Kanban board and status workflows; `/governance/export` route for exporting published BCs as XLSX │ └── audit.py # Immutable change log with filters ├── services/ │ ├── cdisc_api.py # CDISC Library API client with stale-while-refresh caching (5-min fresh TTL, 1-hour stale fallback) │ ├── ncit_api.py # NCI EVS REST API client with in-memory caching (5-min fresh TTL, 1-hour stale fallback); search uses include="summary" for richer metadata; full concept detail with all definitions prioritized by source via `_pick_definition()` helper (CDISC > NCI > first available), parent and child concepts with codes, semantic type, and NCIt Browser reference links │ ├── loinc_api.py # NLM Clinical Tables API client (optional Basic Auth, metadata caching) │ ├── ingestion.py # File parser and AI field mapper -│ └── export.py # XLSX, JSON, ODM-XML export +│ └── export.py # XLSX, JSON, ODM-XML export; `export_governance_xlsx()` exports stage-3 BCs in BC_LB worksheet format (BC fields, DEC fields, History of Change) ├── templates/ │ ├── base.html # Bootstrap 5 sidebar layout │ └── *.html # One template per screen diff --git a/routes/bc.py b/routes/bc.py index 7e5de9a..4bde45d 100644 --- a/routes/bc.py +++ b/routes/bc.py @@ -194,9 +194,9 @@ def create(): bc_categories=request.form.get("bc_categories", ""), synonyms=request.form.get("synonyms", ""), result_scales=request.form.get("result_scales", ""), - system=request.form.get("system", ""), - system_name=request.form.get("system_name", ""), loinc_code=request.form.get("loinc_code", ""), + system=request.form.get("system", "") if request.form.get("loinc_code", "").strip() else "", + system_name=request.form.get("system_name", "") if request.form.get("loinc_code", "").strip() else "", loinc_metadata=request.form.get("loinc_metadata", "") or None, ncit_metadata=request.form.get("ncit_metadata", "") or None, package_date=request.form.get("package_date", ""), @@ -231,10 +231,10 @@ def edit(bc_id): bc.bc_categories = request.form.get("bc_categories", bc.bc_categories) bc.synonyms = request.form.get("synonyms", bc.synonyms) bc.result_scales = request.form.get("result_scales", bc.result_scales) - bc.system = request.form.get("system", bc.system) - bc.system_name = request.form.get("system_name", bc.system_name) new_loinc_code = (request.form.get("loinc_code", "") or "").strip() or None bc.loinc_code = new_loinc_code + bc.system = request.form.get("system", bc.system) if new_loinc_code else "" + bc.system_name = request.form.get("system_name", bc.system_name) if new_loinc_code else "" bc.loinc_metadata = (request.form.get("loinc_metadata", "") or bc.loinc_metadata) if new_loinc_code else None bc.package_date = request.form.get("package_date", bc.package_date) bc.updated_at = datetime.now(timezone.utc) @@ -282,6 +282,8 @@ def clear_loinc(bc_id): before = bc.to_dict() bc.loinc_code = None bc.loinc_metadata = None + bc.system = "" + bc.system_name = "" bc.updated_at = datetime.now(timezone.utc) db.session.add( AuditLog( diff --git a/routes/governance.py b/routes/governance.py index 62050cd..91c81a8 100644 --- a/routes/governance.py +++ b/routes/governance.py @@ -1,9 +1,10 @@ -from flask import Blueprint, render_template, request, redirect, url_for, flash, jsonify +from flask import Blueprint, render_template, request, redirect, url_for, flash, jsonify, Response from models.bc import BiomedicalConcept from models.governance import GovernanceRecord from models.audit import AuditLog from extensions import db from datetime import datetime, timezone +from services.export import export_governance_xlsx bp = Blueprint("governance", __name__) @@ -23,6 +24,23 @@ def board(): ) +@bp.route("/export") +def export(): + filename = request.args.get("filename", "governance_export").strip() or "governance_export" + base = filename.rsplit(".", 1)[0] if "." in filename else filename + safe_filename = f"{base}.xlsx" + + stage3_bc_ids = db.session.query(GovernanceRecord.bc_id).filter(GovernanceRecord.stage == 3).distinct() + bcs = BiomedicalConcept.query.filter(BiomedicalConcept.bc_id.in_(stage3_bc_ids)).all() + + buf = export_governance_xlsx(bcs) + return Response( + buf, + mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + headers={"Content-Disposition": f'attachment; filename="{safe_filename}"'}, + ) + + @bp.route("/advance/", methods=["POST"]) def advance(bc_id): bc = db.get_or_404(BiomedicalConcept, bc_id) diff --git a/services/export.py b/services/export.py index d8ac54f..2dc0dcb 100644 --- a/services/export.py +++ b/services/export.py @@ -64,6 +64,80 @@ def export_xlsx(bc_list): return buf +GOVERNANCE_BC_FIELDS = [ + "package_date", + "short_name", + "bc_id", + "ncit_code", + "parent_bc_id", + "bc_categories", + "synonyms", + "result_scales", + "definition", + "system", + "system_name", + "code", +] +GOVERNANCE_DEC_FIELDS = ["dec_id", "ncit_dec_code", "dec_label", "data_type", "example_set"] +GOVERNANCE_HEADERS = GOVERNANCE_BC_FIELDS + GOVERNANCE_DEC_FIELDS + ["History of Change"] + + +def export_governance_xlsx(bc_objects): + """Export BiomedicalConcept ORM objects in BC_LB worksheet format. + + Matches the column layout of the BC_LB sheet in the reference file: + 18 columns — BC fields, DEC fields, then History of Change. + One BC-only row is written per BC, followed by one row per DEC + (all BC fields repeated on each DEC row). + Returns a BytesIO object. + """ + if openpyxl is None: + raise ImportError("openpyxl is required for XLSX export") + + wb = openpyxl.Workbook() + ws = wb.active + ws.title = "BC_LB" + + header_fill = PatternFill("solid", fgColor="003366") + header_font_white = Font(bold=True, color="FFFFFF") + for col_idx, header in enumerate(GOVERNANCE_HEADERS, start=1): + cell = ws.cell(row=1, column=col_idx, value=header) + cell.font = header_font_white + cell.fill = header_fill + cell.alignment = Alignment(horizontal="center") + + row_idx = 2 + for bc in bc_objects: + bc_vals = {f: getattr(bc, f, "") or "" for f in GOVERNANCE_BC_FIELDS} + bc_vals["code"] = bc.loinc_code or "" + if not bc.loinc_code: + bc_vals["system"] = "" + bc_vals["system_name"] = "" + bc_vals["History of Change"] = bc.history_of_change or "" + + # BC-only row (DEC columns left blank) + for col_idx, header in enumerate(GOVERNANCE_HEADERS, start=1): + ws.cell(row=row_idx, column=col_idx, value=bc_vals.get(header, "")) + row_idx += 1 + + # One row per DEC, repeating BC fields + for dec in bc.decs.order_by("sort_order"): + for col_idx, header in enumerate(GOVERNANCE_HEADERS, start=1): + if header in GOVERNANCE_BC_FIELDS: + val = bc_vals[header] + elif header == "History of Change": + val = bc_vals["History of Change"] + else: + val = getattr(dec, header, "") or "" + ws.cell(row=row_idx, column=col_idx, value=val) + row_idx += 1 + + buf = io.BytesIO() + wb.save(buf) + buf.seek(0) + return buf + + def export_odm_xml(bc_list): """Export BCs as ODM-XML string.""" if etree is None: diff --git a/templates/governance.html b/templates/governance.html index 91c2f7b..85fb318 100644 --- a/templates/governance.html +++ b/templates/governance.html @@ -202,7 +202,42 @@

Governance Actions

View Full Audit Trail + + + + + {% endblock %} diff --git a/tests/test_bc_routes.py b/tests/test_bc_routes.py index 1c99ddf..d0eb660 100644 --- a/tests/test_bc_routes.py +++ b/tests/test_bc_routes.py @@ -81,6 +81,20 @@ def test_create_writes_audit_log(self, client, app): log = AuditLog.query.filter_by(entity_id="C00001", action="created").first() assert log is not None + def test_create_does_not_set_system_without_loinc_code(self, client, app): + client.post("/bc/", data=_bc_form(system="http://loinc.org/", system_name="LOINC", loinc_code="")) + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C00001") + assert bc.system in (None, "") + assert bc.system_name in (None, "") + + def test_create_sets_system_when_loinc_code_provided(self, client, app): + client.post("/bc/", data=_bc_form(system="http://loinc.org/", system_name="LOINC", loinc_code="4548-4")) + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C00001") + assert bc.system == "http://loinc.org/" + assert bc.system_name == "LOINC" + def test_create_with_decs(self, client, app): data = _bc_form() data["dec_label[]"] = ["Systolic", "Diastolic"] @@ -271,6 +285,32 @@ def test_edit_clears_loinc_code_and_metadata_when_submitted_empty(self, client, assert bc.loinc_code is None assert bc.loinc_metadata is None + def test_edit_clears_system_and_system_name_when_loinc_code_cleared(self, client, app, sample_bc): + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C12345") + bc.loinc_code = "4548-4" + bc.system = "http://loinc.org/" + bc.system_name = "LOINC" + db.session.commit() + client.post("/bc/C12345/edit", data={"loinc_code": ""}) + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C12345") + assert bc.system in (None, "") + assert bc.system_name in (None, "") + + def test_edit_preserves_system_when_loinc_code_present(self, client, app, sample_bc): + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C12345") + bc.loinc_code = "4548-4" + bc.system = "http://loinc.org/" + bc.system_name = "LOINC" + db.session.commit() + client.post("/bc/C12345/edit", data={"loinc_code": "4548-4", "system": "http://loinc.org/", "system_name": "LOINC"}) + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C12345") + assert bc.system == "http://loinc.org/" + assert bc.system_name == "LOINC" + def test_edit_strips_whitespace_from_ncit_code(self, client, app, sample_bc): client.post("/bc/C12345/edit", data={"ncit_code": " "}) with app.app_context(): @@ -358,6 +398,19 @@ def test_clear_loinc_writes_audit_log(self, client, app, sample_bc): log = AuditLog.query.filter_by(entity_id="C12345", action="loinc_cleared").first() assert log is not None + def test_clear_loinc_also_clears_system_and_system_name(self, client, app, sample_bc): + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C12345") + bc.loinc_code = "4548-4" + bc.system = "http://loinc.org/" + bc.system_name = "LOINC" + db.session.commit() + client.post("/bc/C12345/clear-loinc", follow_redirects=False) + with app.app_context(): + bc = db.session.get(BiomedicalConcept, "C12345") + assert bc.system in (None, "") + assert bc.system_name in (None, "") + def test_clear_loinc_nonexistent_bc_returns_404(self, client): r = client.post("/bc/NOTREAL/clear-loinc") assert r.status_code == 404 diff --git a/tests/test_governance_routes.py b/tests/test_governance_routes.py index 263c2ee..9aba42a 100644 --- a/tests/test_governance_routes.py +++ b/tests/test_governance_routes.py @@ -104,3 +104,129 @@ def test_reject_ajax_returns_json(self, client, sample_bc): def test_reject_nonexistent_bc_returns_404(self, client): r = client.post("/governance/reject/NOPE") assert r.status_code == 404 + + +class TestGovernanceExport: + def test_export_returns_xlsx(self, client, app, sample_bc): + for _ in range(3): + client.post("/governance/advance/C12345") + r = client.get("/governance/export") + assert r.status_code == 200 + assert "spreadsheetml" in r.content_type + + def test_export_filename_in_content_disposition(self, client, app, sample_bc): + for _ in range(3): + client.post("/governance/advance/C12345") + r = client.get("/governance/export?filename=my_report") + assert "my_report.xlsx" in r.headers["Content-Disposition"] + + def test_export_enforces_xlsx_extension(self, client, app, sample_bc): + for _ in range(3): + client.post("/governance/advance/C12345") + r = client.get("/governance/export?filename=my_report.csv") + assert "my_report.xlsx" in r.headers["Content-Disposition"] + + def test_export_excludes_non_stage3_bcs(self, client, app, sample_bc): + # BC stays provisional — no stage-3 governance record + r = client.get("/governance/export") + assert r.status_code == 200 + import io + import openpyxl + + wb = openpyxl.load_workbook(io.BytesIO(r.data)) + ws = wb.active + assert ws.max_row == 1 # header row only + + def test_export_includes_stage3_bcs(self, client, app, sample_bc): + for _ in range(3): + client.post("/governance/advance/C12345") + r = client.get("/governance/export") + import io + import openpyxl + + wb = openpyxl.load_workbook(io.BytesIO(r.data)) + ws = wb.active + assert ws.max_row >= 2 # at least one BC data row + + def test_export_system_columns_blank_without_loinc_code(self, client, app): + with app.app_context(): + from extensions import db as _db + + bc = BiomedicalConcept( + bc_id="C99998", + short_name="No LOINC Concept", + ncit_code="C99998", + system="http://loinc.org/", + system_name="LOINC", + status="provisional", + ) + _db.session.add(bc) + _db.session.commit() + for _ in range(3): + client.post("/governance/advance/C99998") + r = client.get("/governance/export") + import io + import openpyxl + + wb = openpyxl.load_workbook(io.BytesIO(r.data)) + ws = wb.active + headers = [ws.cell(row=1, column=c).value for c in range(1, ws.max_column + 1)] + system_col = headers.index("system") + 1 + system_name_col = headers.index("system_name") + 1 + assert ws.cell(row=2, column=system_col).value in (None, "") + assert ws.cell(row=2, column=system_name_col).value in (None, "") + + def test_export_system_columns_populated_with_loinc_code(self, client, app): + with app.app_context(): + from extensions import db as _db + + bc = BiomedicalConcept( + bc_id="C99997", + short_name="LOINC System Concept", + ncit_code="C99997", + loinc_code="12345-6", + system="http://loinc.org/", + system_name="LOINC", + status="provisional", + ) + _db.session.add(bc) + _db.session.commit() + for _ in range(3): + client.post("/governance/advance/C99997") + r = client.get("/governance/export") + import io + import openpyxl + + wb = openpyxl.load_workbook(io.BytesIO(r.data)) + ws = wb.active + headers = [ws.cell(row=1, column=c).value for c in range(1, ws.max_column + 1)] + system_col = headers.index("system") + 1 + system_name_col = headers.index("system_name") + 1 + assert ws.cell(row=2, column=system_col).value == "http://loinc.org/" + assert ws.cell(row=2, column=system_name_col).value == "LOINC" + + def test_export_code_column_uses_loinc_code(self, client, app): + with app.app_context(): + from extensions import db as _db + + bc = BiomedicalConcept( + bc_id="C99999", + short_name="LOINC Test Concept", + ncit_code="C99999", + loinc_code="12345-6", + status="provisional", + ) + _db.session.add(bc) + _db.session.commit() + for _ in range(3): + client.post("/governance/advance/C99999") + r = client.get("/governance/export") + import io + import openpyxl + + wb = openpyxl.load_workbook(io.BytesIO(r.data)) + ws = wb.active + headers = [ws.cell(row=1, column=c).value for c in range(1, ws.max_column + 1)] + code_col = headers.index("code") + 1 + data_row = ws.cell(row=2, column=code_col).value + assert data_row == "12345-6" From 2c3825cf8bb3a11cf33d5065635cb53a38608bfd Mon Sep 17 00:00:00 2001 From: pendingintent <3921919+pendingintent@users.noreply.github.com> Date: Wed, 8 Jul 2026 14:47:24 -0400 Subject: [PATCH 17/36] =?UTF-8?q?=F0=9F=93=9D=20Docs/Config:=20fix=20CLAUD?= =?UTF-8?q?E.md=20drift,=20rewrite=20mike=20agent,=20add=20project=20skill?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CLAUDE.md: document loinc blueprint (8 total), correct port to 8081, add Testing section (pytest command, conftest isolation, pre-commit, CI) - Rewrite mike.md PM agent for this project (was mis-templated from another repo; referenced non-existent docs/modules/ and phantom agents) - Add project-level cdisc-concept-explorer agent adopting the orphaned agent-memory (verified COSMOS v2 endpoint reference) - Fix stale cdisc-frontend-dev memory (loinc blueprint, IngestionRecord DB queue not session, cdisc_api is a full client, db imports from extensions) - New skills: run-concept-curation (smoke.sh, 14 checks, throwaway DB; verified passing) and code-review (project checklist) - (.claude/settings.json also updated locally: Stop hook narrowed to README-PROGRESS.md changelog; file is gitignored) --- .../cdisc-frontend-dev/project_foundation.md | 11 +- .claude/agents/cdisc-concept-explorer.md | 67 ++++++ .claude/agents/mike.md | 226 ++++-------------- .claude/skills/code-review/SKILL.md | 59 +++++ .claude/skills/run-concept-curation/SKILL.md | 49 ++++ .claude/skills/run-concept-curation/smoke.sh | 90 +++++++ CLAUDE.md | 24 +- 7 files changed, 343 insertions(+), 183 deletions(-) create mode 100644 .claude/agents/cdisc-concept-explorer.md create mode 100644 .claude/skills/code-review/SKILL.md create mode 100644 .claude/skills/run-concept-curation/SKILL.md create mode 100755 .claude/skills/run-concept-curation/smoke.sh diff --git a/.claude/agent-memory/cdisc-frontend-dev/project_foundation.md b/.claude/agent-memory/cdisc-frontend-dev/project_foundation.md index ed075af..b091d7f 100644 --- a/.claude/agent-memory/cdisc-frontend-dev/project_foundation.md +++ b/.claude/agent-memory/cdisc-frontend-dev/project_foundation.md @@ -11,6 +11,7 @@ The app uses a Flask application factory pattern in `app.py`. `db` and `migrate` - `ingestion` → `/ingestion` - `bc` → `/bc` - `ncit` → `/ncit` +- `loinc` → `/loinc` - `specializations` → `/specializations` - `governance` → `/governance` - `audit` → `/audit` @@ -22,8 +23,10 @@ Route files live in `routes/.py`. Each exports `bp = Blueprint('', _ - `specialization.py` — `DatasetSpecialization` - `governance.py` — `GovernanceRecord` - `audit.py` — `AuditLog` +- `ingestion.py` — `IngestionRecord` (staging rows for the upload queue) -`models/__init__.py` imports `db` from `app`. Models import `db` directly from `app` (not from `models`). +All models import `db` from `extensions` (`from extensions import db`) — never +from `app`. `extensions.py` holds the shared `db`/`migrate` singletons. Database: SQLite at `sqlite:///cdisc_curation.db` by default. Tables are auto-created via `db.create_all()` inside `create_app()`. @@ -34,14 +37,16 @@ Virtual environment: `.venv/` — use `.venv/bin/pip` and `.venv/bin/python` for **Services** (`services/`): - `ingestion.py` — `parse_xlsx`, `parse_csv`, `parse_json`, `deduplicate`, `map_fields`, `validate_bc` - `ncit_api.py` — `NCItApiClient` with `search_concept(term, size)`, `get_concept(ncit_code)`, `get_preferred_name(ncit_code)` +- `loinc_api.py` — `LoincApiClient.search(term)` against NLM Clinical Tables (optional Basic Auth) - `export.py` — `export_json(bc_list)`, `export_xlsx(bc_list)` (returns BytesIO), `export_odm_xml(bc_list)` (returns str) -- `cdisc_api.py` — `CDISCApiClient` (stub for CDISC Library API) +- `cdisc_api.py` — `CDISCApiClient`, a full CDISC Library REST client with a stale-tolerant in-memory cache (not a stub) **Route implementations are complete** (as of 2026-03-27). All 24 URL rules register successfully. Key route decisions: - `bc.export` is a static path `/bc/export` — must be defined before `bc.detail` (`/bc/`) to avoid Flask treating "export" as a bc_id. - `governance` blueprint has no `/` route — board is at `/governance/board`. - `ncit.index` at `/ncit/` redirects to `ncit.mapping`. -- Ingestion queue stored in Flask `session`, capped at 100 records. +- Ingestion queue stored in the `IngestionRecord` DB table (NOT the Flask + session — session storage was replaced to avoid cookie overflow). - `_save_decs()` in `bc.py` does a full delete-then-reinsert of DECs on every save. **Why:** Routes were stubs; wired up 2026-03-27 to connect templates to real model queries and service calls. diff --git a/.claude/agents/cdisc-concept-explorer.md b/.claude/agents/cdisc-concept-explorer.md new file mode 100644 index 0000000..f7fc28f --- /dev/null +++ b/.claude/agents/cdisc-concept-explorer.md @@ -0,0 +1,67 @@ +--- +name: cdisc-concept-explorer +description: "Use this agent when work in the cdisc-concept-curation project needs live CDISC Library data: searching Biomedical Concepts by name or category, comparing a locally curated BC against the published Library version, finding dataset specializations for a BC, or recommending which published concept a curated draft should align with.\n\n\nContext: The user is curating a draft BC and wants to check for an existing published equivalent.\nuser: \"Is there already a published CDISC BC for 'Systolic Blood Pressure' that our draft duplicates?\"\nassistant: \"I'll use the cdisc-concept-explorer agent to search the CDISC Library API and compare candidates against the draft.\"\n\nDuplicate detection against the live Library is this agent's core job in the curation workflow.\n\n\n\n\nContext: The user wants dataset specializations for a BC in the review queue.\nuser: \"What SDTM dataset specializations exist for C64796?\"\nassistant: \"Let me launch the cdisc-concept-explorer agent to query the Library's specializations endpoint for C64796.\"\n\nSpecialization lookup requires live API access — use the cdisc-concept-explorer agent.\n\n" +tools: Read, Bash, WebFetch, ToolSearch, Write, Edit +model: sonnet +memory: project +--- + +You are an expert CDISC standards specialist with deep knowledge of the CDISC +Biomedical Concepts (BC) library, controlled terminology, and clinical trial +data standards. You support the **cdisc-concept-curation** project — a Flask +app where draft BCs move through ingest → SME review → governance approval → +publish. + +## Your Core Mission +Help users search, evaluate, and compare CDISC Biomedical Concepts from the +live CDISC Library API, especially to (a) detect duplicates between locally +curated drafts and published concepts, (b) enrich drafts with authoritative +metadata, and (c) find dataset specializations tied to a BC. + +## Environment & API Access +- Base URL: `https://library.cdisc.org/api/cosmos/v2` (the app's + `services/cdisc_api.py` uses `https://api.library.cdisc.org/api/cosmos/v2` — + both hosts serve the same API) +- **Primary auth header**: `api-key: ` using `CDISC_API_KEY` env var +- Fallback header (only if `CDISC_API_KEY` is unset): + `Ocp-Apim-Subscription-Key` using `CDISC_SUBSCRIPTION_KEY` +- Key endpoints (details, response shapes, and quirks are documented in your + memory file `reference_api_endpoints.md` — consult it first): + - GET /mdr/bc/biomedicalconcepts — search all BCs + - GET /mdr/bc/biomedicalconcepts/{id} — one BC's full detail + - GET /mdr/bc/categories — list categories + - GET /mdr/specializations/datasetspecializations?biomedicalconcept={id} + +## Project Integration Points +- Local drafts live in the `biomedical_concepts` table + (`models/bc.py: BiomedicalConcept`, PK = NCIt C-code `bc_id`). +- The app's own Library client is `services/cdisc_api.py: CDISCApiClient` + (`get_biomedical_concepts()`, `get_bc(id)`, `check_duplicate(short_name)`). + Prefer reading through it when reasoning about app behavior; use curl for + ad-hoc exploration. +- The `/bc/library/` route renders a published BC for comparison. + +## CRITICAL: API-First Policy +- You MUST attempt the CDISC Library API before using any other source. +- If neither `CDISC_API_KEY` nor `CDISC_SUBSCRIPTION_KEY` is set → STOP and + tell the user to set one. Never return training-data C-codes. +- If the API errors → STOP and report the HTTP status. Do not substitute + training-data values. +- Training knowledge MAY be used only to suggest search terms, never for BC + identifiers or C-codes. + +## Output Format +### Search Results — candidate BCs with key details +### Recommendation — primary pick with rationale, alternatives with when-to-prefer +### Curation Notes — duplicate risk vs local drafts, metadata worth copying into the draft, deprecation flags + +## Quality Standards +- Never guess a BC identifier — verify against the API +- Prefer official CDISC terminology over informal names +- Flag deprecated concepts and newer package versions +- If no exact match exists, say so and recommend the closest fit + +**Update your agent memory** as you discover BC mappings, API response +quirks, category coverage, and duplicate-detection patterns in this project. +`reference_api_endpoints.md` in your memory directory already documents +verified endpoint shapes — keep it current. diff --git a/.claude/agents/mike.md b/.claude/agents/mike.md index 30c07fa..acc6332 100644 --- a/.claude/agents/mike.md +++ b/.claude/agents/mike.md @@ -1,69 +1,66 @@ --- name: mike -description: mike is your Project Manager Assistant. She tracks daily progress, updates README-PROGRESS.md, manages GitHub commits with detailed messages, and maintains project documentation. Use PROACTIVELY for progress updates, commits, and project status summaries. +description: mike is the Project Manager Assistant for the cdisc-concept-curation project. Tracks daily progress, updates README-PROGRESS.md, prepares detailed commit messages, and maintains project documentation. Use PROACTIVELY for progress updates, commits, and project status summaries. model: sonnet memory: project --- -You are mike, the dedicated Project Manager Assistant for the Kanojo project. You help track progress, manage documentation, and handle GitHub operations with a professional and organized approach. +You are mike, the dedicated Project Manager Assistant for the +**cdisc-concept-curation** project — a Flask web app for curating CDISC +Biomedical Concepts (ingest → SME review → governance approval → publish). +You help track progress, manage documentation, and handle git operations +with a professional and organized approach. ## Your Responsibilities ### 1. Progress Tracking -- Update `README-PROGRESS.md` with daily changelog entries -- Aggregate status from all `docs/modules/*-status.md` files -- Maintain feature status overview table -- Track milestones and roadmap items +- Update `README-PROGRESS.md` (repo root) with daily changelog entries +- Maintain its feature status overview table +- Track milestones against the project's SMART goals (>=90% mapping + accuracy, <5 min ingest-to-queue) -### 2. GitHub Operations +### 2. Git Operations - Create comprehensive commit messages -- Push changes to remote repository -- **Always ask before committing** - never auto-commit +- **Always ask before committing** — never auto-commit - Support PR creation with proper descriptions ### 3. Documentation Management -- Keep progress documentation current -- Cross-reference module status files -- Maintain project health overview +- Keep `README-PROGRESS.md` current after every work session +- Flag drift between docs (`README.md`, `CLAUDE.md`) and code — e.g. + blueprint counts, ports, env vars — and offer to fix it - Archive completed milestones ## Project Structure Knowledge -### Progress Files +### Progress & docs files ``` -README-PROGRESS.md # Main progress changelog (root) -docs/modules/ -├── markdown-status.md # Markdown Viewer -├── themetypo-status.md # Theme & Typography -├── mike-status.md # Your own status -└── [module]-status.md # Other modules +README-PROGRESS.md # Feature status table + daily changelog (root) +README.md # User/setup documentation +CLAUDE.md # Claude Code guidance (architecture, conventions) ``` +There is no `docs/` directory in this repo — all progress tracking lives +in `README-PROGRESS.md`. -### Subagent Team +### Agent team ``` .claude/agents/ -├── mike.md # You (PM Assistant) -├── usdm-implementation-expert.md # USDM expert -├── security-auditor.md # Security - +├── mike.md # You (PM Assistant) +├── cdisc-frontend-dev.md # Jinja/Flask/CSS/JS front-end work +└── cdisc-concept-explorer.md # CDISC Library / concept lookup ``` -## Progress Update Format - -### README-PROGRESS.md Structure -```markdown -# Project Progress - -## Overview -Brief project description and current phase. +### App shape (for status summaries) +8 Flask blueprints (`dashboard`, `ingestion`, `bc`, `ncit`, `loinc`, +`specializations`, `governance`, `audit`), services layer for external +APIs (CDISC Library, NCI EVS, NLM LOINC), SQLAlchemy models with an +immutable `AuditLog`, tests in `tests/` run with `pytest --tb=short`. -## Feature Status -| Module | Status | Details | -|--------|--------|---------| -| Name | ✅/🚧/📋 | [Link](docs/modules/x-status.md) | +## README-PROGRESS.md Format -## Daily Changelog +Follow the file's existing structure — feature status table plus a daily +changelog with dated sections: +```markdown ### [Date] - ✅ Completed item - 🚧 In progress item @@ -72,163 +69,40 @@ Brief project description and current phase. - 📝 Documentation update ``` -### Status Indicators -- ✅ Complete -- 🚧 In Progress -- 📋 Planned/Todo -- 🐛 Bug Fix -- 📝 Documentation -- ⚠️ Needs Attention -- 🔄 Refactored +Status indicators: ✅ Complete · 🚧 In Progress · 📋 Planned · 🐛 Bug Fix · +📝 Documentation · ⚠️ Needs Attention · 🔄 Refactored ## Git Commit Guidelines ### Before Committing -1. Run `git status` to see all changes -2. Run `git diff` to review changes -3. Check recent commits for message style -4. **Ask user for confirmation** +1. Run `git status` and `git diff` to review changes +2. Check recent commits for message style (`git log --oneline -10`) +3. Confirm the test suite passes (`pytest --tb=short`) +4. **Ask the user for confirmation** ### Commit Message Format ``` [emoji] [Type]: [Brief description] [Detailed bullet points of changes] - -🤖 Generated with [Claude Code](https://claude.com/claude-code) - -Co-Authored-By: Claude ``` -### Commit Type Emojis -- ✅ Feature complete -- 🚧 Work in progress -- 🐛 Bug fix -- 📝 Documentation -- 🔄 Refactor -- ⚡️ Performance -- 🎨 Style/UI -- 🧪 Tests -- 🔧 Config - -### Example Commit Message -``` -✅ Subagents: Created PM Assistant (mike) - -- Created mike.md subagent for project management -- Added README-PROGRESS.md for daily changelog tracking -- Added mike-status.md for self-tracking -- Established progress update workflow - -🤖 Generated with [Claude Code](https://claude.com/claude-code) - -Co-Authored-By: Claude -``` +Type emojis: ✅ Feature · 🚧 WIP · 🐛 Bug fix · 📝 Docs · 🔄 Refactor · +⚡️ Performance · 🎨 Style/UI · 🧪 Tests · 🔧 Config ## Workflow Patterns ### After Work Session 1. Summarize what was accomplished -2. Update `README-PROGRESS.md` with new changelog entry -3. Update relevant `docs/modules/*-status.md` -4. Ask user if they want to commit and push - -### Daily Progress Update -```markdown -### [Today's Date] -- [List all completed items] -- [List items in progress] -- [Note any blockers or issues] -``` - -### Weekly Summary (Optional) -- Aggregate week's progress -- Update milestone status -- Note upcoming priorities - -## Communication Style - -### Professional & Organized -- Clear, concise updates -- Bullet points for readability -- Consistent formatting -- Emoji indicators for quick scanning +2. Update `README-PROGRESS.md` with a new changelog entry +3. Ask the user if they want to commit ### Always Ask Before -- Committing changes -- Pushing to remote +- Committing or pushing - Creating PRs -- Any destructive operations - -### Proactive Updates -- Suggest progress updates after sessions -- Remind about uncommitted changes -- Offer to summarize module statuses - -## Integration with Other Agents - -### After Module Work -When a module agent (security-auditor, usdm-implementation-expert, etc.) completes work: -1. They update their `docs/modules/[module]-status.md` -2. You aggregate into `README-PROGRESS.md` -3. You handle the commit - -### Coordination -- Reference other agents' status files -- Don't duplicate detailed information -- Link to module status for details -- Summarize at high level - -## Documentation Requirements - -**IMPORTANT**: After completing any PM work session, you MUST update `docs/modules/mike-status.md` with: +- Any destructive operation -```markdown -## Session Update - -**Agent ID**: [Your agent ID from this session] -**Date**: [Current date] - -### What We Did Today -- [Progress updates made] -- [Commits created] -- [Documentation updated] - -### Current Status -- [Overall project health] -- [Modules needing attention] -- [Upcoming priorities] - -### Files Modified -- [List of files updated] -``` - ---- - -## Quick Reference - -### Common Tasks - -**Update daily progress:** -``` -> Ask mike to update today's progress -``` - -**Commit and push:** -``` -> Have mike commit the current changes -``` - -**Status summary:** -``` -> Ask mike for a project status summary -``` - -**Weekly report:** -``` -> Have mike create a weekly progress report -``` - ---- - -Remember: You're here to help keep the project organized and well-documented. Always be thorough with commit messages, keep progress tracking current, and maintain clear communication with the user about all operations. \ No newline at end of file +## Communication Style +- Clear, concise updates with bullet points +- Emoji indicators for quick scanning +- Summarize at a high level; link to files for detail diff --git a/.claude/skills/code-review/SKILL.md b/.claude/skills/code-review/SKILL.md new file mode 100644 index 0000000..188e7eb --- /dev/null +++ b/.claude/skills/code-review/SKILL.md @@ -0,0 +1,59 @@ +--- +name: code-review +description: Project-specific code review checklist for cdisc-concept-curation. Use when asked to review changes, a diff, or a PR in this repo. +--- + +# Code review for cdisc-concept-curation + +Review the working diff (`git diff` / `git diff --staged`, or the range the +user names). Verify each finding against the actual code before reporting. +Rank findings by severity; note file:line. + +## Project-specific checks (highest value) + +1. **Layering** — routes handle HTTP only; business logic belongs in + `services/`; DB models in `models/`. New SQL/ORM queries embedded in + templates or scattered helpers are a flag. Models must import `db` from + `extensions`, never from `app`. +2. **Audit coverage** — every mutation of `BiomedicalConcept`, + `DataElementConcept`, `GovernanceRecord`, or `IngestionRecord` must write + an `AuditLog` row (before/after state as dicts). A commit that changes + data without an audit entry is a defect. (Known historical gap: + `ncit.resolve`.) +3. **Schema changes** — any change to a model requires an Alembic revision + in `migrations/versions/` (`flask db migrate`), not just the model edit. + Watch for edits that rely on `db.create_all()` picking up new columns — + it does not alter existing tables. +4. **External API clients** (`services/cdisc_api.py`, `ncit_api.py`, + `loinc_api.py`) — errors are returned as `{"error": ...}` values, and + callers must check for the `"error"` key before using results. New call + sites that index into a result without that check will crash on API + failure. Every caught exception must be logged. +5. **Test isolation** — tests must use `TestConfig` (in-memory SQLite) and + mock external HTTP (`unittest.mock.patch` / `monkeypatch`). Any test that + hits a real API or touches `instance/cdisc_curation.db` is a defect. +6. **TDD convention** — behavior changes should come with test changes in + `tests/`. Flag code changes whose tests were not updated. +7. **Route ordering** — static paths must be registered before parameterized + ones in the same blueprint (e.g. `/bc/export` before `/bc/`). + +## General checks + +- Broad `except Exception` where a specific exception fits; silent failure + paths without logging. +- Secrets: no keys/tokens in code; config comes from `config.py` env vars. +- Duplicated blocks that should be a helper (the repeated + `AuditLog(...) + add + commit` pattern is the canonical example). +- SQL injection is unlikely via the ORM, but flag any raw `text()`/string + SQL with interpolated input. +- Jinja templates: user-supplied values must not be marked `|safe`. + +## Verification + +Run before approving: + +```bash +source .venv/bin/activate +pytest --tb=short +pre-commit run --all-files +``` diff --git a/.claude/skills/run-concept-curation/SKILL.md b/.claude/skills/run-concept-curation/SKILL.md new file mode 100644 index 0000000..e602e98 --- /dev/null +++ b/.claude/skills/run-concept-curation/SKILL.md @@ -0,0 +1,49 @@ +--- +name: run-concept-curation +description: Run, start, smoke-test, or verify the cdisc-concept-curation Flask server. Use when asked to run the app, confirm a change works in the live server, smoke-test the routes, or check an endpoint manually. +--- + +# Run the cdisc-concept-curation app + +## Quick smoke test (preferred) + +```bash +bash .claude/skills/run-concept-curation/smoke.sh +``` + +Starts the app on **port 9881** against a **throwaway SQLite database** in a +temp directory, runs 14 HTTP checks (dashboard, BC list/create/detail/submit/ +delete round-trip, governance board, NCIt pages, audit, ingestion, +specializations, export), prints PASS/FAIL per check, and tears everything +down. Takes ~15s. No API key required — external-API panels degrade +gracefully. + +## Manual dev server + +```bash +source .venv/bin/activate +export CDISC_API_KEY=your_key # optional; dashboard/library panels need it +python app.py # http://localhost:8081 (PORT env overrides) +``` + +## Rules + +- **Never point ad-hoc runs at `instance/cdisc_curation.db`** (the real + curation data). For experiments, always set + `DATABASE_URL=sqlite:////tmp/.db`. +- The dev DB is auto-prepared on startup; tests use in-memory SQLite via + `tests/conftest.py` and need no server. +- Useful manual checks: + ```bash + curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8081/bc/ + curl -s http://127.0.0.1:8081/bc/export | head -c 400 # JSON export + ``` + +## Troubleshooting + +- **Port in use**: `lsof -ti :8081 | xargs kill` (or set `PORT`). +- **Dashboard slow without network**: `/` fans out two CDISC Library calls + with 10s timeouts; offline it renders after the timeout with error panels — + that is expected, not a failure. +- **`CDISC_API_KEY` unset**: app runs fine; only Library-backed panels and + `/bc/library/` show errors. diff --git a/.claude/skills/run-concept-curation/smoke.sh b/.claude/skills/run-concept-curation/smoke.sh new file mode 100755 index 0000000..6c8bd78 --- /dev/null +++ b/.claude/skills/run-concept-curation/smoke.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Smoke test for cdisc-concept-curation. +# Boots the Flask app on a throwaway SQLite DB and exercises the main routes. +set -u + +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +PORT="${SMOKE_PORT:-9881}" +BASE="http://127.0.0.1:${PORT}" +TMP_DIR="$(mktemp -d)" +DB_PATH="${TMP_DIR}/smoke.db" +SERVER_LOG="${TMP_DIR}/server.log" +PASS=0 +FAIL=0 +SERVER_PID="" + +cleanup() { + if [[ -n "${SERVER_PID}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then + kill "${SERVER_PID}" 2>/dev/null + wait "${SERVER_PID}" 2>/dev/null + fi + rm -rf "${TMP_DIR}" +} +trap cleanup EXIT + +check() { + local label="$1" expected="$2" actual="$3" + if [[ "${actual}" == "${expected}" ]]; then + echo "PASS ${label} (${actual})" + PASS=$((PASS + 1)) + else + echo "FAIL ${label} (expected ${expected}, got ${actual})" + FAIL=$((FAIL + 1)) + fi +} + +get_code() { + curl -s -o /dev/null -w '%{http_code}' --max-time 30 "$1" +} + +post_code() { + local url="$1"; shift + curl -s -o /dev/null -w '%{http_code}' --max-time 30 -X POST "$@" "${url}" +} + +cd "${REPO_DIR}" +# shellcheck disable=SC1091 +source .venv/bin/activate + +echo "Starting server on port ${PORT} (throwaway DB: ${DB_PATH})" +DATABASE_URL="sqlite:///${DB_PATH}" PORT="${PORT}" CDISC_API_KEY="" \ + python app.py >"${SERVER_LOG}" 2>&1 & +SERVER_PID=$! + +# Wait for readiness (BC list is DB-only, no external calls) +ready=0 +for _ in $(seq 1 30); do + if [[ "$(get_code "${BASE}/bc/")" == "200" ]]; then + ready=1 + break + fi + sleep 0.5 +done +if [[ "${ready}" != "1" ]]; then + echo "FAIL server did not become ready; log tail:" + tail -20 "${SERVER_LOG}" + exit 1 +fi + +check "GET /bc/ (list)" 200 "$(get_code "${BASE}/bc/")" +check "GET /bc/new (form)" 200 "$(get_code "${BASE}/bc/new")" +check "POST /bc/ (create SMOKE001)" 302 "$(post_code "${BASE}/bc/" \ + --data-urlencode 'bc_id=SMOKE001' \ + --data-urlencode 'short_name=Smoke Test BC' \ + --data-urlencode 'definition=Created by smoke.sh' \ + --data-urlencode 'submitter=smoke')" +check "GET /bc/SMOKE001 (detail)" 200 "$(get_code "${BASE}/bc/SMOKE001")" +check "POST /bc/SMOKE001/submit" 302 "$(post_code "${BASE}/bc/SMOKE001/submit")" +check "GET /governance/board" 200 "$(get_code "${BASE}/governance/board")" +check "GET /ncit/mapping" 200 "$(get_code "${BASE}/ncit/mapping")" +check "GET /ncit/search (no term)" 200 "$(get_code "${BASE}/ncit/search")" +check "GET /audit/" 200 "$(get_code "${BASE}/audit/")" +check "GET /ingestion/" 200 "$(get_code "${BASE}/ingestion/")" +check "GET /specializations/" 200 "$(get_code "${BASE}/specializations/")" +check "GET /bc/export (json)" 200 "$(get_code "${BASE}/bc/export")" +check "POST /bc/SMOKE001/delete" 302 "$(post_code "${BASE}/bc/SMOKE001/delete")" +check "GET / (dashboard, degraded API)" 200 "$(get_code "${BASE}/")" + +echo +echo "Smoke result: ${PASS} passed, ${FAIL} failed" +[[ "${FAIL}" == "0" ]] diff --git a/CLAUDE.md b/CLAUDE.md index ae3707e..1f7b9b7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -17,7 +17,7 @@ pip install -r requirements.txt export CDISC_API_KEY=your_key_here # Start dev server -python app.py # runs on http://localhost:5000 +python app.py # runs on http://localhost:8081 (override with PORT env var) ``` Database (`instance/cdisc_curation.db`) is auto-created on first run via `db.create_all()`. @@ -31,7 +31,19 @@ black --check . Line length is set to 200 (black) / 999 (flake8). Flake8 ignores F401, F841, E711 — see `.flake8` for full ignore list. -All tests are defined in the tests directory. +## Testing + +```bash +pytest --tb=short # full suite (same command CI runs) +pytest tests/test_bc_routes.py -v # single file +``` + +- Tests use an in-memory SQLite database via `TestConfig` in + `tests/conftest.py`; an autouse `clean_db` fixture drops/creates all tables + around every test. No env vars required; external API clients are mocked. +- Pre-commit (`.pre-commit-config.yaml`) runs black, flake8, and pytest on + every commit. +- CI (`.github/workflows/ci.yml`) runs `pytest --tb=short` on Python 3.12. ## Architecture @@ -43,7 +55,7 @@ All tests are defined in the tests directory. - `extensions.py` — Shared `db` and `migrate` instances (avoids circular imports — always import from here) - `tests/` - Unit tests -**7 blueprints** registered in `app.py`: +**8 blueprints** registered in `app.py`: | Blueprint | Prefix | Purpose | |-----------|--------|---------| @@ -51,6 +63,7 @@ All tests are defined in the tests directory. | `ingestion` | `/ingestion` | File upload → parse → queue → approve | | `bc` | `/bc` | BC CRUD, export (XLSX/JSON/ODM-XML) | | `ncit` | `/ncit` | NCI Thesaurus search & mapping | +| `loinc` | `/loinc` | LOINC code search (NLM Clinical Tables) | | `specializations` | `/specializations` | Dataset specialization management | | `governance` | `/governance` | 4-stage Kanban board | | `audit` | `/audit` | Immutable change log | @@ -58,6 +71,7 @@ All tests are defined in the tests directory. **Key services:** - `services/cdisc_api.py` — CDISC Library REST client with 5-min in-memory cache - `services/ncit_api.py` — NCI EVS client (no auth required) +- `services/loinc_api.py` — NLM Clinical Tables LOINC search (optional Basic Auth via `LOINC_USER`/`LOINC_PASSWORD`) - `services/ingestion.py` — XLSX/CSV/JSON parser + fuzzy field mapper (`SequenceMatcher` similarity scoring) - `services/export.py` — XLSX/JSON/ODM-XML exporter @@ -78,7 +92,9 @@ All configuration is in `config.py` via environment variables: |-----|---------|---------| | `CDISC_API_KEY` | `''` | CDISC Library API authentication | | `SECRET_KEY` | `'dev-secret-key-change-in-prod'` | Flask session secret | -| `DATABASE_URL` | `sqlite:///cdisc_curation.db` | Database connection string | +| `DATABASE_URL` | `sqlite:///cdisc_curation.db` | Database connection string (resolves to `instance/cdisc_curation.db`) | +| `PORT` | `8081` | Dev server port | +| `LOINC_USER` / `LOINC_PASSWORD` | unset | Optional Basic Auth for the NLM LOINC API | CDISC API base: `https://api.library.cdisc.org/api/cosmos/v2` NCIt API base: `https://api-evsrest.nci.nih.gov/api/v1` From 54f18456e5731227be0448608b09b4c4a1e85522 Mon Sep 17 00:00:00 2001 From: pendingintent <3921919+pendingintent@users.noreply.github.com> Date: Wed, 8 Jul 2026 14:52:06 -0400 Subject: [PATCH 18/36] =?UTF-8?q?=F0=9F=A7=AA=20Tests:=20cover=20export=20?= =?UTF-8?q?service,=20dashboard,=20specializations,=20cdisc=5Fapi=20cache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - tests/test_export_service.py: JSON/XLSX/governance-XLSX/ODM-XML exports - tests/test_cdisc_api_cache.py: _cached() fresh/stale/expired/error states and cache-key hygiene (no raw API key in keys) - tests/test_dashboard.py: ThreadPoolExecutor fan-out with mocked client, graceful degradation on API errors, local DB stats - tests/test_specializations_routes.py: full route coverage Bug fix found by the new tests: routes/specializations.py passed 'specs=' but templates/specializations.html iterates 'specializations' — the All Specializations table never rendered. Also pointed the row Edit link at the real detail route (spec.id does not exist on the model). Suite: 202 passed. --- routes/specializations.py | 4 +- templates/specializations.html | 2 +- tests/test_cdisc_api_cache.py | 117 ++++++++++++++++ tests/test_dashboard.py | 75 ++++++++++ tests/test_export_service.py | 199 +++++++++++++++++++++++++++ tests/test_specializations_routes.py | 144 +++++++++++++++++++ 6 files changed, 538 insertions(+), 3 deletions(-) create mode 100644 tests/test_cdisc_api_cache.py create mode 100644 tests/test_dashboard.py create mode 100644 tests/test_export_service.py create mode 100644 tests/test_specializations_routes.py diff --git a/routes/specializations.py b/routes/specializations.py index 7465bb5..9e77fa2 100644 --- a/routes/specializations.py +++ b/routes/specializations.py @@ -25,7 +25,7 @@ def index(): library_bcs, local_bcs = _get_bc_options() return render_template( "specializations.html", - specs=specs, + specializations=specs, library_bcs=library_bcs, local_bcs=local_bcs, page_title="Specializations", @@ -53,7 +53,7 @@ def detail(vlm_group_id): library_bcs, local_bcs = _get_bc_options() return render_template( "specializations.html", - specs=specs, + specializations=specs, library_bcs=library_bcs, local_bcs=local_bcs, edit_spec=spec, diff --git a/templates/specializations.html b/templates/specializations.html index c6f8daf..30ba7c3 100644 --- a/templates/specializations.html +++ b/templates/specializations.html @@ -213,7 +213,7 @@

All Specializations

- Edit diff --git a/tests/test_cdisc_api_cache.py b/tests/test_cdisc_api_cache.py new file mode 100644 index 0000000..a286ea0 --- /dev/null +++ b/tests/test_cdisc_api_cache.py @@ -0,0 +1,117 @@ +"""Tests for the stale-tolerant in-memory cache in services/cdisc_api.py.""" + +import time + +import pytest + +import services.cdisc_api as cdisc_api +from services.cdisc_api import CDISCApiClient, _cached + + +class TestCachedHelper: + def setup_method(self): + cdisc_api._cache.clear() + + def test_first_call_fetches_and_stores(self): + calls = [] + + def fn(): + calls.append(1) + return ["data"] + + assert _cached("k", fn) == ["data"] + assert len(calls) == 1 + assert "k" in cdisc_api._cache + + def test_fresh_entry_served_without_refetch(self): + calls = [] + + def fn(): + calls.append(1) + return ["data"] + + _cached("k", fn) + assert _cached("k", fn) == ["data"] + assert len(calls) == 1 # second call served from cache + + def test_stale_entry_refreshes_successfully(self): + cdisc_api._cache["k"] = (time.time() - cdisc_api._CACHE_TTL - 10, ["old"]) + assert _cached("k", lambda: ["new"]) == ["new"] + assert cdisc_api._cache["k"][1] == ["new"] + + def test_stale_entry_served_when_refresh_fails(self): + cdisc_api._cache["k"] = (time.time() - cdisc_api._CACHE_TTL - 10, ["old"]) + + def failing(): + raise ConnectionError("boom") + + assert _cached("k", failing) == ["old"] + + def test_expired_entry_raises_when_refresh_fails(self): + cdisc_api._cache["k"] = (time.time() - cdisc_api._CACHE_STALE_TTL - 10, ["old"]) + + def failing(): + raise ConnectionError("boom") + + with pytest.raises(ConnectionError): + _cached("k", failing) + + def test_no_entry_raises_when_fetch_fails(self): + def failing(): + raise ConnectionError("boom") + + with pytest.raises(ConnectionError): + _cached("missing", failing) + + +class TestClientCacheKeys: + def setup_method(self): + cdisc_api._cache.clear() + + def test_cache_key_excludes_raw_api_key(self, app): + with app.app_context(): + client = CDISCApiClient() + key = client._cache_key("biomedical_concepts") + assert client.api_key not in key or client.api_key == "" + assert key[2] == "biomedical_concepts" + + def test_get_biomedical_concepts_error_encoded_not_raised(self, app, monkeypatch): + """API failure is captured as [{'error': ...}] and cached, not raised.""" + + def boom(*args, **kwargs): + raise ConnectionError("no network") + + monkeypatch.setattr(cdisc_api.requests, "get", boom) + with app.app_context(): + result = CDISCApiClient().get_biomedical_concepts() + assert len(result) == 1 + assert "error" in result[0] + + def test_error_result_replaced_after_ttl(self, app, monkeypatch): + """A cached error list refreshes to real data once the TTL passes.""" + with app.app_context(): + client = CDISCApiClient() + + def boom(*args, **kwargs): + raise ConnectionError("no network") + + monkeypatch.setattr(cdisc_api.requests, "get", boom) + with app.app_context(): + assert "error" in CDISCApiClient().get_biomedical_concepts()[0] + + # Age the entry past the fresh TTL, then make the API succeed + key = client._cache_key("biomedical_concepts") + ts, data = cdisc_api._cache[key] + cdisc_api._cache[key] = (ts - cdisc_api._CACHE_TTL - 10, data) + + class FakeResp: + def raise_for_status(self): + pass + + def json(self): + return {"_links": {"biomedicalConcepts": [{"href": "/x", "title": "X"}]}} + + monkeypatch.setattr(cdisc_api.requests, "get", lambda *a, **kw: FakeResp()) + with app.app_context(): + result = CDISCApiClient().get_biomedical_concepts() + assert result == [{"href": "/x", "title": "X"}] diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py new file mode 100644 index 0000000..fba033e --- /dev/null +++ b/tests/test_dashboard.py @@ -0,0 +1,75 @@ +"""Tests for routes/dashboard.py — concurrent API fan-out and local stats.""" + +from unittest.mock import MagicMock, patch + +from extensions import db +from models.bc import BiomedicalConcept + +LIBRARY_BCS = [ + {"href": "/mdr/bc/biomedicalconcepts/C64849", "title": "Hemoglobin A1c", "type": "BC"}, + {"href": "/mdr/bc/biomedicalconcepts/C25298", "title": "Systolic BP", "type": "BC"}, +] +LIBRARY_SPECS = [ + {"href": "/mdr/specializations/sdtm/datasetspecializations/HBA1C", "title": "HBA1C", "type": "SDTM"}, +] + + +def _mock_client(bcs, specs): + client = MagicMock() + client.get_biomedical_concepts.return_value = bcs + client.get_dataset_specializations.return_value = specs + return client + + +class TestDashboardIndex: + def test_renders_with_api_data(self, client): + with patch("routes.dashboard.CDISCApiClient") as mock_cls: + mock_cls.return_value = _mock_client(LIBRARY_BCS, LIBRARY_SPECS) + resp = client.get("/") + assert resp.status_code == 200 + assert b"Hemoglobin A1c" in resp.data + + def test_renders_when_api_errors(self, client): + """API failures are encoded as error dicts — the page must still render.""" + with patch("routes.dashboard.CDISCApiClient") as mock_cls: + mock_cls.return_value = _mock_client( + [{"error": "401 Unauthorized"}], + [{"error": "timeout"}], + ) + resp = client.get("/") + assert resp.status_code == 200 + assert b"Hemoglobin A1c" not in resp.data + + def test_renders_with_empty_api_results(self, client): + with patch("routes.dashboard.CDISCApiClient") as mock_cls: + mock_cls.return_value = _mock_client([], []) + resp = client.get("/") + assert resp.status_code == 200 + + def test_local_stats_reflect_db(self, app, client, sample_bc): + with app.app_context(): + db.session.add( + BiomedicalConcept( + bc_id="C99999", + short_name="Published Concept", + status="published", + submitter="tester", + ) + ) + db.session.commit() + with patch("routes.dashboard.CDISCApiClient") as mock_cls: + mock_cls.return_value = _mock_client([], []) + resp = client.get("/") + assert resp.status_code == 200 + # Both local BCs appear in recent submissions + assert b"Test Concept" in resp.data + assert b"Published Concept" in resp.data + + def test_both_api_calls_made_concurrently(self, client): + """Both Library endpoints are requested exactly once per page load.""" + mock = _mock_client(LIBRARY_BCS, LIBRARY_SPECS) + with patch("routes.dashboard.CDISCApiClient") as mock_cls: + mock_cls.return_value = mock + client.get("/") + assert mock.get_biomedical_concepts.call_count == 1 + assert mock.get_dataset_specializations.call_count == 1 diff --git a/tests/test_export_service.py b/tests/test_export_service.py new file mode 100644 index 0000000..8a54d2c --- /dev/null +++ b/tests/test_export_service.py @@ -0,0 +1,199 @@ +"""Tests for services/export.py — JSON, XLSX, governance XLSX, ODM-XML.""" + +import json + +import openpyxl +import pytest +from lxml import etree + +from extensions import db +from models.bc import BiomedicalConcept, DataElementConcept +from services.export import ( + BC_EXPORT_FIELDS, + GOVERNANCE_HEADERS, + export_governance_xlsx, + export_json, + export_odm_xml, + export_xlsx, +) + +SAMPLE_BCS = [ + { + "bc_id": "C64849", + "short_name": "Hemoglobin A1c Measurement", + "definition": "A quantitative measurement of HbA1c.", + "ncit_code": "C64849", + "parent_bc_id": None, + "bc_categories": "Laboratory Tests", + "synonyms": "HbA1c;A1C", + "result_scales": "Quantitative", + "system": "http://loinc.org/", + "system_name": "LOINC", + "loinc_code": "4548-4", + "package_date": "2026-01-01", + "status": "provisional", + "submitter": "tester", + }, + { + "bc_id": "C25298", + "short_name": "Systolic Blood Pressure", + "definition": "The maximum arterial pressure.", + "ncit_code": "", + "parent_bc_id": None, + "bc_categories": "Vital Signs", + "synonyms": "", + "result_scales": "Quantitative", + "system": "", + "system_name": "", + "loinc_code": "", + "package_date": "", + "status": "sme_review", + "submitter": "tester", + }, +] + + +class TestExportJson: + def test_round_trip(self): + out = export_json(SAMPLE_BCS) + parsed = json.loads(out) + assert len(parsed) == 2 + assert parsed[0]["bc_id"] == "C64849" + assert parsed[1]["short_name"] == "Systolic Blood Pressure" + + def test_empty_list(self): + assert json.loads(export_json([])) == [] + + def test_non_serializable_values_coerced(self): + from datetime import datetime + + out = export_json([{"bc_id": "X", "created_at": datetime(2026, 1, 1)}]) + assert "2026-01-01" in out + + +class TestExportXlsx: + def test_returns_workbook_with_headers_and_rows(self): + buf = export_xlsx(SAMPLE_BCS) + wb = openpyxl.load_workbook(buf) + ws = wb.active + assert ws.title == "Biomedical Concepts" + headers = [c.value for c in ws[1]] + assert headers[0] == "Bc Id" + assert len(headers) == len(BC_EXPORT_FIELDS) + # Row 2 = first BC + assert ws.cell(row=2, column=1).value == "C64849" + assert ws.cell(row=3, column=2).value == "Systolic Blood Pressure" + + def test_empty_list_has_header_only(self): + wb = openpyxl.load_workbook(export_xlsx([])) + ws = wb.active + assert ws.max_row == 1 + + def test_missing_fields_render_blank(self): + wb = openpyxl.load_workbook(export_xlsx([{"bc_id": "ONLY_ID"}])) + ws = wb.active + assert ws.cell(row=2, column=1).value == "ONLY_ID" + assert ws.cell(row=2, column=2).value in ("", None) + + +class TestExportGovernanceXlsx: + def _make_bc_with_decs(self): + bc = BiomedicalConcept( + bc_id="C64849", + short_name="Hemoglobin A1c Measurement", + definition="A quantitative measurement of HbA1c.", + ncit_code="C64849", + loinc_code="4548-4", + system="http://loinc.org/", + system_name="LOINC", + history_of_change="Initial version", + status="provisional", + submitter="tester", + ) + db.session.add(bc) + db.session.add_all( + [ + DataElementConcept( + dec_id="C64849.DEC.1", + bc_id="C64849", + dec_label="Result Value", + data_type="decimal", + sort_order=0, + ), + DataElementConcept( + dec_id="C64849.DEC.2", + bc_id="C64849", + dec_label="Unit", + data_type="string", + sort_order=1, + ), + ] + ) + db.session.commit() + return bc + + def test_bc_row_then_dec_rows(self, app): + with app.app_context(): + bc = self._make_bc_with_decs() + wb = openpyxl.load_workbook(export_governance_xlsx([bc])) + ws = wb.active + assert ws.title == "BC_LB" + headers = [c.value for c in ws[1]] + assert headers == GOVERNANCE_HEADERS + # Row 2: BC-only row — DEC columns blank, loinc code in "code" col + code_col = GOVERNANCE_HEADERS.index("code") + 1 + dec_label_col = GOVERNANCE_HEADERS.index("dec_label") + 1 + assert ws.cell(row=2, column=code_col).value == "4548-4" + assert ws.cell(row=2, column=dec_label_col).value in ("", None) + # Rows 3-4: one per DEC in sort order, BC fields repeated + assert ws.cell(row=3, column=dec_label_col).value == "Result Value" + assert ws.cell(row=4, column=dec_label_col).value == "Unit" + bc_id_col = GOVERNANCE_HEADERS.index("bc_id") + 1 + assert ws.cell(row=3, column=bc_id_col).value == "C64849" + # History of Change lands in the last column + assert ws.cell(row=2, column=len(GOVERNANCE_HEADERS)).value == "Initial version" + + def test_bc_without_loinc_blanks_system_fields(self, app): + with app.app_context(): + bc = BiomedicalConcept( + bc_id="C25298", + short_name="Systolic Blood Pressure", + system="http://should-be-blanked/", + system_name="STALE", + status="provisional", + ) + db.session.add(bc) + db.session.commit() + wb = openpyxl.load_workbook(export_governance_xlsx([bc])) + ws = wb.active + system_col = GOVERNANCE_HEADERS.index("system") + 1 + system_name_col = GOVERNANCE_HEADERS.index("system_name") + 1 + assert ws.cell(row=2, column=system_col).value in ("", None) + assert ws.cell(row=2, column=system_name_col).value in ("", None) + + +class TestExportOdmXml: + def test_valid_odm_structure(self): + xml = export_odm_xml(SAMPLE_BCS) + root = etree.fromstring(xml.encode()) + ns = {"odm": "http://www.cdisc.org/ns/odm/v1.3"} + assert root.tag == "{http://www.cdisc.org/ns/odm/v1.3}ODM" + assert root.get("FileType") == "Snapshot" + item_defs = root.findall("odm:ItemDef", ns) + assert [i.get("OID") for i in item_defs] == ["C64849", "C25298"] + # Definition text present + text = item_defs[0].find("odm:Description/odm:TranslatedText", ns) + assert text.text == "A quantitative measurement of HbA1c." + + def test_alias_only_when_ncit_code_present(self): + xml = export_odm_xml(SAMPLE_BCS) + root = etree.fromstring(xml.encode()) + ns = {"odm": "http://www.cdisc.org/ns/odm/v1.3"} + item_defs = root.findall("odm:ItemDef", ns) + assert item_defs[0].find("odm:Alias", ns) is not None + assert item_defs[0].find("odm:Alias", ns).get("Name") == "C64849" + assert item_defs[1].find("odm:Alias", ns) is None + + def test_empty_list(self): + root = etree.fromstring(export_odm_xml([]).encode()) + assert len(root) == 0 diff --git a/tests/test_specializations_routes.py b/tests/test_specializations_routes.py new file mode 100644 index 0000000..9e7cf57 --- /dev/null +++ b/tests/test_specializations_routes.py @@ -0,0 +1,144 @@ +"""Tests for routes/specializations.py.""" + +from unittest.mock import MagicMock, patch + +from extensions import db +from models.bc import DataElementConcept +from models.specialization import DatasetSpecialization + +LIBRARY_LINKS = [ + {"href": "/mdr/bc/biomedicalconcepts/C64849", "title": "Hemoglobin A1c"}, +] + + +def _patch_client(**kwargs): + """Patch the CDISCApiClient used by the specializations blueprint.""" + client = MagicMock() + client.get_biomedical_concepts.return_value = kwargs.get("bcs", LIBRARY_LINKS) + client.get_specialization.return_value = kwargs.get("spec", {}) + patcher = patch("routes.specializations.CDISCApiClient", return_value=client) + return patcher, client + + +class TestIndex: + def test_lists_specializations(self, client, app, sample_bc): + with app.app_context(): + spec = DatasetSpecialization(vlm_group_id="C12345.SDTM", bc_id=sample_bc, domain="SDTM", short_name="Test Spec") + db.session.add(spec) + db.session.commit() + patcher, _ = _patch_client() + with patcher: + resp = client.get("/specializations/") + assert resp.status_code == 200 + assert b"C12345.SDTM" in resp.data + + def test_index_survives_library_error(self, client): + patcher, _ = _patch_client(bcs=[{"error": "401"}]) + with patcher: + resp = client.get("/specializations/") + assert resp.status_code == 200 + + +class TestCreate: + def test_create_persists_spec(self, client, app, sample_bc): + patcher, _ = _patch_client() + with patcher: + resp = client.post( + "/specializations/", + data={"vlm_group_id": "VLM1", "bc_id": sample_bc, "domain": "CDASH", "short_name": "Manual Spec"}, + ) + assert resp.status_code == 302 + with app.app_context(): + spec = db.session.get(DatasetSpecialization, "VLM1") + assert spec is not None + assert spec.domain == "CDASH" + assert spec.variables == [] + + def test_create_requires_ids(self, client, app): + patcher, _ = _patch_client() + with patcher: + resp = client.post("/specializations/", data={"vlm_group_id": "", "bc_id": ""}) + assert resp.status_code == 302 + with app.app_context(): + assert DatasetSpecialization.query.count() == 0 + + +class TestDetail: + def test_detail_renders(self, client, app, sample_bc): + with app.app_context(): + db.session.add(DatasetSpecialization(vlm_group_id="VLM1", bc_id=sample_bc, domain="SDTM")) + db.session.commit() + patcher, _ = _patch_client() + with patcher: + resp = client.get("/specializations/VLM1") + assert resp.status_code == 200 + + def test_detail_404_for_unknown(self, client): + patcher, _ = _patch_client() + with patcher: + resp = client.get("/specializations/NOPE") + assert resp.status_code == 404 + + +class TestLibraryDetail: + def test_renders_library_spec(self, client): + patcher, _ = _patch_client(spec={"shortName": "HBA1C Spec", "datasetSpecializationId": "HBA1C"}) + with patcher: + resp = client.get("/specializations/library/mdr/specializations/sdtm/datasetspecializations/HBA1C") + assert resp.status_code == 200 + + def test_error_redirects_to_dashboard(self, client): + patcher, _ = _patch_client(spec={"error": "404 not found"}) + with patcher: + resp = client.get("/specializations/library/mdr/specializations/sdtm/datasetspecializations/NOPE") + assert resp.status_code == 302 + assert resp.headers["Location"] in ("/", "http://localhost/") + + +class TestGenerate: + def _add_decs(self, app, bc_id): + with app.app_context(): + db.session.add_all( + [ + DataElementConcept(dec_id=f"{bc_id}.DEC.1", bc_id=bc_id, dec_label="Result", data_type="decimal", required=True, sort_order=0), + DataElementConcept(dec_id=f"{bc_id}.DEC.2", bc_id=bc_id, dec_label="Unit", data_type="string", sort_order=1), + ] + ) + db.session.commit() + + def test_generate_builds_variables_from_decs(self, client, app, sample_bc): + self._add_decs(app, sample_bc) + resp = client.post(f"/specializations/generate/{sample_bc}", data={"domain": "SDTM"}) + assert resp.status_code == 302 + with app.app_context(): + spec = db.session.get(DatasetSpecialization, f"{sample_bc}.SDTM") + assert spec is not None + assert [v["name"] for v in spec.variables] == ["Result", "Unit"] + assert spec.variables[0]["required"] is True + + def test_generate_duplicate_is_rejected(self, client, app, sample_bc): + self._add_decs(app, sample_bc) + client.post(f"/specializations/generate/{sample_bc}", data={"domain": "SDTM"}) + resp = client.post(f"/specializations/generate/{sample_bc}", data={"domain": "SDTM"}) + assert resp.status_code == 302 + with app.app_context(): + assert DatasetSpecialization.query.count() == 1 + + def test_generate_unknown_bc_404(self, client): + resp = client.post("/specializations/generate/NOPE", data={"domain": "SDTM"}) + assert resp.status_code == 404 + + +class TestGenerateFromDec: + def test_returns_variables_json(self, client, app, sample_bc): + with app.app_context(): + db.session.add(DataElementConcept(dec_id="D1", bc_id=sample_bc, dec_label="Result", data_type="decimal", sort_order=0)) + db.session.commit() + resp = client.post("/specializations/generate-from-dec", json={"bc_id": sample_bc}) + assert resp.status_code == 200 + payload = resp.get_json() + assert payload["variables"][0]["name"] == "Result" + + def test_missing_bc_id_returns_400(self, client): + resp = client.post("/specializations/generate-from-dec", json={}) + assert resp.status_code == 400 From a1a7d9e2f996fd292928e5bf06a0c5fe79100cd6 Mon Sep 17 00:00:00 2001 From: pendingintent <3921919+pendingintent@users.noreply.github.com> Date: Wed, 8 Jul 2026 14:58:58 -0400 Subject: [PATCH 19/36] =?UTF-8?q?=F0=9F=94=84=20Refactor:=20add=20logging?= =?UTF-8?q?=20throughout,=20extract=20audit=20helper=20and=20BC=20form=20m?= =?UTF-8?q?apper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - app.py: configure logging once in create_app (root untouched if another runner already installed handlers) - services/audit.py (new): log_change() queues an AuditLog row on the session; caller owns the commit so change + audit land atomically. All 8 inline AuditLog blocks in routes/bc.py and routes/governance.py now use it. - routes/bc.py: _apply_bc_form() replaces the duplicated form-to-model field mapping in create()/edit() (semantics preserved: create keeps raw values, edit normalizes cleared ncit/parent/loinc to None) - services/{cdisc_api,ncit_api,loinc_api}.py: narrow 'except Exception' to requests.RequestException + parse errors and log every caught failure with context; _cached() keeps its intentional broad catch (documented) and now logs stale-serve and hard-failure paths - services/ingestion.py: parser catches stay broad by design (arbitrary user files) but now log with tracebacks - tests: failure mocks updated to raise requests.RequestException (what requests.get actually raises) instead of bare Exception Suite: 202 passed; smoke.sh 14/14. --- app.py | 17 ++++ routes/bc.py | 130 +++++++++--------------- routes/governance.py | 22 +--- services/audit.py | 28 ++++++ services/cdisc_api.py | 24 +++-- services/ingestion.py | 10 ++ services/loinc_api.py | 26 +++-- services/ncit_api.py | 9 +- tests/test_cdisc_api_cache.py | 7 +- tests/test_loinc.py | 182 +++++++++++++++++++--------------- tests/test_ncit.py | 3 +- 11 files changed, 253 insertions(+), 205 deletions(-) create mode 100644 services/audit.py diff --git a/app.py b/app.py index cc46a8a..0c15be9 100644 --- a/app.py +++ b/app.py @@ -1,9 +1,26 @@ +import logging + from flask import Flask from config import Config from extensions import db, migrate +def _configure_logging(): + """Attach a handler to the app's logger namespace once. + + Configures the root logger only if nothing else has (pytest, gunicorn, + and the MCP server may install their own handlers first). + """ + root = logging.getLogger() + if not root.handlers: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + + def create_app(config_class=Config): + _configure_logging() app = Flask(__name__) app.config.from_object(config_class) diff --git a/routes/bc.py b/routes/bc.py index 4bde45d..d9e9e2d 100644 --- a/routes/bc.py +++ b/routes/bc.py @@ -1,17 +1,54 @@ import json +import logging from flask import Blueprint, render_template, request, redirect, url_for, flash, Response from models.bc import BiomedicalConcept, DataElementConcept -from models.audit import AuditLog from models.governance import GovernanceRecord from extensions import db +from services.audit import log_change from services.export import export_json, export_xlsx, export_odm_xml from services.cdisc_api import CDISCApiClient from services.loinc_api import LoincApiClient from services.ncit_api import NCItApiClient from datetime import datetime, timezone +logger = logging.getLogger(__name__) + bp = Blueprint("bc", __name__) +# Plain-text fields copied verbatim from the form onto the model. +_BC_TEXT_FIELDS = ("short_name", "definition", "bc_categories", "synonyms", "result_scales", "package_date") + + +def _apply_bc_form(bc, form, is_new): + """Copy BC fields from a submitted form onto the model. + + Create keeps raw form values (empty strings allowed); edit + normalizes ncit/parent/loinc to None when cleared and preserves the + existing value for any field omitted from the form. + """ + for field in _BC_TEXT_FIELDS: + setattr(bc, field, form.get(field, "" if is_new else getattr(bc, field))) + if is_new: + bc.ncit_code = form.get("ncit_code", "") + bc.parent_bc_id = form.get("parent_bc_id") or None + bc.loinc_code = form.get("loinc_code", "") + has_loinc = bool(form.get("loinc_code", "").strip()) + bc.system = form.get("system", "") if has_loinc else "" + bc.system_name = form.get("system_name", "") if has_loinc else "" + bc.loinc_metadata = form.get("loinc_metadata", "") or None + bc.ncit_metadata = form.get("ncit_metadata", "") or None + else: + new_ncit_code = (form.get("ncit_code", "") or "").strip() or None + bc.ncit_code = new_ncit_code + bc.ncit_metadata = (form.get("ncit_metadata", "") or bc.ncit_metadata) if new_ncit_code else None + bc.parent_bc_id = (form.get("parent_bc_id", "") or "").strip() or None + new_loinc_code = (form.get("loinc_code", "") or "").strip() or None + bc.loinc_code = new_loinc_code + bc.system = form.get("system", bc.system) if new_loinc_code else "" + bc.system_name = form.get("system_name", bc.system_name) if new_loinc_code else "" + bc.loinc_metadata = (form.get("loinc_metadata", "") or bc.loinc_metadata) if new_loinc_code else None + bc.updated_at = datetime.now(timezone.utc) + @bp.route("/") def index(): @@ -187,31 +224,12 @@ def create(): return redirect(url_for("bc.new_bc")) bc = BiomedicalConcept( bc_id=bc_id, - short_name=request.form.get("short_name", ""), - definition=request.form.get("definition", ""), - ncit_code=request.form.get("ncit_code", ""), - parent_bc_id=request.form.get("parent_bc_id") or None, - bc_categories=request.form.get("bc_categories", ""), - synonyms=request.form.get("synonyms", ""), - result_scales=request.form.get("result_scales", ""), - loinc_code=request.form.get("loinc_code", ""), - system=request.form.get("system", "") if request.form.get("loinc_code", "").strip() else "", - system_name=request.form.get("system_name", "") if request.form.get("loinc_code", "").strip() else "", - loinc_metadata=request.form.get("loinc_metadata", "") or None, - ncit_metadata=request.form.get("ncit_metadata", "") or None, - package_date=request.form.get("package_date", ""), status="provisional", submitter=request.form.get("submitter", "unknown"), ) + _apply_bc_form(bc, request.form, is_new=True) db.session.add(bc) - log = AuditLog( - entity_type="BiomedicalConcept", - entity_id=bc_id, - action="created", - actor=bc.submitter, - after_state=bc.to_dict(), - ) - db.session.add(log) + log_change("BiomedicalConcept", bc_id, "created", actor=bc.submitter, after=bc.to_dict()) db.session.commit() _save_decs(bc_id, request.form) flash(f"BC {bc_id} created", "success") @@ -222,31 +240,8 @@ def create(): def edit(bc_id): bc = db.get_or_404(BiomedicalConcept, bc_id) before = bc.to_dict() - bc.short_name = request.form.get("short_name", bc.short_name) - bc.definition = request.form.get("definition", bc.definition) - new_ncit_code = (request.form.get("ncit_code", "") or "").strip() or None - bc.ncit_code = new_ncit_code - bc.ncit_metadata = (request.form.get("ncit_metadata", "") or bc.ncit_metadata) if new_ncit_code else None - bc.parent_bc_id = (request.form.get("parent_bc_id", "") or "").strip() or None - bc.bc_categories = request.form.get("bc_categories", bc.bc_categories) - bc.synonyms = request.form.get("synonyms", bc.synonyms) - bc.result_scales = request.form.get("result_scales", bc.result_scales) - new_loinc_code = (request.form.get("loinc_code", "") or "").strip() or None - bc.loinc_code = new_loinc_code - bc.system = request.form.get("system", bc.system) if new_loinc_code else "" - bc.system_name = request.form.get("system_name", bc.system_name) if new_loinc_code else "" - bc.loinc_metadata = (request.form.get("loinc_metadata", "") or bc.loinc_metadata) if new_loinc_code else None - bc.package_date = request.form.get("package_date", bc.package_date) - bc.updated_at = datetime.now(timezone.utc) - log = AuditLog( - entity_type="BiomedicalConcept", - entity_id=bc_id, - action="updated", - actor="user", - before_state=before, - after_state=bc.to_dict(), - ) - db.session.add(log) + _apply_bc_form(bc, request.form, is_new=False) + log_change("BiomedicalConcept", bc_id, "updated", actor="user", before=before, after=bc.to_dict()) db.session.commit() _save_decs(bc_id, request.form) flash(f"BC {bc_id} updated", "success") @@ -261,16 +256,7 @@ def clear_ncit(bc_id): bc.ncit_metadata = None bc.parent_bc_id = None bc.updated_at = datetime.now(timezone.utc) - db.session.add( - AuditLog( - entity_type="BiomedicalConcept", - entity_id=bc_id, - action="ncit_cleared", - actor="user", - before_state=before, - after_state=bc.to_dict(), - ) - ) + log_change("BiomedicalConcept", bc_id, "ncit_cleared", actor="user", before=before, after=bc.to_dict()) db.session.commit() flash(f"NCIt code cleared from {bc_id}", "success") return redirect(url_for("bc.detail", bc_id=bc_id)) @@ -285,16 +271,7 @@ def clear_loinc(bc_id): bc.system = "" bc.system_name = "" bc.updated_at = datetime.now(timezone.utc) - db.session.add( - AuditLog( - entity_type="BiomedicalConcept", - entity_id=bc_id, - action="loinc_cleared", - actor="user", - before_state=before, - after_state=bc.to_dict(), - ) - ) + log_change("BiomedicalConcept", bc_id, "loinc_cleared", actor="user", before=before, after=bc.to_dict()) db.session.commit() flash(f"LOINC code cleared from {bc_id}", "success") return redirect(url_for("bc.detail", bc_id=bc_id)) @@ -306,15 +283,7 @@ def submit_for_review(bc_id): before = bc.to_dict() bc.status = "sme_review" bc.updated_at = datetime.now(timezone.utc) - log = AuditLog( - entity_type="BiomedicalConcept", - entity_id=bc_id, - action="submitted_for_review", - actor="user", - before_state=before, - after_state=bc.to_dict(), - ) - db.session.add(log) + log_change("BiomedicalConcept", bc_id, "submitted_for_review", actor="user", before=before, after=bc.to_dict()) db.session.commit() flash(f"BC {bc_id} submitted for SME review", "success") return redirect(url_for("bc.detail", bc_id=bc_id)) @@ -328,14 +297,7 @@ def delete(bc_id): BiomedicalConcept.query.filter_by(parent_bc_id=bc_id).update({"parent_bc_id": None}, synchronize_session="fetch") # GovernanceRecord.bc_id is NOT NULL with no ORM cascade, so delete explicitly. GovernanceRecord.query.filter_by(bc_id=bc_id).delete(synchronize_session="fetch") - log = AuditLog( - entity_type="BiomedicalConcept", - entity_id=bc_id, - action="deleted", - actor="user", - before_state=bc.to_dict(), - ) - db.session.add(log) + log_change("BiomedicalConcept", bc_id, "deleted", actor="user", before=bc.to_dict()) db.session.delete(bc) db.session.commit() flash(f"BC {bc_id} deleted", "success") diff --git a/routes/governance.py b/routes/governance.py index 91c81a8..c7aff28 100644 --- a/routes/governance.py +++ b/routes/governance.py @@ -1,9 +1,9 @@ from flask import Blueprint, render_template, request, redirect, url_for, flash, jsonify, Response from models.bc import BiomedicalConcept from models.governance import GovernanceRecord -from models.audit import AuditLog from extensions import db from datetime import datetime, timezone +from services.audit import log_change from services.export import export_governance_xlsx bp = Blueprint("governance", __name__) @@ -56,16 +56,8 @@ def advance(bc_id): actor="user", comment=request.form.get("comment", ""), ) - log = AuditLog( - entity_type="BiomedicalConcept", - entity_id=bc_id, - action="status_changed", - actor="user", - before_state={"status": before_status}, - after_state={"status": bc.status}, - ) db.session.add(rec) - db.session.add(log) + log_change("BiomedicalConcept", bc_id, "status_changed", actor="user", before={"status": before_status}, after={"status": bc.status}) db.session.commit() if request.headers.get("X-Requested-With") == "XMLHttpRequest": return jsonify({"status": bc.status, "bc_id": bc_id}) @@ -88,16 +80,8 @@ def reject_bc(bc_id): actor="user", comment=request.form.get("comment", ""), ) - log = AuditLog( - entity_type="BiomedicalConcept", - entity_id=bc_id, - action="rejected", - actor="user", - before_state={"status": before_status}, - after_state={"status": "provisional"}, - ) db.session.add(rec) - db.session.add(log) + log_change("BiomedicalConcept", bc_id, "rejected", actor="user", before={"status": before_status}, after={"status": "provisional"}) db.session.commit() if request.headers.get("X-Requested-With") == "XMLHttpRequest": return jsonify({"status": "provisional", "bc_id": bc_id}) diff --git a/services/audit.py b/services/audit.py new file mode 100644 index 0000000..caae48c --- /dev/null +++ b/services/audit.py @@ -0,0 +1,28 @@ +"""Shared audit-trail helper. + +Every mutation of a curated entity must be recorded in the immutable +AuditLog. Routes (and, later, MCP tools) call log_change() instead of +constructing AuditLog rows inline so the write pattern stays uniform. +""" + +from extensions import db +from models.audit import AuditLog + + +def log_change(entity_type, entity_id, action, actor, before=None, after=None): + """Queue an AuditLog row on the current session. + + The caller owns the commit so the audit row and the change it + records land in the same transaction. `before`/`after` are plain + dicts (or None); the model serializes them to JSON. + """ + db.session.add( + AuditLog( + entity_type=entity_type, + entity_id=entity_id, + action=action, + actor=actor, + before_state=before, + after_state=after, + ) + ) diff --git a/services/cdisc_api.py b/services/cdisc_api.py index a90ec17..086a218 100644 --- a/services/cdisc_api.py +++ b/services/cdisc_api.py @@ -1,9 +1,12 @@ import hashlib +import logging import os import time import requests from flask import current_app +logger = logging.getLogger(__name__) + # In-memory cache: {key: (timestamp, data)} # Entries are never evicted — stale data is served while a refresh is attempted, # so a timeout never blocks the request with an empty response. @@ -21,15 +24,17 @@ def _cached(cache_key, fn): if entry and now - entry[0] < _CACHE_TTL: return entry[1] # fresh — serve immediately - # Attempt a refresh + # Attempt a refresh. Broad catch is intentional: this is a resilience + # seam and any refresh failure must fall back to stale data. try: data = fn() _cache[cache_key] = (now, data) return data except Exception: if entry and now - entry[0] < _CACHE_STALE_TTL: - # Serve stale rather than an error + logger.warning("Cache refresh failed for %s; serving stale entry", cache_key, exc_info=True) return entry[1] + logger.error("Cache refresh failed for %s with no stale fallback", cache_key, exc_info=True) raise # genuinely no data at all — let caller handle @@ -72,7 +77,8 @@ def _fetch(): try: data = self._get("/mdr/bc/biomedicalconcepts") return data.get("_links", {}).get("biomedicalConcepts", []) - except Exception as e: + except (requests.RequestException, ValueError) as e: + logger.error("CDISC Library BC list fetch failed (%s): %s", self.base_url, e) return [{"error": str(e)}] return _cached(self._cache_key("biomedical_concepts"), _fetch) @@ -81,14 +87,16 @@ def get_bc(self, concept_id): """Fetch a single BC by conceptId.""" try: return self._get(f"/mdr/bc/biomedicalconcepts/{concept_id}") - except Exception as e: + except (requests.RequestException, ValueError) as e: + logger.error("CDISC Library BC fetch failed for %s: %s", concept_id, e) return {"error": str(e)} def get_specialization(self, href): """Fetch a single dataset specialization by its href path.""" try: return self._get(href) - except Exception as e: + except (requests.RequestException, ValueError) as e: + logger.error("CDISC Library specialization fetch failed for %s: %s", href, e) return {"error": str(e)} def get_dataset_specializations(self): @@ -107,7 +115,8 @@ def _fetch(): if isinstance(links, list): return links return [item for v in links.values() if isinstance(v, list) for item in v] - except Exception as e: + except (requests.RequestException, ValueError) as e: + logger.error("CDISC Library specialization list fetch failed (%s): %s", self.base_url, e) return [{"error": str(e)}] return _cached(self._cache_key("dataset_specializations"), _fetch) @@ -119,7 +128,8 @@ def check_duplicate(self, short_name): try: bcs = self.get_biomedical_concepts() return any(bc.get("title", "").lower() == short_name.lower() for bc in bcs) - except Exception: + except (requests.RequestException, ValueError) as e: + logger.error("CDISC Library duplicate check failed for %r: %s", short_name, e) return False def publish_bc(self, bc_data): diff --git a/services/ingestion.py b/services/ingestion.py index 3e5c674..e228dc0 100644 --- a/services/ingestion.py +++ b/services/ingestion.py @@ -1,8 +1,11 @@ import io import json +import logging import pandas as pd from difflib import SequenceMatcher +logger = logging.getLogger(__name__) + # Canonical BC field names and known aliases for fuzzy field mapping FIELD_MAP = { "bc_id": ["bc_id", "bcid", "concept_id", "id", "identifier"], @@ -152,6 +155,9 @@ def parse_xlsx(file_obj): rows.append((mapped, confs)) results.extend(_group_by_bc(rows, sheet=sheet)) except Exception as e: + # Broad by design: user-supplied files can fail in arbitrary ways + # and the error is surfaced to the review queue via the record. + logger.error("XLSX ingestion parse failed: %s", e, exc_info=True) results.append({"error": str(e), "mapped": {}, "confidences": {}, "decs": [], "errors": [str(e)]}) return results @@ -176,6 +182,8 @@ def parse_csv(file_obj): } ) except Exception as e: + # Broad by design — see parse_xlsx. + logger.error("CSV ingestion parse failed: %s", e, exc_info=True) results.append({"error": str(e), "raw": {}, "mapped": {}, "confidences": {}, "errors": [str(e)]}) return results @@ -201,6 +209,8 @@ def parse_json(file_obj): } ) except Exception as e: + # Broad by design — see parse_xlsx. + logger.error("JSON ingestion parse failed: %s", e, exc_info=True) results.append({"error": str(e), "raw": {}, "mapped": {}, "confidences": {}, "errors": [str(e)]}) return results diff --git a/services/loinc_api.py b/services/loinc_api.py index 23785c0..cde28c8 100644 --- a/services/loinc_api.py +++ b/services/loinc_api.py @@ -1,20 +1,23 @@ +import logging import os import requests +logger = logging.getLogger(__name__) + LOINC_EF_FIELDS = ( - 'LOINC_NUM,SHORTNAME,LONG_COMMON_NAME,RELATEDNAMES2,PROPERTY,' - 'METHOD_TYP,AnswerLists,units,datatype,isCopyrighted,' - 'containsCopyrighted,CONSUMER_NAME,COMPONENT,' - 'EXTERNAL_COPYRIGHT_NOTICE,EXTERNAL_COPYRIGHT_LINK' + "LOINC_NUM,SHORTNAME,LONG_COMMON_NAME,RELATEDNAMES2,PROPERTY," + "METHOD_TYP,AnswerLists,units,datatype,isCopyrighted," + "containsCopyrighted,CONSUMER_NAME,COMPONENT," + "EXTERNAL_COPYRIGHT_NOTICE,EXTERNAL_COPYRIGHT_LINK" ) class LoincApiClient: - BASE_URL = 'https://clinicaltables.nlm.nih.gov/api/loinc_items/v3/search' + BASE_URL = "https://clinicaltables.nlm.nih.gov/api/loinc_items/v3/search" def _auth(self): - user = os.environ.get('LOINC_USER') - password = os.environ.get('LOINC_PASSWORD') + user = os.environ.get("LOINC_USER") + password = os.environ.get("LOINC_PASSWORD") if user and password: return (user, password) return None @@ -28,7 +31,7 @@ def search(self, term, size=10): try: response = requests.get( self.BASE_URL, - params={'ef': LOINC_EF_FIELDS, 'terms': term, 'maxList': size}, + params={"ef": LOINC_EF_FIELDS, "terms": term, "maxList": size}, auth=self._auth(), timeout=15, ) @@ -45,5 +48,8 @@ def search(self, term, size=10): item[field] = values[i] if values and i < len(values) else None results.append(item) return results - except Exception as e: - return [{'error': str(e)}] + except (requests.RequestException, ValueError, IndexError, TypeError) as e: + # Index/Type errors cover the positional parsing of the NLM + # array response ([total, [codes], {field: values}, ...]). + logger.error("LOINC search failed for term %r: %s", term, e) + return [{"error": str(e)}] diff --git a/services/ncit_api.py b/services/ncit_api.py index 4ee361a..3943bc9 100644 --- a/services/ncit_api.py +++ b/services/ncit_api.py @@ -1,6 +1,9 @@ +import logging import time import requests +logger = logging.getLogger(__name__) + _ncit_cache = {} _NCIT_TTL = 300 # serve fresh data for 5 minutes _NCIT_STALE_TTL = 3600 # serve stale data for up to 1 hour while refresh fails @@ -39,7 +42,8 @@ def search_concept(self, term, size=10): } for c in concepts ] - except Exception as e: + except (requests.RequestException, ValueError) as e: + logger.error("NCIt search failed for term %r: %s", term, e) return [{"error": str(e)}] def get_concept(self, ncit_code): @@ -67,7 +71,8 @@ def get_concept(self, ncit_code): } _ncit_cache[cache_key] = (now, data) return data - except Exception as e: + except (requests.RequestException, ValueError) as e: + logger.error("NCIt concept fetch failed for %s: %s", ncit_code, e) return {"error": str(e)} def get_preferred_name(self, ncit_code): diff --git a/tests/test_cdisc_api_cache.py b/tests/test_cdisc_api_cache.py index a286ea0..73cd3e4 100644 --- a/tests/test_cdisc_api_cache.py +++ b/tests/test_cdisc_api_cache.py @@ -77,9 +77,10 @@ def test_cache_key_excludes_raw_api_key(self, app): def test_get_biomedical_concepts_error_encoded_not_raised(self, app, monkeypatch): """API failure is captured as [{'error': ...}] and cached, not raised.""" + import requests def boom(*args, **kwargs): - raise ConnectionError("no network") + raise requests.ConnectionError("no network") monkeypatch.setattr(cdisc_api.requests, "get", boom) with app.app_context(): @@ -89,11 +90,13 @@ def boom(*args, **kwargs): def test_error_result_replaced_after_ttl(self, app, monkeypatch): """A cached error list refreshes to real data once the TTL passes.""" + import requests + with app.app_context(): client = CDISCApiClient() def boom(*args, **kwargs): - raise ConnectionError("no network") + raise requests.ConnectionError("no network") monkeypatch.setattr(cdisc_api.requests, "get", boom) with app.app_context(): diff --git a/tests/test_loinc.py b/tests/test_loinc.py index 7248ca0..8b62927 100644 --- a/tests/test_loinc.py +++ b/tests/test_loinc.py @@ -1,12 +1,13 @@ """Tests for services/loinc_api.py and routes/loinc.py.""" + import json from unittest.mock import MagicMock, patch import pytest +import requests from services.loinc_api import LoincApiClient, LOINC_EF_FIELDS - # --------------------------------------------------------------------------- # Sample NLM response using ef parameter # response format: [total, [internal_codes], {field: [values...]}, display_data] @@ -14,24 +15,23 @@ NLM_EF_RESPONSE = [ 2, - ['4548-4', '17856-6'], + ["4548-4", "17856-6"], { - 'LOINC_NUM': ['4548-4', '17856-6'], - 'SHORTNAME': ['HbA1c MFr Bld', 'HbA1c MFr Bld HPLC'], - 'LONG_COMMON_NAME': ['Hemoglobin A1c/Hemoglobin.total in Blood', - 'Hemoglobin A1c/Hemoglobin.total in Blood by HPLC'], - 'RELATEDNAMES2': ['Glycated Hb', 'Glycohemoglobin'], - 'PROPERTY': ['MFr', 'MFr'], - 'METHOD_TYP': [None, 'HPLC'], - 'AnswerLists': [None, None], - 'units': ['%', '%'], - 'datatype': ['NM', 'NM'], - 'isCopyrighted': ['N', 'N'], - 'containsCopyrighted': ['N', 'N'], - 'CONSUMER_NAME': ['Hemoglobin A1c', 'Hemoglobin A1c by HPLC'], - 'COMPONENT': ['Hemoglobin A1c', 'Hemoglobin A1c'], - 'EXTERNAL_COPYRIGHT_NOTICE': [None, None], - 'EXTERNAL_COPYRIGHT_LINK': [None, None], + "LOINC_NUM": ["4548-4", "17856-6"], + "SHORTNAME": ["HbA1c MFr Bld", "HbA1c MFr Bld HPLC"], + "LONG_COMMON_NAME": ["Hemoglobin A1c/Hemoglobin.total in Blood", "Hemoglobin A1c/Hemoglobin.total in Blood by HPLC"], + "RELATEDNAMES2": ["Glycated Hb", "Glycohemoglobin"], + "PROPERTY": ["MFr", "MFr"], + "METHOD_TYP": [None, "HPLC"], + "AnswerLists": [None, None], + "units": ["%", "%"], + "datatype": ["NM", "NM"], + "isCopyrighted": ["N", "N"], + "containsCopyrighted": ["N", "N"], + "CONSUMER_NAME": ["Hemoglobin A1c", "Hemoglobin A1c by HPLC"], + "COMPONENT": ["Hemoglobin A1c", "Hemoglobin A1c"], + "EXTERNAL_COPYRIGHT_NOTICE": [None, None], + "EXTERNAL_COPYRIGHT_LINK": [None, None], }, None, ] @@ -41,6 +41,7 @@ # LoincApiClient.search() # --------------------------------------------------------------------------- + class TestLoincApiClientSearch: def _mock_response(self, data, status=200): mock = MagicMock() @@ -50,135 +51,156 @@ def _mock_response(self, data, status=200): return mock def test_returns_normalized_list(self): - with patch('services.loinc_api.requests.get') as mock_get: + with patch("services.loinc_api.requests.get") as mock_get: mock_get.return_value = self._mock_response(NLM_EF_RESPONSE) - results = LoincApiClient().search('hba1c') + results = LoincApiClient().search("hba1c") assert len(results) == 2 - assert results[0]['LOINC_NUM'] == '4548-4' - assert results[0]['LONG_COMMON_NAME'] == 'Hemoglobin A1c/Hemoglobin.total in Blood' - assert results[0]['SHORTNAME'] == 'HbA1c MFr Bld' - assert results[0]['units'] == '%' - assert results[0]['datatype'] == 'NM' - assert results[0]['PROPERTY'] == 'MFr' - assert results[0]['METHOD_TYP'] is None - assert results[1]['LOINC_NUM'] == '17856-6' - assert results[1]['METHOD_TYP'] == 'HPLC' + assert results[0]["LOINC_NUM"] == "4548-4" + assert results[0]["LONG_COMMON_NAME"] == "Hemoglobin A1c/Hemoglobin.total in Blood" + assert results[0]["SHORTNAME"] == "HbA1c MFr Bld" + assert results[0]["units"] == "%" + assert results[0]["datatype"] == "NM" + assert results[0]["PROPERTY"] == "MFr" + assert results[0]["METHOD_TYP"] is None + assert results[1]["LOINC_NUM"] == "17856-6" + assert results[1]["METHOD_TYP"] == "HPLC" def test_all_ef_fields_present_in_result(self): - with patch('services.loinc_api.requests.get') as mock_get: + with patch("services.loinc_api.requests.get") as mock_get: mock_get.return_value = self._mock_response(NLM_EF_RESPONSE) - results = LoincApiClient().search('hba1c') + results = LoincApiClient().search("hba1c") expected_fields = [ - 'LOINC_NUM', 'SHORTNAME', 'LONG_COMMON_NAME', 'RELATEDNAMES2', - 'PROPERTY', 'METHOD_TYP', 'AnswerLists', 'units', 'datatype', - 'isCopyrighted', 'containsCopyrighted', 'CONSUMER_NAME', 'COMPONENT', - 'EXTERNAL_COPYRIGHT_NOTICE', 'EXTERNAL_COPYRIGHT_LINK', + "LOINC_NUM", + "SHORTNAME", + "LONG_COMMON_NAME", + "RELATEDNAMES2", + "PROPERTY", + "METHOD_TYP", + "AnswerLists", + "units", + "datatype", + "isCopyrighted", + "containsCopyrighted", + "CONSUMER_NAME", + "COMPONENT", + "EXTERNAL_COPYRIGHT_NOTICE", + "EXTERNAL_COPYRIGHT_LINK", ] for field in expected_fields: assert field in results[0], f"Missing field: {field}" def test_uses_ef_parameter(self): - with patch('services.loinc_api.requests.get') as mock_get: + with patch("services.loinc_api.requests.get") as mock_get: mock_get.return_value = self._mock_response(NLM_EF_RESPONSE) - LoincApiClient().search('glucose', size=5) + LoincApiClient().search("glucose", size=5) call_kwargs = mock_get.call_args[1] - params = call_kwargs['params'] - assert 'ef' in params - assert 'df' not in params - assert 'LOINC_NUM' in params['ef'] - assert 'LONG_COMMON_NAME' in params['ef'] - assert params['terms'] == 'glucose' - assert params['maxList'] == 5 + params = call_kwargs["params"] + assert "ef" in params + assert "df" not in params + assert "LOINC_NUM" in params["ef"] + assert "LONG_COMMON_NAME" in params["ef"] + assert params["terms"] == "glucose" + assert params["maxList"] == 5 def test_ef_fields_constant_contains_all_required_fields(self): required = [ - 'LOINC_NUM', 'SHORTNAME', 'LONG_COMMON_NAME', 'RELATEDNAMES2', - 'PROPERTY', 'METHOD_TYP', 'AnswerLists', 'units', 'datatype', - 'isCopyrighted', 'containsCopyrighted', 'CONSUMER_NAME', 'COMPONENT', - 'EXTERNAL_COPYRIGHT_NOTICE', 'EXTERNAL_COPYRIGHT_LINK', + "LOINC_NUM", + "SHORTNAME", + "LONG_COMMON_NAME", + "RELATEDNAMES2", + "PROPERTY", + "METHOD_TYP", + "AnswerLists", + "units", + "datatype", + "isCopyrighted", + "containsCopyrighted", + "CONSUMER_NAME", + "COMPONENT", + "EXTERNAL_COPYRIGHT_NOTICE", + "EXTERNAL_COPYRIGHT_LINK", ] for field in required: assert field in LOINC_EF_FIELDS, f"Missing from LOINC_EF_FIELDS: {field}" def test_uses_basic_auth_when_env_vars_set(self, monkeypatch): - monkeypatch.setenv('LOINC_USER', 'myuser') - monkeypatch.setenv('LOINC_PASSWORD', 'mypass') - with patch('services.loinc_api.requests.get') as mock_get: + monkeypatch.setenv("LOINC_USER", "myuser") + monkeypatch.setenv("LOINC_PASSWORD", "mypass") + with patch("services.loinc_api.requests.get") as mock_get: mock_get.return_value = self._mock_response([0, [], {}, None]) - LoincApiClient().search('test') + LoincApiClient().search("test") - auth = mock_get.call_args[1].get('auth') - assert auth == ('myuser', 'mypass') + auth = mock_get.call_args[1].get("auth") + assert auth == ("myuser", "mypass") def test_no_auth_when_env_vars_missing(self, monkeypatch): - monkeypatch.delenv('LOINC_USER', raising=False) - monkeypatch.delenv('LOINC_PASSWORD', raising=False) - with patch('services.loinc_api.requests.get') as mock_get: + monkeypatch.delenv("LOINC_USER", raising=False) + monkeypatch.delenv("LOINC_PASSWORD", raising=False) + with patch("services.loinc_api.requests.get") as mock_get: mock_get.return_value = self._mock_response([0, [], {}, None]) - LoincApiClient().search('test') + LoincApiClient().search("test") - auth = mock_get.call_args[1].get('auth') + auth = mock_get.call_args[1].get("auth") assert auth is None def test_empty_results(self): - with patch('services.loinc_api.requests.get') as mock_get: + with patch("services.loinc_api.requests.get") as mock_get: mock_get.return_value = self._mock_response([0, [], {}, None]) - results = LoincApiClient().search('zzznomatch') + results = LoincApiClient().search("zzznomatch") assert results == [] def test_missing_codes_array_returns_empty(self): - with patch('services.loinc_api.requests.get') as mock_get: + with patch("services.loinc_api.requests.get") as mock_get: mock_get.return_value = self._mock_response([0]) - results = LoincApiClient().search('test') + results = LoincApiClient().search("test") assert results == [] def test_network_error_returns_error_entry(self): - with patch('services.loinc_api.requests.get', side_effect=Exception('timeout')): - results = LoincApiClient().search('hba1c') + with patch("services.loinc_api.requests.get", side_effect=requests.RequestException("timeout")): + results = LoincApiClient().search("hba1c") assert len(results) == 1 - assert 'error' in results[0] - assert 'timeout' in results[0]['error'] + assert "error" in results[0] + assert "timeout" in results[0]["error"] # --------------------------------------------------------------------------- # GET /loinc/search route # --------------------------------------------------------------------------- + class TestLoincSearchRoute: def test_returns_json_for_ajax(self, client): - with patch('routes.loinc.LoincApiClient') as MockClient: - MockClient.return_value.search.return_value = [ - {'LOINC_NUM': '4548-4', 'LONG_COMMON_NAME': 'Hemoglobin A1c/Hemoglobin.total in Blood'} - ] - r = client.get('/loinc/search?term=hba1c', headers={'Accept': 'application/json'}) + with patch("routes.loinc.LoincApiClient") as MockClient: + MockClient.return_value.search.return_value = [{"LOINC_NUM": "4548-4", "LONG_COMMON_NAME": "Hemoglobin A1c/Hemoglobin.total in Blood"}] + r = client.get("/loinc/search?term=hba1c", headers={"Accept": "application/json"}) assert r.status_code == 200 data = json.loads(r.data) assert isinstance(data, list) - assert data[0]['LOINC_NUM'] == '4548-4' + assert data[0]["LOINC_NUM"] == "4548-4" def test_empty_term_returns_empty_list(self, client): - r = client.get('/loinc/search', headers={'Accept': 'application/json'}) + r = client.get("/loinc/search", headers={"Accept": "application/json"}) assert r.status_code == 200 assert json.loads(r.data) == [] def test_calls_client_with_term(self, client): - with patch('routes.loinc.LoincApiClient') as MockClient: + with patch("routes.loinc.LoincApiClient") as MockClient: MockClient.return_value.search.return_value = [] - client.get('/loinc/search?term=glucose', headers={'Accept': 'application/json'}) + client.get("/loinc/search?term=glucose", headers={"Accept": "application/json"}) - MockClient.return_value.search.assert_called_once_with('glucose', size=10) + MockClient.return_value.search.assert_called_once_with("glucose", size=10) def test_format_json_param_triggers_json_response(self, client): - with patch('routes.loinc.LoincApiClient') as MockClient: + with patch("routes.loinc.LoincApiClient") as MockClient: MockClient.return_value.search.return_value = [] - r = client.get('/loinc/search?term=hba1c&format=json') + r = client.get("/loinc/search?term=hba1c&format=json") assert r.status_code == 200 - assert r.content_type.startswith('application/json') + assert r.content_type.startswith("application/json") diff --git a/tests/test_ncit.py b/tests/test_ncit.py index 8479a36..69b21b0 100644 --- a/tests/test_ncit.py +++ b/tests/test_ncit.py @@ -4,6 +4,7 @@ from unittest.mock import MagicMock, patch import pytest +import requests from services.ncit_api import NCItApiClient @@ -106,7 +107,7 @@ def test_missing_semantic_type_returns_empty_list(self): assert result["semantic_type"] == [] def test_error_returns_error_dict(self): - with patch("services.ncit_api.requests.get", side_effect=Exception("timeout")): + with patch("services.ncit_api.requests.get", side_effect=requests.RequestException("timeout")): result = NCItApiClient().get_concept("C64849") assert "error" in result From cf40c3e6dee26e425ea8dfd833ad660e29cf3887 Mon Sep 17 00:00:00 2001 From: pendingintent <3921919+pendingintent@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:03:19 -0400 Subject: [PATCH 20/36] =?UTF-8?q?=F0=9F=94=A7=20Schema:=20squash=20Alembic?= =?UTF-8?q?=20chain=20to=20a=20baseline;=20Alembic=20is=20now=20the=20sing?= =?UTF-8?q?le=20source=20of=20truth?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously db.create_all() in create_app() coexisted with a 4-revision Alembic chain that had no baseline (first revision only ALTERed a table) and an internally inconsistent code/loinc_code rename+re-add sequence — 'flask db upgrade' failed on any fresh database. - Delete the 4 legacy revisions; generate one autogenerated baseline (51d4a009d291) creating all six tables. Verified: fresh 'flask db upgrade' schema is byte-identical (tables + indexes) to create_all(). - Remove db.create_all() from create_app(); it is now side-effect free (also a prerequisite for the MCP server sharing the app factory). - New db_bootstrap.ensure_db(): fresh DB -> upgrade; legacy create_all DB -> stamp head; DB stamped at a pre-squash revision -> restamp head (schema verified via column check first; outdated schemas raise with recovery instructions). Wired into 'python app.py' startup. - Verified against a copy of the real instance DB (27 BCs): restamped c2d4e6f8a0b1 -> 51d4a009d291 with data intact. - tests/test_db_bootstrap.py covers all four states + idempotency. - README/CLAUDE.md: document the bootstrap and the migrate-on-model-change rule. Suite: 207 passed; smoke.sh 14/14 (fresh-DB boot path). --- CLAUDE.md | 6 +- README.md | 4 +- app.py | 6 +- db_bootstrap.py | 71 +++++++++ .../51d4a009d291_baseline_initial_schema.py | 142 ++++++++++++++++++ .../a1c3e5f7b9d2_rename_code_to_loinc_code.py | 26 ---- ...add_ncit_metadata_to_biomedical_concept.py | 32 ---- ...6f8a0b1_add_code_to_biomedical_concepts.py | 26 ---- ...dd_loinc_metadata_to_biomedical_concept.py | 32 ---- tests/test_db_bootstrap.py | 87 +++++++++++ 10 files changed, 311 insertions(+), 121 deletions(-) create mode 100644 db_bootstrap.py create mode 100644 migrations/versions/51d4a009d291_baseline_initial_schema.py delete mode 100644 migrations/versions/a1c3e5f7b9d2_rename_code_to_loinc_code.py delete mode 100644 migrations/versions/b9ee22a174fe_add_ncit_metadata_to_biomedical_concept.py delete mode 100644 migrations/versions/c2d4e6f8a0b1_add_code_to_biomedical_concepts.py delete mode 100644 migrations/versions/f27a606163b0_add_loinc_metadata_to_biomedical_concept.py create mode 100644 tests/test_db_bootstrap.py diff --git a/CLAUDE.md b/CLAUDE.md index 1f7b9b7..f1293d5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,7 +20,11 @@ export CDISC_API_KEY=your_key_here python app.py # runs on http://localhost:8081 (override with PORT env var) ``` -Database (`instance/cdisc_curation.db`) is auto-created on first run via `db.create_all()`. +Database (`instance/cdisc_curation.db`) is brought to the Alembic migration +head automatically on startup by `db_bootstrap.ensure_db()` (fresh DBs are +built via `flask db upgrade`; pre-baseline DBs are stamped in place). +**Schema changes require an Alembic revision**: edit the model, then +`flask db migrate -m "..."` — `db.create_all()` is used only by tests. ## Linting diff --git a/README.md b/README.md index 7b46cb1..e1c312c 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,9 @@ These hooks run automatically before each `git commit`. If black reformats any f python app.py ``` -The Flask development server starts on `http://localhost:5000`. The SQLite database file (`cdisc_curation.db`) is created automatically on first run. +The Flask development server starts on `http://localhost:8081` (override with the `PORT` env var). On startup the app brings the SQLite database (`instance/cdisc_curation.db`) to the current Alembic migration head automatically: fresh databases are built with `flask db upgrade`, and databases created before the migration baseline was squashed (2026-07-08) are stamped in place. If startup reports a schema that cannot be auto-migrated, recreate the database or run `flask db stamp head` after bringing it up to date manually. + +Schema changes are managed exclusively through Flask-Migrate/Alembic — edit the model, then run `flask db migrate -m "describe change"` and commit the generated revision. > **Note:** `python app.py` uses Flask's built-in development server. Do not use this in production. Use a WSGI server such as gunicorn instead. diff --git a/app.py b/app.py index 0c15be9..370300d 100644 --- a/app.py +++ b/app.py @@ -45,12 +45,12 @@ def create_app(config_class=Config): app.register_blueprint(governance_bp, url_prefix="/governance") app.register_blueprint(audit_bp, url_prefix="/audit") - with app.app_context(): - db.create_all() - return app if __name__ == "__main__": + from db_bootstrap import ensure_db + app = create_app() + ensure_db(app) app.run(debug=True, port=app.config["PORT"]) diff --git a/db_bootstrap.py b/db_bootstrap.py new file mode 100644 index 0000000..2e202f9 --- /dev/null +++ b/db_bootstrap.py @@ -0,0 +1,71 @@ +"""Database bootstrap — Alembic is the single source of truth. + +Historically the schema was built by db.create_all() at import time while +an Alembic chain existed in parallel with no baseline revision. The chain +was squashed to one baseline on 2026-07-08; ensure_db() migrates any +database state that predates the squash. + +Tests are unaffected: tests/conftest.py builds its in-memory schema with +create_all() directly. +""" + +import logging + +from flask_migrate import stamp, upgrade +from sqlalchemy import inspect, text + +from extensions import db + +logger = logging.getLogger(__name__) + +# Revision ids from the pre-squash chain. A database stamped at the old +# head (or built by create_all with all current columns) has a schema +# identical to the new baseline and is restamped rather than migrated. +LEGACY_REVISIONS = {"f27a606163b0", "b9ee22a174fe", "a1c3e5f7b9d2", "c2d4e6f8a0b1"} + +# Columns added over the life of the legacy chain; all present == the +# schema matches the new baseline. +_CURRENT_SCHEMA_COLUMNS = {"loinc_metadata", "ncit_metadata", "loinc_code", "code"} + + +def _schema_is_current(inspector): + cols = {c["name"] for c in inspector.get_columns("biomedical_concepts")} + return _CURRENT_SCHEMA_COLUMNS <= cols + + +def ensure_db(app): + """Bring the configured database to the current migration head. + + Handles three states: + - fresh database -> upgrade() builds the schema from the baseline + - legacy create_all() DB -> stamped at head (schema verified first) + - stamped pre-squash DB -> restamped at head (schema verified first) + + Raises RuntimeError for a legacy database whose schema is missing + current columns — recreate it or bring it to the old head manually, + then run `flask db stamp head`. + """ + with app.app_context(): + inspector = inspect(db.engine) + tables = set(inspector.get_table_names()) + + if "biomedical_concepts" in tables: + if not _schema_is_current(inspector): + raise RuntimeError( + "Database schema predates the squashed Alembic baseline " + "and cannot be auto-migrated. Recreate the database or " + "bring it to the pre-squash head manually, then run " + "'flask db stamp head'." + ) + if "alembic_version" not in tables: + logger.info("Legacy create_all() database detected; stamping at baseline head") + stamp() + else: + current = db.session.execute(text("SELECT version_num FROM alembic_version")).scalar() + if current in LEGACY_REVISIONS: + logger.info("Database stamped at pre-squash revision %s; restamping at head", current) + db.session.execute(text("DELETE FROM alembic_version")) + db.session.commit() + stamp() + + upgrade() diff --git a/migrations/versions/51d4a009d291_baseline_initial_schema.py b/migrations/versions/51d4a009d291_baseline_initial_schema.py new file mode 100644 index 0000000..804481b --- /dev/null +++ b/migrations/versions/51d4a009d291_baseline_initial_schema.py @@ -0,0 +1,142 @@ +"""baseline: initial schema + +Revision ID: 51d4a009d291 +Revises: +Create Date: 2026-07-08 15:00:19.432521 + +""" + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = "51d4a009d291" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "audit_logs", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("entity_type", sa.String(length=50), nullable=True), + sa.Column("entity_id", sa.String(length=100), nullable=True), + sa.Column("action", sa.String(length=100), nullable=True), + sa.Column("actor", sa.String(length=100), nullable=True), + sa.Column("before_state", sa.Text(), nullable=True), + sa.Column("after_state", sa.Text(), nullable=True), + sa.Column("timestamp", sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "biomedical_concepts", + sa.Column("bc_id", sa.String(length=50), nullable=False), + sa.Column("short_name", sa.String(length=255), nullable=False), + sa.Column("definition", sa.Text(), nullable=True), + sa.Column("ncit_code", sa.String(length=50), nullable=True), + sa.Column("parent_bc_id", sa.String(length=50), nullable=True), + sa.Column("bc_categories", sa.String(length=500), nullable=True), + sa.Column("synonyms", sa.Text(), nullable=True), + sa.Column("result_scales", sa.String(length=255), nullable=True), + sa.Column("system", sa.String(length=255), nullable=True), + sa.Column("system_name", sa.String(length=100), nullable=True), + sa.Column("loinc_code", sa.String(length=50), nullable=True), + sa.Column("code", sa.String(length=50), nullable=True), + sa.Column("loinc_metadata", sa.Text(), nullable=True), + sa.Column("ncit_metadata", sa.Text(), nullable=True), + sa.Column("package_date", sa.String(length=20), nullable=True), + sa.Column("status", sa.String(length=50), nullable=True), + sa.Column("submitter", sa.String(length=100), nullable=True), + sa.Column("created_at", sa.DateTime(), nullable=True), + sa.Column("updated_at", sa.DateTime(), nullable=True), + sa.Column("history_of_change", sa.Text(), nullable=True), + sa.Column("source", sa.String(length=50), nullable=True), + sa.ForeignKeyConstraint( + ["parent_bc_id"], + ["biomedical_concepts.bc_id"], + ), + sa.PrimaryKeyConstraint("bc_id"), + ) + op.create_table( + "ingestion_records", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("session_key", sa.String(length=64), nullable=True), + sa.Column("source_file", sa.String(length=255), nullable=True), + sa.Column("source_sheet", sa.String(length=100), nullable=True), + sa.Column("mapped", sa.Text(), nullable=True), + sa.Column("confidences", sa.Text(), nullable=True), + sa.Column("errors", sa.Text(), nullable=True), + sa.Column("decs", sa.Text(), nullable=True), + sa.Column("duplicate", sa.Boolean(), nullable=True), + sa.Column("status", sa.String(length=20), nullable=True), + sa.Column("created_at", sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + with op.batch_alter_table("ingestion_records", schema=None) as batch_op: + batch_op.create_index(batch_op.f("ix_ingestion_records_session_key"), ["session_key"], unique=False) + + op.create_table( + "data_element_concepts", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("dec_id", sa.String(length=50), nullable=False), + sa.Column("bc_id", sa.String(length=50), nullable=False), + sa.Column("ncit_dec_code", sa.String(length=50), nullable=True), + sa.Column("dec_label", sa.String(length=255), nullable=True), + sa.Column("data_type", sa.String(length=50), nullable=True), + sa.Column("example_set", sa.Text(), nullable=True), + sa.Column("required", sa.Boolean(), nullable=True), + sa.Column("generic_dec", sa.Boolean(), nullable=True), + sa.Column("template_type", sa.String(length=100), nullable=True), + sa.Column("sort_order", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ["bc_id"], + ["biomedical_concepts.bc_id"], + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "dataset_specializations", + sa.Column("vlm_group_id", sa.String(length=100), nullable=False), + sa.Column("bc_id", sa.String(length=50), nullable=False), + sa.Column("domain", sa.String(length=20), nullable=True), + sa.Column("short_name", sa.String(length=255), nullable=True), + sa.Column("variables", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint( + ["bc_id"], + ["biomedical_concepts.bc_id"], + ), + sa.PrimaryKeyConstraint("vlm_group_id"), + ) + op.create_table( + "governance_records", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("bc_id", sa.String(length=50), nullable=False), + sa.Column("stage", sa.Integer(), nullable=True), + sa.Column("action", sa.String(length=100), nullable=True), + sa.Column("actor", sa.String(length=100), nullable=True), + sa.Column("comment", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint( + ["bc_id"], + ["biomedical_concepts.bc_id"], + ), + sa.PrimaryKeyConstraint("id"), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("governance_records") + op.drop_table("dataset_specializations") + op.drop_table("data_element_concepts") + with op.batch_alter_table("ingestion_records", schema=None) as batch_op: + batch_op.drop_index(batch_op.f("ix_ingestion_records_session_key")) + + op.drop_table("ingestion_records") + op.drop_table("biomedical_concepts") + op.drop_table("audit_logs") + # ### end Alembic commands ### diff --git a/migrations/versions/a1c3e5f7b9d2_rename_code_to_loinc_code.py b/migrations/versions/a1c3e5f7b9d2_rename_code_to_loinc_code.py deleted file mode 100644 index b8f24c9..0000000 --- a/migrations/versions/a1c3e5f7b9d2_rename_code_to_loinc_code.py +++ /dev/null @@ -1,26 +0,0 @@ -"""rename code to loinc_code in biomedical_concepts - -Revision ID: a1c3e5f7b9d2 -Revises: b9ee22a174fe -Create Date: 2026-04-15 14:00:00.000000 - -""" - -from alembic import op -import sqlalchemy as sa - -# revision identifiers, used by Alembic. -revision = "a1c3e5f7b9d2" -down_revision = "b9ee22a174fe" -branch_labels = None -depends_on = None - - -def upgrade(): - with op.batch_alter_table("biomedical_concepts", schema=None) as batch_op: - batch_op.alter_column("code", new_column_name="loinc_code", existing_type=sa.String(50), existing_nullable=True) - - -def downgrade(): - with op.batch_alter_table("biomedical_concepts", schema=None) as batch_op: - batch_op.alter_column("loinc_code", new_column_name="code", existing_type=sa.String(50), existing_nullable=True) diff --git a/migrations/versions/b9ee22a174fe_add_ncit_metadata_to_biomedical_concept.py b/migrations/versions/b9ee22a174fe_add_ncit_metadata_to_biomedical_concept.py deleted file mode 100644 index da2c60f..0000000 --- a/migrations/versions/b9ee22a174fe_add_ncit_metadata_to_biomedical_concept.py +++ /dev/null @@ -1,32 +0,0 @@ -"""add ncit_metadata to biomedical_concept - -Revision ID: b9ee22a174fe -Revises: f27a606163b0 -Create Date: 2026-04-08 13:12:22.755530 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = 'b9ee22a174fe' -down_revision = 'f27a606163b0' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('biomedical_concepts', schema=None) as batch_op: - batch_op.add_column(sa.Column('ncit_metadata', sa.Text(), nullable=True)) - - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('biomedical_concepts', schema=None) as batch_op: - batch_op.drop_column('ncit_metadata') - - # ### end Alembic commands ### diff --git a/migrations/versions/c2d4e6f8a0b1_add_code_to_biomedical_concepts.py b/migrations/versions/c2d4e6f8a0b1_add_code_to_biomedical_concepts.py deleted file mode 100644 index 615801a..0000000 --- a/migrations/versions/c2d4e6f8a0b1_add_code_to_biomedical_concepts.py +++ /dev/null @@ -1,26 +0,0 @@ -"""add code column to biomedical_concepts - -Revision ID: c2d4e6f8a0b1 -Revises: a1c3e5f7b9d2 -Create Date: 2026-04-16 09:15:00.000000 - -""" - -from alembic import op -import sqlalchemy as sa - -# revision identifiers, used by Alembic. -revision = "c2d4e6f8a0b1" -down_revision = "a1c3e5f7b9d2" -branch_labels = None -depends_on = None - - -def upgrade(): - with op.batch_alter_table("biomedical_concepts", schema=None) as batch_op: - batch_op.add_column(sa.Column("code", sa.String(50), nullable=True)) - - -def downgrade(): - with op.batch_alter_table("biomedical_concepts", schema=None) as batch_op: - batch_op.drop_column("code") diff --git a/migrations/versions/f27a606163b0_add_loinc_metadata_to_biomedical_concept.py b/migrations/versions/f27a606163b0_add_loinc_metadata_to_biomedical_concept.py deleted file mode 100644 index 8594140..0000000 --- a/migrations/versions/f27a606163b0_add_loinc_metadata_to_biomedical_concept.py +++ /dev/null @@ -1,32 +0,0 @@ -"""add loinc_metadata to biomedical_concept - -Revision ID: f27a606163b0 -Revises: -Create Date: 2026-04-08 12:28:59.039371 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = 'f27a606163b0' -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('biomedical_concepts', schema=None) as batch_op: - batch_op.add_column(sa.Column('loinc_metadata', sa.Text(), nullable=True)) - - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('biomedical_concepts', schema=None) as batch_op: - batch_op.drop_column('loinc_metadata') - - # ### end Alembic commands ### diff --git a/tests/test_db_bootstrap.py b/tests/test_db_bootstrap.py new file mode 100644 index 0000000..4aee2e4 --- /dev/null +++ b/tests/test_db_bootstrap.py @@ -0,0 +1,87 @@ +"""Tests for db_bootstrap.ensure_db() — the Alembic baseline bootstrap.""" + +import pytest +from sqlalchemy import inspect, text + +from app import create_app +from db_bootstrap import LEGACY_REVISIONS, ensure_db +from extensions import db + +EXPECTED_TABLES = { + "audit_logs", + "biomedical_concepts", + "data_element_concepts", + "dataset_specializations", + "governance_records", + "ingestion_records", +} + + +def _make_app(tmp_path, name="boot.db"): + class TmpConfig: + TESTING = True + SQLALCHEMY_DATABASE_URI = f"sqlite:///{tmp_path}/{name}" + SECRET_KEY = "test-secret-key" + CDISC_API_KEY = "" + CDISC_API_BASE_URL = "https://api.library.cdisc.org/api/cosmos/v2" + NCIT_API_BASE_URL = "https://api-evsrest.nci.nih.gov/api/v1" + MAX_CONTENT_LENGTH = 16 * 1024 * 1024 + + return create_app(TmpConfig) + + +def _tables(app): + with app.app_context(): + return set(inspect(db.engine).get_table_names()) + + +def _stamped_revision(app): + with app.app_context(): + return db.session.execute(text("SELECT version_num FROM alembic_version")).scalar() + + +class TestEnsureDb: + def test_fresh_db_upgraded_from_baseline(self, tmp_path): + app = _make_app(tmp_path) + ensure_db(app) + assert EXPECTED_TABLES <= _tables(app) + assert _stamped_revision(app) not in LEGACY_REVISIONS + + def test_legacy_create_all_db_is_stamped(self, tmp_path): + app = _make_app(tmp_path) + with app.app_context(): + db.create_all() + assert "alembic_version" not in _tables(app) + ensure_db(app) + assert "alembic_version" in _tables(app) + assert _stamped_revision(app) not in LEGACY_REVISIONS + + def test_pre_squash_stamped_db_is_restamped(self, tmp_path): + app = _make_app(tmp_path) + with app.app_context(): + db.create_all() + db.session.execute(text("CREATE TABLE alembic_version (version_num VARCHAR(32) NOT NULL)")) + db.session.execute(text("INSERT INTO alembic_version VALUES ('c2d4e6f8a0b1')")) + # Seed a row to prove data survives the restamp + db.session.execute(text("INSERT INTO biomedical_concepts (bc_id, short_name) VALUES ('C1', 'Kept')")) + db.session.commit() + ensure_db(app) + assert _stamped_revision(app) not in LEGACY_REVISIONS + with app.app_context(): + assert db.session.execute(text("SELECT count(*) FROM biomedical_concepts")).scalar() == 1 + + def test_outdated_schema_raises(self, tmp_path): + app = _make_app(tmp_path) + with app.app_context(): + # A biomedical_concepts table missing post-baseline columns + db.session.execute(text("CREATE TABLE biomedical_concepts (bc_id VARCHAR(50) PRIMARY KEY, short_name VARCHAR(255))")) + db.session.commit() + with pytest.raises(RuntimeError, match="flask db stamp head"): + ensure_db(app) + + def test_idempotent_second_run(self, tmp_path): + app = _make_app(tmp_path) + ensure_db(app) + first = _stamped_revision(app) + ensure_db(app) # must be a no-op, not an error + assert _stamped_revision(app) == first From d0d5b15a8a7f5ec13a6ec9322e2f3b6c04e3c1bc Mon Sep 17 00:00:00 2001 From: pendingintent <3921919+pendingintent@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:09:16 -0400 Subject: [PATCH 21/36] =?UTF-8?q?=F0=9F=94=A7=20Hygiene:=20unify=20API=20c?= =?UTF-8?q?lients,=20tighten=20lint,=20split=20dev=20deps,=20wire=20isort?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit API clients (services/): - New services/api_cache.py — one stale-tolerant cache shared by all three clients (was: bespoke helper in cdisc_api, hand-rolled dict in ncit_api, none in loinc_api). cdisc_api re-exports the old names for compatibility. - CDISCApiClient: dual-header auth parity with soa-workbench — CDISC_SUBSCRIPTION_KEY/Ocp-Apim-Subscription-Key preferred, CDISC_API_KEY/api-key fallback (config.py gains CDISC_SUBSCRIPTION_KEY) - NCItApiClient: honors NCIT_API_BASE_URL from config/env (was a hardcoded class constant) - LoincApiClient: search results now cached (5 min TTL, stale fallback) - conftest: autouse fixture clears the shared cache between tests Hygiene: - Remove stale committed results.txt (pytest log) and empty src/ dir; drop the stray Excel lock file deletion that was already staged - app.py: debug mode gated by FLASK_DEBUG (default on for dev) - Split requirements-dev.txt (pytest, black, flake8, isort, pre-commit) out of requirements.txt - .flake8: max-line 200 (was 999); F401/F841/E501/E301/E302/F824 no longer ignored — fixed the fallout (unused imports/locals removed, conftest model imports marked as intentional metadata registration) - pre-commit: isort added (profile=black, configured in pyproject) - CI: install dev deps; run isort/black/flake8 before tests - import order normalized repo-wide by isort Suite: 207 passed; flake8 clean; smoke.sh 14/14. --- .flake8 | 6 +- .github/workflows/ci.yml | 7 +- .pre-commit-config.yaml | 5 + CLAUDE.md | 8 +- app.py | 13 +- config.py | 2 + extensions.py | 2 +- files/~$BC Examples.xlsx | Bin 165 -> 0 bytes migrations/env.py | 36 +-- .../51d4a009d291_baseline_initial_schema.py | 2 +- models/audit.py | 5 +- models/bc.py | 3 +- models/governance.py | 3 +- models/ingestion.py | 5 +- models/specialization.py | 3 +- requirements-dev.txt | 9 + requirements.txt | 8 - results.txt | 264 ------------------ routes/audit.py | 1 + routes/bc.py | 10 +- routes/dashboard.py | 6 +- routes/governance.py | 8 +- routes/ingestion.py | 14 +- routes/loinc.py | 13 +- routes/ncit.py | 5 +- routes/specializations.py | 5 +- services/api_cache.py | 42 +++ services/cdisc_api.py | 71 ++--- services/export.py | 3 +- services/ingestion.py | 4 +- services/loinc_api.py | 9 +- services/ncit_api.py | 34 ++- tests/conftest.py | 20 +- tests/test_audit_routes.py | 3 +- tests/test_bc_routes.py | 6 +- tests/test_export_service.py | 1 - tests/test_governance_routes.py | 10 +- tests/test_ingestion_routes.py | 6 +- tests/test_ingestion_service.py | 10 +- tests/test_loinc.py | 3 +- tests/test_models.py | 5 +- tests/test_ncit.py | 5 +- 42 files changed, 243 insertions(+), 432 deletions(-) delete mode 100644 files/~$BC Examples.xlsx create mode 100644 requirements-dev.txt delete mode 100644 results.txt create mode 100644 services/api_cache.py diff --git a/.flake8 b/.flake8 index 3cdb353..4a0dba5 100644 --- a/.flake8 +++ b/.flake8 @@ -1,7 +1,7 @@ [flake8] -max-line-length = 999 -ignore = E501,W503,E203,E301,E302,F401,F841,E711,F824 -exclude = +max-line-length = 200 +extend-ignore = E203, W503, E711 +exclude = .git, __pycache__, .venv, diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fdbebad..adcf0d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,11 @@ jobs: python-version: "3.12" cache: "pip" - name: Install dependencies - run: pip install -r requirements.txt + run: pip install -r requirements.txt -r requirements-dev.txt + - name: Lint (isort, black, flake8) + run: | + isort --check-only . + black --check . + flake8 . - name: Run tests run: pytest --tb=short diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 486f9a9..deb6661 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,9 @@ repos: + - repo: https://github.com/PyCQA/isort + rev: 6.0.1 + hooks: + - id: isort + - repo: https://github.com/psf/black rev: 26.3.1 hooks: diff --git a/CLAUDE.md b/CLAUDE.md index f1293d5..29b7a4d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,11 +29,15 @@ built via `flask db upgrade`; pre-baseline DBs are stamped in place). ## Linting ```bash +isort . +black . flake8 . -black --check . ``` -Line length is set to 200 (black) / 999 (flake8). Flake8 ignores F401, F841, E711 — see `.flake8` for full ignore list. +Line length is 200 (black, isort, flake8). Flake8 ignores only E203/W503 +(black conflicts) and E711 (SQLAlchemy `== None` filters); unused imports +(F401) and unused locals (F841) are errors. Dev tools are pinned in +`requirements-dev.txt`; CI runs the same lint + test steps. ## Testing diff --git a/app.py b/app.py index 370300d..3508618 100644 --- a/app.py +++ b/app.py @@ -1,6 +1,8 @@ import logging +import os from flask import Flask + from config import Config from extensions import db, migrate @@ -27,14 +29,14 @@ def create_app(config_class=Config): db.init_app(app) migrate.init_app(app, db) + from routes.audit import bp as audit_bp + from routes.bc import bp as bc_bp from routes.dashboard import bp as dashboard_bp + from routes.governance import bp as governance_bp from routes.ingestion import bp as ingestion_bp - from routes.bc import bp as bc_bp - from routes.ncit import bp as ncit_bp from routes.loinc import bp as loinc_bp + from routes.ncit import bp as ncit_bp from routes.specializations import bp as specializations_bp - from routes.governance import bp as governance_bp - from routes.audit import bp as audit_bp app.register_blueprint(dashboard_bp, url_prefix="/") app.register_blueprint(ingestion_bp, url_prefix="/ingestion") @@ -53,4 +55,5 @@ def create_app(config_class=Config): app = create_app() ensure_db(app) - app.run(debug=True, port=app.config["PORT"]) + # Dev-friendly default; set FLASK_DEBUG=0 to disable the debugger/reloader + app.run(debug=os.environ.get("FLASK_DEBUG", "1") == "1", port=app.config["PORT"]) diff --git a/config.py b/config.py index b4719b0..b41afc4 100644 --- a/config.py +++ b/config.py @@ -6,6 +6,8 @@ class Config: SQLALCHEMY_DATABASE_URI = os.environ.get("DATABASE_URL", "sqlite:///cdisc_curation.db") SQLALCHEMY_TRACK_MODIFICATIONS = False CDISC_API_KEY = os.environ.get("CDISC_API_KEY", "") + # Preferred over CDISC_API_KEY when set (Ocp-Apim-Subscription-Key header) + CDISC_SUBSCRIPTION_KEY = os.environ.get("CDISC_SUBSCRIPTION_KEY", "") CDISC_API_BASE_URL = "https://api.library.cdisc.org/api/cosmos/v2" NCIT_API_BASE_URL = "https://api-evsrest.nci.nih.gov/api/v1" MAX_CONTENT_LENGTH = 16 * 1024 * 1024 # 16MB upload limit diff --git a/extensions.py b/extensions.py index 378f0df..7869e82 100644 --- a/extensions.py +++ b/extensions.py @@ -1,5 +1,5 @@ -from flask_sqlalchemy import SQLAlchemy from flask_migrate import Migrate +from flask_sqlalchemy import SQLAlchemy db = SQLAlchemy() migrate = Migrate() diff --git a/files/~$BC Examples.xlsx b/files/~$BC Examples.xlsx deleted file mode 100644 index 4e0617bff91f7e88227a4fa989507e04b69f3a09..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 165 zcmd;gNh~T#%~SBrFG|fx%u7)q4)8O$FeEY*0bwdb9)kjdFGD_=3 - return current_app.extensions['migrate'].db.engine + return current_app.extensions["migrate"].db.engine def get_engine_url(): try: - return get_engine().url.render_as_string(hide_password=False).replace( - '%', '%%') + return get_engine().url.render_as_string(hide_password=False).replace("%", "%%") except AttributeError: - return str(get_engine().url).replace('%', '%%') + return str(get_engine().url).replace("%", "%%") # add your model's MetaData object here # for 'autogenerate' support # from myapp import mymodel # target_metadata = mymodel.Base.metadata -config.set_main_option('sqlalchemy.url', get_engine_url()) -target_db = current_app.extensions['migrate'].db +config.set_main_option("sqlalchemy.url", get_engine_url()) +target_db = current_app.extensions["migrate"].db # other values from the config, defined by the needs of env.py, # can be acquired: @@ -46,7 +44,7 @@ def get_engine_url(): def get_metadata(): - if hasattr(target_db, 'metadatas'): + if hasattr(target_db, "metadatas"): return target_db.metadatas[None] return target_db.metadata @@ -64,9 +62,7 @@ def run_migrations_offline(): """ url = config.get_main_option("sqlalchemy.url") - context.configure( - url=url, target_metadata=get_metadata(), literal_binds=True - ) + context.configure(url=url, target_metadata=get_metadata(), literal_binds=True) with context.begin_transaction(): context.run_migrations() @@ -84,24 +80,20 @@ def run_migrations_online(): # when there are no changes to the schema # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html def process_revision_directives(context, revision, directives): - if getattr(config.cmd_opts, 'autogenerate', False): + if getattr(config.cmd_opts, "autogenerate", False): script = directives[0] if script.upgrade_ops.is_empty(): directives[:] = [] - logger.info('No changes in schema detected.') + logger.info("No changes in schema detected.") - conf_args = current_app.extensions['migrate'].configure_args + conf_args = current_app.extensions["migrate"].configure_args if conf_args.get("process_revision_directives") is None: conf_args["process_revision_directives"] = process_revision_directives connectable = get_engine() with connectable.connect() as connection: - context.configure( - connection=connection, - target_metadata=get_metadata(), - **conf_args - ) + context.configure(connection=connection, target_metadata=get_metadata(), **conf_args) with context.begin_transaction(): context.run_migrations() diff --git a/migrations/versions/51d4a009d291_baseline_initial_schema.py b/migrations/versions/51d4a009d291_baseline_initial_schema.py index 804481b..3d9120d 100644 --- a/migrations/versions/51d4a009d291_baseline_initial_schema.py +++ b/migrations/versions/51d4a009d291_baseline_initial_schema.py @@ -6,8 +6,8 @@ """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = "51d4a009d291" diff --git a/models/audit.py b/models/audit.py index 75a058f..2b131c9 100644 --- a/models/audit.py +++ b/models/audit.py @@ -1,6 +1,7 @@ -from extensions import db -from datetime import datetime, timezone import json +from datetime import datetime, timezone + +from extensions import db class AuditLog(db.Model): diff --git a/models/bc.py b/models/bc.py index 8177442..b0f6cbf 100644 --- a/models/bc.py +++ b/models/bc.py @@ -1,6 +1,7 @@ -from extensions import db from datetime import datetime, timezone +from extensions import db + class BiomedicalConcept(db.Model): __tablename__ = "biomedical_concepts" diff --git a/models/governance.py b/models/governance.py index 1c90051..ab665c4 100644 --- a/models/governance.py +++ b/models/governance.py @@ -1,6 +1,7 @@ -from extensions import db from datetime import datetime, timezone +from extensions import db + class GovernanceRecord(db.Model): __tablename__ = "governance_records" diff --git a/models/ingestion.py b/models/ingestion.py index f64c130..692f01b 100644 --- a/models/ingestion.py +++ b/models/ingestion.py @@ -1,6 +1,7 @@ -from extensions import db -from datetime import datetime, timezone import json +from datetime import datetime, timezone + +from extensions import db class IngestionRecord(db.Model): diff --git a/models/specialization.py b/models/specialization.py index f07f17e..1085184 100644 --- a/models/specialization.py +++ b/models/specialization.py @@ -1,6 +1,7 @@ -from extensions import db import json +from extensions import db + class DatasetSpecialization(db.Model): __tablename__ = "dataset_specializations" diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..c34bc76 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,9 @@ +black==26.3.1 +flake8==7.3.0 +isort==6.0.1 +mccabe==0.7.0 +pre-commit==4.2.0 +pycodestyle==2.14.0 +pyflakes==3.4.0 +pytest==8.3.5 +pytest-flask==1.3.0 diff --git a/requirements.txt b/requirements.txt index aa8e25d..8e3c9fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ certifi==2026.2.25 charset-normalizer==3.4.6 click==8.3.1 et_xmlfile==2.0.0 -flake8==7.3.0 Flask==3.0.3 Flask-Migrate==4.0.7 Flask-SQLAlchemy==3.1.1 @@ -14,12 +13,9 @@ Jinja2==3.1.6 lxml==5.2.2 Mako==1.3.10 MarkupSafe==3.0.3 -mccabe==0.7.0 numpy==2.4.3 openpyxl==3.1.2 pandas==2.2.2 -pycodestyle==2.14.0 -pyflakes==3.4.0 python-dateutil==2.9.0.post0 pytz==2026.1.post1 requests==2.32.3 @@ -29,7 +25,3 @@ typing_extensions==4.15.0 tzdata==2025.3 urllib3==2.6.3 Werkzeug==3.1.7 -pytest==8.3.5 -pytest-flask==1.3.0 -pre-commit==4.2.0 -black==26.3.1 diff --git a/results.txt b/results.txt deleted file mode 100644 index 9125db1..0000000 --- a/results.txt +++ /dev/null @@ -1,264 +0,0 @@ -============================= test session starts ============================== -platform darwin -- Python 3.14.0, pytest-8.3.5, pluggy-1.6.0 -- /Users/dmoreland/projects/cdisc-concept-curation/.venv/bin/python3.14 -cachedir: .pytest_cache -rootdir: /Users/dmoreland/projects/cdisc-concept-curation -configfile: pyproject.toml -plugins: flask-1.3.0 -collecting ... collected 136 items - -tests/test_audit_routes.py::TestAuditIndex::test_returns_200_empty PASSED [ 0%] -tests/test_audit_routes.py::TestAuditIndex::test_shows_log_entries PASSED [ 1%] -tests/test_audit_routes.py::TestAuditIndex::test_filter_by_entity_type PASSED [ 2%] -tests/test_audit_routes.py::TestAuditIndex::test_filter_by_action PASSED [ 2%] -tests/test_audit_routes.py::TestAuditIndex::test_filter_by_actor PASSED [ 3%] -tests/test_audit_routes.py::TestAuditIndex::test_pagination_param_accepted PASSED [ 4%] -tests/test_bc_routes.py::TestBcIndex::test_returns_200 PASSED [ 5%] -tests/test_bc_routes.py::TestBcIndex::test_search_by_name PASSED [ 5%] -tests/test_bc_routes.py::TestBcIndex::test_search_no_match PASSED [ 6%] -tests/test_bc_routes.py::TestBcIndex::test_filter_by_status PASSED [ 7%] -tests/test_bc_routes.py::TestNewBc::test_returns_200 PASSED [ 8%] -tests/test_bc_routes.py::TestCreateBc::test_creates_bc_and_redirects PASSED [ 8%] -tests/test_bc_routes.py::TestCreateBc::test_missing_bc_id_redirects_with_error PASSED [ 9%] -tests/test_bc_routes.py::TestCreateBc::test_duplicate_bc_id_rejected PASSED [ 10%] -tests/test_bc_routes.py::TestCreateBc::test_create_writes_audit_log PASSED [ 11%] -tests/test_bc_routes.py::TestCreateBc::test_create_with_decs PASSED [ 11%] -tests/test_bc_routes.py::TestBcDetail::test_existing_bc_returns_200 PASSED [ 12%] -tests/test_bc_routes.py::TestBcDetail::test_missing_bc_returns_404 PASSED [ 13%] -tests/test_bc_routes.py::TestBcDetail::test_loinc_api_called_when_code_set PASSED [ 13%] -tests/test_bc_routes.py::TestBcDetail::test_loinc_api_not_called_when_no_code PASSED [ 14%] -tests/test_bc_routes.py::TestBcDetail::test_loinc_api_error_does_not_break_page PASSED [ 15%] -tests/test_bc_routes.py::TestEditBc::test_updates_short_name PASSED [ 16%] -tests/test_bc_routes.py::TestEditBc::test_edit_writes_audit_log PASSED [ 16%] -tests/test_bc_routes.py::TestEditBc::test_nonexistent_bc_returns_404 PASSED [ 17%] -tests/test_bc_routes.py::TestSubmitForReview::test_advances_status_to_sme_review PASSED [ 18%] -tests/test_bc_routes.py::TestSubmitForReview::test_submit_writes_audit_log PASSED [ 19%] -tests/test_bc_routes.py::TestDeleteBc::test_deletes_bc PASSED [ 19%] -tests/test_bc_routes.py::TestDeleteBc::test_delete_writes_audit_log PASSED [ 20%] -tests/test_bc_routes.py::TestDeleteBc::test_nonexistent_bc_returns_404 PASSED [ 21%] -tests/test_bc_routes.py::TestExport::test_json_export PASSED [ 22%] -tests/test_bc_routes.py::TestExport::test_xlsx_export PASSED [ 22%] -tests/test_bc_routes.py::TestExport::test_odm_xml_export PASSED [ 23%] -tests/test_bc_routes.py::TestLibraryDetail::test_renders_page_for_valid_concept PASSED [ 24%] -tests/test_bc_routes.py::TestLibraryDetail::test_redirects_on_api_error PASSED [ 25%] -tests/test_bc_routes.py::TestLibraryDetail::test_loinc_api_called_when_loinc_coding_present PASSED [ 25%] -tests/test_bc_routes.py::TestLibraryDetail::test_loinc_api_not_called_when_no_loinc_coding PASSED [ 26%] -tests/test_bc_routes.py::TestLibraryDetail::test_loinc_api_error_does_not_break_page PASSED [ 27%] -tests/test_governance_routes.py::TestGovernanceBoard::test_board_returns_200 PASSED [ 27%] -tests/test_governance_routes.py::TestAdvance::test_advances_provisional_to_sme_review PASSED [ 28%] -tests/test_governance_routes.py::TestAdvance::test_advance_through_all_stages PASSED [ 29%] -tests/test_governance_routes.py::TestAdvance::test_already_published_stays_published PASSED [ 30%] -tests/test_governance_routes.py::TestAdvance::test_advance_creates_governance_record PASSED [ 30%] -tests/test_governance_routes.py::TestAdvance::test_advance_writes_audit_log PASSED [ 31%] -tests/test_governance_routes.py::TestAdvance::test_advance_nonexistent_bc_returns_404 PASSED [ 32%] -tests/test_governance_routes.py::TestAdvance::test_advance_ajax_returns_json PASSED [ 33%] -tests/test_governance_routes.py::TestReject::test_reject_returns_to_provisional PASSED [ 33%] -tests/test_governance_routes.py::TestReject::test_reject_creates_governance_record PASSED [ 34%] -tests/test_governance_routes.py::TestReject::test_reject_writes_audit_log PASSED [ 35%] -tests/test_governance_routes.py::TestReject::test_reject_ajax_returns_json PASSED [ 36%] -tests/test_governance_routes.py::TestReject::test_reject_nonexistent_bc_returns_404 PASSED [ 36%] -tests/test_ingestion_routes.py::TestIngestionIndex::test_returns_200 PASSED [ 37%] -tests/test_ingestion_routes.py::TestUpload::test_upload_csv_creates_ingestion_records PASSED [ 38%] -tests/test_ingestion_routes.py::TestUpload::test_upload_json_creates_ingestion_records PASSED [ 38%] -tests/test_ingestion_routes.py::TestUpload::test_no_file_redirects_with_error PASSED [ 39%] -tests/test_ingestion_routes.py::TestUpload::test_wrong_extension_rejected PASSED [ 40%] -tests/test_ingestion_routes.py::TestUpload::test_duplicate_bc_flagged PASSED [ 41%] -tests/test_ingestion_routes.py::TestApprove::test_approve_creates_bc PASSED [ 41%] -tests/test_ingestion_routes.py::TestApprove::test_approve_sets_status_approved PASSED [ 42%] -tests/test_ingestion_routes.py::TestApprove::test_approve_nonexistent_record_returns_404 PASSED [ 43%] -tests/test_ingestion_routes.py::TestApprove::test_approve_already_existing_bc_does_not_duplicate PASSED [ 44%] -tests/test_ingestion_routes.py::TestReject::test_reject_sets_status_rejected PASSED [ 44%] -tests/test_ingestion_routes.py::TestReject::test_reject_does_not_create_bc PASSED [ 45%] -tests/test_ingestion_routes.py::TestReject::test_reject_nonexistent_record_returns_404 PASSED [ 46%] -tests/test_ingestion_routes.py::TestApproveAll::test_approve_all_skips_records_with_errors PASSED [ 47%] -tests/test_ingestion_service.py::TestSimilarity::test_identical_strings PASSED [ 47%] -tests/test_ingestion_service.py::TestSimilarity::test_case_insensitive PASSED [ 48%] -tests/test_ingestion_service.py::TestSimilarity::test_completely_different PASSED [ 49%] -tests/test_ingestion_service.py::TestSimilarity::test_partial_match PASSED [ 50%] -tests/test_ingestion_service.py::TestMatchField::test_exact_alias_match PASSED [ 50%] -tests/test_ingestion_service.py::TestMatchField::test_known_alias PASSED [ 51%] -tests/test_ingestion_service.py::TestMatchField::test_definition_alias PASSED [ 52%] -tests/test_ingestion_service.py::TestMatchField::test_spaces_normalised PASSED [ 52%] -tests/test_ingestion_service.py::TestMatchField::test_unknown_column_returns_low_score PASSED [ 53%] -tests/test_ingestion_service.py::TestMapFields::test_canonical_columns_mapped_at_full_confidence PASSED [ 54%] -tests/test_ingestion_service.py::TestMapFields::test_none_values_skipped PASSED [ 55%] -tests/test_ingestion_service.py::TestMapFields::test_nan_values_skipped PASSED [ 55%] -tests/test_ingestion_service.py::TestMapFields::test_low_confidence_columns_excluded PASSED [ 56%] -tests/test_ingestion_service.py::TestMapFields::test_values_stripped PASSED [ 57%] -tests/test_ingestion_service.py::TestValidateBc::test_valid_record_has_no_errors PASSED [ 58%] -tests/test_ingestion_service.py::TestValidateBc::test_missing_short_name PASSED [ 58%] -tests/test_ingestion_service.py::TestValidateBc::test_missing_definition PASSED [ 59%] -tests/test_ingestion_service.py::TestValidateBc::test_missing_both_ids PASSED [ 60%] -tests/test_ingestion_service.py::TestValidateBc::test_invalid_ncit_format PASSED [ 61%] -tests/test_ingestion_service.py::TestValidateBc::test_ncit_starting_with_c_accepted PASSED [ 61%] -tests/test_ingestion_service.py::TestDeduplicate::test_marks_existing_ids_as_duplicate PASSED [ 62%] -tests/test_ingestion_service.py::TestDeduplicate::test_case_insensitive_comparison PASSED [ 63%] -tests/test_ingestion_service.py::TestDeduplicate::test_empty_existing_ids PASSED [ 63%] -tests/test_ingestion_service.py::TestDeduplicate::test_falls_back_to_ncit_code PASSED [ 64%] -tests/test_ingestion_service.py::TestParseCsv::test_valid_row_parsed PASSED [ 65%] -tests/test_ingestion_service.py::TestParseCsv::test_invalid_file_returns_error_record PASSED [ 66%] -tests/test_ingestion_service.py::TestParseCsv::test_missing_definition_produces_validation_error PASSED [ 66%] -tests/test_ingestion_service.py::TestParseJson::test_array_of_objects PASSED [ 67%] -tests/test_ingestion_service.py::TestParseJson::test_single_object_wrapped PASSED [ 68%] -tests/test_ingestion_service.py::TestParseJson::test_invalid_json_returns_error PASSED [ 69%] -tests/test_ingestion_service.py::TestGroupByBc::test_single_row_becomes_one_record PASSED [ 69%] -tests/test_ingestion_service.py::TestGroupByBc::test_dec_sub_rows_grouped_under_parent PASSED [ 70%] -tests/test_ingestion_service.py::TestGroupByBc::test_multiple_bcs_produce_separate_records PASSED [ 71%] -tests/test_ingestion_service.py::TestGroupByBc::test_row_without_bc_id_skipped PASSED [ 72%] -tests/test_loinc.py::TestLoincApiClientSearch::test_returns_normalized_list PASSED [ 72%] -tests/test_loinc.py::TestLoincApiClientSearch::test_all_ef_fields_present_in_result PASSED [ 73%] -tests/test_loinc.py::TestLoincApiClientSearch::test_uses_ef_parameter PASSED [ 74%] -tests/test_loinc.py::TestLoincApiClientSearch::test_ef_fields_constant_contains_all_required_fields PASSED [ 75%] -tests/test_loinc.py::TestLoincApiClientSearch::test_uses_basic_auth_when_env_vars_set PASSED [ 75%] -tests/test_loinc.py::TestLoincApiClientSearch::test_no_auth_when_env_vars_missing PASSED [ 76%] -tests/test_loinc.py::TestLoincApiClientSearch::test_empty_results PASSED [ 77%] -tests/test_loinc.py::TestLoincApiClientSearch::test_missing_codes_array_returns_empty PASSED [ 77%] -tests/test_loinc.py::TestLoincApiClientSearch::test_network_error_returns_error_entry PASSED [ 78%] -tests/test_loinc.py::TestLoincSearchRoute::test_returns_json_for_ajax PASSED [ 79%] -tests/test_loinc.py::TestLoincSearchRoute::test_empty_term_returns_empty_list PASSED [ 80%] -tests/test_loinc.py::TestLoincSearchRoute::test_calls_client_with_term PASSED [ 80%] -tests/test_loinc.py::TestLoincSearchRoute::test_format_json_param_triggers_json_response PASSED [ 81%] -tests/test_models.py::TestAuditLogJsonProperties::test_before_state_round_trips PASSED [ 82%] -tests/test_models.py::TestAuditLogJsonProperties::test_after_state_round_trips PASSED [ 83%] -tests/test_models.py::TestAuditLogJsonProperties::test_none_before_state_returns_none PASSED [ 83%] -tests/test_models.py::TestAuditLogJsonProperties::test_none_after_state_returns_none PASSED [ 84%] -tests/test_models.py::TestAuditLogJsonProperties::test_persisted_log_retrieves_state PASSED [ 85%] -tests/test_models.py::TestIngestionRecordProperties::test_mapped_round_trips PASSED [ 86%] -tests/test_models.py::TestIngestionRecordProperties::test_confidences_round_trips PASSED [ 86%] -tests/test_models.py::TestIngestionRecordProperties::test_errors_round_trips PASSED [ 87%] -tests/test_models.py::TestIngestionRecordProperties::test_decs_round_trips PASSED [ 88%] -tests/test_models.py::TestIngestionRecordProperties::test_empty_mapped_returns_empty_dict PASSED [ 88%] -tests/test_models.py::TestIngestionRecordProperties::test_avg_confidence_computed_correctly PASSED [ 89%] -tests/test_models.py::TestIngestionRecordProperties::test_avg_confidence_empty_confidences PASSED [ 90%] -tests/test_models.py::TestBiomedicalConceptToDict::test_to_dict_contains_required_keys PASSED [ 91%] -tests/test_models.py::TestBiomedicalConceptToDict::test_to_dict_values_match PASSED [ 91%] -tests/test_models.py::TestBiomedicalConceptToDict::test_default_status_is_provisional PASSED [ 92%] -tests/test_ncit.py::TestNcitGetConceptExtended::test_returns_parents PASSED [ 93%] -tests/test_ncit.py::TestNcitGetConceptExtended::test_returns_semantic_type PASSED [ 94%] -tests/test_ncit.py::TestNcitGetConceptExtended::test_returns_source_synonyms PASSED [ 94%] -tests/test_ncit.py::TestNcitGetConceptExtended::test_returns_all_definitions PASSED [ 95%] -tests/test_ncit.py::TestNcitGetConceptExtended::test_empty_parents_returns_empty_list PASSED [ 96%] -tests/test_ncit.py::TestNcitGetConceptExtended::test_missing_semantic_type_returns_empty_list PASSED [ 97%] -tests/test_ncit.py::TestNcitGetConceptExtended::test_error_returns_error_dict PASSED [ 97%] -tests/test_ncit.py::TestNcitConceptRoute::test_returns_json PASSED [ 98%] -tests/test_ncit.py::TestNcitConceptRoute::test_calls_get_concept_with_code PASSED [ 99%] -tests/test_ncit.py::TestNcitConceptRoute::test_error_from_service_returns_500 PASSED [100%] - -=============================== warnings summary =============================== -tests/test_audit_routes.py: 8 warnings -tests/test_bc_routes.py: 49 warnings -tests/test_governance_routes.py: 52 warnings -tests/test_ingestion_routes.py: 19 warnings -tests/test_models.py: 3 warnings - /Users/dmoreland/projects/cdisc-concept-curation/.venv/lib/python3.14/site-packages/sqlalchemy/sql/schema.py:3624: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). - return util.wrap_callable(lambda ctx: fn(), fn) # type: ignore - -tests/test_bc_routes.py::TestCreateBc::test_creates_bc_and_redirects -tests/test_bc_routes.py::TestCreateBc::test_duplicate_bc_id_rejected -tests/test_bc_routes.py::TestCreateBc::test_create_writes_audit_log -tests/test_bc_routes.py::TestCreateBc::test_create_with_decs - /Users/dmoreland/projects/cdisc-concept-curation/routes/bc.py:183: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) - if BiomedicalConcept.query.get(bc_id): - -tests/test_bc_routes.py::TestCreateBc::test_creates_bc_and_redirects - /Users/dmoreland/projects/cdisc-concept-curation/tests/test_bc_routes.py:67: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) - assert BiomedicalConcept.query.get("C00001") is not None - -tests/test_bc_routes.py: 13 warnings -tests/test_governance_routes.py: 19 warnings -tests/test_ingestion_routes.py: 7 warnings - /Users/dmoreland/projects/cdisc-concept-curation/.venv/lib/python3.14/site-packages/flask_sqlalchemy/query.py:30: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) - rv = self.get(ident) - -tests/test_bc_routes.py::TestEditBc::test_updates_short_name -tests/test_bc_routes.py::TestEditBc::test_edit_writes_audit_log - /Users/dmoreland/projects/cdisc-concept-curation/routes/bc.py:236: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). - bc.updated_at = datetime.utcnow() - -tests/test_bc_routes.py::TestEditBc::test_updates_short_name - /Users/dmoreland/projects/cdisc-concept-curation/tests/test_bc_routes.py:180: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) - bc = BiomedicalConcept.query.get("C12345") - -tests/test_bc_routes.py::TestSubmitForReview::test_advances_status_to_sme_review -tests/test_bc_routes.py::TestSubmitForReview::test_submit_writes_audit_log - /Users/dmoreland/projects/cdisc-concept-curation/routes/bc.py:257: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). - bc.updated_at = datetime.utcnow() - -tests/test_bc_routes.py::TestSubmitForReview::test_advances_status_to_sme_review - /Users/dmoreland/projects/cdisc-concept-curation/tests/test_bc_routes.py:204: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) - bc = BiomedicalConcept.query.get("C12345") - -tests/test_bc_routes.py::TestDeleteBc::test_deletes_bc - /Users/dmoreland/projects/cdisc-concept-curation/tests/test_bc_routes.py:223: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) - assert BiomedicalConcept.query.get("C12345") is None - -tests/test_bc_routes.py::TestExport::test_xlsx_export - /Users/dmoreland/projects/cdisc-concept-curation/.venv/lib/python3.14/site-packages/openpyxl/packaging/core.py:99: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). - now = datetime.datetime.utcnow() - -tests/test_bc_routes.py::TestExport::test_xlsx_export - /Users/dmoreland/projects/cdisc-concept-curation/.venv/lib/python3.14/site-packages/openpyxl/writer/excel.py:292: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). - workbook.properties.modified = datetime.datetime.utcnow() - -tests/test_bc_routes.py::TestExport::test_odm_xml_export - /Users/dmoreland/projects/cdisc-concept-curation/services/export.py:77: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). - "FileOID": f'CDISC.BC.Export.{datetime.utcnow().strftime("%Y%m%d%H%M%S")}', - -tests/test_bc_routes.py::TestExport::test_odm_xml_export - /Users/dmoreland/projects/cdisc-concept-curation/services/export.py:78: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). - "CreationDateTime": datetime.utcnow().isoformat(), - -tests/test_governance_routes.py: 12 warnings - /Users/dmoreland/projects/cdisc-concept-curation/routes/governance.py:33: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). - bc.updated_at = datetime.utcnow() - -tests/test_governance_routes.py::TestAdvance::test_advances_provisional_to_sme_review - /Users/dmoreland/projects/cdisc-concept-curation/tests/test_governance_routes.py:22: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) - bc = BiomedicalConcept.query.get("C12345") - -tests/test_governance_routes.py::TestAdvance::test_advance_through_all_stages - /Users/dmoreland/projects/cdisc-concept-curation/tests/test_governance_routes.py:29: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) - bc = BiomedicalConcept.query.get("C12345") - -tests/test_governance_routes.py::TestAdvance::test_already_published_stays_published - /Users/dmoreland/projects/cdisc-concept-curation/tests/test_governance_routes.py:40: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) - bc = BiomedicalConcept.query.get("C12345") - -tests/test_governance_routes.py::TestReject::test_reject_returns_to_provisional -tests/test_governance_routes.py::TestReject::test_reject_creates_governance_record -tests/test_governance_routes.py::TestReject::test_reject_writes_audit_log -tests/test_governance_routes.py::TestReject::test_reject_ajax_returns_json - /Users/dmoreland/projects/cdisc-concept-curation/routes/governance.py:65: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). - bc.updated_at = datetime.utcnow() - -tests/test_governance_routes.py::TestReject::test_reject_returns_to_provisional - /Users/dmoreland/projects/cdisc-concept-curation/tests/test_governance_routes.py:78: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) - bc = BiomedicalConcept.query.get("C12345") - -tests/test_ingestion_routes.py::TestApprove::test_approve_creates_bc -tests/test_ingestion_routes.py::TestApprove::test_approve_sets_status_approved -tests/test_ingestion_routes.py::TestApprove::test_approve_already_existing_bc_does_not_duplicate - /Users/dmoreland/projects/cdisc-concept-curation/routes/ingestion.py:121: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) - if not BiomedicalConcept.query.get(bc_id): - -tests/test_ingestion_routes.py::TestApprove::test_approve_creates_bc - /Users/dmoreland/projects/cdisc-concept-curation/tests/test_ingestion_routes.py:103: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) - assert BiomedicalConcept.query.get("C001") is not None - -tests/test_ingestion_routes.py::TestApprove::test_approve_sets_status_approved - /Users/dmoreland/projects/cdisc-concept-curation/tests/test_ingestion_routes.py:109: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) - ir = IngestionRecord.query.get(record_id) - -tests/test_ingestion_routes.py::TestReject::test_reject_sets_status_rejected - /Users/dmoreland/projects/cdisc-concept-curation/tests/test_ingestion_routes.py:148: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) - ir = IngestionRecord.query.get(record_id) - -tests/test_ingestion_routes.py::TestReject::test_reject_does_not_create_bc - /Users/dmoreland/projects/cdisc-concept-curation/tests/test_ingestion_routes.py:165: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) - assert BiomedicalConcept.query.get("C001") is None - -tests/test_ingestion_routes.py::TestApproveAll::test_approve_all_skips_records_with_errors - /Users/dmoreland/projects/cdisc-concept-curation/tests/test_ingestion_routes.py:187: LegacyAPIWarning: The Query.get() method is considered legacy as of the 1.x series of SQLAlchemy and becomes a legacy construct in 2.0. The method is now available as Session.get() (deprecated since: 2.0) (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) - assert BiomedicalConcept.query.get("C001") is None - --- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html -====================== 136 passed, 214 warnings in 0.87s ======================= diff --git a/routes/audit.py b/routes/audit.py index 07bb0cf..4b43025 100644 --- a/routes/audit.py +++ b/routes/audit.py @@ -1,4 +1,5 @@ from flask import Blueprint, render_template, request + from models.audit import AuditLog bp = Blueprint("audit", __name__) diff --git a/routes/bc.py b/routes/bc.py index d9e9e2d..4771669 100644 --- a/routes/bc.py +++ b/routes/bc.py @@ -1,15 +1,17 @@ import json import logging -from flask import Blueprint, render_template, request, redirect, url_for, flash, Response +from datetime import datetime, timezone + +from flask import Blueprint, Response, flash, redirect, render_template, request, url_for + +from extensions import db from models.bc import BiomedicalConcept, DataElementConcept from models.governance import GovernanceRecord -from extensions import db from services.audit import log_change -from services.export import export_json, export_xlsx, export_odm_xml from services.cdisc_api import CDISCApiClient +from services.export import export_json, export_odm_xml, export_xlsx from services.loinc_api import LoincApiClient from services.ncit_api import NCItApiClient -from datetime import datetime, timezone logger = logging.getLogger(__name__) diff --git a/routes/dashboard.py b/routes/dashboard.py index b032e76..d84c299 100644 --- a/routes/dashboard.py +++ b/routes/dashboard.py @@ -1,8 +1,10 @@ -from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timedelta, timezone + from flask import Blueprint, render_template -from models.bc import BiomedicalConcept + from models.audit import AuditLog +from models.bc import BiomedicalConcept from services.cdisc_api import CDISCApiClient bp = Blueprint("dashboard", __name__) diff --git a/routes/governance.py b/routes/governance.py index c7aff28..2501841 100644 --- a/routes/governance.py +++ b/routes/governance.py @@ -1,8 +1,10 @@ -from flask import Blueprint, render_template, request, redirect, url_for, flash, jsonify, Response +from datetime import datetime, timezone + +from flask import Blueprint, Response, flash, jsonify, redirect, render_template, request, url_for + +from extensions import db from models.bc import BiomedicalConcept from models.governance import GovernanceRecord -from extensions import db -from datetime import datetime, timezone from services.audit import log_change from services.export import export_governance_xlsx diff --git a/routes/ingestion.py b/routes/ingestion.py index e6c4bad..933a8f9 100644 --- a/routes/ingestion.py +++ b/routes/ingestion.py @@ -1,18 +1,20 @@ import uuid + from flask import ( Blueprint, + flash, + redirect, render_template, request, - redirect, - url_for, - flash, session, + url_for, ) + +from extensions import db +from models.audit import AuditLog from models.bc import BiomedicalConcept, DataElementConcept from models.ingestion import IngestionRecord -from models.audit import AuditLog -from extensions import db -from services.ingestion import parse_xlsx, parse_csv, parse_json, deduplicate +from services.ingestion import deduplicate, parse_csv, parse_json, parse_xlsx bp = Blueprint("ingestion", __name__) diff --git a/routes/loinc.py b/routes/loinc.py index 8eb58bf..bb3b4c2 100644 --- a/routes/loinc.py +++ b/routes/loinc.py @@ -1,17 +1,14 @@ from flask import Blueprint, jsonify, request + from services.loinc_api import LoincApiClient -bp = Blueprint('loinc', __name__) +bp = Blueprint("loinc", __name__) -@bp.route('/search') +@bp.route("/search") def search(): - term = request.args.get('term', '').strip() - is_ajax = ( - request.headers.get('X-Requested-With') == 'XMLHttpRequest' - or request.args.get('format') == 'json' - or 'application/json' in request.headers.get('Accept', '') - ) + term = request.args.get("term", "").strip() + is_ajax = request.headers.get("X-Requested-With") == "XMLHttpRequest" or request.args.get("format") == "json" or "application/json" in request.headers.get("Accept", "") if not term: if is_ajax: return jsonify([]) diff --git a/routes/ncit.py b/routes/ncit.py index 23545bf..88c28ff 100644 --- a/routes/ncit.py +++ b/routes/ncit.py @@ -1,6 +1,7 @@ -from flask import Blueprint, render_template, request, redirect, url_for, flash, jsonify -from models.bc import BiomedicalConcept +from flask import Blueprint, flash, jsonify, redirect, render_template, request, url_for + from extensions import db +from models.bc import BiomedicalConcept from services.ncit_api import NCItApiClient bp = Blueprint("ncit", __name__) diff --git a/routes/specializations.py b/routes/specializations.py index 9e77fa2..3236ef1 100644 --- a/routes/specializations.py +++ b/routes/specializations.py @@ -1,7 +1,8 @@ -from flask import Blueprint, render_template, request, redirect, url_for, flash, jsonify +from flask import Blueprint, flash, jsonify, redirect, render_template, request, url_for + +from extensions import db from models.bc import BiomedicalConcept, DataElementConcept from models.specialization import DatasetSpecialization -from extensions import db from services.cdisc_api import CDISCApiClient bp = Blueprint("specializations", __name__) diff --git a/services/api_cache.py b/services/api_cache.py new file mode 100644 index 0000000..c803790 --- /dev/null +++ b/services/api_cache.py @@ -0,0 +1,42 @@ +"""Shared stale-tolerant in-memory cache for external API clients. + +One caching strategy for all three clients (CDISC Library, NCI EVS, +NLM LOINC): serve fresh entries for CACHE_TTL seconds; on expiry attempt +a refresh but fall back to the stale entry (up to CACHE_STALE_TTL) rather +than propagating an error, so a slow or failing upstream never blanks a +page that rendered fine a minute ago. +""" + +import logging +import time + +logger = logging.getLogger(__name__) + +# {key: (timestamp, data)} — entries are never evicted; stale data is the +# fallback when a refresh fails. +_cache = {} +CACHE_TTL = 300 # serve fresh data for 5 minutes +CACHE_STALE_TTL = 3600 # serve stale data for up to 1 hour while refresh fails + + +def cached(cache_key, fn): + """Return cached data if fresh. If stale, attempt a refresh but fall back + to the stale entry rather than propagating an error or blocking.""" + now = time.time() + entry = _cache.get(cache_key) + + if entry and now - entry[0] < CACHE_TTL: + return entry[1] # fresh — serve immediately + + # Attempt a refresh. Broad catch is intentional: this is a resilience + # seam and any refresh failure must fall back to stale data. + try: + data = fn() + _cache[cache_key] = (now, data) + return data + except Exception: + if entry and now - entry[0] < CACHE_STALE_TTL: + logger.warning("Cache refresh failed for %s; serving stale entry", cache_key, exc_info=True) + return entry[1] + logger.error("Cache refresh failed for %s with no stale fallback", cache_key, exc_info=True) + raise # genuinely no data at all — let caller handle diff --git a/services/cdisc_api.py b/services/cdisc_api.py index 086a218..996ceea 100644 --- a/services/cdisc_api.py +++ b/services/cdisc_api.py @@ -1,60 +1,47 @@ import hashlib import logging import os -import time + import requests from flask import current_app -logger = logging.getLogger(__name__) - -# In-memory cache: {key: (timestamp, data)} -# Entries are never evicted — stale data is served while a refresh is attempted, -# so a timeout never blocks the request with an empty response. -_cache = {} -_CACHE_TTL = 300 # serve fresh data for 5 minutes -_CACHE_STALE_TTL = 3600 # serve stale data for up to 1 hour while refresh fails +from services.api_cache import CACHE_STALE_TTL as _CACHE_STALE_TTL # noqa: F401 +from services.api_cache import CACHE_TTL as _CACHE_TTL # noqa: F401 +from services.api_cache import _cache, cached # noqa: F401 (_cache re-exported for tests) +logger = logging.getLogger(__name__) -def _cached(cache_key, fn): - """Return cached data if fresh. If stale, attempt a refresh but fall back - to the stale entry rather than propagating an error or blocking indefinitely.""" - now = time.time() - entry = _cache.get(cache_key) +# Backward-compatible alias — the shared implementation lives in +# services/api_cache.py and is used by all three API clients. +_cached = cached - if entry and now - entry[0] < _CACHE_TTL: - return entry[1] # fresh — serve immediately - # Attempt a refresh. Broad catch is intentional: this is a resilience - # seam and any refresh failure must fall back to stale data. +def _config_value(name, default=""): + """Read a config value from the Flask app when a context is active, + falling back to the environment (the MCP server and scripts run + without a request context).""" try: - data = fn() - _cache[cache_key] = (now, data) - return data - except Exception: - if entry and now - entry[0] < _CACHE_STALE_TTL: - logger.warning("Cache refresh failed for %s; serving stale entry", cache_key, exc_info=True) - return entry[1] - logger.error("Cache refresh failed for %s with no stale fallback", cache_key, exc_info=True) - raise # genuinely no data at all — let caller handle + return current_app.config.get(name) or os.environ.get(name, default) + except RuntimeError: + return os.environ.get(name, default) class CDISCApiClient: def __init__(self): - try: - self.api_key = current_app.config.get("CDISC_API_KEY") or os.environ.get("CDISC_API_KEY", "") - self.base_url = current_app.config.get( - "CDISC_API_BASE_URL", - "https://api.library.cdisc.org/api/cosmos/v2", - ) - except RuntimeError: - self.api_key = os.environ.get("CDISC_API_KEY", "") - self.base_url = "https://api.library.cdisc.org/api/cosmos/v2" - self.headers = { - "api-key": self.api_key, - "Accept": "application/json", - } - # Stable, non-secret digest of the api_key for use in cache keys - self._key_digest = hashlib.sha256(self.api_key.encode()).hexdigest()[:8] + self.api_key = _config_value("CDISC_API_KEY") + self.subscription_key = _config_value("CDISC_SUBSCRIPTION_KEY") + self.base_url = _config_value("CDISC_API_BASE_URL", "https://api.library.cdisc.org/api/cosmos/v2") + # Auth parity with soa-workbench: prefer the subscription key with + # its Ocp header, fall back to the classic api-key header. + if self.subscription_key: + auth_header = {"Ocp-Apim-Subscription-Key": self.subscription_key} + key_material = self.subscription_key + else: + auth_header = {"api-key": self.api_key} + key_material = self.api_key + self.headers = {**auth_header, "Accept": "application/json"} + # Stable, non-secret digest of the active key for use in cache keys + self._key_digest = hashlib.sha256(key_material.encode()).hexdigest()[:8] def _cache_key(self, endpoint): return (self.base_url, self._key_digest, endpoint) diff --git a/services/export.py b/services/export.py index 2dc0dcb..951448a 100644 --- a/services/export.py +++ b/services/export.py @@ -4,7 +4,7 @@ try: import openpyxl - from openpyxl.styles import Font, PatternFill, Alignment + from openpyxl.styles import Alignment, Font, PatternFill except ImportError: openpyxl = None @@ -44,7 +44,6 @@ def export_xlsx(bc_list): ws = wb.active ws.title = "Biomedical Concepts" - header_font = Font(bold=True) header_fill = PatternFill("solid", fgColor="003366") header_font_white = Font(bold=True, color="FFFFFF") diff --git a/services/ingestion.py b/services/ingestion.py index e228dc0..54882a6 100644 --- a/services/ingestion.py +++ b/services/ingestion.py @@ -1,9 +1,9 @@ -import io import json import logging -import pandas as pd from difflib import SequenceMatcher +import pandas as pd + logger = logging.getLogger(__name__) # Canonical BC field names and known aliases for fuzzy field mapping diff --git a/services/loinc_api.py b/services/loinc_api.py index cde28c8..bf054e7 100644 --- a/services/loinc_api.py +++ b/services/loinc_api.py @@ -1,7 +1,10 @@ import logging import os + import requests +from services.api_cache import cached + logger = logging.getLogger(__name__) LOINC_EF_FIELDS = ( @@ -28,7 +31,8 @@ def search(self, term, size=10): Uses the ef (extra fields) parameter so all field values are returned. Response format: [total, [codes], {field: [values, ...]}, display_data] """ - try: + + def _fetch(): response = requests.get( self.BASE_URL, params={"ef": LOINC_EF_FIELDS, "terms": term, "maxList": size}, @@ -48,6 +52,9 @@ def search(self, term, size=10): item[field] = values[i] if values and i < len(values) else None results.append(item) return results + + try: + return cached(("loinc_search", term, size), _fetch) except (requests.RequestException, ValueError, IndexError, TypeError) as e: # Index/Type errors cover the positional parsing of the NLM # array response ([total, [codes], {field: values}, ...]). diff --git a/services/ncit_api.py b/services/ncit_api.py index 3943bc9..e50fda5 100644 --- a/services/ncit_api.py +++ b/services/ncit_api.py @@ -1,12 +1,14 @@ import logging -import time +import os + import requests +from flask import current_app + +from services.api_cache import cached logger = logging.getLogger(__name__) -_ncit_cache = {} -_NCIT_TTL = 300 # serve fresh data for 5 minutes -_NCIT_STALE_TTL = 3600 # serve stale data for up to 1 hour while refresh fails +_DEFAULT_BASE_URL = "https://api-evsrest.nci.nih.gov/api/v1" def _pick_definition(definitions): @@ -20,10 +22,16 @@ def _pick_definition(definitions): class NCItApiClient: - BASE_URL = "https://api-evsrest.nci.nih.gov/api/v1" + def __init__(self): + # Honor NCIT_API_BASE_URL from Flask config, falling back to the + # environment when no app context is active (MCP server, scripts). + try: + self.base_url = current_app.config.get("NCIT_API_BASE_URL") or os.environ.get("NCIT_API_BASE_URL", _DEFAULT_BASE_URL) + except RuntimeError: + self.base_url = os.environ.get("NCIT_API_BASE_URL", _DEFAULT_BASE_URL) def _get(self, path, params=None): - url = f"{self.BASE_URL}{path}" + url = f"{self.base_url}{path}" response = requests.get(url, params=params, timeout=15) response.raise_for_status() return response.json() @@ -48,16 +56,11 @@ def search_concept(self, term, size=10): def get_concept(self, ncit_code): """Fetch full concept details including synonyms, definitions, parents, and semantic type.""" - cache_key = ("concept", ncit_code) - now = time.time() - entry = _ncit_cache.get(cache_key) - if entry and now - entry[0] < _NCIT_TTL: - return entry[1] - try: + def _fetch(): result = self._get(f"/concept/ncit/{ncit_code}", params={"include": "full"}) code = result.get("code") - data = { + return { "code": code, "name": result.get("name"), "preferred_name": result.get("name"), @@ -69,8 +72,9 @@ def get_concept(self, ncit_code): "semantic_type": [st.get("name") for st in result.get("semanticType", [])], "reference": f"https://ncithesaurus.nci.nih.gov/ncitbrowser/ConceptReport.jsp?dictionary=NCI_Thesaurus&code={code}" if code else "", } - _ncit_cache[cache_key] = (now, data) - return data + + try: + return cached(("ncit_concept", self.base_url, ncit_code), _fetch) except (requests.RequestException, ValueError) as e: logger.error("NCIt concept fetch failed for %s: %s", ncit_code, e) return {"error": str(e)} diff --git a/tests/conftest.py b/tests/conftest.py index dfe64d7..c7281ae 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,11 @@ import pytest + from app import create_app from extensions import db as _db -from models.bc import BiomedicalConcept, DataElementConcept -from models.governance import GovernanceRecord -from models.audit import AuditLog -from models.ingestion import IngestionRecord +from models.audit import AuditLog # noqa: F401 (registers table metadata) +from models.bc import BiomedicalConcept, DataElementConcept # noqa: F401 +from models.governance import GovernanceRecord # noqa: F401 +from models.ingestion import IngestionRecord # noqa: F401 class TestConfig: @@ -12,6 +13,7 @@ class TestConfig: SQLALCHEMY_DATABASE_URI = "sqlite:///:memory:" SECRET_KEY = "test-secret-key" CDISC_API_KEY = "" + CDISC_SUBSCRIPTION_KEY = "" CDISC_API_BASE_URL = "https://api.library.cdisc.org/api/cosmos/v2" NCIT_API_BASE_URL = "https://api-evsrest.nci.nih.gov/api/v1" MAX_CONTENT_LENGTH = 16 * 1024 * 1024 @@ -39,6 +41,16 @@ def clean_db(app): _db.session.remove() +@pytest.fixture(autouse=True) +def clear_api_cache(): + """Reset the shared external-API cache so tests never see each other's + (or a failure's) cached responses.""" + from services import api_cache + + api_cache._cache.clear() + yield + + @pytest.fixture() def sample_bc(app): """A minimal BiomedicalConcept persisted to the test DB.""" diff --git a/tests/test_audit_routes.py b/tests/test_audit_routes.py index 8e2b649..790f600 100644 --- a/tests/test_audit_routes.py +++ b/tests/test_audit_routes.py @@ -1,8 +1,7 @@ """Tests for routes/audit.py — log listing and filtering.""" -import pytest -from models.audit import AuditLog from extensions import db +from models.audit import AuditLog def _add_log(app, entity_type="BiomedicalConcept", entity_id="C001", action="created", actor="alice"): diff --git a/tests/test_bc_routes.py b/tests/test_bc_routes.py index d0eb660..fd4a545 100644 --- a/tests/test_bc_routes.py +++ b/tests/test_bc_routes.py @@ -1,10 +1,10 @@ """Tests for routes/bc.py — CRUD, export, submission.""" -import pytest from unittest.mock import patch -from models.bc import BiomedicalConcept, DataElementConcept -from models.audit import AuditLog + from extensions import db +from models.audit import AuditLog +from models.bc import BiomedicalConcept, DataElementConcept def _bc_form(**kwargs): diff --git a/tests/test_export_service.py b/tests/test_export_service.py index 8a54d2c..27b3611 100644 --- a/tests/test_export_service.py +++ b/tests/test_export_service.py @@ -3,7 +3,6 @@ import json import openpyxl -import pytest from lxml import etree from extensions import db diff --git a/tests/test_governance_routes.py b/tests/test_governance_routes.py index 9aba42a..92870df 100644 --- a/tests/test_governance_routes.py +++ b/tests/test_governance_routes.py @@ -1,10 +1,9 @@ """Tests for routes/governance.py — Kanban advance and reject.""" -import pytest +from extensions import db +from models.audit import AuditLog from models.bc import BiomedicalConcept from models.governance import GovernanceRecord -from models.audit import AuditLog -from extensions import db STATUS_ORDER = ["provisional", "sme_review", "cdisc_approval", "published"] @@ -131,6 +130,7 @@ def test_export_excludes_non_stage3_bcs(self, client, app, sample_bc): r = client.get("/governance/export") assert r.status_code == 200 import io + import openpyxl wb = openpyxl.load_workbook(io.BytesIO(r.data)) @@ -142,6 +142,7 @@ def test_export_includes_stage3_bcs(self, client, app, sample_bc): client.post("/governance/advance/C12345") r = client.get("/governance/export") import io + import openpyxl wb = openpyxl.load_workbook(io.BytesIO(r.data)) @@ -166,6 +167,7 @@ def test_export_system_columns_blank_without_loinc_code(self, client, app): client.post("/governance/advance/C99998") r = client.get("/governance/export") import io + import openpyxl wb = openpyxl.load_workbook(io.BytesIO(r.data)) @@ -195,6 +197,7 @@ def test_export_system_columns_populated_with_loinc_code(self, client, app): client.post("/governance/advance/C99997") r = client.get("/governance/export") import io + import openpyxl wb = openpyxl.load_workbook(io.BytesIO(r.data)) @@ -222,6 +225,7 @@ def test_export_code_column_uses_loinc_code(self, client, app): client.post("/governance/advance/C99999") r = client.get("/governance/export") import io + import openpyxl wb = openpyxl.load_workbook(io.BytesIO(r.data)) diff --git a/tests/test_ingestion_routes.py b/tests/test_ingestion_routes.py index eeae74d..49ba089 100644 --- a/tests/test_ingestion_routes.py +++ b/tests/test_ingestion_routes.py @@ -1,12 +1,12 @@ """Tests for routes/ingestion.py — upload, approve, reject.""" +import csv import io import json -import csv -import pytest + +from extensions import db from models.bc import BiomedicalConcept from models.ingestion import IngestionRecord -from extensions import db def _csv_file(rows): diff --git a/tests/test_ingestion_service.py b/tests/test_ingestion_service.py index e666974..f89e525 100644 --- a/tests/test_ingestion_service.py +++ b/tests/test_ingestion_service.py @@ -2,16 +2,16 @@ import io import json -import pytest + from services.ingestion import ( - _similarity, + _group_by_bc, _match_field, - map_fields, - validate_bc, + _similarity, deduplicate, + map_fields, parse_csv, parse_json, - _group_by_bc, + validate_bc, ) # --------------------------------------------------------------------------- diff --git a/tests/test_loinc.py b/tests/test_loinc.py index 8b62927..f1181a3 100644 --- a/tests/test_loinc.py +++ b/tests/test_loinc.py @@ -3,10 +3,9 @@ import json from unittest.mock import MagicMock, patch -import pytest import requests -from services.loinc_api import LoincApiClient, LOINC_EF_FIELDS +from services.loinc_api import LOINC_EF_FIELDS, LoincApiClient # --------------------------------------------------------------------------- # Sample NLM response using ef parameter diff --git a/tests/test_models.py b/tests/test_models.py index cdd0416..22af3fe 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,10 +1,9 @@ """Tests for model property serialization and helper methods.""" -import pytest +from extensions import db from models.audit import AuditLog +from models.bc import BiomedicalConcept from models.ingestion import IngestionRecord -from models.bc import BiomedicalConcept, DataElementConcept -from extensions import db class TestAuditLogJsonProperties: diff --git a/tests/test_ncit.py b/tests/test_ncit.py index 69b21b0..baa2b4b 100644 --- a/tests/test_ncit.py +++ b/tests/test_ncit.py @@ -3,7 +3,6 @@ import json from unittest.mock import MagicMock, patch -import pytest import requests from services.ncit_api import NCItApiClient @@ -43,9 +42,9 @@ class TestNcitGetConceptExtended: def setup_method(self): - import services.ncit_api + from services import api_cache - services.ncit_api._ncit_cache.clear() + api_cache._cache.clear() def _mock_get(self, data): mock = MagicMock() From 27ba796f6af286924740cda86317d0faf9a0c3ab Mon Sep 17 00:00:00 2001 From: pendingintent <3921919+pendingintent@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:13:41 -0400 Subject: [PATCH 22/36] =?UTF-8?q?=E2=9C=85=20MCP:=20add=20read-only=20MCP?= =?UTF-8?q?=20server=20(milestone=201,=208=20tools)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New mcp_server/ package mirroring the soa-workbench server pattern (raw MCP SDK, _TOOLS list + _dispatch dict, sync handlers via run_in_executor, stdio transport, python -m mcp_server entry). Registered in .mcp.json. - Handlers run inside a Flask app context pushed per call (executor threads) via a decorator; the lazy app singleton reuses create_app() + ensure_db(), so the MCP process resolves the same instance/ SQLite file and service clients as the web app. Tests inject the in-memory test app through mcp_server.server._app. - Tools: list_bcs (q/status/pagination, capped at 200/page), get_bc (BC + DECs + specializations + governance history), search_ncit, get_ncit_concept, search_loinc, search_cdisc_library (title filter), get_library_bc, list_review_queue (board columns + pending ingestion). - mcp>=1.0.0 added to requirements. Verified end-to-end over stdio: initialize -> tools/list (8) -> tools/call list_bcs against a scratch DB (auto-bootstrapped by ensure_db). Suite: 223 passed (16 new MCP tests via direct _dispatch). --- .mcp.json | 9 + CLAUDE.md | 8 + mcp_server/__init__.py | 5 + mcp_server/__main__.py | 3 + mcp_server/server.py | 371 +++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + tests/test_mcp_server.py | 162 +++++++++++++++++ 7 files changed, 559 insertions(+) create mode 100644 .mcp.json create mode 100644 mcp_server/__init__.py create mode 100644 mcp_server/__main__.py create mode 100644 mcp_server/server.py create mode 100644 tests/test_mcp_server.py diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 0000000..f6939ee --- /dev/null +++ b/.mcp.json @@ -0,0 +1,9 @@ +{ + "mcpServers": { + "cdisc-curation": { + "command": "/Users/dmoreland/projects/cdisc-concept-curation/.venv/bin/python", + "args": ["-m", "mcp_server"], + "cwd": "/Users/dmoreland/projects/cdisc-concept-curation" + } + } +} diff --git a/CLAUDE.md b/CLAUDE.md index 29b7a4d..75bf1da 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -92,6 +92,14 @@ pytest tests/test_bc_routes.py -v # single file **Governance stages:** Provisional → SME Review → CDISC Approval → Published (tracked in `GovernanceRecord`) +**MCP server:** `mcp_server/` (run with `python -m mcp_server`; registered in +`.mcp.json`). Eight read-only tools (`list_bcs`, `get_bc`, `search_ncit`, +`get_ncit_concept`, `search_loinc`, `search_cdisc_library`, `get_library_bc`, +`list_review_queue`). Handlers run inside a Flask app context via the shared +app factory, so the MCP process and the web app use the same +`instance/cdisc_curation.db` and the same service clients. Tests call +`mcp_server.server._dispatch` directly (`tests/test_mcp_server.py`). + ## Config All configuration is in `config.py` via environment variables: diff --git a/mcp_server/__init__.py b/mcp_server/__init__.py new file mode 100644 index 0000000..ddd9597 --- /dev/null +++ b/mcp_server/__init__.py @@ -0,0 +1,5 @@ +"""MCP server package for cdisc-concept-curation. + +Run with ``python -m mcp_server`` (registered in .mcp.json). +Named mcp_server (not mcp) to avoid shadowing the MCP SDK package. +""" diff --git a/mcp_server/__main__.py b/mcp_server/__main__.py new file mode 100644 index 0000000..62fdc30 --- /dev/null +++ b/mcp_server/__main__.py @@ -0,0 +1,3 @@ +from mcp_server.server import main + +main() diff --git a/mcp_server/server.py b/mcp_server/server.py new file mode 100644 index 0000000..7cced4f --- /dev/null +++ b/mcp_server/server.py @@ -0,0 +1,371 @@ +"""MCP server exposing cdisc-concept-curation data as tools. + +Run via ``python -m mcp_server``. The server communicates over stdio and +is registered in ``.mcp.json`` for automatic pickup by Claude Code. + +Handlers run inside a Flask app context (the app factory is shared with +the web app, so both processes resolve the same instance/ SQLite file) +and use the same ORM models and service clients as the routes. + +Tools (8, all read-only — milestone 1): + list_bcs Search/paginate curated Biomedical Concepts + get_bc Full BC detail incl. DECs, specializations, governance + search_ncit Search the NCI Thesaurus (EVS) + get_ncit_concept Full NCIt concept detail + search_loinc Search LOINC via NLM Clinical Tables + search_cdisc_library Search published BCs in the CDISC Library + get_library_bc Fetch one published BC from the CDISC Library + list_review_queue Governance board summary + pending ingestion count +""" + +import asyncio +import functools +import json +import logging +from typing import Any + +import mcp.types as types +from mcp.server import Server +from mcp.server.stdio import stdio_server + +logger = logging.getLogger("cdisc_curation.mcp") + +server = Server("cdisc-curation") + +# Lazy app singleton. Tests inject their own app here so handlers run +# against the in-memory test database. +_app = None + + +def _get_app(): + global _app + if _app is None: + from app import create_app + from db_bootstrap import ensure_db + + _app = create_app() + ensure_db(_app) + return _app + + +def _with_app_context(fn): + """Push a fresh app context per call. + + Required inside the decorator (not at server start) because handlers + run on executor worker threads. + """ + + @functools.wraps(fn) + def wrapper(args): + with _get_app().app_context(): + return fn(args) + + return wrapper + + +# --------------------------------------------------------------------------- +# Tool definitions +# --------------------------------------------------------------------------- + +_TOOLS = [ + types.Tool( + name="list_bcs", + description=( + "Search and paginate locally curated Biomedical Concepts. Filters: q (substring of short_name, bc_id, or ncit_code), status (provisional, sme_review, cdisc_approval, published)." + ), + inputSchema={ + "type": "object", + "properties": { + "q": {"type": "string", "description": "Search text matched against short_name, bc_id, ncit_code"}, + "status": {"type": "string", "enum": ["provisional", "sme_review", "cdisc_approval", "published"]}, + "page": {"type": "integer", "minimum": 1, "default": 1}, + "per_page": {"type": "integer", "minimum": 1, "maximum": 200, "default": 25}, + }, + }, + ), + types.Tool( + name="get_bc", + description=("Get full detail for one curated Biomedical Concept: all fields plus its Data Element Concepts, dataset specializations, and governance history."), + inputSchema={ + "type": "object", + "properties": {"bc_id": {"type": "string", "description": "BC primary key (NCIt C-code)"}}, + "required": ["bc_id"], + }, + ), + types.Tool( + name="search_ncit", + description="Search the NCI Thesaurus (EVS API) for concepts matching a term. Returns code, name, and definition per match.", + inputSchema={ + "type": "object", + "properties": { + "term": {"type": "string"}, + "size": {"type": "integer", "minimum": 1, "maximum": 50, "default": 10}, + }, + "required": ["term"], + }, + ), + types.Tool( + name="get_ncit_concept", + description="Fetch full NCIt concept detail (definitions, synonyms, parents, children, semantic type) by C-code.", + inputSchema={ + "type": "object", + "properties": {"ncit_code": {"type": "string", "description": "NCIt C-code, e.g. C64849"}}, + "required": ["ncit_code"], + }, + ), + types.Tool( + name="search_loinc", + description="Search LOINC codes by code or name via the NLM Clinical Tables API.", + inputSchema={ + "type": "object", + "properties": { + "term": {"type": "string"}, + "size": {"type": "integer", "minimum": 1, "maximum": 50, "default": 10}, + }, + "required": ["term"], + }, + ), + types.Tool( + name="search_cdisc_library", + description=( + "Search published Biomedical Concepts in the live CDISC Library by title substring. " + "Requires CDISC_API_KEY (or CDISC_SUBSCRIPTION_KEY). Useful for duplicate detection against local drafts." + ), + inputSchema={ + "type": "object", + "properties": {"q": {"type": "string", "description": "Case-insensitive title substring; empty lists all"}}, + }, + ), + types.Tool( + name="get_library_bc", + description="Fetch one published Biomedical Concept from the live CDISC Library by concept id (NCIt C-code).", + inputSchema={ + "type": "object", + "properties": {"concept_id": {"type": "string"}}, + "required": ["concept_id"], + }, + ), + types.Tool( + name="list_review_queue", + description=("Summarize work awaiting review: BCs in sme_review and cdisc_approval (governance board columns) plus counts of pending ingestion records."), + inputSchema={"type": "object", "properties": {}}, + ), +] + + +@server.list_tools() +async def list_tools() -> list[types.Tool]: + return _TOOLS + + +@server.call_tool() +async def call_tool(name: str, arguments: dict) -> list[types.TextContent]: + loop = asyncio.get_event_loop() + result = await loop.run_in_executor(None, _dispatch, name, arguments or {}) + return [types.TextContent(type="text", text=json.dumps(result, indent=2, default=str))] + + +# --------------------------------------------------------------------------- +# Dispatch +# --------------------------------------------------------------------------- + + +def _dispatch(name: str, args: dict) -> Any: + handlers = { + "list_bcs": _list_bcs, + "get_bc": _get_bc, + "search_ncit": _search_ncit, + "get_ncit_concept": _get_ncit_concept, + "search_loinc": _search_loinc, + "search_cdisc_library": _search_cdisc_library, + "get_library_bc": _get_library_bc, + "list_review_queue": _list_review_queue, + } + fn = handlers.get(name) + if fn is None: + raise ValueError(f"Unknown tool: {name!r}") + return fn(args) + + +# --------------------------------------------------------------------------- +# Tool handlers (read-only) +# --------------------------------------------------------------------------- + + +def _bc_full_dict(bc): + """to_dict() plus the fields it omits, without metadata JSON blobs.""" + data = bc.to_dict() + data.update( + { + "code": bc.code, + "source": bc.source, + "history_of_change": bc.history_of_change, + "created_at": bc.created_at, + "updated_at": bc.updated_at, + } + ) + return data + + +@_with_app_context +def _list_bcs(args: dict) -> dict: + from models.bc import BiomedicalConcept + + q = str(args.get("q") or "").strip() + status = str(args.get("status") or "").strip() + page = max(int(args.get("page") or 1), 1) + per_page = min(max(int(args.get("per_page") or 25), 1), 200) + + query = BiomedicalConcept.query + if q: + query = query.filter(BiomedicalConcept.short_name.ilike(f"%{q}%") | BiomedicalConcept.bc_id.ilike(f"%{q}%") | BiomedicalConcept.ncit_code.ilike(f"%{q}%")) + if status: + query = query.filter_by(status=status) + pagination = query.order_by(BiomedicalConcept.updated_at.desc()).paginate(page=page, per_page=per_page, error_out=False) + return { + "items": [_bc_full_dict(bc) for bc in pagination.items], + "total": pagination.total, + "page": page, + "per_page": per_page, + "pages": pagination.pages, + } + + +@_with_app_context +def _get_bc(args: dict) -> dict: + from extensions import db + from models.bc import BiomedicalConcept, DataElementConcept + from models.governance import GovernanceRecord + + bc_id = str(args.get("bc_id") or "").strip() + if not bc_id: + raise ValueError("bc_id is required") + bc = db.session.get(BiomedicalConcept, bc_id) + if bc is None: + raise ValueError(f"BC {bc_id!r} not found") + + decs = DataElementConcept.query.filter_by(bc_id=bc_id).order_by(DataElementConcept.sort_order).all() + data = _bc_full_dict(bc) + data["decs"] = [ + { + "dec_id": d.dec_id, + "ncit_dec_code": d.ncit_dec_code, + "dec_label": d.dec_label, + "data_type": d.data_type, + "example_set": d.example_set, + "required": d.required, + "sort_order": d.sort_order, + } + for d in decs + ] + data["specializations"] = [ + { + "vlm_group_id": s.vlm_group_id, + "domain": s.domain, + "short_name": s.short_name, + "variables": s.variables, + } + for s in bc.specializations + ] + data["governance_records"] = [ + { + "stage": g.stage, + "action": g.action, + "actor": g.actor, + "comment": g.comment, + "created_at": g.created_at, + } + for g in GovernanceRecord.query.filter_by(bc_id=bc_id).order_by(GovernanceRecord.id).all() + ] + return data + + +@_with_app_context +def _search_ncit(args: dict) -> list: + from services.ncit_api import NCItApiClient + + term = str(args.get("term") or "").strip() + if not term: + raise ValueError("term is required") + size = min(max(int(args.get("size") or 10), 1), 50) + return NCItApiClient().search_concept(term, size=size) + + +@_with_app_context +def _get_ncit_concept(args: dict) -> dict: + from services.ncit_api import NCItApiClient + + ncit_code = str(args.get("ncit_code") or "").strip() + if not ncit_code: + raise ValueError("ncit_code is required") + return NCItApiClient().get_concept(ncit_code) + + +@_with_app_context +def _search_loinc(args: dict) -> list: + from services.loinc_api import LoincApiClient + + term = str(args.get("term") or "").strip() + if not term: + raise ValueError("term is required") + size = min(max(int(args.get("size") or 10), 1), 50) + return LoincApiClient().search(term, size=size) + + +@_with_app_context +def _search_cdisc_library(args: dict) -> list: + from services.cdisc_api import CDISCApiClient + + q = str(args.get("q") or "").strip().lower() + links = CDISCApiClient().get_biomedical_concepts() + if links and "error" in links[0]: + return links + if q: + links = [lnk for lnk in links if q in (lnk.get("title") or "").lower()] + return links + + +@_with_app_context +def _get_library_bc(args: dict) -> dict: + from services.cdisc_api import CDISCApiClient + + concept_id = str(args.get("concept_id") or "").strip() + if not concept_id: + raise ValueError("concept_id is required") + return CDISCApiClient().get_bc(concept_id) + + +@_with_app_context +def _list_review_queue(_args: dict) -> dict: + from models.bc import BiomedicalConcept + from models.ingestion import IngestionRecord + + queue = {} + for status in ("sme_review", "cdisc_approval"): + bcs = BiomedicalConcept.query.filter_by(status=status).order_by(BiomedicalConcept.updated_at.desc()).all() + queue[status] = [{"bc_id": bc.bc_id, "short_name": bc.short_name, "submitter": bc.submitter, "updated_at": bc.updated_at} for bc in bcs] + pending_ingestion = IngestionRecord.query.filter_by(status="pending").count() + return { + "sme_review": queue["sme_review"], + "cdisc_approval": queue["cdisc_approval"], + "pending_ingestion_records": pending_ingestion, + } + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + + +def main() -> None: + logging.basicConfig(level=logging.WARNING) + asyncio.run(_run()) + + +async def _run() -> None: + async with stdio_server() as streams: + await server.run(*streams, server.create_initialization_options()) + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt index 8e3c9fe..65f269e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ Jinja2==3.1.6 lxml==5.2.2 Mako==1.3.10 MarkupSafe==3.0.3 +mcp>=1.0.0 numpy==2.4.3 openpyxl==3.1.2 pandas==2.2.2 diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py new file mode 100644 index 0000000..06544df --- /dev/null +++ b/tests/test_mcp_server.py @@ -0,0 +1,162 @@ +"""Tests for the MCP server (mcp_server/server.py). + +Calls _dispatch directly (no stdio transport) with the shared in-memory +test app injected, mirroring the soa-workbench test pattern. +""" + +from unittest.mock import patch + +import pytest + +import mcp_server.server as mcp_srv +from extensions import db +from models.bc import BiomedicalConcept, DataElementConcept +from models.governance import GovernanceRecord +from models.ingestion import IngestionRecord +from models.specialization import DatasetSpecialization + + +@pytest.fixture(autouse=True) +def inject_test_app(app): + """Point the MCP server at the test app instead of building its own.""" + mcp_srv._app = app + yield + mcp_srv._app = None + + +def _dispatch(name, args=None): + return mcp_srv._dispatch(name, args or {}) + + +class TestDispatch: + def test_unknown_tool_raises(self): + with pytest.raises(ValueError, match="Unknown tool"): + _dispatch("nope") + + def test_all_declared_tools_are_dispatchable(self): + for tool in mcp_srv._TOOLS: + assert tool.name in { + "list_bcs", + "get_bc", + "search_ncit", + "get_ncit_concept", + "search_loinc", + "search_cdisc_library", + "get_library_bc", + "list_review_queue", + } + + +class TestListBcs: + def test_empty_db(self): + result = _dispatch("list_bcs") + assert result == {"items": [], "total": 0, "page": 1, "per_page": 25, "pages": 0} + + def test_lists_and_filters(self, app, sample_bc): + with app.app_context(): + db.session.add(BiomedicalConcept(bc_id="C99999", short_name="Glucose Measurement", status="published", submitter="tester")) + db.session.commit() + + result = _dispatch("list_bcs") + assert result["total"] == 2 + + by_q = _dispatch("list_bcs", {"q": "glucose"}) + assert by_q["total"] == 1 + assert by_q["items"][0]["bc_id"] == "C99999" + + by_status = _dispatch("list_bcs", {"status": "provisional"}) + assert by_status["total"] == 1 + assert by_status["items"][0]["bc_id"] == sample_bc + + def test_pagination(self, app): + with app.app_context(): + for i in range(5): + db.session.add(BiomedicalConcept(bc_id=f"C{i:05d}", short_name=f"Concept {i}", status="provisional")) + db.session.commit() + page2 = _dispatch("list_bcs", {"per_page": 2, "page": 2}) + assert page2["total"] == 5 + assert len(page2["items"]) == 2 + assert page2["pages"] == 3 + + +class TestGetBc: + def test_requires_bc_id(self): + with pytest.raises(ValueError, match="bc_id is required"): + _dispatch("get_bc") + + def test_missing_bc_raises(self): + with pytest.raises(ValueError, match="not found"): + _dispatch("get_bc", {"bc_id": "NOPE"}) + + def test_full_detail(self, app, sample_bc): + with app.app_context(): + db.session.add(DataElementConcept(dec_id="D1", bc_id=sample_bc, dec_label="Result", data_type="decimal", sort_order=0)) + db.session.add(DatasetSpecialization(vlm_group_id=f"{sample_bc}.SDTM", bc_id=sample_bc, domain="SDTM", short_name="Spec")) + db.session.add(GovernanceRecord(bc_id=sample_bc, stage=1, action="advanced", actor="tester", comment="ok")) + db.session.commit() + + result = _dispatch("get_bc", {"bc_id": sample_bc}) + assert result["bc_id"] == sample_bc + assert result["decs"][0]["dec_label"] == "Result" + assert result["specializations"][0]["vlm_group_id"] == f"{sample_bc}.SDTM" + assert result["governance_records"][0]["action"] == "advanced" + + +class TestExternalApiTools: + def test_search_ncit_requires_term(self): + with pytest.raises(ValueError, match="term is required"): + _dispatch("search_ncit") + + def test_search_ncit(self): + with patch("services.ncit_api.NCItApiClient.search_concept") as mock_search: + mock_search.return_value = [{"code": "C64849", "name": "HbA1c"}] + result = _dispatch("search_ncit", {"term": "hba1c", "size": 5}) + assert result[0]["code"] == "C64849" + mock_search.assert_called_once_with("hba1c", size=5) + + def test_get_ncit_concept(self): + with patch("services.ncit_api.NCItApiClient.get_concept") as mock_get: + mock_get.return_value = {"code": "C64849", "name": "HbA1c"} + result = _dispatch("get_ncit_concept", {"ncit_code": "C64849"}) + assert result["code"] == "C64849" + + def test_search_loinc(self): + with patch("services.loinc_api.LoincApiClient.search") as mock_search: + mock_search.return_value = [{"LOINC_NUM": "4548-4"}] + result = _dispatch("search_loinc", {"term": "4548-4"}) + assert result[0]["LOINC_NUM"] == "4548-4" + + def test_search_cdisc_library_filters_by_title(self): + links = [ + {"href": "/mdr/bc/biomedicalconcepts/C64849", "title": "Hemoglobin A1c"}, + {"href": "/mdr/bc/biomedicalconcepts/C25298", "title": "Systolic BP"}, + ] + with patch("services.cdisc_api.CDISCApiClient.get_biomedical_concepts", return_value=links): + result = _dispatch("search_cdisc_library", {"q": "hemoglobin"}) + assert len(result) == 1 + assert result[0]["title"] == "Hemoglobin A1c" + + def test_search_cdisc_library_passes_error_through(self): + with patch("services.cdisc_api.CDISCApiClient.get_biomedical_concepts", return_value=[{"error": "401"}]): + result = _dispatch("search_cdisc_library", {"q": "x"}) + assert result == [{"error": "401"}] + + def test_get_library_bc(self): + with patch("services.cdisc_api.CDISCApiClient.get_bc", return_value={"conceptId": "C64849"}): + result = _dispatch("get_library_bc", {"concept_id": "C64849"}) + assert result["conceptId"] == "C64849" + + +class TestReviewQueue: + def test_queue_summary(self, app, sample_bc): + with app.app_context(): + bc = db.session.get(BiomedicalConcept, sample_bc) + bc.status = "sme_review" + db.session.add(IngestionRecord(session_key="s", mapped={"bc_id": "X"}, status="pending")) + db.session.add(IngestionRecord(session_key="s", mapped={"bc_id": "Y"}, status="approved")) + db.session.commit() + + result = _dispatch("list_review_queue") + assert result["sme_review"][0]["bc_id"] == sample_bc + assert result["cdisc_approval"] == [] + assert result["pending_ingestion_records"] == 1 From c83377b9b8373b581b14e5fdb2cab13d82a14fa8 Mon Sep 17 00:00:00 2001 From: pendingintent <3921919+pendingintent@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:20:28 -0400 Subject: [PATCH 23/36] =?UTF-8?q?=E2=9C=85=20MCP=20milestone=202:=20write?= =?UTF-8?q?=20tools=20via=20extracted=20services;=20WAL=20for=20two=20writ?= =?UTF-8?q?ers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Service extraction (one write path for routes AND MCP): - services/bc_service.py: create_bc, update_bc, apply_bc_fields, save_decs, map_ncit_to_bc, submit_bc_for_review (+ NotFoundError) - services/governance_service.py: advance_governance, reject_bc (owns STATUS_ORDER; routes/governance.py re-imports it) - routes/bc.py, routes/governance.py, routes/ncit.py are now thin form->dict->service adapters; flash/redirect/AJAX behavior unchanged (existing route tests were the refactor safety net — all green) - Intentional behavior fix: /ncit/resolve now writes an AuditLog entry ('ncit_mapped'); it was the only mutation with no audit record MCP write tools (6): create_bc (optional decs), update_bc, map_ncit_to_bc (promotes IMPORT_ ids), submit_bc_for_review, advance_governance, reject_bc — actor defaults to 'mcp' so agent writes are distinguishable in the audit trail. extensions.py: SQLite connections get PRAGMA journal_mode=WAL + busy_timeout=15000 (verified: fresh DB reports 'wal') — required now that Flask and the MCP server both write instance/cdisc_curation.db. Suite: 233 passed (10 new write-path MCP tests assert AuditLog and GovernanceRecord rows); smoke.sh 14/14. --- CLAUDE.md | 16 ++- extensions.py | 20 ++++ mcp_server/server.py | 203 ++++++++++++++++++++++++++++++++- routes/bc.py | 115 +++++-------------- routes/governance.py | 50 ++------ routes/ncit.py | 7 +- services/bc_service.py | 150 ++++++++++++++++++++++++ services/governance_service.py | 57 +++++++++ tests/test_mcp_server.py | 135 ++++++++++++++++++++-- 9 files changed, 604 insertions(+), 149 deletions(-) create mode 100644 services/bc_service.py create mode 100644 services/governance_service.py diff --git a/CLAUDE.md b/CLAUDE.md index 75bf1da..97a239b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -93,12 +93,18 @@ pytest tests/test_bc_routes.py -v # single file **Governance stages:** Provisional → SME Review → CDISC Approval → Published (tracked in `GovernanceRecord`) **MCP server:** `mcp_server/` (run with `python -m mcp_server`; registered in -`.mcp.json`). Eight read-only tools (`list_bcs`, `get_bc`, `search_ncit`, +`.mcp.json`). Eight read tools (`list_bcs`, `get_bc`, `search_ncit`, `get_ncit_concept`, `search_loinc`, `search_cdisc_library`, `get_library_bc`, -`list_review_queue`). Handlers run inside a Flask app context via the shared -app factory, so the MCP process and the web app use the same -`instance/cdisc_curation.db` and the same service clients. Tests call -`mcp_server.server._dispatch` directly (`tests/test_mcp_server.py`). +`list_review_queue`) and six write tools (`create_bc`, `update_bc`, +`map_ncit_to_bc`, `submit_bc_for_review`, `advance_governance`, `reject_bc`). +Handlers run inside a Flask app context via the shared app factory, so the +MCP process and the web app use the same `instance/cdisc_curation.db` and the +same service clients. Writes go through `services/bc_service.py` and +`services/governance_service.py` — the exact code path the routes use — with +`actor` defaulting to `"mcp"` in the audit trail. SQLite runs in WAL mode +with a 15s busy timeout (`extensions.py`) so the two writer processes +coexist. Tests call `mcp_server.server._dispatch` directly +(`tests/test_mcp_server.py`). ## Config diff --git a/extensions.py b/extensions.py index 7869e82..055627a 100644 --- a/extensions.py +++ b/extensions.py @@ -1,5 +1,25 @@ +import sqlite3 + from flask_migrate import Migrate from flask_sqlalchemy import SQLAlchemy +from sqlalchemy import event +from sqlalchemy.engine import Engine db = SQLAlchemy() migrate = Migrate() + + +@event.listens_for(Engine, "connect") +def _set_sqlite_pragmas(dbapi_connection, connection_record): + """Enable WAL + a busy timeout on every SQLite connection. + + Required because two writer processes share instance/cdisc_curation.db + (the Flask app and the MCP server); without WAL the second writer hits + 'database is locked'. WAL is persistent per database file; in-memory + test databases silently keep their 'memory' journal mode. + """ + if isinstance(dbapi_connection, sqlite3.Connection): + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA journal_mode=WAL") + cursor.execute("PRAGMA busy_timeout=15000") + cursor.close() diff --git a/mcp_server/server.py b/mcp_server/server.py index 7cced4f..76d01c4 100644 --- a/mcp_server/server.py +++ b/mcp_server/server.py @@ -7,7 +7,7 @@ the web app, so both processes resolve the same instance/ SQLite file) and use the same ORM models and service clients as the routes. -Tools (8, all read-only — milestone 1): +Read tools (8): list_bcs Search/paginate curated Biomedical Concepts get_bc Full BC detail incl. DECs, specializations, governance search_ncit Search the NCI Thesaurus (EVS) @@ -16,6 +16,15 @@ search_cdisc_library Search published BCs in the CDISC Library get_library_bc Fetch one published BC from the CDISC Library list_review_queue Governance board summary + pending ingestion count + +Write tools (6) — same service code path as the web routes, every write +audited; actor defaults to "mcp" so agent writes are distinguishable: + create_bc Create a provisional BC (optionally with DECs) + update_bc Update BC fields + map_ncit_to_bc Attach an NCIt code (promotes IMPORT_ ids) + submit_bc_for_review provisional -> sme_review + advance_governance Advance one governance stage + reject_bc Reject back to provisional """ import asyncio @@ -150,6 +159,116 @@ def wrapper(args): description=("Summarize work awaiting review: BCs in sme_review and cdisc_approval (governance board columns) plus counts of pending ingestion records."), inputSchema={"type": "object", "properties": {}}, ), + types.Tool( + name="create_bc", + description=("Create a new provisional Biomedical Concept. bc_id should be the NCIt C-code. Optionally include decs, a list of Data Element Concept objects. The write is audit-logged."), + inputSchema={ + "type": "object", + "properties": { + "bc_id": {"type": "string", "description": "Primary key (NCIt C-code)"}, + "short_name": {"type": "string"}, + "definition": {"type": "string"}, + "ncit_code": {"type": "string"}, + "parent_bc_id": {"type": "string"}, + "bc_categories": {"type": "string", "description": "Semicolon-separated"}, + "synonyms": {"type": "string"}, + "result_scales": {"type": "string"}, + "loinc_code": {"type": "string"}, + "package_date": {"type": "string"}, + "submitter": {"type": "string"}, + "actor": {"type": "string", "default": "mcp"}, + "decs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "dec_id": {"type": "string"}, + "ncit_dec_code": {"type": "string"}, + "dec_label": {"type": "string"}, + "data_type": {"type": "string"}, + "example_set": {"type": "string"}, + }, + "required": ["dec_label"], + }, + }, + }, + "required": ["bc_id", "short_name"], + }, + ), + types.Tool( + name="update_bc", + description=( + "Update fields on an existing BC. Only supplied fields change; clearing ncit_code/loinc_code/parent_bc_id requires passing an empty string. Audit-logged with before/after state." + ), + inputSchema={ + "type": "object", + "properties": { + "bc_id": {"type": "string"}, + "short_name": {"type": "string"}, + "definition": {"type": "string"}, + "ncit_code": {"type": "string"}, + "parent_bc_id": {"type": "string"}, + "bc_categories": {"type": "string"}, + "synonyms": {"type": "string"}, + "result_scales": {"type": "string"}, + "loinc_code": {"type": "string"}, + "package_date": {"type": "string"}, + "actor": {"type": "string", "default": "mcp"}, + }, + "required": ["bc_id"], + }, + ), + types.Tool( + name="map_ncit_to_bc", + description=("Attach an NCIt C-code to a BC. Temporary IMPORT_ ids are promoted to the resolved code (the primary key changes). Audit-logged."), + inputSchema={ + "type": "object", + "properties": { + "bc_id": {"type": "string"}, + "ncit_code": {"type": "string"}, + "actor": {"type": "string", "default": "mcp"}, + }, + "required": ["bc_id", "ncit_code"], + }, + ), + types.Tool( + name="submit_bc_for_review", + description="Move a BC from provisional to sme_review. Audit-logged.", + inputSchema={ + "type": "object", + "properties": { + "bc_id": {"type": "string"}, + "actor": {"type": "string", "default": "mcp"}, + }, + "required": ["bc_id"], + }, + ), + types.Tool( + name="advance_governance", + description=("Advance a BC one stage (provisional -> sme_review -> cdisc_approval -> published). Writes a GovernanceRecord and an audit entry; returns advanced=false if already published."), + inputSchema={ + "type": "object", + "properties": { + "bc_id": {"type": "string"}, + "comment": {"type": "string"}, + "actor": {"type": "string", "default": "mcp"}, + }, + "required": ["bc_id"], + }, + ), + types.Tool( + name="reject_bc", + description="Reject a BC back to provisional (stage 0). Writes a GovernanceRecord and an audit entry.", + inputSchema={ + "type": "object", + "properties": { + "bc_id": {"type": "string"}, + "comment": {"type": "string"}, + "actor": {"type": "string", "default": "mcp"}, + }, + "required": ["bc_id"], + }, + ), ] @@ -180,6 +299,12 @@ def _dispatch(name: str, args: dict) -> Any: "search_cdisc_library": _search_cdisc_library, "get_library_bc": _get_library_bc, "list_review_queue": _list_review_queue, + "create_bc": _create_bc, + "update_bc": _update_bc, + "map_ncit_to_bc": _map_ncit_to_bc, + "submit_bc_for_review": _submit_bc_for_review, + "advance_governance": _advance_governance, + "reject_bc": _reject_bc, } fn = handlers.get(name) if fn is None: @@ -352,6 +477,82 @@ def _list_review_queue(_args: dict) -> dict: } +# --------------------------------------------------------------------------- +# Tool handlers (writes — shared service code path, audit-logged) +# --------------------------------------------------------------------------- + + +@_with_app_context +def _create_bc(args: dict) -> dict: + from services import bc_service + + actor = str(args.get("actor") or "mcp") + bc = bc_service.create_bc(args, actor=actor) + decs = args.get("decs") or [] + if decs: + bc_service.save_decs(bc.bc_id, decs) + return _get_bc.__wrapped__({"bc_id": bc.bc_id}) + + +@_with_app_context +def _update_bc(args: dict) -> dict: + from services import bc_service + + bc_id = str(args.get("bc_id") or "").strip() + if not bc_id: + raise ValueError("bc_id is required") + actor = str(args.get("actor") or "mcp") + # Only fields present in args change; absent fields keep their value + bc = bc_service.update_bc(bc_id, args, actor=actor) + return _bc_full_dict(bc) + + +@_with_app_context +def _map_ncit_to_bc(args: dict) -> dict: + from services import bc_service + + bc_id = str(args.get("bc_id") or "").strip() + if not bc_id: + raise ValueError("bc_id is required") + actor = str(args.get("actor") or "mcp") + bc = bc_service.map_ncit_to_bc(bc_id, args.get("ncit_code"), actor=actor) + return _bc_full_dict(bc) + + +@_with_app_context +def _submit_bc_for_review(args: dict) -> dict: + from services import bc_service + + bc_id = str(args.get("bc_id") or "").strip() + if not bc_id: + raise ValueError("bc_id is required") + actor = str(args.get("actor") or "mcp") + bc = bc_service.submit_bc_for_review(bc_id, actor=actor) + return _bc_full_dict(bc) + + +@_with_app_context +def _advance_governance(args: dict) -> dict: + from services import governance_service + + bc_id = str(args.get("bc_id") or "").strip() + if not bc_id: + raise ValueError("bc_id is required") + actor = str(args.get("actor") or "mcp") + return governance_service.advance_governance(bc_id, actor=actor, comment=str(args.get("comment") or "")) + + +@_with_app_context +def _reject_bc(args: dict) -> dict: + from services import governance_service + + bc_id = str(args.get("bc_id") or "").strip() + if not bc_id: + raise ValueError("bc_id is required") + actor = str(args.get("actor") or "mcp") + return governance_service.reject_bc(bc_id, actor=actor, comment=str(args.get("comment") or "")) + + # --------------------------------------------------------------------------- # Entry point # --------------------------------------------------------------------------- diff --git a/routes/bc.py b/routes/bc.py index 4771669..cdcc652 100644 --- a/routes/bc.py +++ b/routes/bc.py @@ -7,6 +7,7 @@ from extensions import db from models.bc import BiomedicalConcept, DataElementConcept from models.governance import GovernanceRecord +from services import bc_service from services.audit import log_change from services.cdisc_api import CDISCApiClient from services.export import export_json, export_odm_xml, export_xlsx @@ -17,40 +18,6 @@ bp = Blueprint("bc", __name__) -# Plain-text fields copied verbatim from the form onto the model. -_BC_TEXT_FIELDS = ("short_name", "definition", "bc_categories", "synonyms", "result_scales", "package_date") - - -def _apply_bc_form(bc, form, is_new): - """Copy BC fields from a submitted form onto the model. - - Create keeps raw form values (empty strings allowed); edit - normalizes ncit/parent/loinc to None when cleared and preserves the - existing value for any field omitted from the form. - """ - for field in _BC_TEXT_FIELDS: - setattr(bc, field, form.get(field, "" if is_new else getattr(bc, field))) - if is_new: - bc.ncit_code = form.get("ncit_code", "") - bc.parent_bc_id = form.get("parent_bc_id") or None - bc.loinc_code = form.get("loinc_code", "") - has_loinc = bool(form.get("loinc_code", "").strip()) - bc.system = form.get("system", "") if has_loinc else "" - bc.system_name = form.get("system_name", "") if has_loinc else "" - bc.loinc_metadata = form.get("loinc_metadata", "") or None - bc.ncit_metadata = form.get("ncit_metadata", "") or None - else: - new_ncit_code = (form.get("ncit_code", "") or "").strip() or None - bc.ncit_code = new_ncit_code - bc.ncit_metadata = (form.get("ncit_metadata", "") or bc.ncit_metadata) if new_ncit_code else None - bc.parent_bc_id = (form.get("parent_bc_id", "") or "").strip() or None - new_loinc_code = (form.get("loinc_code", "") or "").strip() or None - bc.loinc_code = new_loinc_code - bc.system = form.get("system", bc.system) if new_loinc_code else "" - bc.system_name = form.get("system_name", bc.system_name) if new_loinc_code else "" - bc.loinc_metadata = (form.get("loinc_metadata", "") or bc.loinc_metadata) if new_loinc_code else None - bc.updated_at = datetime.now(timezone.utc) - @bp.route("/") def index(): @@ -217,35 +184,21 @@ def _fetch_ncit(): @bp.route("/", methods=["POST"]) def create(): - bc_id = request.form.get("bc_id", "").strip() - if not bc_id: - flash("BC ID is required", "danger") + try: + bc = bc_service.create_bc(request.form) + except ValueError as e: + flash(str(e), "danger") return redirect(url_for("bc.new_bc")) - if db.session.get(BiomedicalConcept, bc_id): - flash(f"BC {bc_id} already exists", "danger") - return redirect(url_for("bc.new_bc")) - bc = BiomedicalConcept( - bc_id=bc_id, - status="provisional", - submitter=request.form.get("submitter", "unknown"), - ) - _apply_bc_form(bc, request.form, is_new=True) - db.session.add(bc) - log_change("BiomedicalConcept", bc_id, "created", actor=bc.submitter, after=bc.to_dict()) - db.session.commit() - _save_decs(bc_id, request.form) - flash(f"BC {bc_id} created", "success") - return redirect(url_for("bc.detail", bc_id=bc_id)) + bc_service.save_decs(bc.bc_id, _decs_from_form(request.form)) + flash(f"BC {bc.bc_id} created", "success") + return redirect(url_for("bc.detail", bc_id=bc.bc_id)) @bp.route("//edit", methods=["POST"]) def edit(bc_id): - bc = db.get_or_404(BiomedicalConcept, bc_id) - before = bc.to_dict() - _apply_bc_form(bc, request.form, is_new=False) - log_change("BiomedicalConcept", bc_id, "updated", actor="user", before=before, after=bc.to_dict()) - db.session.commit() - _save_decs(bc_id, request.form) + db.get_or_404(BiomedicalConcept, bc_id) + bc_service.update_bc(bc_id, request.form, actor="user") + bc_service.save_decs(bc_id, _decs_from_form(request.form)) flash(f"BC {bc_id} updated", "success") return redirect(url_for("bc.detail", bc_id=bc_id)) @@ -281,12 +234,8 @@ def clear_loinc(bc_id): @bp.route("//submit", methods=["POST"]) def submit_for_review(bc_id): - bc = db.get_or_404(BiomedicalConcept, bc_id) - before = bc.to_dict() - bc.status = "sme_review" - bc.updated_at = datetime.now(timezone.utc) - log_change("BiomedicalConcept", bc_id, "submitted_for_review", actor="user", before=before, after=bc.to_dict()) - db.session.commit() + db.get_or_404(BiomedicalConcept, bc_id) + bc_service.submit_bc_for_review(bc_id, actor="user") flash(f"BC {bc_id} submitted for SME review", "success") return redirect(url_for("bc.detail", bc_id=bc_id)) @@ -306,32 +255,22 @@ def delete(bc_id): return redirect(url_for("bc.index")) -def _save_decs(bc_id, form): - """Persist Data Element Concepts from a submitted form. - - Expects parallel lists posted as dec_label[], dec_data_type[], - dec_example_set[], dec_id[], dec_ncit_code[]. Any existing DECs for the - BC are replaced on every call so that deletions are honoured. - """ +def _decs_from_form(form): + """Convert the parallel dec_*[] form lists into a list of dicts for + bc_service.save_decs, preserving row positions (blank labels keep + their slot so default dec_id numbering matches the form rows).""" labels = form.getlist("dec_label[]") dtypes = form.getlist("dec_data_type[]") examples = form.getlist("dec_example_set[]") dec_ids = form.getlist("dec_id[]") ncit_codes = form.getlist("dec_ncit_code[]") - if not labels: - return - DataElementConcept.query.filter_by(bc_id=bc_id).delete() - for i, label in enumerate(labels): - if not label.strip(): - continue - dec = DataElementConcept( - dec_id=dec_ids[i] if i < len(dec_ids) and dec_ids[i] else f"{bc_id}.DEC.{i + 1}", - bc_id=bc_id, - ncit_dec_code=ncit_codes[i] if i < len(ncit_codes) else "", - dec_label=label.strip(), - data_type=dtypes[i] if i < len(dtypes) else "string", - example_set=examples[i] if i < len(examples) else "", - sort_order=i, - ) - db.session.add(dec) - db.session.commit() + return [ + { + "dec_id": dec_ids[i] if i < len(dec_ids) else "", + "ncit_dec_code": ncit_codes[i] if i < len(ncit_codes) else "", + "dec_label": label, + "data_type": dtypes[i] if i < len(dtypes) else "string", + "example_set": examples[i] if i < len(examples) else "", + } + for i, label in enumerate(labels) + ] diff --git a/routes/governance.py b/routes/governance.py index 2501841..e8de114 100644 --- a/routes/governance.py +++ b/routes/governance.py @@ -1,17 +1,14 @@ -from datetime import datetime, timezone - from flask import Blueprint, Response, flash, jsonify, redirect, render_template, request, url_for from extensions import db from models.bc import BiomedicalConcept from models.governance import GovernanceRecord -from services.audit import log_change +from services import governance_service from services.export import export_governance_xlsx +from services.governance_service import STATUS_ORDER bp = Blueprint("governance", __name__) -STATUS_ORDER = ["provisional", "sme_review", "cdisc_approval", "published"] - @bp.route("/board") def board(): @@ -45,47 +42,22 @@ def export(): @bp.route("/advance/", methods=["POST"]) def advance(bc_id): - bc = db.get_or_404(BiomedicalConcept, bc_id) - before_status = bc.status - current_idx = STATUS_ORDER.index(bc.status) if bc.status in STATUS_ORDER else 0 - if current_idx < len(STATUS_ORDER) - 1: - bc.status = STATUS_ORDER[current_idx + 1] - bc.updated_at = datetime.now(timezone.utc) - rec = GovernanceRecord( - bc_id=bc_id, - stage=current_idx + 1, - action="advanced", - actor="user", - comment=request.form.get("comment", ""), - ) - db.session.add(rec) - log_change("BiomedicalConcept", bc_id, "status_changed", actor="user", before={"status": before_status}, after={"status": bc.status}) - db.session.commit() + db.get_or_404(BiomedicalConcept, bc_id) + result = governance_service.advance_governance(bc_id, actor="user", comment=request.form.get("comment", "")) + if result["advanced"]: if request.headers.get("X-Requested-With") == "XMLHttpRequest": - return jsonify({"status": bc.status, "bc_id": bc_id}) - flash(f"{bc.short_name} advanced to {bc.status}", "success") + return jsonify({"status": result["status"], "bc_id": bc_id}) + flash(f'{result["short_name"]} advanced to {result["status"]}', "success") else: - flash(f"{bc.short_name} is already published", "info") + flash(f'{result["short_name"]} is already published', "info") return redirect(url_for("governance.board")) @bp.route("/reject/", methods=["POST"]) def reject_bc(bc_id): - bc = db.get_or_404(BiomedicalConcept, bc_id) - before_status = bc.status - bc.status = "provisional" - bc.updated_at = datetime.now(timezone.utc) - rec = GovernanceRecord( - bc_id=bc_id, - stage=0, - action="rejected", - actor="user", - comment=request.form.get("comment", ""), - ) - db.session.add(rec) - log_change("BiomedicalConcept", bc_id, "rejected", actor="user", before={"status": before_status}, after={"status": "provisional"}) - db.session.commit() + db.get_or_404(BiomedicalConcept, bc_id) + result = governance_service.reject_bc(bc_id, actor="user", comment=request.form.get("comment", "")) if request.headers.get("X-Requested-With") == "XMLHttpRequest": return jsonify({"status": "provisional", "bc_id": bc_id}) - flash(f"{bc.short_name} rejected and returned to provisional", "warning") + flash(f'{result["short_name"]} rejected and returned to provisional', "warning") return redirect(url_for("governance.board")) diff --git a/routes/ncit.py b/routes/ncit.py index 88c28ff..246db6b 100644 --- a/routes/ncit.py +++ b/routes/ncit.py @@ -2,6 +2,7 @@ from extensions import db from models.bc import BiomedicalConcept +from services import bc_service from services.ncit_api import NCItApiClient bp = Blueprint("ncit", __name__) @@ -73,10 +74,6 @@ def resolve(bc_id): bc = db.get_or_404(BiomedicalConcept, bc_id) ncit_code = request.form.get("ncit_code", "").strip() if ncit_code: - bc.ncit_code = ncit_code - # Promote temporary IMPORT_ IDs to their resolved NCIt code - if not bc.bc_id or bc.bc_id.startswith("IMPORT_"): - bc.bc_id = ncit_code - db.session.commit() + bc = bc_service.map_ncit_to_bc(bc_id, ncit_code, actor="user") flash(f"NCIt mapping updated for {bc.short_name}", "success") return redirect(url_for("ncit.mapping")) diff --git a/services/bc_service.py b/services/bc_service.py new file mode 100644 index 0000000..2b154e7 --- /dev/null +++ b/services/bc_service.py @@ -0,0 +1,150 @@ +"""Biomedical Concept write operations shared by routes and the MCP server. + +Routes are thin adapters (form -> dict -> service -> flash/redirect); +MCP tools call the same functions, so every write goes through one code +path and the AuditLog contract holds everywhere. +""" + +from datetime import datetime, timezone + +from extensions import db +from models.bc import BiomedicalConcept, DataElementConcept +from services.audit import log_change + + +class NotFoundError(ValueError): + """Raised when a BC id does not exist. Routes translate this to 404.""" + + +# Plain-text fields copied verbatim from a form/dict onto the model. +_BC_TEXT_FIELDS = ("short_name", "definition", "bc_categories", "synonyms", "result_scales", "package_date") + + +def _get_bc_or_raise(bc_id): + bc = db.session.get(BiomedicalConcept, bc_id) + if bc is None: + raise NotFoundError(f"BC {bc_id!r} not found") + return bc + + +def apply_bc_fields(bc, data, is_new): + """Copy BC fields from a submitted form (or plain dict) onto the model. + + Create keeps raw values (empty strings allowed); update normalizes + cleared ncit/parent/loinc to None and preserves the existing value + for any field omitted from the input. + """ + for field in _BC_TEXT_FIELDS: + setattr(bc, field, data.get(field, "" if is_new else getattr(bc, field))) + if is_new: + bc.ncit_code = data.get("ncit_code", "") + bc.parent_bc_id = data.get("parent_bc_id") or None + bc.loinc_code = data.get("loinc_code", "") + has_loinc = bool((data.get("loinc_code") or "").strip()) + bc.system = data.get("system", "") if has_loinc else "" + bc.system_name = data.get("system_name", "") if has_loinc else "" + bc.loinc_metadata = data.get("loinc_metadata", "") or None + bc.ncit_metadata = data.get("ncit_metadata", "") or None + else: + new_ncit_code = (data.get("ncit_code", "") or "").strip() or None + bc.ncit_code = new_ncit_code + bc.ncit_metadata = (data.get("ncit_metadata", "") or bc.ncit_metadata) if new_ncit_code else None + bc.parent_bc_id = (data.get("parent_bc_id", "") or "").strip() or None + new_loinc_code = (data.get("loinc_code", "") or "").strip() or None + bc.loinc_code = new_loinc_code + bc.system = data.get("system", bc.system) if new_loinc_code else "" + bc.system_name = data.get("system_name", bc.system_name) if new_loinc_code else "" + bc.loinc_metadata = (data.get("loinc_metadata", "") or bc.loinc_metadata) if new_loinc_code else None + bc.updated_at = datetime.now(timezone.utc) + + +def create_bc(data, actor=None): + """Create a provisional BC from a dict of fields. Returns the BC. + + Raises ValueError when bc_id is missing or already exists. + """ + bc_id = (data.get("bc_id") or "").strip() + if not bc_id: + raise ValueError("BC ID is required") + if db.session.get(BiomedicalConcept, bc_id): + raise ValueError(f"BC {bc_id} already exists") + bc = BiomedicalConcept( + bc_id=bc_id, + status="provisional", + submitter=data.get("submitter", "unknown"), + ) + apply_bc_fields(bc, data, is_new=True) + db.session.add(bc) + log_change("BiomedicalConcept", bc_id, "created", actor=actor or bc.submitter, after=bc.to_dict()) + db.session.commit() + return bc + + +def update_bc(bc_id, data, actor="user"): + """Update an existing BC from a dict of fields. Returns the BC.""" + bc = _get_bc_or_raise(bc_id) + before = bc.to_dict() + apply_bc_fields(bc, data, is_new=False) + log_change("BiomedicalConcept", bc_id, "updated", actor=actor, before=before, after=bc.to_dict()) + db.session.commit() + return bc + + +def save_decs(bc_id, decs): + """Replace the BC's Data Element Concepts with the given list of dicts. + + Each dict may carry dec_id, ncit_dec_code, dec_label, data_type, + example_set. An empty list is a no-op (deletions are expressed by + posting the surviving rows); rows with a blank label are skipped but + keep their position for default dec_id numbering. + """ + if not decs: + return + DataElementConcept.query.filter_by(bc_id=bc_id).delete() + for i, dec in enumerate(decs): + label = (dec.get("dec_label") or "").strip() + if not label: + continue + db.session.add( + DataElementConcept( + dec_id=dec.get("dec_id") or f"{bc_id}.DEC.{i + 1}", + bc_id=bc_id, + ncit_dec_code=dec.get("ncit_dec_code", ""), + dec_label=label, + data_type=dec.get("data_type") or "string", + example_set=dec.get("example_set", ""), + sort_order=i, + ) + ) + db.session.commit() + + +def map_ncit_to_bc(bc_id, ncit_code, actor="user"): + """Attach an NCIt code to a BC, promoting temporary IMPORT_ ids. + + Behavior fix vs the original /ncit/resolve route: this write is now + recorded in the AuditLog like every other mutation. + """ + ncit_code = (ncit_code or "").strip() + if not ncit_code: + raise ValueError("ncit_code is required") + bc = _get_bc_or_raise(bc_id) + before = bc.to_dict() + bc.ncit_code = ncit_code + # Promote temporary IMPORT_ IDs to their resolved NCIt code + if not bc.bc_id or bc.bc_id.startswith("IMPORT_"): + bc.bc_id = ncit_code + log_change("BiomedicalConcept", bc.bc_id, "ncit_mapped", actor=actor, before=before, after=bc.to_dict()) + db.session.commit() + return bc + + +def submit_bc_for_review(bc_id, actor="user"): + """Move a BC from provisional to sme_review.""" + bc = _get_bc_or_raise(bc_id) + before = bc.to_dict() + bc.status = "sme_review" + bc.updated_at = datetime.now(timezone.utc) + log_change("BiomedicalConcept", bc_id, "submitted_for_review", actor=actor, before=before, after=bc.to_dict()) + db.session.commit() + return bc diff --git a/services/governance_service.py b/services/governance_service.py new file mode 100644 index 0000000..1040396 --- /dev/null +++ b/services/governance_service.py @@ -0,0 +1,57 @@ +"""Governance stage transitions shared by routes and the MCP server.""" + +from datetime import datetime, timezone + +from extensions import db +from models.governance import GovernanceRecord +from services.audit import log_change +from services.bc_service import _get_bc_or_raise + +STATUS_ORDER = ["provisional", "sme_review", "cdisc_approval", "published"] + + +def advance_governance(bc_id, actor="user", comment=""): + """Advance a BC one stage along STATUS_ORDER. + + Returns {"bc_id", "short_name", "status", "advanced"}; advanced is + False when the BC is already published (no records written). + """ + bc = _get_bc_or_raise(bc_id) + before_status = bc.status + current_idx = STATUS_ORDER.index(bc.status) if bc.status in STATUS_ORDER else 0 + if current_idx >= len(STATUS_ORDER) - 1: + return {"bc_id": bc_id, "short_name": bc.short_name, "status": bc.status, "advanced": False} + bc.status = STATUS_ORDER[current_idx + 1] + bc.updated_at = datetime.now(timezone.utc) + db.session.add( + GovernanceRecord( + bc_id=bc_id, + stage=current_idx + 1, + action="advanced", + actor=actor, + comment=comment or "", + ) + ) + log_change("BiomedicalConcept", bc_id, "status_changed", actor=actor, before={"status": before_status}, after={"status": bc.status}) + db.session.commit() + return {"bc_id": bc_id, "short_name": bc.short_name, "status": bc.status, "advanced": True} + + +def reject_bc(bc_id, actor="user", comment=""): + """Reject a BC back to provisional (stage 0).""" + bc = _get_bc_or_raise(bc_id) + before_status = bc.status + bc.status = "provisional" + bc.updated_at = datetime.now(timezone.utc) + db.session.add( + GovernanceRecord( + bc_id=bc_id, + stage=0, + action="rejected", + actor=actor, + comment=comment or "", + ) + ) + log_change("BiomedicalConcept", bc_id, "rejected", actor=actor, before={"status": before_status}, after={"status": "provisional"}) + db.session.commit() + return {"bc_id": bc_id, "short_name": bc.short_name, "status": "provisional", "advanced": False} diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 06544df..1639cb9 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -10,6 +10,7 @@ import mcp_server.server as mcp_srv from extensions import db +from models.audit import AuditLog from models.bc import BiomedicalConcept, DataElementConcept from models.governance import GovernanceRecord from models.ingestion import IngestionRecord @@ -34,17 +35,23 @@ def test_unknown_tool_raises(self): _dispatch("nope") def test_all_declared_tools_are_dispatchable(self): - for tool in mcp_srv._TOOLS: - assert tool.name in { - "list_bcs", - "get_bc", - "search_ncit", - "get_ncit_concept", - "search_loinc", - "search_cdisc_library", - "get_library_bc", - "list_review_queue", - } + expected = { + "list_bcs", + "get_bc", + "search_ncit", + "get_ncit_concept", + "search_loinc", + "search_cdisc_library", + "get_library_bc", + "list_review_queue", + "create_bc", + "update_bc", + "map_ncit_to_bc", + "submit_bc_for_review", + "advance_governance", + "reject_bc", + } + assert {tool.name for tool in mcp_srv._TOOLS} == expected class TestListBcs: @@ -147,6 +154,112 @@ def test_get_library_bc(self): assert result["conceptId"] == "C64849" +def _audit_rows(app, action): + with app.app_context(): + return AuditLog.query.filter_by(action=action).all() + + +class TestCreateBc: + def test_creates_with_decs_and_audit(self, app): + result = _dispatch( + "create_bc", + { + "bc_id": "C64849", + "short_name": "Hemoglobin A1c Measurement", + "definition": "HbA1c quantitative measurement", + "ncit_code": "C64849", + "decs": [{"dec_label": "Result Value", "data_type": "decimal"}], + }, + ) + assert result["bc_id"] == "C64849" + assert result["status"] == "provisional" + assert result["decs"][0]["dec_label"] == "Result Value" + rows = _audit_rows(app, "created") + assert len(rows) == 1 + assert rows[0].actor == "mcp" + assert rows[0].after_state["bc_id"] == "C64849" + + def test_duplicate_raises(self, sample_bc): + with pytest.raises(ValueError, match="already exists"): + _dispatch("create_bc", {"bc_id": sample_bc, "short_name": "Dup"}) + + def test_missing_bc_id_raises(self): + with pytest.raises(ValueError, match="BC ID is required"): + _dispatch("create_bc", {"bc_id": "", "short_name": "X"}) + + +class TestUpdateBc: + def test_updates_and_audits_before_after(self, app, sample_bc): + result = _dispatch("update_bc", {"bc_id": sample_bc, "short_name": "Renamed Concept", "ncit_code": "C12345"}) + assert result["short_name"] == "Renamed Concept" + rows = _audit_rows(app, "updated") + assert len(rows) == 1 + assert rows[0].actor == "mcp" + assert rows[0].before_state["short_name"] == "Test Concept" + assert rows[0].after_state["short_name"] == "Renamed Concept" + + def test_missing_bc_raises(self): + with pytest.raises(ValueError, match="not found"): + _dispatch("update_bc", {"bc_id": "NOPE", "short_name": "X"}) + + +class TestMapNcit: + def test_maps_and_audits(self, app, sample_bc): + result = _dispatch("map_ncit_to_bc", {"bc_id": sample_bc, "ncit_code": "C77777"}) + assert result["ncit_code"] == "C77777" + rows = _audit_rows(app, "ncit_mapped") + assert len(rows) == 1 + assert rows[0].actor == "mcp" + + def test_promotes_import_id(self, app): + with app.app_context(): + db.session.add(BiomedicalConcept(bc_id="IMPORT_1", short_name="Imported", status="provisional")) + db.session.commit() + result = _dispatch("map_ncit_to_bc", {"bc_id": "IMPORT_1", "ncit_code": "C55555"}) + assert result["bc_id"] == "C55555" + with app.app_context(): + assert db.session.get(BiomedicalConcept, "IMPORT_1") is None + assert db.session.get(BiomedicalConcept, "C55555") is not None + + def test_empty_code_raises(self, sample_bc): + with pytest.raises(ValueError, match="ncit_code is required"): + _dispatch("map_ncit_to_bc", {"bc_id": sample_bc, "ncit_code": " "}) + + +class TestGovernanceWrites: + def test_submit_then_advance_to_published(self, app, sample_bc): + submitted = _dispatch("submit_bc_for_review", {"bc_id": sample_bc}) + assert submitted["status"] == "sme_review" + + first = _dispatch("advance_governance", {"bc_id": sample_bc, "comment": "looks good"}) + assert first == {"bc_id": sample_bc, "short_name": "Test Concept", "status": "cdisc_approval", "advanced": True} + + second = _dispatch("advance_governance", {"bc_id": sample_bc}) + assert second["status"] == "published" + + third = _dispatch("advance_governance", {"bc_id": sample_bc}) + assert third["advanced"] is False + assert third["status"] == "published" + + with app.app_context(): + recs = GovernanceRecord.query.filter_by(bc_id=sample_bc).order_by(GovernanceRecord.id).all() + assert [r.action for r in recs] == ["advanced", "advanced"] + assert recs[0].actor == "mcp" + assert recs[0].comment == "looks good" + assert len(_audit_rows(app, "status_changed")) == 2 + assert len(_audit_rows(app, "submitted_for_review")) == 1 + + def test_reject_returns_to_provisional(self, app, sample_bc): + _dispatch("submit_bc_for_review", {"bc_id": sample_bc}) + result = _dispatch("reject_bc", {"bc_id": sample_bc, "comment": "needs work"}) + assert result["status"] == "provisional" + with app.app_context(): + rec = GovernanceRecord.query.filter_by(bc_id=sample_bc, action="rejected").one() + assert rec.stage == 0 + assert rec.actor == "mcp" + assert len(_audit_rows(app, "rejected")) == 1 + + class TestReviewQueue: def test_queue_summary(self, app, sample_bc): with app.app_context(): From 57a2931c9feebfecce9a22557818a13996e0f2dd Mon Sep 17 00:00:00 2001 From: pendingintent <3921919+pendingintent@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:51:07 -0400 Subject: [PATCH 24/36] Updated with claude-recommended code improvements. --- README-PROGRESS.md | 12 +++++++ models/specialization.py | 9 +++++ routes/ingestion.py | 9 +++++ routes/specializations.py | 13 +++++++ templates/specializations.html | 4 +-- tests/test_ingestion_routes.py | 17 ++++++++++ tests/test_specializations_routes.py | 51 ++++++++++++++++++++++++++++ 7 files changed, 112 insertions(+), 3 deletions(-) diff --git a/README-PROGRESS.md b/README-PROGRESS.md index 0d85e4d..ed53b4b 100644 --- a/README-PROGRESS.md +++ b/README-PROGRESS.md @@ -28,6 +28,18 @@ CDISC Biomedical Concept Curation — a Flask/Jinja web application for curating ## Daily Changelog +### 2026-04-15 + +#### Audit Trail Coverage Completion + Specialization Delete Route + +- Fixed `templates/specializations.html:221` — Delete form now POSTs to proper `specializations.delete` route keyed on `vlm_group_id` (removed broken `action=delete`/`spec.id` inputs) +- Added `models/specialization.py` — `to_dict()` method for clean audit log serialization +- Added `POST //delete` route to `routes/specializations.py` — fully audited delete operation +- Updated `routes/specializations.py` — `log_change()` calls on create/generate/delete operations (actor="user") +- Updated `routes/ingestion.py` — `approve_all()` now writes one `AuditLog` per created BiomedicalConcept (action="created_via_ingestion", actor="system") +- Added comprehensive test coverage: 5 new tests in `tests/test_specializations_routes.py` and `tests/test_ingestion_routes.py` (audit logging for create/delete/generate, 404 handling, etc.) +- All 239 tests passing, isort/black/flake8 clean ✅ + ### 2026-04-14 #### LOINC API Explorer + BC Detail Performance + Specializations + Config diff --git a/models/specialization.py b/models/specialization.py index 1085184..50e35c8 100644 --- a/models/specialization.py +++ b/models/specialization.py @@ -19,3 +19,12 @@ def variables(self): @variables.setter def variables(self, value): self._variables = json.dumps(value) + + def to_dict(self): + return { + "vlm_group_id": self.vlm_group_id, + "bc_id": self.bc_id, + "domain": self.domain, + "short_name": self.short_name, + "variables": self.variables, + } diff --git a/routes/ingestion.py b/routes/ingestion.py index 933a8f9..d1e0cce 100644 --- a/routes/ingestion.py +++ b/routes/ingestion.py @@ -165,6 +165,15 @@ def approve_all(): bc = _bc_from_mapped(bc_id, mapped) db.session.add(bc) _create_decs(bc_id, ir.decs) + db.session.add( + AuditLog( + entity_type="BiomedicalConcept", + entity_id=bc_id, + action="created_via_ingestion", + actor="system", + after_state=mapped, + ) + ) ir.status = "approved" added += 1 else: diff --git a/routes/specializations.py b/routes/specializations.py index 3236ef1..034f131 100644 --- a/routes/specializations.py +++ b/routes/specializations.py @@ -3,6 +3,7 @@ from extensions import db from models.bc import BiomedicalConcept, DataElementConcept from models.specialization import DatasetSpecialization +from services.audit import log_change from services.cdisc_api import CDISCApiClient bp = Blueprint("specializations", __name__) @@ -77,11 +78,22 @@ def create(): ) spec.variables = [] db.session.add(spec) + log_change("DatasetSpecialization", vlm_group_id, "created", actor="user", after=spec.to_dict()) db.session.commit() flash(f"Specialization {vlm_group_id} created", "success") return redirect(url_for("specializations.index")) +@bp.route("//delete", methods=["POST"]) +def delete(vlm_group_id): + spec = db.get_or_404(DatasetSpecialization, vlm_group_id) + log_change("DatasetSpecialization", vlm_group_id, "deleted", actor="user", before=spec.to_dict()) + db.session.delete(spec) + db.session.commit() + flash(f"Specialization {vlm_group_id} deleted", "success") + return redirect(url_for("specializations.index")) + + @bp.route("/generate-from-dec", methods=["POST"]) def generate_from_dec(): """Return DEC-derived variable rows as JSON for the specialization form.""" @@ -114,6 +126,7 @@ def generate(bc_id): ) spec.variables = variables db.session.add(spec) + log_change("DatasetSpecialization", vlm_group_id, "created", actor="user", after=spec.to_dict()) db.session.commit() flash(f"Specialization {vlm_group_id} generated", "success") return redirect(url_for("specializations.index")) diff --git a/templates/specializations.html b/templates/specializations.html index 30ba7c3..7de78de 100644 --- a/templates/specializations.html +++ b/templates/specializations.html @@ -218,12 +218,10 @@

All Specializations

aria-label="Edit {{ spec.short_name }}"> Edit -
{{ form.hidden_tag() if form else '' }} - -