diff --git a/Dockerfile b/Dockerfile index 2e38774..25f5a0d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,6 +49,7 @@ COPY --chown=1000:1000 . /jupyter RUN chown -R 1000:1000 /jupyter WORKDIR /jupyter + # Clean up sensitive files and install local package RUN rm -f /jupyter/.beaker.conf /jupyter/.env && \ uv pip install --system -e /jupyter @@ -71,8 +72,11 @@ ENV BEAKER_AGENT_USER=jupyter \ BEAKER_RUN_PATH=/var/run/beaker \ BEAKER_APP=biome.app.BiomeApp +# Entrypoint (runs startup tasks before launching the server) +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + # Run as jupyter user USER jupyter -# Service -CMD ["python", "-m", "beaker_kernel.service.server", "--ip", "0.0.0.0"] \ No newline at end of file +CMD ["/entrypoint.sh"] \ No newline at end of file diff --git a/adhoc_data/specifications/idc/attachments/idc.md b/adhoc_data/specifications/idc/attachments/idc.md index 6b21f26..95fddaa 100644 --- a/adhoc_data/specifications/idc/attachments/idc.md +++ b/adhoc_data/specifications/idc/attachments/idc.md @@ -1,546 +1,843 @@ -Open In Colab +--- +name: imaging-data-commons +description: Query and download public cancer imaging data from NCI Imaging Data Commons using idc-index. Use for accessing large-scale radiology (CT, MR, PET) and pathology datasets for AI training or research. No authentication required. Query by metadata, visualize in browser, check licenses. +license: This skill is provided under the MIT License. IDC data itself has individual licensing (mostly CC-BY, some CC-NC) that must be respected when using the data. +metadata: + version: 1.4.0 + skill-author: Andrey Fedorov, @fedorov + idc-index: "0.11.10" + idc-data-version: "v23" + repository: https://github.com/ImagingDataCommons/idc-claude-skill +--- -# Getting started with IDC using `idc-index` python package +# Imaging Data Commons ---- +## Overview +Use the `idc-index` Python package to query and download public cancer imaging data from the National Cancer Institute Imaging Data Commons (IDC). No authentication required for data access. -## Summary +**Current IDC Data Version: v23** (always verify with `IDCClient().get_idc_version()`) -This notebook is introducing NCI Imaging Data Commons to the users who want to interact with IDC programmatically using [`idc-index` python package](https://github.com/ImagingDataCommons/idc-index). +**Primary tool:** `idc-index` ([GitHub](https://github.com/imagingdatacommons/idc-index)) -In this notebook you will be introduced into how IDC organizes the metadata accompanying images available in IDC, and how that metadata can be used to define subsets of data. This documentation page complements this notebook with the more detailed discussion of the metadata organization in IDC: https://learn.canceridc.dev/data/organization-of-data/files-and-metadata. +**CRITICAL - Check package version and upgrade if needed (run this FIRST):** -Please note that it is important that you run each cell of this notebook in sequence the first time you go over the notebook. Out of order execution may result in runtime errors. +```python +import idc_index + +REQUIRED_VERSION = "0.11.10" # Must match metadata.idc-index in this file +installed = idc_index.__version__ + +if installed < REQUIRED_VERSION: + print(f"Upgrading idc-index from {installed} to {REQUIRED_VERSION}...") + import subprocess + subprocess.run(["pip3", "install", "--upgrade", "--break-system-packages", "idc-index"], check=True) + print("Upgrade complete. Restart Python to use new version.") +else: + print(f"idc-index {installed} meets requirement ({REQUIRED_VERSION})") +``` ---- -Initial version: Nov 2023 +**Verify IDC data version and check current data scale:** -## What is IDC? +```python +from idc_index import IDCClient +client = IDCClient() + +# Verify IDC data version (should be "v23") +print(f"IDC data version: {client.get_idc_version()}") + +# Get collection count and total series +stats = client.sql_query(""" + SELECT + COUNT(DISTINCT collection_id) as collections, + COUNT(DISTINCT analysis_result_id) as analysis_results, + COUNT(DISTINCT PatientID) as patients, + COUNT(DISTINCT StudyInstanceUID) as studies, + COUNT(DISTINCT SeriesInstanceUID) as series, + SUM(instanceCount) as instances, + SUM(series_size_MB)/1000000 as size_TB + FROM index +""") +print(stats) +``` -NCI Imaging Data Commons (IDC) is a cloud-based environment containing publicly available cancer imaging data co-located with the analysis and exploration tools and resources. IDC is a node within the broader NCI Cancer Research Data Commons (CRDC) infrastructure that provides secure access to a large, comprehensive, and expanding collection of cancer research data. +**Core workflow:** +1. Query metadata → `client.sql_query()` +2. Download DICOM files → `client.download_from_selection()` +3. Visualize in browser → `client.get_viewer_URL(seriesInstanceUID=...)` -## IDC metadata search: why, what and how +## When to Use This Skill -**Why?** +- Finding publicly available radiology (CT, MR, PET) or pathology (slide microscopy) images +- Selecting image subsets by cancer type, modality, anatomical site, or other metadata +- Downloading DICOM data from IDC +- Checking data licenses before use in research or commercial applications +- Visualizing medical images in a browser without local DICOM viewer software -Think of IDC as a library. Image files are books, and we have ~45 TB of those. When you go to a library, you want to check out just the books that you want to read. In order to find a book in a large library you need a catalog. Without a good catalog it is difficult to make use of a library, or any other resource containing significant amount of information. +## Quick Navigation -**What?** +**Core Sections (inline):** +- IDC Data Model - Collection and analysis result hierarchy +- Index Tables - Available tables and joining patterns +- Installation - Package setup and version verification +- Core Capabilities - Essential API patterns (query, download, visualize, license, citations, batch) +- Best Practices - Usage guidelines +- Troubleshooting - Common issues and solutions -Just as in the library, IDC maintains a catalog that indexes a variety of metadata fields describing the files we curate. That metadata catalog is accessible in a large database table that you should be using to search and subset the images. All of the data in IDC is in DICOM format, and DICOM format is all about metadata! A typical DICOM image will contain dozens if not hundreds of attributes describing its content. +**Reference Guides (load on demand):** -We extract all of the DICOM attributes, and store the result in a large table, where each row of that table corresponds to a file (and most often, one slice of a CT or MR image corresponds to one DICOM file), and each column corresponds to a metadata attribute. IDC catalog of all the metadata is enormous: as of writing, for IDC release v16, this table contains >42M rows and 869 columns! We will refer to this gigantic catalog as _IDC BigQuery index_ (you can learn how to use this index in another tutorial). +| Guide | When to Load | +|-------|--------------| +| `index_tables_guide.md` | Complex JOINs, schema discovery, DataFrame access | +| `use_cases.md` | End-to-end workflow examples (training datasets, batch downloads) | +| `sql_patterns.md` | Quick SQL patterns for filter discovery, annotations, size estimation | +| `clinical_data_guide.md` | Clinical/tabular data, imaging+clinical joins, value mapping | +| `cloud_storage_guide.md` | Direct S3/GCS access, versioning, UUID mapping | +| `dicomweb_guide.md` | DICOMweb endpoints, PACS integration | +| `digital_pathology_guide.md` | Slide microscopy (SM), annotations (ANN), pathology workflows | +| `bigquery_guide.md` | Full DICOM metadata, private elements (requires GCP) | +| `cli_guide.md` | Command-line tools (`idc download`, manifest files) | -The issue is, _IDC BigQuery index_ is very large, requires rather powerful resources to search it, can be intimidating to the novice IDC users, and is not necessary for most common search tasks. This is why we developed a small index of IDC data that contains just a fraction of the metadata attributes. Since this index is small, it can be distributed and searched easily. The downside of course is that you cannot search all of the metadata. +## IDC Data Model -We wrapped the IDC "mini" index into the `idc-index` python package, which we will be focusing on in this tutorial. +IDC adds two grouping levels above the standard DICOM hierarchy (Patient → Study → Series → Instance): -**How?** +- **collection_id**: Groups patients by disease, modality, or research focus (e.g., `tcga_luad`, `nlst`). A patient belongs to exactly one collection. +- **analysis_result_id**: Identifies derived objects (segmentations, annotations, radiomics features) across one or more original collections. -When you search, or _query_ IDC catalog, you specify what criteria should the metadata describing the selected files satisfy. +Use `collection_id` to find original imaging data, may include annotations deposited along with the images; use `analysis_result_id` to find AI-generated or expert annotations. -Queries can be as simple as +**Key identifiers for queries:** +| Identifier | Scope | Use for | +|------------|-------|---------| +| `collection_id` | Dataset grouping | Filtering by project/study | +| `PatientID` | Patient | Grouping images by patient | +| `StudyInstanceUID` | DICOM study | Grouping of related series, visualization | +| `SeriesInstanceUID` | DICOM series | Grouping of related series, visualization | -* "_everything in collection X_", +## Index Tables -or as complex as +The `idc-index` package provides multiple metadata index tables, accessible via SQL or as pandas DataFrames. -* "_files corresponding to CT images of female patients that are accompanied by annotations of lung tumors that are larger than 1500 mm^3 in volume_". +**Complete index table documentation:** Use https://idc-index.readthedocs.io/en/latest/indices_reference.html for quick check of available tables and columns without executing any code. -Although it would be very nice to just state what you need in free form (and let AI write the query for you - which is becoming more and more feasible - see one such early results in [this preprint](https://arxiv.org/abs/2305.07637)!), in practice most often queries need to be written in a formal way. +**Important:** Use `client.indices_overview` to get current table descriptions and column schemas. This is the authoritative source for available columns and their types — always query it when writing SQL or exploring data structure. -To query IDC index, you can utilize Standard Query Language (SQL). You can use SQL with both the IDC BigQuery index, and with the IDC "mini" index available in the `idc-index` python package. +### Available Tables -In the following steps of the tutorial we will use just a few of the attributes (SQL table columns) to get started. You will be able to use the same principles and SQL queries to extend your search criteria to include any of the other attributes indexed by IDC. +| Table | Row Granularity | Loaded | Description | +|-------|-----------------|--------|-------------| +| `index` | 1 row = 1 DICOM series | Auto | Primary metadata for all current IDC data | +| `prior_versions_index` | 1 row = 1 DICOM series | Auto | Series from previous IDC releases; for downloading deprecated data | +| `collections_index` | 1 row = 1 collection | fetch_index() | Collection-level metadata and descriptions | +| `analysis_results_index` | 1 row = 1 analysis result collection | fetch_index() | Metadata about derived datasets (annotations, segmentations) | +| `clinical_index` | 1 row = 1 clinical data column | fetch_index() | Dictionary mapping clinical table columns to collections | +| `sm_index` | 1 row = 1 slide microscopy series | fetch_index() | Slide Microscopy (pathology) series metadata | +| `sm_instance_index` | 1 row = 1 slide microscopy instance | fetch_index() | Instance-level (SOPInstanceUID) metadata for slide microscopy | +| `seg_index` | 1 row = 1 DICOM Segmentation series | fetch_index() | Segmentation metadata: algorithm, segment count, reference to source image series | +| `ann_index` | 1 row = 1 DICOM ANN series | fetch_index() | Microscopy Bulk Simple Annotations series metadata; references annotated image series | +| `ann_group_index` | 1 row = 1 annotation group | fetch_index() | Detailed annotation group metadata: graphic type, annotation count, property codes, algorithm | +| `contrast_index` | 1 row = 1 series with contrast info | fetch_index() | Contrast agent metadata: agent name, ingredient, administration route (CT, MR, PT, XA, RF) | -## Prerequisites +**Auto** = loaded automatically when `IDCClient()` is instantiated +**fetch_index()** = requires `client.fetch_index("table_name")` to load -Prerequisites for using IDC "mini" index are very simple: use `pip` to install the `idc-index` package! In the cell below we use a fixed (currently, latest) version of the package for the sake of reproducibility. +### Joining Tables -Note how we install the specific version of the package. We do this because `idc-index` is in the early stages of development, and its API and capabilities are evolving. By fixing the release version we ensure that the notebook remains functional even if the package API changes in a breaking manner. +**Key columns are not explicitly labeled, the following is a subset that can be used in joins.** -Once the package is installed, we instantiate `IDCClient` class that is a wrapper around the "mini" index. +| Join Column | Tables | Use Case | +|-------------|--------|----------| +| `collection_id` | index, prior_versions_index, collections_index, clinical_index | Link series to collection metadata or clinical data | +| `SeriesInstanceUID` | index, prior_versions_index, sm_index, sm_instance_index | Link series across tables; connect to slide microscopy details | +| `StudyInstanceUID` | index, prior_versions_index | Link studies across current and historical data | +| `PatientID` | index, prior_versions_index | Link patients across current and historical data | +| `analysis_result_id` | index, analysis_results_index | Link series to analysis result metadata (annotations, segmentations) | +| `source_DOI` | index, analysis_results_index | Link by publication DOI | +| `crdc_series_uuid` | index, prior_versions_index | Link by CRDC unique identifier | +| `Modality` | index, prior_versions_index | Filter by imaging modality | +| `SeriesInstanceUID` | index, seg_index, ann_index, ann_group_index, contrast_index | Link segmentation/annotation/contrast series to its index metadata | +| `segmented_SeriesInstanceUID` | seg_index → index | Link segmentation to its source image series (join seg_index.segmented_SeriesInstanceUID = index.SeriesInstanceUID) | +| `referenced_SeriesInstanceUID` | ann_index → index | Link annotation to its source image series (join ann_index.referenced_SeriesInstanceUID = index.SeriesInstanceUID) | +**Note:** `Subjects`, `Updated`, and `Description` appear in multiple tables but have different meanings (counts vs identifiers, different update contexts). + +For detailed join examples, schema discovery patterns, key columns reference, and DataFrame access, see `references/index_tables_guide.md`. + +### Clinical Data Access ```python -!pip install idc-index --upgrade +# Fetch clinical index (also downloads clinical data tables) +client.fetch_index("clinical_index") -from idc_index import index +# Query clinical index to find available tables and their columns +tables = client.sql_query("SELECT DISTINCT table_name, column_label FROM clinical_index") -client = index.IDCClient() +# Load a specific clinical table as DataFrame +clinical_df = client.get_clinical_table("table_name") ``` -## First query +See `references/clinical_data_guide.md` for detailed workflows including value mapping patterns and joining clinical data with imaging. -As the very first query, let's get the list of all the image collections available in IDC. Here is that query: +## Data Access Options -```sql -SELECT - DISTINCT(collection_id) -FROM - index -``` +| Method | Auth Required | Best For | +|--------|---------------|----------| +| `idc-index` | No | Key queries and downloads (recommended) | +| IDC Portal | No | Interactive exploration, manual selection, browser-based download | +| BigQuery | Yes (GCP account) | Complex queries, full DICOM metadata | +| DICOMweb proxy | No | Tool integration via DICOMweb API | +| Cloud storage (S3/GCS) | No | Direct file access, bulk downloads, custom pipelines | -Let's look into how this query works: +**Cloud storage organization** -* `SELECT` defines the list of columns that should be returned by the query, -* `DISTINCT` indicates that we want to see the distinct values encountered in the selected column, -* `FROM` defines which table should be queried. Here, `index` is the internal table availabe within the `idc-index` package. +IDC maintains all DICOM files in public cloud storage buckets mirrored between AWS S3 and Google Cloud Storage. Files are organized by CRDC UUIDs (not DICOM UIDs) to support versioning. -Next, let's execute that query using the `client` instantiated earlier. The `sql_query` function will return a `pandas` `DataFrame` with the result. +| Bucket (AWS / GCS) | License | Content | +|--------------------|---------|---------| +| `idc-open-data` / `idc-open-data` | No commercial restriction | >90% of IDC data | +| `idc-open-data-two` / `idc-open-idc1` | No commercial restriction | Collections with potential head scans | +| `idc-open-data-cr` / `idc-open-cr` | Commercial use restricted (CC BY-NC) | ~4% of data | +Files are stored as `/.dcm`. Access is free (no egress fees) via AWS CLI, gsutil, or s5cmd with anonymous access. Use `series_aws_url` column from the index for S3 URLs; GCS uses the same path structure. -```python -# formatting is not required, but makes queries easier to read! -query = """ -SELECT - DISTINCT(collection_id) -FROM - index -""" -client.sql_query(query) -``` +See `references/cloud_storage_guide.md` for bucket details, access commands, UUID mapping, and versioning. + +**DICOMweb access** + +IDC data is available via DICOMweb interface (Google Cloud Healthcare API implementation) for integration with PACS systems and DICOMweb-compatible tools. + +| Endpoint | Auth | Use Case | +|----------|------|----------| +| Public proxy | No | Testing, moderate queries, daily quota | +| Google Healthcare | Yes (GCP) | Production use, higher quotas | -What other attributes are available in this "mini" index? +See `references/dicomweb_guide.md` for endpoint URLs, code examples, supported operations, and implementation details. + +## Installation and Setup + +**Required (for basic access):** +```bash +pip install --upgrade idc-index +``` +**Important:** New IDC data release will always trigger a new version of `idc-index`. Always use `--upgrade` flag while installing, unless an older version is needed for reproducibility. +**IMPORTANT:** IDC data version v23 is current. Always verify your version: ```python -print(client.index.columns) +print(client.get_idc_version()) # Should return "v23" ``` +If you see an older version, upgrade with: `pip install --upgrade idc-index` -## Summarizing the content of the individual collections +**Tested with:** idc-index 0.11.10 (IDC data version v23) -Before discussing what each of those attributes means, let's consider a bit more complicated query that utilizes the `Modality` and `BodyPartExamined` to create summary of the collections available in IDC. +**Optional (for data analysis):** +```bash +pip install pandas numpy pydicom +``` -In the query above, we use the familiar operators `SELECT` and `FROM`, but also couple of new ones: +## Core Capabilities -* `GROUP BY` in the end of the query indicates that we want to get a single row per the distinct value of the `collection_id` -* `STRING_AGG` and `DISTINCT` indicate how the values of the selected columns should be aggregated while combining into single row per collection_id: we take all the distinct values per individual `collection_id`, and the concatenate them into a single string +### 1. Data Discovery and Exploration +Discover what imaging collections and data are available in IDC: ```python +from idc_index import IDCClient + +client = IDCClient() + +# Get summary statistics from primary index query = """ SELECT collection_id, - STRING_AGG(DISTINCT(Modality)) as modalities, - STRING_AGG(DISTINCT(BodyPartExamined)) as body_parts -FROM - index -GROUP BY - collection_id -ORDER BY - collection_id ASC + COUNT(DISTINCT PatientID) as patients, + COUNT(DISTINCT SeriesInstanceUID) as series, + SUM(series_size_MB) as size_mb +FROM index +GROUP BY collection_id +ORDER BY patients DESC """ -client.sql_query(query) +collections_summary = client.sql_query(query) + +# For richer collection metadata, use collections_index +client.fetch_index("collections_index") +collections_info = client.sql_query(""" + SELECT collection_id, CancerTypes, TumorLocations, Species, Subjects, SupportingData + FROM collections_index +""") + +# For analysis results (annotations, segmentations), use analysis_results_index +client.fetch_index("analysis_results_index") +analysis_info = client.sql_query(""" + SELECT analysis_result_id, analysis_result_title, Subjects, Collections, Modalities + FROM analysis_results_index +""") ``` -As you look at the result of the query, consider that ... -* `Modality` abbreviations are disambiguated in this part of the standard: https://dicom.nema.org/medical/dicom/current/output/chtml/part03/sect_C.7.3.html#sect_C.7.3.1.1.1 -* the values of `BodyPartExamined` were curated by IDC to improve conformance to the value set prescribed by the standard (you can see it here: https://dicom.nema.org/medical/dicom/current/output/chtml/part16/chapter_L.html#chapter_L) -* Slide Microscopy modality (`SM`) does not use `BodyPartExamined`, and therefore it is expected that the values of this attribute are blank for the slide microscopy collections. - -## Selecting collections based on the specific characteristics +**`collections_index`** provides curated metadata per collection: cancer types, tumor locations, species, subject counts, and supporting data types — without needing to aggregate from the primary index. -In the following query, we use several of the attributes in the index to select collections that meet specific search criteria: those that contain MR modality, and have Liver as the body part examined. +**`analysis_results_index`** lists derived datasets (AI segmentations, expert annotations, radiomics features) with their source collections and modalities. -Note that standard compliant values of `BodyPartExamined` are ALL CAPS! If you wanted to make the search case-insensitive, you could use `UPPER(BodyPartExamined)`. +### 2. Querying Metadata with SQL +Query the IDC mini-index using SQL to find specific datasets. +**First, explore available values for filter columns:** ```python -query = """ -SELECT - DISTINCT(collection_id) -FROM - index -WHERE - Modality = 'MR' - AND BodyPartExamined = 'LIVER' -""" -client.sql_query(query) +from idc_index import IDCClient + +client = IDCClient() + +# Check what Modality values exist +modalities = client.sql_query(""" + SELECT DISTINCT Modality, COUNT(*) as series_count + FROM index + GROUP BY Modality + ORDER BY series_count DESC +""") +print(modalities) + +# Check what BodyPartExamined values exist for MR modality +body_parts = client.sql_query(""" + SELECT DISTINCT BodyPartExamined, COUNT(*) as series_count + FROM index + WHERE Modality = 'MR' AND BodyPartExamined IS NOT NULL + GROUP BY BodyPartExamined + ORDER BY series_count DESC + LIMIT 20 +""") +print(body_parts) ``` -## DICOM data model: Patients, studies, series and instances - -Up to now we searched the data at the granularity of the collections. In practice, we often want to know how many patients meet our search criteria, or what are the specific images that we need to download. - -IDC is using DICOM for data representation, and in the DICOM data model, patients (identified by `PatientID`) undergo imaging exams (or _studies_, in DICOM nomenclature). - -Each patient will have one or more studies, with each study identified uniquely by the attribute `StudyInstanceUID`. During each of the imaging studies one or more imaging _series_ will be collected. As an example, a Computed Tomography (CT) imaging study may include a volume sweep before and after administration of the contrast agent. Imaging series are uniqiely identified by `SeriesInstanceUID`. - -Finally, each imaging series contains one or more _instances_, where each instance corresponds to a file. Most often, one instance corresponds to a single slice from a cross-sectional image. Individual instances are identified by unique `SOPInstanceUID` values. - -The figure below, borrowed from the DICOM standard [here](http://dicom.nema.org/medical/dicom/current/output/chtml/part03/chapter_7.html), captures the discussed data model. - -![DICOM data model](https://2103490465-files.gitbook.io/~/files/v0/b/gitbook-x-prod.appspot.com/o/spaces%2F-MCTG4fXybYgGMalZnmf-2668963341%2Fuploads%2Fgit-blob-0f639d56e22ae53a03c2ca59c96306c5db51b158%2FPS3.3_7-1a-DICOM_model.png?alt=media) - -**image description**: DICOM data model representing the following relationships: -```yaml -entities: -- Patient: - relationships: - - makes: - target: Study - cardinality: "1-n" - - has: - target: Study - cardinality: "1-n" - -- Study: - relationships: - - comprised_of: - target: Modality_Performed_Procedure_Step - cardinality: "1-n" - - includes: - target: Modality_Performed_Procedure_Step - cardinality: "1-n" - - contains: - target: Series - cardinality: "1-n" - -- Modality_Performed_Procedure_Step: - relationships: - - includes: - target: Series - cardinality: "1-n" - -- Equipment: - relationships: - - creates: - target: Series - cardinality: "1-n" - -- Frame_of_Reference: - relationships: - - spatially_defines: - target: Series - cardinality: "0-1" - - spatially_defines: - target: Registration - cardinality: "1-n" - -- Series: - relationships: - - contains: - target: - - Presentation_State - - MR_Spectroscopy - - Radiotherapy_Objects - - Encapsulated_Document - - Real_World_Value_Mapping - - Stereometric_Relationship - - Measurements - - Tractography_Results - - Registration - - Fiducials - - Image - - Raw_Data - - Waveform - - SR_Document - - Surface - - Annotations - cardinality: "0-n" - - contains: - target: Registration - cardinality: "0-n" - -- Presentation_State: - relationships: [] - -- MR_Spectroscopy: - relationships: [] - -- Radiotherapy_Objects: - relationships: [] - -- Encapsulated_Document: - relationships: [] - -- Real_World_Value_Mapping: - relationships: [] - -- Stereometric_Relationship: - relationships: [] - -- Measurements: - relationships: [] - -- Tractography_Results: - relationships: [] - -- Registration: - relationships: - - spatially_defines: - target: Frame_of_Reference - cardinality: "1-n" - -- Fiducials: - relationships: [] - -- Image: - relationships: [] - -- Raw_Data: - relationships: [] - -- Waveform: - relationships: [] - -- SR_Document: - relationships: [] - -- Surface: - relationships: [] - -- Annotations: - relationships: [] +**Then query with validated filter values:** +```python +# Find breast MRI scans (use actual values from exploration above) +results = client.sql_query(""" + SELECT + collection_id, + PatientID, + SeriesInstanceUID, + Modality, + SeriesDescription, + license_short_name + FROM index + WHERE Modality = 'MR' + AND BodyPartExamined = 'BREAST' + LIMIT 20 +""") + +# Access results as pandas DataFrame +for idx, row in results.iterrows(): + print(f"Patient: {row['PatientID']}, Series: {row['SeriesInstanceUID']}") ``` -## `idc-index` organization +**To filter by cancer type, join with `collections_index`:** +```python +client.fetch_index("collections_index") +results = client.sql_query(""" + SELECT i.collection_id, i.PatientID, i.SeriesInstanceUID, i.Modality + FROM index i + JOIN collections_index c ON i.collection_id = c.collection_id + WHERE c.CancerTypes LIKE '%Breast%' + AND i.Modality = 'MR' + LIMIT 20 +""") +``` -Having reviewed the DICOM data model, it is time to go over the attributes (columns) included in `idc-index`. +**Available metadata fields** (use `client.indices_overview` for complete list): +- Identifiers: collection_id, PatientID, StudyInstanceUID, SeriesInstanceUID +- Imaging: Modality, BodyPartExamined, Manufacturer, ManufacturerModelName +- Clinical: PatientAge, PatientSex, StudyDate +- Descriptions: StudyDescription, SeriesDescription +- Licensing: license_short_name -But first, it is important to know that **`idc-index` is series-based**, i.e, it has one row per DICOM series. AI analysis is most often performed at the granularity of the individual series, and therefore selection of the series is a task we address with `idc-index`. +**Note:** Cancer type is in `collections_index.CancerTypes`, not in the primary `index` table. -We "compressed" the IDC BigQuery index (which is instance-based) for the sake of efficiency to support basic search capabilities. The convention followed in naming the columns is that every column that is named in CamelCase notation corresponds one-for-one to the DICOM attribute with the same name. Columns starting with a lower-case letter are those that were introduced by the IDC team as part of curation, and do do not fit the DICOM data model. For each of those series our "mini" index contains the following columns. +### 3. Downloading DICOM Files -* non-DICOM attributes assigned/curated by IDC: - * `collection_id`: short string with the identifier of the collection the series belongs to - * `analysis_result_id`: this string is not empty if the specific series is part of an analysis results collection; analysis results can be added to a given collection over time - * `source_DOI`: Digital Object Identifier of the dataset that contains the given series; note that a given collection can include one or more DOIs, since analysis results added to the collection would typically have independent DOI values! - * `instanceCount`: number of files in the series (typically, this matches the number of slices in cross-sectional modalities) - * `license_short_name`: short name of the license that governs the use of the files corresponding to the series - * `series_aws_url`: location of the series files in a public AWS bucket - * `series_size_MB`: total disk size needed to store the series -* DICOM attributes extracted from the files - * `PatientID`: identifier of the patient - * `PatientAge` and `PatientSex`: attributes containing patient age and sex - * `StudyInstanceUID`: unique identifier of the DICOM study - * `StudyDescription`: textual description of the study content - * `StudyDate`: date of the study (note that those dates are shifted, and are not real dates when images were acquired, to protect patient privacy) - * `SeriesInstanceUID`: unique identifier of the DICOM series - * `SeriesDate`: date when the series was acquired - * `SeriesDescription`: textual description of the series content - * `SeriesNumber`: series number - * `BodyPartExamined`: body part imaged - * `Modality`: acquisition modality - * `Manufacturer`: manufacturer of the equipment that generated the series - * `ManufacturerModelName`: model name of the equipment +Download imaging data efficiently from IDC's cloud storage: - Similar to how we searched collections, we can filter suitable DICOM series with SQL queries. In the following query we search for MR series that have "PROSTATE" as the `BodyPartExamined`, and list the collections that contain those series, along with the total number of patients, studies, series count, manufacturer values and size on disk in GB. +**Download entire collection:** +```python +from idc_index import IDCClient +client = IDCClient() +# Download small collection (RIDER Pilot ~1GB) +client.download_from_selection( + collection_id="rider_pilot", + downloadDir="./data/rider" +) +``` +**Download specific series:** ```python -selection_query = """ -SELECT - collection_id, - COUNT(DISTINCT(PatientID)) as patient_count, - COUNT(DISTINCT(StudyInstanceUID)) as study_count, - COUNT(DISTINCT(SeriesInstanceUID)) as series_count, - STRING_AGG(DISTINCT(Manufacturer)) as manufacturers, - SUM(series_size_MB)/1000 as total_size_GB -FROM - index -WHERE - Modality = 'MR' - AND BodyPartExamined = 'PROSTATE' -GROUP BY - collection_id -ORDER BY - patient_count DESC -""" -client.sql_query(selection_query) +# First, query for series UIDs +series_df = client.sql_query(""" + SELECT SeriesInstanceUID + FROM index + WHERE Modality = 'CT' + AND BodyPartExamined = 'CHEST' + AND collection_id = 'nlst' + LIMIT 5 +""") + +# Download only those series +client.download_from_selection( + seriesInstanceUID=list(series_df['SeriesInstanceUID'].values), + downloadDir="./data/lung_ct" +) ``` -Instead of creating a collection-level summary, we can instead get the list of series that match our selection criteria. +**Custom directory structure:** +Default `dirTemplate`: `%collection_id/%PatientID/%StudyInstanceUID/%Modality_%SeriesInstanceUID` ```python -selection_query = """ -SELECT - SeriesInstanceUID, - SeriesDescription, - series_aws_url, - license_short_name, - source_DOI -FROM - index -WHERE - Modality = 'MR' - AND BodyPartExamined = 'PROSTATE' -""" -client.sql_query(selection_query) +# Simplified hierarchy (omit StudyInstanceUID level) +client.download_from_selection( + collection_id="tcga_luad", + downloadDir="./data", + dirTemplate="%collection_id/%PatientID/%Modality" +) +# Results in: ./data/tcga_luad/TCGA-05-4244/CT/ + +# Flat structure (all files in one directory) +client.download_from_selection( + seriesInstanceUID=list(series_df['SeriesInstanceUID'].values), + downloadDir="./data/flat", + dirTemplate="" +) +# Results in: ./data/flat/*.dcm ``` -# Key operations with IDC cohorts +**Downloaded file names:** -A "cohort"? What cohort? +Individual DICOM files are named using their CRDC instance UUID: `.dcm` (e.g., `0d73f84e-70ae-4eeb-96a0-1c613b5d9229.dcm`). This UUID-based naming: +- Enables version tracking (UUIDs change when file content changes) +- Matches cloud storage organization (`s3://idc-open-data//.dcm`) +- Differs from DICOM UIDs (SOPInstanceUID) which are preserved inside the file metadata -In IDC, a _cohort_ is set of objects stored in IDC that share certain characteristics as defined by metadata. In the previous section we defined a query that selects all MR series of the prostate. You can think of that selection as a cohort. +To identify files, use the `crdc_instance_uuid` column in queries or read DICOM metadata (SOPInstanceUID) from the files. -We will show you how to download the series in your cohort, but first - let's learn what you can do without having to download anything! +### Command-Line Download -## Checking licenses and attribution requirements +The `idc download` command provides command-line access to download functionality without writing Python code. Available after installing `idc-index`. -IDC collects data from various data coordination centers and program. It is important to appreciate that different components of IDC data are covered by different licenses and have attribution requirements that you must follow when using the data! +**Auto-detects input type:** manifest file path, or identifiers (collection_id, PatientID, StudyInstanceUID, SeriesInstanceUID, crdc_series_uuid). -To get information about license check the `license_short_name`. Most of the data in IDC is covered by the CC-BY (Creative Commons By Attribution). +```bash +# Download entire collection +idc download rider_pilot --download-dir ./data +# Download specific series by UID +idc download "1.3.6.1.4.1.9328.50.1.69736" --download-dir ./data + +# Download multiple items (comma-separated) +idc download "tcga_luad,tcga_lusc" --download-dir ./data + +# Download from manifest file (auto-detected) +idc download manifest.txt --download-dir ./data +``` -To make sure you properly comply with the attribution clause, use `source_DOI`. If we prepend `source_DOI` with "https://doi.org/", we will get a URL that you can open to learn more about the dataset that contains the specific series. +**Options:** -If you use data from IDC, we would also appreciate if you acknowledge IDC by citing one of the publications below: +| Option | Description | +|--------|-------------| +| `--download-dir` | Output directory (default: current directory) | +| `--dir-template` | Directory hierarchy template (default: `%collection_id/%PatientID/%StudyInstanceUID/%Modality_%SeriesInstanceUID`) | +| `--log-level` | Verbosity: debug, info, warning, error, critical | -> Fedorov, A., Longabaugh, W. J. R., Pot, D., Clunie, D. A., Pieper, S. D., Gibbs, D. L., Bridge, C., Herrmann, M. D., Homeyer, A., Lewis, R., Aerts, H. J. W., Krishnaswamy, D., Thiriveedhi, V. K., Ciausu, C., Schacherer, D. P., Bontempi, D., Pihl, T., Wagner, U., Farahani, K., Kim, E. & Kikinis, R. _National Cancer Institute Imaging Data Commons: Toward Transparency, Reproducibility, and Scalability in Imaging Artificial Intelligence_. RadioGraphics (2023). https://doi.org/10.1148/rg.230180 +**Manifest files:** +Manifest files contain S3 URLs (one per line) and can be: +- Exported from the IDC Portal after cohort selection +- Shared by collaborators for reproducible data access +- Generated programmatically from query results + +Format (one S3 URL per line): +``` +s3://idc-open-data/cb09464a-c5cc-4428-9339-d7fa87cfe837/* +s3://idc-open-data/88f3990d-bdef-49cd-9b2b-4787767240f2/* +``` + +**Example: Generate manifest from Python query:** ```python -from IPython.display import HTML +from idc_index import IDCClient -selection_query = """ -SELECT - SeriesInstanceUID, - SeriesDescription, - series_aws_url, - license_short_name, - CONCAT('https://doi.org/',source_DOI) as source_DOI_URL -FROM - index -WHERE - Modality = 'MR' - AND BodyPartExamined = 'PROSTATE' -""" -df = client.sql_query(selection_query) +client = IDCClient() -# make the DOI URL clickable -def make_clickable(val): - return '{}'.format(val,val) +# Query for series URLs +results = client.sql_query(""" + SELECT series_aws_url + FROM index + WHERE collection_id = 'rider_pilot' AND Modality = 'CT' +""") -df['source_DOI_URL'] = df['source_DOI_URL'].apply(make_clickable) +# Save as manifest file +with open('ct_manifest.txt', 'w') as f: + for url in results['series_aws_url']: + f.write(url + '\n') +``` -# Display the DataFrame with clickable URLs -HTML(df.to_html(escape=False)) +Then download: +```bash +idc download ct_manifest.txt --download-dir ./ct_data ``` -## Visualizing individual series +### 4. Visualizing IDC Images -You can easily visualize any of the series in IDC from the convenience of your web browser. Moreover, there are several viewers that are available. Given the information in `idc-index`, all you have to do is build the URL to point the viewers IDC team is maintaining to the specific series of interest. +View DICOM data in browser without downloading: -IDC supports visualization using the hosted instances of the following open source viewers: -* OHIF Viewer v2 (legacy) and v3 (radiology): https://github.com/OHIF/Viewers -* VolView (hosted by Kitware Inc, radiology): https://volview.kitware.com/ -* Slim (slide microscopy): https://github.com/ImagingDataCommons/slim +```python +from idc_index import IDCClient +import webbrowser + +client = IDCClient() + +# First query to get valid UIDs +results = client.sql_query(""" + SELECT SeriesInstanceUID, StudyInstanceUID + FROM index + WHERE collection_id = 'rider_pilot' AND Modality = 'CT' + LIMIT 1 +""") + +# View single series +viewer_url = client.get_viewer_URL(seriesInstanceUID=results.iloc[0]['SeriesInstanceUID']) +webbrowser.open(viewer_url) + +# View all series in a study (useful for multi-series exams like MRI protocols) +viewer_url = client.get_viewer_URL(studyInstanceUID=results.iloc[0]['StudyInstanceUID']) +webbrowser.open(viewer_url) +``` -In the following query we build URLs for each of the series to open those in OHIF v2, OHIF v3 and VolView. We apply extra trick to make the links clickable, and limit the search results to 10 series to make the output more readable. +The method automatically selects OHIF v3 for radiology or SLIM for slide microscopy. Viewing by study is useful when a DICOM Study contains multiple Series (e.g., T1, T2, DWI sequences from a single MRI session). -WARNING: Due to a [last-minute bug in `idc-index`](https://github.com/ImagingDataCommons/idc-index/issues/16), VolView URL will not work, but this will be resolved in an upcoming release of `idc-index`. +### 5. Understanding and Checking Licenses +Check data licensing before use (critical for commercial applications): ```python -from IPython.display import HTML +from idc_index import IDCClient -# remember to use the single quotes ' ' for constant strings! -# " " will cause error -selection_query = """ -SELECT - SeriesDescription, - CONCAT('https://viewer.imaging.datacommons.cancer.gov/viewer/', - StudyInstanceUID, - '?SeriesInstanceUID=', - SeriesInstanceUID) as ohif_v2_url, - CONCAT('https://viewer.imaging.datacommons.cancer.gov/v3/viewer/?StudyInstanceUIDs=', - StudyInstanceUID, - '&SeriesInstanceUID=', - SeriesInstanceUID) as ohif_v3_url, - CONCAT('https://volview.kitware.app/?urls=[', - series_aws_url, - ']') as volview_url -FROM - index -WHERE - Modality = 'MR' - AND BodyPartExamined = 'PROSTATE' -LIMIT - 10 +client = IDCClient() + +# Check licenses for all collections +query = """ +SELECT DISTINCT + collection_id, + license_short_name, + COUNT(DISTINCT SeriesInstanceUID) as series_count +FROM index +GROUP BY collection_id, license_short_name +ORDER BY collection_id """ -df = client.sql_query(selection_query) -# if you remove `target="_blank"`, the viewer will open directly -# in the notebook cell! -def make_clickable(val): - return '{}'.format(val,val) +licenses = client.sql_query(query) +print(licenses) +``` + +**License types in IDC:** +- **CC BY 4.0** / **CC BY 3.0** (~97% of data) - Allows commercial use with attribution +- **CC BY-NC 4.0** / **CC BY-NC 3.0** (~3% of data) - Non-commercial use only +- **Custom licenses** (rare) - Some collections have specific terms (e.g., NLM Terms and Conditions) -df['ohif_v2_url'] = df['ohif_v2_url'].apply(make_clickable) -df['ohif_v3_url'] = df['ohif_v3_url'].apply(make_clickable) -df['volview_url'] = df['volview_url'].apply(make_clickable) +**Important:** Always check the license before using IDC data in publications or commercial applications. Each DICOM file is tagged with its specific license in metadata. -# Display the DataFrame with clickable URLs -HTML(df.to_html(escape=False)) +### Generating Citations for Attribution + +The `source_DOI` column contains DOIs linking to publications describing how the data was generated. To satisfy attribution requirements, use `citations_from_selection()` to generate properly formatted citations: + +```python +from idc_index import IDCClient + +client = IDCClient() + +# Get citations for a collection (APA format by default) +citations = client.citations_from_selection(collection_id="rider_pilot") +for citation in citations: + print(citation) + +# Get citations for specific series +results = client.sql_query(""" + SELECT SeriesInstanceUID FROM index + WHERE collection_id = 'tcga_luad' LIMIT 5 +""") +citations = client.citations_from_selection( + seriesInstanceUID=list(results['SeriesInstanceUID'].values) +) + +# Alternative format: BibTeX (for LaTeX documents) +bibtex_citations = client.citations_from_selection( + collection_id="tcga_luad", + citation_format=IDCClient.CITATION_FORMAT_BIBTEX +) ``` -## Downloading the content of the cohort +**Parameters:** +- `collection_id`: Filter by collection(s) +- `patientId`: Filter by patient ID(s) +- `studyInstanceUID`: Filter by study UID(s) +- `seriesInstanceUID`: Filter by series UID(s) +- `citation_format`: Use `IDCClient.CITATION_FORMAT_*` constants: + - `CITATION_FORMAT_APA` (default) - APA style + - `CITATION_FORMAT_BIBTEX` - BibTeX for LaTeX + - `CITATION_FORMAT_JSON` - CSL JSON + - `CITATION_FORMAT_TURTLE` - RDF Turtle -Earlier we mentioned that `series_aws_url` contains the location of the series files in a AWS bucket. Download is as simple as copying the files from the bucket. To perform the download operation, we rely on the open source [`s5cmd` tool](https://github.com/peak/s5cmd), which was installed as part of the `idc-index` package installation. We first prepare `s5cmd` download manifest that contains the list of download commands for all series, and then pass that manifest to a dedicated function. +**Best practice:** When publishing results using IDC data, include the generated citations to properly attribute the data sources and satisfy license requirements. -Let's build and save the manifest first. +### 6. Batch Processing and Filtering +Process large datasets efficiently with filtering: ```python -downloadDir = '/content/idc_downloads' +from idc_index import IDCClient +import pandas as pd -!rm -rf {downloadDir} -!mkdir -p {downloadDir} +client = IDCClient() -selection_query = """ +# Find chest CT scans from GE scanners +query = """ SELECT - CONCAT('cp ',series_aws_url,' /content/idc_downloads') as cp_command -FROM - index -WHERE - Modality = 'MR' - AND BodyPartExamined = 'PROSTATE' -LIMIT - 10 + SeriesInstanceUID, + PatientID, + collection_id, + ManufacturerModelName +FROM index +WHERE Modality = 'CT' + AND BodyPartExamined = 'CHEST' + AND Manufacturer = 'GE MEDICAL SYSTEMS' + AND license_short_name = 'CC BY 4.0' +LIMIT 100 """ -df = client.sql_query(selection_query) -with open('/content/idc_downloads/download_manifest.txt', 'w') as f: - f.write('\n'.join(df['cp_command'])) +results = client.sql_query(query) + +# Save manifest for later +results.to_csv('lung_ct_manifest.csv', index=False) + +# Download in batches to avoid timeout +batch_size = 10 +for i in range(0, len(results), batch_size): + batch = results.iloc[i:i+batch_size] + client.download_from_selection( + seriesInstanceUID=list(batch['SeriesInstanceUID'].values), + downloadDir=f"./data/batch_{i//batch_size}" + ) ``` +### 7. Advanced Queries with BigQuery + +For queries requiring full DICOM metadata, complex JOINs, clinical data tables, or private DICOM elements, use Google BigQuery. Requires GCP account with billing enabled. + +**Quick reference:** +- Dataset: `bigquery-public-data.idc_current.*` +- Main table: `dicom_all` (combined metadata) +- Full metadata: `dicom_metadata` (all DICOM tags) +- Private elements: `OtherElements` column (vendor-specific tags like diffusion b-values) + +See `references/bigquery_guide.md` for setup, table schemas, query patterns, private element access, and cost optimization. +**Before using BigQuery**, always check if a specialized index table already has the metadata you need: +1. Use `client.indices_overview` or the [idc-index indices reference](https://idc-index.readthedocs.io/en/latest/indices_reference.html) to discover all available tables and their columns +2. Fetch the relevant index: `client.fetch_index("table_name")` +3. Query locally with `client.sql_query()` (free, no GCP account needed) + +Common specialized indices: `seg_index` (segmentations), `ann_index` / `ann_group_index` (microscopy annotations), `sm_index` (slide microscopy), `collections_index` (collection metadata). Only use BigQuery if you need private DICOM elements or attributes not in any index. + +### 8. Tool Selection Guide + +| Task | Tool | Reference | +|------|------|-----------| +| Programmatic queries & downloads | `idc-index` | This document | +| Interactive exploration | IDC Portal | https://portal.imaging.datacommons.cancer.gov/ | +| Complex metadata queries | BigQuery | `references/bigquery_guide.md` | +| 3D visualization & analysis | SlicerIDCBrowser | https://github.com/ImagingDataCommons/SlicerIDCBrowser | + +**Default choice:** Use `idc-index` for most tasks (no auth, easy API, batch downloads). + +### 9. Integration with Analysis Pipelines + +Integrate IDC data into imaging analysis workflows: + +**Read downloaded DICOM files:** ```python -# examine the content of the manifest -!head /content/idc_downloads/download_manifest.txt +import pydicom +import os + +# Read DICOM files from downloaded series +series_dir = "./data/rider/rider_pilot/RIDER-1007893286/CT_1.3.6.1..." + +dicom_files = [os.path.join(series_dir, f) for f in os.listdir(series_dir) + if f.endswith('.dcm')] + +# Load first image +ds = pydicom.dcmread(dicom_files[0]) +print(f"Patient ID: {ds.PatientID}") +print(f"Modality: {ds.Modality}") +print(f"Image shape: {ds.pixel_array.shape}") ``` +**Build 3D volume from CT series:** +```python +import pydicom +import numpy as np +from pathlib import Path + +def load_ct_series(series_path): + """Load CT series as 3D numpy array""" + files = sorted(Path(series_path).glob('*.dcm')) + slices = [pydicom.dcmread(str(f)) for f in files] + + # Sort by slice location + slices.sort(key=lambda x: float(x.ImagePositionPatient[2])) + + # Stack into 3D array + volume = np.stack([s.pixel_array for s in slices]) + return volume, slices[0] # Return volume and first slice for metadata + +volume, metadata = load_ct_series("./data/lung_ct/series_dir") +print(f"Volume shape: {volume.shape}") # (z, y, x) +``` + +**Integrate with SimpleITK:** ```python -# efficiently download the files corresponding to the manifest -s5cmd_binary = client.s5cmdPath +import SimpleITK as sitk +from pathlib import Path + +# Read DICOM series +series_path = "./data/ct_series" +reader = sitk.ImageSeriesReader() +dicom_names = reader.GetGDCMSeriesFileNames(series_path) +reader.SetFileNames(dicom_names) +image = reader.Execute() -!{s5cmd_binary} --no-sign-request --endpoint-url https://s3.amazonaws.com run /content/idc_downloads/download_manifest.txt +# Apply processing +smoothed = sitk.CurvatureFlow(image1=image, timeStep=0.125, numberOfIterations=5) + +# Save as NIfTI +sitk.WriteImage(smoothed, "processed_volume.nii.gz") ``` -## Summary +## Common Use Cases + +See `references/use_cases.md` for complete end-to-end workflow examples including: +- Building deep learning training datasets from lung CT scans +- Comparing image quality across scanner manufacturers +- Previewing data in browser before downloading +- License-aware batch downloads for commercial use + +## Best Practices + +- **Verify IDC version before generating responses** - Always call `client.get_idc_version()` at the start of a session to confirm you're using the expected data version (currently v23). If using an older version, recommend `pip install --upgrade idc-index` +- **Check licenses before use** - Always query the `license_short_name` field and respect licensing terms (CC BY vs CC BY-NC) +- **Generate citations for attribution** - Use `citations_from_selection()` to get properly formatted citations from `source_DOI` values; include these in publications +- **Start with small queries** - Use `LIMIT` clause when exploring to avoid long downloads and understand data structure +- **Use mini-index for simple queries** - Only use BigQuery when you need comprehensive metadata or complex JOINs +- **Organize downloads with dirTemplate** - Use meaningful directory structures like `%collection_id/%PatientID/%Modality` +- **Cache query results** - Save DataFrames to CSV files to avoid re-querying and ensure reproducibility +- **Estimate size first** - Check collection size before downloading - some collection sizes are in terabytes! +- **Save manifests** - Always save query results with Series UIDs for reproducibility and data provenance +- **Read documentation** - IDC data structure and metadata fields are documented at https://learn.canceridc.dev/ +- **Use IDC forum** - Search for questons/answers and ask your questions to the IDC maintainers and users at https://discourse.canceridc.dev/ + +## Troubleshooting + +**Issue: `ModuleNotFoundError: No module named 'idc_index'`** +- **Cause:** idc-index package not installed +- **Solution:** Install with `pip install --upgrade idc-index` + +**Issue: Download fails with connection timeout** +- **Cause:** Network instability or large download size +- **Solution:** + - Download smaller batches (e.g., 10-20 series at a time) + - Check network connection + - Use `dirTemplate` to organize downloads by batch + - Implement retry logic with delays + +**Issue: `BigQuery quota exceeded` or billing errors** +- **Cause:** BigQuery requires billing-enabled GCP project +- **Solution:** Use idc-index mini-index for simple queries (no billing required), or see `references/bigquery_guide.md` for cost optimization tips + +**Issue: Series UID not found or no data returned** +- **Cause:** Typo in UID, data not in current IDC version, or wrong field name +- **Solution:** + - Check if data is in current IDC version (some old data may be deprecated) + - Use `LIMIT 5` to test query first + - Check field names against metadata schema documentation + +**Issue: Downloaded DICOM files won't open** +- **Cause:** Corrupted download or incompatible viewer +- **Solution:** + - Check DICOM object type (Modality and SOPClassUID attributes) - some object types require specialized tools + - Verify file integrity (check file sizes) + - Use pydicom to validate: `pydicom.dcmread(file, force=True)` + - Try different DICOM viewer (3D Slicer, Horos, RadiAnt, QuPath) + - Re-download the series -After completing this tutorial, you hopefully: -* developed basic understanding of the IDC image metadata and its organization -* learned about `idc-index` as the tool for searching IDC metadata -* are motivated to start experimenting with the SQL interface to select subsets of IDC data at different levels of data model (collection, patient, study, series) -* understood how to script download of the data available in IDC +## Common SQL Query Patterns + +See `references/sql_patterns.md` for quick-reference SQL patterns including: +- Filter value discovery (modalities, body parts, manufacturers) +- Annotation and segmentation queries (including seg_index, ann_index joins) +- Slide microscopy queries (sm_index patterns) +- Download size estimation +- Clinical data linking + +For segmentation and annotation details, also see `references/digital_pathology_guide.md`. + +## Related Skills + +The following skills complement IDC workflows for downstream analysis and visualization: + +### DICOM Processing +- **pydicom** - Read, write, and manipulate downloaded DICOM files. Use for extracting pixel data, reading metadata, anonymization, and format conversion. Essential for working with IDC radiology data (CT, MR, PET). + +### Pathology and Slide Microscopy +See `references/digital_pathology_guide.md` for DICOM-compatible tools (highdicom, wsidicom, TIA-Toolbox, Slim viewer). + +### Metadata Visualization +- **matplotlib** - Low-level plotting for full customization. Use for creating static figures summarizing IDC query results (bar charts of modalities, histograms of series counts, etc.). +- **seaborn** - Statistical visualization with pandas integration. Use for quick exploration of IDC metadata distributions, relationships between variables, and categorical comparisons with attractive defaults. +- **plotly** - Interactive visualization. Use when you need hover info, zoom, and pan for exploring IDC metadata, or for creating web-embeddable dashboards of collection statistics. + +### Data Exploration +- **exploratory-data-analysis** - Comprehensive EDA on scientific data files. Use after downloading IDC data to understand file structure, quality, and characteristics before analysis. + +## Resources + +### Schema Reference (Primary Source) + +**Always use `client.indices_overview` for current column schemas.** This ensures accuracy with the installed idc-index version: + +```python +# Get all column names and types for any table +schema = client.indices_overview["index"]["schema"] +columns = [(c['name'], c['type'], c.get('description', '')) for c in schema['columns']] +``` -If you have any questions about this tutorial, or about searching IDC metadata, please send us an email to support@canceridc.dev or posting your question on [IDC User forum](https://discourse.cancer.dev)! +### Reference Documentation -This tutorial barely scratches the surface of what you can do with BigQuery SQL. If you are interested in a comprehensive tutorial about BigQuery SQL, check out this ["Intro to SQL" course on Kaggle](https://www.kaggle.com/learn/intro-to-sql)! +See the Quick Navigation section at the top for the full list of reference guides with decision triggers. -If you are interested to do a deeper dive into SQL and experiment with IDC BigQuery index, check out the "original" ["Getting Started with IDC"](https://github.com/ImagingDataCommons/IDC-Tutorials/tree/master/notebooks/getting_started) series that utilizes BigQuery! +- **[indices_reference](https://idc-index.readthedocs.io/en/latest/indices_reference.html)** - External documentation for index tables (may be ahead of the installed version) -## Acknowledgments +### External Links -Imaging Data Commons has been funded in whole or in part with Federal funds from the National Cancer Institute, National Institutes of Health, under Task Order No. HHSN26110071 under Contract No. HHSN261201500003l. +- **IDC Portal**: https://portal.imaging.datacommons.cancer.gov/explore/ +- **Documentation**: https://learn.canceridc.dev/ +- **Tutorials**: https://github.com/ImagingDataCommons/IDC-Tutorials +- **User Forum**: https://discourse.canceridc.dev/ +- **idc-index GitHub**: https://github.com/ImagingDataCommons/idc-index +- **Citation**: Fedorov, A., et al. "National Cancer Institute Imaging Data Commons: Toward Transparency, Reproducibility, and Scalability in Imaging Artificial Intelligence." RadioGraphics 43.12 (2023). https://doi.org/10.1148/rg.230180 -If you use IDC in your research, please cite the following publication: +### Skill Updates -> Fedorov, A., Longabaugh, W. J. R., Pot, D., Clunie, D. A., Pieper, S. D., Gibbs, D. L., Bridge, C., Herrmann, M. D., Homeyer, A., Lewis, R., Aerts, H. J. W., Krishnaswamy, D., Thiriveedhi, V. K., Ciausu, C., Schacherer, D. P., Bontempi, D., Pihl, T., Wagner, U., Farahani, K., Kim, E. & Kikinis, R. _National Cancer Institute Imaging Data Commons: Toward Transparency, Reproducibility, and Scalability in Imaging Artificial Intelligence_. RadioGraphics (2023). https://doi.org/10.1148/rg.230180 \ No newline at end of file +This skill version is available in skill metadata. To check for updates: +- Visit the [releases page](https://github.com/ImagingDataCommons/idc-claude-skill/releases) +- Watch the repository on GitHub (Watch → Custom → Releases) diff --git a/docker-compose.yaml b/docker-compose.yaml index 772d46f..ebc2d94 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -26,6 +26,7 @@ services: - PYTHONBREAKPOINT=debugpy.breakpoint - BIOME_LIT_REVIEW_DIR=/jupyter/_literature_review - INTEGRATION_PATH=/jupyter/adhoc_data + - IDC_SKILL_REF=main # idc-claude-skill branch/tag (e.g. main or v1.4.0) env_file: - .env volumes: diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..c8ee0ac --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,18 @@ +#!/bin/sh +set -e + +# Fetch latest IDC skill documentation at startup. +# adhoc_data is bind-mounted from the host, so this must happen at runtime rather than during build. +IDC_SKILL_URL="https://raw.githubusercontent.com/ImagingDataCommons/idc-claude-skill/${IDC_SKILL_REF:-main}/SKILL.md" +IDC_MD_PATH="/jupyter/adhoc_data/specifications/idc/attachments/idc.md" + +echo "Fetching IDC skill documentation (ref: ${IDC_SKILL_REF:-main})..." +if curl -sf "${IDC_SKILL_URL}" -o "${IDC_MD_PATH}.tmp"; then + mv "${IDC_MD_PATH}.tmp" "${IDC_MD_PATH}" + echo "IDC skill documentation updated." +else + echo "Warning: could not fetch IDC skill docs, using cached version." + rm -f "${IDC_MD_PATH}.tmp" +fi + +exec python -m beaker_kernel.service.server --ip 0.0.0.0 diff --git a/src/biome/context.py b/src/biome/context.py index afcb13c..318701d 100644 --- a/src/biome/context.py +++ b/src/biome/context.py @@ -26,18 +26,18 @@ class BiomeContext(BeakerContext): agent_cls: "BaseAgent" = BiomeAgent def __init__(self, beaker_kernel: "LLMKernel", config: Dict[str, Any]): - from adhoc_api.uaii import gpt_41, o3_mini, claude_37_sonnet, gemini_15_pro + from adhoc_api.uaii import gpt_52, o3_mini, claude_46_sonnet, gemini_3_flash ttl_seconds = 1800 - drafter_config_gemini = {**gemini_15_pro, 'ttl_seconds': ttl_seconds, 'api_key': os.environ.get("GEMINI_API_KEY", "")} - drafter_config_anthropic = {**claude_37_sonnet, 'api_key': os.environ.get("ANTHROPIC_API_KEY")} + drafter_config_anthropic = {**claude_46_sonnet, 'api_key': os.environ.get("ANTHROPIC_API_KEY")} + drafter_config_gpt = {**gpt_52, 'api_key': os.environ.get("OPENAI_API_KEY")} + drafter_config_gemini = {**gemini_3_flash, 'ttl_seconds': ttl_seconds, 'api_key': os.environ.get("GEMINI_API_KEY", "")} curator_config = {**o3_mini, 'api_key': os.environ.get("OPENAI_API_KEY")} - gpt_41_config = {**gpt_41, 'api_key': os.environ.get("OPENAI_API_KEY")} # Initialize adhoc integration adhoc_integration = BiomeAdhocIntegrations( - drafter_config=[gpt_41_config, drafter_config_anthropic, drafter_config_gemini], + drafter_config=[drafter_config_anthropic, drafter_config_gpt, drafter_config_gemini], curator_config=curator_config, - contextualizer_config=gpt_41_config, + contextualizer_config=drafter_config_gpt, logger=logger, display_name="Specialist Agents" )