diff --git a/Dockerfile b/Dockerfile index 249d9b1..2183f88 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ RUN pip install \ requests \ google-generativeai \ PyYAML \ - adhoc-api~=1.0.0 \ + adhoc-api~=2.0.2 \ idc-index \ seaborn \ biopython \ diff --git a/pyproject.toml b/pyproject.toml index e0bdbef..f0d32ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ "requests", "google-generativeai", "PyYAML", - "adhoc-api~=1.0.0", + "adhoc-api~=2.0.2", "idc-index", "seaborn", "biopython", diff --git a/src/biome/agent.py b/src/biome/agent.py index 1b6b579..1994aea 100644 --- a/src/biome/agent.py +++ b/src/biome/agent.py @@ -18,6 +18,7 @@ from pathlib import Path from adhoc_api.tool import AdhocApi from adhoc_api.loader import load_yaml_api +from adhoc_api.uaii import gpt_4o, o3_mini, claude_35_sonnet, gemini_pro logger = logging.getLogger(__name__) @@ -82,8 +83,10 @@ def __init__(self, context: BaseContext = None, tools: list = None, **kwargs): # Note: not all providers support ttl_seconds ttl_seconds = 1800 - drafter_config_gemini={'provider': 'google', 'model': 'gemini-1.5-pro-001', 'ttl_seconds': ttl_seconds, 'api_key': os.environ.get("GEMINI_API_KEY", "")} - drafter_config_anthropic={'provider': 'anthropic', 'model': 'claude-3-5-sonnet-latest', 'api_key': os.environ.get("ANTHROPIC_API_KEY")} + drafter_config_gemini = {**gemini_pro, 'ttl_seconds': ttl_seconds, 'api_key': os.environ.get("GEMINI_API_KEY", "")} + drafter_config_anthropic = {**claude_35_sonnet, 'api_key': os.environ.get("ANTHROPIC_API_KEY")} + curator_config = {**o3_mini, 'api_key': os.environ.get("OPENAI_API_KEY")} + contextualizer_config = {**gpt_4o, 'api_key': os.environ.get("OPENAI_API_KEY")} specs = self.api_specs instructions_dir = os.path.join(self.root_folder, 'instructions') @@ -102,7 +105,13 @@ def __init__(self, context: BaseContext = None, tools: list = None, **kwargs): self.logger = MessageLogger(self.context) try: - self.api = AdhocApi(logger=logger, drafter_config=[drafter_config_anthropic, drafter_config_gemini], apis=specs) + self.api = AdhocApi( + apis=specs, + drafter_config=[drafter_config_anthropic, drafter_config_gemini], + curator_config=curator_config, + contextualizer_config=contextualizer_config, + logger=logger, + ) except ValueError as e: self.add_context(f"The APIs failed to load for this reason: {str(e)}. Please inform the user immediately.") self.api = None diff --git a/src/biome/api_agent.yaml b/src/biome/api_agent.yaml index ca29ae3..8b0d20d 100644 --- a/src/biome/api_agent.yaml +++ b/src/biome/api_agent.yaml @@ -4,7 +4,7 @@ drafter: ttl_seconds: 1800 finalizer: model: gpt-4o -default_cache_body: | +default_cache_body: !fill | You will be given the entire API documentation. When you write code against this API, you should avail yourself of the appropriate query parameters, your understanding of the response model, and be cognizant that not all data is public and thus may require a token, etc. diff --git a/src/biome/api_definitions/cbioportal/api.yaml b/src/biome/api_definitions/cbioportal/api.yaml index 49f9929..f634f58 100644 --- a/src/biome/api_definitions/cbioportal/api.yaml +++ b/src/biome/api_definitions/cbioportal/api.yaml @@ -10,11 +10,11 @@ description: | Institute, Princess Margaret Cancer Centre in Toronto, Children's Hospital of Philadelphia, Caris Life Sciences, The Hyve and SE4BIO in the Netherlands, and Bilkent University in Ankara, Turkey. -raw_documentation: !load documentation/cbioportal.json -examples: !load documentation/examples.md +raw_documentation: !load_txt documentation/cbioportal.json +examples: !load_yaml documentation/examples.yaml cache_body: default: true -documentation: | +documentation: !fill | {raw_documentation} # Additional Instructions: @@ -24,7 +24,4 @@ documentation: | That is the base URL for all requests. There are not very many studies (less than 1000) so if the user is searching for a specific study, you can just fetch all studies and then filter the list yourself since cbioportals filters are tricky. It is often most - productive to simply filter based on the study name. - - Here are some examples of how to use the API: - {examples} \ No newline at end of file + productive to simply filter based on the study name. \ No newline at end of file diff --git a/src/biome/api_definitions/cbioportal/documentation/examples.md b/src/biome/api_definitions/cbioportal/documentation/examples.md deleted file mode 100644 index 22aaba8..0000000 --- a/src/biome/api_definitions/cbioportal/documentation/examples.md +++ /dev/null @@ -1,602 +0,0 @@ -# Examples - - -## Example 1: Query and display all colorectal cancer studies from cBioPortal. This example searches for studies with cancer type IDs 'coadread', 'coad', and 'read' (representing colorectal adenocarcinoma, colon adenocarcinoma, and rectal adenocarcinoma respectively). For each study, it displays the name, study ID, description, and cancer type ID. - -``` -import requests - -url = "https://www.cbioportal.org/api/studies" -response = requests.get(url) - -if response.status_code == 200: - studies = response.json() - colorectal_studies = [study for study in studies if study['cancerTypeId'].lower() in ['coadread', 'coad', 'read']] - print(f"Found {len(colorectal_studies)} colorectal cancer studies.") - print("\nStudy Details:") - for study in colorectal_studies: - print(f"\nName: {study['name']}") - print(f"Study ID: {study['studyId']}") - print(f"Description: {study['description']}") - print(f"Cancer Type ID: {study['cancerTypeId']}") - print("-" * 80) -``` - -## Example 2: Query mutation data from cBioPortal for a specific study (Genentech colorectal cancer study) and analyze mutations in key cancer genes (APC, TP53, KRAS, PIK3CA). The example demonstrates how to: -1. Get sample IDs for a study -2. Fetch mutation data using the molecular profiles endpoint -3. Filter and analyze mutations for specific genes -4. Count mutation types and protein changes -The code provides a detailed breakdown of mutation types and frequencies for each gene. - -``` -import requests -import json - -# Get samples for the study -study_id = "coadread_genentech" -samples_url = f"https://www.cbioportal.org/api/studies/{study_id}/samples" -samples_response = requests.get(samples_url) - -if samples_response.status_code == 200: - samples = json.loads(samples_response.content) - sample_ids = [sample['sampleId'] for sample in samples] - print(f"Found {len(sample_ids)} samples") - - # Get mutation data - mutations_url = f"https://www.cbioportal.org/api/molecular-profiles/{study_id}_mutations/mutations/fetch" - - # Create the filter with sample IDs - data = { - "sampleIds": sample_ids - } - - # Make the POST request - mutations_response = requests.post(mutations_url, json=data) - - if mutations_response.status_code == 200: - mutations = json.loads(mutations_response.content) - print(f"\nRetrieved {len(mutations)} mutations") - - # Focus on key cancer genes - key_genes = { - 'APC': 324, # EntrezGeneID for APC - 'TP53': 7157, # EntrezGeneID for TP53 - 'KRAS': 3845, # EntrezGeneID for KRAS - 'PIK3CA': 5290 # EntrezGeneID for PIK3CA - } - - # Analyze mutations for these genes - gene_details = {} - for gene_symbol, gene_id in key_genes.items(): - gene_mutations = [m for m in mutations if m['entrezGeneId'] == gene_id] - - # Count mutation types - mutation_types = {} - protein_changes = [] - - for mutation in gene_mutations: - mut_type = mutation['mutationType'] - if mut_type in mutation_types: - mutation_types[mut_type] += 1 - else: - mutation_types[mut_type] = 1 - - if mutation['proteinChange']: - protein_changes.append(mutation['proteinChange']) - - gene_details[gene_symbol] = { - 'total_mutations': len(gene_mutations), - 'mutation_types': mutation_types, - 'protein_changes': protein_changes - } - - # Print detailed analysis - print("\nDetailed Analysis of Key Cancer Genes:\n") - for gene, details in gene_details.items(): - print(f"\n{gene} Analysis:") - print(f"Total mutations: {details['total_mutations']}") - - print("Mutation types:") - for mut_type, count in details['mutation_types'].items(): - print(f" - {mut_type}: {count}") - - print("Protein changes (top 5):") - for change in details['protein_changes'][:5]: - print(f" - {change}") - if len(details['protein_changes']) > 5: - print(f" ... and {len(details['protein_changes'])-5} more changes") -``` - - -## Example 3: Query for studies related to colorectal cancer using the cancerTypeId 'coadread'. - -``` -import requests -import pandas as pd - -# cBioPortal API base URL -base_url = "https://www.cbioportal.org/api" - -# Endpoint to fetch studies -endpoint = "/studies" - -# Parameters for the query -params = {"cancerTypeId": "coadread"} - -# Make the API request -response = requests.get(base_url + endpoint, params=params) - -# Check for successful response -response.raise_for_status() - -# Parse the JSON response -studies = response.json() - -# Create a pandas DataFrame from the results -df = pd.DataFrame(studies) - -# Print the DataFrame -print(df) -``` - - -## Example 4: Fetch and filter studies related to Acute Myeloid Leukemia (AML) from cBioPortal API. - -``` -import requests -import pandas as pd - -def fetch_aml_studies(): - # Base URL for cBioPortal API - base_url = "https://www.cbioportal.org/api" - - # Get all studies - response = requests.get(f"{base_url}/studies") - - if response.status_code != 200: - raise Exception(f"API request failed with status code {response.status_code}") - - # Get all studies as JSON - studies = response.json() - - # Filter for AML related studies - aml_studies = [] - for study in studies: - # Convert all fields to lowercase for case-insensitive search - study_id = study.get('studyId', '').lower() - name = study.get('name', '').lower() - cancer_type = study.get('cancerTypeId', '').lower() - description = study.get('description', '').lower() - - # Check if 'aml' appears in any of the relevant fields - if any('aml' in field for field in [study_id, name, cancer_type, description]): - aml_studies.append({ - 'studyId': study['studyId'], - 'name': study['name'], - 'description': study.get('description', 'N/A'), - 'cancerTypeId': study.get('cancerTypeId', 'N/A'), - 'sampleCount': study.get('allSampleCount', 0), - 'status': study.get('status', 'N/A'), - 'publicStudy': study.get('publicStudy', False) - }) - - # Convert to DataFrame for better display - df = pd.DataFrame(aml_studies) - - # Sort by sample count descending - df = df.sort_values('sampleCount', ascending=False) - - return df - -# Execute the function -aml_studies_df = fetch_aml_studies() -print(f"Found {len(aml_studies_df)} AML-related studies") -print(aml_studies_df) -``` - - -## Example 5: This example demonstrates how to fetch and analyze mutation data for a specific study in cBioPortal. It shows: -1. How to properly construct the API endpoint URL with sample list ID -2. How to fetch mutation data for a study -3. How to get gene symbol information using entrez IDs -4. How to create a comprehensive mutation analysis including: - - Total mutation counts - - Gene frequency analysis - - Mutation type distribution - - Sample statistics -The example uses the TARGET AML (GDC) study but can be modified for any study by changing the study_id. - -``` -import requests -import pandas as pd -from collections import Counter -import numpy as np - -# Base URL for cBioPortal API -base_url = "https://www.cbioportal.org/api" -study_id = "aml_target_gdc" # Example study ID - -# Get mutations using the sample list ID -mutations_url = f"{base_url}/molecular-profiles/{study_id}_mutations/mutations?sampleListId={study_id}_all" -mutations_response = requests.get(mutations_url) - -if mutations_response.status_code == 200: - mutations_data = mutations_response.json() - mutations_df = pd.DataFrame(mutations_data) - - # Get gene information for the entrez IDs - entrez_ids = [int(x) for x in mutations_df['entrezGeneId'].unique()] # Convert to regular Python integers - genes_url = f"{base_url}/genes/fetch" - genes_response = requests.post(genes_url, json=entrez_ids) - - if genes_response.status_code == 200: - genes_data = genes_response.json() - gene_map = {gene['entrezGeneId']: gene['hugoGeneSymbol'] for gene in genes_data} - - # Add gene symbols to mutations dataframe - mutations_df['gene_symbol'] = mutations_df['entrezGeneId'].map(gene_map) - - # Print summary statistics - print(f"Total mutations found: {len(mutations_df)}") - print(f"Number of affected genes: {len(mutations_df['gene_symbol'].unique())}") - - print("\nMost frequently mutated genes:") - print(mutations_df['gene_symbol'].value_counts().head(10)) - - print("\nMutation types distribution:") - print(mutations_df['mutationType'].value_counts()) - - print("\nSample of mutations (first 5 rows):") - display_cols = ['gene_symbol', 'sampleId', 'mutationType', 'proteinChange', 'chr', 'startPosition', 'variantType'] - print(mutations_df[display_cols].head()) - - # Calculate sample statistics - samples_with_mutations = len(mutations_df['sampleId'].unique()) - print(f"\nNumber of samples with mutations: {samples_with_mutations}") - - avg_mutations_per_sample = len(mutations_df) / samples_with_mutations - print(f"Average mutations per sample: {avg_mutations_per_sample:.2f}") -else: - print(f"Error fetching mutations: {mutations_response.status_code}") - print("Response:", mutations_response.text) -``` - - -## Example 6: Retrieve and analyze mutation data from multiple AML studies, including data processing and summary statistics. This example demonstrates how to fetch mutations across multiple studies, handle the nested JSON response, and create a comprehensive mutation dataset with key information. - -``` -import requests -import pandas as pd -import json - -# Base URL -BASE_URL = "https://www.cbioportal.org/api" - -# List of AML studies -AML_STUDIES = [ - 'aml_target_2018_pub', - 'aml_ohsu_2018', - 'aml_ohsu_2022', - 'laml_tcga', - 'laml_tcga_pan_can_atlas_2018', - 'laml_tcga_gdac', - 'aml_target_gdc', - 'washu_pdi_2016', - 'laml_tcga_pub' -] - -# Function to get mutations for a study -def get_study_mutations(study_id): - print(f"\nProcessing study: {study_id}") - - # Get the molecular profile ID - molecular_profile_id = f"{study_id}_mutations" - - # Get the sample list ID - sample_list_id = f"{study_id}_all" - - # Fetch mutations - mutations_response = requests.get( - f"{BASE_URL}/molecular-profiles/{molecular_profile_id}/mutations", - params={"sampleListId": sample_list_id, "projection": "DETAILED"} - ) - - if mutations_response.status_code == 200: - mutations = mutations_response.json() - print(f"Found {len(mutations)} mutations") - - # Convert to DataFrame - mutations_df = pd.DataFrame(mutations) - - # Extract gene symbols from the nested dictionary - mutations_df['gene_symbol'] = mutations_df['gene'].apply(lambda x: x['hugoGeneSymbol']) - - # Create a more focused DataFrame with key columns - focused_df = mutations_df[[ - 'gene_symbol', - 'sampleId', - 'proteinChange', - 'mutationType', - 'chr', - 'startPosition', - 'endPosition', - 'referenceAllele', - 'variantAllele' - ]] - - return focused_df - else: - print(f"Error fetching mutations: {mutations_response.text}") - return None - -# Process all studies -all_mutations_dfs = [] - -for study_id in AML_STUDIES: - study_df = get_study_mutations(study_id) - if study_df is not None: - study_df['study_id'] = study_id # Add study ID column - all_mutations_dfs.append(study_df) - -# Combine all mutations -if all_mutations_dfs: - combined_df = pd.concat(all_mutations_dfs, ignore_index=True) - - print("\nOverall mutation data summary:") - print(f"Total number of mutations: {len(combined_df)}") - print(f"Number of unique genes: {combined_df['gene_symbol'].nunique()}") - print(f"Number of unique samples: {combined_df['sampleId'].nunique()}") - - print("\nTop 20 most frequently mutated genes across all studies:") - print(combined_df['gene_symbol'].value_counts().head(20)) - - print("\nDistribution of mutation types:") - print(combined_df['mutationType'].value_counts()) - - print("\nNumber of mutations by study:") - print(combined_df['study_id'].value_counts()) - - # Save to CSV - combined_df.to_csv('all_aml_mutations.csv', index=False) -``` - - -## Example 7: Example showing how to retrieve and analyze the molecular profiles available for multiple studies from cBioPortal. The code demonstrates how to fetch metadata on the molecular profiles, organize them into DataFrames, and analyze specific information about each profile like the types of profiles available such as RNA expression. - -``` -import requests -import pandas as pd -from typing import List, Dict - -def get_molecular_profiles(study_ids: List[str]) -> Dict[str, pd.DataFrame]: - """ - Retrieve molecular profiles for multiple studies from cBioPortal. - - Args: - study_ids: List of cBioPortal study IDs - - Returns: - Dictionary mapping study IDs to DataFrames containing their molecular profiles - """ - # Base URL for cBioPortal's web API - base_url = "https://www.cbioportal.org/api" - - # Dictionary to store results - results = {} - - print("Fetching molecular profiles...") - for study_id in study_ids: - print(f"\nStudy: {study_id}") - print("-" * 40) - - # Get molecular profiles for this study - response = requests.get(f"{base_url}/studies/{study_id}/molecular-profiles") - - if response.status_code == 200: - profiles = response.json() - - # Create a summary DataFrame - profile_data = [] - for profile in profiles: - profile_data.append({ - 'molecularProfileId': profile['molecularProfileId'], - 'name': profile['name'], - 'datatype': profile['datatype'], - 'molecularAlterationType': profile['molecularAlterationType'], - 'description': profile.get('description', 'N/A') - }) - - if profile_data: - df = pd.DataFrame(profile_data) - results[study_id] = df - print(f"Found {len(df)} molecular profiles") - - # Display summary of profile types - print("\nProfile types:") - type_summary = df.groupby('molecularAlterationType')['molecularProfileId'].count() - print(type_summary) - else: - print("No molecular profiles found") - results[study_id] = pd.DataFrame() - else: - print(f"Error accessing study: {response.status_code}") - results[study_id] = pd.DataFrame() - - return results - -# Example usage with AML studies -aml_studies = [ - 'aml_target_2018_pub', - 'aml_ohsu_2018', - 'aml_ohsu_2022', - 'laml_tcga', - 'laml_tcga_pan_can_atlas_2018', - 'aml_target_gdc', - 'mnm_washu_2016' -] - -# Get molecular profiles for all studies -profiles_by_study = get_molecular_profiles(aml_studies) - -# Example of how to work with the results - get RNA expression profiles for first study -first_study = list(profiles_by_study.keys())[0] -if not profiles_by_study[first_study].empty: - rna_profiles = profiles_by_study[first_study][ - profiles_by_study[first_study]['molecularAlterationType'] == 'MRNA_EXPRESSION' - ] - print(f"\nRNA expression profiles for {first_study}:") - print(rna_profiles[['molecularProfileId', 'name', 'datatype']].to_string()) -``` - - -## Example 8: Example of how to fetch RNA-seq z-scores for STAT5A and STAT5B across multiple AML studies. This example demonstrates: -1. How to properly fetch sample IDs for each study -2. How to use these sample IDs when requesting molecular data -3. How to handle large requests by processing samples in chunks -4. How to combine data from multiple studies into a single dataframe -5. How to create both long and wide format versions of the data - -Key points: -- Must provide explicit sample IDs in the molecular data request -- Z-scores are pre-calculated by cBioPortal for each study -- Different studies may use different measurement types (TPM, RPKM, RSEM) -- Processing in chunks helps avoid timeouts with large datasets - -``` -import requests -import pandas as pd -import time - -# Configuration -BASE_URL = "https://www.cbioportal.org/api" - -# Define studies and their RNA-seq profiles -STUDY_PROFILES = { - 'aml_target_gdc': { - 'name': 'TARGET-AML (GDC)', - 'profile': 'aml_target_gdc_mrna_seq_tpm_Zscores', - 'type': 'TPM' - }, - 'aml_ohsu_2022': { - 'name': 'OHSU AML 2022', - 'profile': 'aml_ohsu_2022_mrna_median_Zscores', - 'type': 'RPKM' - } - # ... other studies ... -} - -# Define STAT5 genes with their Entrez IDs -STAT5_GENES = { - 6776: 'STAT5A', - 6777: 'STAT5B' -} - -# Function to get RNA-seq samples for a study -def get_rna_seq_samples(study_id): - """Get list of sample IDs that have RNA-seq data for a study.""" - url = f"{BASE_URL}/studies/{study_id}/sample-lists" - response = requests.get(url) - if response.status_code == 200: - sample_lists = response.json() - # Find RNA-seq sample list - rna_list = next((sl['sampleListId'] for sl in sample_lists - if sl['category'] == 'all_cases_with_mrna_rnaseq_data'), None) - if rna_list: - # Get sample IDs from the list - url = f"{BASE_URL}/sample-lists/{rna_list}/sample-ids" - response = requests.get(url) - if response.status_code == 200: - return response.json() - return None - -# Function to get expression data -def get_expression_data(profile_id, gene_ids, sample_ids): - """Fetch expression z-scores for specific genes and samples.""" - url = f"{BASE_URL}/molecular-profiles/{profile_id}/molecular-data/fetch" - # IMPORTANT: Must provide sample IDs explicitly - data = { - "sampleIds": sample_ids, # List of specific sample IDs - "entrezGeneIds": gene_ids # List of Entrez gene IDs - } - response = requests.post(url, json=data) - if response.status_code == 200: - return response.json() - return None - -# Process each study -all_study_data = [] - -for study_id, study_info in STUDY_PROFILES.items(): - print(f"\nProcessing {study_info['name']}...") - - # First get the sample IDs for the study - sample_ids = get_rna_seq_samples(study_id) - - if sample_ids: - print(f"Found {len(sample_ids)} RNA-seq samples") - - # Process samples in chunks to avoid large requests - chunk_size = 100 - study_data = [] - - for i in range(0, len(sample_ids), chunk_size): - chunk = sample_ids[i:i + chunk_size] - print(f"Fetching data for samples {i+1}-{i+len(chunk)}...") - - # Get expression data for this chunk of samples - expression_data = get_expression_data( - study_info['profile'], - list(STAT5_GENES.keys()), - chunk - ) - - if expression_data: - study_data.extend(expression_data) - time.sleep(0.2) # Small delay between requests - - if study_data: - # Convert to DataFrame - df = pd.DataFrame(study_data) - - # Add metadata - df['gene_symbol'] = df['entrezGeneId'].map(STAT5_GENES) - df['study_id'] = study_id - df['study_name'] = study_info['name'] - df['measurement_type'] = study_info['type'] - - # Select and rename columns - df = df[[ - 'study_id', 'study_name', 'measurement_type', - 'gene_symbol', 'sampleId', 'value' - ]] - df.columns = [ - 'study_id', 'study_name', 'measurement_type', - 'gene', 'sample_id', 'zscore' - ] - - all_study_data.append(df) - -# Combine all data -if all_study_data: - # Create long format - combined_df = pd.concat(all_study_data, ignore_index=True) - - # Create wide format (samples as rows, genes as columns) - combined_wide = combined_df.pivot_table( - index=['study_id', 'study_name', 'measurement_type', 'sample_id'], - columns='gene', - values='zscore' - ).reset_index() - - # Save both versions - stat5_all_studies = combined_df # Long format - stat5_all_studies_wide = combined_wide # Wide format - -# Example of the resulting data structure: -print("\nLong format example (first few rows):") -print(stat5_all_studies.head()) - -print("\nWide format example (first few rows):") -print(stat5_all_studies_wide.head()) -``` diff --git a/src/biome/api_definitions/cbioportal/documentation/examples.yaml b/src/biome/api_definitions/cbioportal/documentation/examples.yaml new file mode 100644 index 0000000..ab1167f --- /dev/null +++ b/src/biome/api_definitions/cbioportal/documentation/examples.yaml @@ -0,0 +1,589 @@ +- query: "Query and display all colorectal cancer studies from cBioPortal. This example searches for studies with cancer type IDs 'coadread', 'coad', and 'read' (representing colorectal adenocarcinoma, colon adenocarcinoma, and rectal adenocarcinoma respectively). For each study, it displays the name, study ID, description, and cancer type ID." + code: | + import requests + + url = "https://www.cbioportal.org/api/studies" + response = requests.get(url) + + if response.status_code == 200: + studies = response.json() + colorectal_studies = [study for study in studies if study['cancerTypeId'].lower() in ['coadread', 'coad', 'read']] + print(f"Found {len(colorectal_studies)} colorectal cancer studies.") + print("\nStudy Details:") + for study in colorectal_studies: + print(f"\nName: {study['name']}") + print(f"Study ID: {study['studyId']}") + print(f"Description: {study['description']}") + print(f"Cancer Type ID: {study['cancerTypeId']}") + print("-" * 80) + +- query: "Query mutation data from cBioPortal for a specific study (Genentech colorectal cancer study) and analyze mutations in key cancer genes (APC, TP53, KRAS, PIK3CA)" + notes: | + The example demonstrates how to: + 1. Get sample IDs for a study + 2. Fetch mutation data using the molecular profiles endpoint + 3. Filter and analyze mutations for specific genes + 4. Count mutation types and protein changes + The code provides a detailed breakdown of mutation types and frequencies for each gene. + code: | + import requests + import json + + # Get samples for the study + study_id = "coadread_genentech" + samples_url = f"https://www.cbioportal.org/api/studies/{study_id}/samples" + samples_response = requests.get(samples_url) + + if samples_response.status_code == 200: + samples = json.loads(samples_response.content) + sample_ids = [sample['sampleId'] for sample in samples] + print(f"Found {len(sample_ids)} samples") + + # Get mutation data + mutations_url = f"https://www.cbioportal.org/api/molecular-profiles/{study_id}_mutations/mutations/fetch" + + # Create the filter with sample IDs + data = { + "sampleIds": sample_ids + } + + # Make the POST request + mutations_response = requests.post(mutations_url, json=data) + + if mutations_response.status_code == 200: + mutations = json.loads(mutations_response.content) + print(f"\nRetrieved {len(mutations)} mutations") + + # Focus on key cancer genes + key_genes = { + 'APC': 324, # EntrezGeneID for APC + 'TP53': 7157, # EntrezGeneID for TP53 + 'KRAS': 3845, # EntrezGeneID for KRAS + 'PIK3CA': 5290 # EntrezGeneID for PIK3CA + } + + # Analyze mutations for these genes + gene_details = {} + for gene_symbol, gene_id in key_genes.items(): + gene_mutations = [m for m in mutations if m['entrezGeneId'] == gene_id] + + # Count mutation types + mutation_types = {} + protein_changes = [] + + for mutation in gene_mutations: + mut_type = mutation['mutationType'] + if mut_type in mutation_types: + mutation_types[mut_type] += 1 + else: + mutation_types[mut_type] = 1 + + if mutation['proteinChange']: + protein_changes.append(mutation['proteinChange']) + + gene_details[gene_symbol] = { + 'total_mutations': len(gene_mutations), + 'mutation_types': mutation_types, + 'protein_changes': protein_changes + } + + # Print detailed analysis + print("\nDetailed Analysis of Key Cancer Genes:\n") + for gene, details in gene_details.items(): + print(f"\n{gene} Analysis:") + print(f"Total mutations: {details['total_mutations']}") + + print("Mutation types:") + for mut_type, count in details['mutation_types'].items(): + print(f" - {mut_type}: {count}") + + print("Protein changes (top 5):") + for change in details['protein_changes'][:5]: + print(f" - {change}") + if len(details['protein_changes']) > 5: + print(f" ... and {len(details['protein_changes'])-5} more changes") + + +- query: "Query for studies related to colorectal cancer using the cancerTypeId 'coadread'." + code: | + import requests + import pandas as pd + + # cBioPortal API base URL + base_url = "https://www.cbioportal.org/api" + + # Endpoint to fetch studies + endpoint = "/studies" + + # Parameters for the query + params = {"cancerTypeId": "coadread"} + + # Make the API request + response = requests.get(base_url + endpoint, params=params) + + # Check for successful response + response.raise_for_status() + + # Parse the JSON response + studies = response.json() + + # Create a pandas DataFrame from the results + df = pd.DataFrame(studies) + + # Print the DataFrame + print(df) + + +- query: "Fetch and filter studies related to Acute Myeloid Leukemia (AML) from cBioPortal API." + code: | + import requests + import pandas as pd + + def fetch_aml_studies(): + # Base URL for cBioPortal API + base_url = "https://www.cbioportal.org/api" + + # Get all studies + response = requests.get(f"{base_url}/studies") + + if response.status_code != 200: + raise Exception(f"API request failed with status code {response.status_code}") + + # Get all studies as JSON + studies = response.json() + + # Filter for AML related studies + aml_studies = [] + for study in studies: + # Convert all fields to lowercase for case-insensitive search + study_id = study.get('studyId', '').lower() + name = study.get('name', '').lower() + cancer_type = study.get('cancerTypeId', '').lower() + description = study.get('description', '').lower() + + # Check if 'aml' appears in any of the relevant fields + if any('aml' in field for field in [study_id, name, cancer_type, description]): + aml_studies.append({ + 'studyId': study['studyId'], + 'name': study['name'], + 'description': study.get('description', 'N/A'), + 'cancerTypeId': study.get('cancerTypeId', 'N/A'), + 'sampleCount': study.get('allSampleCount', 0), + 'status': study.get('status', 'N/A'), + 'publicStudy': study.get('publicStudy', False) + }) + + # Convert to DataFrame for better display + df = pd.DataFrame(aml_studies) + + # Sort by sample count descending + df = df.sort_values('sampleCount', ascending=False) + + return df + + # Execute the function + aml_studies_df = fetch_aml_studies() + print(f"Found {len(aml_studies_df)} AML-related studies") + print(aml_studies_df) + + +- query: "This example demonstrates how to fetch and analyze mutation data for a specific study in cBioPortal" + notes: | + It shows: + 1. How to properly construct the API endpoint URL with sample list ID + 2. How to fetch mutation data for a study + 3. How to get gene symbol information using entrez IDs + 4. How to create a comprehensive mutation analysis including: + - Total mutation counts + - Gene frequency analysis + - Mutation type distribution + - Sample statistics + The example uses the TARGET AML (GDC) study but can be modified for any study by changing the study_id. + code: | + import requests + import pandas as pd + from collections import Counter + import numpy as np + + # Base URL for cBioPortal API + base_url = "https://www.cbioportal.org/api" + study_id = "aml_target_gdc" # Example study ID + + # Get mutations using the sample list ID + mutations_url = f"{base_url}/molecular-profiles/{study_id}_mutations/mutations?sampleListId={study_id}_all" + mutations_response = requests.get(mutations_url) + + if mutations_response.status_code == 200: + mutations_data = mutations_response.json() + mutations_df = pd.DataFrame(mutations_data) + + # Get gene information for the entrez IDs + entrez_ids = [int(x) for x in mutations_df['entrezGeneId'].unique()] # Convert to regular Python integers + genes_url = f"{base_url}/genes/fetch" + genes_response = requests.post(genes_url, json=entrez_ids) + + if genes_response.status_code == 200: + genes_data = genes_response.json() + gene_map = {gene['entrezGeneId']: gene['hugoGeneSymbol'] for gene in genes_data} + + # Add gene symbols to mutations dataframe + mutations_df['gene_symbol'] = mutations_df['entrezGeneId'].map(gene_map) + + # Print summary statistics + print(f"Total mutations found: {len(mutations_df)}") + print(f"Number of affected genes: {len(mutations_df['gene_symbol'].unique())}") + + print("\nMost frequently mutated genes:") + print(mutations_df['gene_symbol'].value_counts().head(10)) + + print("\nMutation types distribution:") + print(mutations_df['mutationType'].value_counts()) + + print("\nSample of mutations (first 5 rows):") + display_cols = ['gene_symbol', 'sampleId', 'mutationType', 'proteinChange', 'chr', 'startPosition', 'variantType'] + print(mutations_df[display_cols].head()) + + # Calculate sample statistics + samples_with_mutations = len(mutations_df['sampleId'].unique()) + print(f"\nNumber of samples with mutations: {samples_with_mutations}") + + avg_mutations_per_sample = len(mutations_df) / samples_with_mutations + print(f"Average mutations per sample: {avg_mutations_per_sample:.2f}") + else: + print(f"Error fetching mutations: {mutations_response.status_code}") + print("Response:", mutations_response.text) + + +- query: "Retrieve and analyze mutation data from multiple AML studies, including data processing and summary statistics. This example demonstrates how to fetch mutations across multiple studies, handle the nested JSON response, and create a comprehensive mutation dataset with key information." + code: | + import requests + import pandas as pd + import json + + # Base URL + BASE_URL = "https://www.cbioportal.org/api" + + # List of AML studies + AML_STUDIES = [ + 'aml_target_2018_pub', + 'aml_ohsu_2018', + 'aml_ohsu_2022', + 'laml_tcga', + 'laml_tcga_pan_can_atlas_2018', + 'laml_tcga_gdac', + 'aml_target_gdc', + 'washu_pdi_2016', + 'laml_tcga_pub' + ] + + # Function to get mutations for a study + def get_study_mutations(study_id): + print(f"\nProcessing study: {study_id}") + + # Get the molecular profile ID + molecular_profile_id = f"{study_id}_mutations" + + # Get the sample list ID + sample_list_id = f"{study_id}_all" + + # Fetch mutations + mutations_response = requests.get( + f"{BASE_URL}/molecular-profiles/{molecular_profile_id}/mutations", + params={"sampleListId": sample_list_id, "projection": "DETAILED"} + ) + + if mutations_response.status_code == 200: + mutations = mutations_response.json() + print(f"Found {len(mutations)} mutations") + + # Convert to DataFrame + mutations_df = pd.DataFrame(mutations) + + # Extract gene symbols from the nested dictionary + mutations_df['gene_symbol'] = mutations_df['gene'].apply(lambda x: x['hugoGeneSymbol']) + + # Create a more focused DataFrame with key columns + focused_df = mutations_df[[ + 'gene_symbol', + 'sampleId', + 'proteinChange', + 'mutationType', + 'chr', + 'startPosition', + 'endPosition', + 'referenceAllele', + 'variantAllele' + ]] + + return focused_df + else: + print(f"Error fetching mutations: {mutations_response.text}") + return None + + # Process all studies + all_mutations_dfs = [] + + for study_id in AML_STUDIES: + study_df = get_study_mutations(study_id) + if study_df is not None: + study_df['study_id'] = study_id # Add study ID column + all_mutations_dfs.append(study_df) + + # Combine all mutations + if all_mutations_dfs: + combined_df = pd.concat(all_mutations_dfs, ignore_index=True) + + print("\nOverall mutation data summary:") + print(f"Total number of mutations: {len(combined_df)}") + print(f"Number of unique genes: {combined_df['gene_symbol'].nunique()}") + print(f"Number of unique samples: {combined_df['sampleId'].nunique()}") + + print("\nTop 20 most frequently mutated genes across all studies:") + print(combined_df['gene_symbol'].value_counts().head(20)) + + print("\nDistribution of mutation types:") + print(combined_df['mutationType'].value_counts()) + + print("\nNumber of mutations by study:") + print(combined_df['study_id'].value_counts()) + + # Save to CSV + combined_df.to_csv('all_aml_mutations.csv', index=False) + + +- query: "Example showing how to retrieve and analyze the molecular profiles available for multiple studies from cBioPortal. The code demonstrates how to fetch metadata on the molecular profiles, organize them into DataFrames, and analyze specific information about each profile like the types of profiles available such as RNA expression." + code: | + import requests + import pandas as pd + from typing import List, Dict + + def get_molecular_profiles(study_ids: List[str]) -> Dict[str, pd.DataFrame]: + """ + Retrieve molecular profiles for multiple studies from cBioPortal. + + Args: + study_ids: List of cBioPortal study IDs + + Returns: + Dictionary mapping study IDs to DataFrames containing their molecular profiles + """ + # Base URL for cBioPortal's web API + base_url = "https://www.cbioportal.org/api" + + # Dictionary to store results + results = {} + + print("Fetching molecular profiles...") + for study_id in study_ids: + print(f"\nStudy: {study_id}") + print("-" * 40) + + # Get molecular profiles for this study + response = requests.get(f"{base_url}/studies/{study_id}/molecular-profiles") + + if response.status_code == 200: + profiles = response.json() + + # Create a summary DataFrame + profile_data = [] + for profile in profiles: + profile_data.append({ + 'molecularProfileId': profile['molecularProfileId'], + 'name': profile['name'], + 'datatype': profile['datatype'], + 'molecularAlterationType': profile['molecularAlterationType'], + 'description': profile.get('description', 'N/A') + }) + + if profile_data: + df = pd.DataFrame(profile_data) + results[study_id] = df + print(f"Found {len(df)} molecular profiles") + + # Display summary of profile types + print("\nProfile types:") + type_summary = df.groupby('molecularAlterationType')['molecularProfileId'].count() + print(type_summary) + else: + print("No molecular profiles found") + results[study_id] = pd.DataFrame() + else: + print(f"Error accessing study: {response.status_code}") + results[study_id] = pd.DataFrame() + + return results + + # Example usage with AML studies + aml_studies = [ + 'aml_target_2018_pub', + 'aml_ohsu_2018', + 'aml_ohsu_2022', + 'laml_tcga', + 'laml_tcga_pan_can_atlas_2018', + 'aml_target_gdc', + 'mnm_washu_2016' + ] + + # Get molecular profiles for all studies + profiles_by_study = get_molecular_profiles(aml_studies) + + # Example of how to work with the results - get RNA expression profiles for first study + first_study = list(profiles_by_study.keys())[0] + if not profiles_by_study[first_study].empty: + rna_profiles = profiles_by_study[first_study][ + profiles_by_study[first_study]['molecularAlterationType'] == 'MRNA_EXPRESSION' + ] + print(f"\nRNA expression profiles for {first_study}:") + print(rna_profiles[['molecularProfileId', 'name', 'datatype']].to_string()) + + +- query: "Example of how to fetch RNA-seq z-scores for STAT5A and STAT5B across multiple AML studies" + notes: | + This example demonstrates: + 1. How to properly fetch sample IDs for each study + 2. How to use these sample IDs when requesting molecular data + 3. How to handle large requests by processing samples in chunks + 4. How to combine data from multiple studies into a single dataframe + 5. How to create both long and wide format versions of the data + + Key points: + - Must provide explicit sample IDs in the molecular data request + - Z-scores are pre-calculated by cBioPortal for each study + - Different studies may use different measurement types (TPM, RPKM, RSEM) + - Processing in chunks helps avoid timeouts with large datasets + code: | + import requests + import pandas as pd + import time + + # Configuration + BASE_URL = "https://www.cbioportal.org/api" + + # Define studies and their RNA-seq profiles + STUDY_PROFILES = { + 'aml_target_gdc': { + 'name': 'TARGET-AML (GDC)', + 'profile': 'aml_target_gdc_mrna_seq_tpm_Zscores', + 'type': 'TPM' + }, + 'aml_ohsu_2022': { + 'name': 'OHSU AML 2022', + 'profile': 'aml_ohsu_2022_mrna_median_Zscores', + 'type': 'RPKM' + } + # ... other studies ... + } + + # Define STAT5 genes with their Entrez IDs + STAT5_GENES = { + 6776: 'STAT5A', + 6777: 'STAT5B' + } + + # Function to get RNA-seq samples for a study + def get_rna_seq_samples(study_id): + """Get list of sample IDs that have RNA-seq data for a study.""" + url = f"{BASE_URL}/studies/{study_id}/sample-lists" + response = requests.get(url) + if response.status_code == 200: + sample_lists = response.json() + # Find RNA-seq sample list + rna_list = next((sl['sampleListId'] for sl in sample_lists + if sl['category'] == 'all_cases_with_mrna_rnaseq_data'), None) + if rna_list: + # Get sample IDs from the list + url = f"{BASE_URL}/sample-lists/{rna_list}/sample-ids" + response = requests.get(url) + if response.status_code == 200: + return response.json() + return None + + # Function to get expression data + def get_expression_data(profile_id, gene_ids, sample_ids): + """Fetch expression z-scores for specific genes and samples.""" + url = f"{BASE_URL}/molecular-profiles/{profile_id}/molecular-data/fetch" + # IMPORTANT: Must provide sample IDs explicitly + data = { + "sampleIds": sample_ids, # List of specific sample IDs + "entrezGeneIds": gene_ids # List of Entrez gene IDs + } + response = requests.post(url, json=data) + if response.status_code == 200: + return response.json() + return None + + # Process each study + all_study_data = [] + + for study_id, study_info in STUDY_PROFILES.items(): + print(f"\nProcessing {study_info['name']}...") + + # First get the sample IDs for the study + sample_ids = get_rna_seq_samples(study_id) + + if sample_ids: + print(f"Found {len(sample_ids)} RNA-seq samples") + + # Process samples in chunks to avoid large requests + chunk_size = 100 + study_data = [] + + for i in range(0, len(sample_ids), chunk_size): + chunk = sample_ids[i:i + chunk_size] + print(f"Fetching data for samples {i+1}-{i+len(chunk)}...") + + # Get expression data for this chunk of samples + expression_data = get_expression_data( + study_info['profile'], + list(STAT5_GENES.keys()), + chunk + ) + + if expression_data: + study_data.extend(expression_data) + time.sleep(0.2) # Small delay between requests + + if study_data: + # Convert to DataFrame + df = pd.DataFrame(study_data) + + # Add metadata + df['gene_symbol'] = df['entrezGeneId'].map(STAT5_GENES) + df['study_id'] = study_id + df['study_name'] = study_info['name'] + df['measurement_type'] = study_info['type'] + + # Select and rename columns + df = df[[ + 'study_id', 'study_name', 'measurement_type', + 'gene_symbol', 'sampleId', 'value' + ]] + df.columns = [ + 'study_id', 'study_name', 'measurement_type', + 'gene', 'sample_id', 'zscore' + ] + + all_study_data.append(df) + + # Combine all data + if all_study_data: + # Create long format + combined_df = pd.concat(all_study_data, ignore_index=True) + + # Create wide format (samples as rows, genes as columns) + combined_wide = combined_df.pivot_table( + index=['study_id', 'study_name', 'measurement_type', 'sample_id'], + columns='gene', + values='zscore' + ).reset_index() + + # Save both versions + stat5_all_studies = combined_df # Long format + stat5_all_studies_wide = combined_wide # Wide format + + # Example of the resulting data structure: + print("\nLong format example (first few rows):") + print(stat5_all_studies.head()) + + print("\nWide format example (first few rows):") + print(stat5_all_studies_wide.head()) diff --git a/src/biome/api_definitions/cda/api.yaml b/src/biome/api_definitions/cda/api.yaml index 9613081..3213fc3 100644 --- a/src/biome/api_definitions/cda/api.yaml +++ b/src/biome/api_definitions/cda/api.yaml @@ -54,10 +54,10 @@ description: > ## Data Standards Services (DSS) The DSS provides us with harmonized values mapped to the data sources above. In our current release, DSS has provided values for: ethnicity, file_format, morphology, primary_diagnosis, race, species, therapeutic_agent, source_material_type (cancer/normal), treatment_type, and vital_status. -raw_documentation: !load documentation/cda.yaml -examples: !load documentation/examples.md +raw_documentation: !load_txt documentation/cda.yaml +examples: !load_yaml documentation/examples.yaml cache_key: api_assistant_cda_yaml -documentation: | +documentation: !fill | {raw_documentation} # Additional Instructions: @@ -71,7 +71,3 @@ documentation: | You should make use of the python requests library to interact with the API. Note that the base URL of the API is 'https://cda.datacommons.cancer.gov/' - - Here are some examples of how to use the API: - {examples} - diff --git a/src/biome/api_definitions/cda/documentation/examples.md b/src/biome/api_definitions/cda/documentation/examples.yaml similarity index 100% rename from src/biome/api_definitions/cda/documentation/examples.md rename to src/biome/api_definitions/cda/documentation/examples.yaml diff --git a/src/biome/api_definitions/gdc/api.yaml b/src/biome/api_definitions/gdc/api.yaml index 4a63f2a..d5e8b3f 100644 --- a/src/biome/api_definitions/gdc/api.yaml +++ b/src/biome/api_definitions/gdc/api.yaml @@ -4,12 +4,12 @@ description: | platform for cancer researchers who need to understand cancer, its clinical progression, and response to therapy. The GDC supports several cancer genome programs at the NCI Center for Cancer Genomics (CCG), including The Cancer Genome Atlas (TCGA) and Therapeutically Applicable Research to Generate Effective Treatments (TARGET). -raw_documentation: !load documentation/gdc.md -facets: !load documentation/facets.txt -mappings: !load documentation/gdc_mappings.json -examples: !load documentation/examples.md +raw_documentation: !load_txt documentation/gdc.md +facets: !load_txt documentation/facets.txt +mappings: !load_txt documentation/gdc_mappings.json +examples: !load_yaml documentation/examples.yaml cache_key: api_assistant_gdc_faceted -documentation: | +documentation: !fill | # GDC API Documentation @@ -53,7 +53,4 @@ documentation: | the wildcard operator so it might not be as specific as they would like. If a user asks you to filter based on the type of cancer, it is often productive to filter on - disease type, primary diagnosis, or primary site. - - Here are some examples of how to use the API: - {examples} \ No newline at end of file + disease type, primary diagnosis, or primary site. \ No newline at end of file diff --git a/src/biome/api_definitions/gdc/documentation/examples.md b/src/biome/api_definitions/gdc/documentation/examples.md deleted file mode 100644 index 3d0318d..0000000 --- a/src/biome/api_definitions/gdc/documentation/examples.md +++ /dev/null @@ -1,269 +0,0 @@ -## Example 1: Query the GDC for cases of myeloid leukemia with a JAK2 somatic mutation and return the results as a pandas dataframe. - -``` -import pandas as pd -from pandas import json_normalize - -import requests -import json - -# Define the endpoint and filters -endpoint = "https://api.gdc.cancer.gov/ssm_occurrences" -filters = { - "op": "and", - "content": [ - { - "op": "=", - "content": { - "field": "case.disease_type", - "value": "*myeloid leukemia*" - } - }, - { - "op": "in", - "content": { - "field": "ssm.consequence.transcript.gene.symbol", - "value": ["JAK2"] - } - } - ] -} - -# Define the fields to be returned -fields = [ - "ssm_id", - "ssm.consequence.transcript.gene.symbol", - "ssm.mutation_type", - "ssm.genomic_dna_change", - "ssm.consequence.transcript.aa_change", - "ssm.consequence.transcript.consequence_type", - "case.project.project_id", - "case.submitter_id", - "case.case_id", - "case.diagnoses.primary_diagnosis" -] - -# Construct the request parameters -params = { - "filters": json.dumps(filters), - "fields": ",".join(fields), - "format": "JSON", - "size": "1000" -} - -# Send the request -response = requests.get(endpoint, params=params) -print(f"total hits: {response.json()['data']['pagination']['total']}") -all_ssms = response.json()['data']['hits'] - -ssms = pd.DataFrame(json_normalize(all_ssms)) -ssms.head() -``` - -## Example 2: Get all cases where disease type is myeloid leukemia from GDC and return the results as a pandas dataframe. Paginate through the results to fetch all available cases - -``` -import pandas as pd -from pandas import json_normalize - -import requests -import json - -# Define the endpoint and filters -endpoint = "https://api.gdc.cancer.gov/cases" -filters = { - "op": "=", - "content": { - "field": "disease_type", - "value": "*myeloid leukemia*" - } -} - -# Define the fields to be returned -fields = [ - "submitter_id", - "case_id", - "primary_site", - "disease_type", - "diagnoses.age_at_diagnosis", - "diagnoses.primary_diagnosis", - "demographic.gender", - "exposures.tobacco_smoking_status", - "files.file_id", - "files.file_name", - "files.data_type", - "files.experimental_strategy" -] - -# Initialize pagination variables -all_cases = [] -current_page = 1 -page_size = 1000 - -while True: - # Construct the request parameters - params = { - "filters": json.dumps(filters), - "fields": ",".join(fields), - "format": "JSON", - "size": page_size, - "from": (current_page - 1) * page_size - } - - # Send the request - response = requests.get(endpoint, params=params) - - # Check for successful response - if response.status_code == 200: - data = response.json() - cases = data["data"]["hits"] - all_cases.extend(cases) - if len(cases) < page_size: - break - current_page += 1 - else: - print(f"Error: {response.status_code} - {response.text}") - break - -# Convert the results to a DataFrame -cases_df = pd.DataFrame(json_normalize(all_cases)) -print(f"There are {len(cases_df)} cases in GDC for AML") -# Display the DataFrame -cases_df.head() -``` - -## Example 3: Get all cases from GDC with no filters. Fetch only the case ids and paginate through the results to fetch all available cases. Store the results as a dataframe - -``` -import requests -import json -import pandas as pd -from pandas import json_normalize - -# Define the fields to be returned -fields = [ - "case_id", -] - -# Initialize pagination variables -all_cases = [] -current_page = 1 -page_size = 1000 - -while True: - # Construct the request parameters - params = { - "fields": ",".join(fields), - "format": "JSON", - "size": page_size, - "from": (current_page - 1) * page_size - } - - # Send the request - response = requests.get(endpoint, params=params) - - # Check for successful response - if response.status_code == 200: - data = response.json() - cases = data["data"]["hits"] - all_cases.extend(cases) - if len(cases) < page_size: - break - current_page += 1 - else: - print(f"Error: {response.status_code} - {response.text}") - break - -# Convert the results to a DataFrame -all_cases_df = pd.DataFrame(json_normalize(all_cases)) -print(f"There are {len(all_cases_df)} cases in GDC in total") -# Display the DataFrame -all_cases_df.head() -``` - -## Example 4: Get simple somatic mutation occurrences (cases) from GDC where the primary site is bronchus and lung, the gender is male, the age at diagnosis is less than 45 years old, and the patient is a lifelong non-smoker. Return the results as a pandas dataframe. - -``` -import pandas as pd -from pandas import json_normalize - -import requests -import json - -# Define the endpoint and filters -endpoint = "https://api.gdc.cancer.gov/ssm_occurrences" -filters = { - "op": "and", - "content": [ - { - "op": "in", - "content": { - "field": "case.primary_site", - "value": ["Bronchus and lung"] - } - }, - { - "op": "in", - "content": { - "field": "case.demographic.gender", - "value": ["male"] - } - }, - { - "op": "<", - "content": { - "field": "case.diagnoses.age_at_diagnosis", - "value": 16436 # 45 years in days - } - }, - { - "op": "in", - "content": { - "field": "case.exposures.tobacco_smoking_status", - "value": ["Lifelong Non-smoker"] - } - } - ] -} - -# Define the fields to be returned -fields = [ - "ssm_id", - "ssm.consequence.transcript.gene.symbol", - "ssm.mutation_type", - "ssm.genomic_dna_change", - "ssm.consequence.transcript.aa_change", - "ssm.consequence.transcript.consequence_type", - "case.project.project_id", - "case.submitter_id", - "case.case_id", - "case.diagnoses.primary_diagnosis", - "case.primary_site", - "case.demographic.gender", - "case.diagnoses.age_at_diagnosis", - "case.exposures.tobacco_smoking_status" -] - -# Construct the request parameters -params = { - "filters": json.dumps(filters), - "fields": ",".join(fields), - "format": "JSON", - "size": "1000" -} - -# Send the request -response = requests.get(endpoint, params=params) - -# Check for successful response -if response.status_code == 200: - data = response.json() - # Extract and display the mutation data - mutations = data["data"]["hits"] - print(f"Found {len(mutations)} mutations in lung cancer cases for males under 45 who never smoked:") - lung_mutation_df = pd.DataFrame(json_normalize(mutations)) -else: - print(f"Error: {response.status_code} - {response.text}") - -lung_mutation_df.head() -``` \ No newline at end of file diff --git a/src/biome/api_definitions/gdc/documentation/examples.yaml b/src/biome/api_definitions/gdc/documentation/examples.yaml new file mode 100644 index 0000000..de9bb75 --- /dev/null +++ b/src/biome/api_definitions/gdc/documentation/examples.yaml @@ -0,0 +1,262 @@ +- query: "Query the GDC for cases of myeloid leukemia with a JAK2 somatic mutation and return the results as a pandas dataframe." + code: | + import pandas as pd + from pandas import json_normalize + + import requests + import json + + # Define the endpoint and filters + endpoint = "https://api.gdc.cancer.gov/ssm_occurrences" + filters = { + "op": "and", + "content": [ + { + "op": "=", + "content": { + "field": "case.disease_type", + "value": "*myeloid leukemia*" + } + }, + { + "op": "in", + "content": { + "field": "ssm.consequence.transcript.gene.symbol", + "value": ["JAK2"] + } + } + ] + } + + # Define the fields to be returned + fields = [ + "ssm_id", + "ssm.consequence.transcript.gene.symbol", + "ssm.mutation_type", + "ssm.genomic_dna_change", + "ssm.consequence.transcript.aa_change", + "ssm.consequence.transcript.consequence_type", + "case.project.project_id", + "case.submitter_id", + "case.case_id", + "case.diagnoses.primary_diagnosis" + ] + + # Construct the request parameters + params = { + "filters": json.dumps(filters), + "fields": ",".join(fields), + "format": "JSON", + "size": "1000" + } + + # Send the request + response = requests.get(endpoint, params=params) + print(f"total hits: {response.json()['data']['pagination']['total']}") + all_ssms = response.json()['data']['hits'] + + ssms = pd.DataFrame(json_normalize(all_ssms)) + ssms.head() + + +- query: "Get all cases where disease type is myeloid leukemia from GDC and return the results as a pandas dataframe. Paginate through the results to fetch all available cases" + code: | + import pandas as pd + from pandas import json_normalize + + import requests + import json + + # Define the endpoint and filters + endpoint = "https://api.gdc.cancer.gov/cases" + filters = { + "op": "=", + "content": { + "field": "disease_type", + "value": "*myeloid leukemia*" + } + } + + # Define the fields to be returned + fields = [ + "submitter_id", + "case_id", + "primary_site", + "disease_type", + "diagnoses.age_at_diagnosis", + "diagnoses.primary_diagnosis", + "demographic.gender", + "exposures.tobacco_smoking_status", + "files.file_id", + "files.file_name", + "files.data_type", + "files.experimental_strategy" + ] + + # Initialize pagination variables + all_cases = [] + current_page = 1 + page_size = 1000 + + while True: + # Construct the request parameters + params = { + "filters": json.dumps(filters), + "fields": ",".join(fields), + "format": "JSON", + "size": page_size, + "from": (current_page - 1) * page_size + } + + # Send the request + response = requests.get(endpoint, params=params) + + # Check for successful response + if response.status_code == 200: + data = response.json() + cases = data["data"]["hits"] + all_cases.extend(cases) + if len(cases) < page_size: + break + current_page += 1 + else: + print(f"Error: {response.status_code} - {response.text}") + break + + # Convert the results to a DataFrame + cases_df = pd.DataFrame(json_normalize(all_cases)) + print(f"There are {len(cases_df)} cases in GDC for AML") + # Display the DataFrame + cases_df.head() + +- query: "Get all cases from GDC with no filters. Fetch only the case ids and paginate through the results to fetch all available cases. Store the results as a dataframe" + code: | + import requests + import json + import pandas as pd + from pandas import json_normalize + + # Define the fields to be returned + fields = [ + "case_id", + ] + + # Initialize pagination variables + all_cases = [] + current_page = 1 + page_size = 1000 + + while True: + # Construct the request parameters + params = { + "fields": ",".join(fields), + "format": "JSON", + "size": page_size, + "from": (current_page - 1) * page_size + } + + # Send the request + response = requests.get(endpoint, params=params) + + # Check for successful response + if response.status_code == 200: + data = response.json() + cases = data["data"]["hits"] + all_cases.extend(cases) + if len(cases) < page_size: + break + current_page += 1 + else: + print(f"Error: {response.status_code} - {response.text}") + break + + # Convert the results to a DataFrame + all_cases_df = pd.DataFrame(json_normalize(all_cases)) + print(f"There are {len(all_cases_df)} cases in GDC in total") + # Display the DataFrame + all_cases_df.head() + +- query: "Get simple somatic mutation occurrences (cases) from GDC where the primary site is bronchus and lung, the gender is male, the age at diagnosis is less than 45 years old, and the patient is a lifelong non-smoker. Return the results as a pandas dataframe." + code: | + import pandas as pd + from pandas import json_normalize + + import requests + import json + + # Define the endpoint and filters + endpoint = "https://api.gdc.cancer.gov/ssm_occurrences" + filters = { + "op": "and", + "content": [ + { + "op": "in", + "content": { + "field": "case.primary_site", + "value": ["Bronchus and lung"] + } + }, + { + "op": "in", + "content": { + "field": "case.demographic.gender", + "value": ["male"] + } + }, + { + "op": "<", + "content": { + "field": "case.diagnoses.age_at_diagnosis", + "value": 16436 # 45 years in days + } + }, + { + "op": "in", + "content": { + "field": "case.exposures.tobacco_smoking_status", + "value": ["Lifelong Non-smoker"] + } + } + ] + } + + # Define the fields to be returned + fields = [ + "ssm_id", + "ssm.consequence.transcript.gene.symbol", + "ssm.mutation_type", + "ssm.genomic_dna_change", + "ssm.consequence.transcript.aa_change", + "ssm.consequence.transcript.consequence_type", + "case.project.project_id", + "case.submitter_id", + "case.case_id", + "case.diagnoses.primary_diagnosis", + "case.primary_site", + "case.demographic.gender", + "case.diagnoses.age_at_diagnosis", + "case.exposures.tobacco_smoking_status" + ] + + # Construct the request parameters + params = { + "filters": json.dumps(filters), + "fields": ",".join(fields), + "format": "JSON", + "size": "1000" + } + + # Send the request + response = requests.get(endpoint, params=params) + + # Check for successful response + if response.status_code == 200: + data = response.json() + # Extract and display the mutation data + mutations = data["data"]["hits"] + print(f"Found {len(mutations)} mutations in lung cancer cases for males under 45 who never smoked:") + lung_mutation_df = pd.DataFrame(json_normalize(mutations)) + else: + print(f"Error: {response.status_code} - {response.text}") + + lung_mutation_df.head() \ No newline at end of file diff --git a/src/biome/api_definitions/hpa/api.yaml b/src/biome/api_definitions/hpa/api.yaml index 44b159e..db61b20 100644 --- a/src/biome/api_definitions/hpa/api.yaml +++ b/src/biome/api_definitions/hpa/api.yaml @@ -32,11 +32,8 @@ description: | In addition the Human Protein Atlas has been appointed Global Core Biodata Resource (GCBR) by the Global Biodata Coalition. The Human Protein Atlas consortium is mainly funded by the Knut and Alice Wallenberg Foundation. -raw_documentation: !load documentation/hpa_docs.md -examples: !load documentation/examples.md +raw_documentation: !load_txt documentation/hpa_docs.md +examples: !load_yaml documentation/examples.yaml cache_key: "api_assistant_pdc_graphql" -documentation: | - {raw_documentation} - - Here are some examples of how to use the API: - {examples} \ No newline at end of file +documentation: !fill | + {raw_documentation} \ No newline at end of file diff --git a/src/biome/api_definitions/hpa/documentation/examples.md b/src/biome/api_definitions/hpa/documentation/examples.md deleted file mode 100644 index ad9c9b8..0000000 --- a/src/biome/api_definitions/hpa/documentation/examples.md +++ /dev/null @@ -1,39 +0,0 @@ - -## Example 1: Query the Human Protein Atlas API for RNA and protein expression summary of a gene (JAK2) using its Ensembl ID. The example demonstrates how to retrieve and display tissue specificity, distribution, and subcellular localization information. - -``` -import requests -import json - -# Query JAK2 data from HPA API -jak2_id = "ENSG00000096968" -url = f"https://www.proteinatlas.org/{jak2_id}.json" - -# Get the data -response = requests.get(url) -data = response.json() - -# Print basic gene information -print(f"Gene: {data.get('Gene', 'N/A')}") -print(f"Description: {data.get('Gene description', 'N/A')}") -print("-" * 80) - -# RNA Expression Summary -print("\nRNA Expression Summary:") -print(f"Tissue specificity: {data.get('RNA tissue specificity', 'N/A')}") -print(f"Tissue distribution: {data.get('RNA tissue distribution', 'N/A')}") -print(f"Tissue specificity score: {data.get('RNA tissue specificity score', 'N/A')}") - -# Blood cell specific information -print("\nRNA Blood Cell Expression:") -print(f"Blood cell specificity: {data.get('RNA blood cell specificity', 'N/A')}") -print(f"Blood cell distribution: {data.get('RNA blood cell distribution', 'N/A')}") -print(f"Blood cell specificity score: {data.get('RNA blood cell specificity score', 'N/A')}") - -# Protein Expression Summary -print("\nProtein Expression Summary:") -print(f"Reliability (Immunohistochemistry): {data.get('Reliability (IH)', 'N/A')}") -print(f"Subcellular main location: {data.get('Subcellular main location', 'N/A')}") -if 'Subcellular additional location' in data: - print(f"Subcellular additional location: {data.get('Subcellular additional location', 'N/A')}") -``` diff --git a/src/biome/api_definitions/hpa/documentation/examples.yaml b/src/biome/api_definitions/hpa/documentation/examples.yaml new file mode 100644 index 0000000..9de4459 --- /dev/null +++ b/src/biome/api_definitions/hpa/documentation/examples.yaml @@ -0,0 +1,37 @@ + +- query: "Query the Human Protein Atlas API for RNA and protein expression summary of a gene (JAK2) using its Ensembl ID. The example demonstrates how to retrieve and display tissue specificity, distribution, and subcellular localization information." + code: | + import requests + import json + + # Query JAK2 data from HPA API + jak2_id = "ENSG00000096968" + url = f"https://www.proteinatlas.org/{jak2_id}.json" + + # Get the data + response = requests.get(url) + data = response.json() + + # Print basic gene information + print(f"Gene: {data.get('Gene', 'N/A')}") + print(f"Description: {data.get('Gene description', 'N/A')}") + print("-" * 80) + + # RNA Expression Summary + print("\nRNA Expression Summary:") + print(f"Tissue specificity: {data.get('RNA tissue specificity', 'N/A')}") + print(f"Tissue distribution: {data.get('RNA tissue distribution', 'N/A')}") + print(f"Tissue specificity score: {data.get('RNA tissue specificity score', 'N/A')}") + + # Blood cell specific information + print("\nRNA Blood Cell Expression:") + print(f"Blood cell specificity: {data.get('RNA blood cell specificity', 'N/A')}") + print(f"Blood cell distribution: {data.get('RNA blood cell distribution', 'N/A')}") + print(f"Blood cell specificity score: {data.get('RNA blood cell specificity score', 'N/A')}") + + # Protein Expression Summary + print("\nProtein Expression Summary:") + print(f"Reliability (Immunohistochemistry): {data.get('Reliability (IH)', 'N/A')}") + print(f"Subcellular main location: {data.get('Subcellular main location', 'N/A')}") + if 'Subcellular additional location' in data: + print(f"Subcellular additional location: {data.get('Subcellular additional location', 'N/A')}") diff --git a/src/biome/api_definitions/idc/api.yaml b/src/biome/api_definitions/idc/api.yaml index 1502123..9e5c0a5 100644 --- a/src/biome/api_definitions/idc/api.yaml +++ b/src/biome/api_definitions/idc/api.yaml @@ -8,10 +8,10 @@ description: | - Commercial-friendly: >95% of the data in IDC is covered by the permissive CC-BY license, which allows commercial reuse (small subset of data is covered by the CC-NC license); each file in IDC is tagged with the license to make it easier for you to understand and follow the rules - Cloud-based: All of the data in IDC is available from both Google and AWS public buckets: fast and free to download, no out-of-cloud egress fees - Harmonized: All of the images and image-derived data in IDC is harmonized into standard DICOM representation -raw_documentation: !load documentation/idc.md -examples: !load documentation/examples.md +raw_documentation: !load_txt documentation/idc.md +examples: !load_yaml documentation/examples.yaml cache_key: "api_assistant_idc_yaml" -documentation: | +documentation: !fill | {raw_documentation} # Additional Instructions: @@ -20,8 +20,4 @@ documentation: | ```python from idc_index import index client = index.IDCClient() - ``` - - Here are some examples of how to use the API: - {examples} - + ``` \ No newline at end of file diff --git a/src/biome/api_definitions/idc/documentation/examples.md b/src/biome/api_definitions/idc/documentation/examples.md deleted file mode 100644 index 4a7d588..0000000 --- a/src/biome/api_definitions/idc/documentation/examples.md +++ /dev/null @@ -1,472 +0,0 @@ -# Examples - -## Example 1: Get columns available in IDC index - -``` -from idc_index import index -import pandas as pd - -client = index.IDCClient() - -# First, let's see what columns are available -query_columns = """ -SELECT * FROM index LIMIT 1 -""" - -try: - df = client.sql_query(query_columns) - print("Available columns in the index:") - print(df.columns.tolist()) -except Exception as e: - print(f"An error occurred: {str(e)}") -``` - -## Example 2: Get slide microscopy data - -``` -from idc_index import index -import pandas as pd - -client = index.IDCClient() - -# Query for slide microscopy data -query = """ -SELECT DISTINCT - collection_id, - PatientID, - SeriesDescription, - StudyDescription, - BodyPartExamined, - SeriesDate -FROM - index -WHERE - Modality = 'SM' -""" - -try: - df = client.sql_query(query) - print("Slide Microscopy Data Available:") - print("\nCollections with slide microscopy data:") - print(df['collection_id'].unique()) - print("\nSample of the data:") - print(df.head()) - print("\nTotal number of records:", len(df)) -except Exception as e: - print(f"An error occurred: {str(e)}") -``` - -## Example 3: Get slide microscopy data for a specific collection - -``` -from idc_index import index -import pandas as pd - -client = index.IDCClient() - -# Query for AML-specific slide microscopy data -query = """ -SELECT DISTINCT - collection_id, - PatientID, - SeriesDescription, - StudyDescription, - BodyPartExamined, - SeriesDate, - Manufacturer, - ManufacturerModelName, - SeriesNumber, - instanceCount -FROM - index -WHERE - Modality = 'SM' - AND collection_id IN ('cptac_aml', 'cmb_aml') -ORDER BY - PatientID, SeriesDate -""" - -try: - df = client.sql_query(query) - print("AML Slide Microscopy Data:") - print("\nTotal number of records:", len(df)) - - print("\nUnique patients:", len(df['PatientID'].unique())) - - print("\nTypes of series descriptions (slide types):") - print(df['SeriesDescription'].unique()) - - print("\nSample of the data:") - print(df.head(10)) - - # Save to a CSV file for further analysis - df.to_csv('aml_pathology_data.csv', index=False) - print("\nData has been saved to 'aml_pathology_data.csv'") - -except Exception as e: - print(f"An error occurred: {str(e)}") -``` - -## Example 4: Get additional information for microscopy data for a specific collection - -``` -from idc_index import index -import pandas as pd - -client = index.IDCClient() - -# Query for all available fields for AML slides -query = """ -SELECT * -FROM - index -WHERE - Modality = 'SM' - AND collection_id IN ('cptac_aml', 'cmb_aml') - AND SeriesDescription LIKE '%bone marrow%' -ORDER BY - PatientID, SeriesDate -""" - -try: - df = client.sql_query(query) - print("AML Bone Marrow Slide Data:") - print("\nTotal number of bone marrow slides:", len(df)) - - print("\nAvailable columns (metadata fields):") - print(df.columns.tolist()) - - # Look for any columns that might contain annotations or clinical data - clinical_cols = [col for col in df.columns if any(term in col.lower() - for term in ['annotation', 'clinical', 'blast', 'cell', - 'pathology', 'diagnosis', 'grade', 'stage'])] - print("\nPotentially relevant clinical/annotation columns:") - print(clinical_cols) - - # Get the source DOIs which might lead to additional clinical data - print("\nUnique source DOIs (for finding additional clinical data):") - print(df['source_DOI'].unique()) - -except Exception as e: - print(f"An error occurred: {str(e)}") - -# Let's also check if there are any study-level annotations -query_study = """ -SELECT DISTINCT - StudyDescription, - collection_id, - source_DOI, - license_short_name -FROM - index -WHERE - collection_id IN ('cptac_aml', 'cmb_aml') -""" - -try: - df_study = client.sql_query(query_study) - print("\nStudy-level information:") - print(df_study) -except Exception as e: - print(f"An error occurred querying study data: {str(e)}") -``` - -Example 5: Query to get download URLs for bone marrow slides - -``` -from idc_index import index -import pandas as pd - -client = index.IDCClient() - -# Query to get download URLs for bone marrow slides -query = """ -SELECT - PatientID, - SeriesDescription, - series_aws_url, - series_size_MB -FROM - index -WHERE - Modality = 'SM' - AND collection_id IN ('cptac_aml', 'cmb_aml') - AND SeriesDescription LIKE '%bone marrow%' -LIMIT 1 -""" - -try: - df = client.sql_query(query) - print("Sample slide information:") - print(df) - - if not df.empty: - url = df['series_aws_url'].iloc[0] - size_mb = df['series_size_MB'].iloc[0] - print(f"\nFile size: {size_mb:.2f} MB") - print(f"Download URL: {url}") -except Exception as e: - print(f"An error occurred: {str(e)}") -``` - -## Example 6: Get list of files in an S3 bucket for a specific collection on IDC - -``` -import boto3 -from botocore import UNSIGNED -from botocore.config import Config -import os -import tempfile - -# Function to list contents of an S3 path -def list_s3_contents(bucket, prefix): - s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED)) - try: - result = s3.list_objects_v2(Bucket=bucket, Prefix=prefix) - if 'Contents' in result: - return result['Contents'] - return [] - except Exception as e: - print(f"Error listing S3 contents: {str(e)}") - return [] - -# Parse S3 URL and list contents -s3_url = "s3://idc-open-data/d53e7fc3-8ef9-4de5-8059-47b21a67eb4f" -bucket = s3_url.split('/')[2] -prefix = '/'.join(s3_url.split('/')[3:]) - -print(f"Checking contents of bucket: {bucket}") -print(f"With prefix: {prefix}") - -contents = list_s3_contents(bucket, prefix) -print("\nFound files:") -for item in contents: - print(f"- {item['Key']} ({item['Size']/1024/1024:.2f} MB)") - -# Let's also check if we can get any pre-signed URLs or public access URLs -s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED)) -try: - # Try to get the bucket location - location = s3.get_bucket_location(Bucket=bucket) - print(f"\nBucket location: {location}") - - # Try to get bucket policy - try: - policy = s3.get_bucket_policy(Bucket=bucket) - print("\nBucket policy:", policy) - except Exception as e: - print("\nCouldn't get bucket policy:", str(e)) - -except Exception as e: - print(f"\nError getting bucket information: {str(e)}") -``` - -## Example 7: Download and visualize a slide microscopy image with pydicom and matplotlib - -``` -import sys -import subprocess - -# Install required package -subprocess.check_call([sys.executable, "-m", "pip", "install", "pydicom"]) - -import boto3 -from botocore import UNSIGNED -from botocore.config import Config -import os -import tempfile -import pydicom -from PIL import Image -import numpy as np -import matplotlib.pyplot as plt - -# Create S3 client -s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED)) - -# Create temporary directory -temp_dir = tempfile.mkdtemp() - -# Let's try the smallest DICOM file first -file_key = "d53e7fc3-8ef9-4de5-8059-47b21a67eb4f/40006642-6ee5-44ca-bc5f-e1cc2bd7ee70.dcm" -local_file = os.path.join(temp_dir, "slide.dcm") - -try: - print(f"Downloading file: {file_key}") - s3.download_file('idc-open-data', file_key, local_file) - print("Download successful!") - - # Try to read the DICOM file - print("\nReading DICOM file...") - ds = pydicom.dcmread(local_file) - - print("\nDICOM metadata:") - print(f"Patient ID: {ds.PatientID}") - print(f"Modality: {ds.Modality}") - print(f"Image Type: {ds.ImageType}") - - # Convert to image and display - if hasattr(ds, 'pixel_array'): - print("\nConverting to image...") - pixel_array = ds.pixel_array - - # Normalize the pixel values - if pixel_array.dtype != np.uint8: - pixel_array = ((pixel_array - pixel_array.min()) * 255.0 / - (pixel_array.max() - pixel_array.min())).astype(np.uint8) - - # Create image - img = Image.fromarray(pixel_array) - - # Display - plt.figure(figsize=(15, 10)) - plt.imshow(img, cmap='gray') - plt.axis('off') - plt.title(f"DICOM Image - Patient: {ds.PatientID}") - plt.show() - else: - print("No pixel data found in the DICOM file") - -except Exception as e: - print(f"An error occurred: {str(e)}") -finally: - # Clean up - if os.path.exists(local_file): - os.remove(local_file) - os.rmdir(temp_dir) -``` - - -## Example 8: Find collections in the Imaging Data Commons with 'aml' in their name, likely related to Acute Myeloid Leukemia. - -``` -from idc_index import index -import pandas as pd - -client = index.IDCClient() - -# Query to find collection names containing 'aml' -query = """ -SELECT DISTINCT collection_id -FROM - index -WHERE - LOWER(collection_id) LIKE '%aml%' -""" - -try: - df_collections = client.sql_query(query) - print("Collections with 'aml' in their name:") - print(df_collections) -except Exception as e: - print(f"An error occurred: {str(e)}") -``` - -## Example 9: Download bone marrow biopsy images from the 'cmb_aml' collection for a specific study using IDC's public S3 bucket. - -``` -import pandas as pd -from idc_index import index - -client = index.IDCClient() - -# Query for the desired images -query = """ -SELECT series_aws_url -FROM index -WHERE collection_id = 'cmb_aml' -AND StudyDescription = 'XR_Biopsy_BoneMarrow' -""" -df = client.sql_query(query) - -# Create a download manifest using the current directory -with open('download_manifest.txt', 'w') as f: - for url in df['series_aws_url']: - f.write(f'cp {url} ./\n') - -# Execute the download -s5cmd_binary = client.s5cmdPath -!{s5cmd_binary} --no-sign-request --endpoint-url https://s3.amazonaws.com run download_manifest.txt -``` - -## Example 10: Basic plot of dicom image from file - -``` -import pydicom -import matplotlib.pyplot as plt - -# Load the DICOM file -filename = '73f18e49-8699-4c1f-90ad-5db94b6b5602.dcm' -ds = pydicom.dcmread(filename) - -# Display the image -plt.imshow(ds.pixel_array, cmap=plt.cm.gray) -plt.title('Bone Marrow Biopsy Image') -plt.axis('off') -plt.show() -``` - -## Example 11: List all studies for a given collection - -``` -# Query to list all studies in the 'cmb_aml' collection -query_all_studies = """ -SELECT DISTINCT StudyDescription -FROM - index -WHERE - collection_id = 'cmb_aml' -""" - -try: - df_all_studies = client.sql_query(query_all_studies) - print("All studies in the 'cmb_aml' collection:") - print(df_all_studies) -except Exception as e: - print(f"An error occurred: {str(e)}") -``` - -## Example 12: Retrieve imagery URLs for the 'XR_Biopsy_BoneMarrow' study in the 'cmb_aml' collection from IDC. - -``` -import pandas as pd -from idc_index import index - -client = index.IDCClient() - -query = """ -SELECT series_aws_url -FROM index -WHERE collection_id = 'cmb_aml' -AND StudyDescription = 'XR_Biopsy_BoneMarrow' -""" -df = client.sql_query(query) -print(df.head()) -``` - - -## Example 12: Fetch studies for a given collection in the Imaging Data Commons (IDC), specifically for the 'cptac_aml' collection. - -``` -from idc_index import index -import pandas as pd - -# Initialize the IDC client -client = index.IDCClient() - -# Query to find studies within the 'cptac_aml' collection -query = """ -SELECT DISTINCT StudyDescription -FROM - index -WHERE - collection_id = 'cptac_aml' -""" - -try: - # Execute the query - df_studies = client.sql_query(query) - print("Studies in 'cptac_aml':") - print(df_studies) -except Exception as e: - print(f"An error occurred: {str(e)}") -``` diff --git a/src/biome/api_definitions/idc/documentation/examples.yaml b/src/biome/api_definitions/idc/documentation/examples.yaml new file mode 100644 index 0000000..65ff8df --- /dev/null +++ b/src/biome/api_definitions/idc/documentation/examples.yaml @@ -0,0 +1,444 @@ +- query: "Get columns available in IDC index" + code: | + from idc_index import index + import pandas as pd + + client = index.IDCClient() + + # First, let's see what columns are available + query_columns = """ + SELECT * FROM index LIMIT 1 + """ + + try: + df = client.sql_query(query_columns) + print("Available columns in the index:") + print(df.columns.tolist()) + except Exception as e: + print(f"An error occurred: {str(e)}") + +- query: "Get slide microscopy data" + code: | + from idc_index import index + import pandas as pd + + client = index.IDCClient() + + # Query for slide microscopy data + query = """ + SELECT DISTINCT + collection_id, + PatientID, + SeriesDescription, + StudyDescription, + BodyPartExamined, + SeriesDate + FROM + index + WHERE + Modality = 'SM' + """ + + try: + df = client.sql_query(query) + print("Slide Microscopy Data Available:") + print("\nCollections with slide microscopy data:") + print(df['collection_id'].unique()) + print("\nSample of the data:") + print(df.head()) + print("\nTotal number of records:", len(df)) + except Exception as e: + print(f"An error occurred: {str(e)}") + +- query: "Get slide microscopy data for a specific collection" + code: | + from idc_index import index + import pandas as pd + + client = index.IDCClient() + + # Query for AML-specific slide microscopy data + query = """ + SELECT DISTINCT + collection_id, + PatientID, + SeriesDescription, + StudyDescription, + BodyPartExamined, + SeriesDate, + Manufacturer, + ManufacturerModelName, + SeriesNumber, + instanceCount + FROM + index + WHERE + Modality = 'SM' + AND collection_id IN ('cptac_aml', 'cmb_aml') + ORDER BY + PatientID, SeriesDate + """ + + try: + df = client.sql_query(query) + print("AML Slide Microscopy Data:") + print("\nTotal number of records:", len(df)) + + print("\nUnique patients:", len(df['PatientID'].unique())) + + print("\nTypes of series descriptions (slide types):") + print(df['SeriesDescription'].unique()) + + print("\nSample of the data:") + print(df.head(10)) + + # Save to a CSV file for further analysis + df.to_csv('aml_pathology_data.csv', index=False) + print("\nData has been saved to 'aml_pathology_data.csv'") + + except Exception as e: + print(f"An error occurred: {str(e)}") + +- query: "Get additional information for microscopy data for a specific collection" + code: | + from idc_index import index + import pandas as pd + + client = index.IDCClient() + + # Query for all available fields for AML slides + query = """ + SELECT * + FROM + index + WHERE + Modality = 'SM' + AND collection_id IN ('cptac_aml', 'cmb_aml') + AND SeriesDescription LIKE '%bone marrow%' + ORDER BY + PatientID, SeriesDate + """ + + try: + df = client.sql_query(query) + print("AML Bone Marrow Slide Data:") + print("\nTotal number of bone marrow slides:", len(df)) + + print("\nAvailable columns (metadata fields):") + print(df.columns.tolist()) + + # Look for any columns that might contain annotations or clinical data + clinical_cols = [col for col in df.columns if any(term in col.lower() + for term in ['annotation', 'clinical', 'blast', 'cell', + 'pathology', 'diagnosis', 'grade', 'stage'])] + print("\nPotentially relevant clinical/annotation columns:") + print(clinical_cols) + + # Get the source DOIs which might lead to additional clinical data + print("\nUnique source DOIs (for finding additional clinical data):") + print(df['source_DOI'].unique()) + + except Exception as e: + print(f"An error occurred: {str(e)}") + + # Let's also check if there are any study-level annotations + query_study = """ + SELECT DISTINCT + StudyDescription, + collection_id, + source_DOI, + license_short_name + FROM + index + WHERE + collection_id IN ('cptac_aml', 'cmb_aml') + """ + + try: + df_study = client.sql_query(query_study) + print("\nStudy-level information:") + print(df_study) + except Exception as e: + print(f"An error occurred querying study data: {str(e)}") + +- query: "Query to get download URLs for bone marrow slides" + code: | + from idc_index import index + import pandas as pd + + client = index.IDCClient() + + # Query to get download URLs for bone marrow slides + query = """ + SELECT + PatientID, + SeriesDescription, + series_aws_url, + series_size_MB + FROM + index + WHERE + Modality = 'SM' + AND collection_id IN ('cptac_aml', 'cmb_aml') + AND SeriesDescription LIKE '%bone marrow%' + LIMIT 1 + """ + + try: + df = client.sql_query(query) + print("Sample slide information:") + print(df) + + if not df.empty: + url = df['series_aws_url'].iloc[0] + size_mb = df['series_size_MB'].iloc[0] + print(f"\nFile size: {size_mb:.2f} MB") + print(f"Download URL: {url}") + except Exception as e: + print(f"An error occurred: {str(e)}") + +- query: "Get list of files in an S3 bucket for a specific collection on IDC" + code: | + import boto3 + from botocore import UNSIGNED + from botocore.config import Config + import os + import tempfile + + # Function to list contents of an S3 path + def list_s3_contents(bucket, prefix): + s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED)) + try: + result = s3.list_objects_v2(Bucket=bucket, Prefix=prefix) + if 'Contents' in result: + return result['Contents'] + return [] + except Exception as e: + print(f"Error listing S3 contents: {str(e)}") + return [] + + # Parse S3 URL and list contents + s3_url = "s3://idc-open-data/d53e7fc3-8ef9-4de5-8059-47b21a67eb4f" + bucket = s3_url.split('/')[2] + prefix = '/'.join(s3_url.split('/')[3:]) + + print(f"Checking contents of bucket: {bucket}") + print(f"With prefix: {prefix}") + + contents = list_s3_contents(bucket, prefix) + print("\nFound files:") + for item in contents: + print(f"- {item['Key']} ({item['Size']/1024/1024:.2f} MB)") + + # Let's also check if we can get any pre-signed URLs or public access URLs + s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED)) + try: + # Try to get the bucket location + location = s3.get_bucket_location(Bucket=bucket) + print(f"\nBucket location: {location}") + + # Try to get bucket policy + try: + policy = s3.get_bucket_policy(Bucket=bucket) + print("\nBucket policy:", policy) + except Exception as e: + print("\nCouldn't get bucket policy:", str(e)) + + except Exception as e: + print(f"\nError getting bucket information: {str(e)}") + +- query: "Download and visualize a slide microscopy image with pydicom and matplotlib" + code: | + import sys + import subprocess + + # Install required package + subprocess.check_call([sys.executable, "-m", "pip", "install", "pydicom"]) + + import boto3 + from botocore import UNSIGNED + from botocore.config import Config + import os + import tempfile + import pydicom + from PIL import Image + import numpy as np + import matplotlib.pyplot as plt + + # Create S3 client + s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED)) + + # Create temporary directory + temp_dir = tempfile.mkdtemp() + + # Let's try the smallest DICOM file first + file_key = "d53e7fc3-8ef9-4de5-8059-47b21a67eb4f/40006642-6ee5-44ca-bc5f-e1cc2bd7ee70.dcm" + local_file = os.path.join(temp_dir, "slide.dcm") + + try: + print(f"Downloading file: {file_key}") + s3.download_file('idc-open-data', file_key, local_file) + print("Download successful!") + + # Try to read the DICOM file + print("\nReading DICOM file...") + ds = pydicom.dcmread(local_file) + + print("\nDICOM metadata:") + print(f"Patient ID: {ds.PatientID}") + print(f"Modality: {ds.Modality}") + print(f"Image Type: {ds.ImageType}") + + # Convert to image and display + if hasattr(ds, 'pixel_array'): + print("\nConverting to image...") + pixel_array = ds.pixel_array + + # Normalize the pixel values + if pixel_array.dtype != np.uint8: + pixel_array = ((pixel_array - pixel_array.min()) * 255.0 / + (pixel_array.max() - pixel_array.min())).astype(np.uint8) + + # Create image + img = Image.fromarray(pixel_array) + + # Display + plt.figure(figsize=(15, 10)) + plt.imshow(img, cmap='gray') + plt.axis('off') + plt.title(f"DICOM Image - Patient: {ds.PatientID}") + plt.show() + else: + print("No pixel data found in the DICOM file") + + except Exception as e: + print(f"An error occurred: {str(e)}") + finally: + # Clean up + if os.path.exists(local_file): + os.remove(local_file) + os.rmdir(temp_dir) + + +- query: "Find collections in the Imaging Data Commons with 'aml' in their name, likely related to Acute Myeloid Leukemia." + code: | + from idc_index import index + import pandas as pd + + client = index.IDCClient() + + # Query to find collection names containing 'aml' + query = """ + SELECT DISTINCT collection_id + FROM + index + WHERE + LOWER(collection_id) LIKE '%aml%' + """ + + try: + df_collections = client.sql_query(query) + print("Collections with 'aml' in their name:") + print(df_collections) + except Exception as e: + print(f"An error occurred: {str(e)}") + +- query: "Download bone marrow biopsy images from the 'cmb_aml' collection for a specific study using IDC's public S3 bucket." + code: | + import pandas as pd + from idc_index import index + + client = index.IDCClient() + + # Query for the desired images + query = """ + SELECT series_aws_url + FROM index + WHERE collection_id = 'cmb_aml' + AND StudyDescription = 'XR_Biopsy_BoneMarrow' + """ + df = client.sql_query(query) + + # Create a download manifest using the current directory + with open('download_manifest.txt', 'w') as f: + for url in df['series_aws_url']: + f.write(f'cp {url} ./\n') + + # Execute the download + s5cmd_binary = client.s5cmdPath + !{s5cmd_binary} --no-sign-request --endpoint-url https://s3.amazonaws.com run download_manifest.txt + +- query: "Basic plot of dicom image from file" + code: | + import pydicom + import matplotlib.pyplot as plt + + # Load the DICOM file + filename = '73f18e49-8699-4c1f-90ad-5db94b6b5602.dcm' + ds = pydicom.dcmread(filename) + + # Display the image + plt.imshow(ds.pixel_array, cmap=plt.cm.gray) + plt.title('Bone Marrow Biopsy Image') + plt.axis('off') + plt.show() + +- query: "List all studies for a given collection" + code: | + # Query to list all studies in the 'cmb_aml' collection + query_all_studies = """ + SELECT DISTINCT StudyDescription + FROM + index + WHERE + collection_id = 'cmb_aml' + """ + + try: + df_all_studies = client.sql_query(query_all_studies) + print("All studies in the 'cmb_aml' collection:") + print(df_all_studies) + except Exception as e: + print(f"An error occurred: {str(e)}") + +- query: "Retrieve imagery URLs for the 'XR_Biopsy_BoneMarrow' study in the 'cmb_aml' collection from IDC." + code: | + import pandas as pd + from idc_index import index + + client = index.IDCClient() + + query = """ + SELECT series_aws_url + FROM index + WHERE collection_id = 'cmb_aml' + AND StudyDescription = 'XR_Biopsy_BoneMarrow' + """ + df = client.sql_query(query) + print(df.head()) + +- query: "Fetch studies for a given collection in the Imaging Data Commons (IDC), specifically for the 'cptac_aml' collection." + code: | + from idc_index import index + import pandas as pd + + # Initialize the IDC client + client = index.IDCClient() + + # Query to find studies within the 'cptac_aml' collection + query = """ + SELECT DISTINCT StudyDescription + FROM + index + WHERE + collection_id = 'cptac_aml' + """ + + try: + # Execute the query + df_studies = client.sql_query(query) + print("Studies in 'cptac_aml':") + print(df_studies) + except Exception as e: + print(f"An error occurred: {str(e)}") + \ No newline at end of file diff --git a/src/biome/api_definitions/indra/api.yaml b/src/biome/api_definitions/indra/api.yaml index bf99709..a005c89 100644 --- a/src/biome/api_definitions/indra/api.yaml +++ b/src/biome/api_definitions/indra/api.yaml @@ -11,10 +11,10 @@ description: > data scientists, and healthcare professionals to explore and analyze rich biomedical data efficiently, supporting advancements in drug discovery, clinical decision-making, and research. With its robust architecture, the INDRA CoGEx Query API facilitates the discovery of meaningful patterns and supports informed decision-making in complex biomedical scenarios. -raw_documentation: !load documentation/indra.json -examples: !load documentation/examples.md +raw_documentation: !load_txt documentation/indra.json +examples: !load_yaml documentation/examples.yaml cache_key: "api_assistant_indra_json" -documentation: | +documentation: !fill | {raw_documentation} # Additional Instructions: @@ -35,6 +35,4 @@ documentation: | ], "include_indirect": true }}' - ``` - Here are some examples of how to use the API: - {examples} \ No newline at end of file + ``` \ No newline at end of file diff --git a/src/biome/api_definitions/indra/documentation/examples.md b/src/biome/api_definitions/indra/documentation/examples.md deleted file mode 100644 index 3efc620..0000000 --- a/src/biome/api_definitions/indra/documentation/examples.md +++ /dev/null @@ -1,183 +0,0 @@ - - -## Example 1: Retrieve literature evidence related to AML using MeSH term queries, including child terms, and summarize the findings. - -``` -import requests -import json - -def get_mesh_literature(): - # API base URL - base_url = "https://discovery.indra.bio/api/" - - # Get papers with evidence for AML MeSH term - payload = { - "mesh_term": ["MESH", "D015470"], - "include_child_terms": True - } - - # Make request to get evidences - response = requests.post( - base_url + "get_evidences_for_mesh", - json=payload, - headers={"Content-Type": "application/json"} - ) - - if response.status_code == 200: - # Process and return results - results = response.json() - - # Convert results to a more readable format - evidence_by_statement = {} - for stmt_hash, evidences in results.items(): - evidence_by_statement[stmt_hash] = { - 'pmids': list(set([ev['text_refs'].get('PMID') for ev in evidences if ev.get('text_refs', {}).get('PMID')])), - 'evidence_count': len(evidences) - } - - return evidence_by_statement - else: - print(f"Error: {response.status_code}") - return None - -if __name__ == "__main__": - evidence_data = get_mesh_literature() - - if evidence_data: - # Print summary of findings - print(f"Found {len(evidence_data)} statements with evidence") - - # Get total evidence count - total_evidence = sum(data['evidence_count'] for data in evidence_data.values()) - print(f"Total evidence count: {total_evidence}") - - # Get unique PMIDs - all_pmids = set() - for data in evidence_data.values(): - all_pmids.update(data['pmids']) - print(f"Total unique PMIDs: {len(all_pmids)}") -``` - - -## Example 2: Perform a gene pathway analysis for JAK2 to find associated pathways and shared pathways with other genes like STAT3, JAK3, and EGFR. - -``` -import requests - -base_url = "https://discovery.indra.bio/api/" - -# Get pathways for JAK2 -jak2_payload = { - "gene": ["HGNC", "6192"] # JAK2 HGNC ID -} - -# Get JAK2 pathways -jak2_pathways_response = requests.post( - base_url + "get_pathways_for_gene", - json=jak2_payload -) - -print("JAK2 Pathways Response:") -print(jak2_pathways_response.json()) - -# Get shared pathways with common partner genes -partner_genes = [ - ["HGNC", "6192"], # JAK2 - ["HGNC", "3603"], # STAT3 - ["HGNC", "6193"], # JAK3 - ["HGNC", "3236"] # EGFR -] - -shared_payload = { - "genes": partner_genes -} - -# Get shared pathways -shared_pathways_response = requests.post( - base_url + "get_shared_pathways_for_genes", - json=shared_payload -) - -print("\nShared Pathways Response:") -print(shared_pathways_response.json()) -``` - - -## Example 3: This example demonstrates how to analyze the JAK2 gene using the INDRA CoGEx API to retrieve GO terms, pathway associations, and check if it is a kinase, phosphatase, or transcription factor. - -``` -import requests - -def analyze_jak2(): - # API base URL - base_url = "https://discovery.indra.bio/api/" - - # JAK2 HGNC ID - jak2_node = ["HGNC", "6192"] - - # Get GO terms - go_terms_payload = { - "gene": jak2_node, - "include_indirect": True - } - go_response = requests.post( - base_url + "get_go_terms_for_gene", - json=go_terms_payload - ) - - # Get pathways - pathways_payload = { - "gene": jak2_node - } - pathways_response = requests.post( - base_url + "get_pathways_for_gene", - json=pathways_payload - ) - - # Check if kinase - kinase_payload = { - "genes": ["JAK2"] - } - kinase_response = requests.post( - base_url + "is_kinase", - json=kinase_payload - ) - - # Check if phosphatase - phosphatase_payload = { - "genes": ["JAK2"] - } - phosphatase_response = requests.post( - base_url + "is_phosphatase", - json=phosphatase_payload - ) - - # Check if transcription factor - tf_payload = { - "genes": ["JAK2"] - } - tf_response = requests.post( - base_url + "is_transcription_factor", - json=tf_payload - ) - - # Compile and return results - results = { - "go_terms": go_response.json(), - "pathways": pathways_response.json(), - "is_kinase": kinase_response.json().get("JAK2", False), - "is_phosphatase": phosphatase_response.json().get("JAK2", False), - "is_transcription_factor": tf_response.json().get("JAK2", False) - } - - return results - -if __name__ == "__main__": - results = analyze_jak2() - print(f"Analysis Results for JAK2:") - print(f"Number of GO terms: {len(results['go_terms'])}") - print(f"Number of pathways: {len(results['pathways'])}") - print(f"Is kinase: {results['is_kinase']}") - print(f"Is phosphatase: {results['is_phosphatase']}") - print(f"Is transcription factor: {results['is_transcription_factor']}") -``` diff --git a/src/biome/api_definitions/indra/documentation/examples.yaml b/src/biome/api_definitions/indra/documentation/examples.yaml new file mode 100644 index 0000000..546a8a2 --- /dev/null +++ b/src/biome/api_definitions/indra/documentation/examples.yaml @@ -0,0 +1,175 @@ +- query: "Retrieve literature evidence related to AML using MeSH term queries, including child terms, and summarize the findings." + code: | + import requests + import json + + def get_mesh_literature(): + # API base URL + base_url = "https://discovery.indra.bio/api/" + + # Get papers with evidence for AML MeSH term + payload = { + "mesh_term": ["MESH", "D015470"], + "include_child_terms": True + } + + # Make request to get evidences + response = requests.post( + base_url + "get_evidences_for_mesh", + json=payload, + headers={"Content-Type": "application/json"} + ) + + if response.status_code == 200: + # Process and return results + results = response.json() + + # Convert results to a more readable format + evidence_by_statement = {} + for stmt_hash, evidences in results.items(): + evidence_by_statement[stmt_hash] = { + 'pmids': list(set([ev['text_refs'].get('PMID') for ev in evidences if ev.get('text_refs', {}).get('PMID')])), + 'evidence_count': len(evidences) + } + + return evidence_by_statement + else: + print(f"Error: {response.status_code}") + return None + + if __name__ == "__main__": + evidence_data = get_mesh_literature() + + if evidence_data: + # Print summary of findings + print(f"Found {len(evidence_data)} statements with evidence") + + # Get total evidence count + total_evidence = sum(data['evidence_count'] for data in evidence_data.values()) + print(f"Total evidence count: {total_evidence}") + + # Get unique PMIDs + all_pmids = set() + for data in evidence_data.values(): + all_pmids.update(data['pmids']) + print(f"Total unique PMIDs: {len(all_pmids)}") + + +- query: "Perform a gene pathway analysis for JAK2 to find associated pathways and shared pathways with other genes like STAT3, JAK3, and EGFR." + code: | + import requests + + base_url = "https://discovery.indra.bio/api/" + + # Get pathways for JAK2 + jak2_payload = { + "gene": ["HGNC", "6192"] # JAK2 HGNC ID + } + + # Get JAK2 pathways + jak2_pathways_response = requests.post( + base_url + "get_pathways_for_gene", + json=jak2_payload + ) + + print("JAK2 Pathways Response:") + print(jak2_pathways_response.json()) + + # Get shared pathways with common partner genes + partner_genes = [ + ["HGNC", "6192"], # JAK2 + ["HGNC", "3603"], # STAT3 + ["HGNC", "6193"], # JAK3 + ["HGNC", "3236"] # EGFR + ] + + shared_payload = { + "genes": partner_genes + } + + # Get shared pathways + shared_pathways_response = requests.post( + base_url + "get_shared_pathways_for_genes", + json=shared_payload + ) + + print("\nShared Pathways Response:") + print(shared_pathways_response.json()) + + +- query: "This example demonstrates how to analyze the JAK2 gene using the INDRA CoGEx API to retrieve GO terms, pathway associations, and check if it is a kinase, phosphatase, or transcription factor." + code: | + import requests + + def analyze_jak2(): + # API base URL + base_url = "https://discovery.indra.bio/api/" + + # JAK2 HGNC ID + jak2_node = ["HGNC", "6192"] + + # Get GO terms + go_terms_payload = { + "gene": jak2_node, + "include_indirect": True + } + go_response = requests.post( + base_url + "get_go_terms_for_gene", + json=go_terms_payload + ) + + # Get pathways + pathways_payload = { + "gene": jak2_node + } + pathways_response = requests.post( + base_url + "get_pathways_for_gene", + json=pathways_payload + ) + + # Check if kinase + kinase_payload = { + "genes": ["JAK2"] + } + kinase_response = requests.post( + base_url + "is_kinase", + json=kinase_payload + ) + + # Check if phosphatase + phosphatase_payload = { + "genes": ["JAK2"] + } + phosphatase_response = requests.post( + base_url + "is_phosphatase", + json=phosphatase_payload + ) + + # Check if transcription factor + tf_payload = { + "genes": ["JAK2"] + } + tf_response = requests.post( + base_url + "is_transcription_factor", + json=tf_payload + ) + + # Compile and return results + results = { + "go_terms": go_response.json(), + "pathways": pathways_response.json(), + "is_kinase": kinase_response.json().get("JAK2", False), + "is_phosphatase": phosphatase_response.json().get("JAK2", False), + "is_transcription_factor": tf_response.json().get("JAK2", False) + } + + return results + + if __name__ == "__main__": + results = analyze_jak2() + print(f"Analysis Results for JAK2:") + print(f"Number of GO terms: {len(results['go_terms'])}") + print(f"Number of pathways: {len(results['pathways'])}") + print(f"Is kinase: {results['is_kinase']}") + print(f"Is phosphatase: {results['is_phosphatase']}") + print(f"Is transcription factor: {results['is_transcription_factor']}") diff --git a/src/biome/api_definitions/pdc/api.yaml b/src/biome/api_definitions/pdc/api.yaml index 8a71ede..f14408f 100644 --- a/src/biome/api_definitions/pdc/api.yaml +++ b/src/biome/api_definitions/pdc/api.yaml @@ -12,17 +12,14 @@ description: | * Application programming interface (API) provides cloud-agnostic data access and allows third parties to extend the functionality beyond the PDC. * A highly structured workspace that serves as a private user data store and also data submission portal. * Distributes controlled access data, such as the patient-specific protein fasta sequence databases, with dbGaP authorization and eRA Commons authentication. -raw_documentation: !load documentation/pdc_schema.graphql -examples: !load documentation/examples.md +raw_documentation: !load_txt documentation/pdc_schema.graphql +examples: !load_yaml documentation/examples.yaml cache_key: "api_assistant_pdc_graphql" -documentation: | +documentation: !fill | {raw_documentation} # Additional Instructions: This API is through GraphQL. You will be provided the schema. All requests go to the following URL: https://pdc.cancer.gov/graphql - Make all GraphQL requests to that URL. - - Here are some examples of how to use the API: - {examples} \ No newline at end of file + Make all GraphQL requests to that URL. \ No newline at end of file diff --git a/src/biome/api_definitions/pdc/documentation/examples.md b/src/biome/api_definitions/pdc/documentation/examples.md deleted file mode 100644 index 249f3a5..0000000 --- a/src/biome/api_definitions/pdc/documentation/examples.md +++ /dev/null @@ -1,858 +0,0 @@ -# Examples - -## Example 1: obtain quantitative mass spec data on a specific protein (STAT5) - -``` -import requests -import json - -url = 'https://pdc.cancer.gov/graphql' - -# Query for STAT5B protein expression data -query = """ -query ($gene_name: String!) { - getPaginatedUIGeneAliquotSpectralCount(gene_name: $gene_name, offset: 0, limit: 10) { - total - uiGeneAliquotSpectralCounts { - aliquot_id - plex - experiment_type - spectral_count - distinct_peptide - unshared_peptide - precursor_area - log2_ratio - } - } -} -""" - -variables = { - "gene_name": "STAT5B" -} - -try: - response = requests.post(url, json={'query': query, 'variables': variables}) - response.raise_for_status() - data = response.json() - print("STAT5B Protein Expression Data:") - print(json.dumps(data, indent=2)) -except requests.exceptions.RequestException as e: - print(f"An error occurred: {e}") -except json.JSONDecodeError as e: - print(f"Error decoding JSON response: {e}") - print("Response content:", response.text) -``` - -## Example 2: obtain quantitative data - -``` -import requests -import json - -url = 'https://pdc.cancer.gov/graphql' - -# Corrected query structure -query = """ -query ($gene_name: String!) { - getPaginatedUIGeneAliquotSpectralCount( - gene_name: $gene_name, - offset: 0, - limit: 10 - ) { - total - uiGeneAliquotSpectralCounts { - aliquot_id - plex - experiment_type - spectral_count - distinct_peptide - unshared_peptide - log2_ratio - study_submitter_id - } - } -} -""" - -def print_results(data, gene_name): - if 'data' in data and 'getPaginatedUIGeneAliquotSpectralCount' in data['data']: - results = data['data']['getPaginatedUIGeneAliquotSpectralCount'] - total = results['total'] - samples = results['uiGeneAliquotSpectralCounts'] - - print(f"\n{gene_name} Results:") - print(f"Total samples available: {total}") - print("\nSample details (first 10 samples):") - - for i, sample in enumerate(samples): - print(f"\nSample {i+1}:") - print(f" Study ID: {sample['study_submitter_id']}") - print(f" Experiment Type: {sample['experiment_type']}") - print(f" Spectral Count: {sample['spectral_count']}") - print(f" Distinct Peptides: {sample['distinct_peptide']}") - print(f" Unshared Peptides: {sample['unshared_peptide']}") - print(f" Log2 Ratio: {sample['log2_ratio']}") - print(f" Plex: {sample['plex']}") - -try: - # Query STAT5A - response = requests.post(url, json={'query': query, 'variables': {'gene_name': 'STAT5A'}}) - response.raise_for_status() - stat5a_data = response.json() - print_results(stat5a_data, 'STAT5A') - - # Query STAT5B - response = requests.post(url, json={'query': query, 'variables': {'gene_name': 'STAT5B'}}) - response.raise_for_status() - stat5b_data = response.json() - print_results(stat5b_data, 'STAT5B') - -except requests.exceptions.RequestException as e: - print(f"An error occurred: {e}") - if hasattr(e.response, 'text'): - print("Response content:", e.response.text) -except json.JSONDecodeError as e: - print(f"Error decoding JSON response: {e}") - print("Response content:", response.text) -``` - -## Example 3: Load samples into a pandas dataframe - -``` -import requests -import json -import pandas as pd - -url = 'https://pdc.cancer.gov/graphql' - -# Updated query with correct schema -query = """ -query ($gene_name: String!) { - getPaginatedUIGeneAliquotSpectralCount( - gene_name: $gene_name, - offset: 0, - limit: 100 # Increased limit to get more samples - ) { - total - uiGeneAliquotSpectralCounts { - aliquot_id - plex - experiment_type - spectral_count - distinct_peptide - unshared_peptide - precursor_area - log2_ratio - study_submitter_id - } - } -} -""" - -def get_protein_data(gene_name): - response = requests.post(url, json={'query': query, 'variables': {'gene_name': gene_name}}) - response.raise_for_status() - data = response.json() - - if 'data' in data and 'getPaginatedUIGeneAliquotSpectralCount' in data['data']: - results = data['data']['getPaginatedUIGeneAliquotSpectralCount'] - samples = results['uiGeneAliquotSpectralCounts'] - - # Convert to DataFrame - df = pd.DataFrame(samples) - # Add gene name column - df['gene_name'] = gene_name - return df - return None - -try: - # Get data for both proteins - stat5a_df = get_protein_data('STAT5A') - stat5b_df = get_protein_data('STAT5B') - - # Combine the dataframes - combined_df = pd.concat([stat5a_df, stat5b_df], ignore_index=True) - - # Display basic information about the dataset - print("DataFrame Info:") - print(combined_df.info()) - - print("\nFirst few rows of the dataset:") - print(combined_df.head()) - - print("\nSummary statistics:") - print(combined_df.describe()) - - # Save the DataFrame for later use - combined_df.to_csv('stat5_protein_data.csv', index=False) - print("\nData has been saved to 'stat5_protein_data.csv'") - -except requests.exceptions.RequestException as e: - print(f"An error occurred: {e}") - if hasattr(e, 'response') and e.response is not None: - print("Response content:", e.response.text) -except json.JSONDecodeError as e: - print(f"Error decoding JSON response: {e}") - print("Response content:", response.text) -``` - -## Example 4: Query the Proteomics Data Commons (PDC) to find studies with primary site 'Lung' or 'Bronchus and lung' using separate queries and combining results. - -``` -import requests -import json - -# Define the GraphQL endpoint -url = "https://pdc.cancer.gov/graphql" - -# Define the GraphQL query to filter for studies with primary site "Lung" or "Bronchus and lung" -query_lung = ''' -{ - getPaginatedUIStudy(primary_site: "Lung") { - uiStudies { - study_id - submitter_id_name - pdc_study_id - disease_type - primary_site - } - } -} -''' - -query_bronchus_and_lung = ''' -{ - getPaginatedUIStudy(primary_site: "Bronchus and lung") { - uiStudies { - study_id - submitter_id_name - pdc_study_id - disease_type - primary_site - } - } -} -''' - -# Send the request for "Lung" -response_lung = requests.post(url, json={'query': query_lung}) - -# Send the request for "Bronchus and lung" -response_bronchus_and_lung = requests.post(url, json={'query': query_bronchus_and_lung}) - -lung_cancer_studies = [] - -# Check for successful response for "Lung" -if response_lung.status_code == 200: - data_lung = response_lung.json() - lung_cancer_studies.extend(data_lung['data']['getPaginatedUIStudy']['uiStudies']) -else: - print(f"Error: {response_lung.status_code} - {response_lung.text}") - -# Check for successful response for "Bronchus and lung" -if response_bronchus_and_lung.status_code == 200: - data_bronchus_and_lung = response_bronchus_and_lung.json() - lung_cancer_studies.extend(data_bronchus_and_lung['data']['getPaginatedUIStudy']['uiStudies']) -else: - print(f"Error: {response_bronchus_and_lung.status_code} - {response_bronchus_and_lung.text}") - -# Remove duplicates -lung_cancer_studies = {study['study_id']: study for study in lung_cancer_studies}.values() - -print(f"Found {len(lung_cancer_studies)} studies with primary site 'Lung' or 'Bronchus and lung'.") -# Display the first few studies -for study in list(lung_cancer_studies)[:5]: - print(study) -``` - - -## Example 5: Fetch all cases based on a study name - -``` -import requests -import json - -# Define the GraphQL endpoint -url = "https://pdc.cancer.gov/graphql" - -# Define the GraphQL query to get cases for the Georgetown Lung Cancer Proteomics Study using study name -query = ''' -{ - getPaginatedUICase(study_name: "Georgetown Lung Cancer Proteomics Study") { - uiCases { - case_id - case_submitter_id - disease_type - primary_site - } - } -} -''' - -# Send the request for cases -response = requests.post(url, json={'query': query}) - -# Check for successful response for cases -if response.status_code == 200: - data_cases = response.json() - cases_info = data_cases['data']['getPaginatedUICase']['uiCases'] - print("Cases information for the Georgetown Lung Cancer Proteomics Study:") - print(json.dumps(cases_info, indent=2)) -else: - print(f"Error: {response.status_code} - {response.text}") -``` - - -## Example 6: Fetch detailed information about a case using its case ID in the Proteomics Data Commons (PDC). - -``` -import requests -import json - -# Define the GraphQL endpoint -url = "https://pdc.cancer.gov/graphql" - -# Define the GraphQL query to get detailed information about a case -query = ''' -{ - case(case_id: "9e8e8f51-d732-11ea-b1fd-0aad30af8a83") { - case_id - case_submitter_id - disease_type - primary_site - demographics { - gender - race - ethnicity - age_at_index - } - diagnoses { - diagnosis_id - diagnosis_submitter_id - tumor_stage - tumor_grade - } - samples { - sample_id - sample_submitter_id - sample_type - } - } -} -''' - -# Send the request -response = requests.post(url, json={'query': query}) - -# Check for successful response -if response.status_code == 200: - data = response.json() - case_info = data['data']['case'] - print("Detailed information about the case:") - print(json.dumps(case_info, indent=2)) -else: - print(f"Error: {response.status_code} - {response.text}") -``` - - -## Example 7: Fetch raw mass spec data for a specific study using study name, data category, and file type in the Proteomics Data Commons (PDC). - -``` -import requests -import json - -# Define the GraphQL endpoint -url = "https://pdc.cancer.gov/graphql" - -# Define the GraphQL query to get raw mass spec data for the Georgetown Lung Cancer Proteomics Study -query = ''' -{ - getPaginatedUIFile( - study_name: "Georgetown Lung Cancer Proteomics Study", - data_category: "Raw Mass Spectra", - file_type: "Proprietary" - ) { - uiFiles { - file_id - file_name - file_type - file_size - md5sum - downloadable - } - } -} -''' - -# Send the request -response = requests.post(url, json={'query': query}) - -# Check for successful response -if response.status_code == 200: - data = response.json() - file_info = data['data']['getPaginatedUIFile']['uiFiles'] - print("Raw mass spec data for the Georgetown Lung Cancer Proteomics Study:") - print(json.dumps(file_info, indent=2)) -else: - print(f"Error: {response.status_code} - {response.text}") -``` - - -## Example 8: Retrieve a signed URL for downloading a specific file (e.g. raw mass spec file) in a study using the Proteomics Data Commons (PDC) API. - -``` -import requests -import json - -# Define the GraphQL endpoint -url = "https://pdc.cancer.gov/graphql" - -# Define the GraphQL query to get the signed URL for the file -query = ''' -query FilesDataQuery($file_name: String!, $study_id: String!) { - uiFilesPerStudy(file_name: $file_name, study_id: $study_id) { - file_id - file_name - signedUrl { - url - } - } -} -''' - -# Set the variables for the query -variables = { - "file_name": "Ctrl_1-set_1-label_113-frac_2-F9.wiff", - "study_id": "17d5bccf-d028-11ea-b1fd-0aad30af8a83" -} - -# Send the request -response = requests.post(url, json={'query': query, 'variables': variables}) - -# Check for successful response -if response.status_code == 200: - data = response.json() - signed_url = data['data']['uiFilesPerStudy'][0]['signedUrl']['url'] - print(f"Signed URL for the file: {signed_url}") -else: - print(f"Error: {response.status_code} - {response.text}") -``` - - -## Example 9: Retrieve open standard data files for a specific study using the Proteomics Data Commons (PDC) API. - -``` -import requests -import json - -# Define the GraphQL endpoint -url = "https://pdc.cancer.gov/graphql" - -# Define the GraphQL query to get open standard data for a study -query = ''' -{ - getPaginatedUIFile( - study_name: "Georgetown Lung Cancer Proteomics Study", - file_type: "Open Standard" - ) { - uiFiles { - file_id - file_name - file_type - file_size - md5sum - downloadable - } - } -} -''' - -# Send the request -response = requests.post(url, json={'query': query}) - -# Check for successful response -if response.status_code == 200: - data = response.json() - file_info = data['data']['getPaginatedUIFile']['uiFiles'] - print("Open standard data for the study:") - print(json.dumps(file_info, indent=2)) -else: - print(f"Error: {response.status_code} - {response.text}") -``` - - -## Example 10: This example demonstrates how to find lung cancer studies in the Proteomics Data Commons by querying for studies with primary sites 'Lung' and 'Bronchus and lung', and combining the results. - -``` -import requests -import json - -# Define the GraphQL endpoint -url = "https://pdc.cancer.gov/graphql" - -# Define the GraphQL query to filter for studies with primary site "Lung" -query_lung = ''' -{ - getPaginatedUIStudy(primary_site: "Lung") { - uiStudies { - study_id - submitter_id_name - pdc_study_id - disease_type - primary_site - } - } -} -''' - -# Define the GraphQL query to filter for studies with primary site "Bronchus and lung" -query_bronchus_and_lung = ''' -{ - getPaginatedUIStudy(primary_site: "Bronchus and lung") { - uiStudies { - study_id - submitter_id_name - pdc_study_id - disease_type - primary_site - } - } -} -''' - -# Send the request for "Lung" -response_lung = requests.post(url, json={'query': query_lung}) - -# Send the request for "Bronchus and lung" -response_bronchus_and_lung = requests.post(url, json={'query': query_bronchus_and_lung}) - -# Combine the results from both queries -lung_cancer_studies = [] - -if response_lung.status_code == 200: - data_lung = response_lung.json() - lung_cancer_studies.extend(data_lung['data']['getPaginatedUIStudy']['uiStudies']) -else: - print(f"Error: {response_lung.status_code} - {response_lung.text}") - -if response_bronchus_and_lung.status_code == 200: - data_bronchus_and_lung = response_bronchus_and_lung.json() - lung_cancer_studies.extend(data_bronchus_and_lung['data']['getPaginatedUIStudy']['uiStudies']) -else: - print(f"Error: {response_bronchus_and_lung.status_code} - {response_bronchus_and_lung.text}") - -# Remove duplicates -lung_cancer_studies = {study['study_id']: study for study in lung_cancer_studies}.values() - -# Print the number of studies found and the first 5 studies -print(f"Found {len(lung_cancer_studies)} studies with primary site 'Lung' or 'Bronchus and lung'.") -print(list(lung_cancer_studies)[:5]) -``` - - -## Example 11: This example demonstrates how to query the Proteomics Data Commons for studies with open standard data for processed mass spec using the getPaginatedUIFile query. - -``` -import requests -import json - -# Define the GraphQL endpoint -url = "https://pdc.cancer.gov/graphql" - -# Define the GraphQL query to find studies with open standard data for processed mass spec -query = ''' -{ - getPaginatedUIFile( - data_category: "Processed Mass Spectra", - file_type: "Open Standard", - offset: 0, - limit: 10 - ) { - uiFiles { - file_id - file_name - study_id - pdc_study_id - } - } -} -''' - -# Send the request -response = requests.post(url, json={'query': query}) - -# Check for successful response -if response.status_code == 200: - data = response.json() - files = data['data']['getPaginatedUIFile']['uiFiles'] - print("Studies with open standard data for processed mass spec:") - print(json.dumps(files, indent=2)) -else: - print(f"Error: {response.status_code} - {response.text}") -``` - -## Example 12: Query PDC to find studies with a specific disease type (in this case 'Acute Myeloid Leukemia') - -``` -import requests -import json - -url = 'https://pdc.cancer.gov/graphql' - -query = """ -{ - getPaginatedUIStudy(disease_type: "Acute Myeloid Leukemia") { - uiStudies { - study_id - submitter_id_name - pdc_study_id - disease_type - primary_site - } - } -} -""" - -try: - response = requests.post(url, json={'query': query}) - response.raise_for_status() - - # Process the response data - data = response.json() - studies = data['data']['getPaginatedUIStudy']['uiStudies'] - - # Output the results - print(f"Found {len(studies)} studies related to Acute Myeloid Leukemia.") - for study in studies[:5]: # Print the first 5 studies - print(study) - -except requests.exceptions.RequestException as e: - print(f"An error occurred: {e}") -``` - -## Example 13: Fetch all available metadata for a study using its PDC Study ID. - -``` -import requests -import json - -url = 'https://pdc.cancer.gov/graphql' -study_id = 'PDC000478' # Using the PDC Study ID instead - -query = """ -query getStudyMetadata($pdc_study_id: String!) { - getPaginatedUIStudy(pdc_study_id: $pdc_study_id) { - uiStudies { - study_id - pdc_study_id - submitter_id_name - study_description - program_name - project_name - disease_type - primary_site - analytical_fraction - experiment_type - embargo_date - cases_count - aliquots_count - filesCount { - file_type - data_category - files_count - } - supplementaryFilesCount { - data_category - file_type - files_count - } - nonSupplementaryFilesCount { - data_category - file_type - files_count - } - contacts { - name - institution - email - url - } - versions { - number - } - } - } -} -""" - -variables = { - "pdc_study_id": study_id -} - -try: - response = requests.post(url, json={'query': query, 'variables': variables}) - response.raise_for_status() - data = response.json() - # Print a subset of the data - print(json.dumps(data['data']['getPaginatedUIStudy']['uiStudies'][0], indent=2)) - -except requests.exceptions.RequestException as e: - print(f"An error occurred: {e}") - if hasattr(e, 'response') and e.response is not None: - print("Response content:", e.response.text) -except json.JSONDecodeError as e: - print(f"Error decoding JSON response: {e}") - print("Response content:", response.text) -``` - - -## Example 14: Retrieve quantitative mass spectrometry data for STAT5A and STAT5B proteins using the getPaginatedUIGeneAliquotSpectralCount query. - -``` -import requests -import json - -url = 'https://pdc.cancer.gov/graphql' - -query = """ -query ($gene_name: String!, $offset: Int, $limit: Int) { - getPaginatedUIGeneAliquotSpectralCount(gene_name: $gene_name, offset: $offset, limit: $limit) { - total - uiGeneAliquotSpectralCounts { - aliquot_id - plex - experiment_type - spectral_count - distinct_peptide - unshared_peptide - precursor_area - log2_ratio - } - } -} -""" - -variables_stat5a = { - "gene_name": "STAT5A", - "offset": 0, - "limit": 10 # Retrieve the first 10 results -} - -variables_stat5b = { - "gene_name": "STAT5B", - "offset": 0, - "limit": 10 # Retrieve the first 10 results -} - -try: - response_stat5a = requests.post(url, json={'query': query, 'variables': variables_stat5a}) - response_stat5a.raise_for_status() - data_stat5a = response_stat5a.json() - - response_stat5b = requests.post(url, json={'query': query, 'variables': variables_stat5b}) - response_stat5b.raise_for_status() - data_stat5b = response_stat5b.json() - - print("STAT5A Protein Expression Data (First 10 Results):") - print(json.dumps(data_stat5a['data']['getPaginatedUIGeneAliquotSpectralCount']['uiGeneAliquotSpectralCounts'], indent=2)) - - print("\nSTAT5B Protein Expression Data (First 10 Results):") - print(json.dumps(data_stat5b['data']['getPaginatedUIGeneAliquotSpectralCount']['uiGeneAliquotSpectralCounts'], indent=2)) - -except requests.exceptions.RequestException as e: - print(f"An error occurred: {e}") -except json.JSONDecodeError as e: - print(f"Error decoding JSON response: {e}") - print("Response content:", response_stat5a.text) - print("Response content:", response_stat5b.text) -``` - - -## Example 15: Get all clinical data (cases) from PDC and filter for those that have external references to GDC. Return the results as a pandas dataframe. - -``` -import requests -import json -import pandas as pd -from pandas import json_normalize - -url = 'https://pdc.cancer.gov/graphql' - -# Define the GraphQL query -query = """ -query FilteredClinicalDataPaginated($offset_value: Int, $limit_value: Int, $source: String!) { - getPaginatedUIClinical( - offset: $offset_value, - limit: $limit_value, - source: $source - ) { - total - uiClinical { - case_id - case_submitter_id - externalReferences { - reference_resource_shortname - reference_entity_location - } - } - pagination { - count - total - pages - size - } - } -} -""" - -# Set initial query variables -offset_value = 0 -limit_value = 3993 # Fetch 1000 records per request -source = "PDC" - -# Initialize list to store all case IDs -all_case_ids = [] - -total_cases = 3993 # Total number of cases to fetch - -while offset_value < total_cases: - variables = { - "offset_value": offset_value, - "limit_value": limit_value, - "source": source - } - - # Make the API request - response = requests.post(url, json={'query': query, 'variables': variables}) - - if response.status_code == 200: - data = response.json() - - # Extract case IDs - cases = data['data']['getPaginatedUIClinical']['uiClinical'] - all_case_ids.extend([case['case_id'] for case in cases]) - - # Increment offset for next page - offset_value += limit_value - else: - print(f"Error: {response.status_code}") - print(response.text) - break - -# Print total number of case IDs fetched -print(f"Total cases fetched: {len(all_case_ids)}") - -pdc_cases_in_gdc = [] - -for case in data['data']['getPaginatedUIClinical']['uiClinical']: - for exref in case['externalReferences']: - if exref.get('reference_resource_shortname', '') == 'GDC': - case['gdc_case_id'] = exref['reference_entity_location'].split('cases/')[1].strip('\r') - pdc_cases_in_gdc.append(case) - -print(f"Found {len(pdc_cases_in_gdc)} cases that have external references to GDC.") - -pdc_in_gdc_df = pd.DataFrame(json_normalize(pdc_cases_in_gdc)) -pdc_in_gdc_df.head() -``` \ No newline at end of file diff --git a/src/biome/api_definitions/pdc/documentation/examples.yaml b/src/biome/api_definitions/pdc/documentation/examples.yaml new file mode 100644 index 0000000..2b913c1 --- /dev/null +++ b/src/biome/api_definitions/pdc/documentation/examples.yaml @@ -0,0 +1,826 @@ +- query: "obtain quantitative mass spec data on a specific protein (STAT5)" + code: | + import requests + import json + + url = 'https://pdc.cancer.gov/graphql' + + # Query for STAT5B protein expression data + query = """ + query ($gene_name: String!) { + getPaginatedUIGeneAliquotSpectralCount(gene_name: $gene_name, offset: 0, limit: 10) { + total + uiGeneAliquotSpectralCounts { + aliquot_id + plex + experiment_type + spectral_count + distinct_peptide + unshared_peptide + precursor_area + log2_ratio + } + } + } + """ + + variables = { + "gene_name": "STAT5B" + } + + try: + response = requests.post(url, json={'query': query, 'variables': variables}) + response.raise_for_status() + data = response.json() + print("STAT5B Protein Expression Data:") + print(json.dumps(data, indent=2)) + except requests.exceptions.RequestException as e: + print(f"An error occurred: {e}") + except json.JSONDecodeError as e: + print(f"Error decoding JSON response: {e}") + print("Response content:", response.text) + +- query: "obtain quantitative data" + code: | + import requests + import json + + url = 'https://pdc.cancer.gov/graphql' + + # Corrected query structure + query = """ + query ($gene_name: String!) { + getPaginatedUIGeneAliquotSpectralCount( + gene_name: $gene_name, + offset: 0, + limit: 10 + ) { + total + uiGeneAliquotSpectralCounts { + aliquot_id + plex + experiment_type + spectral_count + distinct_peptide + unshared_peptide + log2_ratio + study_submitter_id + } + } + } + """ + + def print_results(data, gene_name): + if 'data' in data and 'getPaginatedUIGeneAliquotSpectralCount' in data['data']: + results = data['data']['getPaginatedUIGeneAliquotSpectralCount'] + total = results['total'] + samples = results['uiGeneAliquotSpectralCounts'] + + print(f"\n{gene_name} Results:") + print(f"Total samples available: {total}") + print("\nSample details (first 10 samples):") + + for i, sample in enumerate(samples): + print(f"\nSample {i+1}:") + print(f" Study ID: {sample['study_submitter_id']}") + print(f" Experiment Type: {sample['experiment_type']}") + print(f" Spectral Count: {sample['spectral_count']}") + print(f" Distinct Peptides: {sample['distinct_peptide']}") + print(f" Unshared Peptides: {sample['unshared_peptide']}") + print(f" Log2 Ratio: {sample['log2_ratio']}") + print(f" Plex: {sample['plex']}") + + try: + # Query STAT5A + response = requests.post(url, json={'query': query, 'variables': {'gene_name': 'STAT5A'}}) + response.raise_for_status() + stat5a_data = response.json() + print_results(stat5a_data, 'STAT5A') + + # Query STAT5B + response = requests.post(url, json={'query': query, 'variables': {'gene_name': 'STAT5B'}}) + response.raise_for_status() + stat5b_data = response.json() + print_results(stat5b_data, 'STAT5B') + + except requests.exceptions.RequestException as e: + print(f"An error occurred: {e}") + if hasattr(e.response, 'text'): + print("Response content:", e.response.text) + except json.JSONDecodeError as e: + print(f"Error decoding JSON response: {e}") + print("Response content:", response.text) + +- query: "Load samples into a pandas dataframe" + code: | + import requests + import json + import pandas as pd + + url = 'https://pdc.cancer.gov/graphql' + + # Updated query with correct schema + query = """ + query ($gene_name: String!) { + getPaginatedUIGeneAliquotSpectralCount( + gene_name: $gene_name, + offset: 0, + limit: 100 # Increased limit to get more samples + ) { + total + uiGeneAliquotSpectralCounts { + aliquot_id + plex + experiment_type + spectral_count + distinct_peptide + unshared_peptide + precursor_area + log2_ratio + study_submitter_id + } + } + } + """ + + def get_protein_data(gene_name): + response = requests.post(url, json={'query': query, 'variables': {'gene_name': gene_name}}) + response.raise_for_status() + data = response.json() + + if 'data' in data and 'getPaginatedUIGeneAliquotSpectralCount' in data['data']: + results = data['data']['getPaginatedUIGeneAliquotSpectralCount'] + samples = results['uiGeneAliquotSpectralCounts'] + + # Convert to DataFrame + df = pd.DataFrame(samples) + # Add gene name column + df['gene_name'] = gene_name + return df + return None + + try: + # Get data for both proteins + stat5a_df = get_protein_data('STAT5A') + stat5b_df = get_protein_data('STAT5B') + + # Combine the dataframes + combined_df = pd.concat([stat5a_df, stat5b_df], ignore_index=True) + + # Display basic information about the dataset + print("DataFrame Info:") + print(combined_df.info()) + + print("\nFirst few rows of the dataset:") + print(combined_df.head()) + + print("\nSummary statistics:") + print(combined_df.describe()) + + # Save the DataFrame for later use + combined_df.to_csv('stat5_protein_data.csv', index=False) + print("\nData has been saved to 'stat5_protein_data.csv'") + + except requests.exceptions.RequestException as e: + print(f"An error occurred: {e}") + if hasattr(e, 'response') and e.response is not None: + print("Response content:", e.response.text) + except json.JSONDecodeError as e: + print(f"Error decoding JSON response: {e}") + print("Response content:", response.text) + +- query: "Query the Proteomics Data Commons (PDC) to find studies with primary site 'Lung' or 'Bronchus and lung' using separate queries and combining results." + code: | + import requests + import json + + # Define the GraphQL endpoint + url = "https://pdc.cancer.gov/graphql" + + # Define the GraphQL query to filter for studies with primary site "Lung" or "Bronchus and lung" + query_lung = ''' + { + getPaginatedUIStudy(primary_site: "Lung") { + uiStudies { + study_id + submitter_id_name + pdc_study_id + disease_type + primary_site + } + } + } + ''' + + query_bronchus_and_lung = ''' + { + getPaginatedUIStudy(primary_site: "Bronchus and lung") { + uiStudies { + study_id + submitter_id_name + pdc_study_id + disease_type + primary_site + } + } + } + ''' + + # Send the request for "Lung" + response_lung = requests.post(url, json={'query': query_lung}) + + # Send the request for "Bronchus and lung" + response_bronchus_and_lung = requests.post(url, json={'query': query_bronchus_and_lung}) + + lung_cancer_studies = [] + + # Check for successful response for "Lung" + if response_lung.status_code == 200: + data_lung = response_lung.json() + lung_cancer_studies.extend(data_lung['data']['getPaginatedUIStudy']['uiStudies']) + else: + print(f"Error: {response_lung.status_code} - {response_lung.text}") + + # Check for successful response for "Bronchus and lung" + if response_bronchus_and_lung.status_code == 200: + data_bronchus_and_lung = response_bronchus_and_lung.json() + lung_cancer_studies.extend(data_bronchus_and_lung['data']['getPaginatedUIStudy']['uiStudies']) + else: + print(f"Error: {response_bronchus_and_lung.status_code} - {response_bronchus_and_lung.text}") + + # Remove duplicates + lung_cancer_studies = {study['study_id']: study for study in lung_cancer_studies}.values() + + print(f"Found {len(lung_cancer_studies)} studies with primary site 'Lung' or 'Bronchus and lung'.") + # Display the first few studies + for study in list(lung_cancer_studies)[:5]: + print(study) + + +- query: "Fetch all cases based on a study name" + code: | + import requests + import json + + # Define the GraphQL endpoint + url = "https://pdc.cancer.gov/graphql" + + # Define the GraphQL query to get cases for the Georgetown Lung Cancer Proteomics Study using study name + query = ''' + { + getPaginatedUICase(study_name: "Georgetown Lung Cancer Proteomics Study") { + uiCases { + case_id + case_submitter_id + disease_type + primary_site + } + } + } + ''' + + # Send the request for cases + response = requests.post(url, json={'query': query}) + + # Check for successful response for cases + if response.status_code == 200: + data_cases = response.json() + cases_info = data_cases['data']['getPaginatedUICase']['uiCases'] + print("Cases information for the Georgetown Lung Cancer Proteomics Study:") + print(json.dumps(cases_info, indent=2)) + else: + print(f"Error: {response.status_code} - {response.text}") + + +- query: "Fetch detailed information about a case using its case ID in the Proteomics Data Commons (PDC)." + code: | + import requests + import json + + # Define the GraphQL endpoint + url = "https://pdc.cancer.gov/graphql" + + # Define the GraphQL query to get detailed information about a case + query = ''' + { + case(case_id: "9e8e8f51-d732-11ea-b1fd-0aad30af8a83") { + case_id + case_submitter_id + disease_type + primary_site + demographics { + gender + race + ethnicity + age_at_index + } + diagnoses { + diagnosis_id + diagnosis_submitter_id + tumor_stage + tumor_grade + } + samples { + sample_id + sample_submitter_id + sample_type + } + } + } + ''' + + # Send the request + response = requests.post(url, json={'query': query}) + + # Check for successful response + if response.status_code == 200: + data = response.json() + case_info = data['data']['case'] + print("Detailed information about the case:") + print(json.dumps(case_info, indent=2)) + else: + print(f"Error: {response.status_code} - {response.text}") + + +- query: "Fetch raw mass spec data for a specific study using study name, data category, and file type in the Proteomics Data Commons (PDC)." + code: | + import requests + import json + + # Define the GraphQL endpoint + url = "https://pdc.cancer.gov/graphql" + + # Define the GraphQL query to get raw mass spec data for the Georgetown Lung Cancer Proteomics Study + query = ''' + { + getPaginatedUIFile( + study_name: "Georgetown Lung Cancer Proteomics Study", + data_category: "Raw Mass Spectra", + file_type: "Proprietary" + ) { + uiFiles { + file_id + file_name + file_type + file_size + md5sum + downloadable + } + } + } + ''' + + # Send the request + response = requests.post(url, json={'query': query}) + + # Check for successful response + if response.status_code == 200: + data = response.json() + file_info = data['data']['getPaginatedUIFile']['uiFiles'] + print("Raw mass spec data for the Georgetown Lung Cancer Proteomics Study:") + print(json.dumps(file_info, indent=2)) + else: + print(f"Error: {response.status_code} - {response.text}") + + +- query: "Retrieve a signed URL for downloading a specific file (e.g. raw mass spec file) in a study using the Proteomics Data Commons (PDC) API." + code: | + import requests + import json + + # Define the GraphQL endpoint + url = "https://pdc.cancer.gov/graphql" + + # Define the GraphQL query to get the signed URL for the file + query = ''' + query FilesDataQuery($file_name: String!, $study_id: String!) { + uiFilesPerStudy(file_name: $file_name, study_id: $study_id) { + file_id + file_name + signedUrl { + url + } + } + } + ''' + + # Set the variables for the query + variables = { + "file_name": "Ctrl_1-set_1-label_113-frac_2-F9.wiff", + "study_id": "17d5bccf-d028-11ea-b1fd-0aad30af8a83" + } + + # Send the request + response = requests.post(url, json={'query': query, 'variables': variables}) + + # Check for successful response + if response.status_code == 200: + data = response.json() + signed_url = data['data']['uiFilesPerStudy'][0]['signedUrl']['url'] + print(f"Signed URL for the file: {signed_url}") + else: + print(f"Error: {response.status_code} - {response.text}") + + +- query: "Retrieve open standard data files for a specific study using the Proteomics Data Commons (PDC) API." + code: | + import requests + import json + + # Define the GraphQL endpoint + url = "https://pdc.cancer.gov/graphql" + + # Define the GraphQL query to get open standard data for a study + query = ''' + { + getPaginatedUIFile( + study_name: "Georgetown Lung Cancer Proteomics Study", + file_type: "Open Standard" + ) { + uiFiles { + file_id + file_name + file_type + file_size + md5sum + downloadable + } + } + } + ''' + + # Send the request + response = requests.post(url, json={'query': query}) + + # Check for successful response + if response.status_code == 200: + data = response.json() + file_info = data['data']['getPaginatedUIFile']['uiFiles'] + print("Open standard data for the study:") + print(json.dumps(file_info, indent=2)) + else: + print(f"Error: {response.status_code} - {response.text}") + + +- query: "This example demonstrates how to find lung cancer studies in the Proteomics Data Commons by querying for studies with primary sites 'Lung' and 'Bronchus and lung', and combining the results." + code: | + import requests + import json + + # Define the GraphQL endpoint + url = "https://pdc.cancer.gov/graphql" + + # Define the GraphQL query to filter for studies with primary site "Lung" + query_lung = ''' + { + getPaginatedUIStudy(primary_site: "Lung") { + uiStudies { + study_id + submitter_id_name + pdc_study_id + disease_type + primary_site + } + } + } + ''' + + # Define the GraphQL query to filter for studies with primary site "Bronchus and lung" + query_bronchus_and_lung = ''' + { + getPaginatedUIStudy(primary_site: "Bronchus and lung") { + uiStudies { + study_id + submitter_id_name + pdc_study_id + disease_type + primary_site + } + } + } + ''' + + # Send the request for "Lung" + response_lung = requests.post(url, json={'query': query_lung}) + + # Send the request for "Bronchus and lung" + response_bronchus_and_lung = requests.post(url, json={'query': query_bronchus_and_lung}) + + # Combine the results from both queries + lung_cancer_studies = [] + + if response_lung.status_code == 200: + data_lung = response_lung.json() + lung_cancer_studies.extend(data_lung['data']['getPaginatedUIStudy']['uiStudies']) + else: + print(f"Error: {response_lung.status_code} - {response_lung.text}") + + if response_bronchus_and_lung.status_code == 200: + data_bronchus_and_lung = response_bronchus_and_lung.json() + lung_cancer_studies.extend(data_bronchus_and_lung['data']['getPaginatedUIStudy']['uiStudies']) + else: + print(f"Error: {response_bronchus_and_lung.status_code} - {response_bronchus_and_lung.text}") + + # Remove duplicates + lung_cancer_studies = {study['study_id']: study for study in lung_cancer_studies}.values() + + # Print the number of studies found and the first 5 studies + print(f"Found {len(lung_cancer_studies)} studies with primary site 'Lung' or 'Bronchus and lung'.") + print(list(lung_cancer_studies)[:5]) + + +- query: "This example demonstrates how to query the Proteomics Data Commons for studies with open standard data for processed mass spec using the getPaginatedUIFile query." + code: | + import requests + import json + + # Define the GraphQL endpoint + url = "https://pdc.cancer.gov/graphql" + + # Define the GraphQL query to find studies with open standard data for processed mass spec + query = ''' + { + getPaginatedUIFile( + data_category: "Processed Mass Spectra", + file_type: "Open Standard", + offset: 0, + limit: 10 + ) { + uiFiles { + file_id + file_name + study_id + pdc_study_id + } + } + } + ''' + + # Send the request + response = requests.post(url, json={'query': query}) + + # Check for successful response + if response.status_code == 200: + data = response.json() + files = data['data']['getPaginatedUIFile']['uiFiles'] + print("Studies with open standard data for processed mass spec:") + print(json.dumps(files, indent=2)) + else: + print(f"Error: {response.status_code} - {response.text}") + +- query: "Query PDC to find studies with a specific disease type (in this case 'Acute Myeloid Leukemia')" + code: | + import requests + import json + + url = 'https://pdc.cancer.gov/graphql' + + query = """ + { + getPaginatedUIStudy(disease_type: "Acute Myeloid Leukemia") { + uiStudies { + study_id + submitter_id_name + pdc_study_id + disease_type + primary_site + } + } + } + """ + + try: + response = requests.post(url, json={'query': query}) + response.raise_for_status() + + # Process the response data + data = response.json() + studies = data['data']['getPaginatedUIStudy']['uiStudies'] + + # Output the results + print(f"Found {len(studies)} studies related to Acute Myeloid Leukemia.") + for study in studies[:5]: # Print the first 5 studies + print(study) + + except requests.exceptions.RequestException as e: + print(f"An error occurred: {e}") + +- query: "Fetch all available metadata for a study using its PDC Study ID." + code: | + import requests + import json + + url = 'https://pdc.cancer.gov/graphql' + study_id = 'PDC000478' # Using the PDC Study ID instead + + query = """ + query getStudyMetadata($pdc_study_id: String!) { + getPaginatedUIStudy(pdc_study_id: $pdc_study_id) { + uiStudies { + study_id + pdc_study_id + submitter_id_name + study_description + program_name + project_name + disease_type + primary_site + analytical_fraction + experiment_type + embargo_date + cases_count + aliquots_count + filesCount { + file_type + data_category + files_count + } + supplementaryFilesCount { + data_category + file_type + files_count + } + nonSupplementaryFilesCount { + data_category + file_type + files_count + } + contacts { + name + institution + email + url + } + versions { + number + } + } + } + } + """ + + variables = { + "pdc_study_id": study_id + } + + try: + response = requests.post(url, json={'query': query, 'variables': variables}) + response.raise_for_status() + data = response.json() + # Print a subset of the data + print(json.dumps(data['data']['getPaginatedUIStudy']['uiStudies'][0], indent=2)) + + except requests.exceptions.RequestException as e: + print(f"An error occurred: {e}") + if hasattr(e, 'response') and e.response is not None: + print("Response content:", e.response.text) + except json.JSONDecodeError as e: + print(f"Error decoding JSON response: {e}") + print("Response content:", response.text) + + +- query: "Retrieve quantitative mass spectrometry data for STAT5A and STAT5B proteins using the getPaginatedUIGeneAliquotSpectralCount query." + code: | + import requests + import json + + url = 'https://pdc.cancer.gov/graphql' + + query = """ + query ($gene_name: String!, $offset: Int, $limit: Int) { + getPaginatedUIGeneAliquotSpectralCount(gene_name: $gene_name, offset: $offset, limit: $limit) { + total + uiGeneAliquotSpectralCounts { + aliquot_id + plex + experiment_type + spectral_count + distinct_peptide + unshared_peptide + precursor_area + log2_ratio + } + } + } + """ + + variables_stat5a = { + "gene_name": "STAT5A", + "offset": 0, + "limit": 10 # Retrieve the first 10 results + } + + variables_stat5b = { + "gene_name": "STAT5B", + "offset": 0, + "limit": 10 # Retrieve the first 10 results + } + + try: + response_stat5a = requests.post(url, json={'query': query, 'variables': variables_stat5a}) + response_stat5a.raise_for_status() + data_stat5a = response_stat5a.json() + + response_stat5b = requests.post(url, json={'query': query, 'variables': variables_stat5b}) + response_stat5b.raise_for_status() + data_stat5b = response_stat5b.json() + + print("STAT5A Protein Expression Data (First 10 Results):") + print(json.dumps(data_stat5a['data']['getPaginatedUIGeneAliquotSpectralCount']['uiGeneAliquotSpectralCounts'], indent=2)) + + print("\nSTAT5B Protein Expression Data (First 10 Results):") + print(json.dumps(data_stat5b['data']['getPaginatedUIGeneAliquotSpectralCount']['uiGeneAliquotSpectralCounts'], indent=2)) + + except requests.exceptions.RequestException as e: + print(f"An error occurred: {e}") + except json.JSONDecodeError as e: + print(f"Error decoding JSON response: {e}") + print("Response content:", response_stat5a.text) + print("Response content:", response_stat5b.text) + + +- query: "Get all clinical data (cases) from PDC and filter for those that have external references to GDC. Return the results as a pandas dataframe." + code: | + import requests + import json + import pandas as pd + from pandas import json_normalize + + url = 'https://pdc.cancer.gov/graphql' + + # Define the GraphQL query + query = """ + query FilteredClinicalDataPaginated($offset_value: Int, $limit_value: Int, $source: String!) { + getPaginatedUIClinical( + offset: $offset_value, + limit: $limit_value, + source: $source + ) { + total + uiClinical { + case_id + case_submitter_id + externalReferences { + reference_resource_shortname + reference_entity_location + } + } + pagination { + count + total + pages + size + } + } + } + """ + + # Set initial query variables + offset_value = 0 + limit_value = 3993 # Fetch 1000 records per request + source = "PDC" + + # Initialize list to store all case IDs + all_case_ids = [] + + total_cases = 3993 # Total number of cases to fetch + + while offset_value < total_cases: + variables = { + "offset_value": offset_value, + "limit_value": limit_value, + "source": source + } + + # Make the API request + response = requests.post(url, json={'query': query, 'variables': variables}) + + if response.status_code == 200: + data = response.json() + + # Extract case IDs + cases = data['data']['getPaginatedUIClinical']['uiClinical'] + all_case_ids.extend([case['case_id'] for case in cases]) + + # Increment offset for next page + offset_value += limit_value + else: + print(f"Error: {response.status_code}") + print(response.text) + break + + # Print total number of case IDs fetched + print(f"Total cases fetched: {len(all_case_ids)}") + + pdc_cases_in_gdc = [] + + for case in data['data']['getPaginatedUIClinical']['uiClinical']: + for exref in case['externalReferences']: + if exref.get('reference_resource_shortname', '') == 'GDC': + case['gdc_case_id'] = exref['reference_entity_location'].split('cases/')[1].strip('\r') + pdc_cases_in_gdc.append(case) + + print(f"Found {len(pdc_cases_in_gdc)} cases that have external references to GDC.") + + pdc_in_gdc_df = pd.DataFrame(json_normalize(pdc_cases_in_gdc)) + pdc_in_gdc_df.head() \ No newline at end of file