diff --git a/Dockerfile b/Dockerfile index aa834de..d03c776 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,9 @@ COPY --chown=1000:1000 . /jupyter RUN rm -f /jupyter/.beaker.conf RUN rm -f /jupyter/.env -RUN uv pip install --system -v -e /jupyter +RUN uv pip install --system /jupyter + +RUN chown jupyter:jupyter -R /usr/local/lib/python3.11/site-packages/biome/adhoc_data/ RUN mkdir -m 777 /var/run/beaker @@ -28,11 +30,5 @@ ENV BEAKER_SUBKERNEL_USER=user ENV BEAKER_RUN_PATH=/var/run/beaker ENV BEAKER_APP=biome.app.BiomeApp -RUN ln -s /jupyter/src/biome/datasources /home/user/datasources -# subkernel access / file pane -RUN ln -s /jupyter/data /home/user/data -# agent path resolution -RUN ln -s /jupyter/data /home/jupyter/data - # Service CMD ["python", "-m", "beaker_kernel.service.server", "--ip", "0.0.0.0"] diff --git a/pyproject.toml b/pyproject.toml index 9aa9e14..5eae0f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "biome" dynamic = ["version"] description = '' readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.10" license = "MIT" keywords = [] authors = [ @@ -25,9 +25,9 @@ classifiers = [ "Programming Language :: Python :: Implementation :: PyPy", ] dependencies = [ - "beaker-kernel~=1.11.0", + "beaker-kernel~=1.12.0", "adhoc-api~=2.4.0", - "archytas~=1.4.0", + "archytas~=1.5.0", "requests", "PyYAML", "idc-index", diff --git a/rest_api/Dockerfile b/rest_api/Dockerfile index 329de1b..16f9b5d 100644 --- a/rest_api/Dockerfile +++ b/rest_api/Dockerfile @@ -5,7 +5,6 @@ FROM python:3.11.5 EXPOSE 8001 COPY ./rest_api /rest_api -COPY ./data /data WORKDIR /rest_api RUN pip install --upgrade --no-cache-dir pip diff --git a/data/census-acs/2023-1yr-api-changes.csv b/src/biome/adhoc_data/data/census-acs/2023-1yr-api-changes.csv similarity index 100% rename from data/census-acs/2023-1yr-api-changes.csv rename to src/biome/adhoc_data/data/census-acs/2023-1yr-api-changes.csv diff --git a/data/census-ahs/census_ahs_codebook.csv b/src/biome/adhoc_data/data/census-ahs/census_ahs_codebook.csv similarity index 100% rename from data/census-ahs/census_ahs_codebook.csv rename to src/biome/adhoc_data/data/census-ahs/census_ahs_codebook.csv diff --git a/data/census-ahs/survey_1999.csv b/src/biome/adhoc_data/data/census-ahs/survey_1999.csv similarity index 100% rename from data/census-ahs/survey_1999.csv rename to src/biome/adhoc_data/data/census-ahs/survey_1999.csv diff --git a/data/census-ahs/survey_2001.csv b/src/biome/adhoc_data/data/census-ahs/survey_2001.csv similarity index 100% rename from data/census-ahs/survey_2001.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2001.csv diff --git a/data/census-ahs/survey_2003.csv b/src/biome/adhoc_data/data/census-ahs/survey_2003.csv similarity index 100% rename from data/census-ahs/survey_2003.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2003.csv diff --git a/data/census-ahs/survey_2005.csv b/src/biome/adhoc_data/data/census-ahs/survey_2005.csv similarity index 100% rename from data/census-ahs/survey_2005.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2005.csv diff --git a/data/census-ahs/survey_2007.csv b/src/biome/adhoc_data/data/census-ahs/survey_2007.csv similarity index 100% rename from data/census-ahs/survey_2007.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2007.csv diff --git a/data/census-ahs/survey_2009.csv b/src/biome/adhoc_data/data/census-ahs/survey_2009.csv similarity index 100% rename from data/census-ahs/survey_2009.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2009.csv diff --git a/data/census-ahs/survey_2011.csv b/src/biome/adhoc_data/data/census-ahs/survey_2011.csv similarity index 100% rename from data/census-ahs/survey_2011.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2011.csv diff --git a/data/census-ahs/survey_2013.csv b/src/biome/adhoc_data/data/census-ahs/survey_2013.csv similarity index 100% rename from data/census-ahs/survey_2013.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2013.csv diff --git a/data/census-ahs/survey_2015.csv b/src/biome/adhoc_data/data/census-ahs/survey_2015.csv similarity index 100% rename from data/census-ahs/survey_2015.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2015.csv diff --git a/data/census-ahs/survey_2017.csv b/src/biome/adhoc_data/data/census-ahs/survey_2017.csv similarity index 100% rename from data/census-ahs/survey_2017.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2017.csv diff --git a/data/census-ahs/survey_2019.csv b/src/biome/adhoc_data/data/census-ahs/survey_2019.csv similarity index 100% rename from data/census-ahs/survey_2019.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2019.csv diff --git a/data/census-ahs/survey_2021.csv b/src/biome/adhoc_data/data/census-ahs/survey_2021.csv similarity index 100% rename from data/census-ahs/survey_2021.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2021.csv diff --git a/data/census-ahs/survey_2023.csv b/src/biome/adhoc_data/data/census-ahs/survey_2023.csv similarity index 100% rename from data/census-ahs/survey_2023.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2023.csv diff --git a/data/census-nsch/2016e_topical.sas7bdat b/src/biome/adhoc_data/data/census-nsch/2016e_topical.sas7bdat similarity index 100% rename from data/census-nsch/2016e_topical.sas7bdat rename to src/biome/adhoc_data/data/census-nsch/2016e_topical.sas7bdat diff --git a/data/census-nsch/2017e_topical.sas7bdat b/src/biome/adhoc_data/data/census-nsch/2017e_topical.sas7bdat similarity index 100% rename from data/census-nsch/2017e_topical.sas7bdat rename to src/biome/adhoc_data/data/census-nsch/2017e_topical.sas7bdat diff --git a/data/census-nsch/2018e_topical.sas7bdat b/src/biome/adhoc_data/data/census-nsch/2018e_topical.sas7bdat similarity index 100% rename from data/census-nsch/2018e_topical.sas7bdat rename to src/biome/adhoc_data/data/census-nsch/2018e_topical.sas7bdat diff --git a/data/census-nsch/2019e_topical.sas7bdat b/src/biome/adhoc_data/data/census-nsch/2019e_topical.sas7bdat similarity index 100% rename from data/census-nsch/2019e_topical.sas7bdat rename to src/biome/adhoc_data/data/census-nsch/2019e_topical.sas7bdat diff --git a/data/census-nsch/2020e_topical.sas7bdat b/src/biome/adhoc_data/data/census-nsch/2020e_topical.sas7bdat similarity index 100% rename from data/census-nsch/2020e_topical.sas7bdat rename to src/biome/adhoc_data/data/census-nsch/2020e_topical.sas7bdat diff --git a/data/census-nsch/2021e_topical.sas7bdat b/src/biome/adhoc_data/data/census-nsch/2021e_topical.sas7bdat similarity index 100% rename from data/census-nsch/2021e_topical.sas7bdat rename to src/biome/adhoc_data/data/census-nsch/2021e_topical.sas7bdat diff --git a/data/census-nsch/2022e_topical.sas7bdat b/src/biome/adhoc_data/data/census-nsch/2022e_topical.sas7bdat similarity index 100% rename from data/census-nsch/2022e_topical.sas7bdat rename to src/biome/adhoc_data/data/census-nsch/2022e_topical.sas7bdat diff --git a/data/census-nsch/2023e_topical.sas7bdat b/src/biome/adhoc_data/data/census-nsch/2023e_topical.sas7bdat similarity index 100% rename from data/census-nsch/2023e_topical.sas7bdat rename to src/biome/adhoc_data/data/census-nsch/2023e_topical.sas7bdat diff --git a/data/census-nsch/nsch_dictionary_codebook.csv b/src/biome/adhoc_data/data/census-nsch/nsch_dictionary_codebook.csv similarity index 100% rename from data/census-nsch/nsch_dictionary_codebook.csv rename to src/biome/adhoc_data/data/census-nsch/nsch_dictionary_codebook.csv diff --git a/data/chis_california_asthma/current-asthma-prevalence-by-county-2015_2022.csv b/src/biome/adhoc_data/data/chis_california_asthma/current-asthma-prevalence-by-county-2015_2022.csv similarity index 100% rename from data/chis_california_asthma/current-asthma-prevalence-by-county-2015_2022.csv rename to src/biome/adhoc_data/data/chis_california_asthma/current-asthma-prevalence-by-county-2015_2022.csv diff --git a/data/epa-tri/EPA_TRI_Toxics_2014_2023.csv b/src/biome/adhoc_data/data/epa-tri/EPA_TRI_Toxics_2014_2023.csv similarity index 100% rename from data/epa-tri/EPA_TRI_Toxics_2014_2023.csv rename to src/biome/adhoc_data/data/epa-tri/EPA_TRI_Toxics_2014_2023.csv diff --git a/data/fda-gras/GRASNotices.csv b/src/biome/adhoc_data/data/fda-gras/GRASNotices.csv similarity index 100% rename from data/fda-gras/GRASNotices.csv rename to src/biome/adhoc_data/data/fda-gras/GRASNotices.csv diff --git a/data/nhanes_dietary/1999_2000_DEMOGRAPHICS.xpt b/src/biome/adhoc_data/data/nhanes_dietary/1999_2000_DEMOGRAPHICS.xpt similarity index 100% rename from data/nhanes_dietary/1999_2000_DEMOGRAPHICS.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/1999_2000_DEMOGRAPHICS.xpt diff --git a/data/nhanes_dietary/1999_2000_DRXFMT_food_codes.xpt b/src/biome/adhoc_data/data/nhanes_dietary/1999_2000_DRXFMT_food_codes.xpt similarity index 100% rename from data/nhanes_dietary/1999_2000_DRXFMT_food_codes.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/1999_2000_DRXFMT_food_codes.xpt diff --git a/data/nhanes_dietary/1999_2000_DRXIFF_individual_foods.xpt b/src/biome/adhoc_data/data/nhanes_dietary/1999_2000_DRXIFF_individual_foods.xpt similarity index 100% rename from data/nhanes_dietary/1999_2000_DRXIFF_individual_foods.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/1999_2000_DRXIFF_individual_foods.xpt diff --git a/data/nhanes_dietary/2005_2006_DEMOGRAPHICS.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2005_2006_DEMOGRAPHICS.xpt similarity index 100% rename from data/nhanes_dietary/2005_2006_DEMOGRAPHICS.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2005_2006_DEMOGRAPHICS.xpt diff --git a/data/nhanes_dietary/2005_2006_DR1IFF_individual_foods.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2005_2006_DR1IFF_individual_foods.xpt similarity index 100% rename from data/nhanes_dietary/2005_2006_DR1IFF_individual_foods.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2005_2006_DR1IFF_individual_foods.xpt diff --git a/data/nhanes_dietary/2005_2006_DRXFCD_food_codes.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2005_2006_DRXFCD_food_codes.xpt similarity index 100% rename from data/nhanes_dietary/2005_2006_DRXFCD_food_codes.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2005_2006_DRXFCD_food_codes.xpt diff --git a/data/nhanes_dietary/2005_2006_DRXMCD_modification_codes.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2005_2006_DRXMCD_modification_codes.xpt similarity index 100% rename from data/nhanes_dietary/2005_2006_DRXMCD_modification_codes.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2005_2006_DRXMCD_modification_codes.xpt diff --git a/data/nhanes_dietary/2009-2010_DEMOGRAPHICS.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2009-2010_DEMOGRAPHICS.xpt similarity index 100% rename from data/nhanes_dietary/2009-2010_DEMOGRAPHICS.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2009-2010_DEMOGRAPHICS.xpt diff --git a/data/nhanes_dietary/2009_2010_DR1IFF_individual_foods.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2009_2010_DR1IFF_individual_foods.xpt similarity index 100% rename from data/nhanes_dietary/2009_2010_DR1IFF_individual_foods.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2009_2010_DR1IFF_individual_foods.xpt diff --git a/data/nhanes_dietary/2009_2010_DRXFCD_food_codes.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2009_2010_DRXFCD_food_codes.xpt similarity index 100% rename from data/nhanes_dietary/2009_2010_DRXFCD_food_codes.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2009_2010_DRXFCD_food_codes.xpt diff --git a/data/nhanes_dietary/2009_2010_DRXMCD_modification_codes.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2009_2010_DRXMCD_modification_codes.xpt similarity index 100% rename from data/nhanes_dietary/2009_2010_DRXMCD_modification_codes.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2009_2010_DRXMCD_modification_codes.xpt diff --git a/data/nhanes_dietary/2015_2016_DEMOGRAPHICS.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2015_2016_DEMOGRAPHICS.xpt similarity index 100% rename from data/nhanes_dietary/2015_2016_DEMOGRAPHICS.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2015_2016_DEMOGRAPHICS.xpt diff --git a/data/nhanes_dietary/2015_2016_DR1IFF_individual_foods.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2015_2016_DR1IFF_individual_foods.xpt similarity index 100% rename from data/nhanes_dietary/2015_2016_DR1IFF_individual_foods.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2015_2016_DR1IFF_individual_foods.xpt diff --git a/data/nhanes_dietary/2015_2016_DRXFCD_food_codes.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2015_2016_DRXFCD_food_codes.xpt similarity index 100% rename from data/nhanes_dietary/2015_2016_DRXFCD_food_codes.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2015_2016_DRXFCD_food_codes.xpt diff --git a/data/nhanes_dietary/2017_2020_DEMOGRAPHICS.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2017_2020_DEMOGRAPHICS.xpt similarity index 100% rename from data/nhanes_dietary/2017_2020_DEMOGRAPHICS.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2017_2020_DEMOGRAPHICS.xpt diff --git a/data/nhanes_dietary/2017_2020_DR1IFF_individual_foods.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2017_2020_DR1IFF_individual_foods.xpt similarity index 100% rename from data/nhanes_dietary/2017_2020_DR1IFF_individual_foods.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2017_2020_DR1IFF_individual_foods.xpt diff --git a/data/nhanes_dietary/2017_2020_DRXFCD_food_codes.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2017_2020_DRXFCD_food_codes.xpt similarity index 100% rename from data/nhanes_dietary/2017_2020_DRXFCD_food_codes.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2017_2020_DRXFCD_food_codes.xpt diff --git a/data/nhanes_dietary/2021_2023_DEMOGRAPHICS.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2021_2023_DEMOGRAPHICS.xpt similarity index 100% rename from data/nhanes_dietary/2021_2023_DEMOGRAPHICS.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2021_2023_DEMOGRAPHICS.xpt diff --git a/data/nhanes_dietary/2021_2023_DR1IFF_individual_foods.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2021_2023_DR1IFF_individual_foods.xpt similarity index 100% rename from data/nhanes_dietary/2021_2023_DR1IFF_individual_foods.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2021_2023_DR1IFF_individual_foods.xpt diff --git a/data/nhanes_dietary/2021_2023_DRXFCD_food_codes.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2021_2023_DRXFCD_food_codes.xpt similarity index 100% rename from data/nhanes_dietary/2021_2023_DRXFCD_food_codes.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2021_2023_DRXFCD_food_codes.xpt diff --git a/data/usgs_pesticide/EPest.county.estimates.2000.txt b/src/biome/adhoc_data/data/usgs_pesticide/EPest.county.estimates.2000.txt similarity index 100% rename from data/usgs_pesticide/EPest.county.estimates.2000.txt rename to src/biome/adhoc_data/data/usgs_pesticide/EPest.county.estimates.2000.txt diff --git a/data/usgs_pesticide/EPest.county.estimates.2005.txt b/src/biome/adhoc_data/data/usgs_pesticide/EPest.county.estimates.2005.txt similarity index 100% rename from data/usgs_pesticide/EPest.county.estimates.2005.txt rename to src/biome/adhoc_data/data/usgs_pesticide/EPest.county.estimates.2005.txt diff --git a/data/usgs_pesticide/EPest.county.estimates.2010.txt b/src/biome/adhoc_data/data/usgs_pesticide/EPest.county.estimates.2010.txt similarity index 100% rename from data/usgs_pesticide/EPest.county.estimates.2010.txt rename to src/biome/adhoc_data/data/usgs_pesticide/EPest.county.estimates.2010.txt diff --git a/data/usgs_pesticide/EPest_county_estimates_2013_2017_v2.txt b/src/biome/adhoc_data/data/usgs_pesticide/EPest_county_estimates_2013_2017_v2.txt similarity index 100% rename from data/usgs_pesticide/EPest_county_estimates_2013_2017_v2.txt rename to src/biome/adhoc_data/data/usgs_pesticide/EPest_county_estimates_2013_2017_v2.txt diff --git a/data/usgs_pesticide/EPest_county_estimates_2019.txt b/src/biome/adhoc_data/data/usgs_pesticide/EPest_county_estimates_2019.txt similarity index 100% rename from data/usgs_pesticide/EPest_county_estimates_2019.txt rename to src/biome/adhoc_data/data/usgs_pesticide/EPest_county_estimates_2019.txt diff --git a/src/biome/adhoc_data/datasources/ahs/api.yaml b/src/biome/adhoc_data/datasources/ahs/api.yaml new file mode 100644 index 0000000..cf591d1 --- /dev/null +++ b/src/biome/adhoc_data/datasources/ahs/api.yaml @@ -0,0 +1,446 @@ +datatype: dataset +description: | + You have access to knowledge to all the Census Housing survey data, every 2 years from the + year 1999 to 2023. You can use this API for answering questions on american housing, specifically their + housing conditions (mold, water problems, etc), which can be used to correlate to health conditions, + or other purposes. +img_url: null +last_updated: null +name: Census American Housing Survey (AHS) +provider: adhoc:specialist_agents +resources: + 3a5b7607-80c5-44ae-a240-a9a388f78afd: + filepath: docs.md + integration: beb85142-d32f-42df-a438-b84829ddd4fd + name: more_instructions + resource_id: 3a5b7607-80c5-44ae-a240-a9a388f78afd + resource_type: file + 5025a179-2b90-46ab-ab7e-deef3aa8b129: + code: | + import pandas as pd + + def get_code_name(code): + # Read codebook to get metro area meanings + # ensure to set index_col=False, since the codebook has no index column + codebook = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/census-ahs/census_ahs_codebook.csv', index_col=False) + + # Find the row for OMB13CBSA + ombs_row = codebook[codebook['Variable'] == 'OMB13CBSA'] + + response_codes_text = ombs_row['Response Codes'].iloc[0] # Get the string value from the Series + code_pairs = response_codes_text.split("||") + + code_pairs_dict = {k.strip(): v.strip() for k, v in (code_pair.split(":") for code_pair in code_pairs)} + + return code_pairs_dict.get(code) + integration: beb85142-d32f-42df-a438-b84829ddd4fd + notes: null + query: |- + Use the codebook to get the meaning of the codes for the OMB13CBSA variable, matching one code to the meaning by creating a dictionary of code:meaning pairs and returning the statistical metro area name for a given code + resource_id: 5025a179-2b90-46ab-ab7e-deef3aa8b129 + resource_type: example + 569a9b73-d61a-4b9a-8ee1-9c85c1637e1a: + code: "import pandas as pd\nimport numpy as np\n\n# Load the AHS dataset (2023)\ + \ with asthma-related variables\nfile_path = '{{DATASET_FILES_BASE_PATH}}/census-ahs/survey_2023.csv'\n\ + \n# Define variables of interest\nasthma_vars = ['HHLDASTHMA'] # Household\ + \ member ever diagnosed with asthma\nhousing_vars = [\n 'MOLDBASEM', 'MOLDBATH',\ + \ 'MOLDBEDRM', 'MOLDKITCH', 'MOLDLROOM', 'MOLDOTHER', # Mold\n 'LEAKI',\ + \ 'LEAKO' # Water\ + \ leaks\n]\nchild_vars = ['NUMOLDKIDS', 'NUMYNGKIDS']\ngeo_vars = ['OMB13CBSA']\n\ + \n# Combine all variables of interest\ncols_of_interest = asthma_vars + housing_vars\ + \ + child_vars + geo_vars\n\n# Load a sample of the data\nsample_size = 10000\ + \ # Adjust based on memory constraints\nahs_sample = pd.read_csv(file_path,\ + \ usecols=cols_of_interest, nrows=sample_size)\n\n# Clean the data by removing\ + \ quotes from string values\nfor col in ahs_sample.columns:\n if ahs_sample[col].dtype\ + \ == 'object':\n ahs_sample[col] = ahs_sample[col].str.replace(\"'\"\ + , \"\")\n\n# Create a binary variable for presence of children\nahs_sample['has_children']\ + \ = ((ahs_sample['NUMOLDKIDS'] > 0) | (ahs_sample['NUMYNGKIDS'] > 0)).astype(int)\n\ + \n# Convert asthma variable to binary (Yes/No)\nahs_sample['asthma'] = ahs_sample['HHLDASTHMA'].apply(\n\ + \ lambda x: 1 if x == '1' else (0 if x == '2' else np.nan)\n)\n\n# Convert\ + \ housing condition variables to binary (Yes/No)\nfor col in housing_vars:\n\ + \ ahs_sample[col + '_binary'] = ahs_sample[col].apply(\n lambda x:\ + \ 1 if x == '1' else (0 if x == '2' else np.nan)\n )\n\n# Analyze relationship\ + \ between housing conditions and asthma\nprint(\"Relationship between asthma\ + \ and housing conditions:\")\nfor col in housing_vars:\n binary_col = col\ + \ + '_binary'\n valid_data = ahs_sample.dropna(subset=['asthma', binary_col])\n\ + \ \n if len(valid_data) > 0:\n with_asthma = valid_data[valid_data['asthma']\ + \ == 1][binary_col].mean() * 100\n without_asthma = valid_data[valid_data['asthma']\ + \ == 0][binary_col].mean() * 100\n \n if not np.isnan(with_asthma)\ + \ and not np.isnan(without_asthma):\n print(f\"\\n{col}:\")\n \ + \ print(f\" - Homes with asthma: {with_asthma:.1f}%\")\n \ + \ print(f\" - Homes without asthma: {without_asthma:.1f}%\")\n \ + \ print(f\" - Difference: {with_asthma - without_asthma:.1f} percentage points\"\ + )\n\n# Calculate the prevalence of asthma in households with and without children\n\ + asthma_by_children = ahs_sample.dropna(subset=['asthma', 'has_children']).groupby('has_children')['asthma'].mean()\ + \ * 100\nprint(\"\\nAsthma Prevalence by Presence of Children:\")\nprint(f\"\ + Households with children: {asthma_by_children.get(1, 'N/A'):.1f}%\")\nprint(f\"\ + Households without children: {asthma_by_children.get(0, 'N/A'):.1f}%\")\nprint(f\"\ + Difference: {asthma_by_children.get(1, 0) - asthma_by_children.get(0, 0):.1f}\ + \ percentage points\")\n" + integration: beb85142-d32f-42df-a438-b84829ddd4fd + notes: null + query: |- + Analyzing the relationship between housing conditions, asthma, and the presence of children using the 2023 American Housing Survey data. This code calculates and compares the prevalence of various housing conditions (mold, water leaks) in homes with and without reported asthma cases, and also examines the prevalence of asthma in households with and without children. + resource_id: 569a9b73-d61a-4b9a-8ee1-9c85c1637e1a + resource_type: example + 80f97f58-e289-4afe-9959-37212def3ce6: + code: "import pandas as pd\nimport numpy as np\n\n# Load the 2007 AHS dataset\ + \ with Houston data\nfile_path = '{{DATASET_FILES_BASE_PATH}}/census-ahs/survey_2007.csv'\n\ + \n# Define Houston SMSA code\nhouston_smsa = '3360'\n\n# Define housing condition\ + \ variables based on our exploration\nhousing_vars = [\n 'LEAK', 'ILEAK',\ + \ # Water leaks\n 'RATS', 'MICE' \ + \ # Pests\n]\n\n# Define household size\ + \ variable\nhousehold_var = 'PER'\n\n# Combine all columns of interest\ncols_of_interest\ + \ = ['SMSA'] + housing_vars + [household_var]\n\n# Load the Houston data\nhouston_data\ + \ = pd.read_csv(file_path, usecols=cols_of_interest)\n\n# Clean the data by\ + \ removing quotes if needed\nfor col in houston_data.columns:\n if houston_data[col].dtype\ + \ == 'object':\n houston_data[col] = houston_data[col].str.replace(\"\ + '\", \"\")\n\n# Filter for Houston\nhouston_data = houston_data[houston_data['SMSA']\ + \ == houston_smsa]\n\nprint(f\"Loaded {len(houston_data)} Houston records from\ + \ 2007 survey\")\n\n# Create a binary variable for households likely to have\ + \ children (3+ persons)\n# This is a proxy since we don't have direct child\ + \ indicators\nhouston_data['likely_has_children'] = houston_data[household_var].apply(\n\ + \ lambda x: 1 if x not in ['-6', '-9'] and int(x) >= 3 else 0\n)\n\nprint(f\"\ + \\nHouseholds likely to have children (3+ persons): {houston_data['likely_has_children'].sum()}\ + \ ({houston_data['likely_has_children'].mean()*100:.1f}% of Houston sample)\"\ + )\n\n# Convert housing condition variables to binary (Yes/No)\nfor col in housing_vars:\n\ + \ houston_data[col + '_binary'] = houston_data[col].apply(\n lambda\ + \ x: 1 if x == '1' else (0 if x == '2' else np.nan)\n )\n\n# Analyze housing\ + \ conditions\nprint(\"\\nPrevalence of housing conditions in Houston (2007):\"\ + )\nfor col in housing_vars:\n binary_col = col + '_binary'\n valid_data\ + \ = houston_data.dropna(subset=[binary_col])\n if len(valid_data) > 0:\n\ + \ yes_pct = valid_data[binary_col].mean() * 100\n print(f\"{col}:\ + \ {yes_pct:.1f}% reported 'Yes' (based on {len(valid_data)} valid responses)\"\ + )\n\n# Analyze relationship between household size and housing conditions\n\ + print(\"\\nRelationship between household size and housing conditions in Houston\ + \ (2007):\")\nfor col in housing_vars:\n binary_col = col + '_binary'\n \ + \ valid_data = houston_data.dropna(subset=[binary_col, 'likely_has_children'])\n\ + \ \n if len(valid_data) > 0:\n with_children = valid_data[valid_data['likely_has_children']\ + \ == 1][binary_col].mean() * 100\n without_children = valid_data[valid_data['likely_has_children']\ + \ == 0][binary_col].mean() * 100\n \n if not np.isnan(with_children)\ + \ and not np.isnan(without_children):\n print(f\"\\n{col}:\")\n \ + \ print(f\" - Households with 3+ persons: {with_children:.1f}%\"\ + )\n print(f\" - Households with 1-2 persons: {without_children:.1f}%\"\ + )\n print(f\" - Difference: {with_children - without_children:.1f}\ + \ percentage points\")\n" + integration: beb85142-d32f-42df-a438-b84829ddd4fd + notes: null + query: |- + Analyzing housing conditions in Houston, TX using the 2007 American Housing Survey data. This code identifies Houston records using the SMSA code, analyzes the prevalence of water leaks and pest problems, and compares these conditions between larger households (3+ persons, likely to have children) and smaller households (1-2 persons). + resource_id: 80f97f58-e289-4afe-9959-37212def3ce6 + resource_type: example + ad42a806-0e81-4e63-97cf-2f7aad6a1a35: + code: | + import pandas as pd + + # Read codebook to get metro area meanings + # ensure to set index_col=False, since the codebook has no index column + codebook = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/census-ahs/census_ahs_codebook.csv', index_col=False) + + # Find the row for OMB13CBSA + ombs_row = codebook[codebook['Variable'] == 'OMB13CBSA'] + + # Return the Response Codes column + ombs_row['Response Codes'] + integration: beb85142-d32f-42df-a438-b84829ddd4fd + notes: null + query: What is the meaning of the codes for the OMB13CBSA variable? + resource_id: ad42a806-0e81-4e63-97cf-2f7aad6a1a35 + resource_type: example + badf3293-de67-4c9c-944e-6dbbb8b609d6: + code: "import pandas as pd\nimport numpy as np\n\n# Load the codebook to find\ + \ information about variables\ncodebook_path = '{{DATASET_FILES_BASE_PATH}}/census-ahs/census_ahs_codebook.csv'\n\ + codebook = pd.read_csv(codebook_path, dtype=str, index_col=False)\n\n# Define\ + \ search terms related to asthma and respiratory health\nasthma_terms = [\n\ + \ 'asthma', 'allerg', 'respir', 'breath', 'lung', 'air', 'ventil', \n \ + \ 'humid', 'mold', 'mildew', 'dust', 'smoke', 'pest', 'roach', 'rodent', \n\ + \ 'insect', 'heat', 'cool', 'toxic', 'pollut', 'chemical', 'lead'\n]\n\n\ + # Search for variables related to these terms\nasthma_vars = []\nfor term in\ + \ asthma_terms:\n term_vars = codebook[\n (codebook['Description'].str.contains(term,\ + \ case=False, na=False)) |\n (codebook['Question Text'].str.contains(term,\ + \ case=False, na=False)) |\n (codebook['Variable'].str.contains(term,\ + \ case=False, na=False))\n ]\n if not term_vars.empty:\n for _,\ + \ row in term_vars.iterrows():\n var_info = {\n 'Variable':\ + \ row['Variable'],\n 'Description': row['Description'] if 'Description'\ + \ in row and not pd.isna(row['Description']) else 'No description',\n \ + \ 'Question_Text': row['Question Text'] if 'Question Text' in row and\ + \ not pd.isna(row['Question Text']) else 'No question text',\n \ + \ 'Survey_Years': row['Survey Years'] if 'Survey Years' in row and not pd.isna(row['Survey\ + \ Years']) else 'Unknown',\n 'Response_Codes': row['Response\ + \ Codes'] if 'Response Codes' in row and not pd.isna(row['Response Codes'])\ + \ else 'No codes'\n }\n # Check if this variable is already\ + \ in our list\n if not any(v['Variable'] == var_info['Variable']\ + \ for v in asthma_vars):\n asthma_vars.append(var_info)\n\n#\ + \ Create a DataFrame for easier viewing\nasthma_vars_df = pd.DataFrame(asthma_vars)\n\ + \n# Check if we have any variables specifically about asthma\nasthma_specific\ + \ = asthma_vars_df[\n asthma_vars_df['Description'].str.contains('asthma',\ + \ case=False, na=False) |\n asthma_vars_df['Question_Text'].str.contains('asthma',\ + \ case=False, na=False)\n]\n\nprint(f\"Found {len(asthma_vars_df)} variables\ + \ potentially related to asthma and respiratory health\")\nprint(f\"Found {len(asthma_specific)}\ + \ variables specifically mentioning asthma\")\n\n# Display the asthma-specific\ + \ variables if any\nif not asthma_specific.empty:\n print(\"\\nVariables\ + \ specifically mentioning asthma:\")\n for _, row in asthma_specific.head(5).iterrows():\ + \ # Show first 5 for brevity\n print(f\"\\nVariable: {row['Variable']}\"\ + )\n print(f\"Description: {row['Description']}\")\n print(f\"\ + Question Text: {row['Question_Text']}\")\n print(f\"Survey Years: {row['Survey_Years']}\"\ + )\n\n# Check for variables related to indoor air quality\nair_quality = asthma_vars_df[\n\ + \ asthma_vars_df['Description'].str.contains('air quality|ventil|humid',\ + \ case=False, na=False) |\n asthma_vars_df['Question_Text'].str.contains('air\ + \ quality|ventil|humid', case=False, na=False)\n]\n\nprint(f\"\\nFound {len(air_quality)}\ + \ variables related to indoor air quality\")\n\n# Check for variables related\ + \ to pests\npest_vars = asthma_vars_df[\n asthma_vars_df['Description'].str.contains('pest|roach|rodent|insect|rat|mice|mouse',\ + \ case=False, na=False) |\n asthma_vars_df['Question_Text'].str.contains('pest|roach|rodent|insect|rat|mice|mouse',\ + \ case=False, na=False)\n]\n\nprint(f\"\\nFound {len(pest_vars)} variables related\ + \ to pests\")\n\n# Check for variables related to heating and cooling systems\n\ + hvac_vars = asthma_vars_df[\n asthma_vars_df['Description'].str.contains('heat|cool|air\ + \ condition|hvac', case=False, na=False) |\n asthma_vars_df['Question_Text'].str.contains('heat|cool|air\ + \ condition|hvac', case=False, na=False)\n]\n\nprint(f\"\\nFound {len(hvac_vars)}\ + \ variables related to heating and cooling systems\")" + integration: beb85142-d32f-42df-a438-b84829ddd4fd + notes: null + query: |- + Exploring the American Housing Survey codebook to identify variables related to asthma, respiratory health, and housing conditions. This code searches the codebook for terms related to asthma, air quality, pests, and HVAC systems, and provides a summary of available variables that could be relevant for analyzing the relationship between housing conditions and respiratory health. + resource_id: badf3293-de67-4c9c-944e-6dbbb8b609d6 + resource_type: example + e72b358a-6a63-4d79-9ee5-e81b5d99af2b: + code: "import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\ + import seaborn as sns\nfrom matplotlib.ticker import PercentFormatter\n\n# Load\ + \ a sample of the AHS dataset (2023)\nfile_path = '{{DATASET_FILES_BASE_PATH}}/census-ahs/survey_2023.csv'\n\ + \n# Define columns of interest based on our exploration\ngeo_cols = ['OMB13CBSA']\n\ + mold_cols = ['MOLDBASEM', 'MOLDBATH', 'MOLDBEDRM', 'MOLDKITCH', 'MOLDLROOM',\ + \ 'MOLDOTHER']\nleak_cols = ['LEAKI', 'LEAKO']\nchild_cols = ['NUMOLDKIDS',\ + \ 'NUMYNGKIDS']\n\n# Combine all columns of interest\ncols_of_interest = geo_cols\ + \ + mold_cols + leak_cols + child_cols\n\n# Load a sample of the data with only\ + \ the columns we need\nsample_size = 10000 # Adjust based on memory constraints\n\ + ahs_sample = pd.read_csv(file_path, usecols=cols_of_interest, nrows=sample_size)\n\ + \n# Clean the data by removing quotes from string values\nfor col in ahs_sample.columns:\n\ + \ if ahs_sample[col].dtype == 'object':\n ahs_sample[col] = ahs_sample[col].str.replace(\"\ + '\", \"\")\n\n# Create a binary variable for presence of children\nahs_sample['has_children']\ + \ = ((ahs_sample['NUMOLDKIDS'] > 0) | (ahs_sample['NUMYNGKIDS'] > 0)).astype(int)\n\ + \n# Convert categorical variables to numeric for analysis\nfor col in mold_cols\ + \ + leak_cols:\n # Convert '1' (Yes) to 1, '2' (No) to 0, and others to NaN\n\ + \ ahs_sample[col] = ahs_sample[col].apply(\n lambda x: 1 if x == '1'\ + \ else (0 if x == '2' else np.nan)\n )\n\n# Create a more descriptive mapping\ + \ for the variables\nvariable_descriptions = {\n 'MOLDBASEM': 'Mold in Basement',\n\ + \ 'MOLDBATH': 'Mold in Bathroom',\n 'MOLDBEDRM': 'Mold in Bedroom',\n\ + \ 'MOLDKITCH': 'Mold in Kitchen',\n 'MOLDLROOM': 'Mold in Living Room',\n\ + \ 'MOLDOTHER': 'Mold in Other Areas',\n 'LEAKI': 'Inside Water Leaks',\n\ + \ 'LEAKO': 'Outside Water Leaks'\n}\n\n# Prepare data for visualization\n\ + conditions = []\nwith_children_vals = []\nwithout_children_vals = []\nsample_sizes\ + \ = []\n\nfor condition in mold_cols + leak_cols:\n valid_data = ahs_sample.dropna(subset=[condition,\ + \ 'has_children'])\n \n if len(valid_data) > 0:\n with_children_val\ + \ = valid_data[valid_data['has_children'] == 1][condition].mean() * 100\n \ + \ without_children_val = valid_data[valid_data['has_children'] == 0][condition].mean()\ + \ * 100\n \n if not np.isnan(with_children_val) and not np.isnan(without_children_val):\n\ + \ conditions.append(variable_descriptions.get(condition, condition))\n\ + \ with_children_vals.append(with_children_val)\n without_children_vals.append(without_children_val)\n\ + \ sample_sizes.append(len(valid_data))\n\n# Create a DataFrame for\ + \ plotting\nplot_data = pd.DataFrame({\n 'Condition': conditions,\n 'With\ + \ Children (%)': with_children_vals,\n 'Without Children (%)': without_children_vals,\n\ + \ 'Sample Size': sample_sizes\n})\n\n# Sort by prevalence in homes with children\n\ + plot_data = plot_data.sort_values(by='With Children (%)', ascending=False)\n\ + \n# Create a horizontal bar chart\nplt.figure(figsize=(12, 10))\nsns.set_style(\"\ + whitegrid\")\n\n# Create the horizontal bar chart\nax = sns.barplot(\n x='With\ + \ Children (%)', \n y='Condition', \n data=plot_data, \n color='#3498db',\n\ + \ label='Homes with Children'\n)\n\nsns.barplot(\n x='Without Children\ + \ (%)', \n y='Condition', \n data=plot_data, \n color='#e74c3c',\n\ + \ label='Homes without Children'\n)\n\n# Add percentage labels to the bars\n\ + for i, (with_val, without_val) in enumerate(zip(plot_data['With Children (%)'],\ + \ plot_data['Without Children (%)'])):\n if with_val > without_val:\n \ + \ ax.text(with_val + 0.5, i, f'{with_val:.1f}%', va='center')\n ax.text(without_val\ + \ - 2.5, i, f'{without_val:.1f}%', va='center', color='white')\n else:\n\ + \ ax.text(with_val - 2.5, i, f'{with_val:.1f}%', va='center', color='white')\n\ + \ ax.text(without_val + 0.5, i, f'{without_val:.1f}%', va='center')\n\ + \n# Add sample size information to the y-axis labels\nlabels = [f\"{condition}\\\ + n(n={size:,})\" for condition, size in zip(plot_data['Condition'], plot_data['Sample\ + \ Size'])]\nax.set_yticklabels(labels)\n\n# Format the x-axis as percentages\n\ + ax.xaxis.set_major_formatter(PercentFormatter())\n\n# Add a title and labels\n\ + plt.title('Housing Conditions by Presence of Children\\nAmerican Housing Survey\ + \ (2023)', fontsize=16, pad=20)\nplt.xlabel('Prevalence (%)', fontsize=12)\n\ + plt.ylabel('Housing Condition', fontsize=12)\nplt.legend(title='Household Type',\ + \ loc='lower right')\n\n# Add a note about the data\nplt.figtext(0.5, 0.01,\ + \ \n \"Note: Data from 2023 American Housing Survey. Sample sizes\ + \ vary by condition.\\n\"\n \"Percentages represent the proportion\ + \ of households reporting each condition.\", \n ha='center', fontsize=10,\ + \ style='italic')\n\nplt.tight_layout(rect=[0, 0.03, 1, 0.97])\nplt.show()\n\ + \n# Create a summary table of the results\nsummary_table = plot_data.copy()\n\ + summary_table['Difference (percentage points)'] = summary_table['With Children\ + \ (%)'] - summary_table['Without Children (%)']\nsummary_table = summary_table.sort_values(by='Difference\ + \ (percentage points)', ascending=False)\n\nprint(\"Summary of Housing Conditions\ + \ by Presence of Children:\")\nprint(summary_table[['Condition', 'With Children\ + \ (%)', 'Without Children (%)', 'Difference (percentage points)', 'Sample Size']].to_string(index=False,\ + \ float_format=lambda x: f\"{x:.1f}\"))\n" + integration: beb85142-d32f-42df-a438-b84829ddd4fd + notes: null + query: |- + Analyzing and visualizing the relationship between housing conditions (mold, water leaks) and the presence of children in households using the 2023 American Housing Survey data. This code creates a horizontal bar chart comparing the prevalence of various housing conditions in homes with and without children, and generates a summary table of the differences. + resource_id: e72b358a-6a63-4d79-9ee5-e81b5d99af2b + resource_type: example + ea7d2aec-2d76-4b95-ae85-177db08ff019: + code: "import pandas as pd\n\ndata_dir = \"{{DATASET_FILES_BASE_PATH}}/census-ahs\"\ + \n\ndef get_florida_mold_data():\n # Store results\n florida_mold_data\ + \ = []\n \n # Years to analyze\n years = range(2001, 2012, 2)\n \ + \ \n for year in years:\n try:\n # Read data file for year\n\ + \ file_path = os.path.join(data_dir, f'survey_{year}.csv')\n \ + \ \n df = pd.read_csv(file_path, index_col=False)\n \ + \ \n # Find mold-related columns\n mold_cols = [col\ + \ for col in df.columns if 'mold' in col.lower()]\n \n \ + \ # Find geographic identifier columns\n geo_cols = [col for col\ + \ in df.columns if col in ['SMSA', 'METRO3', 'OMB13CBSA', 'DIVISION']]\n \ + \ \n if len(mold_cols) == 0:\n continue\n\ + \ \n if len(geo_cols) == 0:\n continue\n\ + \ \n # Select relevant columns\n cols_to_use\ + \ = geo_cols + mold_cols\n df_subset = df[cols_to_use]\n \ + \ \n # Read codebook to identify Florida metro areas\n \ + \ codebook_path = os.path.join(data_dir, 'census_ahs_codebook.csv')\n \ + \ \n # ensure to pass index_col=False, else sometimes data\ + \ shifts\n codebook = pd.read_csv(codebook_path, index_col=False)\n\ + \ \n # Get Florida metro codes for the geographic identifier\ + \ being used\n geo_col = geo_cols[len(geo_cols) - 1]\n \ + \ \n geo_codes = codebook[codebook['Variable'] == geo_col]\n \ + \ if geo_codes.empty:\n continue\n \n\ + \ florida_codes = [code.split(':')[0].strip() \n \ + \ for code in geo_codes['Response Codes'].iloc[0].split('||')\n \ + \ if 'FL' in code.upper()]\n \n \ + \ # Filter for Florida metros using substring matching\n df_fl\ + \ = df_subset[df_subset[geo_col].apply(lambda x: any(code in str(x) for code\ + \ in florida_codes))]\n \n # Add year column\n \ + \ df_fl.loc[:, 'YEAR'] = year\n \n florida_mold_data.append(df_fl)\n\ + \ \n except Exception as e:\n import traceback\n\ + \ print(f\"Traceback: {traceback.format_exc()}\")\n continue\n\ + \ \n # Combine all years\n if florida_mold_data:\n combined_data\ + \ = pd.concat(florida_mold_data, ignore_index=True)\n \n # Replace\ + \ values in the 'MOLD' columns with number to string mappings\n # where\ + \ MOLD columns are columns that contain MOLD in the column name (substring match)\n\ + \ mold_cols = [col for col in combined_data.columns if 'mold' in col.lower()]\n\ + \ \n for col in mold_cols:\n def convert_value(x):\n\ + \ # Remove any extra quotes that might be present\n \ + \ # many times codes are wrapped in quotes..\n x_str =\ + \ str(x).strip().strip(\"'\\\"\")\n if x_str in ['Y', '1', '1.0']:\n\ + \ return \"Yes\"\n elif x_str in ['N', '2',\ + \ '2.0']:\n return \"No\"\n elif x_str in\ + \ ['-6', '-6.0', 'N/A', 'NA', '']:\n return \"N/A\"\n \ + \ else:\n return x\n \n \ + \ # Remember that this example converts to \"Yes\" / \"No\" / \"N/A\"\n \ + \ # if you plot it, you'll want to \"count\" these text ocurrences\n\ + \ # also remember that values soemtimes are wrapped in quotes, such\ + \ as \"'6'\",\n # that is why we strip them or sometimes compare\ + \ to substrings\n combined_data[col] = combined_data[col].apply(convert_value)\n\ + \ \n return combined_data\n else:\n return pd.DataFrame()\n" + integration: beb85142-d32f-42df-a438-b84829ddd4fd + notes: null + query: |- + I wonder if there are any trends for metro areas in Florida, US in which mold presence increased between years 2001 and 2011? Can you get the data for matching mold presence? + resource_id: ea7d2aec-2d76-4b95-ae85-177db08ff019 + resource_type: example +slug: adhocspecificationintegration_census_american_housing_survey__ahs_ +source: | + You have access to a file-based Census American Housing Survey + (National Public Use Files CSV) from every 2 years from 1999 to 2023, under the `${DATASET_FILES_BASE_PATH}/census-ahs` directory. + The housing files are located and named per year as follows: + - ${DATASET_FILES_BASE_PATH}/census-ahs/survey_2023.csv + - ${DATASET_FILES_BASE_PATH}/census-ahs/survey_2021.csv + - ... + - ${DATASET_FILES_BASE_PATH}/census-ahs/survey_1999.csv + the more recent onces are at the metropolitan area level, while older ones + are only avaiable at the national level. Use whichever year files you need, depending + on the user question. + + {more_instructions} + + In particular, we have the household data file, which means we + have the OMB13CBSA as a constantly availablecolumn to identify geographic area of survey( + State-based Metropolitan and Micropolitan Statistical Areas). Pre-2015 have more geographical + column/information. Don't try to use state/county/etc if those are not present on + the census housing file for that year. + + Try to alreay have planned which columns you need for each, in order to process less columns/excess data since + merging both datasets will result in more than 1000 columns. Plan it out, select relevant columns + (list the column names once filtered to knkow which are available), + then merge the datasets with those columns selected. + + Once you identify columns of interest available, you can use this csv codebook dictionary that explains what each column code means: + + - ${DATASET_FILES_BASE_PATH}/census-ahs/census_ahs_codebook.csv + + from the codebook,you can use the "Variable" column to match the variable you have access to (say "OMB13CBSA") and use the column "Response Codes" to get + the meaning of the codes or the numeric code range values, for other types of variables. + + you may open it and use it at your discretion- usually better once you have some columns of interest + in order to select a subset of the codebook, since it is pretty big and confusing. + + Don't try to find the codebook columns without checking if they are present, since + the codebook is very exhaustive and contains all the columns or other types of files. + + You can also use knowledge on the usual column codenames from the census to answer questions, but ensure + to check the columns names with pandas (python lib) and _always_ check if the column + is present in the dataset itself instead of assuming it's present. + + Instead of assuming a specific column exist, or if you wish to match with a specific column pattern, + try to compare substrings instead. For example, if you need data for mold, try + to check for the 'mold' substring in the column names, since these csv files sometimes append + soe prefixes. + + The dataset csv files are a bit big, in the sense that they're in + wide format and contain too many columns. When planning what you need, select a subset + of the columns needed, including the geo/housing-characteristics columns for your analysis, + but don't include the thousands of columns. + + # Overview of the American Housing Survey (AHS) + The American Housing Survey (AHS) is a comprehensive national housing survey conducted by the U.S. Census Bureau. It provides detailed information about housing conditions, household characteristics, and neighborhood features across the United States. + + ## Key Dataset Characteristics + + ### Temporal Coverage + - Biennial surveys from 1999 to 2023 + - Data is collected every two years, allowing for trend analysis + - Most recent survey available is from 2023 + + ### Geospatial Resolution + - Metropolitan area level using Core-Based Statistical Areas (CBSA codes) + - Recent surveys (post-2015) use 2013 OMB CBSA codes + - Older surveys (pre-2015) use different geographic identifiers (SMSA codes) + - County-level data is limited, especially in recent surveys + - Harris County/Houston Data: Available in older surveys (1999-2007) using SMSA code 3360, but not found in more recent surveys (2015-2023) + + ### Housing Condition Variables + - Mold: Presence of mold in different areas of the home (basement, bathroom, bedroom, kitchen, living room) + - Water Leaks: Inside and outside water leaks, including specific sources (roof, plumbing, basement) + - Pests: Presence of rodents (rats, mice) and insects + - Air Quality: Indoor air quality ratings + - HVAC Systems: Heating and cooling system types and adequacy + + ### Health-Related Variables + - Asthma: Recent surveys (2015, 2023) include questions about household members diagnosed with asthma + - Emergency Room Visits: Data on ER visits due to asthma + - Asthma Medication: Some surveys include information about asthma medication use + + ### Household Composition + - Number of persons in household + - Presence of children (various age groups) + - Number of adults and elders + + ### Limitations and Challenges + #### Geographic Specificity: + - Recent surveys (2015-2023) do not contain Houston/Harris County specific data + - Only metropolitan area level data is available in recent surveys + + #### Data Consistency: + - Variable definitions and coding schemes change across survey years + - Some variables are only available in certain years + + #### Direct Causality: + - The survey provides correlational data but cannot establish causal relationships between housing conditions and health outcomes + + #### Sample Size: + - When filtering for specific conditions, sample sizes can become small, limiting statistical power + + # Additional Instructions: + + Read the CSV fiels with index_col=False, like so: + ``` + codebook = pd.read_csv('${DATASET_FILES_BASE_PATH}/census-ahs/census_ahs_codebook.csv', dtype=str, index_col=False) + ``` + If you don't add the index_col=False, the first column values will disappear and the whole dataframe will be shifted! +url: null +uuid: beb85142-d32f-42df-a438-b84829ddd4fd diff --git a/src/biome/datasources/ahs/documentation/docs.md b/src/biome/adhoc_data/datasources/ahs/attachments/docs.md similarity index 100% rename from src/biome/datasources/ahs/documentation/docs.md rename to src/biome/adhoc_data/datasources/ahs/attachments/docs.md diff --git a/src/biome/adhoc_data/datasources/alphagenome/api.yaml b/src/biome/adhoc_data/datasources/alphagenome/api.yaml new file mode 100644 index 0000000..07e71f4 --- /dev/null +++ b/src/biome/adhoc_data/datasources/alphagenome/api.yaml @@ -0,0 +1,660 @@ +datatype: api +description: |- + AlphaGenome is a Google DeepMind AI model for analyzing DNA sequences, providing multimodal predictions of genomic features. + It can analyze DNA sequences up to 1 million base pairs and offers predictions at single base-pair resolution. + The model provides functional outputs including gene expression, splicing patterns, chromatin features, and contact maps. + It offers both a Python client library and API access for genomic analysis. +img_url: null +last_updated: null +name: Alphagenome +provider: adhoc:specialist_agents +resources: + 049421e9-9b36-493a-bdd2-2c4b90a896c2: + code: "import os\nfrom alphagenome.data import genome\nfrom alphagenome.models\ + \ import dna_client\nfrom alphagenome.interpretation import mutagenesis\nimport\ + \ numpy as np\nimport pandas as pd\n\n# Setup\nAPI_KEY = os.environ.get(\"ALPHAGENOME_KEY\"\ + )\nmodel = dna_client.create(API_KEY)\n\n# Define smaller interval for mutagenesis\n\ + interval = genome.Interval(\n chromosome='chr22',\n start=36201680,\n\ + \ end=36201720 # Small 40bp region\n)\n\n# Get baseline prediction\nbaseline\ + \ = model.predict(\n interval=interval,\n ontology_terms=['UBERON:0001157'],\n\ + \ requested_outputs=[dna_client.OutputType.RNA_SEQ],\n)\nbaseline_mean =\ + \ baseline.rna_seq.values.mean()\n\n# Perform systematic mutagenesis\nbases\ + \ = ['A', 'C', 'G', 'T']\nmutagenesis_results = []\n\nfor pos in range(interval.start,\ + \ interval.end):\n for alt_base in bases:\n # Create variant\n \ + \ variant = genome.Variant(\n chromosome=interval.chromosome,\n\ + \ position=pos,\n reference_bases='N', # Placeholder\n\ + \ alternate_bases=alt_base,\n )\n \n try:\n\ + \ # Predict effect\n outputs = model.predict_variant(\n\ + \ interval=interval,\n variant=variant,\n \ + \ ontology_terms=['UBERON:0001157'],\n requested_outputs=[dna_client.OutputType.RNA_SEQ],\n\ + \ )\n \n alt_mean = outputs.alternate.rna_seq.values.mean()\n\ + \ effect = (alt_mean - baseline_mean) / baseline_mean\n \ + \ \n mutagenesis_results.append({\n 'Position':\ + \ pos,\n 'Alt_Base': alt_base,\n 'Effect_Size':\ + \ effect,\n 'Abs_Effect': abs(effect)\n })\n \ + \ \n except Exception as e:\n print(f\"Error at position\ + \ {pos} with {alt_base}: {e}\")\n\n# Analyze results\nif mutagenesis_results:\n\ + \ df = pd.DataFrame(mutagenesis_results)\n \n # Find most sensitive\ + \ positions\n position_effects = df.groupby('Position')['Abs_Effect'].max().sort_values(ascending=False)\n\ + \ \n print(\"In Silico Mutagenesis Results:\")\n print(\"-\" * 35)\n\ + \ print(f\"Total mutations tested: {len(df)}\")\n print(f\"Mean absolute\ + \ effect: {df['Abs_Effect'].mean():.6f}\")\n print(f\"Max absolute effect:\ + \ {df['Abs_Effect'].max():.6f}\")\n \n print(\"\\nTop 5 most sensitive\ + \ positions:\")\n for pos, effect in position_effects.head().items():\n \ + \ print(f\" Position {pos}: max effect = {effect:.6f}\")\n \n #\ + \ Find most impactful mutations\n top_mutations = df.nlargest(5, 'Abs_Effect')\n\ + \ print(\"\\nTop 5 most impactful mutations:\")\n for _, row in top_mutations.iterrows():\n\ + \ print(f\" {row['Position']}{row['Alt_Base']}: effect = {row['Effect_Size']:.6f}\"\ + )\n\nelse:\n print(\"No mutagenesis results obtained.\")" + integration: 959aaeff-d99a-4bef-afeb-6f10b0ed9c10 + notes: |- + This example demonstrates in silico mutagenesis analysis to identify functionally important positions within a genomic region. It systematically tests all possible single nucleotide changes and identifies positions that are most sensitive to mutation, helping to understand sequence-function relationships. + query: In silico mutagenesis analysis + resource_id: 049421e9-9b36-493a-bdd2-2c4b90a896c2 + resource_type: example + 13930fb5-3bf1-4c0c-b851-f1753d9d46e9: + code: "import os\nimport pandas as pd\nfrom tqdm import tqdm\nfrom alphagenome.data\ + \ import genome\nfrom alphagenome.models import dna_client\n\n# Setup\nAPI_KEY\ + \ = os.environ.get(\"ALPHAGENOME_KEY\")\nmodel = dna_client.create(API_KEY)\n\ + \n# Create example VCF data\nvcf_data = {\n 'variant_id': ['rs1234567', 'rs2345678',\ + \ 'rs3456789', 'rs4567890'],\n 'CHROM': ['chr1', 'chr2', 'chr3', 'chr4'],\n\ + \ 'POS': [1000000, 2000000, 3000000, 4000000],\n 'REF': ['A', 'G', 'C',\ + \ 'T'],\n 'ALT': ['G', 'T', 'A', 'C']\n}\nvcf = pd.DataFrame(vcf_data)\n\n\ + # Define analysis parameters\nsequence_length = 50000 # 50kb context window\n\ + tissue_ontology = 'UBERON:0001157' # Liver tissue\n\n# Configure multi-modal\ + \ scoring\nscoring_types = [\n dna_client.OutputType.RNA_SEQ,\n dna_client.OutputType.CAGE,\n\ + \ dna_client.OutputType.ATAC,\n dna_client.OutputType.DNASE,\n]\n\n# Process\ + \ variants with progress tracking\nresults = []\nfailed_variants = []\n\nfor\ + \ i, vcf_row in tqdm(vcf.iterrows(), total=len(vcf), desc=\"Processing variants\"\ + ):\n variant = genome.Variant(\n chromosome=str(vcf_row.CHROM),\n\ + \ position=int(vcf_row.POS),\n reference_bases=vcf_row.REF,\n\ + \ alternate_bases=vcf_row.ALT,\n name=vcf_row.variant_id,\n \ + \ )\n \n # Define interval around variant\n interval = genome.Interval(\n\ + \ chromosome=variant.chromosome,\n start=max(1, variant.position\ + \ - sequence_length//2),\n end=variant.position + sequence_length//2\n\ + \ )\n \n try:\n # Score variant across multiple modalities\n\ + \ output = model.predict_variant(\n interval=interval,\n \ + \ variant=variant,\n ontology_terms=[tissue_ontology],\n\ + \ requested_outputs=scoring_types,\n )\n \n \ + \ # Calculate effect sizes for each modality\n rna_ref = output.reference.rna_seq.values.mean()\n\ + \ rna_alt = output.alternate.rna_seq.values.mean()\n rna_effect\ + \ = (rna_alt - rna_ref) / rna_ref if rna_ref != 0 else 0\n \n \ + \ cage_ref = output.reference.cage.values.mean()\n cage_alt = output.alternate.cage.values.mean()\n\ + \ cage_effect = (cage_alt - cage_ref) / cage_ref if cage_ref != 0 else\ + \ 0\n \n atac_ref = output.reference.atac.values.mean()\n \ + \ atac_alt = output.alternate.atac.values.mean()\n atac_effect = (atac_alt\ + \ - atac_ref) / atac_ref if atac_ref != 0 else 0\n \n # Compile\ + \ results\n result = {\n 'variant_id': variant.name,\n \ + \ 'chromosome': variant.chromosome,\n 'position': variant.position,\n\ + \ 'ref_allele': variant.reference_bases,\n 'alt_allele':\ + \ variant.alternate_bases,\n 'rna_seq_effect': rna_effect,\n \ + \ 'cage_effect': cage_effect,\n 'atac_effect': atac_effect,\n\ + \ 'max_abs_effect': max(abs(rna_effect), abs(cage_effect), abs(atac_effect)),\n\ + \ 'high_impact': max(abs(rna_effect), abs(cage_effect), abs(atac_effect))\ + \ > 0.1\n }\n results.append(result)\n \n except Exception\ + \ as e:\n failed_variants.append({\n 'variant_id': variant.name,\n\ + \ 'error': str(e)\n })\n continue\n\n# Process results\n\ + if results:\n results_df = pd.DataFrame(results)\n results_df = results_df.sort_values('max_abs_effect',\ + \ ascending=False)\n \n print(f\"\\nBatch variant scoring completed:\"\ + )\n print(f\" Successfully processed: {len(results_df)} variants\")\n \ + \ print(f\" Failed: {len(failed_variants)} variants\")\n print(f\" High-impact\ + \ variants: {results_df['high_impact'].sum()}\")\n \n # Display top variants\n\ + \ print(f\"\\nTop 3 variants by maximum effect:\")\n top_variants = results_df.head(3)\n\ + \ for _, row in top_variants.iterrows():\n print(f\" {row['variant_id']}:\ + \ max effect = {row['max_abs_effect']:.4f}\")\n \n # Export results\n\ + \ results_df.to_csv('variant_scoring_results.csv', index=False)\n print(f\"\ + \\nResults exported to 'variant_scoring_results.csv'\")\n \nelse:\n print(\"\ + No variants successfully processed.\")" + integration: 959aaeff-d99a-4bef-afeb-6f10b0ed9c10 + notes: |- + This example demonstrates efficient batch processing of VCF variants with comprehensive multi-modal scoring. It includes progress tracking with tqdm, error handling, effect size calculations, and result export. The workflow is optimized for processing hundreds to thousands of variants across multiple genomic data modalities. + query: Batch VCF variant scoring with progress tracking + resource_id: 13930fb5-3bf1-4c0c-b851-f1753d9d46e9 + resource_type: example + 324ca421-3e36-42c3-91b4-28f003978f32: + code: "import os\nfrom alphagenome.data import genome\nfrom alphagenome.models\ + \ import dna_client\nimport pandas as pd\n\n# Setup\nAPI_KEY = os.environ.get(\"\ + ALPHAGENOME_KEY\")\nmodel = dna_client.create(API_KEY)\n\n# Define genomic interval\n\ + interval = genome.Interval(\n chromosome='chr22',\n start=35677410,\n\ + \ end=36725986\n)\n\n# Define tissues of interest\ntissues = {\n 'Liver':\ + \ 'UBERON:0001157',\n 'Lung': 'UBERON:0002048',\n 'Brain': 'UBERON:0000955',\n\ + \ 'Heart': 'UBERON:0000948',\n 'Kidney': 'UBERON:0002113'\n}\n\n# Predict\ + \ expression across tissues\ntissue_results = {}\nfor tissue_name, ontology_id\ + \ in tissues.items():\n try:\n outputs = model.predict(\n \ + \ interval=interval,\n ontology_terms=[ontology_id],\n \ + \ requested_outputs=[dna_client.OutputType.RNA_SEQ],\n )\n \ + \ tissue_results[tissue_name] = outputs.rna_seq.values.mean()\n except Exception\ + \ as e:\n print(f\"Error predicting {tissue_name}: {e}\")\n tissue_results[tissue_name]\ + \ = None\n\n# Create summary DataFrame\ndf = pd.DataFrame(list(tissue_results.items()),\ + \ \n columns=['Tissue', 'Mean_Expression'])\ndf = df.dropna()\n\ + df = df.sort_values('Mean_Expression', ascending=False)\n\nprint(\"Expression\ + \ levels across tissues:\")\nprint(df.to_string(index=False))\n\n# Find tissue\ + \ with highest expression\ntop_tissue = df.iloc[0]['Tissue']\ntop_expression\ + \ = df.iloc[0]['Mean_Expression']\nprint(f\"\\nHighest expression in {top_tissue}:\ + \ {top_expression:.4f}\")" + integration: 959aaeff-d99a-4bef-afeb-6f10b0ed9c10 + notes: |- + This example demonstrates multi-tissue expression analysis by predicting gene expression across different tissue types. It uses common tissue ontology terms and compares expression levels to identify tissue-specific patterns. + query: Multi-tissue expression analysis + resource_id: 324ca421-3e36-42c3-91b4-28f003978f32 + resource_type: example + 3ed26d8b-0c8b-4c02-ac69-23999dae1fc9: + filepath: alphagenome_overview.md + integration: 959aaeff-d99a-4bef-afeb-6f10b0ed9c10 + name: alphagenome_overview + resource_id: 3ed26d8b-0c8b-4c02-ac69-23999dae1fc9 + resource_type: file + 5e1e86ff-38eb-421f-811f-3b4c77ccf727: + code: "import os\nfrom alphagenome.data import genome\nfrom alphagenome.models\ + \ import dna_client\nfrom alphagenome.visualization import plot_components\n\ + import matplotlib.pyplot as plt\n\n# 1. Set up AlphaGenome model with API authentication\n\ + API_KEY = os.environ.get(\"ALPHAGENOME_KEY\")\nmodel = dna_client.create(API_KEY)\n\ + \n# 2. Define cancer mutation for analysis (EGFR L858R)\nmutation_pos = 55191822\n\ + interval = genome.Interval(\n chromosome='chr7',\n start=mutation_pos\ + \ - 1024, # Use supported 2048bp window\n end=mutation_pos + 1024\n)\n\n\ + variant = genome.Variant(\n chromosome='chr7',\n position=mutation_pos,\n\ + \ reference_bases='T',\n alternate_bases='C',\n name='EGFR_L858R'\n\ + )\n\n# 3. Predict variant effects in lung tissue context\noutputs = model.predict_variant(\n\ + \ interval=interval,\n variant=variant,\n ontology_terms=['UBERON:0002048'],\ + \ # Lung tissue\n requested_outputs=[\n dna_client.OutputType.RNA_SEQ,\n\ + \ dna_client.OutputType.ATAC,\n dna_client.OutputType.DNASE,\n\ + \ dna_client.OutputType.SPLICE_JUNCTIONS\n ],\n)\n\n# 4. Calculate\ + \ quantitative effect sizes\ndef calc_effect(ref, alt, modality_name):\n \ + \ ref_mean = ref.values.mean()\n alt_mean = alt.values.mean()\n ref_max\ + \ = ref.values.max()\n alt_max = alt.values.max()\n \n mean_effect\ + \ = (alt_mean - ref_mean) / ref_mean if ref_mean != 0 else 0\n max_effect\ + \ = (alt_max - ref_max) / ref_max if ref_max != 0 else 0\n \n print(f\"\ + {modality_name}:\")\n print(f\" Mean effect: {mean_effect:.4f} ({mean_effect*100:.2f}%)\"\ + )\n print(f\" Max effect: {max_effect:.4f} ({max_effect*100:.2f}%)\")\n\ + \ \n return mean_effect, max_effect\n\n# Analyze each modality\neffects\ + \ = {}\nif hasattr(outputs.reference, 'rna_seq') and hasattr(outputs.alternate,\ + \ 'rna_seq'):\n effects['RNA_Expression'] = calc_effect(outputs.reference.rna_seq,\ + \ outputs.alternate.rna_seq, \"RNA Expression\")\n\nif hasattr(outputs.reference,\ + \ 'atac') and hasattr(outputs.alternate, 'atac'):\n effects['Chromatin_Access']\ + \ = calc_effect(outputs.reference.atac, outputs.alternate.atac, \"Chromatin\ + \ Accessibility\")\n\nif hasattr(outputs.reference, 'dnase') and hasattr(outputs.alternate,\ + \ 'dnase'):\n effects['DNase'] = calc_effect(outputs.reference.dnase, outputs.alternate.dnase,\ + \ \"DNase Hypersensitivity\")\n\nif hasattr(outputs.reference, 'splice_junctions')\ + \ and hasattr(outputs.alternate, 'splice_junctions'):\n effects['Splicing']\ + \ = calc_effect(outputs.reference.splice_junctions, outputs.alternate.splice_junctions,\ + \ \"Splice Junctions\")\n\n# 5. Create visualization\ntracks = []\nif hasattr(outputs.reference,\ + \ 'rna_seq'):\n tracks.append(\n plot_components.OverlaidTracks(\n\ + \ tdata={\n 'Reference RNA-seq': outputs.reference.rna_seq,\n\ + \ 'Alternate RNA-seq': outputs.alternate.rna_seq,\n \ + \ },\n colors={'Reference RNA-seq': 'blue', 'Alternate RNA-seq':\ + \ 'red'},\n )\n )\n\nif hasattr(outputs.reference, 'atac'):\n tracks.append(\n\ + \ plot_components.OverlaidTracks(\n tdata={\n \ + \ 'Reference ATAC': outputs.reference.atac,\n 'Alternate ATAC':\ + \ outputs.alternate.atac,\n },\n colors={'Reference ATAC':\ + \ 'green', 'Alternate ATAC': 'orange'},\n )\n )\n\nif tracks:\n \ + \ plot_components.plot(\n tracks,\n interval=interval,\n \ + \ annotations=[plot_components.VariantAnnotation([variant], alpha=0.8)],\n\ + \ )\n plt.title(f\"EGFR L858R Functional Impact in Lung Tissue\")\n \ + \ plt.tight_layout()\n plt.show()\n\nprint(f\"\\nAnalysis complete for {variant.name}\ + \ in lung tissue context\")\nprint(\"Functional predictions can be compared\ + \ with clinical expression data\")" + integration: 959aaeff-d99a-4bef-afeb-6f10b0ed9c10 + notes: |- + This example demonstrates comprehensive variant effect prediction for cancer + mutations using AlphaGenome. Key features: 1) Uses proper 2048bp genomic intervals + (supported sequence length), 2) Analyzes multiple functional modalities (RNA-seq, + ATAC-seq, DNase-seq, splicing), 3) Applies tissue-specific context (lung tissue + for lung cancer), 4) Calculates quantitative effect sizes for comparison with + clinical data, 5) Creates multi-track visualizations, 6) Handles API authentication + properly. This workflow is ideal for validating clinical genomics findings with + functional predictions and understanding the mechanistic basis of cancer mutations.' + query: |- + Predict functional effects of cancer mutations on gene expression and chromatin accessibility in tissue-specific context + resource_id: 5e1e86ff-38eb-421f-811f-3b4c77ccf727 + resource_type: example + 625914ad-d89e-4151-ba3a-fd4d44751526: + code: "import os\nfrom alphagenome.data import genome\nfrom alphagenome.models\ + \ import dna_client\nimport pandas as pd\n\n# Setup\nAPI_KEY = os.environ.get(\"\ + ALPHAGENOME_KEY\")\nmodel = dna_client.create(API_KEY)\n\n# Define genomic interval\n\ + interval = genome.Interval(\n chromosome='chr22',\n start=35677410,\n\ + \ end=36725986\n)\n\n# Define variants to analyze\nvariants = [\n genome.Variant(chromosome='chr22',\ + \ position=36201698, reference_bases='A', alternate_bases='C'),\n genome.Variant(chromosome='chr22',\ + \ position=36201700, reference_bases='G', alternate_bases='T'),\n genome.Variant(chromosome='chr22',\ + \ position=36201702, reference_bases='C', alternate_bases='G'),\n genome.Variant(chromosome='chr22',\ + \ position=36201704, reference_bases='T', alternate_bases='A'),\n]\n\n# Score\ + \ all variants\nresults = []\nfor i, variant in enumerate(variants):\n try:\n\ + \ outputs = model.predict_variant(\n interval=interval,\n\ + \ variant=variant,\n ontology_terms=['UBERON:0001157'],\ + \ # Liver tissue\n requested_outputs=[dna_client.OutputType.RNA_SEQ],\n\ + \ )\n \n # Calculate effect metrics\n ref_mean =\ + \ outputs.reference.rna_seq.values.mean()\n alt_mean = outputs.alternate.rna_seq.values.mean()\n\ + \ effect_size = (alt_mean - ref_mean) / ref_mean if ref_mean != 0 else\ + \ 0\n \n results.append({\n 'Variant_ID': f\"var_{i+1}\"\ + ,\n 'Chromosome': variant.chromosome,\n 'Position': variant.position,\n\ + \ 'Reference': variant.reference_bases,\n 'Alternate':\ + \ variant.alternate_bases,\n 'Ref_Expression': ref_mean,\n \ + \ 'Alt_Expression': alt_mean,\n 'Effect_Size': effect_size,\n\ + \ 'Abs_Effect': abs(effect_size)\n })\n \n except\ + \ Exception as e:\n print(f\"Error processing variant {i+1}: {e}\")\n\ + \n# Create results DataFrame\ndf = pd.DataFrame(results)\ndf = df.sort_values('Abs_Effect',\ + \ ascending=False)\n\nprint(\"Variant scoring results:\")\nprint(df.to_string(index=False,\ + \ float_format='%.6f'))\n\n# Identify high-impact variants\nhigh_impact = df[df['Abs_Effect']\ + \ > 0.1]\nprint(f\"\\nHigh-impact variants (|effect| > 0.1): {len(high_impact)}\"\ + )\nif len(high_impact) > 0:\n print(high_impact[['Variant_ID', 'Position',\ + \ 'Effect_Size']].to_string(index=False))" + integration: 959aaeff-d99a-4bef-afeb-6f10b0ed9c10 + notes: |- + This example demonstrates batch variant scoring for multiple SNPs within a genomic region. It calculates effect sizes for each variant and identifies high-impact variants that significantly alter gene expression predictions. + query: Batch variant scoring for multiple SNPs + resource_id: 625914ad-d89e-4151-ba3a-fd4d44751526 + resource_type: example + 81258fcb-9071-4836-b379-05147401fb91: + code: "import os\nimport pandas as pd\nfrom alphagenome.data import genome\nfrom\ + \ alphagenome.models import dna_client\nfrom alphagenome.visualization import\ + \ plot_components\nimport matplotlib.pyplot as plt\n\n# Setup\nAPI_KEY = os.environ.get(\"\ + ALPHAGENOME_KEY\")\nmodel = dna_client.create(API_KEY)\n\n# TAL1 locus definition\ + \ (T-cell acute lymphocytic leukemia 1)\ntal1_interval = genome.Interval(\n\ + \ chromosome='chr1',\n start=47200000,\n end=47300000 # 100kb region\ + \ around TAL1\n)\n\n# Known oncogenic variants in TAL1 regulatory region\noncogenic_variants\ + \ = [\n genome.Variant(\n chromosome='chr1',\n position=47240000,\n\ + \ reference_bases='G',\n alternate_bases='A',\n name='TAL1_regulatory_variant_1'\n\ + \ ),\n genome.Variant(\n chromosome='chr1',\n position=47245000,\n\ + \ reference_bases='C',\n alternate_bases='T',\n name='TAL1_regulatory_variant_2'\n\ + \ ),\n genome.Variant(\n chromosome='chr1',\n position=47250000,\n\ + \ reference_bases='A',\n alternate_bases='G',\n name='TAL1_regulatory_variant_3'\n\ + \ ),\n]\n\n# Analyze in hematopoietic context (CD34+ stem cells)\ncd34_ontology\ + \ = 'CL:0000037' # CD34+ stem cell\n\n# Define analysis modalities\nanalysis_outputs\ + \ = [\n dna_client.OutputType.RNA_SEQ,\n dna_client.OutputType.ATAC,\n\ + \ dna_client.OutputType.HISTONE_MARKS,\n dna_client.OutputType.TF_BINDING\n\ + ]\n\n# Analyze each variant\nvariant_results = {}\n\nfor variant in oncogenic_variants:\n\ + \ print(f\"Analyzing variant: {variant.name}\")\n \n # Predict variant\ + \ effects\n output = model.predict_variant(\n interval=tal1_interval,\n\ + \ variant=variant,\n ontology_terms=[cd34_ontology],\n \ + \ requested_outputs=analysis_outputs,\n )\n \n # Calculate effect sizes\n\ + \ rna_ref = output.reference.rna_seq.values.mean()\n rna_alt = output.alternate.rna_seq.values.mean()\n\ + \ rna_effect = (rna_alt - rna_ref) / rna_ref if rna_ref != 0 else 0\n \ + \ \n atac_ref = output.reference.atac.values.mean()\n atac_alt = output.alternate.atac.values.mean()\n\ + \ atac_effect = (atac_alt - atac_ref) / atac_ref if atac_ref != 0 else 0\n\ + \ \n # Calculate TAL1 upregulation potential\n tal1_upregulation_score\ + \ = max(rna_effect, 0) # Only positive effects\n chromatin_disruption_score\ + \ = abs(atac_effect)\n \n # Oncogenic potential assessment\n oncogenic_score\ + \ = (tal1_upregulation_score * 0.7 + chromatin_disruption_score * 0.3)\n \ + \ is_high_risk = oncogenic_score > 0.15\n \n variant_results[variant.name]\ + \ = {\n 'position': variant.position,\n 'ref_alt': f\"{variant.reference_bases}>{variant.alternate_bases}\"\ + ,\n 'rna_expression_effect': rna_effect,\n 'chromatin_accessibility_effect':\ + \ atac_effect,\n 'tal1_upregulation_score': tal1_upregulation_score,\n\ + \ 'chromatin_disruption_score': chromatin_disruption_score,\n \ + \ 'oncogenic_score': oncogenic_score,\n 'high_risk_classification': is_high_risk,\n\ + \ 'output_data': output\n }\n \n print(f\" RNA effect: {rna_effect:.4f}\"\ + )\n print(f\" ATAC effect: {atac_effect:.4f}\")\n print(f\" Oncogenic\ + \ score: {oncogenic_score:.4f}\")\n print(f\" High risk: {is_high_risk}\"\ + )\n\n# Summary analysis\nprint(f\"\\nTAL1 Oncogenic Variant Analysis Summary:\"\ + )\nprint(f\"=\" * 45)\n\nhigh_risk_variants = [v for v, r in variant_results.items()\ + \ if r['high_risk_classification']]\nprint(f\"High-risk variants identified:\ + \ {len(high_risk_variants)}\")\n\n# Create visualization for high-risk variants\n\ + for variant_name in high_risk_variants:\n result = variant_results[variant_name]\n\ + \ output = result['output_data']\n \n # Create multi-track visualization\n\ + \ fig, axes = plt.subplots(2, 1, figsize=(12, 8))\n \n # RNA expression\ + \ comparison\n plt.subplot(2, 1, 1)\n plot_components.plot([\n \ + \ plot_components.OverlaidTracks(\n tdata={\n 'Reference':\ + \ output.reference.rna_seq,\n 'Alternate': output.alternate.rna_seq,\n\ + \ },\n colors={'Reference': 'blue', 'Alternate': 'red'},\n\ + \ ),\n ], interval=tal1_interval)\n plt.title(f\"RNA Expression:\ + \ {variant_name}\")\n \n # Chromatin accessibility comparison\n plt.subplot(2,\ + \ 1, 2)\n plot_components.plot([\n plot_components.OverlaidTracks(\n\ + \ tdata={\n 'Reference': output.reference.atac,\n\ + \ 'Alternate': output.alternate.atac,\n },\n \ + \ colors={'Reference': 'green', 'Alternate': 'orange'},\n ),\n\ + \ ], interval=tal1_interval)\n plt.title(f\"Chromatin Accessibility: {variant_name}\"\ + )\n \n plt.tight_layout()\n plt.show()\n \n print(f\"\\nVisualization\ + \ created for {variant_name}\")\n\n# Clinical interpretation\nprint(f\"\\nClinical\ + \ Interpretation:\")\nprint(f\"- {len(high_risk_variants)} variants show high\ + \ oncogenic potential\")\nprint(f\"- These variants likely contribute to TAL1\ + \ upregulation\")\nprint(f\"- Mechanism involves disruption of regulatory chromatin\ + \ structure\")\nprint(f\"- May serve as therapeutic targets for precision medicine\"\ + )\n\n# Export results\nresults_df = pd.DataFrame.from_dict(variant_results,\ + \ orient='index')\nresults_df.to_csv('tal1_oncogenic_analysis.csv')\nprint(f\"\ + \\nDetailed results exported to 'tal1_oncogenic_analysis.csv'\")" + integration: 959aaeff-d99a-4bef-afeb-6f10b0ed9c10 + notes: |- + This example demonstrates comprehensive oncogenic variant analysis focused on the TAL1 locus, a critical gene in T-cell acute lymphocytic leukemia. The workflow analyzes regulatory variants in hematopoietic stem cell context, calculates oncogenic potential scores, and provides clinical interpretation. It represents a complete research workflow from variant identification to mechanistic understanding and clinical relevance. + query: TAL1 oncogenic variant analysis workflow + resource_id: 81258fcb-9071-4836-b379-05147401fb91 + resource_type: example + 85f40f69-621a-4488-a4bd-fd9ee0f84bb2: + code: |- + import os + from alphagenome.data import genome + from alphagenome.models import dna_client + + # Setup AlphaGenome client + API_KEY = os.environ.get("ALPHAGENOME_KEY") + model = dna_client.create(API_KEY) + + # Define genomic interval + interval = genome.Interval( + chromosome='chr22', + start=35677410, + end=36725986 + ) + + # Make prediction for liver tissue + outputs = model.predict( + interval=interval, + ontology_terms=['UBERON:0001157'], # Liver tissue + requested_outputs=[dna_client.OutputType.RNA_SEQ], + ) + + # Access expression data + expression_data = outputs.rna_seq + print(f"Expression prediction shape: {expression_data.shape}") + print(f"Genomic interval: {expression_data.interval}") + integration: 959aaeff-d99a-4bef-afeb-6f10b0ed9c10 + notes: |- + This example demonstrates basic gene expression prediction for a genomic interval in liver tissue. The TAL1 locus is used as an example region known for its role in hematopoiesis. The prediction returns RNA-seq expression levels at single base-pair resolution across the specified interval. + query: Basic gene expression prediction for a genomic interval + resource_id: 85f40f69-621a-4488-a4bd-fd9ee0f84bb2 + resource_type: example + 87b55cec-ace6-4c3d-a16f-075552369b2a: + code: "import os\nfrom alphagenome.data import genome\nfrom alphagenome.models\ + \ import dna_client\nfrom alphagenome.visualization import plot_components\n\ + import matplotlib.pyplot as plt\nimport pandas as pd\n\n# Setup\nAPI_KEY = os.environ.get(\"\ + ALPHAGENOME_KEY\")\nmodel = dna_client.create(API_KEY)\n\n# Define genomic interval\n\ + interval = genome.Interval(\n chromosome='chr22',\n start=35677410,\n\ + \ end=36725986\n)\n\n# Load transcript annotations (example data structure)\n\ + longest_transcripts = pd.DataFrame({\n 'gene_id': ['ENSG00000100055'],\n\ + \ 'transcript_id': ['ENST00000216714'],\n 'chromosome': ['chr22'],\n \ + \ 'start': [35677410],\n 'end': [36725986],\n 'strand': ['+']\n})\n\n\ + # Get comprehensive predictions\noutput = model.predict(\n interval=interval,\n\ + \ ontology_terms=['UBERON:0001157'], # Liver tissue\n requested_outputs=[\n\ + \ dna_client.OutputType.RNA_SEQ,\n dna_client.OutputType.CAGE,\n\ + \ dna_client.OutputType.ATAC,\n ],\n)\n\n# Create multi-track visualization\n\ + plot_components.plot([\n # Transcript annotations\n plot_components.TranscriptAnnotation(longest_transcripts),\n\ + \ \n # RNA-seq expression track\n plot_components.Tracks(\n \ + \ tdata=output.rna_seq,\n ylabel_template='RNA_SEQ: {biosample_name}'\n\ + \ ),\n \n # CAGE transcription start sites\n plot_components.Tracks(\n\ + \ tdata=output.cage,\n ylabel_template='CAGE: {biosample_name}'\n\ + \ ),\n \n # ATAC-seq accessibility\n plot_components.Tracks(\n \ + \ tdata=output.atac,\n ylabel_template='ATAC: {biosample_name}'\n\ + \ ),\n], interval=interval)\n\nplt.title(\"Multi-Modal Genomic Predictions\"\ + )\nplt.tight_layout()\nplt.show()\n\nprint(f\"Visualization created for interval:\ + \ {interval}\")\nprint(f\"Number of tracks displayed: 4 (transcripts + 3 data\ + \ tracks)\")" + integration: 959aaeff-d99a-4bef-afeb-6f10b0ed9c10 + notes: |- + This example demonstrates comprehensive multi-track genomic visualization combining transcript annotations with multiple prediction modalities. It shows how to create publication-quality figures that integrate gene structure with functional predictions from RNA-seq, CAGE, and ATAC-seq data. + query: Multi-track genomic visualization with transcript annotations + resource_id: 87b55cec-ace6-4c3d-a16f-075552369b2a + resource_type: example + d8eea40b-987c-4d18-a4d5-646d6ab58758: + code: |- + import os + from alphagenome.data import genome + from alphagenome.models import dna_client + from alphagenome.visualization import plot_components + import matplotlib.pyplot as plt + + # Setup + API_KEY = os.environ.get("ALPHAGENOME_KEY") + model = dna_client.create(API_KEY) + + # Define genomic interval and variant + interval = genome.Interval( + chromosome='chr22', + start=35677410, + end=36725986 + ) + + variant = genome.Variant( + chromosome='chr22', + position=36201698, + reference_bases='A', + alternate_bases='C', + ) + + # Predict variant effects + outputs = model.predict_variant( + interval=interval, + variant=variant, + ontology_terms=['UBERON:0001157'], + requested_outputs=[dna_client.OutputType.RNA_SEQ], + ) + + # Visualize reference vs alternate + plot_components.plot([ + plot_components.OverlaidTracks( + tdata={ + 'REF': outputs.reference.rna_seq, + 'ALT': outputs.alternate.rna_seq, + }, + colors={'REF': 'dimgrey', 'ALT': 'red'}, + ), + ], + interval=outputs.reference.rna_seq.interval.resize(2**15), + annotations=[plot_components.VariantAnnotation([variant], alpha=0.8)], + ) + plt.title("Variant Effect on Gene Expression") + plt.show() + + # Calculate effect size + ref_mean = outputs.reference.rna_seq.values.mean() + alt_mean = outputs.alternate.rna_seq.values.mean() + effect_size = (alt_mean - ref_mean) / ref_mean + print(f"Variant effect size: {effect_size:.4f}") + integration: 959aaeff-d99a-4bef-afeb-6f10b0ed9c10 + notes: |- + This example demonstrates variant effect prediction and visualization. It compares reference and alternate allele effects on gene expression and calculates the relative effect size. The visualization shows both predictions overlaid with the variant position annotated. + query: Variant effect prediction and visualization + resource_id: d8eea40b-987c-4d18-a4d5-646d6ab58758 + resource_type: example + ebc000b0-c1af-4e24-b7ae-72d8f6e64f8c: + code: |- + import os + from alphagenome.models import dna_client + import pandas as pd + + # Setup + API_KEY = os.environ.get("ALPHAGENOME_KEY") + model = dna_client.create(API_KEY) + + # Get comprehensive ontology metadata + metadata = model.get_output_metadata('homo_sapiens') + df_metadata = pd.DataFrame(metadata) + + print(f"Total ontology entries: {len(df_metadata)}") + print(f"Ontology systems covered: {df_metadata['ontology_system'].unique()}") + + # Filter by tissue types + brain_entries = df_metadata[df_metadata['tissue_name'].str.contains('brain', case=False, na=False)] + liver_entries = df_metadata[df_metadata['tissue_name'].str.contains('liver', case=False, na=False)] + blood_entries = df_metadata[df_metadata['tissue_name'].str.contains('blood|hematopoietic', case=False, na=False)] + + print(f"\nTissue-specific entries:") + print(f" Brain-related: {len(brain_entries)}") + print(f" Liver-related: {len(liver_entries)}") + print(f" Blood/hematopoietic: {len(blood_entries)}") + + # Explore cell line vs primary tissue breakdown + cell_lines = df_metadata[df_metadata['sample_type'] == 'cell_line'] + primary_tissues = df_metadata[df_metadata['sample_type'] == 'primary_tissue'] + + print(f"\nSample type breakdown:") + print(f" Cell lines: {len(cell_lines)}") + print(f" Primary tissues: {len(primary_tissues)}") + + # Get ontology terms for specific use cases + stem_cell_terms = df_metadata[df_metadata['tissue_name'].str.contains('stem cell', case=False, na=False)] + cancer_terms = df_metadata[df_metadata['tissue_name'].str.contains('cancer|tumor', case=False, na=False)] + + print(f"\nSpecialized contexts:") + print(f" Stem cell contexts: {len(stem_cell_terms)}") + print(f" Cancer contexts: {len(cancer_terms)}") + + # Display sample ontology terms + print(f"\nExample stem cell ontology terms:") + for i, row in stem_cell_terms.head(3).iterrows(): + print(f" {row['ontology_id']}: {row['tissue_name']}") + + # Filter by developmental stage + embryonic = df_metadata[df_metadata['life_stage'].str.contains('embryonic', case=False, na=False)] + adult = df_metadata[df_metadata['life_stage'].str.contains('adult', case=False, na=False)] + + print(f"\nDevelopmental stage breakdown:") + print(f" Embryonic: {len(embryonic)}") + print(f" Adult: {len(adult)}") + integration: 959aaeff-d99a-4bef-afeb-6f10b0ed9c10 + notes: |- + This example demonstrates comprehensive navigation of AlphaGenome's tissue ontology system with 5,563 entries. It shows how to filter by tissue types, sample types, developmental stages, and specialized contexts like stem cells and cancer. The ontology system supports UBERON, CL, and EFO standards for precise tissue-specific genomic analysis. + query: Tissue ontology navigation and filtering + resource_id: ebc000b0-c1af-4e24-b7ae-72d8f6e64f8c + resource_type: example + fbbc293d-5001-48e1-af48-592c8ffcbe25: + filepath: alphagenome_tutorials.md + integration: 959aaeff-d99a-4bef-afeb-6f10b0ed9c10 + name: alphagenome_tutorials + resource_id: fbbc293d-5001-48e1-af48-592c8ffcbe25 + resource_type: file + fcfe22fc-0386-4bc1-b5b2-424cd51c674f: + code: |- + import os + from alphagenome.data import genome + from alphagenome.models import dna_client + from alphagenome.visualization import plot_components + import matplotlib.pyplot as plt + + # Setup + API_KEY = os.environ.get("ALPHAGENOME_KEY") + model = dna_client.create(API_KEY) + + # Define genomic interval + interval = genome.Interval( + chromosome='chr22', + start=35677410, + end=36725986 + ) + + # Request multiple output types + output_types = [ + dna_client.OutputType.RNA_SEQ, + dna_client.OutputType.SPLICING, + dna_client.OutputType.CHROMATIN + ] + + # Make comprehensive prediction + outputs = model.predict( + interval=interval, + ontology_terms=['UBERON:0001157'], # Liver tissue + requested_outputs=output_types, + ) + + # Analyze each modality + print("Multi-modal Analysis Results:") + print("-" * 40) + + # RNA-seq analysis + if hasattr(outputs, 'rna_seq'): + rna_data = outputs.rna_seq + print(f"RNA-seq Expression:") + print(f" Mean expression: {rna_data.values.mean():.4f}") + print(f" Max expression: {rna_data.values.max():.4f}") + print(f" Expression variance: {rna_data.values.var():.4f}") + + # Splicing analysis + if hasattr(outputs, 'splicing'): + splicing_data = outputs.splicing + print(f"Splicing Predictions:") + print(f" Mean splicing score: {splicing_data.values.mean():.4f}") + print(f" Max splicing score: {splicing_data.values.max():.4f}") + + # Chromatin analysis + if hasattr(outputs, 'chromatin'): + chromatin_data = outputs.chromatin + print(f"Chromatin Features:") + print(f" Mean accessibility: {chromatin_data.values.mean():.4f}") + print(f" Max accessibility: {chromatin_data.values.max():.4f}") + + # Create multi-track visualization + tracks = [] + if hasattr(outputs, 'rna_seq'): + tracks.append(plot_components.Track( + tdata={'RNA-seq': outputs.rna_seq}, + colors={'RNA-seq': 'blue'} + )) + + if hasattr(outputs, 'chromatin'): + tracks.append(plot_components.Track( + tdata={'Chromatin': outputs.chromatin}, + colors={'Chromatin': 'green'} + )) + + if tracks: + plot_components.plot(tracks, interval=interval) + plt.title("Multi-modal Genomic Predictions") + plt.tight_layout() + plt.show() + + print(f"\nAnalysis complete for interval: {interval}") + integration: 959aaeff-d99a-4bef-afeb-6f10b0ed9c10 + notes: |- + This example demonstrates comprehensive multi-modal analysis using AlphaGenome. It requests multiple output types (RNA-seq, splicing, chromatin) and analyzes each modality to provide a complete picture of the genomic region's regulatory landscape. + query: Comprehensive multi-modal analysis + resource_id: fcfe22fc-0386-4bc1-b5b2-424cd51c674f + resource_type: example +slug: adhocspecificationintegration_alphagenome +source: |- + # Additional Instructions: + + AlphaGenome is a Python package and API service for genomic analysis. It requires an API key from Google DeepMind. + This API key is available to you in the environment variable `ALPHAGENOME_KEY`. + + When drafting code, use the following pattern to access the API key: `os.environ.get("ALPHAGENOME_KEY")`. + Do not use placeholder auth values, just use the environment variable. + + ## Key Features + - Multimodal genomic predictions + - Single base-pair resolution analysis + - DNA sequences up to 1 million base pairs + - Variant effect prediction + - Gene expression analysis + - Splicing pattern analysis + - Chromatin feature analysis + - Contact map generation + + ## Main Modules + - `alphagenome.data.genome`: Data structures for genomic intervals and variants + - `alphagenome.models.dna_client`: Main client for making predictions + - `alphagenome.visualization.plot_components`: Visualization tools + - `alphagenome.interpretation`: Sequence interpretation tools + + ## Example Usage + ```python + from alphagenome.data import genome + from alphagenome.models import dna_client + + API_KEY = os.environ.get("ALPHAGENOME_KEY") + model = dna_client.create(API_KEY) + + interval = genome.Interval(chromosome='chr22', start=35677410, end=36725986) + variant = genome.Variant( + chromosome='chr22', + position=36201698, + reference_bases='A', + alternate_bases='C', + ) + + outputs = model.predict_variant( + interval=interval, + variant=variant, + ontology_terms=['UBERON:0001157'], + requested_outputs=[dna_client.OutputType.RNA_SEQ], + ) + ``` + + ## Supported Output Types + - RNA_SEQ: Gene expression predictions + - SPLICING: Splicing pattern predictions + - CHROMATIN: Chromatin feature predictions + - CONTACT_MAPS: 3D genomic contact predictions + + # Overview Documentation: + ${alphagenome_overview} + + # Tutorial Documentation: + ${alphagenome_tutorials} +url: null +uuid: 959aaeff-d99a-4bef-afeb-6f10b0ed9c10 diff --git a/src/biome/datasources/alphagenome/documentation/alphagenome_overview.md b/src/biome/adhoc_data/datasources/alphagenome/attachments/alphagenome_overview.md similarity index 100% rename from src/biome/datasources/alphagenome/documentation/alphagenome_overview.md rename to src/biome/adhoc_data/datasources/alphagenome/attachments/alphagenome_overview.md diff --git a/src/biome/datasources/alphagenome/documentation/alphagenome_tutorials.md b/src/biome/adhoc_data/datasources/alphagenome/attachments/alphagenome_tutorials.md similarity index 100% rename from src/biome/datasources/alphagenome/documentation/alphagenome_tutorials.md rename to src/biome/adhoc_data/datasources/alphagenome/attachments/alphagenome_tutorials.md diff --git a/src/biome/adhoc_data/datasources/aqs/api.yaml b/src/biome/adhoc_data/datasources/aqs/api.yaml new file mode 100644 index 0000000..26c1f2c --- /dev/null +++ b/src/biome/adhoc_data/datasources/aqs/api.yaml @@ -0,0 +1,483 @@ +datatype: api +description: | + This API is the primary place to obtain row-level data + from the EPA's Air Quality System (AQS) database. AQS contains + ambient air sample data collected by state, local, tribal, and federal + air pollution control agencies from thousands of monitors around the nation. + It also contains meteorological data, descriptive information about + each monitoring station (including its geographic location and its operator), + and information about the quality of the samples. Note, AQS does not contain + real-time air quality data (it can take 6 months from the time + data is collected until it is in AQS). For real-time data, please use + the AirNow API. +img_url: null +last_updated: null +name: EPA Air Quality System +provider: adhoc:specialist_agents +resources: + 00b2ebdf-94ff-475f-be91-a1f69e1b4403: + code: "import os\nimport requests\nimport pandas as pd\nfrom datetime import datetime\n\ + import calendar\n\n# API configuration\nbase_url = \"https://aqs.epa.gov/data/api\"\ + \nemail = os.environ.get(\"API_EPA_AQS_EMAIL\")\nkey = os.environ.get(\"API_EPA_AQS\"\ + )\n\n# Define parameters\nparameters = {\n 'Ozone': 44201,\n 'CO': 42101,\ + \ \n 'NO2': 42602,\n 'PM2.5': 88101\n}\n\n# Harris County, TX codes\n\ + state_code = '48'\ncounty_code = '201'\nyear = 2022\n\ndef get_monthly_data(param_code,\ + \ month):\n # Get number of days in month\n num_days = calendar.monthrange(year,\ + \ month)[1]\n \n params = {\n 'email': email,\n 'key': key,\n\ + \ 'param': param_code,\n 'bdate': f'{year}{month:02d}01',\n \ + \ 'edate': f'{year}{month:02d}{num_days}',\n 'state': state_code,\n\ + \ 'county': county_code\n }\n \n response = requests.get(f\"\ + {base_url}/dailyData/byCounty\", params=params)\n return response.json()\n\ + \n# Initialize empty list to store all data\nall_data = []\n\n# Loop through\ + \ each parameter and month\nfor param_name, param_code in parameters.items():\n\ + \ for month in range(1, 13):\n try:\n data = get_monthly_data(param_code,\ + \ month)\n \n if 'Data' in data:\n for\ + \ record in data['Data']:\n all_data.append({\n \ + \ 'date': record['date_local'],\n 'parameter':\ + \ param_name,\n 'value': record['arithmetic_mean'],\n\ + \ 'units': record['units_of_measure'],\n \ + \ 'site': record['site_number'],\n 'latitude':\ + \ record['latitude'],\n 'longitude': record['longitude']\n\ + \ })\n except Exception as e:\n print(f\"\ + Error fetching data for {param_name} in month {month}: {str(e)}\")\n\n# Convert\ + \ to DataFrame\ndf = pd.DataFrame(all_data)\n\n# Convert date string to datetime\n\ + df['date'] = pd.to_datetime(df['date'])\n\n# Sort by date and parameter\ndf\ + \ = df.sort_values(['date', 'parameter'])\n\n# Save to CSV\ndf.to_csv('harris_county_air_quality_2022.csv',\ + \ index=False)\n" + integration: 18e5974c-8cea-4fe7-88db-7c5779a17d5d + notes: null + query: Retrieve daily air quality data for ozone, CO, NO2, and PM2.5 in Harris + County, TX for each month of the year 2022. + resource_id: 00b2ebdf-94ff-475f-be91-a1f69e1b4403 + resource_type: example + 07d42e21-2e7d-467a-878b-e0f7572f3b8d: + code: | + import requests + import os + + # Get API credentials from environment variables + email = os.environ.get("API_EPA_AQS_EMAIL") + key = os.environ.get("API_EPA_AQS") + + # Define the API endpoint and parameters + state_code = 12 # "Florida" + county_code = 117 # "Seminole" county + year = 2000 + # codes can be fetched from /list/parametersByClass?email={email}&key={key}&pc=ALL + # These are asthma related pollutant/parameter codes: + pollutant_codes = [ + '44201', # ozone + '81102', # PM10 + '81104', # PM2.5 + '42602', # Nitrogen dioxide (NO2) + '42401', # Sulfur dioxide + '42101', # Carbon monoxide + ] + + bdate = f"{year}0501" + edate = f"{year}0501" + + # Construct the API URL + param_string = ','.join(pollutant_codes) + + url = f'https://aqs.epa.gov/data/api/annualData/byCounty?email={email}&key={key}¶m={param_string}&bdate={bdate}&edate={edate}&state={state_code}&county={county_code}' + + # Make the API request + response = requests.get(url) + + data = None + # Check if the request was successful + if response.status_code == 200: + data = response.json() + + data + integration: 18e5974c-8cea-4fe7-88db-7c5779a17d5d + notes: null + query: Query annual air pollution data for Seminole County, Florida for the year + 2000. Use pollutants relating to asthma. + resource_id: 07d42e21-2e7d-467a-878b-e0f7572f3b8d + resource_type: example + 14b7c79f-eec0-4fa0-b6bc-42a7b81c85c6: + code: "import os\nimport requests\nimport pandas as pd\nimport matplotlib.pyplot\ + \ as plt\nimport seaborn as sns\n\n# Get authentication details from environment\ + \ variables\nemail = os.environ.get(\"API_EPA_AQS_EMAIL\")\napi_key = os.environ.get(\"\ + API_EPA_AQS\")\n\n# Base URL for the EPA AQS API\nbase_url = \"https://aqs.epa.gov/data/api\"\ + \n\n# Function to make API requests\ndef get_aqs_data(endpoint, params=None):\n\ + \ \"\"\"\n Make a request to the EPA AQS API\n \n Parameters:\n\ + \ -----------\n endpoint : str\n The API endpoint to query\n \ + \ params : dict\n Additional parameters for the request\n \n\ + \ Returns:\n --------\n dict\n The JSON response from the API\n\ + \ \"\"\"\n # Add authentication parameters\n if params is None:\n \ + \ params = {}\n params['email'] = email\n params['key'] = api_key\n\ + \ \n # Make the request\n url = f\"{base_url}/{endpoint}\"\n response\ + \ = requests.get(url, params=params)\n \n # Check if the request was successful\n\ + \ if response.status_code == 200:\n return response.json()\n else:\n\ + \ print(f\"Error: {response.status_code}\")\n print(response.text)\n\ + \ return None\n\n# Get list of parameter classes\nclasses_response =\ + \ get_aqs_data(\"list/classes\")\n\n# Check if we got a valid response for classes\n\ + if classes_response and classes_response.get('Header', [{}])[0].get('status')\ + \ == 'Success':\n # Convert to DataFrame for easier viewing\n classes_data\ + \ = classes_response.get('Data', [])\n classes_df = pd.DataFrame(classes_data)\n\ + \ \n print(\"Parameter Classes:\")\n print(classes_df)\n \n #\ + \ Now get parameters for the CRITERIA class (most relevant to air quality standards)\n\ + \ criteria_params_response = get_aqs_data(\"list/parametersByClass\", {\"\ + pc\": \"CRITERIA\"})\n \n if criteria_params_response and criteria_params_response.get('Header',\ + \ [{}])[0].get('status') == 'Success':\n # Convert to DataFrame for easier\ + \ viewing\n criteria_params_data = criteria_params_response.get('Data',\ + \ [])\n criteria_params_df = pd.DataFrame(criteria_params_data)\n \ + \ \n print(f\"\\nFound {len(criteria_params_data)} criteria pollutant\ + \ parameters\")\n print(\"\\nCriteria pollutant parameters:\")\n \ + \ print(criteria_params_df)\n \n # Create a dictionary of key\ + \ parameter codes for asthma research\n asthma_param_codes = {\n \ + \ 'Ozone': '44201',\n 'PM2.5': '88101',\n 'PM10':\ + \ '81102',\n 'NO2': '42602',\n 'SO2': '42401',\n \ + \ 'CO': '42101'\n }\n \n print(\"\\nKey parameter\ + \ codes for asthma research:\")\n for pollutant, code in asthma_param_codes.items():\n\ + \ print(f\"{pollutant}: {code}\")\n else:\n print(\"Failed\ + \ to retrieve criteria parameters data\")\nelse:\n print(\"Failed to retrieve\ + \ parameter classes data\")\n" + integration: 18e5974c-8cea-4fe7-88db-7c5779a17d5d + notes: null + query: |- + Retrieving and exploring parameter classes and codes for air quality research. This example demonstrates how to get a list of parameter classes, then retrieve specific parameters within the CRITERIA class, which contains the most important air pollutants for health research. It also identifies key parameter codes relevant to asthma research. + resource_id: 14b7c79f-eec0-4fa0-b6bc-42a7b81c85c6 + resource_type: example + 19afbed5-92dc-48af-9354-55a544e417ce: + code: "import requests\nimport os\n\n# Get API credentials from environment variables\n\ + email = os.environ.get(\"API_EPA_AQS_EMAIL\")\nkey = os.environ.get(\"API_EPA_AQS\"\ + )\n\n# Define parameters\nstate_code = \"12\" # Florida\ncounty_code = \"095\"\ + \ # Orange County\nparam_codes = [\n \"44201\", # Ozone\n \"42401\",\ + \ # SO2\n \"42101\", # CO \n \"42602\", # NO2\n \"88101\" # PM2.5\n\ + ]\n\n# Use annual data API endpoint since we want historical data\n# Sample\ + \ every 5 years from 1980-2020 to avoid too many API calls\nyears = range(1980,\ + \ 2021, 5)\n\n# Initialize list to store results\nall_data = []\n\n# Make API\ + \ requests for each year\nfor year in years:\n bdate = f\"{year}0101\"\n\ + \ edate = f\"{year}1231\"\n \n url = f\"https://aqs.epa.gov/data/api/annualData/byCounty\"\ + \n params = {\n \"email\": email,\n \"key\": key,\n \ + \ \"param\": \",\".join(param_codes),\n \"bdate\": bdate,\n \"\ + edate\": edate,\n \"state\": state_code,\n \"county\": county_code\n\ + \ }\n \n response = requests.get(url, params=params)\n \n if\ + \ response.status_code == 200:\n data = response.json()\n if \"\ + Data\" in data:\n all_data.extend(data[\"Data\"])\n else:\n\ + \ print(f\"No data found for year {year}\")\n else:\n print(f\"\ + Error {response.status_code}: {response.text}\")\n\n# Check the structure of\ + \ the collected data\nif all_data:\n print(all_data[0]) # Print the first\ + \ entry to see the structure\n df = pd.DataFrame(all_data)\n \n # Clean\ + \ up the DataFrame\n if not df.empty:\n # Select relevant columns\n\ + \ # parameter_name doesn't exist- use `parameter` or `parameter_code`:\n\ + \ cols = ['parameter', 'year', 'arithmetic_mean', \n \ + \ 'first_max_value', 'first_max_datetime',\n 'observation_count']\n\ + \ df = df[cols]\n \n # Sort by year and parameter\n \ + \ df = df.sort_values(['year', 'parameter'])\n \n # Convert\ + \ year to int\n df['year'] = df['year'].astype(int)\n \n \ + \ # Display first few rows\n print(df.head())\n else:\n print(\"\ + No data found for the specified parameters\")\nelse:\n print(\"No data collected.\"\ + )\n" + integration: 18e5974c-8cea-4fe7-88db-7c5779a17d5d + notes: null + query: Collect historical air quality data from the EPA Air Quality System (AQS) + for Orange County, FL. + resource_id: 19afbed5-92dc-48af-9354-55a544e417ce + resource_type: example + 365697ce-eb50-444b-8a82-637a2a0131f4: + filepath: aqs.json + integration: 18e5974c-8cea-4fe7-88db-7c5779a17d5d + name: raw_documentation + resource_id: 365697ce-eb50-444b-8a82-637a2a0131f4 + resource_type: file + 39430090-3680-4065-9bd1-7755ba712889: + code: "def get_pollutant_data(state_code, county_code, param_code, year, pollutant_name):\n\ + \ \"\"\"\n Get and process data for a specific pollutant\n \n Parameters:\n\ + \ -----------\n state_code : str\n Two-digit state code\n county_code\ + \ : str\n Three-digit county code\n param_code : str\n Parameter\ + \ code for the pollutant\n year : str\n Year to retrieve data for\n\ + \ pollutant_name : str\n Name of the pollutant (for display purposes)\n\ + \ \n Returns:\n --------\n pandas.DataFrame or None\n \ + \ DataFrame containing the processed data, or None if the request failed\n\ + \ \"\"\"\n print(f\"Retrieving daily {pollutant_name} data for Harris\ + \ County, TX for {year}...\")\n \n bdate = f\"{year}0101\" # January\ + \ 1st of the year\n edate = f\"{year}1231\" # December 31st of the year\n\ + \ \n data_response = get_aqs_data(\"dailyData/byCounty\", {\n \"\ + state\": state_code,\n \"county\": county_code,\n \"param\": param_code,\n\ + \ \"bdate\": bdate,\n \"edate\": edate\n })\n \n if data_response\ + \ and data_response.get('Header', [{}])[0].get('status') == 'Success':\n \ + \ data = data_response.get('Data', [])\n df = pd.DataFrame(data)\n\ + \ \n # Convert date_local to datetime\n df['date'] = pd.to_datetime(df['date_local'])\n\ + \ \n # Group by site and date to get daily averages for each site\n\ + \ site_daily_avg = df.groupby(['local_site_name', 'date'])['arithmetic_mean'].mean().reset_index()\n\ + \ \n # Get the overall daily average across all sites\n \ + \ daily_avg = site_daily_avg.groupby('date')['arithmetic_mean'].mean().reset_index()\n\ + \ \n # Rename the arithmetic_mean column to the pollutant name\ + \ for clarity\n daily_avg = daily_avg.rename(columns={'arithmetic_mean':\ + \ pollutant_name.lower()})\n \n return daily_avg, df['units_of_measure'].iloc[0]\n\ + \ else:\n print(f\"Failed to retrieve {pollutant_name} data\")\n \ + \ return None, None\n" + integration: 18e5974c-8cea-4fe7-88db-7c5779a17d5d + notes: null + query: 'Here''s a reusable function for retrieving and processing data for multiple + pollutants:' + resource_id: 39430090-3680-4065-9bd1-7755ba712889 + resource_type: example + 4245dd8c-005b-4bfa-b50e-ba076b53ffbe: + code: "import requests\nfrom genson import SchemaBuilder\nimport os\n\n# Get API\ + \ credentials from environment variables\nemail = os.environ.get(\"API_EPA_AQS_EMAIL\"\ + )\nkey = os.environ.get(\"API_EPA_AQS\")\n\n# Define parameters\nstate_code\ + \ = \"48\" # Texas\ncounty_code = \"201\" # Harris County\nparam_codes = [\n\ + \ \"44201\", # Ozone\n \"42401\", # SO2\n \"42101\", # CO \n \ + \ \"42602\", # NO2\n \"88101\", # PM2.5\n \"81102\" # PM10\n]\n\n# Sample\ + \ one year to inspect the response structure\nyear = 2020\nbdate = f\"{year}0101\"\ + \nedate = f\"{year}1231\"\n\nurl = f\"https://aqs.epa.gov/data/api/annualData/byCounty\"\ + \nparams = {\n \"email\": email,\n \"key\": key,\n \"param\": \",\"\ + .join(param_codes),\n \"bdate\": bdate,\n \"edate\": edate,\n \"state\"\ + : state_code,\n \"county\": county_code\n}\n\nresponse = requests.get(url,\ + \ params=params)\nresponse_data = response.json()\n\n# annual data by county\ + \ can include a lot of data, especially if we specify a broad date range and\ + \ a county with a lot of data- lets generate a schema and inspect that instead\ + \ of inspecting the raw data\nbuilder = SchemaBuilder()\nbuilder.add_object(response_data)\n\ + schema = builder.to_schema()\n\nprint(schema[\"properties\"])\n" + integration: 18e5974c-8cea-4fe7-88db-7c5779a17d5d + notes: null + query: |- + Inspect API response change by generating a schema from an API response so that we don't inspect the raw data and blow up the context window. + resource_id: 4245dd8c-005b-4bfa-b50e-ba076b53ffbe + resource_type: example + e643cf2e-66f1-428e-8f2c-2626c4c59192: + code: "import os\nimport requests\nimport pandas as pd\nimport matplotlib.pyplot\ + \ as plt\nimport seaborn as sns\nfrom datetime import datetime\n\n# Set the\ + \ style for our plots\nsns.set(style=\"whitegrid\")\nplt.rcParams['figure.figsize']\ + \ = (12, 6)\n\n# Get authentication details from environment variables\nemail\ + \ = os.environ.get(\"API_EPA_AQS_EMAIL\")\napi_key = os.environ.get(\"API_EPA_AQS\"\ + )\n\n# Base URL for the EPA AQS API\nbase_url = \"https://aqs.epa.gov/data/api\"\ + \n\n# Function to make API requests\ndef get_aqs_data(endpoint, params=None):\n\ + \ \"\"\"\n Make a request to the EPA AQS API\n \n Parameters:\n\ + \ -----------\n endpoint : str\n The API endpoint to query\n \ + \ params : dict\n Additional parameters for the request\n \n\ + \ Returns:\n --------\n dict\n The JSON response from the API\n\ + \ \"\"\"\n # Add authentication parameters\n if params is None:\n \ + \ params = {}\n params['email'] = email\n params['key'] = api_key\n\ + \ \n # Make the request\n url = f\"{base_url}/{endpoint}\"\n response\ + \ = requests.get(url, params=params)\n \n # Check if the request was successful\n\ + \ if response.status_code == 200:\n return response.json()\n else:\n\ + \ print(f\"Error: {response.status_code}\")\n print(response.text)\n\ + \ return None\n\n# Function to get and process data for a specific pollutant\n\ + def get_pollutant_data(state_code, county_code, param_code, year, pollutant_name):\n\ + \ \"\"\"\n Get and process data for a specific pollutant\n \n Parameters:\n\ + \ -----------\n state_code : str\n Two-digit state code\n county_code\ + \ : str\n Three-digit county code\n param_code : str\n Parameter\ + \ code for the pollutant\n year : str\n Year to retrieve data for\n\ + \ pollutant_name : str\n Name of the pollutant (for display purposes)\n\ + \ \n Returns:\n --------\n pandas.DataFrame or None\n \ + \ DataFrame containing the processed data, or None if the request failed\n\ + \ \"\"\"\n print(f\"Retrieving daily {pollutant_name} data for {year}...\"\ + )\n \n bdate = f\"{year}0101\" # January 1st of the year\n edate =\ + \ f\"{year}1231\" # December 31st of the year\n \n data_response = get_aqs_data(\"\ + dailyData/byCounty\", {\n \"state\": state_code,\n \"county\"\ + : county_code,\n \"param\": param_code,\n \"bdate\": bdate,\n\ + \ \"edate\": edate\n })\n \n if data_response and data_response.get('Header',\ + \ [{}])[0].get('status') == 'Success':\n data = data_response.get('Data',\ + \ [])\n df = pd.DataFrame(data)\n \n print(f\"\\nFound\ + \ {len(data)} daily {pollutant_name} records for {year}\")\n \n \ + \ # Convert date_local to datetime\n df['date'] = pd.to_datetime(df['date_local'])\n\ + \ \n # Extract month and day of week for analysis\n df['month']\ + \ = df['date'].dt.month\n df['month_name'] = df['date'].dt.month_name()\n\ + \ \n # Group by site and date to get daily averages for each site\n\ + \ site_daily_avg = df.groupby(['local_site_name', 'date'])['arithmetic_mean'].mean().reset_index()\n\ + \ \n # Get the overall daily average across all sites\n \ + \ daily_avg = site_daily_avg.groupby('date')['arithmetic_mean'].mean().reset_index()\n\ + \ \n # Rename the arithmetic_mean column to the pollutant name\ + \ for clarity\n daily_avg = daily_avg.rename(columns={'arithmetic_mean':\ + \ pollutant_name.lower()})\n \n return daily_avg, df['units_of_measure'].iloc[0]\n\ + \ else:\n print(f\"Failed to retrieve {pollutant_name} data\")\n \ + \ return None, None\n\n# Set up parameters\nstate_code = \"48\" # Texas\n\ + county_code = \"201\" # Harris County\nyear = \"2022\"\n\n# Define pollutants\ + \ to retrieve\npollutants = {\n 'Ozone': '44201',\n 'PM2.5': '88101',\n\ + \ 'NO2': '42602',\n 'SO2': '42401'\n}\n\n# Get data for each pollutant\n\ + pollutant_data = {}\nunits = {}\n\nfor pollutant_name, param_code in pollutants.items():\n\ + \ daily_avg, unit = get_pollutant_data(state_code, county_code, param_code,\ + \ year, pollutant_name)\n if daily_avg is not None:\n pollutant_data[pollutant_name.lower()]\ + \ = daily_avg\n units[pollutant_name.lower()] = unit\n\n# Merge all pollutant\ + \ data on date\nmerged_df = None\n\nfor pollutant_name, daily_avg in pollutant_data.items():\n\ + \ if merged_df is None:\n merged_df = daily_avg\n else:\n \ + \ merged_df = pd.merge(merged_df, daily_avg[['date', pollutant_name]], on='date',\ + \ how='outer')\n\n# Calculate correlations between pollutants\nif merged_df\ + \ is not None:\n print(\"\\nCorrelations between pollutants:\")\n correlation_matrix\ + \ = merged_df.drop('date', axis=1).corr()\n print(correlation_matrix)\n \ + \ \n # Plot correlation heatmap\n plt.figure(figsize=(10, 8))\n sns.heatmap(correlation_matrix,\ + \ annot=True, cmap='coolwarm', vmin=-1, vmax=1, center=0)\n plt.title(f'Correlation\ + \ Matrix of Air Pollutants ({year})', fontsize=16)\n plt.tight_layout()\n\ + \ plt.show()\n" + integration: 18e5974c-8cea-4fe7-88db-7c5779a17d5d + notes: null + query: |- + Multi-pollutant correlation analysis for air quality research. This example demonstrates how to retrieve data for multiple pollutants (Ozone, PM2.5, NO2, and SO2), process the data to get daily averages, merge the datasets, and analyze correlations between different pollutants. It includes a visualization of the correlation matrix using a heatmap. + resource_id: e643cf2e-66f1-428e-8f2c-2626c4c59192 + resource_type: example +slug: adhocspecificationintegration_epa_air_quality_system +source: "# OpenAPI Spec JSON:\n${raw_documentation}\n\n# Additional Instructions:\n\ + \nThis API is through OpenAPI Spec (Swagger). You will be provided the schema.\n\ + All requests go to the following URL: https://aqs.epa.gov/data/api\nThat is the\ + \ base URL for all requests.\n\nTo use the AQS API always retrieve the email/key\ + \ credentials from environment variables:\n- email: os.environ.get(\"API_EPA_AQS_EMAIL\"\ + )\n- api_key: os.environ.get(\"API_EPA_AQS\")\nDo not use placeholder auth values\ + \ from the OpenAPI spec nor prompt the user for these values, just use the environment\ + \ variables.\n\nEnsure you know the fields that come back from the OpenAPI spec\ + \ in order to process the AQS data.\n\nDaily/yearly data can only be accessed one\ + \ year at a time. If working for that in year/decade ranges, prefer to use the annualData\ + \ endpoints, as the daily data may be too large to fit into context.\nIf the user\ + \ requests data for a year range, you will need to make multiple requests to the\ + \ API- potentially sampling in between, or creating averages, in order to not make\ + \ requests for every single year in the range.\nThe earliest sample in the data\ + \ set is from 1957. Year 1980 marks the beginning of nationally consistent operational\ + \ and quality assurance procedures.\n\nMeasured entities in the AQS data set are\ + \ referred to as parameters (which includes pollutants/substances), and use integer\ + \ codes specific to AQS.\n\nIf a monitor is scheduled to collect data and does not\ + \ it will contain `null` data. This is a placeholder to let EPA know that more data\ + \ will not be forthcoming. AQS does place absolute limits on the values that\ + \ can be submitted.\nHowever, these are fairly liberal limits- for many parameters,\ + \ negative values are acceptable.\nAQS has a nominal quarterly reporting deadline:\ + \ data must be reported by 90 days after the end of the calendar quarter in which\ + \ they are collected.\n\nIf you need to inspect the data of a JSON response before\ + \ processing, and you're dealing with annual data, or multi-daily data, or\nfor\ + \ a geographic region that could yield too much data to process (or fit into an\ + \ LLM context window),\n\nbetter use this endpoint:\n```\n/metaData/fieldsByService?email={{email}}&key={{key}}&service=annualData\n\ + ```\n(example above is checking shape of annualData endpoint). It will provide fields\ + \ documentation for each property in the response.\nIf you use `print(response_data)`\ + \ on a huge JSON response, you will not be able to proceed- you can also try checking\ + \ the length as string,\nand if that's too long, inspect whatever amount of characters\ + \ that would be enough.\nThat is, never do:\n```\ndata = response.json()\nprint(data)\n\ + ```\ninstead, either use fieldsByService endpoint as described earlier, or use the\ + \ Example 3 below (in the #Examples section) to generate a schema and inspect the\ + \ data.\n\nIMPORTANT: `parameter_name` is not a property in the respponse usually,\ + \ check `parameter` or `parameter_code` in responses first instead!\n\n# Output\ + \ Format for the AQS API - JSON\nThe only output format available from the services\ + \ is JSON. This is in conformity with the 18f API standards for government.\n\n\ + The JSON response has two top-level elements. A header and a body. The header contains\ + \ information about the request and the body contains the data.\n\nThe header contains\ + \ the following elements:\n\n- `status`. \"SUCCESS\" if the request returned data.\ + \ \"FAILED\" if the request could not be processed. \"No data matched your selection\"\ + \ if the request was processed but resulted in no data.\n- `request_time`. The date\ + \ and time (at the location of the server) when the request was received.\n- `url`.\ + \ The original URL that was submitted to make the request.\n- `row`. The number\ + \ of rows of data in the body object.\n- `errors`. Any errors encountered when attempting\ + \ to respond to the request.\nThe body is an array with one object per data row.\ + \ The contents of the object will vary based on the request made. (E.g. the columns\ + \ of data returned are different for each query.\n\nHere is a sample of the output\ + \ of a Raw Data request with one data point:\n```\n {\n \"Header\": [\n\ + \ {\n \"status\": \"success\",\n \"request_time\":\ + \ \"2018-06-13T07:45:01-04:00\",\n \"url\": \"https://...\",\n \ + \ \"rows\": 1\n }\n ],\n \"Body\": [\n {\n \ + \ \"state_code\": \"01\",\n \"county_code\": \"073\",\n \ + \ \"site_number\": \"0023\",\n \"parameter_code\": \"88101\",\n\ + \ \"poc\": 1.0,\n \"latitude\": 33.0,\n \"longitude\"\ + : -86.0,\n \"datum\": \"WGS84\",\n \"parameter_name\": \"\ + PM2.5 - Local Conditions\",\n \"date_local\": \"2017-04-01\",\n \ + \ \"time_local\": \"00:00\",\n \"date_gmt\": \"2017-04-01\",\n\ + \ \"time_gmt\": \"06:00\",\n \"sample_measurement\": 6.2,\n\ + \ \"unit_of_measure\": \"Micrograms/cubic meter (LC)\",\n \ + \ \"detection_limit\": 2.0,\n \"uncertainty\": null,\n \"\ + qualifiers\": null,\n \"method_type\": \"FRM\",\n \"method_code\"\ + : \"142\",\n \"method_name\": \"BGI Models PQ200-VSCC or PQ200A-VSCC\ + \ - Gravimetric\",\n \"state_name\": \"Alabama\",\n \"county_name\"\ + : \"Jefferson\",\n \"date_of_last_change\": \"2017-05-30\",\n \ + \ \"cbsa_code\": \"13820\"\n }\n ]\n }\n```\n\n# Error Handling\ + \ and Status Codes\nIf the API is able to parse your request it will return the\ + \ JSON described above and an HTTP status of 200.\n\nIf the API is not able to parse\ + \ your request it will return a status code of 400 and only the header, which will\ + \ include an array of error messages instead of a row count.\nFor example:\n```\n\ + \ {\n \"Header\": [{\n \"status\": \"Failed\",\n \ + \ \"request_time\": \"2018-06-13T08:05:46.588-04:00\",\n \"url\": \"\ + https://...\",\n \"error\": [\n \"value is missing or\ + \ the value is empty: param\"\n ]\n }],\n \"Body\"\ + : []\n }\n```\n\nIf the API is able to parse your request but no data matches\ + \ your selections, it will return the\nJSON header and an HTTP status of 200. The\ + \ row count will be zero,\nthe status will be a message that no data matched your\ + \ selection criteria,\nand the body will be empty.\n```\n{\n \"Header\": [\n\ + \ {\n \"status\": \"No data matched your selection\",\n \"request_time\"\ + : \"2018-06-13T10:15:42-04:00\",\n \"url\": \"https://...\",\n \"\ + rows\": 0\n }\n ],\n \"Body\": []\n}\n```\n\n# Request Limits and Terms\ + \ of Service\nThe API has the following limits imposed on request size:\n\n- Length\ + \ of time. All services (except Monitor) must have the end date (edate field) be\ + \ in the same year as the begin date (bdate field).\n- Number of parameters. Most\ + \ services allow for the selection of multiple parameter codes (param field). A\ + \ maximum of 5 parameter codes may be listed in a single request.\n\nPlease adhere\ + \ to the following when using the API:\n\n- _Limit the size of queries_. Our database\ + \ contains billions of values and you may request more than you intend. If you are\ + \ unsure of the amount of data, start small and work your way up. We request that\ + \ you limit queries to 1,000,000 rows of data each. You can use the \"observation\ + \ count\" field on the annualData service to determine how much data exists for\ + \ a time-parameter-geography combination. If you have any questions or need advice,\ + \ please contact us.\n- _Limit the frequency of queries_. Our system can process\ + \ a limited load. If scripting requests, please wait for one request to complete\ + \ before submitting another and do not make more than 10 requests per minute. Also,\ + \ we request a pause of 5 seconds between requests and adjust accordingly for response\ + \ time and size.\nIf you violate these terms, we may disable your account without\ + \ notice (but we will be in contact via the email address provided).\n\n# Usage\ + \ tips\nThis section contains suggestions for completing certain data related tasks\ + \ and links to software tools using the API.\n\n- Determine if or how much data\ + \ exists for a time-parameter-geography combination:\n - Retrieve data using\ + \ the annualData service.\n - If no records are returned, we do not have the\ + \ data.\n - If records are returned, use the observation count to determine the\ + \ temporal and geographic distribution of the data.\n\n- Monthly averages:\n \ + \ - AQS does not routinely calculate monthly aggregate statistics.\n - If you\ + \ need these, you must calculate them yourself.\n - These can be calculated from\ + \ the sample data or the daily data without loss of fidelity.\n\n- Determine a single\ + \ value for a site with collocated monitors:\n - Many sites will have collocated\ + \ monitors - monitors collecting the same parameter at the same time.\n - The\ + \ API currently provides only monitor level values. (We anticipate adding site-level\ + \ values at some point.)\n - For some criteria pollutants (PM2.5, ozone, lead,\ + \ and NO2), the regulations define procedures for defining a single site-level value.\n\ + \ - For other pollutants, determining a single site-level value is left to the\ + \ investigator.\n\n# More Details\n## API Structure and Capabilities\n- The EPA\ + \ AQS API provides access to air quality data from monitoring stations across the\ + \ US\n- Data is available at multiple temporal resolutions: annual, quarterly, daily,\ + \ and sample-level.\n- Data can be queried at various geographic levels: state,\ + \ county, site, CBSA, and bounding box\n- The API returns data in JSON format with\ + \ a standard structure\n- Key pollutants relevant to asthma (ozone, PM2.5, NO2,\ + \ SO2, CO) are available through the API\n- Data can be used to analyze both acute\ + \ exposures (daily data) and chronic exposures (annual data)\n- Seasonal patterns\ + \ in air quality can be correlated with seasonal patterns in asthma exacerbations\n\ + - The API enables spatial analysis to identify potential hotspots of poor air quality\n\ + \n## Data Integration Strategies\n- Geographic alignment: Use census tract or ZIP\ + \ code as common geographic units\n- Temporal alignment: Monthly aggregation captures\ + \ seasonal patterns while managing data volume\n- Spatial interpolation can estimate\ + \ air quality values between monitoring stations\n- Population-weighted averages\ + \ can be used when aggregating to larger geographic areas\n- A standardized spatial\ + \ grid can facilitate integration with other environmental datasets\n\n## Key Parameters\ + \ for Asthma Research\nThe API provides access to several air pollutants known to\ + \ affect respiratory health:\n```\nParameter Code\tParameter Name\tRelevance to\ + \ Asthma\n44201\tOzone\tMajor trigger for asthma attacks\n88101\tPM2.5\tCan penetrate\ + \ deep into lungs\n81102\tPM10\tCan irritate airways\n42602\tNitrogen Dioxide (NO2)\t\ + Common in vehicle emissions\n42401\tSulfur Dioxide (SO2)\tIndustrial emissions linked\ + \ to respiratory issues\n42101\tCarbon Monoxide\t(CO) Affects oxygen transport\n\ + ```\n\n## Key Endpoints and Parameters\nThe API offers several endpoint categories:\n\ + \n- Monitoring Sites: monitors/byCounty - Returns information about monitoring stations\n\ + - Daily Data: dailyData/byCounty - Daily measurements for parameters\n- Annual Data:\ + \ annualData/byCounty - Yearly aggregated measurements\n- Sample Data: sampleData/byCounty\ + \ - Individual sample measurements\n- Metadata: Various endpoints for parameter\ + \ codes, classes, etc.\n\nAll data requests require:\n- Geographic parameters (state/county\ + \ codes)\n- Date parameters (bdate/edate in YYYYMMDD format)\n- Parameter code for\ + \ the pollutant\n\n# Implementation Challenges and Solutions\n- Date Parameter Requirements:\ + \ Despite documentation suggesting otherwise, all data endpoints require explicit\ + \ date parameters (bdate and edate), not just the year parameter.\n- Data Volume\ + \ Management: Harris County has numerous monitoring stations (we found 24,532 daily\ + \ ozone records for 2022 alone), requiring efficient data aggregation strategies.\n\ + - Parameter Identification: The API requires specific parameter codes rather than\ + \ names, necessitating lookup of these codes before querying data.\n\nThe EPA AQS\ + \ API provides rich, detailed air quality data that can be invaluable\nfor researching\ + \ environmental factors affecting childhood asthma (and others).\nBy combining this\ + \ data with health outcome data from other sources, researchers\ncan develop comprehensive\ + \ analyses of how air quality impacts respiratory health\nin specific geographic\ + \ areas like a county or region.\nThe API's structure allows for flexible querying\ + \ by location, time period, and\nspecific pollutants, making it a powerful tool\ + \ for environmental health research.\nHowever, users should be aware of specific\ + \ parameter requirements and prepare\nfor handling large volumes of data when working\ + \ with densely monitored areas.\n\nNote: If/when using the `metaData/isAvailable`\ + \ endpoint to check if the API is available,\nuse this sample response as guide:\n\ + ```\n{\n \"Header\": [\n {\n \"status\": \"API service is up\ + \ and running healthy. Status: connection_pool: size: 5, connections: 1, in use:\ + \ 1, waiting_in_queue: 0\",\n \"request_time\": \"2025-03-21T08:21:41-04:00\"\ + ,\n \"url\": \"http://aqs.epa.gov/api/metaData/isAvailable?email={{email}}.com&key={{key}}\"\ + \n }\n ],\n \"Data\": []\n}\n```\nNever simulate this API- if it is\ + \ not available ask the user how to proceed.\n" +url: null +uuid: 18e5974c-8cea-4fe7-88db-7c5779a17d5d diff --git a/src/biome/datasources/aqs/documentation/aqs.json b/src/biome/adhoc_data/datasources/aqs/attachments/aqs.json similarity index 100% rename from src/biome/datasources/aqs/documentation/aqs.json rename to src/biome/adhoc_data/datasources/aqs/attachments/aqs.json diff --git a/src/biome/adhoc_data/datasources/cbioportal/api.yaml b/src/biome/adhoc_data/datasources/cbioportal/api.yaml new file mode 100644 index 0000000..ab36919 --- /dev/null +++ b/src/biome/adhoc_data/datasources/cbioportal/api.yaml @@ -0,0 +1,642 @@ +datatype: api +description: | + The cBioPortal for Cancer Genomics is an open-access, open-source resource for interactive exploration of multidimensional cancer genomics data sets. The goal of cBioPortal is to significantly lower the barriers between complex genomic data and cancer researchers by providing rapid, intuitive, and high-quality access to molecular profiles and clinical attributes from large-scale cancer genomics projects, and therefore to empower researchers to translate these rich data sets into biologic insights and clinical applications. +img_url: null +last_updated: null +name: cbioportal +provider: adhoc:specialist_agents +resources: + 21897f3f-9451-4338-b2b4-8ae3faeafc20: + code: | + import requests + import pandas as pd + + def fetch_aml_studies(): + # Base URL for cBioPortal API + base_url = "https://www.cbioportal.org/api" + + # Get all studies + response = requests.get(f"{base_url}/studies") + + if response.status_code != 200: + raise Exception(f"API request failed with status code {response.status_code}") + + # Get all studies as JSON + studies = response.json() + + # Filter for AML related studies + aml_studies = [] + for study in studies: + # Convert all fields to lowercase for case-insensitive search + study_id = study.get('studyId', '').lower() + name = study.get('name', '').lower() + cancer_type = study.get('cancerTypeId', '').lower() + description = study.get('description', '').lower() + + # Check if 'aml' appears in any of the relevant fields + if any('aml' in field for field in [study_id, name, cancer_type, description]): + aml_studies.append({ + 'studyId': study['studyId'], + 'name': study['name'], + 'description': study.get('description', 'N/A'), + 'cancerTypeId': study.get('cancerTypeId', 'N/A'), + 'sampleCount': study.get('allSampleCount', 0), + 'status': study.get('status', 'N/A'), + 'publicStudy': study.get('publicStudy', False) + }) + + # Convert to DataFrame for better display + df = pd.DataFrame(aml_studies) + + # Sort by sample count descending + df = df.sort_values('sampleCount', ascending=False) + + return df + + # Execute the function + aml_studies_df = fetch_aml_studies() + print(f"Found {len(aml_studies_df)} AML-related studies") + print(aml_studies_df) + integration: 33e3119f-5504-4615-981e-55000c066e80 + notes: null + query: Fetch and filter studies related to Acute Myeloid Leukemia (AML) from cBioPortal + API. + resource_id: 21897f3f-9451-4338-b2b4-8ae3faeafc20 + resource_type: example + 3449a203-e65b-4a8d-91cb-593ccdbef58c: + code: | + import requests + import pandas as pd + import time + + # Configuration + BASE_URL = "https://www.cbioportal.org/api" + + # Define studies and their RNA-seq profiles + STUDY_PROFILES = { + 'aml_target_gdc': { + 'name': 'TARGET-AML (GDC)', + 'profile': 'aml_target_gdc_mrna_seq_tpm_Zscores', + 'type': 'TPM' + }, + 'aml_ohsu_2022': { + 'name': 'OHSU AML 2022', + 'profile': 'aml_ohsu_2022_mrna_median_Zscores', + 'type': 'RPKM' + } + # ... other studies ... + } + + # Define STAT5 genes with their Entrez IDs + STAT5_GENES = { + 6776: 'STAT5A', + 6777: 'STAT5B' + } + + # Function to get RNA-seq samples for a study + def get_rna_seq_samples(study_id): + """Get list of sample IDs that have RNA-seq data for a study.""" + url = f"{BASE_URL}/studies/{study_id}/sample-lists" + response = requests.get(url) + if response.status_code == 200: + sample_lists = response.json() + # Find RNA-seq sample list + rna_list = next((sl['sampleListId'] for sl in sample_lists + if sl['category'] == 'all_cases_with_mrna_rnaseq_data'), None) + if rna_list: + # Get sample IDs from the list + url = f"{BASE_URL}/sample-lists/{rna_list}/sample-ids" + response = requests.get(url) + if response.status_code == 200: + return response.json() + return None + + # Function to get expression data + def get_expression_data(profile_id, gene_ids, sample_ids): + """Fetch expression z-scores for specific genes and samples.""" + url = f"{BASE_URL}/molecular-profiles/{profile_id}/molecular-data/fetch" + # IMPORTANT: Must provide sample IDs explicitly + data = { + "sampleIds": sample_ids, # List of specific sample IDs + "entrezGeneIds": gene_ids # List of Entrez gene IDs + } + response = requests.post(url, json=data) + if response.status_code == 200: + return response.json() + return None + + # Process each study + all_study_data = [] + + for study_id, study_info in STUDY_PROFILES.items(): + print(f"\nProcessing {study_info['name']}...") + + # First get the sample IDs for the study + sample_ids = get_rna_seq_samples(study_id) + + if sample_ids: + print(f"Found {len(sample_ids)} RNA-seq samples") + + # Process samples in chunks to avoid large requests + chunk_size = 100 + study_data = [] + + for i in range(0, len(sample_ids), chunk_size): + chunk = sample_ids[i:i + chunk_size] + print(f"Fetching data for samples {i+1}-{i+len(chunk)}...") + + # Get expression data for this chunk of samples + expression_data = get_expression_data( + study_info['profile'], + list(STAT5_GENES.keys()), + chunk + ) + + if expression_data: + study_data.extend(expression_data) + time.sleep(0.2) # Small delay between requests + + if study_data: + # Convert to DataFrame + df = pd.DataFrame(study_data) + + # Add metadata + df['gene_symbol'] = df['entrezGeneId'].map(STAT5_GENES) + df['study_id'] = study_id + df['study_name'] = study_info['name'] + df['measurement_type'] = study_info['type'] + + # Select and rename columns + df = df[[ + 'study_id', 'study_name', 'measurement_type', + 'gene_symbol', 'sampleId', 'value' + ]] + df.columns = [ + 'study_id', 'study_name', 'measurement_type', + 'gene', 'sample_id', 'zscore' + ] + + all_study_data.append(df) + + # Combine all data + if all_study_data: + # Create long format + combined_df = pd.concat(all_study_data, ignore_index=True) + + # Create wide format (samples as rows, genes as columns) + combined_wide = combined_df.pivot_table( + index=['study_id', 'study_name', 'measurement_type', 'sample_id'], + columns='gene', + values='zscore' + ).reset_index() + + # Save both versions + stat5_all_studies = combined_df # Long format + stat5_all_studies_wide = combined_wide # Wide format + + # Example of the resulting data structure: + print("\nLong format example (first few rows):") + print(stat5_all_studies.head()) + + print("\nWide format example (first few rows):") + print(stat5_all_studies_wide.head()) + integration: 33e3119f-5504-4615-981e-55000c066e80 + notes: | + This example demonstrates: + 1. How to properly fetch sample IDs for each study + 2. How to use these sample IDs when requesting molecular data + 3. How to handle large requests by processing samples in chunks + 4. How to combine data from multiple studies into a single dataframe + 5. How to create both long and wide format versions of the data + + Key points: + - Must provide explicit sample IDs in the molecular data request + - Z-scores are pre-calculated by cBioPortal for each study + - Different studies may use different measurement types (TPM, RPKM, RSEM) + - Processing in chunks helps avoid timeouts with large datasets + query: Example of how to fetch RNA-seq z-scores for STAT5A and STAT5B across multiple + AML studies + resource_id: 3449a203-e65b-4a8d-91cb-593ccdbef58c + resource_type: example + 52adfa23-1b07-462e-968b-d09ed12848f0: + filepath: cbioportal.json + integration: 33e3119f-5504-4615-981e-55000c066e80 + name: cbioportal + resource_id: 52adfa23-1b07-462e-968b-d09ed12848f0 + resource_type: file + 61d5403b-63fe-4f11-9e0d-04c5aec310df: + code: | + import requests + import pandas as pd + from collections import Counter + import numpy as np + + # Base URL for cBioPortal API + base_url = "https://www.cbioportal.org/api" + study_id = "aml_target_gdc" # Example study ID + + # Get mutations using the sample list ID + mutations_url = f"{base_url}/molecular-profiles/{study_id}_mutations/mutations?sampleListId={study_id}_all" + mutations_response = requests.get(mutations_url) + + if mutations_response.status_code == 200: + mutations_data = mutations_response.json() + mutations_df = pd.DataFrame(mutations_data) + + # Get gene information for the entrez IDs + entrez_ids = [int(x) for x in mutations_df['entrezGeneId'].unique()] # Convert to regular Python integers + genes_url = f"{base_url}/genes/fetch" + genes_response = requests.post(genes_url, json=entrez_ids) + + if genes_response.status_code == 200: + genes_data = genes_response.json() + gene_map = {gene['entrezGeneId']: gene['hugoGeneSymbol'] for gene in genes_data} + + # Add gene symbols to mutations dataframe + mutations_df['gene_symbol'] = mutations_df['entrezGeneId'].map(gene_map) + + # Print summary statistics + print(f"Total mutations found: {len(mutations_df)}") + print(f"Number of affected genes: {len(mutations_df['gene_symbol'].unique())}") + + print("\nMost frequently mutated genes:") + print(mutations_df['gene_symbol'].value_counts().head(10)) + + print("\nMutation types distribution:") + print(mutations_df['mutationType'].value_counts()) + + print("\nSample of mutations (first 5 rows):") + display_cols = ['gene_symbol', 'sampleId', 'mutationType', 'proteinChange', 'chr', 'startPosition', 'variantType'] + print(mutations_df[display_cols].head()) + + # Calculate sample statistics + samples_with_mutations = len(mutations_df['sampleId'].unique()) + print(f"\nNumber of samples with mutations: {samples_with_mutations}") + + avg_mutations_per_sample = len(mutations_df) / samples_with_mutations + print(f"Average mutations per sample: {avg_mutations_per_sample:.2f}") + else: + print(f"Error fetching mutations: {mutations_response.status_code}") + print("Response:", mutations_response.text) + integration: 33e3119f-5504-4615-981e-55000c066e80 + notes: | + It shows: + 1. How to properly construct the API endpoint URL with sample list ID + 2. How to fetch mutation data for a study + 3. How to get gene symbol information using entrez IDs + 4. How to create a comprehensive mutation analysis including: + - Total mutation counts + - Gene frequency analysis + - Mutation type distribution + - Sample statistics + The example uses the TARGET AML (GDC) study but can be modified for any study by changing the study_id. + query: This example demonstrates how to fetch and analyze mutation data for a + specific study in cBioPortal + resource_id: 61d5403b-63fe-4f11-9e0d-04c5aec310df + resource_type: example + 7f738f92-b1d2-463b-826f-db0dc1331e56: + code: | + import requests + import pandas as pd + from typing import List, Dict + + def get_molecular_profiles(study_ids: List[str]) -> Dict[str, pd.DataFrame]: + """ + Retrieve molecular profiles for multiple studies from cBioPortal. + + Args: + study_ids: List of cBioPortal study IDs + + Returns: + Dictionary mapping study IDs to DataFrames containing their molecular profiles + """ + # Base URL for cBioPortal's web API + base_url = "https://www.cbioportal.org/api" + + # Dictionary to store results + results = {} + + print("Fetching molecular profiles...") + for study_id in study_ids: + print(f"\nStudy: {study_id}") + print("-" * 40) + + # Get molecular profiles for this study + response = requests.get(f"{base_url}/studies/{study_id}/molecular-profiles") + + if response.status_code == 200: + profiles = response.json() + + # Create a summary DataFrame + profile_data = [] + for profile in profiles: + profile_data.append({ + 'molecularProfileId': profile['molecularProfileId'], + 'name': profile['name'], + 'datatype': profile['datatype'], + 'molecularAlterationType': profile['molecularAlterationType'], + 'description': profile.get('description', 'N/A') + }) + + if profile_data: + df = pd.DataFrame(profile_data) + results[study_id] = df + print(f"Found {len(df)} molecular profiles") + + # Display summary of profile types + print("\nProfile types:") + type_summary = df.groupby('molecularAlterationType')['molecularProfileId'].count() + print(type_summary) + else: + print("No molecular profiles found") + results[study_id] = pd.DataFrame() + else: + print(f"Error accessing study: {response.status_code}") + results[study_id] = pd.DataFrame() + + return results + + # Example usage with AML studies + aml_studies = [ + 'aml_target_2018_pub', + 'aml_ohsu_2018', + 'aml_ohsu_2022', + 'laml_tcga', + 'laml_tcga_pan_can_atlas_2018', + 'aml_target_gdc', + 'mnm_washu_2016' + ] + + # Get molecular profiles for all studies + profiles_by_study = get_molecular_profiles(aml_studies) + + # Example of how to work with the results - get RNA expression profiles for first study + first_study = list(profiles_by_study.keys())[0] + if not profiles_by_study[first_study].empty: + rna_profiles = profiles_by_study[first_study][ + profiles_by_study[first_study]['molecularAlterationType'] == 'MRNA_EXPRESSION' + ] + print(f"\nRNA expression profiles for {first_study}:") + print(rna_profiles[['molecularProfileId', 'name', 'datatype']].to_string()) + integration: 33e3119f-5504-4615-981e-55000c066e80 + notes: null + query: |- + Example showing how to retrieve and analyze the molecular profiles available for multiple studies from cBioPortal. The code demonstrates how to fetch metadata on the molecular profiles, organize them into DataFrames, and analyze specific information about each profile like the types of profiles available such as RNA expression. + resource_id: 7f738f92-b1d2-463b-826f-db0dc1331e56 + resource_type: example + 84612cb9-2439-4e07-96ae-edfac09374e9: + code: | + import requests + import pandas as pd + import json + + # Base URL + BASE_URL = "https://www.cbioportal.org/api" + + # List of AML studies + AML_STUDIES = [ + 'aml_target_2018_pub', + 'aml_ohsu_2018', + 'aml_ohsu_2022', + 'laml_tcga', + 'laml_tcga_pan_can_atlas_2018', + 'laml_tcga_gdac', + 'aml_target_gdc', + 'washu_pdi_2016', + 'laml_tcga_pub' + ] + + # Function to get mutations for a study + def get_study_mutations(study_id): + print(f"\nProcessing study: {study_id}") + + # Get the molecular profile ID + molecular_profile_id = f"{study_id}_mutations" + + # Get the sample list ID + sample_list_id = f"{study_id}_all" + + # Fetch mutations + mutations_response = requests.get( + f"{BASE_URL}/molecular-profiles/{molecular_profile_id}/mutations", + params={"sampleListId": sample_list_id, "projection": "DETAILED"} + ) + + if mutations_response.status_code == 200: + mutations = mutations_response.json() + print(f"Found {len(mutations)} mutations") + + # Convert to DataFrame + mutations_df = pd.DataFrame(mutations) + + # Extract gene symbols from the nested dictionary + mutations_df['gene_symbol'] = mutations_df['gene'].apply(lambda x: x['hugoGeneSymbol']) + + # Create a more focused DataFrame with key columns + focused_df = mutations_df[[ + 'gene_symbol', + 'sampleId', + 'proteinChange', + 'mutationType', + 'chr', + 'startPosition', + 'endPosition', + 'referenceAllele', + 'variantAllele' + ]] + + return focused_df + else: + print(f"Error fetching mutations: {mutations_response.text}") + return None + + # Process all studies + all_mutations_dfs = [] + + for study_id in AML_STUDIES: + study_df = get_study_mutations(study_id) + if study_df is not None: + study_df['study_id'] = study_id # Add study ID column + all_mutations_dfs.append(study_df) + + # Combine all mutations + if all_mutations_dfs: + combined_df = pd.concat(all_mutations_dfs, ignore_index=True) + + print("\nOverall mutation data summary:") + print(f"Total number of mutations: {len(combined_df)}") + print(f"Number of unique genes: {combined_df['gene_symbol'].nunique()}") + print(f"Number of unique samples: {combined_df['sampleId'].nunique()}") + + print("\nTop 20 most frequently mutated genes across all studies:") + print(combined_df['gene_symbol'].value_counts().head(20)) + + print("\nDistribution of mutation types:") + print(combined_df['mutationType'].value_counts()) + + print("\nNumber of mutations by study:") + print(combined_df['study_id'].value_counts()) + + # Save to CSV + combined_df.to_csv('all_aml_mutations.csv', index=False) + integration: 33e3119f-5504-4615-981e-55000c066e80 + notes: null + query: |- + Retrieve and analyze mutation data from multiple AML studies, including data processing and summary statistics. This example demonstrates how to fetch mutations across multiple studies, handle the nested JSON response, and create a comprehensive mutation dataset with key information. + resource_id: 84612cb9-2439-4e07-96ae-edfac09374e9 + resource_type: example + a1958b77-329b-4c77-a1b6-8cbdefd56eea: + code: | + import requests + + url = "https://www.cbioportal.org/api/studies" + response = requests.get(url) + + if response.status_code == 200: + studies = response.json() + colorectal_studies = [study for study in studies if study['cancerTypeId'].lower() in ['coadread', 'coad', 'read']] + print(f"Found {len(colorectal_studies)} colorectal cancer studies.") + print("\nStudy Details:") + for study in colorectal_studies: + print(f"\nName: {study['name']}") + print(f"Study ID: {study['studyId']}") + print(f"Description: {study['description']}") + print(f"Cancer Type ID: {study['cancerTypeId']}") + print("-" * 80) + integration: 33e3119f-5504-4615-981e-55000c066e80 + notes: null + query: |- + Query and display all colorectal cancer studies from cBioPortal. This example searches for studies with cancer type IDs 'coadread', 'coad', and 'read' (representing colorectal adenocarcinoma, colon adenocarcinoma, and rectal adenocarcinoma respectively). For each study, it displays the name, study ID, description, and cancer type ID. + resource_id: a1958b77-329b-4c77-a1b6-8cbdefd56eea + resource_type: example + c68b74b2-c97c-4cf5-b8b4-7e514db2db65: + code: | + import requests + import pandas as pd + + # cBioPortal API base URL + base_url = "https://www.cbioportal.org/api" + + # Endpoint to fetch studies + endpoint = "/studies" + + # Parameters for the query + params = {"cancerTypeId": "coadread"} + + # Make the API request + response = requests.get(base_url + endpoint, params=params) + + # Check for successful response + response.raise_for_status() + + # Parse the JSON response + studies = response.json() + + # Create a pandas DataFrame from the results + df = pd.DataFrame(studies) + + # Print the DataFrame + print(df) + integration: 33e3119f-5504-4615-981e-55000c066e80 + notes: null + query: Query for studies related to colorectal cancer using the cancerTypeId 'coadread'. + resource_id: c68b74b2-c97c-4cf5-b8b4-7e514db2db65 + resource_type: example + d54d8188-81f5-4fff-a641-230480369fe4: + code: | + import requests + import json + + # Get samples for the study + study_id = "coadread_genentech" + samples_url = f"https://www.cbioportal.org/api/studies/{study_id}/samples" + samples_response = requests.get(samples_url) + + if samples_response.status_code == 200: + samples = json.loads(samples_response.content) + sample_ids = [sample['sampleId'] for sample in samples] + print(f"Found {len(sample_ids)} samples") + + # Get mutation data + mutations_url = f"https://www.cbioportal.org/api/molecular-profiles/{study_id}_mutations/mutations/fetch" + + # Create the filter with sample IDs + data = { + "sampleIds": sample_ids + } + + # Make the POST request + mutations_response = requests.post(mutations_url, json=data) + + if mutations_response.status_code == 200: + mutations = json.loads(mutations_response.content) + print(f"\nRetrieved {len(mutations)} mutations") + + # Focus on key cancer genes + key_genes = { + 'APC': 324, # EntrezGeneID for APC + 'TP53': 7157, # EntrezGeneID for TP53 + 'KRAS': 3845, # EntrezGeneID for KRAS + 'PIK3CA': 5290 # EntrezGeneID for PIK3CA + } + + # Analyze mutations for these genes + gene_details = {} + for gene_symbol, gene_id in key_genes.items(): + gene_mutations = [m for m in mutations if m['entrezGeneId'] == gene_id] + + # Count mutation types + mutation_types = {} + protein_changes = [] + + for mutation in gene_mutations: + mut_type = mutation['mutationType'] + if mut_type in mutation_types: + mutation_types[mut_type] += 1 + else: + mutation_types[mut_type] = 1 + + if mutation['proteinChange']: + protein_changes.append(mutation['proteinChange']) + + gene_details[gene_symbol] = { + 'total_mutations': len(gene_mutations), + 'mutation_types': mutation_types, + 'protein_changes': protein_changes + } + + # Print detailed analysis + print("\nDetailed Analysis of Key Cancer Genes:\n") + for gene, details in gene_details.items(): + print(f"\n{gene} Analysis:") + print(f"Total mutations: {details['total_mutations']}") + + print("Mutation types:") + for mut_type, count in details['mutation_types'].items(): + print(f" - {mut_type}: {count}") + + print("Protein changes (top 5):") + for change in details['protein_changes'][:5]: + print(f" - {change}") + if len(details['protein_changes']) > 5: + print(f" ... and {len(details['protein_changes'])-5} more changes") + integration: 33e3119f-5504-4615-981e-55000c066e80 + notes: | + The example demonstrates how to: + 1. Get sample IDs for a study + 2. Fetch mutation data using the molecular profiles endpoint + 3. Filter and analyze mutations for specific genes + 4. Count mutation types and protein changes + The code provides a detailed breakdown of mutation types and frequencies for each gene. + query: |- + Query mutation data from cBioPortal for a specific study (Genentech colorectal cancer study) and analyze mutations in key cancer genes (APC, TP53, KRAS, PIK3CA) + resource_id: d54d8188-81f5-4fff-a641-230480369fe4 + resource_type: example +slug: adhocspecificationintegration_cbioportal +source: | + This API is through Open API Spec (Swagger). You will be provided the schema below. + All requests go to the following URL: https://www.cbioportal.org/ + + ${cbioportal} +url: null +uuid: 33e3119f-5504-4615-981e-55000c066e80 diff --git a/src/biome/datasources/cbioportal/documentation/cbioportal.json b/src/biome/adhoc_data/datasources/cbioportal/attachments/cbioportal.json similarity index 100% rename from src/biome/datasources/cbioportal/documentation/cbioportal.json rename to src/biome/adhoc_data/datasources/cbioportal/attachments/cbioportal.json diff --git a/src/biome/adhoc_data/datasources/cda/api.yaml b/src/biome/adhoc_data/datasources/cda/api.yaml new file mode 100644 index 0000000..c8d5920 --- /dev/null +++ b/src/biome/adhoc_data/datasources/cda/api.yaml @@ -0,0 +1,123 @@ +datatype: api +description: "The Cancer Data Aggregator (CDA) is a service of the National Cancer\ + \ Institutes' (NCI) Cancer Research Data Commons. We pull metadata for thousands\ + \ of studies hosted at multiple data repositories across NCI, and make it available\ + \ for search from a single tool so researchers can more easily find and reuse existing\ + \ cancer research data. In between pulling and publishing, we thoroughly clean,\ + \ harmonize, and cross-reference the metadata so you can easily do things like find\ + \ subjects that have participated in multiple studies, discover data from a disease\ + \ that was originally described in different ways at each repository, and compile\ + \ all the data from your favorite program such as CPTAC - no matter where it ended\ + \ up.\n\n# Conceptual Overview\nYou can think of the CDA as a really, really enormous\ + \ spreadsheet full of data. To search this enormous spreadsheet, you'd want to select\ + \ columns that have data you're interested in, and then filter the rows to only\ + \ the values you care about.\n\nCDA data comes from six sources:\n- The Proteomic\ + \ Data Commons (PDC)\n- The Genomic Data Commons (GDC)\n- The Imaging Data Commons\ + \ (IDC)\n- The Cancer Data Service (CDS)\n- The Integrated Canine Data Commons (ICDC)\n\ + - The ISB Cancer Gateway in the Cloud (ISB-CGC)\n\nThe CDA makes this data searchable\ + \ in five main endpoints:\n- subject: A patient entity captures the study-independent\ + \ metadata for research subjects. Human research subjects are usually not traceable\ + \ to a particular person to protect the subjects privacy.\n- researchsubject: A\ + \ research subject is the entity of interest in a specific research study or project,\ + \ typically a human being or an animal, but can also be a device, group of humans\ + \ or animals, or a tissue sample. Human research subjects are usually not traceable\ + \ to a particular person to protect the subjects privacy. This entity plays the\ + \ role of the case_id in existing data. A subject who participates in 3 studies\ + \ will have 3 researchsubject IDs.\n- specimen: Any material taken as a sample from\ + \ a biological entity (living or dead), or from a physical object or the environment.\ + \ Specimens are usually collected as an example of their kind, often for use in\ + \ some investigation.\n- mutation: Molecular data about specific mutations, currently\ + \ limited to the TCGA-READ project from GDC.\n- file: A unit of data about subjects,\ + \ researchsubjects, specimens, or their associated information.\n\nand two endpoints\ + \ that offer deeper information about data in the researchsubject endpoint:\n- diagnosis:\ + \ A collection of characteristics that describe an abnormal condition of the body\ + \ as assessed at a point in time. May be used to capture information about neoplastic\ + \ and non-neoplastic conditions.\n- treatment: Represent medication administration\ + \ or other treatment types.\n\nIf the CDA works like a giant spreadsheet, the endpoints\ + \ are sets of specific columns that always go together. Any metadata field can be\ + \ searched from any endpoint, the only difference between search types is what type\ + \ of data is returned by default.\n\nIf you are looking to build a cohort of distinct\ + \ individuals who meet some criteria, you would search by subject, and the result\ + \ will be a table of information with one row per subject. If you are looking for\ + \ biosamples that can be ordered or a specific format of information (e.g. histological\ + \ slides) start with specimen. This search will return a table of information with\ + \ one row per biosample. If you want to build a cohort, but are particularly interested\ + \ in studies rather than the participants, search by researchsubject.\n\n# Data\ + \ Sources\n## Genomics Data Commons\nThe Genomic Data Commons (GDC) is a cancer\ + \ knowledge network that supports hosting, standardization, and analysis of genomic,\ + \ clinical, and biospecimen data from cancer research programs. The GDC harmonizes\ + \ raw sequencing data, identifies and applies state-of-the-art bioinformatics methods\ + \ for generating mutation calls, structural variants and other high-level data,\ + \ and provides scalable downloads and web-based analysis tools. Because of the personal\ + \ nature of genomic data, some genomic data in the GDC may be controlled access,\ + \ requiring eRA Commons authentication and dbGaP authorization to access the data.\n\ + \n## Proteomic Data Commons\nThe Proteomic Data Commons (PDC) was developed to advance\ + \ understanding of how proteins help to shape the risk, diagnosis, development,\ + \ progression, and treatment of cancer. In-depth analysis of proteomic data allows\ + \ the study of both how and why cancer develops and informs ways of tailoring treatment\ + \ for individual patients using precision medicine. All proteomic data in the PDC\ + \ are open access and, with appropriate attribution, can be included in publications.\n\ + \n## Imaging Data Commons\nNCI Imaging Data Commons (IDC) is a cloud-based repository\ + \ of publicly available cancer imaging data co-located with the analysis and exploration\ + \ tools and resources. IDC is a node within the broader NCI Cancer Research Data\ + \ Commons (CRDC) infrastructure that provides secure access to a large, comprehensive,\ + \ and expanding collection of cancer research data.\n\nAll data hosted by IDC is\ + \ available publicly. The current content of IDC is populated using the radiology\ + \ collections from The Cancer Imaging Archive (TCIA), as well as data collected\ + \ by other major NCI initiatives, such as TCGA, CPTAC, NLST and HTAN. IDC does not\ + \ perform de-identification of images but accepts data de-identified by TCIA or\ + \ other Data Coordinating Centers that are approved by NCI Security.\n\n## Cancer\ + \ Data Services\nThe CDS provides data storage and sharing capabilities for NCI-funded\ + \ studies that fall under the following categories: \u2022 Studies with data that\ + \ do not match an existing CRDC data commons \u2022 Studies with data that do not\ + \ fit current data type criteria and/or the minimum metadata standards for a CRDC\ + \ data commons.\n\nCDS currently hosts a variety of data types from NCI projects\ + \ such as the Human Tumor Atlas Network (HTAN), Division of Cancer Control and Population\ + \ Sciences (DCCPS), and Childhood Cancer Data Initiative (CCDI) as well as data\ + \ from independent research projects. The CDS is home to both open and controlled\ + \ access data.\n\n## Integrated Canine Data Commons\nThe Integrated Canine Data\ + \ Commons (ICDC) is a cloud-based repository of spontaneously-arising canine cancer\ + \ data. ICDC was established to further research on human cancers by enabling comparative\ + \ analysis with canine cancer. The data in the ICDC is sourced from multiple different\ + \ programs and projects; all focused on canine subjects. The data is harmonized\ + \ into an integrated data model and then made available to the research community.\n\ + \n## ISB Cancer Gateway in the Cloud\nThe ISB Cancer Gateway in the Cloud (ISB-CGC)\ + \ is one of three National Cancer Institute (NCI) Cloud Resources tasked with enabling\ + \ researchers to combine cancer data and cloud computation. The ISB-CGC cloud resource\ + \ hosts data from a variety of sources such as HTAN and TCGA, CPTAC, and TARGET\ + \ from the GDC and PDC in Google BigQuery columnar data tables. This includes file,\ + \ case, clinical, and open access derived data that can be accessed both programmatically\ + \ and through interactive web applications, eliminating the need to download and\ + \ store large data sets.\n\n## Data Standards Services (DSS)\nThe DSS provides us\ + \ with harmonized values mapped to the data sources above. In our current release,\ + \ DSS has provided values for: ethnicity, file_format, morphology, primary_diagnosis,\ + \ race, species, therapeutic_agent, source_material_type (cancer/normal), treatment_type,\ + \ and vital_status.\n" +img_url: null +last_updated: null +name: Cancer Data Aggregator +provider: adhoc:specialist_agents +resources: + 5361ce45-01fb-4c16-aa22-31d5397a0bd1: + filepath: cda.yaml + integration: 3a97a492-4947-4cb1-9b40-4dcfbdeee3f3 + name: raw_documentation + resource_id: 5361ce45-01fb-4c16-aa22-31d5397a0bd1 + resource_type: file +slug: adhocspecificationintegration_cancer_data_aggregator +source: | + ${raw_documentation} + + # Additional Instructions: + + Typically metadata retrieved from CDA will locate the raw data files via Data Repository Service (DRS) URIs. + DRS URIs are not to the data itself, but will return metadata about the location of the data. + The location metadata (which should include the actual data download URL) can be retrieved by using the drs_uri_info tool if it is available, + or by querying "https://nci-crdc.datacommons.io/ga4gh/drs/v1/objects/" directly. + + All data in CDA should be publicly accessible without any special permissions or credentials. + + You should make use of the python requests library to interact with the API. + Note that the base URL of the API is 'https://cda.datacommons.cancer.gov/' +url: null +uuid: 3a97a492-4947-4cb1-9b40-4dcfbdeee3f3 diff --git a/src/biome/datasources/cda/documentation/cda.yaml b/src/biome/adhoc_data/datasources/cda/attachments/cda.yaml similarity index 100% rename from src/biome/datasources/cda/documentation/cda.yaml rename to src/biome/adhoc_data/datasources/cda/attachments/cda.yaml diff --git a/src/biome/adhoc_data/datasources/cdc_tracking_network/api.yaml b/src/biome/adhoc_data/datasources/cdc_tracking_network/api.yaml new file mode 100644 index 0000000..ed2197d --- /dev/null +++ b/src/biome/adhoc_data/datasources/cdc_tracking_network/api.yaml @@ -0,0 +1,51 @@ +datatype: api +description: | + The National Environmental Public Health Tracking Network (Tracking Network) + brings together health data and environmental data from national, state, county, + and city sources and provides supporting information to make the data easier + to understand. + The Tracking Network has data and information on environments and hazards, + health effects, and population health. + This resource includes childhood emergency department visits and hospitalizations at the county level. It provides: + - Age-stratified data including children under 18 + - County-level resolution + - Information on asthma, and other conditions +img_url: null +last_updated: null +name: CDC Tracking Network +provider: adhoc:specialist_agents +resources: + 5b62b4c6-536c-4a8d-8e4b-bd295ac8ed72: + filepath: api_examples.md + integration: 62306be3-9b4b-4988-aac9-19d50b64b336 + name: api_examples + resource_id: 5b62b4c6-536c-4a8d-8e4b-bd295ac8ed72 + resource_type: file + 7c5e6353-7e1f-4639-91ee-9dc568887a69: + filepath: user_guide.md + integration: 62306be3-9b4b-4988-aac9-19d50b64b336 + name: user_guide + resource_id: 7c5e6353-7e1f-4639-91ee-9dc568887a69 + resource_type: file +slug: adhocspecificationintegration_cdc_tracking_network +source: | + The Tracking Program collects, integrates, analyzes, and disseminates non-infectious disease, + environmental, and socio-economic data from various sources. + These include a collective of national, state, and local partners. + + This is an HTTP API, where the usage is free but rate-limiting is applied heavily when not using an API KEY. + You will need an API Key. It is available in the environment variable: + - apiToken: os.environ.get("API_CDC_TRACKING_NETWORK") + + To fully understand specifics about what data is or is not available, + you must run code against the API to see what features are available at + what level of geography. + + ${user_guide} + + Never simulate this API- if it is not available ask the user how to proceed. + + The list of content areas is available at: + ${api_examples} +url: null +uuid: 62306be3-9b4b-4988-aac9-19d50b64b336 diff --git a/src/biome/datasources/cdc_tracking_network/documentation/api_examples.md b/src/biome/adhoc_data/datasources/cdc_tracking_network/attachments/api_examples.md similarity index 100% rename from src/biome/datasources/cdc_tracking_network/documentation/api_examples.md rename to src/biome/adhoc_data/datasources/cdc_tracking_network/attachments/api_examples.md diff --git a/src/biome/datasources/cdc_tracking_network/documentation/user_guide.md b/src/biome/adhoc_data/datasources/cdc_tracking_network/attachments/user_guide.md similarity index 100% rename from src/biome/datasources/cdc_tracking_network/documentation/user_guide.md rename to src/biome/adhoc_data/datasources/cdc_tracking_network/attachments/user_guide.md diff --git a/src/biome/adhoc_data/datasources/census_acs/api.yaml b/src/biome/adhoc_data/datasources/census_acs/api.yaml new file mode 100644 index 0000000..d9bc469 --- /dev/null +++ b/src/biome/adhoc_data/datasources/census_acs/api.yaml @@ -0,0 +1,77 @@ +datatype: api +description: | + The Census ACS (American Community Survey) and SF1 (Decennial Census) API returns data that has been collected from + the Census Bureau, a database that contains information on the population of the United States. + Use this to control for socioeconomic factors (or for other purposes). +img_url: null +last_updated: null +name: Census ACS (American Community Survey) and SF1 +provider: adhoc:specialist_agents +resources: + 76813161-e3dd-400e-b1db-c6534c3fb484: + filepath: census_web.md + integration: d6d4d49c-b157-43cb-995a-d66fa3398a72 + name: web_documentation + resource_id: 76813161-e3dd-400e-b1db-c6534c3fb484 + resource_type: file + 93d4da90-fe43-4121-8e1b-d9d677637143: + code: "import pandas as pd\nfrom census import Census\nfrom us import states\n\ + import matplotlib.pyplot as plt\n\nc = Census(os.environ.get(\"API_CENSUS\"\ + ))\n\n# Get ACS data for Harris County census tracts\nharris_ses = c.acs5.get(\n\ + \ ('NAME', 'B19013_001E', 'B17001_002E', 'B25064_001E', 'B01003_001E'),\n\ + \ {'for': 'tract:*', 'in': f'state:{states.TX.fips} county:201'}\n)\n\n#\ + \ Convert to DataFrame\nharris_ses_df = pd.DataFrame(harris_ses)\nharris_ses_df.columns\ + \ = ['Name', 'Median_Income', 'Poverty_Count', \n 'Median_Rent',\ + \ 'Population', 'state', 'county', 'tract']\n\n# Calculate poverty rate\nharris_ses_df['Poverty_Rate']\ + \ = harris_ses_df['Poverty_Count'] / harris_ses_df['Population'] * 100\n" + integration: d6d4d49c-b157-43cb-995a-d66fa3398a72 + notes: null + query: Calculate the poverty rate in Harris County, Texas- Descriptive analysis + resource_id: 93d4da90-fe43-4121-8e1b-d9d677637143 + resource_type: example + 9a1dd209-4e65-49af-a5cc-2f9d73cd8d4b: + code: "import pandas as pd\nfrom census import Census\nfrom us import states\n\ + import matplotlib.pyplot as plt\n\nc = Census(os.environ.get(\"API_CENSUS\"\ + ))\n\n# Get ACS data for Harris County census tracts\nharris_ses = c.acs5.get(\n\ + \ ('NAME', 'B19013_001E', 'B17001_002E', 'B25064_001E', 'B01003_001E'),\n\ + \ {'for': 'tract:*', 'in': f'state:{states.TX.fips} county:201'}\n)\n\n#\ + \ Convert to DataFrame\nharris_ses_df = pd.DataFrame(harris_ses)\nharris_ses_df.columns\ + \ = ['Name', 'Median_Income', 'Poverty_Count', \n 'Median_Rent',\ + \ 'Population', 'state', 'county', 'tract']\n\n# Group census tracts by income\ + \ quartiles\nharris_ses_df['Income_Quartile'] = pd.qcut(harris_ses_df['Median_Income'],\ + \ 4, labels=False)\n\n# Analyze environmental exposures by income quartile\n\ + exposure_by_ses = pd.merge(environmental_data, harris_ses_df, on='tract')\n\n\ + # Calculate average exposures by income quartile\nquartile_summary = exposure_by_ses.groupby('Income_Quartile')['PM25_Annual',\ + \ 'Ozone_Days_Exceeded', 'TRI_Releases'].mean()" + integration: d6d4d49c-b157-43cb-995a-d66fa3398a72 + notes: null + query: Calculate the poverty rate in Harris County, Texas- Stratified analysis + resource_id: 9a1dd209-4e65-49af-a5cc-2f9d73cd8d4b + resource_type: example + cb8c091d-ff82-46d7-bdea-893aa5b0c6c5: + filepath: census_sdk.md + integration: d6d4d49c-b157-43cb-995a-d66fa3398a72 + name: sdk_documentation + resource_id: cb8c091d-ff82-46d7-bdea-893aa5b0c6c5 + resource_type: file +slug: adhocspecificationintegration_census_acs__american_community_survey__and_sf1 +source: "The American Community Survey (ACS) is an ongoing survey that provides data\ + \ every\nyear\u2014giving communities the current information they need to make\ + \ important decisions.\nThe ACS covers a broad range of topics about social, economic,\ + \ housing, and demographic\ncharacteristics of the U.S. population.\n\n${web_documentation}\n\ + \nYou will need an API Key to use the Census ACS API.\nIt is available in the environment\ + \ variable:\n- api_key: os.environ.get(\"API_CENSUS\")\n\nYou have an api key, so\ + \ you have access to the census pypi package, and also direct access to the census\ + \ web api.\nPrefer the pypi package if you can, it's easier to use.\n\nData ranges\ + \ from 2005 to 2023.\n\n${sdk_documentation}\n\n## For 1-year datasets when loading\ + \ as file (may or may not apply when using python libraries...)\n### Variable Changes\n\ + Variables, and the values they represent, may change over time. Use the file in\ + \ `${DATASET_FILES_BASE_PATH}/census-acs/2023-1yr-api-changes.csv` as a guide for\ + \ which variables have changed from the prior year for 2023 ACS 1-Year Detailed\ + \ Tables, Data Profiles and Subject Tables. See below for a description of each\ + \ change type.\nNo Change - The variable has not changed from the prior year (most\ + \ variables).\nUpdated - That variable has changed from the prior year and a matching\ + \ variable for the current year has been found.\nNo Match - The variable has changed\ + \ from the prior year and no matching or comparable variable has been found.\n" +url: null +uuid: d6d4d49c-b157-43cb-995a-d66fa3398a72 diff --git a/src/biome/datasources/census_acs/documentation/census_sdk.md b/src/biome/adhoc_data/datasources/census_acs/attachments/census_sdk.md similarity index 100% rename from src/biome/datasources/census_acs/documentation/census_sdk.md rename to src/biome/adhoc_data/datasources/census_acs/attachments/census_sdk.md diff --git a/src/biome/datasources/census_acs/documentation/census_web.md b/src/biome/adhoc_data/datasources/census_acs/attachments/census_web.md similarity index 100% rename from src/biome/datasources/census_acs/documentation/census_web.md rename to src/biome/adhoc_data/datasources/census_acs/attachments/census_web.md diff --git a/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml b/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml new file mode 100644 index 0000000..53ce898 --- /dev/null +++ b/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml @@ -0,0 +1,51 @@ +datatype: dataset +description: | + This dataset contains current asthma prevalence, the estimated percentage of + Californians who have ever been diagnosed with asthma by a health care provider + AND report they still have asthma and/or had an asthma episode or attack within + the past 12 months, statewide and by county. The data are stratified by age group + (all ages, 0-17, 18+, 0-4, 5-17, 18-64, 65+) and reported for 2-year periods. +img_url: null +last_updated: null +name: CHIS California Asthma +provider: adhoc:specialist_agents +resources: + ca7ff323-7bef-489d-9d2c-760628f40825: + code: "import pandas as pd\nimport os\n\n# Define the file path using the correct\ + \ filename\nfile_path = os.path.join('${DATASET_FILES_BASE_PATH}', 'chis_california_asthma',\ + \ 'current-asthma-prevalence-by-county-2015_2022.csv')\n\ndf = pd.read_csv(file_path,\ + \ encoding='latin1') # latin1 encoding has worked before, not specifying it\ + \ hasn't\n\n# Check the column names to make sure we're using the right ones\n\ + print(\"Column names:\", df.columns.tolist())\n\n# Display a sample of the data\ + \ to understand its structure\nprint(\"\\nSample data:\")\nprint(df.head(2))\n\ + \ " + integration: 5aefdb54-5aa3-4c53-a7a4-d0483e4577fd + notes: null + query: Open the california chis asthma dataset and display columns/first 2 rows + to understand the structure + resource_id: ca7ff323-7bef-489d-9d2c-760628f40825 + resource_type: example +slug: adhocspecificationintegration_chis_california_asthma +source: | + You have access to a file-based California CHIS Data-Current Asthma Prevalence by County 2015-2022 file, + located at `${DATASET_FILES_BASE_PATH}/chis_california_asthma/current-asthma-prevalence-by-county-2015_2022.csv`. + + open it like so: + ```python + import pandas as pd + df = pd.read_csv('${DATASET_FILES_BASE_PATH}/chis_california_asthma/current-asthma-prevalence-by-county-2015_2022.csv', encoding='latin1') + ``` + + This dataset contains current asthma prevalence, the estimated percentage of + Californians who have ever been diagnosed with asthma by a health care provider + AND report they still have asthma and/or had an asthma episode or attack within + the past 12 months, statewide and by county. The data are stratified by age group + (all ages, 0-17, 18+, 0-4, 5-17, 18-64, 65+) and reported for 2-year periods. + + The columns are: "COUNTY", "YEARS", "STRATA", "AGE GROUP", "CURRENT PREVALENCE", "95% CONFIDENCE INTERVAL", "COUNTIES GROUPED", "COMMENT" + + "Agre Group" possible values are: "All Ages", "0-17 years", "18+ years", "0-4 years", "5-17 years", "18-64 years", "65+ years" + + You can use this API to answer questions on the current asthma prevalence in California by county and age group. +url: null +uuid: 5aefdb54-5aa3-4c53-a7a4-d0483e4577fd diff --git a/src/biome/adhoc_data/datasources/epa_tri/api.yaml b/src/biome/adhoc_data/datasources/epa_tri/api.yaml new file mode 100644 index 0000000..1e76445 --- /dev/null +++ b/src/biome/adhoc_data/datasources/epa_tri/api.yaml @@ -0,0 +1,460 @@ +datatype: dataset +description: | + The Toxics Release Inventory (TRI) is a resource for learning about toxic chemical + releases and pollution prevention activities reported by industrial and federal + facilities. TRI data support informed decision-making by communities, + government agencies, companies, and others. +img_url: null +last_updated: null +name: EPA Toxic Release Inventory (TRI) +provider: adhoc:specialist_agents +resources: + 125d85a4-0a2a-4306-b0d3-cc3c9451bb8e: + code: "import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\n\ + \n# Load and prepare data (assuming tri_data is already loaded and cleaned)\n\ + file_path = '{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv'\n\ + tri_data = pd.read_csv(file_path, low_memory=False)\n\n# Fix the numeric columns\ + \ that might be stored as strings with commas\nfor col in ['Releases (lb)',\ + \ 'Waste Managed (lb)', 'RSEI Hazard']:\n if tri_data[col].dtype == 'object':\n\ + \ # Remove commas and convert to float\n tri_data[col] = tri_data[col].str.replace(',',\ + \ '').astype(float)\n\n# Filter for a specific county\ncounty_data = tri_data[(tri_data['County']\ + \ == 'HARRIS, TX') | \n (tri_data['County'] == 'HARRIS COUNTY,\ + \ TX')]\n\n# Analyze data by ZIP code\nzip_summary = county_data.groupby('ZIP\ + \ Code').agg({\n 'TRI Facility Name': 'nunique',\n 'Releases (lb)': 'sum',\n\ + \ 'RSEI Hazard': 'sum'\n}).reset_index()\n\nzip_summary.columns = ['ZIP Code',\ + \ 'Number of Facilities', 'Total Releases (lb)', 'Total RSEI Hazard']\nzip_summary\ + \ = zip_summary.sort_values('Total Releases (lb)', ascending=False)\n\nprint(\"\ + Top 10 ZIP Codes by Total Releases:\")\nfor i, (zipcode, facilities, releases,\ + \ hazard) in enumerate(zip_summary.head(10).values, 1):\n print(f\"{i}. ZIP\ + \ {zipcode}: {facilities} facilities, {releases:,.2f} lb, RSEI Hazard: {hazard:,.2f}\"\ + )\n\n# Create a bar chart of top 10 ZIP codes by releases\nplt.figure(figsize=(12,\ + \ 6))\ntop_zips = zip_summary.head(10)\nbars = plt.bar(top_zips['ZIP Code'].astype(str),\ + \ top_zips['Total Releases (lb)'] / 1e6)\nplt.title('Top 10 ZIP Codes by Total\ + \ Toxic Releases')\nplt.xlabel('ZIP Code')\nplt.ylabel('Total Releases (Million\ + \ lb)')\nplt.xticks(rotation=45)\nplt.grid(True, axis='y', linestyle='--', alpha=0.7)\n\ + \n# Add value labels on top of bars\nfor bar in bars:\n height = bar.get_height()\n\ + \ plt.text(bar.get_x() + bar.get_width()/2., height + 0.5,\n f'{height:.1f}M',\ + \ ha='center', va='bottom')\n\nplt.tight_layout()\nplt.savefig('top_zips_by_releases.png')\n\ + plt.show()\n" + integration: b57b39a7-882d-4f69-93c9-c3e3dee96063 + notes: null + query: |- + Analyzing and visualizing toxic releases by ZIP code. This example shows how to aggregate TRI data by geographic units (ZIP codes), calculate total releases and hazard scores, and create a bar chart visualization of the top areas with highest toxic releases. + resource_id: 125d85a4-0a2a-4306-b0d3-cc3c9451bb8e + resource_type: example + 25907f4c-47dc-4c98-8ca4-584e6cd8556c: + code: | + import pandas as pd + + # Always the TRI csv with index_col=False param, since pandas is inferring the wrong index column and shifting the data without it + df = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv', dtype=str, index_col=False) + + # Filter for any entries containing 'Orange' in the County column + # sample name without the county suffix, all uppercase and with correct spacing + # In general, check contains/case=False instead of exact matches + orange_county = df[df['County'].str.contains('ORANGE, FL', case=False, na=False)] + + # Display results + print(f"Found {len(orange_county)} toxic release records containing 'ORANGE, FL' in the County name.") + print("\nSample of the data:") + print(orange_county.head()) + integration: b57b39a7-882d-4f69-93c9-c3e3dee96063 + notes: null + query: Open the TRI data file with index_col=False, since otherwise columns are + shifted, and find data for Orange County, FL. + resource_id: 25907f4c-47dc-4c98-8ca4-584e6cd8556c + resource_type: example + 42c5c855-df3e-4b20-9144-5083a691ce5f: + code: | + import pandas as pd + + # Load the TRI data + tri_data = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv', index_col=False) + + # Define respiratory/asthma related chemicals + # Updated to match the actual format in the dataset (with CAS numbers) + respiratory_chemicals = [ + 'Ammonia (7664-41-7)', + 'Chlorine (7782-50-5)', + 'Sulfuric acid (7664-93-9)', + 'Hydrochloric acid (7647-01-0)', + 'Nitric acid (7697-37-2)', + 'Ozone', # Keep checking if this has a CAS number format + 'Toluene (108-88-3)', + 'Benzene (71-43-2)', + 'Formaldehyde (50-00-0)', + 'Methanol (67-56-1)' + ] + + # Filter data for: + # - Year 2022 + # - Harris County (case-insensitive match) + # - Respiratory chemicals + respiratory_incidents = tri_data[ + (tri_data['Year'] == 2022) & + (tri_data['County'].str.upper() == 'HARRIS, TX') & + (tri_data['Chemical'].isin(respiratory_chemicals)) + ] + + # Convert 'Releases (lb)' to numeric, coercing errors to NaN since ascending= on sort_values messes with it + respiratory_incidents['Releases (lb)'] = pd.to_numeric(respiratory_incidents['Releases (lb)'], errors='coerce') + + # Sort by release amount + respiratory_incidents = respiratory_incidents.sort_values('Releases (lb)', ascending=False) + + # Select relevant columns + result = respiratory_incidents[[ + 'TRI Facility Name', + 'Chemical', + 'Releases (lb)', + 'RSEI Hazard', + 'Latitude', + 'Longitude' + ]] + + result + integration: b57b39a7-882d-4f69-93c9-c3e3dee96063 + notes: null + query: |- + Capture any asthma-respiratory related hazard incident from the TRI- EPA's Toxic Release Inventory data- you have access to this api. FOr now focus on problems in 2022, in the county of Harris, TX. + resource_id: 42c5c855-df3e-4b20-9144-5083a691ce5f + resource_type: example + 494e3a73-da61-4132-a169-56880ce5ea60: + code: "import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\n\ + \n# Load and prepare data (assuming tri_data is already loaded and cleaned)\n\ + file_path = '{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv'\n\ + tri_data = pd.read_csv(file_path, low_memory=False)\n\n# Fix the numeric columns\ + \ that might be stored as strings with commas\nfor col in ['Releases (lb)',\ + \ 'Waste Managed (lb)', 'RSEI Hazard']:\n if tri_data[col].dtype == 'object':\n\ + \ # Remove commas and convert to float\n tri_data[col] = tri_data[col].str.replace(',',\ + \ '').astype(float)\n\n# Filter for a specific county\ncounty_data = tri_data[(tri_data['County']\ + \ == 'HARRIS, TX') | \n (tri_data['County'] == 'HARRIS COUNTY,\ + \ TX')]\n\n# List of chemicals known to potentially trigger or worsen a specific\ + \ health condition (e.g., asthma)\ncondition_related_chemicals = [\n 'Ammonia',\ + \ 'Chlorine', 'Formaldehyde', 'Sulfuric acid', 'Hydrochloric acid',\n 'Nitrogen\ + \ oxides', 'Sulfur dioxide', 'Particulate matter', 'Isocyanates',\n 'Toluene\ + \ diisocyanate', 'Phthalates', 'Volatile organic compounds', 'VOCs',\n 'Ozone',\ + \ 'Acrolein', 'Benzene', 'Styrene', 'Xylene', 'Toluene'\n]\n\n# Create a function\ + \ to check if a chemical is related to the health condition\ndef is_condition_related(chemical_name):\n\ + \ for chem in condition_related_chemicals:\n if chem.lower() in chemical_name.lower():\n\ + \ return True\n return False\n\n# Add a column to identify condition-related\ + \ chemicals\ncounty_data['Condition_Related'] = county_data['Chemical'].apply(is_condition_related)\n\ + \n# Calculate total condition-related releases\ncondition_releases = county_data[county_data['Condition_Related']]['Releases\ + \ (lb)'].sum()\ntotal_releases = county_data['Releases (lb)'].sum()\ncondition_percentage\ + \ = (condition_releases / total_releases) * 100\n\nprint(f\"Total toxic releases:\ + \ {total_releases:,.2f} lb\")\nprint(f\"Condition-related chemical releases:\ + \ {condition_releases:,.2f} lb\")\nprint(f\"Percentage of condition-related\ + \ releases: {condition_percentage:.2f}%\")\n\n# Analyze trends over time for\ + \ condition-related chemicals\nyearly_condition = county_data[county_data['Condition_Related']].groupby('Year')['Releases\ + \ (lb)'].sum()\nyearly_total = county_data.groupby('Year')['Releases (lb)'].sum()\n\ + yearly_percentage = (yearly_condition / yearly_total) * 100\n\n# Create a time\ + \ series plot\nplt.figure(figsize=(12, 6))\nplt.plot(yearly_condition.index,\ + \ yearly_condition.values / 1e6, 'o-', color='red', \n label='Condition-related\ + \ Releases')\nplt.plot(yearly_total.index, yearly_total.values / 1e6, 'o-',\ + \ color='blue', \n label='Total Releases')\nplt.title('Yearly Trends\ + \ in Toxic Releases')\nplt.xlabel('Year')\nplt.ylabel('Releases (Million lb)')\n\ + plt.legend()\nplt.grid(True, linestyle='--', alpha=0.7)\nplt.tight_layout()\n\ + plt.savefig('yearly_condition_trends.png')\nplt.show()\n" + integration: b57b39a7-882d-4f69-93c9-c3e3dee96063 + notes: null + query: |- + Identifying and analyzing chemicals related to specific health conditions. This example demonstrates how to filter TRI data for chemicals associated with a particular health condition (like asthma), calculate their proportion of total releases, and analyze trends over time. This approach can be adapted for various health conditions by modifying the list of relevant chemicals. + resource_id: 494e3a73-da61-4132-a169-56880ce5ea60 + resource_type: example + 7dcb643d-1565-415e-99c7-04c0c48e6366: + code: "import pandas as pd\nimport folium\nfrom folium.plugins import MarkerCluster\n\ + import numpy as np\n\n# Load and prepare data (assuming tri_data is already\ + \ loaded and cleaned)\nfile_path = '{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv'\n\ + tri_data = pd.read_csv(file_path, low_memory=False)\n\n# Fix the numeric columns\ + \ that might be stored as strings with commas\nfor col in ['Releases (lb)',\ + \ 'Waste Managed (lb)', 'RSEI Hazard']:\n if tri_data[col].dtype == 'object':\n\ + \ # Remove commas and convert to float\n tri_data[col] = tri_data[col].str.replace(',',\ + \ '').astype(float)\n\n# Filter for a specific county\ncounty_data = tri_data[(tri_data['County']\ + \ == 'HARRIS, TX') | \n (tri_data['County'] == 'HARRIS COUNTY,\ + \ TX')]\n\n# Create a summary of facilities\nfacility_summary = county_data.groupby(['TRI\ + \ Facility Name', 'Latitude', 'Longitude']).agg({\n 'Releases (lb)': 'sum',\n\ + \ 'RSEI Hazard': 'sum',\n 'Chemical': 'nunique',\n 'Year': 'nunique'\n\ + }).reset_index()\n\nfacility_summary.columns = ['Facility Name', 'Latitude',\ + \ 'Longitude', 'Total Releases (lb)', \n 'Total RSEI\ + \ Hazard', 'Number of Chemicals', 'Years Reported']\n\n# Sort by total releases\n\ + facility_summary = facility_summary.sort_values('Total Releases (lb)', ascending=False)\n\ + \n# Create an interactive map\n# Center the map on the county\ncounty_center\ + \ = [29.7604, -95.3698] # Houston coordinates as center\nm = folium.Map(location=county_center,\ + \ zoom_start=10, tiles='CartoDB positron')\n\n# Add a marker cluster for better\ + \ visualization\nmarker_cluster = MarkerCluster().add_to(m)\n\n# Add markers\ + \ for each facility\nfor i, row in facility_summary.iterrows():\n # Scale\ + \ the circle size based on the log of releases (to make it more visible)\n \ + \ radius = np.log1p(row['Total Releases (lb)']) * 0.5\n \n # Create\ + \ popup content\n popup_content = f\"\"\"\n {row['Facility Name']}
\n\ + \ Total Releases: {row['Total Releases (lb)']:,.2f} lb
\n RSEI Hazard:\ + \ {row['Total RSEI Hazard']:,.2f}
\n Number of Chemicals: {row['Number\ + \ of Chemicals']}
\n Years Reported: {row['Years Reported']}\n \"\"\ + \"\n \n # Add a circle marker\n folium.CircleMarker(\n location=[row['Latitude'],\ + \ row['Longitude']],\n radius=radius,\n popup=folium.Popup(popup_content,\ + \ max_width=300),\n color='red',\n fill=True,\n fill_color='red',\n\ + \ fill_opacity=0.6,\n opacity=0.8\n ).add_to(marker_cluster)\n\ + \n# Save the map\nmap_file = 'facilities_map.html'\nm.save(map_file)\nprint(f\"\ + Interactive map saved to: {map_file}\")\n" + integration: b57b39a7-882d-4f69-93c9-c3e3dee96063 + notes: null + query: |- + Creating an interactive map of toxic release facilities. This example shows how to use the folium library to generate an interactive web map displaying the locations of facilities, with circle sizes proportional to release amounts and popup information showing detailed facility data. This visualization helps identify spatial patterns and hotspots of toxic releases. + resource_id: 7dcb643d-1565-415e-99c7-04c0c48e6366 + resource_type: example + 848b37b3-e119-42e7-a85d-b8a11e58ae19: + code: | + import pandas as pd + + # Load the TRI data + tri_data = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv', index_col=False) + + # List of respiratory-related chemicals + respiratory_chemicals = [ + 'Ammonia', + 'Chlorine', + 'Sulfuric acid', + 'Hydrochloric acid', + 'Hydrogen fluoride', + 'Nitric acid', + 'Ozone', + 'Phosgene', + 'Toluene diisocyanate', + 'Methylene diphenyl diisocyanate' + ] + + # Filter the data + respiratory_incidents = tri_data[ + (tri_data['Year'] == 2022) & + (tri_data['County'] == 'HARRIS, TX') & + (tri_data['Chemical'].str.contains('|'.join(respiratory_chemicals), case=False, na=False)) + ] + + # Sort by release amount + respiratory_incidents = respiratory_incidents.sort_values('Releases (lb)', ascending=False) + + # Select relevant columns + result = respiratory_incidents[[ + 'TRI Facility Name', + 'Chemical', + 'Releases (lb)', + 'RSEI Hazard' + ]] + + print(result) + integration: b57b39a7-882d-4f69-93c9-c3e3dee96063 + notes: null + query: |- + Query to retrieve asthma-respiratory related hazard incidents from the EPA's TRI for the year 2022 in Harris County, TX. + resource_id: 848b37b3-e119-42e7-a85d-b8a11e58ae19 + resource_type: example + 8de5ce51-3ff8-4ab3-a347-311b499acf7a: + code: "import pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as\ + \ sns\nimport numpy as np\nfrom scipy.stats import pearsonr\n\n# Load and prepare\ + \ data (assuming tri_data is already loaded and cleaned)\nfile_path = '{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv'\n\ + tri_data = pd.read_csv(file_path, low_memory=False)\n\n# Fix the numeric columns\ + \ that might be stored as strings with commas\nfor col in ['Releases (lb)',\ + \ 'Waste Managed (lb)', 'RSEI Hazard']:\n if tri_data[col].dtype == 'object':\n\ + \ # Remove commas and convert to float\n tri_data[col] = tri_data[col].str.replace(',',\ + \ '').astype(float)\n\n# Filter for a specific county\ncounty_data = tri_data[(tri_data['County']\ + \ == 'HARRIS, TX') | \n (tri_data['County'] == 'HARRIS COUNTY,\ + \ TX')]\n\n# Analyze data by Census Block Group\ncensus_summary = county_data.groupby('Census\ + \ Block Group').agg({\n 'TRI Facility Name': 'nunique',\n 'Releases (lb)':\ + \ 'sum',\n 'RSEI Hazard': 'sum',\n 'Chemical': 'nunique'\n}).reset_index()\n\ + \ncensus_summary.columns = ['Census Block Group', 'Number of Facilities', \n\ + \ 'Total Releases (lb)', 'Total RSEI Hazard', 'Number\ + \ of Chemicals']\n\n# Create a scatter plot of total releases vs RSEI Hazard\ + \ by Census Block Group\nplt.figure(figsize=(10, 8))\nplt.scatter(\n np.log10(census_summary['Total\ + \ Releases (lb)']), \n np.log10(census_summary['Total RSEI Hazard'] + 1),\ + \ # Add 1 to avoid log(0)\n alpha=0.7,\n s=50,\n c='blue'\n)\nplt.title('Relationship\ + \ Between Total Releases and RSEI Hazard Score (Log Scale)')\nplt.xlabel('Log10(Total\ + \ Releases in lb)')\nplt.ylabel('Log10(RSEI Hazard Score)')\nplt.grid(True,\ + \ linestyle='--', alpha=0.7)\n\n# Add a trend line\nz = np.polyfit(np.log10(census_summary['Total\ + \ Releases (lb)']), \n np.log10(census_summary['Total RSEI Hazard']\ + \ + 1), 1)\np = np.poly1d(z)\nplt.plot(np.log10(census_summary['Total Releases\ + \ (lb)']), \n p(np.log10(census_summary['Total Releases (lb)'])), \n\ + \ \"r--\", linewidth=2)\n\n# Add correlation coefficient\ncorr, _ = pearsonr(np.log10(census_summary['Total\ + \ Releases (lb)']), \n np.log10(census_summary['Total RSEI Hazard']\ + \ + 1))\nplt.annotate(f\"Correlation: {corr:.2f}\", xy=(0.05, 0.95), xycoords='axes\ + \ fraction', \n fontsize=12, bbox=dict(boxstyle=\"round,pad=0.3\"\ + , fc=\"white\", ec=\"gray\", alpha=0.8))\n\nplt.tight_layout()\nplt.savefig('releases_vs_hazard.png')\n\ + plt.show()" + integration: b57b39a7-882d-4f69-93c9-c3e3dee96063 + notes: null + query: |- + Analyzing the relationship between release amounts and hazard scores. This example demonstrates how to investigate the correlation between the quantity of toxic releases and their associated RSEI Hazard scores using scatter plots, trend lines, and correlation statistics. This analysis helps understand whether larger releases necessarily correspond to higher health risks. + resource_id: 8de5ce51-3ff8-4ab3-a347-311b499acf7a + resource_type: example + 9ec78574-65e7-4a7c-bb6b-7752a2cdf5a7: + code: "import pandas as pd\n\n# Load the data\nfile_path = '{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv'\n\ + tri_data = pd.read_csv(file_path, low_memory=False)\n\n# Fix the numeric columns\ + \ that might be stored as strings with commas\nfor col in ['Releases (lb)',\ + \ 'Waste Managed (lb)', 'RSEI Hazard']:\n if tri_data[col].dtype == 'object':\n\ + \ # Remove commas and convert to float\n tri_data[col] = tri_data[col].str.replace(',',\ + \ '').astype(float)\n\n# Filter for a specific county\nharris_data = tri_data[(tri_data['County']\ + \ == 'HARRIS, TX') | \n (tri_data['County'] == 'HARRIS COUNTY,\ + \ TX')]\n\n# Create a summary of facilities\nfacility_summary = harris_data.groupby(['TRI\ + \ Facility Name', 'Latitude', 'Longitude']).agg({\n 'Releases (lb)': 'sum',\n\ + \ 'RSEI Hazard': 'sum',\n 'Chemical': 'nunique',\n 'Year': 'nunique'\n\ + }).reset_index()\n\nfacility_summary.columns = ['Facility Name', 'Latitude',\ + \ 'Longitude', 'Total Releases (lb)', \n 'Total RSEI\ + \ Hazard', 'Number of Chemicals', 'Years Reported']\n\n# Sort by total releases\n\ + facility_summary = facility_summary.sort_values('Total Releases (lb)', ascending=False)\n\ + \nprint(\"Summary of Facilities:\")\nprint(f\"Total number of facilities: {len(facility_summary)}\"\ + )\nprint(f\"Total toxic releases: {facility_summary['Total Releases (lb)'].sum():,.2f}\ + \ lb\")\nprint(f\"Total RSEI Hazard score: {facility_summary['Total RSEI Hazard'].sum():,.2f}\"\ + )\n\n# Print top 10 facilities by releases\nprint(\"\\nTop 10 Facilities by\ + \ Total Releases:\")\nfor i, (name, lat, lon, releases, hazard, chemicals, years)\ + \ in enumerate(facility_summary.head(10).values, 1):\n print(f\"{i}. {name}:\ + \ {releases:,.2f} lb, RSEI Hazard: {hazard:,.2f}, Chemicals: {chemicals}\")\n" + integration: b57b39a7-882d-4f69-93c9-c3e3dee96063 + notes: null + query: |- + Basic loading, cleaning, and summarizing of EPA TRI data for a specific county. This example demonstrates how to handle the comma-separated numeric values in the dataset, filter for a specific geographic area, and create a summary of facilities with their total releases, hazard scores, and chemical counts. + resource_id: 9ec78574-65e7-4a7c-bb6b-7752a2cdf5a7 + resource_type: example + e2bd90f9-24a6-41e5-a3ba-1aca807d8990: + code: | + import pandas as pd + + # Load the data + tri_data = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv', index_col=False) + + print("Sample of County values:") + print(tri_data['County'].unique()[:10]) # Print first 10 unique county values + + print("\nSample of Chemical values containing 'Ammonia':") + ammonia_chemicals = [chem for chem in tri_data['Chemical'].unique() if 'Ammonia' in str(chem)] + print(ammonia_chemicals) + + # Try a more flexible approach + print("\nTrying a more flexible filter:") + filtered_data = tri_data[ + tri_data['Chemical'].str.contains('Ammonia', case=False, na=False) + ] + + print(f"Found {len(filtered_data)} records for any Ammonia") + + # Now try to find Harris county with a flexible approach + print("\nLooking for Harris county:") + harris_data = tri_data[ + tri_data['County'].str.contains('Harris', case=False, na=False) + ] + + print(f"Found {len(harris_data)} records for Harris county") + print("Sample county values in Harris results:") + print(harris_data['County'].unique()) + + # Now try the combined filter with the correct formats + print("\nTrying combined filter with formats from the dataset:") + final_filtered = tri_data[ + tri_data['County'].str.contains('Harris', case=False, na=False) & + tri_data['Chemical'].str.contains('Ammonia', case=False, na=False) + ] + + print(f"Found {len(final_filtered)} records for Ammonia in Harris county") + if len(final_filtered) > 0: + # Select relevant columns and sort by release amount + result = final_filtered[['TRI Facility Name', 'Chemical', 'Releases (lb)', 'RSEI Hazard', 'Latitude', 'Longitude']] + result['Releases (lb)'] = pd.to_numeric(result['Releases (lb)'], errors='coerce') + result = result.sort_values('Releases (lb)', ascending=False) + print("\nResults:") + print(result.head()) + integration: b57b39a7-882d-4f69-93c9-c3e3dee96063 + notes: null + query: |- + When not finding results for a combined query, try a more systematic approach by finding items and counting length by one by one, in order to discover if there is matching data, but the query is not returning any results. This is to discover amonia references for the county of 'HARRIS, TX'. + resource_id: e2bd90f9-24a6-41e5-a3ba-1aca807d8990 + resource_type: example +slug: adhocspecificationintegration_epa_toxic_release_inventory__tri_ +source: | + You have access to a file-based EPA Toxic Release Inventory (TRI) data, from 2014 to 2023, at the `${DATASET_FILES_BASE_PATH}/epa-tri/EPA_TRI_Toxics_2014_2023.csv` file. + The report is just one file- there's a year column, and then a list of chemicals released, and the amount released. + It also contains geospatial information such as latitude, longitude, state/zip, and even EPA region. + + # Overview of the EPA TRI Dataset + TRI tracks the waste management of certain toxic chemicals that may pose a threat to human health and the environment. U.S. facilities in different industry sectors must report annually how much of each chemical they release into the environment and/or managed through recycling, energy recovery and treatment, as well as any practices implemented to prevent or reduce the generation of chemical waste. + + Contains information about toxic chemical releases reported by industrial and federal facilities; key features of this dataset include: + - Facility information: Names, IDs, geographic coordinates, and locations + - Chemical data: Chemical names and release amounts (in pounds) + - Geographic information: Latitude, longitude, state, county, EPA region, ZIP codes, and Census Block Groups + - Risk metrics: RSEI (Risk-Screening Environmental Indicators) Hazard scores + - Temporal coverage: 10 years of data (2014-2023) + + In general, chemicals covered by the TRI Program are those that cause: + - Cancer or other chronic human health effects + - Significant adverse acute human health effects + - Significant adverse environmental effects + + There are currently 799 individually listed chemicals and 33 chemical categories covered by the TRI Program. Facilities that manufacture, process or otherwise use these chemicals in amounts above established levels must submit annual reporting forms for each chemical. Note that the TRI chemical list doesn't include all toxic chemicals used in the U.S. + + The Toxics Release Inventory (TRI) is a resource for learning about toxic chemical releases and pollution prevention activities reported by industrial and federal facilities. TRI data support informed decision-making by communities, government agencies, companies, and others. + + ## Data Dictionary: + - "TRI Facility Name": Name of the facility reporting chemical releases + - "TRI Facility ID": Unique identifier for the facility + - "Year": Reporting year (2014-2023) + - "Census Block Group": Geographic identifier for census block group + - "ZIP Code": Postal code of the facility + - "City": City where the facility is located + - "County": County where the facility is located- format: "COUNTY-NAME-ONLY, STATE 2-DIGIT CODE" (value in all caps) + - "State": State where the facility is located + - "EPA Region": EPA administrative region (numbered 1-10) + - "Latitude"/"Longitude": Geographic coordinates of the facility + - "Chemical": Name of the toxic chemical being reported + - "Releases (lb)": Total amount of chemical released to the environment in pounds + - "Waste Managed (lb)": Total amount of chemical waste managed in pounds + - "RSEI Hazard": Risk-Screening Environmental Indicators score, which measures the relative hazard of chemicals + + ## Working with the Data: + 1. When loading the data, ensure you specify the index_col=False parameter, since pandas is inferring the wrong index column and shifting the data without it: + ```python + import pandas as pd + tri_data = pd.read_csv('${DATASET_FILES_BASE_PATH}/epa-tri/EPA_TRI_Toxics_2014_2023.csv', index_col=False) + ``` + + 2. Common data analysis tasks: + - Filter by year: `tri_data[tri_data['Year'] == 2023]` + - Filter by chemical: `tri_data[tri_data['Chemical'] == 'Lead']` + - Filter by state: `tri_data[tri_data['State'] == 'California']` + - Find top polluters: `tri_data.sort_values('Releases (lb)', ascending=False).head(10)` + - Analyze trends over time: `tri_data.groupby('Year')['Releases (lb)'].sum().plot()` + + 3. Geospatial analysis: + - Create a map of facilities: Use latitude/longitude with libraries like folium or geopandas + - Regional analysis: Group by EPA Region or State to compare pollution levels + + 4. Data quality considerations: + - Check for missing values: `tri_data.isnull().sum()` + - Some chemicals may have different reporting thresholds + - RSEI Hazard scores provide context on relative toxicity beyond just release amounts + + Since this dataset contains a column for EPA Region, we can use this when relating to other more anonymous data sources + that only use these properties- but do store latitude/longitude to correlate with the other data sources that have a higher resolution. + + When checking for respiratory chemicals, always include the CAS number, not just the name- either that or check that the + non-CAS-included name you use is a substring of the value in the Chemical column when trying to make a match. + Example Chemical column values: Manganese compounds (N450), Ammonia (7664-41-7), Nitrate compounds (water dissociable) (N511) + + # Web Documentation + + ## What industries are included in the TRI? + Facilities that report to TRI are typically larger facilities involved in manufacturing, metal mining, electric power generation, chemical manufacturing and hazardous waste treatment. Not all industry sectors are covered by the TRI Program, and not all facilities in covered sectors are required to report to TRI. + + The TRI Program is also different because the data it collects are: + + - annual, collected each July and made publicly available online; + - multimedia, reflecting chemical emissions to air, water and land; and + - broad, encompassing source reduction and other pollution prevention practices. + + Numerous EPA programs use TRI data and information to: + + - provide a more complete picture of environmental performance at facilities and corporations; + - identify facilities that are potentially out of compliance with regulations or operating permits; + - support technical analysis for regulation; + - improve data quality across EPA; and + - develop program priorities and projects. + + Under various environmental laws (identified in the figure below), other EPA programs also collect information about regulated chemicals. Users looking for information not available in the TRI can check the databases associated with these other programs. The Clean Air Act's National Emissions Inventory (NEI), for example, can be used to find estimates of air releases for facilities that do not report to the TRI or for mobile sources such as cars, which are not covered by the TRI. +url: null +uuid: b57b39a7-882d-4f69-93c9-c3e3dee96063 diff --git a/src/biome/adhoc_data/datasources/faers/api.yaml b/src/biome/adhoc_data/datasources/faers/api.yaml new file mode 100644 index 0000000..a62bb39 --- /dev/null +++ b/src/biome/adhoc_data/datasources/faers/api.yaml @@ -0,0 +1,76 @@ +datatype: api +description: | + Drug Adverse Event Overview + The openFDA drug adverse event API returns data that has been collected from + the FDA Adverse Event Reporting System (FAERS), a database that contains + information on adverse event and medication error reports submitted to FDA. +img_url: null +last_updated: null +name: FDA drug adverse event FAERS API +provider: adhoc:specialist_agents +resources: + 66215565-2562-455a-aedc-80cdb06f6c1a: + filepath: faers_fields_reference.csv + integration: baf45232-38d9-401b-b1e5-8c5621917442 + name: fields_reference + resource_id: 66215565-2562-455a-aedc-80cdb06f6c1a + resource_type: file + a02272d5-d7da-4070-942e-750f5f5f9088: + code: |- + import requests + import os + import pandas as pd + + # Base URL for the API + base_url = "https://api.fda.gov/drug/event.json?api_key=" + os.environ.get('API_OPENFDA') + + # Query parameters + params = { + 'count': 'patient.drug.medicinalproduct.exact', + 'limit': 100 # Get top 100 medications by report count + } + + # Make API request + response = requests.get(base_url, params=params) + + # Convert to DataFrame + results = response.json()['results'] + df = pd.DataFrame(results, columns=['term', 'count']) + + # Sort by count in descending order + df = df.sort_values('count', ascending=False) + + # Display top medications with highest adverse event counts + print("Top medications by adverse event count:") + print(df.head(10)) + + # Get medication with highest count + top_med = df.iloc[0] + print(f"\nMedication with highest number of adverse events:") + print(f"{top_med['term']}: {top_med['count']} reports") + integration: baf45232-38d9-401b-b1e5-8c5621917442 + notes: null + query: Can you tell me which medication have the highest adverse effects reported + from all of the available? + resource_id: a02272d5-d7da-4070-942e-750f5f5f9088 + resource_type: example + f9afc1aa-371f-4318-9fb1-cd5030db39fe: + filepath: faers_web.md + integration: baf45232-38d9-401b-b1e5-8c5621917442 + name: web_documentation + resource_id: f9afc1aa-371f-4318-9fb1-cd5030db39fe + resource_type: file +slug: adhocspecificationintegration_fda_drug_adverse_event_faers_api +source: | + ${web_documentation} + + # Additional Instructions: + + You will need an API Key to use any openFDA APIs. It is available in the environment variable: + - API Key: os.environ.get("API_OPENFDA") + Do not use placeholder auth values from the OpenAPI spec nor prompt the user for these values, just use the environment variable. + + # Searchable Fields Reference: + ${fields_reference} +url: null +uuid: baf45232-38d9-401b-b1e5-8c5621917442 diff --git a/src/biome/datasources/faers/documentation/faers_fields_reference.csv b/src/biome/adhoc_data/datasources/faers/attachments/faers_fields_reference.csv similarity index 99% rename from src/biome/datasources/faers/documentation/faers_fields_reference.csv rename to src/biome/adhoc_data/datasources/faers/attachments/faers_fields_reference.csv index a1bfada..e98fa7b 100644 --- a/src/biome/datasources/faers/documentation/faers_fields_reference.csv +++ b/src/biome/adhoc_data/datasources/faers/attachments/faers_fields_reference.csv @@ -1,70 +1,70 @@ -Section,Field Name,Type,Description, -,authoritynumb,string,"Populated with the Regulatory Authority’s case report number, when available.", -,companynumb,string,Identifier for the company providing the report. This is self-assigned., -,duplicate,string,This value is `1` if earlier versions of this report were submitted to FDA. openFDA only shows the most recent version., -,fulfillexpeditecriteria,string,Identifies expedited reports (those that were processed within 15 days)., -,occurcountry,string,The name of the country where the event occurred., -patient.drug,actiondrug,string,Actions taken with the drug., -patient.drug,abuse,string,"Information about the types of abuse that can occur with the drug and adverse reactions pertinent to those types of abuse, primarily based on human data. May include descriptions of particularly susceptible patient populations.", -patient.drug.activesubstance,activesubstancename,string,"Product active ingredient, which may be different than other drug identifiers (when provided).", -patient.drug,drugadditional,string,Dechallenge outcome information—whether the event abated after product use stopped or the dose was reduced. Only present when this was attempted and the data was provided., -patient.drug,drugadministrationroute,string,The drug’s route of administration., -patient.drug,drugauthorizationnumb,string,"Drug authorization or application number (NDA or ANDA), if provided.", -patient.drug,drugbatchnumb,string,"Drug product lot number, if provided.", -patient.drug,drugcharacterization,string,Reported role of the drug in the adverse event report. These values are not validated by FDA., -patient.drug,drugcumulativedosagenumb,string,"The cumulative dose taken until the first reaction was experienced, if provided.", -patient.drug,drugcumulativedosageunit,string,The unit for `drugcumulativedosagenumb`., -patient.drug,drugdosagetext,string,"Additional detail about the dosage taken. Frequently unknown, but occasionally including information like a brief textual description of the schedule of administration.", -patient.drug,drugenddate,string,Date the patient stopped taking the drug., -patient.drug,drugenddateformat,string,Encoding format of the field `drugenddateformat`. Always set to `102` (YYYYMMDD)., -patient.drug,drugindication,string,Indication for the drug’s use., -patient.drug,drugintervaldosagedefinition,string,The unit for the interval in the field `drugintervaldosageunitnumb.`, -patient.drug,drugintervaldosageunitnumb,string,Information about the drug product’s indications for use., -patient.drug,drugrecurreadministration,string,Whether the reaction occured after readministration of the drug., -patient.drug.drugrecurrence,drugrecuractionmeddraversion,string,The version of MedDRA from which the term in `drugrecuraction` is drawn., -patient.drug.drugrecurrence,drugrecuraction,string,Populated with the Reaction/Event information if/when `drugrecurreadministration` equals `1`., -patient.drug,drugseparatedosagenumb,string,The number of separate doses that were administered., -patient.drug,drugstartdate,string,Date the patient began taking the drug., -patient.drug,drugstartdateformat,string,Encoding format of the field `drugstartdate`. Always set to `102` (YYYYMMDD)., -patient.drug,drugstructuredosagenumb,string,"The number portion of a dosage; when combined with `drugstructuredosageunit` the complete dosage information is represented. For example, *300* in `300 mg`.", -patient.drug,drugstructuredosageunit,string,"The unit for the field `drugstructuredosagenumb`. For example, *mg* in `300 mg`.", -patient.drug,drugtreatmentduration,string,The interval of the field `drugtreatmentdurationunit` for which the patient was taking the drug., -patient.drug,drugtreatmentdurationunit,string,Information about when a doctor or pharmacist should be consulted about drug/drug or drug/food interactions before using a drug product., -patient.drug,medicinalproduct,string,"Drug name. This may be the valid trade name of the product (such as `ADVIL` or `ALEVE`) or the generic name (such as `IBUPROFEN`). This field is not systematically normalized. It may contain misspellings or idiosyncratic descriptions of drugs, such as combination products such as those used for birth control.", -patient,patientagegroup,string,Populated with Patient Age Group code., -patient.patientdeath,patientdeathdate,string,"If the patient died, the date that the patient died.", -patient.patientdeath,patientdeathdateformat,string,Encoding format of the field `patientdeathdate`. Always set to `102` (YYYYMMDD)., -patient,patientonsetage,string,Age of the patient when the event first occured., -patient,patientonsetageunit,string,The unit for the interval in the field `patientonsetage.`, -patient,patientsex,string,The sex of the patient., -patient,patientweight,string,"The patient weight, in kg (kilograms).", -patient.reaction,reactionmeddrapt,string,"Patient reaction, as a MedDRA term. Note that these terms are encoded in British English. For instance, diarrhea is spelled `diarrohea`. MedDRA is a standardized medical terminology.", -patient.reaction,reactionmeddraversionpt,string,The version of MedDRA from which the term in `reactionmeddrapt` is drawn., -patient.reaction,reactionoutcome,string,Outcome of the reaction in `reactionmeddrapt` at the time of last observation., -patient.summary,narrativeincludeclinical,string,"Populated with Case Event Date, when available; does `NOT` include Case Narrative.", -primarysource,literaturereference,string,"Populated with the Literature Reference information, when available.""", -primarysource,qualification,string,Category of individual who submitted the report., -primarysource,reportercountry,string,Country from which the report was submitted., -,primarysourcecountry,string,Country of the reporter of the event., -,receiptdate,string,Date that the _most recent_ information in the report was received by FDA., -,receiptdateformat,string,Encoding format of the `receiptdate` field. Always set to 102 (YYYYMMDD)., -,receivedate,string,"Date that the report was _first_ received by FDA. If this report has multiple versions, this will be the date the first version was received by FDA.", -,receivedateformat,string,Encoding format of the `receivedate` field. Always set to 102 (YYYYMMDD)., -receiver,receiverorganization,string,"Name of the organization receiving the report. Because FDA received the report, the value is always `FDA`.", -receiver,receivertype,string,"The type of organization receiving the report. The value,`6`, is only specified if it is `other`, otherwise it is left blank.", -reportduplicate,duplicatenumb,string,The case identifier for the duplicate., -reportduplicate,duplicatesource,string,The name of the organization providing the duplicate., -,reporttype,string,Code indicating the circumstances under which the report was generated., -,safetyreportid,string,"The 8-digit Safety Report ID number, also known as the case report number or case ID. The first 7 digits (before the hyphen) identify an individual report and the last digit (after the hyphen) is a checksum. This field can be used to identify or find a specific adverse event report.", -,safetyreportversion,string,"The version number of the `safetyreportid`. Multiple versions of the same report may exist, it is generally best to only count the latest report and disregard others. openFDA will only return the latest version of a report.", -sender,senderorganization,string,"Name of the organization sending the report. Because FDA is providing these reports to you, the value is always `FDA-Public Use.`", -sender,sendertype,string,"The name of the organization sending the report. Because FDA is providing these reports to you, the value is always `2`.", -,serious,string,Seriousness of the adverse event., -,seriousnesscongenitalanomali,string,"This value is `1` if the adverse event resulted in a congenital anomaly, and absent otherwise.", -,seriousnessdeath,string,"This value is `1` if the adverse event resulted in death, and absent otherwise.", -,seriousnessdisabling,string,"This value is `1` if the adverse event resulted in disability, and absent otherwise.", -,seriousnesshospitalization,string,"This value is `1` if the adverse event resulted in a hospitalization, and absent otherwise.", -,seriousnesslifethreatening,string,"This value is `1` if the adverse event resulted in a life threatening condition, and absent otherwise.", -,seriousnessother,string,"This value is `1` if the adverse event resulted in some other serious condition, and absent otherwise.", -,transmissiondate,string,Date that the record was created. This may be earlier than the date the record was received by the FDA., +Section,Field Name,Type,Description, +,authoritynumb,string,"Populated with the Regulatory Authority’s case report number, when available.", +,companynumb,string,Identifier for the company providing the report. This is self-assigned., +,duplicate,string,This value is `1` if earlier versions of this report were submitted to FDA. openFDA only shows the most recent version., +,fulfillexpeditecriteria,string,Identifies expedited reports (those that were processed within 15 days)., +,occurcountry,string,The name of the country where the event occurred., +patient.drug,actiondrug,string,Actions taken with the drug., +patient.drug,abuse,string,"Information about the types of abuse that can occur with the drug and adverse reactions pertinent to those types of abuse, primarily based on human data. May include descriptions of particularly susceptible patient populations.", +patient.drug.activesubstance,activesubstancename,string,"Product active ingredient, which may be different than other drug identifiers (when provided).", +patient.drug,drugadditional,string,Dechallenge outcome information—whether the event abated after product use stopped or the dose was reduced. Only present when this was attempted and the data was provided., +patient.drug,drugadministrationroute,string,The drug’s route of administration., +patient.drug,drugauthorizationnumb,string,"Drug authorization or application number (NDA or ANDA), if provided.", +patient.drug,drugbatchnumb,string,"Drug product lot number, if provided.", +patient.drug,drugcharacterization,string,Reported role of the drug in the adverse event report. These values are not validated by FDA., +patient.drug,drugcumulativedosagenumb,string,"The cumulative dose taken until the first reaction was experienced, if provided.", +patient.drug,drugcumulativedosageunit,string,The unit for `drugcumulativedosagenumb`., +patient.drug,drugdosagetext,string,"Additional detail about the dosage taken. Frequently unknown, but occasionally including information like a brief textual description of the schedule of administration.", +patient.drug,drugenddate,string,Date the patient stopped taking the drug., +patient.drug,drugenddateformat,string,Encoding format of the field `drugenddateformat`. Always set to `102` (YYYYMMDD)., +patient.drug,drugindication,string,Indication for the drug’s use., +patient.drug,drugintervaldosagedefinition,string,The unit for the interval in the field `drugintervaldosageunitnumb.`, +patient.drug,drugintervaldosageunitnumb,string,Information about the drug product’s indications for use., +patient.drug,drugrecurreadministration,string,Whether the reaction occured after readministration of the drug., +patient.drug.drugrecurrence,drugrecuractionmeddraversion,string,The version of MedDRA from which the term in `drugrecuraction` is drawn., +patient.drug.drugrecurrence,drugrecuraction,string,Populated with the Reaction/Event information if/when `drugrecurreadministration` equals `1`., +patient.drug,drugseparatedosagenumb,string,The number of separate doses that were administered., +patient.drug,drugstartdate,string,Date the patient began taking the drug., +patient.drug,drugstartdateformat,string,Encoding format of the field `drugstartdate`. Always set to `102` (YYYYMMDD)., +patient.drug,drugstructuredosagenumb,string,"The number portion of a dosage; when combined with `drugstructuredosageunit` the complete dosage information is represented. For example, *300* in `300 mg`.", +patient.drug,drugstructuredosageunit,string,"The unit for the field `drugstructuredosagenumb`. For example, *mg* in `300 mg`.", +patient.drug,drugtreatmentduration,string,The interval of the field `drugtreatmentdurationunit` for which the patient was taking the drug., +patient.drug,drugtreatmentdurationunit,string,Information about when a doctor or pharmacist should be consulted about drug/drug or drug/food interactions before using a drug product., +patient.drug,medicinalproduct,string,"Drug name. This may be the valid trade name of the product (such as `ADVIL` or `ALEVE`) or the generic name (such as `IBUPROFEN`). This field is not systematically normalized. It may contain misspellings or idiosyncratic descriptions of drugs, such as combination products such as those used for birth control.", +patient,patientagegroup,string,Populated with Patient Age Group code., +patient.patientdeath,patientdeathdate,string,"If the patient died, the date that the patient died.", +patient.patientdeath,patientdeathdateformat,string,Encoding format of the field `patientdeathdate`. Always set to `102` (YYYYMMDD)., +patient,patientonsetage,string,Age of the patient when the event first occured., +patient,patientonsetageunit,string,The unit for the interval in the field `patientonsetage.`, +patient,patientsex,string,The sex of the patient., +patient,patientweight,string,"The patient weight, in kg (kilograms).", +patient.reaction,reactionmeddrapt,string,"Patient reaction, as a MedDRA term. Note that these terms are encoded in British English. For instance, diarrhea is spelled `diarrohea`. MedDRA is a standardized medical terminology.", +patient.reaction,reactionmeddraversionpt,string,The version of MedDRA from which the term in `reactionmeddrapt` is drawn., +patient.reaction,reactionoutcome,string,Outcome of the reaction in `reactionmeddrapt` at the time of last observation., +patient.summary,narrativeincludeclinical,string,"Populated with Case Event Date, when available; does `NOT` include Case Narrative.", +primarysource,literaturereference,string,"Populated with the Literature Reference information, when available.""", +primarysource,qualification,string,Category of individual who submitted the report., +primarysource,reportercountry,string,Country from which the report was submitted., +,primarysourcecountry,string,Country of the reporter of the event., +,receiptdate,string,Date that the _most recent_ information in the report was received by FDA., +,receiptdateformat,string,Encoding format of the `receiptdate` field. Always set to 102 (YYYYMMDD)., +,receivedate,string,"Date that the report was _first_ received by FDA. If this report has multiple versions, this will be the date the first version was received by FDA.", +,receivedateformat,string,Encoding format of the `receivedate` field. Always set to 102 (YYYYMMDD)., +receiver,receiverorganization,string,"Name of the organization receiving the report. Because FDA received the report, the value is always `FDA`.", +receiver,receivertype,string,"The type of organization receiving the report. The value,`6`, is only specified if it is `other`, otherwise it is left blank.", +reportduplicate,duplicatenumb,string,The case identifier for the duplicate., +reportduplicate,duplicatesource,string,The name of the organization providing the duplicate., +,reporttype,string,Code indicating the circumstances under which the report was generated., +,safetyreportid,string,"The 8-digit Safety Report ID number, also known as the case report number or case ID. The first 7 digits (before the hyphen) identify an individual report and the last digit (after the hyphen) is a checksum. This field can be used to identify or find a specific adverse event report.", +,safetyreportversion,string,"The version number of the `safetyreportid`. Multiple versions of the same report may exist, it is generally best to only count the latest report and disregard others. openFDA will only return the latest version of a report.", +sender,senderorganization,string,"Name of the organization sending the report. Because FDA is providing these reports to you, the value is always `FDA-Public Use.`", +sender,sendertype,string,"The name of the organization sending the report. Because FDA is providing these reports to you, the value is always `2`.", +,serious,string,Seriousness of the adverse event., +,seriousnesscongenitalanomali,string,"This value is `1` if the adverse event resulted in a congenital anomaly, and absent otherwise.", +,seriousnessdeath,string,"This value is `1` if the adverse event resulted in death, and absent otherwise.", +,seriousnessdisabling,string,"This value is `1` if the adverse event resulted in disability, and absent otherwise.", +,seriousnesshospitalization,string,"This value is `1` if the adverse event resulted in a hospitalization, and absent otherwise.", +,seriousnesslifethreatening,string,"This value is `1` if the adverse event resulted in a life threatening condition, and absent otherwise.", +,seriousnessother,string,"This value is `1` if the adverse event resulted in some other serious condition, and absent otherwise.", +,transmissiondate,string,Date that the record was created. This may be earlier than the date the record was received by the FDA., ,transmissiondateformat,string,Encoding format of the `transmissiondate` field. Always set to 102 (YYYYMMDD)., \ No newline at end of file diff --git a/src/biome/datasources/faers/documentation/faers_web.md b/src/biome/adhoc_data/datasources/faers/attachments/faers_web.md similarity index 100% rename from src/biome/datasources/faers/documentation/faers_web.md rename to src/biome/adhoc_data/datasources/faers/attachments/faers_web.md diff --git a/src/biome/adhoc_data/datasources/gdc/api.yaml b/src/biome/adhoc_data/datasources/gdc/api.yaml new file mode 100644 index 0000000..a4554c4 --- /dev/null +++ b/src/biome/adhoc_data/datasources/gdc/api.yaml @@ -0,0 +1,304 @@ +datatype: api +description: | + The NCI's Genomic Data Commons (GDC) provides the cancer research community with a repository and computational + platform for cancer researchers who need to understand cancer, its clinical progression, and response to therapy. + The GDC supports several cancer genome programs at the NCI Center for Cancer Genomics (CCG), + including The Cancer Genome Atlas (TCGA) and Therapeutically Applicable Research to Generate Effective Treatments (TARGET). +img_url: null +last_updated: null +name: Genomics Data Commons +provider: adhoc:specialist_agents +resources: + 4b9845c7-d61b-440d-880e-b3d2b315a415: + filepath: facets.txt + integration: 39771f58-c791-43e8-a19d-c2095c338567 + name: facets + resource_id: 4b9845c7-d61b-440d-880e-b3d2b315a415 + resource_type: file + 56f47a1d-b966-4669-8ffd-c3b9de388afd: + code: | + import requests + import json + import pandas as pd + from pandas import json_normalize + + # Define the fields to be returned + fields = [ + "case_id", + ] + + # Initialize pagination variables + all_cases = [] + current_page = 1 + page_size = 1000 + + while True: + # Construct the request parameters + params = { + "fields": ",".join(fields), + "format": "JSON", + "size": page_size, + "from": (current_page - 1) * page_size + } + + # Send the request + response = requests.get(endpoint, params=params) + + # Check for successful response + if response.status_code == 200: + data = response.json() + cases = data["data"]["hits"] + all_cases.extend(cases) + if len(cases) < page_size: + break + current_page += 1 + else: + print(f"Error: {response.status_code} - {response.text}") + break + + # Convert the results to a DataFrame + all_cases_df = pd.DataFrame(json_normalize(all_cases)) + print(f"There are {len(all_cases_df)} cases in GDC in total") + # Display the DataFrame + all_cases_df.head() + integration: 39771f58-c791-43e8-a19d-c2095c338567 + notes: null + query: |- + Get all cases from GDC with no filters. Fetch only the case ids and paginate through the results to fetch all available cases. Store the results as a dataframe + resource_id: 56f47a1d-b966-4669-8ffd-c3b9de388afd + resource_type: example + 57f85658-1659-44e4-a9b8-ec9eb41f7aae: + code: | + import pandas as pd + from pandas import json_normalize + + import requests + import json + + # Define the endpoint and filters + endpoint = "https://api.gdc.cancer.gov/ssm_occurrences" + filters = { + "op": "and", + "content": [ + { + "op": "=", + "content": { + "field": "case.disease_type", + "value": "*myeloid leukemia*" + } + }, + { + "op": "in", + "content": { + "field": "ssm.consequence.transcript.gene.symbol", + "value": ["JAK2"] + } + } + ] + } + + # Define the fields to be returned + fields = [ + "ssm_id", + "ssm.consequence.transcript.gene.symbol", + "ssm.mutation_type", + "ssm.genomic_dna_change", + "ssm.consequence.transcript.aa_change", + "ssm.consequence.transcript.consequence_type", + "case.project.project_id", + "case.submitter_id", + "case.case_id", + "case.diagnoses.primary_diagnosis" + ] + + # Construct the request parameters + params = { + "filters": json.dumps(filters), + "fields": ",".join(fields), + "format": "JSON", + "size": "1000" + } + + # Send the request + response = requests.get(endpoint, params=params) + print(f"total hits: {response.json()['data']['pagination']['total']}") + all_ssms = response.json()['data']['hits'] + + ssms = pd.DataFrame(json_normalize(all_ssms)) + ssms.head() + integration: 39771f58-c791-43e8-a19d-c2095c338567 + notes: null + query: Query the GDC for cases of myeloid leukemia with a JAK2 somatic mutation + and return the results as a pandas dataframe. + resource_id: 57f85658-1659-44e4-a9b8-ec9eb41f7aae + resource_type: example + 74735957-03a3-4706-aa40-f8bf200553cc: + filepath: gdc.md + integration: 39771f58-c791-43e8-a19d-c2095c338567 + name: raw_documentation + resource_id: 74735957-03a3-4706-aa40-f8bf200553cc + resource_type: file + 7a94f76a-0854-41a3-bb97-69a4dcbcaad9: + code: "import pandas as pd\nfrom pandas import json_normalize\n\nimport requests\n\ + import json\n\n# Define the endpoint and filters\nendpoint = \"https://api.gdc.cancer.gov/ssm_occurrences\"\ + \nfilters = {\n \"op\": \"and\",\n \"content\": [\n {\n \ + \ \"op\": \"in\",\n \"content\": {\n \"field\"\ + : \"case.primary_site\",\n \"value\": [\"Bronchus and lung\"\ + ]\n }\n },\n {\n \"op\": \"in\",\n \ + \ \"content\": {\n \"field\": \"case.demographic.gender\"\ + ,\n \"value\": [\"male\"]\n }\n },\n \ + \ {\n \"op\": \"<\",\n \"content\": {\n \ + \ \"field\": \"case.diagnoses.age_at_diagnosis\",\n \"value\"\ + : 16436 # 45 years in days\n }\n },\n {\n \ + \ \"op\": \"in\",\n \"content\": {\n \"field\":\ + \ \"case.exposures.tobacco_smoking_status\",\n \"value\": [\"\ + Lifelong Non-smoker\"]\n }\n }\n ]\n}\n\n# Define the fields\ + \ to be returned\nfields = [\n \"ssm_id\",\n \"ssm.consequence.transcript.gene.symbol\"\ + ,\n \"ssm.mutation_type\",\n \"ssm.genomic_dna_change\",\n \"ssm.consequence.transcript.aa_change\"\ + ,\n \"ssm.consequence.transcript.consequence_type\",\n \"case.project.project_id\"\ + ,\n \"case.submitter_id\",\n \"case.case_id\",\n \"case.diagnoses.primary_diagnosis\"\ + ,\n \"case.primary_site\",\n \"case.demographic.gender\",\n \"case.diagnoses.age_at_diagnosis\"\ + ,\n \"case.exposures.tobacco_smoking_status\"\n]\n\n# Construct the request\ + \ parameters\nparams = {\n \"filters\": json.dumps(filters),\n \"fields\"\ + : \",\".join(fields),\n \"format\": \"JSON\",\n \"size\": \"1000\"\n}\n\ + \n# Send the request\nresponse = requests.get(endpoint, params=params)\n\n#\ + \ Check for successful response\nif response.status_code == 200:\n data =\ + \ response.json()\n # Extract and display the mutation data\n mutations\ + \ = data[\"data\"][\"hits\"]\n print(f\"Found {len(mutations)} mutations\ + \ in lung cancer cases for males under 45 who never smoked:\")\n lung_mutation_df\ + \ = pd.DataFrame(json_normalize(mutations))\nelse:\n print(f\"Error: {response.status_code}\ + \ - {response.text}\")\n \nlung_mutation_df.head()" + integration: 39771f58-c791-43e8-a19d-c2095c338567 + notes: null + query: |- + Get simple somatic mutation occurrences (cases) from GDC where the primary site is bronchus and lung, the gender is male, the age at diagnosis is less than 45 years old, and the patient is a lifelong non-smoker. Return the results as a pandas dataframe. + resource_id: 7a94f76a-0854-41a3-bb97-69a4dcbcaad9 + resource_type: example + 9f1cc9a9-6acd-4797-b050-73afbaeb74c3: + code: | + import pandas as pd + from pandas import json_normalize + + import requests + import json + + # Define the endpoint and filters + endpoint = "https://api.gdc.cancer.gov/cases" + filters = { + "op": "=", + "content": { + "field": "disease_type", + "value": "*myeloid leukemia*" + } + } + + # Define the fields to be returned + fields = [ + "submitter_id", + "case_id", + "primary_site", + "disease_type", + "diagnoses.age_at_diagnosis", + "diagnoses.primary_diagnosis", + "demographic.gender", + "exposures.tobacco_smoking_status", + "files.file_id", + "files.file_name", + "files.data_type", + "files.experimental_strategy" + ] + + # Initialize pagination variables + all_cases = [] + current_page = 1 + page_size = 1000 + + while True: + # Construct the request parameters + params = { + "filters": json.dumps(filters), + "fields": ",".join(fields), + "format": "JSON", + "size": page_size, + "from": (current_page - 1) * page_size + } + + # Send the request + response = requests.get(endpoint, params=params) + + # Check for successful response + if response.status_code == 200: + data = response.json() + cases = data["data"]["hits"] + all_cases.extend(cases) + if len(cases) < page_size: + break + current_page += 1 + else: + print(f"Error: {response.status_code} - {response.text}") + break + + # Convert the results to a DataFrame + cases_df = pd.DataFrame(json_normalize(all_cases)) + print(f"There are {len(cases_df)} cases in GDC for AML") + # Display the DataFrame + cases_df.head() + integration: 39771f58-c791-43e8-a19d-c2095c338567 + notes: null + query: |- + Get all cases where disease type is myeloid leukemia from GDC and return the results as a pandas dataframe. Paginate through the results to fetch all available cases + resource_id: 9f1cc9a9-6acd-4797-b050-73afbaeb74c3 + resource_type: example + a96216e1-8a2b-4fab-90e8-565d1db62fbe: + filepath: gdc_mappings.json + integration: 39771f58-c791-43e8-a19d-c2095c338567 + name: mappings + resource_id: a96216e1-8a2b-4fab-90e8-565d1db62fbe + resource_type: file +slug: adhocspecificationintegration_genomics_data_commons +source: | + # GDC API Documentation + + Below you will find the GDC API documentation. Note that + there are several endpoints available--some are discussed in the `Search and Retrieval` + section, and others are discussed in the `Data Analysis` section. + + When a user is searching for cases related to gene mutations you should use the `/ssm_occurrences` endpoint. + + ${raw_documentation} + + # Additional Instructions: + + You will be given python code to query the GDC API. + + When querying GDC, change the `format` from "TSV" to "JSON" + + If you download a file by file-id, the output will be TSV. + Save the file with the proper extension. + + When filtering you should rely on the fields provided for a given endpoint in the following mapping. + This mapping is a dictionary where the keys are the endpoints and the values are the fields that are available for that endpoint. + + ``` + ${mappings} + ``` + + These are the fields that are available for filtering and also the fields that can be returned. You can use + the facets to filter the fields that are available for a given endpoint. + + A list of fields and their respective choices/facets are as follows: + + ``` + ${facets} + ``` + + Note this list of facets may be incomplete as it changes over time. Remember to consult it + when you are filtering. If you are unsure about options available for a given field, you should consider + just using the wildcard operator. For example, if you are looking for cases for "paragangliomas and glomus tumors" + and getting no results, you should consider just using `*paragangliomas*`. Be sure to let the user know that you are using + the wildcard operator so it might not be as specific as they would like. + + If a user asks you to filter based on the type of cancer, it is often productive to filter on + disease type, primary diagnosis, or primary site. +url: null +uuid: 39771f58-c791-43e8-a19d-c2095c338567 diff --git a/src/biome/datasources/gdc/documentation/facets.txt b/src/biome/adhoc_data/datasources/gdc/attachments/facets.txt similarity index 100% rename from src/biome/datasources/gdc/documentation/facets.txt rename to src/biome/adhoc_data/datasources/gdc/attachments/facets.txt diff --git a/src/biome/datasources/gdc/documentation/gdc.md b/src/biome/adhoc_data/datasources/gdc/attachments/gdc.md similarity index 100% rename from src/biome/datasources/gdc/documentation/gdc.md rename to src/biome/adhoc_data/datasources/gdc/attachments/gdc.md diff --git a/src/biome/datasources/gdc/documentation/gdc_mappings.json b/src/biome/adhoc_data/datasources/gdc/attachments/gdc_mappings.json similarity index 100% rename from src/biome/datasources/gdc/documentation/gdc_mappings.json rename to src/biome/adhoc_data/datasources/gdc/attachments/gdc_mappings.json diff --git a/src/biome/adhoc_data/datasources/gras/api.yaml b/src/biome/adhoc_data/datasources/gras/api.yaml new file mode 100644 index 0000000..993e8c4 --- /dev/null +++ b/src/biome/adhoc_data/datasources/gras/api.yaml @@ -0,0 +1,70 @@ +datatype: dataset +description: | + The FDA Generally Recognized as Safe (GRAS) is a csv file (export) data with notices filed from the FDA GRAS database from years 1998 to 2024. + You may use this tool to check for food additives introductions or changes. + Use this tool to answer questions such as: + In which year was Pea fiber food additive/preservative introduced into the US food supply? +img_url: null +last_updated: null +name: FDA Generally Recognized as Safe (GRAS) +provider: adhoc:specialist_agents +resources: + fd30b48e-e83b-41a2-a969-49ebc58da112: + code: |- + import pandas as pd + + # Load GRAS notices data + df = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/fda-gras/GRASNotices.csv', dtype=str, index_col=False) + + # print head + print(f"First 5 rows of GRAS notices:") + print(df.head()) + + # Search for notices containing Dioctyl sodium sulfosuccinate (case insensitive) + search_term = 'Dioctyl sodium sulfosuccinate' + + print(f"Searching for notices containing {search_term}...") + + matches = df[df['Substance'].str.contains(search_term, case=False, na=False)] + + # Display relevant columns + if len(matches) > 0: + print(matches[['GRAS Notice (GRN) No.', 'Substance', 'Date of filing', "FDA's Letter"]]) + else: + print(f"No GRAS notices found for {search_term}") + integration: d83c8b7e-e51d-499a-8927-79f109fe3f66 + notes: null + query: Is there any FDA GRAS notice for Dioctyl sodium sulfosuccinate as an additive? + resource_id: fd30b48e-e83b-41a2-a969-49ebc58da112 + resource_type: example +slug: adhocspecificationintegration_fda_generally_recognized_as_safe__gras_ +source: "You have access to the FDA Generally Recognized as Safe (GRAS), which is\ + \ a csv file data with notices filed from the FDA GRAS database from years 1998\ + \ to 2024.\nYou may use this to check for food additives introductions, and correlate\ + \ with other data sources for analysis.\n\n\"GRAS\" is an acronym for the phrase\ + \ Generally Recognized As Safe.\nUnder sections 201(s) and 409 of the Federal Food,\ + \ Drug, and Cosmetic Act,\nany substance that is intentionally added to food is\ + \ a food additive.\n\n# Additional Instructions:\nUse the raw csv notices file to\ + \ answer questions about GRAS substances. Load the file from:\n\n${DATASET_FILES_BASE_PATH}/fda-gras/GRASNotices.csv\n\ + \nThese are the columns available (separated by semicolons, some newlines/spaces):\n\ + GRAS Notice (GRN) No.\t; Substance; Intended Use; Basis; Notifier;\tNotifier Address;\n\ + Date of filing;\tGRN Part 1;\tGRN Part 2;\tGRN Part 3;\tGRN Part 4;\tGRN Part 5;\t\ + GRN Part 6;\tGRN Part 7;\nDate of closure; Date of correction letter;\tFDA's Letter;\t\ + Date additional correspondence;\tAdditional correspondence;\nDate additinoal correspondence\ + \ 2;\tAdditional correspondence 2; Date additional correspondence 3;\nAdditional\ + \ correspondence 3; Date additional correspondence 4; Additional correspondence\ + \ 4;\nResubmission; Resubmitted; Notes;\tRelated submission\n\nThe Gras Notice GRN\ + \ No is wrapped by spredsheet-software-like function..\nExample:\n=T(\"1\") for\ + \ \"GRAS Notice (GRN) No.\" = 1\n\nThe file which was downloaded from FDA GRAS Notices;\ + \ http://www.hfpappexternal.fda.gov/scripts/fdcc/?set=GRASNotices;\nFile last updated\ + \ by FDA on 3/3/2025.\n\nIf the \"FDA's Letter\" column contains \" FDA has no questions\"\ + \ it means the FDA did not challenge the notice, closest thing to \"approved\".\n\ + Id the \"FDA's Letter\" column contains \"FDA has questions\" or \"Pending\", or\ + \ similar, it means the FDA has questions about the notice, and the notice is not\ + \ approved yet.\nIf the \"FDA's Letter\" column contains \"Notice does not provide\ + \ a basis for a GRAS determination\"- it means it challenged the notice, and the\ + \ notice was not approved.\nUse the stats of the \"FDA's Letter\" column to know\ + \ if a food additive is actually approved to be introduced into the US food supply\ + \ or not.\n" +url: null +uuid: d83c8b7e-e51d-499a-8927-79f109fe3f66 diff --git a/src/biome/adhoc_data/datasources/hpa/api.yaml b/src/biome/adhoc_data/datasources/hpa/api.yaml new file mode 100644 index 0000000..2279b7b --- /dev/null +++ b/src/biome/adhoc_data/datasources/hpa/api.yaml @@ -0,0 +1,93 @@ +datatype: api +description: | + The Human Protein Atlas is a Swedish-based program initiated in 2003 with the aim to map all + the human proteins in cells, tissues, and organs using an integration of various omics + technologies, including antibody-based imaging, mass spectrometry-based proteomics, + transcriptomics, and systems biology. All the data in the knowledge resource is open access + to allow scientists both in academia and industry to freely access the data for exploration + of the human proteome. + + The Human Protein Atlas consists of eight separate resources, each focusing on a particular + aspect of the genome-wide analysis of the human proteins: + + - The Tissue resource, showing the distribution of the proteins across all major tissues and + organs in the human body + - The Brain resource, exploring the distribution of proteins in various regions of the + mammalian brain + - The Single Cell resource, showing expression of protein-coding genes in immune cells and + human single cell types based on bulk and single cell RNA-seq + - The Subcellular resource, showing the subcellular localization of proteins in single cells + - The Cancer resource, showing the impact of protein levels for the survival of patients + with cancer + - The Blood resource, describing proteins detected in blood and showing protein levels in + blood in patients with different diseases + - The Cell line resource, showing expression of protein-coding genes in human cancer cell + lines + - The Structure & Interaction resource, showing predicted 3D structures and exploring + protein-coding genes in the context of protein-protein and metabolic interaction networks. + + The Human Protein Atlas program has already contributed to several thousands of publications + in the field of human biology and disease and is selected by the organization ELIXIR as a + European core resource due to its fundamental importance for a wider life science community. + In addition the Human Protein Atlas has been appointed Global Core Biodata Resource (GCBR) + by the Global Biodata Coalition. The Human Protein Atlas consortium is mainly funded by the + Knut and Alice Wallenberg Foundation. +img_url: null +last_updated: null +name: Human Protein Atlas +provider: adhoc:specialist_agents +resources: + 04c53147-cc0a-42f5-8e1d-6427a595948b: + filepath: hpa_docs.md + integration: 9653bf97-8668-4ce2-8c88-653f2d2b3749 + name: raw_documentation + resource_id: 04c53147-cc0a-42f5-8e1d-6427a595948b + resource_type: file + 92183e68-5e56-4487-a92f-8a1163b4e178: + code: | + import requests + import json + + # Query JAK2 data from HPA API + jak2_id = "ENSG00000096968" + url = f"https://www.proteinatlas.org/{jak2_id}.json" + + # Get the data + response = requests.get(url) + data = response.json() + + # Print basic gene information + print(f"Gene: {data.get('Gene', 'N/A')}") + print(f"Description: {data.get('Gene description', 'N/A')}") + print("-" * 80) + + # RNA Expression Summary + print("\nRNA Expression Summary:") + print(f"Tissue specificity: {data.get('RNA tissue specificity', 'N/A')}") + print(f"Tissue distribution: {data.get('RNA tissue distribution', 'N/A')}") + print(f"Tissue specificity score: {data.get('RNA tissue specificity score', 'N/A')}") + + # Blood cell specific information + print("\nRNA Blood Cell Expression:") + print(f"Blood cell specificity: {data.get('RNA blood cell specificity', 'N/A')}") + print(f"Blood cell distribution: {data.get('RNA blood cell distribution', 'N/A')}") + print(f"Blood cell specificity score: {data.get('RNA blood cell specificity score', 'N/A')}") + + # Protein Expression Summary + print("\nProtein Expression Summary:") + print(f"Reliability (Immunohistochemistry): {data.get('Reliability (IH)', 'N/A')}") + print(f"Subcellular main location: {data.get('Subcellular main location', 'N/A')}") + if 'Subcellular additional location' in data: + print(f"Subcellular additional location: {data.get('Subcellular additional location', 'N/A')}") + integration: 9653bf97-8668-4ce2-8c88-653f2d2b3749 + notes: null + query: |- + Query the Human Protein Atlas API for RNA and protein expression summary of a gene (JAK2) using its Ensembl ID. The example demonstrates how to retrieve and display tissue specificity, distribution, and subcellular localization information. + resource_id: 92183e68-5e56-4487-a92f-8a1163b4e178 + resource_type: example +slug: adhocspecificationintegration_human_protein_atlas +source: '${raw_documentation} + + ' +url: null +uuid: 9653bf97-8668-4ce2-8c88-653f2d2b3749 diff --git a/src/biome/datasources/hpa/documentation/hpa_docs.md b/src/biome/adhoc_data/datasources/hpa/attachments/hpa_docs.md similarity index 100% rename from src/biome/datasources/hpa/documentation/hpa_docs.md rename to src/biome/adhoc_data/datasources/hpa/attachments/hpa_docs.md diff --git a/src/biome/adhoc_data/datasources/idc/api.yaml b/src/biome/adhoc_data/datasources/idc/api.yaml new file mode 100644 index 0000000..849f120 --- /dev/null +++ b/src/biome/adhoc_data/datasources/idc/api.yaml @@ -0,0 +1,352 @@ +datatype: api +description: | + NCI Imaging Data Commons (IDC) is a cloud-based environment containing publicly available cancer imaging data co-located with the analysis and exploration tools and resources. + + Features: + - >65 TB: IDC contains radiology, brightfield (H&E) and fluorescence slide microscopy images, along with image-derived data (annotations, segmentations, quantitative measurements) and accompanying clinical data + - Free: All of the data in IDC is publicly available: no registration, no access requests + - Commercial-friendly: >95% of the data in IDC is covered by the permissive CC-BY license, which allows commercial reuse (small subset of data is covered by the CC-NC license); each file in IDC is tagged with the license to make it easier for you to understand and follow the rules + - Cloud-based: All of the data in IDC is available from both Google and AWS public buckets: fast and free to download, no out-of-cloud egress fees + - Harmonized: All of the images and image-derived data in IDC is harmonized into standard DICOM representation +img_url: null +last_updated: null +name: Imaging Data Commons +provider: adhoc:specialist_agents +resources: + 090cdd92-54e0-4a30-82ab-a2e9c67b50c7: + code: "# Query to list all studies in the 'cmb_aml' collection\nquery_all_studies\ + \ = \"\"\"\nSELECT DISTINCT StudyDescription\nFROM \n index\nWHERE \n \ + \ collection_id = 'cmb_aml'\n\"\"\"\n\ntry:\n df_all_studies = client.sql_query(query_all_studies)\n\ + \ print(\"All studies in the 'cmb_aml' collection:\")\n print(df_all_studies)\n\ + except Exception as e:\n print(f\"An error occurred: {str(e)}\")\n" + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 + notes: null + query: List all studies for a given collection + resource_id: 090cdd92-54e0-4a30-82ab-a2e9c67b50c7 + resource_type: example + 1be1fe03-4bb2-4240-a95e-96efb9c1d4d8: + code: | + from idc_index import index + import pandas as pd + + client = index.IDCClient() + + # Query for slide microscopy data + query = """ + SELECT DISTINCT + collection_id, + PatientID, + SeriesDescription, + StudyDescription, + BodyPartExamined, + SeriesDate + FROM + index + WHERE + Modality = 'SM' + """ + + try: + df = client.sql_query(query) + print("Slide Microscopy Data Available:") + print("\nCollections with slide microscopy data:") + print(df['collection_id'].unique()) + print("\nSample of the data:") + print(df.head()) + print("\nTotal number of records:", len(df)) + except Exception as e: + print(f"An error occurred: {str(e)}") + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 + notes: null + query: Get slide microscopy data + resource_id: 1be1fe03-4bb2-4240-a95e-96efb9c1d4d8 + resource_type: example + 25d73a5f-adb2-495c-9588-8237bb39883d: + code: "from idc_index import index\nimport pandas as pd\n\nclient = index.IDCClient()\n\ + \n# Query for all available fields for AML slides\nquery = \"\"\"\nSELECT *\n\ + FROM\n index\nWHERE\n Modality = 'SM'\n AND collection_id IN ('cptac_aml',\ + \ 'cmb_aml')\n AND SeriesDescription LIKE '%bone marrow%'\nORDER BY\n \ + \ PatientID, SeriesDate\n\"\"\"\n\ntry:\n df = client.sql_query(query)\n\ + \ print(\"AML Bone Marrow Slide Data:\")\n print(\"\\nTotal number of\ + \ bone marrow slides:\", len(df))\n\n print(\"\\nAvailable columns (metadata\ + \ fields):\")\n print(df.columns.tolist())\n\n # Look for any columns\ + \ that might contain annotations or clinical data\n clinical_cols = [col\ + \ for col in df.columns if any(term in col.lower() \n \ + \ for term in ['annotation', 'clinical', 'blast',\ + \ 'cell', \n 'pathology',\ + \ 'diagnosis', 'grade', 'stage'])]\n print(\"\\nPotentially relevant clinical/annotation\ + \ columns:\")\n print(clinical_cols)\n\n # Get the source DOIs which might\ + \ lead to additional clinical data\n print(\"\\nUnique source DOIs (for finding\ + \ additional clinical data):\")\n print(df['source_DOI'].unique())\n\nexcept\ + \ Exception as e:\n print(f\"An error occurred: {str(e)}\")\n\n# Let's also\ + \ check if there are any study-level annotations\nquery_study = \"\"\"\nSELECT\ + \ DISTINCT\n StudyDescription,\n collection_id,\n source_DOI,\n \ + \ license_short_name\nFROM\n index\nWHERE\n collection_id IN ('cptac_aml',\ + \ 'cmb_aml')\n\"\"\"\n\ntry:\n df_study = client.sql_query(query_study)\n\ + \ print(\"\\nStudy-level information:\")\n print(df_study)\nexcept Exception\ + \ as e:\n print(f\"An error occurred querying study data: {str(e)}\")\n" + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 + notes: null + query: Get additional information for microscopy data for a specific collection + resource_id: 25d73a5f-adb2-495c-9588-8237bb39883d + resource_type: example + 3078cb17-9102-4e54-a381-d0b546032cef: + code: "from idc_index import index\nimport pandas as pd\n\nclient = index.IDCClient()\n\ + \n# Query to find collection names containing 'aml'\nquery = \"\"\"\nSELECT\ + \ DISTINCT collection_id\nFROM \n index\nWHERE \n LOWER(collection_id)\ + \ LIKE '%aml%'\n\"\"\"\n\ntry:\n df_collections = client.sql_query(query)\n\ + \ print(\"Collections with 'aml' in their name:\")\n print(df_collections)\n\ + except Exception as e:\n print(f\"An error occurred: {str(e)}\")\n" + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 + notes: null + query: Find collections in the Imaging Data Commons with 'aml' in their name, + likely related to Acute Myeloid Leukemia. + resource_id: 3078cb17-9102-4e54-a381-d0b546032cef + resource_type: example + 49f63b7d-8e99-4e88-ac9d-86db0d831a8f: + code: | + from idc_index import index + import pandas as pd + + client = index.IDCClient() + + # First, let's see what columns are available + query_columns = """ + SELECT * FROM index LIMIT 1 + """ + + try: + df = client.sql_query(query_columns) + print("Available columns in the index:") + print(df.columns.tolist()) + except Exception as e: + print(f"An error occurred: {str(e)}") + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 + notes: null + query: Get columns available in IDC index + resource_id: 49f63b7d-8e99-4e88-ac9d-86db0d831a8f + resource_type: example + 4adff70f-bc39-4728-a215-b8f7908ff5d0: + code: | + import pandas as pd + from idc_index import index + + client = index.IDCClient() + + # Query for the desired images + query = """ + SELECT series_aws_url + FROM index + WHERE collection_id = 'cmb_aml' + AND StudyDescription = 'XR_Biopsy_BoneMarrow' + """ + df = client.sql_query(query) + + # Create a download manifest using the current directory + with open('download_manifest.txt', 'w') as f: + for url in df['series_aws_url']: + f.write(f'cp {url} ./\n') + + # Execute the download + s5cmd_binary = client.s5cmdPath + !{s5cmd_binary} --no-sign-request --endpoint-url https://s3.amazonaws.com run download_manifest.txt + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 + notes: null + query: Download bone marrow biopsy images from the 'cmb_aml' collection for a + specific study using IDC's public S3 bucket. + resource_id: 4adff70f-bc39-4728-a215-b8f7908ff5d0 + resource_type: example + 51d2dd36-412e-492e-b52f-37b60219dd8c: + code: "from idc_index import index\nimport pandas as pd\n\n# Initialize the IDC\ + \ client\nclient = index.IDCClient()\n\n# Query to find studies within the 'cptac_aml'\ + \ collection\nquery = \"\"\"\nSELECT DISTINCT StudyDescription\nFROM \n index\n\ + WHERE \n collection_id = 'cptac_aml'\n\"\"\"\n\ntry:\n # Execute the query\n\ + \ df_studies = client.sql_query(query)\n print(\"Studies in 'cptac_aml':\"\ + )\n print(df_studies)\nexcept Exception as e:\n print(f\"An error occurred:\ + \ {str(e)}\")\n" + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 + notes: null + query: Fetch studies for a given collection in the Imaging Data Commons (IDC), + specifically for the 'cptac_aml' collection. + resource_id: 51d2dd36-412e-492e-b52f-37b60219dd8c + resource_type: example + 590305c8-9e62-49a5-a28f-8cc74df03667: + code: | + import pandas as pd + from idc_index import index + + client = index.IDCClient() + + query = """ + SELECT series_aws_url + FROM index + WHERE collection_id = 'cmb_aml' + AND StudyDescription = 'XR_Biopsy_BoneMarrow' + """ + df = client.sql_query(query) + print(df.head()) + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 + notes: null + query: Retrieve imagery URLs for the 'XR_Biopsy_BoneMarrow' study in the 'cmb_aml' + collection from IDC. + resource_id: 590305c8-9e62-49a5-a28f-8cc74df03667 + resource_type: example + a155ea7d-f42f-4546-8d4c-b1135994ea47: + filepath: idc.md + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 + name: raw_documentation + resource_id: a155ea7d-f42f-4546-8d4c-b1135994ea47 + resource_type: file + bbfbce09-abfe-4565-a50b-6afc299df2d3: + code: | + import pydicom + import matplotlib.pyplot as plt + + # Load the DICOM file + filename = '73f18e49-8699-4c1f-90ad-5db94b6b5602.dcm' + ds = pydicom.dcmread(filename) + + # Display the image + plt.imshow(ds.pixel_array, cmap=plt.cm.gray) + plt.title('Bone Marrow Biopsy Image') + plt.axis('off') + plt.show() + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 + notes: null + query: Basic plot of dicom image from file + resource_id: bbfbce09-abfe-4565-a50b-6afc299df2d3 + resource_type: example + c19b78ab-29cb-4be3-8069-1ef22302dfb2: + code: "import sys\nimport subprocess\n\n# Install required package\nsubprocess.check_call([sys.executable,\ + \ \"-m\", \"pip\", \"install\", \"pydicom\"])\n\nimport boto3\nfrom botocore\ + \ import UNSIGNED\nfrom botocore.config import Config\nimport os\nimport tempfile\n\ + import pydicom\nfrom PIL import Image\nimport numpy as np\nimport matplotlib.pyplot\ + \ as plt\n\n# Create S3 client\ns3 = boto3.client('s3', config=Config(signature_version=UNSIGNED))\n\ + \n# Create temporary directory\ntemp_dir = tempfile.mkdtemp()\n\n# Let's try\ + \ the smallest DICOM file first\nfile_key = \"d53e7fc3-8ef9-4de5-8059-47b21a67eb4f/40006642-6ee5-44ca-bc5f-e1cc2bd7ee70.dcm\"\ + \nlocal_file = os.path.join(temp_dir, \"slide.dcm\")\n\ntry:\n print(f\"\ + Downloading file: {file_key}\")\n s3.download_file('idc-open-data', file_key,\ + \ local_file)\n print(\"Download successful!\")\n\n # Try to read the\ + \ DICOM file\n print(\"\\nReading DICOM file...\")\n ds = pydicom.dcmread(local_file)\n\ + \n print(\"\\nDICOM metadata:\")\n print(f\"Patient ID: {ds.PatientID}\"\ + )\n print(f\"Modality: {ds.Modality}\")\n print(f\"Image Type: {ds.ImageType}\"\ + )\n\n # Convert to image and display\n if hasattr(ds, 'pixel_array'):\n\ + \ print(\"\\nConverting to image...\")\n pixel_array = ds.pixel_array\n\ + \n # Normalize the pixel values\n if pixel_array.dtype != np.uint8:\n\ + \ pixel_array = ((pixel_array - pixel_array.min()) * 255.0 / \n \ + \ (pixel_array.max() - pixel_array.min())).astype(np.uint8)\n\ + \n # Create image\n img = Image.fromarray(pixel_array)\n\n \ + \ # Display\n plt.figure(figsize=(15, 10))\n plt.imshow(img,\ + \ cmap='gray')\n plt.axis('off')\n plt.title(f\"DICOM Image -\ + \ Patient: {ds.PatientID}\")\n plt.show()\n else:\n print(\"\ + No pixel data found in the DICOM file\")\n\nexcept Exception as e:\n print(f\"\ + An error occurred: {str(e)}\")\nfinally:\n # Clean up\n if os.path.exists(local_file):\n\ + \ os.remove(local_file)\n os.rmdir(temp_dir)\n" + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 + notes: null + query: Download and visualize a slide microscopy image with pydicom and matplotlib + resource_id: c19b78ab-29cb-4be3-8069-1ef22302dfb2 + resource_type: example + dc0ce97c-46a2-406b-a582-a51be49367c9: + code: "from idc_index import index\nimport pandas as pd\n\nclient = index.IDCClient()\n\ + \n# Query to get download URLs for bone marrow slides\nquery = \"\"\"\nSELECT\n\ + \ PatientID,\n SeriesDescription,\n series_aws_url,\n series_size_MB\n\ + FROM\n index\nWHERE\n Modality = 'SM'\n AND collection_id IN ('cptac_aml',\ + \ 'cmb_aml')\n AND SeriesDescription LIKE '%bone marrow%'\nLIMIT 1\n\"\"\"\ + \n\ntry:\n df = client.sql_query(query)\n print(\"Sample slide information:\"\ + )\n print(df)\n \n if not df.empty:\n url = df['series_aws_url'].iloc[0]\n\ + \ size_mb = df['series_size_MB'].iloc[0]\n print(f\"\\nFile size:\ + \ {size_mb:.2f} MB\")\n print(f\"Download URL: {url}\")\nexcept Exception\ + \ as e:\n print(f\"An error occurred: {str(e)}\")\n" + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 + notes: null + query: Query to get download URLs for bone marrow slides + resource_id: dc0ce97c-46a2-406b-a582-a51be49367c9 + resource_type: example + e0279959-d257-438e-8513-fb40391082c1: + code: "from idc_index import index\nimport pandas as pd\n\nclient = index.IDCClient()\n\ + \n# Query for AML-specific slide microscopy data\nquery = \"\"\"\nSELECT DISTINCT\n\ + \ collection_id,\n PatientID,\n SeriesDescription,\n StudyDescription,\n\ + \ BodyPartExamined,\n SeriesDate,\n Manufacturer,\n ManufacturerModelName,\n\ + \ SeriesNumber,\n instanceCount\nFROM\n index\nWHERE\n Modality\ + \ = 'SM'\n AND collection_id IN ('cptac_aml', 'cmb_aml')\nORDER BY\n PatientID,\ + \ SeriesDate\n\"\"\"\n\ntry:\n df = client.sql_query(query)\n print(\"\ + AML Slide Microscopy Data:\")\n print(\"\\nTotal number of records:\", len(df))\n\ + \n print(\"\\nUnique patients:\", len(df['PatientID'].unique()))\n\n print(\"\ + \\nTypes of series descriptions (slide types):\")\n print(df['SeriesDescription'].unique())\n\ + \n print(\"\\nSample of the data:\")\n print(df.head(10))\n\n # Save\ + \ to a CSV file for further analysis\n df.to_csv('aml_pathology_data.csv',\ + \ index=False)\n print(\"\\nData has been saved to 'aml_pathology_data.csv'\"\ + )\n\nexcept Exception as e:\n print(f\"An error occurred: {str(e)}\") \n" + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 + notes: null + query: Get slide microscopy data for a specific collection + resource_id: e0279959-d257-438e-8513-fb40391082c1 + resource_type: example + f5ffd878-c1c9-46e9-b83c-6fd54fe975a5: + code: | + import boto3 + from botocore import UNSIGNED + from botocore.config import Config + import os + import tempfile + + # Function to list contents of an S3 path + def list_s3_contents(bucket, prefix): + s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED)) + try: + result = s3.list_objects_v2(Bucket=bucket, Prefix=prefix) + if 'Contents' in result: + return result['Contents'] + return [] + except Exception as e: + print(f"Error listing S3 contents: {str(e)}") + return [] + + # Parse S3 URL and list contents + s3_url = "s3://idc-open-data/d53e7fc3-8ef9-4de5-8059-47b21a67eb4f" + bucket = s3_url.split('/')[2] + prefix = '/'.join(s3_url.split('/')[3:]) + + print(f"Checking contents of bucket: {bucket}") + print(f"With prefix: {prefix}") + + contents = list_s3_contents(bucket, prefix) + print("\nFound files:") + for item in contents: + print(f"- {item['Key']} ({item['Size']/1024/1024:.2f} MB)") + + # Let's also check if we can get any pre-signed URLs or public access URLs + s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED)) + try: + # Try to get the bucket location + location = s3.get_bucket_location(Bucket=bucket) + print(f"\nBucket location: {location}") + + # Try to get bucket policy + try: + policy = s3.get_bucket_policy(Bucket=bucket) + print("\nBucket policy:", policy) + except Exception as e: + print("\nCouldn't get bucket policy:", str(e)) + + except Exception as e: + print(f"\nError getting bucket information: {str(e)}") + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 + notes: null + query: Get list of files in an S3 bucket for a specific collection on IDC + resource_id: f5ffd878-c1c9-46e9-b83c-6fd54fe975a5 + resource_type: example +slug: adhocspecificationintegration_imaging_data_commons +source: | + ${raw_documentation} + + # Additional Instructions: + + Be sure to import and instantiate the client for the Imaging Data Commons API: + ```python + from idc_index import index + client = index.IDCClient() + ``` +url: null +uuid: b9f7983b-97b3-43c6-9b12-13b810e91ac0 diff --git a/src/biome/datasources/idc/documentation/idc.md b/src/biome/adhoc_data/datasources/idc/attachments/idc.md similarity index 100% rename from src/biome/datasources/idc/documentation/idc.md rename to src/biome/adhoc_data/datasources/idc/attachments/idc.md diff --git a/src/biome/adhoc_data/datasources/indra/api.yaml b/src/biome/adhoc_data/datasources/indra/api.yaml new file mode 100644 index 0000000..1f4c39e --- /dev/null +++ b/src/biome/adhoc_data/datasources/indra/api.yaml @@ -0,0 +1,238 @@ +datatype: api +description: | + INDRA CoGEx (Context Graph Extension) is an automatically assembled biomedical knowledge graph which integrates causal mechanisms from INDRA + with non-causal contextual relations including properties, ontology, and data. + + The INDRA CoGEx Query API is a RESTful service designed to streamline complex queries within the biomedical domain, + powered by the Contextual Graph Explorer (CoGEx) framework. It provides an interface to retrieve detailed insights + by integrating and analyzing data from diverse biomedical sources. Users can execute queries to uncover relationships + such as diseases associated with specific clinical trials, drugs linked to particular side effects, or other intricate + connections in biomedical knowledge. By leveraging its contextual graph-based approach, the API enables researchers, + data scientists, and healthcare professionals to explore and analyze rich biomedical data efficiently, supporting + advancements in drug discovery, clinical decision-making, and research. With its robust architecture, the INDRA CoGEx + Query API facilitates the discovery of meaningful patterns and supports informed decision-making in complex biomedical scenarios. +img_url: null +last_updated: null +name: INDRA Context Graph Extension (CoGEx) +provider: adhoc:specialist_agents +resources: + b7c4d45f-287b-47e2-aa9b-5ee5bae07353: + code: | + import requests + + base_url = "https://discovery.indra.bio/api/" + + # Get pathways for JAK2 + jak2_payload = { + "gene": ["HGNC", "6192"] # JAK2 HGNC ID + } + + # Get JAK2 pathways + jak2_pathways_response = requests.post( + base_url + "get_pathways_for_gene", + json=jak2_payload + ) + + print("JAK2 Pathways Response:") + print(jak2_pathways_response.json()) + + # Get shared pathways with common partner genes + partner_genes = [ + ["HGNC", "6192"], # JAK2 + ["HGNC", "3603"], # STAT3 + ["HGNC", "6193"], # JAK3 + ["HGNC", "3236"] # EGFR + ] + + shared_payload = { + "genes": partner_genes + } + + # Get shared pathways + shared_pathways_response = requests.post( + base_url + "get_shared_pathways_for_genes", + json=shared_payload + ) + + print("\nShared Pathways Response:") + print(shared_pathways_response.json()) + integration: 417f20f1-4f56-4f6c-b11a-c40d84258557 + notes: null + query: |- + Perform a gene pathway analysis for JAK2 to find associated pathways and shared pathways with other genes like STAT3, JAK3, and EGFR. + resource_id: b7c4d45f-287b-47e2-aa9b-5ee5bae07353 + resource_type: example + c8f8db77-c0bb-4ee2-9525-925707496b5a: + code: | + import requests + import json + + def get_mesh_literature(): + # API base URL + base_url = "https://discovery.indra.bio/api/" + + # Get papers with evidence for AML MeSH term + payload = { + "mesh_term": ["MESH", "D015470"], + "include_child_terms": True + } + + # Make request to get evidences + response = requests.post( + base_url + "get_evidences_for_mesh", + json=payload, + headers={"Content-Type": "application/json"} + ) + + if response.status_code == 200: + # Process and return results + results = response.json() + + # Convert results to a more readable format + evidence_by_statement = {} + for stmt_hash, evidences in results.items(): + evidence_by_statement[stmt_hash] = { + 'pmids': list(set([ev['text_refs'].get('PMID') for ev in evidences if ev.get('text_refs', {}).get('PMID')])), + 'evidence_count': len(evidences) + } + + return evidence_by_statement + else: + print(f"Error: {response.status_code}") + return None + + if __name__ == "__main__": + evidence_data = get_mesh_literature() + + if evidence_data: + # Print summary of findings + print(f"Found {len(evidence_data)} statements with evidence") + + # Get total evidence count + total_evidence = sum(data['evidence_count'] for data in evidence_data.values()) + print(f"Total evidence count: {total_evidence}") + + # Get unique PMIDs + all_pmids = set() + for data in evidence_data.values(): + all_pmids.update(data['pmids']) + print(f"Total unique PMIDs: {len(all_pmids)}") + integration: 417f20f1-4f56-4f6c-b11a-c40d84258557 + notes: null + query: Retrieve literature evidence related to AML using MeSH term queries, including + child terms, and summarize the findings. + resource_id: c8f8db77-c0bb-4ee2-9525-925707496b5a + resource_type: example + e57fbdf9-732a-407a-8469-921076b69054: + code: | + import requests + + def analyze_jak2(): + # API base URL + base_url = "https://discovery.indra.bio/api/" + + # JAK2 HGNC ID + jak2_node = ["HGNC", "6192"] + + # Get GO terms + go_terms_payload = { + "gene": jak2_node, + "include_indirect": True + } + go_response = requests.post( + base_url + "get_go_terms_for_gene", + json=go_terms_payload + ) + + # Get pathways + pathways_payload = { + "gene": jak2_node + } + pathways_response = requests.post( + base_url + "get_pathways_for_gene", + json=pathways_payload + ) + + # Check if kinase + kinase_payload = { + "genes": ["JAK2"] + } + kinase_response = requests.post( + base_url + "is_kinase", + json=kinase_payload + ) + + # Check if phosphatase + phosphatase_payload = { + "genes": ["JAK2"] + } + phosphatase_response = requests.post( + base_url + "is_phosphatase", + json=phosphatase_payload + ) + + # Check if transcription factor + tf_payload = { + "genes": ["JAK2"] + } + tf_response = requests.post( + base_url + "is_transcription_factor", + json=tf_payload + ) + + # Compile and return results + results = { + "go_terms": go_response.json(), + "pathways": pathways_response.json(), + "is_kinase": kinase_response.json().get("JAK2", False), + "is_phosphatase": phosphatase_response.json().get("JAK2", False), + "is_transcription_factor": tf_response.json().get("JAK2", False) + } + + return results + + if __name__ == "__main__": + results = analyze_jak2() + print(f"Analysis Results for JAK2:") + print(f"Number of GO terms: {len(results['go_terms'])}") + print(f"Number of pathways: {len(results['pathways'])}") + print(f"Is kinase: {results['is_kinase']}") + print(f"Is phosphatase: {results['is_phosphatase']}") + print(f"Is transcription factor: {results['is_transcription_factor']}") + integration: 417f20f1-4f56-4f6c-b11a-c40d84258557 + notes: null + query: |- + This example demonstrates how to analyze the JAK2 gene using the INDRA CoGEx API to retrieve GO terms, pathway associations, and check if it is a kinase, phosphatase, or transcription factor. + resource_id: e57fbdf9-732a-407a-8469-921076b69054 + resource_type: example + f49c8aa7-727e-444e-a3a2-314a5f0d6631: + filepath: indra.json + integration: 417f20f1-4f56-4f6c-b11a-c40d84258557 + name: raw_documentation + resource_id: f49c8aa7-727e-444e-a3a2-314a5f0d6631 + resource_type: file +slug: adhocspecificationintegration_indra_context_graph_extension__cogex_ +source: | + ${raw_documentation} + + # Additional Instructions: + + You should make use of the python requests library to interact with the API. + Note that the base URL of the API is 'https://discovery.indra.bio/api/' + + For example, here is an example request: + + curl -X 'POST' \ + 'https://discovery.indra.bio/api/get_go_terms_for_gene' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "gene": [ + "HGNC", + "9896" + ], + "include_indirect": true + }' + ``` +url: null +uuid: 417f20f1-4f56-4f6c-b11a-c40d84258557 diff --git a/src/biome/datasources/indra/documentation/indra.json b/src/biome/adhoc_data/datasources/indra/attachments/indra.json similarity index 100% rename from src/biome/datasources/indra/documentation/indra.json rename to src/biome/adhoc_data/datasources/indra/attachments/indra.json diff --git a/src/biome/adhoc_data/datasources/netrias/api.yaml b/src/biome/adhoc_data/datasources/netrias/api.yaml new file mode 100644 index 0000000..1105817 --- /dev/null +++ b/src/biome/adhoc_data/datasources/netrias/api.yaml @@ -0,0 +1,84 @@ +datatype: api +description: | + The Netrias Harmonization REST API leverages AI-driven harmonization approaches to standardize + cancer-related terminologies from the Cancer Data Service (CDS), managed by the Center for + Biomedical Informatics and Information Technology (CBIIT) at NIH. + + This API is designed to process and harmonize textual variations of terms, mapping them to + standardized terms defined in the CDS data model ontology (version 5.0.5). By utilizing + advanced machine learning techniques, the service enhances data consistency, improves + interoperability, and streamlines standardization for cancer research and biomedical + informatics applications. + + The API enables automated term reconciliation, ensuring uniform terminology across datasets + for more reliable data integration and analysis. The current intended application is for + harmonization of terms in study metadata from cancer researchers delivering research data + to the Cancer Research Data Commons. +img_url: null +last_updated: null +name: Netrias Harmonization API +provider: adhoc:specialist_agents +resources: + 2030d84d-840e-44a1-89d4-f027e2158a31: + code: "import requests\nimport os\nimport json\nimport difflib\n\n# Get API key\ + \ from environment variable\napi_key = os.environ.get(\"NETRIAS_KEY\")\n\n#\ + \ Set up the request headers\nheaders = {\n \"x-api-key\": api_key,\n \ + \ \"Content-Type\": \"application/json\"\n}\n\n# Base URL for the API\nbase_url\ + \ = \"https://apivm.netriasbdf.cloud\"\n\n# The drug term to harmonize\n\ + drug_term = \"zorubicin_9012\"\n\n# Get the standards to find therapeutic agents\n\ + standards_url = f\"{base_url}/v1/standards\"\nstandards_response = requests.get(standards_url,\ + \ headers=headers)\n\nif standards_response.status_code == 200:\n standards\ + \ = standards_response.json()\n \n # Look for therapeutic agents in the\ + \ DataHub standards\n if \"datahub\" in standards and \"therapeutic_agents\"\ + \ in standards[\"datahub\"]:\n therapeutic_agents = standards[\"datahub\"\ + ][\"therapeutic_agents\"]\n \n # Find closest matches using difflib\n\ + \ matches = difflib.get_close_matches(drug_term, therapeutic_agents,\ + \ n=5, cutoff=0.5)\n \n print(f\"Found closest matches for '{drug_term}':\"\ + )\n for match in matches:\n print(f\" {match}\")\n \ + \ \n if matches:\n print(f\"\\nBest harmonization result:\ + \ '{matches[0]}'\")\n else:\n print(f\"No close matches found\ + \ for '{drug_term}'\")\n else:\n print(\"Could not find therapeutic_agents\ + \ in DataHub standards\")\nelse:\n print(f\"Error getting standards: {standards_response.status_code}\"\ + )\n print(standards_response.text)\n" + integration: 9fc4893e-ad7d-4ade-b7ab-2a70653602a6 + notes: |- + This example demonstrates how to harmonize a drug term that includes a non-standard suffix (e.g., "zorubicin_9012") by: + 1. Retrieving the therapeutic_agents standard from the DataHub + 2. Using Python's difflib to find the closest matches + 3. Identifying "Zorubicin" as the standard term + + The approach uses string similarity matching when the direct API harmonization doesn't return results. This is useful for handling terms with identifiers, suffixes, or other non-standard additions that aren't in the controlled vocabulary. + query: Harmonizing a drug term with a non-standard suffix + resource_id: 2030d84d-840e-44a1-89d4-f027e2158a31 + resource_type: example + 8242a126-8ef1-48de-aa94-e0ccc7e58a28: + filepath: netrias.json + integration: 9fc4893e-ad7d-4ade-b7ab-2a70653602a6 + name: raw_documentation + resource_id: 8242a126-8ef1-48de-aa94-e0ccc7e58a28 + resource_type: file +slug: adhocspecificationintegration_netrias_harmonization_api +source: "${raw_documentation}\n\n# Additional Instructions:\n\nThe REST API is accessible\ + \ at https://apivm.netriasbdf.cloud/.\nIt uses API Keys to track access. It is\ + \ just applied as an additional HTTP header `x-api-key: `\n\n**IMPORTANT**:\ + \ the API Key is available in the environment variable `NETRIAS_KEY`.\n\nFor CDS\ + \ standard terms, commonly referred to as Common Data Elements or CDEs, with large\ + \ numbers of permissible values,\nNetrias builds fine tuned LLMs from synthetically\ + \ generated training data. If a CDE has a small number of permissible values,\n\ + we apply in-context learning approaches where we craft a prompt to feed an LLM API\ + \ chat endpoint (e.g. OpenAI) requesting it\ngenerate harmonizations with no or\ + \ few examples. When used as a tool, this may lead to significant variations in\ + \ response\nlatency as the fine tuned models may take some time to load into a GPU,\ + \ tokenize the input, and infer the intended response.\n\nThe REST API also supports\ + \ the harmonization of terms from ontologies supporting Sage Bionetworks' data repositories\ + \ focused\non rare diseases and cancers. In a demonstration integration of the API\ + \ with Sage's Synapse chatbot application the teams\nfocused on four types of variations.\ + \ Here are some examples:\n\n1. Typos\n a. neurofibromatisis type ! \u2192 Neurofibromatosis\ + \ type 1\n b. whole exme squencing \u2192 whole exome sequencing\n\n2. Exact\ + \ synonyms\n a. NF-1 \u2192 Neurofibromatosis type 1\n b. WEX \u2192 whole\ + \ exome sequencing\n\n3. Narrow synonyms\n a. Watson Syndrome \u2192 Neurofibromatosis\ + \ type 1\n b. clinical whole-exome sequencing \u2192 whole exome sequencing\n\ + \n4. Broad synonyms\n a. NF \u2192 Neurofibromatosis type 1\n b. DNA sequencing\ + \ \u2192 whole exome sequencing\n" +url: null +uuid: 9fc4893e-ad7d-4ade-b7ab-2a70653602a6 diff --git a/src/biome/datasources/netrias/documentation/netrias.json b/src/biome/adhoc_data/datasources/netrias/attachments/netrias.json similarity index 100% rename from src/biome/datasources/netrias/documentation/netrias.json rename to src/biome/adhoc_data/datasources/netrias/attachments/netrias.json diff --git a/src/biome/adhoc_data/datasources/nf_synapse/api.yaml b/src/biome/adhoc_data/datasources/nf_synapse/api.yaml new file mode 100644 index 0000000..2883c4d --- /dev/null +++ b/src/biome/adhoc_data/datasources/nf_synapse/api.yaml @@ -0,0 +1,161 @@ +datatype: api +description: |- + The NF Synapse API is a RESTful API that allows you to interact with the Neurofibromatosis Data Portal. The code drafter has access both to the openapi spec and the web docs. +img_url: null +last_updated: null +name: NF Synapse +provider: adhoc:specialist_agents +resources: + 19e09360-7ad6-4d53-9d27-e5bf68bab700: + filepath: SynapseWebDocs.md + integration: 365c5121-5819-4492-b653-6ed0e85703de + name: SynapseWebDocs + resource_id: 19e09360-7ad6-4d53-9d27-e5bf68bab700 + resource_type: file + d92c04cb-0ece-4684-aebf-bce150e79d75: + filepath: SynapseOpenApiSpec.json + integration: 365c5121-5819-4492-b653-6ed0e85703de + name: SynapseOpenApiSpec + resource_id: d92c04cb-0ece-4684-aebf-bce150e79d75 + resource_type: file + feca1051-3952-4a0f-bddf-09191aa17f15: + code: |- + import os + import requests + import pandas as pd + + BASE_URL = "https://repo-prod.prod.sagebase.org" + AUTH_TOKEN = os.environ.get("API_SYNAPSE") + HEADERS = {"Authorization": f"Bearer {AUTH_TOKEN}"} + + search_terms = [ + "dementia", + "Alzheimer", + "cognitive decline", + "neurodegenerative", + "memory loss", + "Parkinson", + "Lewy body", + "frontotemporal", + "MCI", + "mild cognitive impairment" + ] + + results = [] + for term in search_terms: + payload = { + "queryTerm": [term], + "rangeQuery": [], + "facetOptions": [], + "returnFields": [], + "start": 0, + "size": 50 + } + resp = requests.post( + f"{BASE_URL}/repo/v1/search", + headers=HEADERS, + json=payload + ) + resp.raise_for_status() + data = resp.json() + for hit in data.get("hits", []): + results.append({ + "id": hit.get("id"), + "name": hit.get("name"), + "description": hit.get("description"), + "type": hit.get("node_type"), + "created_on": hit.get("created_on"), + "modified_on": hit.get("modified_on"), + "created_by": hit.get("created_by"), + "modified_by": hit.get("modified_by"), + }) + + df = pd.DataFrame(results).drop_duplicates(subset=["id"]) + # Filter for likely datasets, projects, or studies by type or name/description + mask = ( + df["type"].str.lower().isin(["project", "dataset", "study", "folder", "entityview", "submissionview", "materializedview", "table"]) + | df["name"].str.contains("study|dataset|project", case=False, na=False) + | df["description"].str.contains("study|dataset|project", case=False, na=False) + ) + filtered = df[mask] + + print(f"Found {len(filtered)} dementia-related studies/datasets in Synapse") + print("\nBreakdown by type:") + print(filtered['type'].value_counts()) + print("\nTop 10 results:") + print(filtered.head(10)[['id', 'name', 'type']].to_string()) + integration: 365c5121-5819-4492-b653-6ed0e85703de + notes: |- + This example demonstrates how to search Synapse for studies related to dementia + and neurodegenerative diseases. It uses multiple search terms to capture various + types of dementia research (Alzheimer's, Parkinson's, MCI, etc.) and filters results + to focus on actual studies, datasets, and projects rather than individual files. + The search found 229 relevant studies across different research areas including + biomarker discovery, genomics, proteomics, and clinical assessments. Key studies + include the Alzheimer's Disease Community Portal, multi-modal datasets for automatic + dementia detection, and various biomarker investigation projects. + query: Search for dementia studies in Synapse + resource_id: feca1051-3952-4a0f-bddf-09191aa17f15 + resource_type: example +slug: adhocspecificationintegration_nf_synapse +source: |- + # Additional Instructions: + + This API is through OpenAPI Spec. You have been provided the schema. + All requests go to the following URL: https://repo-prod.prod.sagebase.org + That is the base URL for all requests. + + To use the Synapse API always retrieve the auth header token from the environment variable: + - bearer token: os.environ.get("API_SYNAPSE") + Do not use placeholder auth values from the OpenAPI spec nor prompt the user for these values, just use the environment variables. + + Ensure you know the fields that come back from the OpenAPI spec in order to process the Synapse data. + + Never simulate this API- if it is not available ask the user how to proceed. + + You will mostly have to use the /repo/v1 and /file/v1 paths. + For example you can start search with: + POST /repo/v1/search + + and once you find a project or dataset you like, you can proceed to exploreother entities in + order to find a file / file handle and download it. + + An example file handle we found: syn4988822 with it, you can issue a download request. + You'll need to find the file ID first depending on your needs and the user query. + + You only have read/view/download assess/permissions- don't try to create or upload data- the openapi spec may have create/upload endpoints but do not use them. + + # OpenAPI Spec JSON: + ${SynapseOpenApiSpec} + + # More Web Docs: + ${SynapseWebDocs} + + # Example issuing a POST to search for `High throughput analysis of pNF cell lines` (omitting auth bearer token header; eg Authorization: Bearer ) + ``` + POST https://repo-prod.prod.sagebase.org/repo/v1/search + { + "queryTerm": [ + "Blakeley" + ], + "rangeQuery": [], + "facetOptions": [], + "returnFields": [], + "start": 0, + "size": "10" + } + ``` + + # About the NF synapse subdomain + + NF Data Portal + A home for Neurofibromatosis research resources + + Files are data collected from from human samples, animal models, and cell lines from a variety of assays. + Datasets are curated collections of files organized to facilitate data sharing. + Initiatives are funder-organized programs, groups, consortia, cohorts, or awards, usually focused on a specific research area in neurofibromatosis. + Studies are hypothesis-driven projects with the goal of uncovering new knowledge about neurofibromatosis type 1, type 2, or schwannomatosis. + Papers are pre-prints and peer-reviewed articles produced by NF Data Portal studies + Tools include animal models, cell lines, genetic reagents, antibodies, and biobanks related to NF! +url: null +uuid: 365c5121-5819-4492-b653-6ed0e85703de diff --git a/src/biome/datasources/synapse/documentation/SynapseOpenApiSpec.json b/src/biome/adhoc_data/datasources/nf_synapse/attachments/SynapseOpenApiSpec.json similarity index 99% rename from src/biome/datasources/synapse/documentation/SynapseOpenApiSpec.json rename to src/biome/adhoc_data/datasources/nf_synapse/attachments/SynapseOpenApiSpec.json index 10e915b..96aea74 100644 --- a/src/biome/datasources/synapse/documentation/SynapseOpenApiSpec.json +++ b/src/biome/adhoc_data/datasources/nf_synapse/attachments/SynapseOpenApiSpec.json @@ -84,7 +84,7 @@ }, { "name": "Evaluation Services", - "description": "

\n The Evaluation API is designed to support open-access data analysis and modeling challenges in\n Synapse. This framework provides tools for administrators to collect and analyze data models\n from Synapse users created for a specific goal or purpose.\n <\/p>\n \n

\n The data model of the Evaluation API is built around around two primary objects:\n