From dbc4ec799bd3b2de01126d0005f032ce0ee9011d Mon Sep 17 00:00:00 2001 From: Satchel Baldwin Date: Fri, 16 May 2025 17:37:05 -0500 Subject: [PATCH 01/21] change tool names and update prompts and send examples to beaker --- src/biome/agent.py | 63 ++++++++++--------- src/biome/context.py | 10 ++- src/biome/prompts/agent_prompt.md | 20 +++--- src/biome/prompts/consult_api_docs.md | 15 ----- src/biome/prompts/consult_integration_docs.md | 15 +++++ ..._api_code.md => draft_integration_code.md} | 30 ++++----- 6 files changed, 82 insertions(+), 71 deletions(-) delete mode 100644 src/biome/prompts/consult_api_docs.md create mode 100644 src/biome/prompts/consult_integration_docs.md rename src/biome/prompts/{draft_api_code.md => draft_integration_code.md} (68%) diff --git a/src/biome/agent.py b/src/biome/agent.py index cabc3a8..4913a3a 100644 --- a/src/biome/agent.py +++ b/src/biome/agent.py @@ -57,8 +57,8 @@ def ignore_tags(loader, tag_suffix, node): yaml.add_multi_constructor('', ignore_tags, yaml.SafeLoader) -DRAFT_API_CODE_DOC = load_docstring('draft_api_code.md') -CONSULT_API_DOCS_DOC = load_docstring('consult_api_docs.md') +DRAFT_API_CODE_DOC = load_docstring('draft_integration_code.md') +CONSULT_API_DOCS_DOC = load_docstring('consult_integration_docs.md') class BiomeAgent(BeakerAgent): """ @@ -121,11 +121,6 @@ def fetch_specs(self): api_spec = load_yaml_api(api_yaml) raw_contents = api_yaml.read_text() raw_spec = yaml.safe_load(raw_contents) - try: - ensure_name_slug_compatibility(raw_spec) - self.raw_specs.append((os.path.join(datasource_dir, 'api.yaml'), raw_spec)) - except Exception as e: - logger.error(f"Failed to load datasource `{api_yaml}` from raw yaml: {e}") # Replace {DATASET_FILES_BASE_PATH} with data_dir path; { and {{ to reduce mental overhead api_spec['documentation'] = api_spec['documentation'].replace('{DATASET_FILES_BASE_PATH}', str(data_dir)) @@ -137,6 +132,14 @@ def fetch_specs(self): example['code'] = example['code'].replace('{{DATASET_FILES_BASE_PATH}}', str(data_dir)) example['code'] = example['code'].replace('{DATASET_FILES_BASE_PATH}', str(data_dir)) + try: + ensure_name_slug_compatibility(raw_spec) + # add the loaded examples in too, since we want that tag parsed but also the raw text as well + raw_spec['loaded_examples'] = api_spec.get('examples', []) + self.raw_specs.append((os.path.join(datasource_dir, 'api.yaml'), raw_spec)) + except Exception as e: + logger.error(f"Failed to load datasource `{api_yaml}` from raw yaml: {e}") + ensure_name_slug_compatibility(api_spec) self.api_specs.append(api_spec) self.api_directories[api_spec['slug']] = datasource_dir @@ -171,11 +174,11 @@ def log(self, event_type: str, content = None) -> None: ) @tool() - @with_docstring('draft_api_code.md') - async def draft_api_code(self, api: str, goal: str, agent: AgentRef, loop: LoopControllerRef, react_context: ReactContextRef) -> str: - logger.info(f"using api: {api}") + @with_docstring('draft_integration_code.md') + async def draft_integration_code(self, integration: str, goal: str, agent: AgentRef, loop: LoopControllerRef, react_context: ReactContextRef) -> str: + logger.info(f"using integration: {integration}") try: - code = self.api.use_api(api, goal) + code = self.api.use_api(integration, goal) return f"Here is the code the drafter created to use the API to accomplish the goal: \n\n```\n{code}\n```" except Exception as e: if self.api is None: @@ -184,18 +187,18 @@ async def draft_api_code(self, api: str, goal: str, agent: AgentRef, loop: LoopC return f"An error occurred while using the API. The error was: {str(e)}. Please try again with a different goal." @tool() - @with_docstring('consult_api_docs.md') - async def consult_api_docs(self, api: str, query: str, agent: AgentRef, loop: LoopControllerRef, react_context: ReactContextRef) -> str: - logger.info(f"asking api: {api}") + @with_docstring('consult_integration_docs.md') + async def consult_integration_docs(self, integration: str, query: str, agent: AgentRef, loop: LoopControllerRef, react_context: ReactContextRef) -> str: + logger.info(f"asking integration: {integration}") try: - results = self.api.ask_api(api, query) + results = self.api.ask_api(integration, query) return f"Here is the information I found about how to use the API: \n{results}" except Exception as e: if self.api is None: return "Do not attempt to fix this result: there is no API for the agent that creates the request. Inform the user that they need to specify GEMINI_API_KEY and consider this a successful tool invocation." logger.error(str(e)) return f"An error occurred while asking the API. The error was: {str(e)}. Please try again with a different question." - + @tool() async def drs_uri_info(self, uris: List[str]) -> List[dict]: @@ -206,7 +209,7 @@ async def drs_uri_info(self, uris: List[str]) -> List[dict]: Args: uris (list): A list of DRS URIs to get information about. URIs should be of the form 'drs://:'. - + Returns: list: The information from looking up each DRS URI. """ @@ -220,7 +223,7 @@ async def drs_uri_info(self, uris: List[str]) -> List[dict]: object_id = uri.split(":")[-1] except IndexError: raise ValueError("Invalid DRS URI: Missing object ID") - + # Get information about the object from the DRS server url = f"https://nci-crdc.datacommons.io/ga4gh/drs/v1/objects/{object_id}" response = requests.get(url) @@ -230,7 +233,7 @@ async def drs_uri_info(self, uris: List[str]) -> List[dict]: responses.append(response.json()) return responses - + @tool() async def add_example(self, api: str, code: str, query: str, notes: str = None) -> str: """ @@ -250,23 +253,23 @@ async def add_example(self, api: str, code: str, query: str, notes: str = None) """ if api not in self.api_list: raise ValueError(f"Error: the API name must match one of the names in the {self.api_list}. The API name provided was {api}.") - + try: api_folder = self.api_directories[api] # Construct path to examples.yaml file examples_path = os.path.join(self.root_folder, DATASOURCES_FOLDER, api_folder, "documentation", "examples.yaml") os.makedirs(os.path.dirname(examples_path), exist_ok=True) - + # Create new example entry as a dictionary new_example = { "query": query, "code": code # Will be formatted with block scalar style } - + # Add notes if provided if notes: new_example["notes"] = notes - + # Read existing examples if file exists examples = [] if os.path.exists(examples_path) and os.path.getsize(examples_path) > 0: @@ -275,28 +278,28 @@ async def add_example(self, api: str, code: str, query: str, notes: str = None) examples = yaml.safe_load(f) or [] if not isinstance(examples, list): examples = [] - + # Add new example examples.append(new_example) - + # Write updated examples back to file import yaml - + # Custom YAML dumper class that always uses block style for multiline strings class BlockStyleDumper(yaml.SafeDumper): pass - + # Always use block style (|) for strings with newlines def represent_str_as_block(dumper, data): if '\n' in data: return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') return dumper.represent_scalar('tag:yaml.org,2002:str', data) - + BlockStyleDumper.add_representer(str, represent_str_as_block) - + with open(examples_path, 'w') as f: yaml.dump(examples, f, Dumper=BlockStyleDumper, sort_keys=False, default_flow_style=False) - + return f"Successfully added example to {examples_path}" except Exception as e: diff --git a/src/biome/context.py b/src/biome/context.py index 1f1a081..a5aa639 100644 --- a/src/biome/context.py +++ b/src/biome/context.py @@ -66,7 +66,8 @@ async def get_datasources(self) -> list[Datasource]: "cache_key", "documentation", "examples", - "cache_body" + "cache_body", + "loaded_examples" ] ]: if not isinstance(spec[attachment_key], str): @@ -88,6 +89,9 @@ async def get_datasources(self) -> list[Datasource]: is_empty_file=False )) + # manually load examples + + return [ Datasource( slug=spec['slug'], @@ -95,7 +99,8 @@ async def get_datasources(self) -> list[Datasource]: name=spec['name'], description=spec.get('description'), source=spec.get('documentation').replace('!fill', ''), - attached_files=attached_files[spec['name']] + attached_files=attached_files[spec['name']], + examples=spec.get("loaded_examples", []) ) for (yaml_location, spec) in self.agent.raw_specs ] @@ -122,3 +127,4 @@ async def save_datasource(self, message): self.agent.fetch_specs() self.agent.initialize_adhoc() self.agent.add_context(f"A new datasource has been added: `{slug}`. You may now use this with `draft_api_code`.") + diff --git a/src/biome/prompts/agent_prompt.md b/src/biome/prompts/agent_prompt.md index 55a77a2..0b1f03c 100644 --- a/src/biome/prompts/agent_prompt.md +++ b/src/biome/prompts/agent_prompt.md @@ -2,13 +2,13 @@ You are Biome, a chat assistant that helps the analyst user with their questions You have the ability to look up information regarding the environment via the tools that are provided. You should use these tools whenever are not able to satisfy the request to a high level of reliability. You should avoid guessing at how to do something in favor of using the provided tools to look up more information. Do not make assumptions, always check the documentation instead of assuming. -You must be extremely careful about what you print to stdout when running code--be cognizant of the size and make sure you don't print out huge things! Never, ever just print out the entire result of a workload or API search result. Always slice it and show just a subset to the user. You can save it as a variable for later use, etc. +You must be extremely careful about what you print to stdout when running code--be cognizant of the size and make sure you don't print out huge things! Never, ever just print out the entire result of a workload or API search result. Always slice it and show just a subset to the user. You can save it as a variable for later use, etc. You will have access to external integrations, which are things like: APIs, datasets, or other external sources. An integration can be thought of as something that is an external source you can work with. An API is a type of integration you have access to. -When you use a tool you must ALWAYS indicate which tool you are using by explicitly stating the tool name in your thinking. Wrap the tool name in backticks. You do not need to include the class name, just the method. For example you should not indicate `BiomeAgent.draft_api_code` but rather just `draft_api_code`. +When you use a tool you must ALWAYS indicate which tool you are using by explicitly stating the tool name in your thinking. Wrap the tool name in backticks. You do not need to include the class name, just the method. For example you should not indicate `BiomeAgent.draft_integration_code` but rather just `draft_integration_code`. -A very common workflow is to use the `draft_api_code` tool to get code to interact with an API. Once you have the code, you MUST use the `run_code` tool to execute the code otherwise the code will NOT be run. If you work out something tricky on behalf of the user, let's capture your success: you should ask the user if they would like to use the `add_example` tool to add the code as an example to the API's documentation. Never add examples without being asked to do so or without the user's confirmation. +A very common workflow is to use the `draft_integration_code` tool to get code to interact with an API. Once you have the code, you MUST use the `run_code` tool to execute the code otherwise the code will NOT be run. If you work out something tricky on behalf of the user, let's capture your success: you should ask the user if they would like to use the `add_example` tool to add the code as an example to the integration's documentation. Never add examples without being asked to do so or without the user's confirmation. -You will often be asked to integrate information from multiple sources to answer a question. For example, you may be asked to find a dataset from one API and integrate it with information from another API. In this case, you should be explicit about the steps needed to accomplish the task and where the information from each API is used. When you summarize your findings or results you should be clear about which information came from which API. +You will often be asked to integrate information from multiple sources to answer a question. For example, you may be asked to find a dataset from one API and integrate it with information from another API. In this case, you should be explicit about the steps needed to accomplish the task and where the information from each API is used. When you summarize your findings or results you should be clear about which information came from which API. When using `run_code` and in general you must NEVER print out the entire result of a workload or API search result. Always slice it and show just a subset to the user. You can save it as a variable for later use, etc. Be extremely CAREFUL about this. If you need to print something, be extremely CAREFUL--don't print the whole thing! You must ALWAYS indicate which tool you are using by explicitly stating the tool name in your thinking. Wrap the tool name in backticks. @@ -16,16 +16,18 @@ When using `run_code` you may execute code that has an error. Sometimes the code Users may ask you to load and munge data and many other tasks. Try to identify all of the steps needed, and all of the tools, but do not assume the user wants to do all of the steps at once. -You should NEVER use `draft_api_code` without then using the `run_code` tool to run the returned code UNLESS the user explicitly asks you to generate code but NOT run it. Otherwise, you MUST use these two tools in tandem. +You should NEVER use `draft_integration_code` without then using the `run_code` tool to run the returned code UNLESS the user explicitly asks you to generate code but NOT run it. Otherwise, you MUST use these two tools in tandem. -Importantly, you have special tooling for a set of core Biome APIs. The APIs that you have specialized tooling for are: +Importantly, you have special tooling for a set of core Biome integrations. The integrations that you have specialized tooling for are: ``` {api_list} ``` -For these APIs you should utilize the `draft_api_code` tool before using the `run_code` tool to ensure that you obtain expert level information about how to interact with the API. +For these APIs you should utilize the `draft_integration_code` tool before using the `run_code` tool to ensure that you obtain expert level information about how to interact with the integration. -However, you can utilize other APIs as well, you just CANNOT use the `draft_api_code` tool to interact with them. For example, you can use the `run_code` tool to interact with other APIs by writing your own code to do so, e.g. using the `requests` library or using Biopython, etc. For certain queries that you can make via Biopython or simply using requests to Entrez, NCBI's database. Here are some specific instructions and examples of how to do this: +CRITICAL: If the integration or API or source has a codebook to explain the meaning of variables and other data, if you are asked about features or variables or their data, you MUST run code to look up the codebook to explain the variable or feature. + +However, you can utilize other APIs as well, you just CANNOT use the `draft_integration_code` tool to interact with them. For example, you can use the `run_code` tool to interact with other APIs by writing your own code to do so, e.g. using the `requests` library or using Biopython, etc. For certain queries that you can make via Biopython or simply using requests to Entrez, NCBI's database. Here are some specific instructions and examples of how to do this: ``` {instructions} @@ -35,4 +37,4 @@ When responding to user queries where those instructions are relevant, you shoul When you are asked to generate plots or charts, you should use `seaborn` wherever possible and should think about how to make your plots aesthetically pleasing, readable, and informative. -You are running code inside a Jupyter notebook. You may need to use the `display` function to show your plots. If you need to install a library ask the user if it is okay to install it, then you can do so using the `run_code` tool and the `!pip install` command. \ No newline at end of file +You are running code inside a Jupyter notebook. You may need to use the `display` function to show your plots. If you need to install a library ask the user if it is okay to install it, then you can do so using the `run_code` tool and the `!pip install` command. diff --git a/src/biome/prompts/consult_api_docs.md b/src/biome/prompts/consult_api_docs.md deleted file mode 100644 index 6a073e0..0000000 --- a/src/biome/prompts/consult_api_docs.md +++ /dev/null @@ -1,15 +0,0 @@ -This tool is used to ask a question of an API. It allows you to ask a question of an API's documentation and get results in -natural language, which may include code snippets or other information that you can then use to write code to interact with the API -(e.g. using the `BiomeAgent__run_code` tool). You can also use this information to refine your goal when using the `draft_api_code` tool. - -You can ask questions related to endpoints, payloads, etc. For example, you can ask "What are the endpoints for this API?" -or "What payload do I need to send to this API?" or "How do I query for datasets by keyword?" etc, etc. - -If you use this tool, you MUST indicate so in your thinking. Wrap the tool name in backticks. - -Args: - api (str): The name of the API to use - query (str): The question you want to ask about the API. - -Returns: - str: returns instructions on how to utilize the API based on the question asked. \ No newline at end of file diff --git a/src/biome/prompts/consult_integration_docs.md b/src/biome/prompts/consult_integration_docs.md new file mode 100644 index 0000000..49f93ff --- /dev/null +++ b/src/biome/prompts/consult_integration_docs.md @@ -0,0 +1,15 @@ +This tool is used to ask a question of an an external integration. It allows you to ask a question of an integration's documentation and get results in +natural language, which may include code snippets or other information that you can then use to write code to interact with the API +(e.g. using the `BiomeAgent__run_code` tool). You can also use this information to refine your goal when using the `draft_api_code` tool. + +You can ask questions related to endpoints, payloads, etc. For example, you can ask "What are the endpoints for this integration?" +or "What payload do I need to send to this integration?" or "How do I query for datasets by keyword?" etc, etc. + +If you use this tool, you MUST indicate so in your thinking. Wrap the tool name in backticks. + +Args: + integration (str): The name of the integration to use + query (str): The question you want to ask about the integration. + +Returns: + str: returns instructions on how to utilize the integration based on the question asked. diff --git a/src/biome/prompts/draft_api_code.md b/src/biome/prompts/draft_integration_code.md similarity index 68% rename from src/biome/prompts/draft_api_code.md rename to src/biome/prompts/draft_integration_code.md index c1522d1..093dc44 100644 --- a/src/biome/prompts/draft_api_code.md +++ b/src/biome/prompts/draft_integration_code.md @@ -1,13 +1,13 @@ -Drafts python code for an API request given a specified goal, such as a query for a specific study. You can use this tool to -get code to interact with the APIs that are available to you. When the user asks you to use an API and you are unsure how to do so, you should -be sure to use this tool. Once you've learned how to do common tasks with an API you may not need this tool, but for accomplishing -new tasks, you should use this tool. - -The way this tool functions is that it will provide the goal to an external agent, which we refer to as the "drafter". -The drafter has access to the API documentation for the API in question. The drafter will then generate the code to perform the desired operation. -However, the drafter requires a very specific goal in order to do their job and does not have the ability to guess or infer. -Therefore, you must provide a very specific goal. It also does not have awareness of information outside of what you provide in the goal. -Therefore, if you have run code previously that returned information such as names of studies, `ids` of datasets, etc, you must provide that +Drafts python code for an integration request given a specified goal, such as a query for a specific study. You can use this tool to +get code to interact with the integrations that are available to you. When the user asks you to use an integration and you are unsure how to do so, you should +be sure to use this tool. Once you've learned how to do common tasks with an integration you may not need this tool, but for accomplishing +new tasks, you should use this tool. + +The way this tool functions is that it will provide the goal to an external agent, which we refer to as the "drafter". +The drafter has access to the integration's documentation for the integration in question. The drafter will then generate the code to perform the desired operation. +However, the drafter requires a very specific goal in order to do their job and does not have the ability to guess or infer. +Therefore, you must provide a very specific goal. It also does not have awareness of information outside of what you provide in the goal. +Therefore, if you have run code previously that returned information such as names of studies, `ids` of datasets, etc, you must provide that information in the goal if it is needed to perform the desired operation. If you are asked to query for something specific, e.g. a study, you MUST provide the relevant `id` as part of the goal if you have access to it. @@ -19,20 +19,20 @@ VERBOSE goals so that the drafter of the code has all the information needed to You must never reference code that has already been run since the drafter does not have awareness of that information. For example, don't set as the goal: "Run the previous code but modify it to do X"; you must provide that code in the goal if you want it to be utilized. -The code that is drafted will generally be a complete, small program including relevant imports and other boilerplate code. Much of this +The code that is drafted will generally be a complete, small program including relevant imports and other boilerplate code. Much of this may already be implemented in the code you have run previously; if that is the case you should not repeat it. Feel free to streamline the code generated by removing any unnecessary steps before sending it to the `BiomeAgent__run_code` tool. -If you use this tool, you MUST indicate so in your thinking. Wrap the tool name in backticks. +If you use this tool, you MUST indicate so in your thinking. Wrap the tool name in backticks. You MUST also be explicit about the goal in your thinking. Args: - api (str): The name of the API to use - goal (str): The task to be performed by the API request. This should be as specific as possible and include any relevant details such as the `ids` or other parameters that should be used to get the desired result. + integration (str): The name of the integration to use + goal (str): The task to be performed by the integration request. This should be as specific as possible and include any relevant details such as the `ids` or other parameters that should be used to get the desired result. Returns: str: Depending on the user defined configuration will do one of two things. Either A) return the raw generated code. Or B) Will attempt to run the code and return the result or any errors that occurred (along with the original code). if an error is returned, you may consider - trying to fix the code yourself rather than reusing the tool. \ No newline at end of file + trying to fix the code yourself rather than reusing the tool. From f59f13da953bc4edfaaeefedbadd0447bdd301f9 Mon Sep 17 00:00:00 2001 From: Satchel Baldwin Date: Tue, 27 May 2025 17:54:54 -0500 Subject: [PATCH 02/21] add add_integration example and make current with beaker, change api to integration --- src/biome/agent.py | 191 +++++++++++++------------------------------ src/biome/context.py | 29 +++---- 2 files changed, 69 insertions(+), 151 deletions(-) diff --git a/src/biome/agent.py b/src/biome/agent.py index 4913a3a..cf61f97 100644 --- a/src/biome/agent.py +++ b/src/biome/agent.py @@ -24,7 +24,7 @@ JSON_OUTPUT = False -DATASOURCES_FOLDER = "datasources" +INTEGRATIONS_FOLDER = "datasources" class MessageLogger(): def __init__(self, agent_log_function, print_logger): @@ -57,16 +57,16 @@ def ignore_tags(loader, tag_suffix, node): yaml.add_multi_constructor('', ignore_tags, yaml.SafeLoader) -DRAFT_API_CODE_DOC = load_docstring('draft_integration_code.md') -CONSULT_API_DOCS_DOC = load_docstring('consult_integration_docs.md') +DRAFT_INTEGRATION_CODE_DOC = load_docstring('draft_integration_code.md') +CONSULT_INTEGRATIONS_DOCS_DOC = load_docstring('consult_integration_docs.md') class BiomeAgent(BeakerAgent): """ You are the Biome Agent, a chat assistant that helps users with biomedical research tasks. - A 'datasource' is defined as an API or dataset or general collection of knowledge that you have access to. + An 'integration' is defined as an API or dataset or general collection of knowledge that you have access to. - An API should be considered a type of datasource. + An API should be considered a type of integration. """ def __init__(self, context: BaseContext = None, tools: list = None, **kwargs): logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO')) @@ -88,11 +88,11 @@ def __init__(self, context: BaseContext = None, tools: list = None, **kwargs): prompts_dir = os.path.join(self.root_folder, 'prompts') with open(os.path.join(prompts_dir, 'agent_prompt.md'), 'r') as f: template = f.read() - self.__doc__ = template.format(api_list=self.api_list, instructions=self.instructions) + self.__doc__ = template.format(api_list=self.integration_list, instructions=self.instructions) self.add_context(self.__doc__) def fetch_specs(self): - datasource_root = os.path.join(self.root_folder, DATASOURCES_FOLDER) + integration_root = os.path.join(self.root_folder, INTEGRATIONS_FOLDER) data_dir_raw = os.environ.get("BIOME_DATA_DIR", "./data") try: @@ -104,22 +104,22 @@ def fetch_specs(self): self.data_dir = data_dir # Get API specs and directories in one pass - self.api_specs = [] + self.integration_specs = [] self.raw_specs = [] # no interpolation - self.api_directories = {} - for datasource_dir in os.listdir(datasource_root): - if datasource_dir == '.ipynb_checkpoints': - os.rmdir(os.path.join(datasource_root, datasource_dir)) - - datasource_full_path = os.path.join(datasource_root, datasource_dir) - if os.path.isdir(datasource_full_path): - api_yaml = Path(os.path.join(datasource_full_path, 'api.yaml')) - if not api_yaml.is_file(): - logger.warning(f"Ignoring malformed API: {api_yaml}") + self.integration_directories = {} + for integration_dir in os.listdir(integration_root): + if integration_dir == '.ipynb_checkpoints': + os.rmdir(os.path.join(integration_root, integration_dir)) + + integration_full_path = os.path.join(integration_root, integration_dir) + if os.path.isdir(integration_full_path): + integration_yaml = Path(os.path.join(integration_full_path, 'api.yaml')) + if not integration_yaml.is_file(): + logger.warning(f"Ignoring malformed API: {integration_yaml}") continue - api_spec = load_yaml_api(api_yaml) - raw_contents = api_yaml.read_text() + api_spec = load_yaml_api(integration_yaml) + raw_contents = integration_yaml.read_text() raw_spec = yaml.safe_load(raw_contents) # Replace {DATASET_FILES_BASE_PATH} with data_dir path; { and {{ to reduce mental overhead @@ -136,13 +136,13 @@ def fetch_specs(self): ensure_name_slug_compatibility(raw_spec) # add the loaded examples in too, since we want that tag parsed but also the raw text as well raw_spec['loaded_examples'] = api_spec.get('examples', []) - self.raw_specs.append((os.path.join(datasource_dir, 'api.yaml'), raw_spec)) + self.raw_specs.append((os.path.join(integration_dir, 'api.yaml'), raw_spec)) except Exception as e: - logger.error(f"Failed to load datasource `{api_yaml}` from raw yaml: {e}") + logger.error(f"Failed to load integration `{integration_yaml}` from raw yaml: {e}") ensure_name_slug_compatibility(api_spec) - self.api_specs.append(api_spec) - self.api_directories[api_spec['slug']] = datasource_dir + self.integration_specs.append(api_spec) + self.integration_directories[api_spec['slug']] = integration_dir def initialize_adhoc(self): # Note: not all providers support ttl_seconds @@ -151,7 +151,7 @@ def initialize_adhoc(self): drafter_config_anthropic = {**claude_37_sonnet, 'api_key': os.environ.get("ANTHROPIC_API_KEY")} curator_config = {**o3_mini, 'api_key': os.environ.get("OPENAI_API_KEY")} gpt_41_config = {**gpt_41, 'api_key': os.environ.get("OPENAI_API_KEY")} - specs = self.api_specs + specs = self.integration_specs try: self.api = AdhocApi( @@ -165,7 +165,7 @@ def initialize_adhoc(self): self.add_context(f"The datasources failed to load for this reason: {str(e)}. Please inform the user immediately.") self.api = None - self.api_list = [spec['slug'] for spec in specs] + self.integration_list = [spec['slug'] for spec in specs] def log(self, event_type: str, content = None) -> None: self.context.beaker_kernel.log( @@ -235,15 +235,15 @@ async def drs_uri_info(self, uris: List[str]) -> List[dict]: return responses @tool() - async def add_example(self, api: str, code: str, query: str, notes: str = None) -> str: + async def add_example(self, integration: str, code: str, query: str, notes: str = None) -> str: """ - Add a successful code example to the datasource's examples.yaml documentation file. - This tool should be used after successfully completing a task with an datasource to capture the working code for future reference. + Add a successful code example to the integration's examples.yaml documentation file. + This tool should be used after successfully completing a task with an integration to capture the working code for future reference. - The API names must match one of the names in the agent's datasource list. + The API names must match one of the names in the agent's integration list. Args: - api (str): The name of the datasource the example is for + integration (str): The name of the integration the example is for code (str): The working, successful code to add as an example query (str): A brief description of what the example demonstrates notes (str, optional): Additional notes about the example, such as implementation details @@ -251,87 +251,38 @@ async def add_example(self, api: str, code: str, query: str, notes: str = None) Returns: str: Message indicating success or failure of adding the example """ - if api not in self.api_list: - raise ValueError(f"Error: the API name must match one of the names in the {self.api_list}. The API name provided was {api}.") + if integration not in self.integration_list: + raise ValueError(f"Error: the API name must match one of the names in the {self.integration_list}. The API name provided was {api}.") - try: - api_folder = self.api_directories[api] - # Construct path to examples.yaml file - examples_path = os.path.join(self.root_folder, DATASOURCES_FOLDER, api_folder, "documentation", "examples.yaml") - os.makedirs(os.path.dirname(examples_path), exist_ok=True) - - # Create new example entry as a dictionary - new_example = { + self.context.beaker_kernel.send_response( + "iopub", "add_example", content={ + "integration": integration, + "code": code, "query": query, - "code": code # Will be formatted with block scalar style + "notes": notes } - - # Add notes if provided - if notes: - new_example["notes"] = notes - - # Read existing examples if file exists - examples = [] - if os.path.exists(examples_path) and os.path.getsize(examples_path) > 0: - import yaml - with open(examples_path, 'r') as f: - examples = yaml.safe_load(f) or [] - if not isinstance(examples, list): - examples = [] - - # Add new example - examples.append(new_example) - - # Write updated examples back to file - import yaml - - # Custom YAML dumper class that always uses block style for multiline strings - class BlockStyleDumper(yaml.SafeDumper): - pass - - # Always use block style (|) for strings with newlines - def represent_str_as_block(dumper, data): - if '\n' in data: - return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') - return dumper.represent_scalar('tag:yaml.org,2002:str', data) - - BlockStyleDumper.add_representer(str, represent_str_as_block) - - with open(examples_path, 'w') as f: - yaml.dump(examples, f, Dumper=BlockStyleDumper, sort_keys=False, default_flow_style=False) - - return f"Successfully added example to {examples_path}" - - except Exception as e: - self.logger.error(str(e)) - raise ValueError(f"Failed to add example: {str(e)}") - + ) + return "Successfully added example." @tool() - async def add_datasource(self, - datasource: str, + async def add_integration(self, + integration: str, description: str, base_url: str, schema_location: str) -> str: """ - Adds a datasource to the list of supported datasources usable within Biome. + Adds an integration to the list of supported integrations usable within Biome. This will be added to the API and data source list. Args: - datasource (str): The name of the target data source or API that will be added. + integration (str): The name of the target data source or API that will be added. description (str): A plain text description of what the data source is based on your knowledge of what the user is asking for, combined with their description if their description is relevant, or, if you do not know about the target data source. If the user does not provide any information, rely on what you know. Target a paragraph in length. schema_location (str): A URL or local filepath to fetch an OpenAPI schema from. If the user does not provide one, ask them for the URL or local filepath to the schema. base_url (str): The base URL for the datasource that will be used for making OpenAPI calls. If the user does not provide one, ask them for the base URL of the API. Returns: - str: Message indicating success or failure of adding the datasource. + str: Message indicating success or failure of adding the integration. """ - datasources_path = os.path.join(self.root_folder, DATASOURCES_FOLDER) - datasource_name = datasource.lower().replace(' ', '_') - datasource_folder = os.path.join(datasources_path, datasource_name) - if (os.path.isdir(datasource_folder)): - return f"Failed to add datasource: {datasource}: {DATASOURCES_FOLDER}/{datasource_folder} already exists." - logger.info(f'adding datasource {datasource} to {datasource_folder}') try: if schema_location.startswith('http'): response = requests.get(schema_location) @@ -341,53 +292,29 @@ async def add_datasource(self, else: with open(schema_location, 'r') as f: schema = f.read() - - documentation_folder = os.path.join(datasource_folder, 'documentation') - os.makedirs(documentation_folder, exist_ok=True) - with open(os.path.join(documentation_folder, 'schema.yaml'), 'w') as f: - f.write(str(schema)) - with open(os.path.join(documentation_folder, 'examples.yaml'), 'a'): - pass - except Exception as e: return f'Failed to get OpenAPI schema: {e}' - formatted_description = description.replace('\n', ' ') - template = f""" -name: {datasource} -description: {formatted_description} -examples: !load_yaml documentation/examples.yaml -cache_key: "api_assistant_{datasource_name}" -raw_documentation: !load_txt documentation/schema.yaml - -documentation: !fill | - The base URL for the service is `{base_url}` - Below is the OpenAPI schema for the desired service. - - {{raw_documentation}} -""" - with open(os.path.join(datasource_folder, 'api.yaml'), 'w') as f: - f.write(template) - try: - datasource_yaml = Path(os.path.join(datasource_folder, 'api.yaml')) - datasource_spec = load_yaml_api(datasource_yaml) - self.api_specs.append(datasource_spec) - self.api.add_api(datasource_spec) - - except Exception as e: - return f"Failed to add datasource: {e}" - logger.info(f'Successfully added {datasource} to {datasource_folder} and it is now available for use.') - return f'Successfully added {datasource} and it is now available for use.' + # calls save_integration in context.py as an action after finishing + self.context.beaker_kernel.send_response( + "iopub", "add_integration", content={ + "integration": integration, + "description": description, + "base_url": base_url, + "schema": schema + } + ) + return f"Added integration `{integration}`." @tool() - async def get_available_datasources(self) -> list: + async def get_available_integrations(self) -> list: """ - Get list of datasources that the agent is designed to interact with. + Get list of integrations that the agent is designed to interact with. Returns: - list: The list of available datasources. + list: The list of available integrations. """ - return self.api_list + return self.integration_list @tool() diff --git a/src/biome/context.py b/src/biome/context.py index a5aa639..56349bc 100644 --- a/src/biome/context.py +++ b/src/biome/context.py @@ -10,7 +10,7 @@ from beaker_kernel.subkernels.python import PythonSubkernel from beaker_kernel.lib.types import Datasource, DatasourceAttachment -from .agent import DATASOURCES_FOLDER, BiomeAgent +from .agent import BiomeAgent if TYPE_CHECKING: from beaker_kernel.kernel import LLMKernel @@ -105,26 +105,17 @@ async def get_datasources(self) -> list[Datasource]: for (yaml_location, spec) in self.agent.raw_specs ] - @action(action_name="save_datasource") - async def save_datasource(self, message): + @action(action_name="save_integration") + async def save_integration(self, message): content = message.content + self.agent.fetch_specs() + self.agent.initialize_adhoc() + self.agent.add_context(f"A new integration has been added: `{content.get('slug')}`. You may now use this with `draft_integration_code`.") - datasource = Datasource( - name=content.get('name'), - slug=content.get('slug'), - url=content.get('url'), - description=content.get('description'), - source=content.get('source'), - attached_files=[ - DatasourceAttachment( - name=payload['name'], - filepath=payload['filepath'] - ) - for payload in content.get('attached_files')] - ) - - slug = datasource.slug + @action(action_name="add_example") + async def add_example(self, message): + content = message.content self.agent.fetch_specs() self.agent.initialize_adhoc() - self.agent.add_context(f"A new datasource has been added: `{slug}`. You may now use this with `draft_api_code`.") + self.agent.add_context(f"A new example has been added to `{content.get('slug')}.`") From 26a021e2f8c1fc7cbba05b24956b56af8c26efaf Mon Sep 17 00:00:00 2001 From: Satchel Baldwin Date: Fri, 6 Jun 2025 11:59:57 -0500 Subject: [PATCH 03/21] add integration yaml handling with FileContentsManager --- src/biome/context.py | 34 +++++---- src/biome/integration.py | 144 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+), 12 deletions(-) create mode 100644 src/biome/integration.py diff --git a/src/biome/context.py b/src/biome/context.py index 56349bc..64548cb 100644 --- a/src/biome/context.py +++ b/src/biome/context.py @@ -1,8 +1,12 @@ -from typing import TYPE_CHECKING, Any, Dict, List +from typing import TYPE_CHECKING, Any, Dict import os -import json import re import logging +import json + +from jupyter_server.services.contents.filemanager import ( + FileContentsManager, +) from pathlib import Path @@ -11,11 +15,13 @@ from beaker_kernel.lib.types import Datasource, DatasourceAttachment from .agent import BiomeAgent +from .integration import write_integration if TYPE_CHECKING: from beaker_kernel.kernel import LLMKernel from beaker_kernel.lib.agent import BaseAgent + logger = logging.getLogger(__name__) @@ -57,7 +63,7 @@ async def get_datasources(self) -> list[Datasource]: # get list of keys not inherent to a datasource for the user-files category attached_files = {} for (_, spec) in self.agent.raw_specs: - attached_files[spec['name']] = [] + attached_files[spec["name"]] = [] for attachment_key in [ key for key in spec.keys() if key not in [ "name", @@ -77,12 +83,12 @@ async def get_datasources(self) -> list[Datasource]: # trim yaml tags since they will be readded at save time # TODO: handle not-eliding documentation/ filepath_raw = re.sub( - r'!load_[a-zA-Z]+', - '', + r"!load_[a-zA-Z]+", + "", spec[attachment_key].strip() - ).strip().replace('documentation/', '') + ).strip().replace("documentation/", "") - attached_files[spec['name']].append(DatasourceAttachment( + attached_files[spec["name"]].append(DatasourceAttachment( name=attachment_key, filepath=filepath_raw, content=None, @@ -94,12 +100,12 @@ async def get_datasources(self) -> list[Datasource]: return [ Datasource( - slug=spec['slug'], + slug=spec["slug"], url=str(yaml_location), - name=spec['name'], - description=spec.get('description'), - source=spec.get('documentation').replace('!fill', ''), - attached_files=attached_files[spec['name']], + name=spec["name"], + description=spec.get("description"), + source=spec.get("documentation").replace("!fill", ""), + attached_files=attached_files[spec["name"]], examples=spec.get("loaded_examples", []) ) for (yaml_location, spec) in self.agent.raw_specs @@ -107,14 +113,18 @@ async def get_datasources(self) -> list[Datasource]: @action(action_name="save_integration") async def save_integration(self, message): + manager = FileContentsManager() content = message.content + write_integration(manager, content) self.agent.fetch_specs() self.agent.initialize_adhoc() self.agent.add_context(f"A new integration has been added: `{content.get('slug')}`. You may now use this with `draft_integration_code`.") @action(action_name="add_example") async def add_example(self, message): + manager = FileContentsManager() content = message.content + write_integration(manager, content) self.agent.fetch_specs() self.agent.initialize_adhoc() self.agent.add_context(f"A new example has been added to `{content.get('slug')}.`") diff --git a/src/biome/integration.py b/src/biome/integration.py new file mode 100644 index 0000000..fde5db2 --- /dev/null +++ b/src/biome/integration.py @@ -0,0 +1,144 @@ +import yaml +from pathlib import Path +import os +import logging + +logger = logging.getLogger(__name__) + +def str_presenter(dumper, data): + if len(data.splitlines()) > 1: # check for multiline string + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + return dumper.represent_scalar("tag:yaml.org,2002:str", data) +yaml.add_representer(str, str_presenter) +yaml.representer.SafeRepresenter.add_representer(str, str_presenter) + +# add dumpers but not loaders, since we need to get raw specifications elsewhere to give to beaker + +class LoadYamlTag(yaml.YAMLObject): + yaml_tag = "!load_yaml" + def __init__(self, payload): + self.payload = payload + def __repr__(self): + return f"LoadYamlTag({self.payload})" + @classmethod + def from_yaml(cls, loader, node): + return LoadYamlTag(node.value) + @classmethod + def to_yaml(cls, dumper, data): + logger.warning(msg="load_yaml dump") + return dumper.represent_scalar(cls.yaml_tag, data.payload) + +# yaml.SafeLoader.add_constructor("!load_yaml", LoadYamlTag.from_yaml) +yaml.SafeDumper.add_multi_representer(LoadYamlTag, LoadYamlTag.to_yaml) + + +class LoadTextTag(yaml.YAMLObject): + yaml_tag = "!load_txt" + def __init__(self, payload): + self.payload = payload + def __repr__(self): + return f"LoadTextTag({self.payload})" + @classmethod + def from_yaml(cls, loader, node): + return LoadTextTag(node.value) + @classmethod + def to_yaml(cls, dumper, data): + return dumper.represent_scalar(cls.yaml_tag, data.payload) + +# yaml.SafeLoader.add_constructor("!load_txt", LoadTextTag.from_yaml) +yaml.SafeDumper.add_multi_representer(LoadTextTag, LoadTextTag.to_yaml) + + +class FillTag(yaml.YAMLObject): + yaml_tag = "!fill" + def __init__(self, payload): + self.payload = payload + def __repr__(self): + return f"FillTag({self.payload})" + @classmethod + def from_yaml(cls, loader, node): + return FillTag(node.value) + @classmethod + def to_yaml(cls, dumper, data): + return dumper.represent_scalar(cls.yaml_tag, data.payload, style="|") + +# yaml.SafeLoader.add_constructor("!fill", FillTag.from_yaml) +yaml.SafeDumper.add_multi_representer(FillTag, FillTag.to_yaml) + + +def create_file_model(content: str): + return { + "type": "file", + "format": "text", + "content": content + } + + +def create_directory_model(): + return { + "type": "directory", + "format": "json", + "mimetype": None + } + + +def get_integration_slug(integration): + return integration.get("slug", integration.get("name", "").lower().replace(" ", "_")) + + +def get_integration_folder(integration): + url: str = integration.get("url", "") + if url == "": + return get_integration_slug(integration) + elif url.endswith("api.yaml"): + return url[:-(len("api.yaml"))] + else: + return url + + +def get_integration_base_path(integration): + return (Path(os.environ.get("BIOME_INTEGRATIONS_DIR", "./")) + / get_integration_folder(integration)).resolve() + + +def create_folder_structure_for_integration(manager, integration): + base_path = get_integration_base_path(integration) + if not manager.dir_exists(str(base_path)): + manager.save(create_directory_model(), str(base_path)) + documentation_path = base_path / "documentation" + if not manager.dir_exists(str(documentation_path)): + manager.save(create_directory_model(), str(documentation_path)) + + +def format_integration(integration): + saved_fields = { + "name": integration.get("name"), + "slug": get_integration_slug(integration), + "cache_key": f"integration_{get_integration_slug(integration)}", + "examples": LoadYamlTag("documentation/examples.yaml"), + "description": integration.get("description"), + "documentation": FillTag(integration.get("source")) + } + for file in integration.get("attached_files", []): + print('a') + saved_fields[file.get("name")] = LoadTextTag( + f"documentation/{file.get('filepath')}" + ) + return yaml.safe_dump(saved_fields) + + +def write_integration(manager, integration): + base_path = get_integration_base_path(integration) + create_folder_structure_for_integration(manager, integration) + + example_path = base_path / "documentation" / "examples.yaml" + manager.save( + create_file_model(yaml.safe_dump(integration.get("examples", ""))), + str(example_path) + ) + + integration_yaml_path = base_path / "api.yaml" + manager.save( + create_file_model(format_integration(integration)), + str(integration_yaml_path) + ) From eb80574aa96ecb98bd5db5172fb139b87c064420 Mon Sep 17 00:00:00 2001 From: Satchel Baldwin Date: Fri, 6 Jun 2025 15:42:38 -0500 Subject: [PATCH 04/21] add fixes for frontend required actions --- src/biome/context.py | 25 ++++++++++++++++++++----- src/biome/integration.py | 10 +++++++++- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/biome/context.py b/src/biome/context.py index 64548cb..342cf78 100644 --- a/src/biome/context.py +++ b/src/biome/context.py @@ -15,7 +15,7 @@ from beaker_kernel.lib.types import Datasource, DatasourceAttachment from .agent import BiomeAgent -from .integration import write_integration +from .integration import create_folder_structure_for_integration, get_integration_folder, write_integration if TYPE_CHECKING: from beaker_kernel.kernel import LLMKernel @@ -52,9 +52,6 @@ async def setup(self, context_info=None, parent_header=None): }) await self.execute(command) - async def get_datasource_root(self) -> str: - return os.environ.get("BIOME_INTEGRATIONS_DIR", "") - async def get_datasources(self) -> list[Datasource]: """ fetch all of the adhoc-api datasources to pass to beaker. @@ -97,7 +94,6 @@ async def get_datasources(self) -> list[Datasource]: # manually load examples - return [ Datasource( slug=spec["slug"], @@ -111,6 +107,25 @@ async def get_datasources(self) -> list[Datasource]: for (yaml_location, spec) in self.agent.raw_specs ] + # frontend can request where to upload files to given a specific integration + @action(action_name="get_integration_root") + async def get_integration_root(self, message): + content = message.content + return str( + Path(os.environ.get("BIOME_INTEGRATIONS_DIR", "/")) + / get_integration_folder(content.get("integration")) + ) + + # handles a case of uploading a new file to a temporary, unsaved datasource + # that way the frontend can upload directly rather than sending it in an action message + @action(action_name="create_integration_folders_for_upload") + async def create_integration_folders_for_upload(self, message): + manager = FileContentsManager() + content = message.content + create_folder_structure_for_integration(manager, content.get("integration")) + return True + + # @action(action_name="save_integration") async def save_integration(self, message): manager = FileContentsManager() diff --git a/src/biome/integration.py b/src/biome/integration.py index fde5db2..bf4a862 100644 --- a/src/biome/integration.py +++ b/src/biome/integration.py @@ -120,7 +120,6 @@ def format_integration(integration): "documentation": FillTag(integration.get("source")) } for file in integration.get("attached_files", []): - print('a') saved_fields[file.get("name")] = LoadTextTag( f"documentation/{file.get('filepath')}" ) @@ -142,3 +141,12 @@ def write_integration(manager, integration): create_file_model(format_integration(integration)), str(integration_yaml_path) ) + + for file in integration.get("attached_files", []): + # files that are not empty content have yet to be saved -- uploading can be handled by frontend. + # so if a tool creates a new file, it needs to be written here. + if (content := file.get("content", None)) is not None: + manager.save( + create_file_model(content), + str(base_path / "documentation" / file.get('filepath', 'invalid_filepath')) + ) From cab4dc15977d5682b22c0c904b9770f7675da51f Mon Sep 17 00:00:00 2001 From: Satchel Baldwin Date: Tue, 17 Jun 2025 12:54:50 -0500 Subject: [PATCH 05/21] fix: backend folder directories --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index aa834de..85e14b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,11 +28,12 @@ ENV BEAKER_SUBKERNEL_USER=user ENV BEAKER_RUN_PATH=/var/run/beaker ENV BEAKER_APP=biome.app.BiomeApp -RUN ln -s /jupyter/src/biome/datasources /home/user/datasources # subkernel access / file pane -RUN ln -s /jupyter/data /home/user/data -# agent path resolution +RUN ln -s /jupyter/src/biome/datasources /home/jupyter/datasources RUN ln -s /jupyter/data /home/jupyter/data +RUN ln -s /jupyter/src/biome/datasources /home/user/datasources +RUN ln -s /jupyter/data /home/user/data + # Service CMD ["python", "-m", "beaker_kernel.service.server", "--ip", "0.0.0.0"] From fa14f94b2ba66537f3e88c2e0123f9aad0853d27 Mon Sep 17 00:00:00 2001 From: Satchel Baldwin Date: Tue, 17 Jun 2025 14:29:59 -0500 Subject: [PATCH 06/21] handle actions in beaker-kernel --- src/biome/context.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/biome/context.py b/src/biome/context.py index 342cf78..f5b9389 100644 --- a/src/biome/context.py +++ b/src/biome/context.py @@ -108,7 +108,6 @@ async def get_datasources(self) -> list[Datasource]: ] # frontend can request where to upload files to given a specific integration - @action(action_name="get_integration_root") async def get_integration_root(self, message): content = message.content return str( @@ -118,16 +117,15 @@ async def get_integration_root(self, message): # handles a case of uploading a new file to a temporary, unsaved datasource # that way the frontend can upload directly rather than sending it in an action message - @action(action_name="create_integration_folders_for_upload") async def create_integration_folders_for_upload(self, message): manager = FileContentsManager() content = message.content create_folder_structure_for_integration(manager, content.get("integration")) return True - # - @action(action_name="save_integration") + async def save_integration(self, message): + logger.warning('called save_integration') manager = FileContentsManager() content = message.content write_integration(manager, content) @@ -135,12 +133,12 @@ async def save_integration(self, message): self.agent.initialize_adhoc() self.agent.add_context(f"A new integration has been added: `{content.get('slug')}`. You may now use this with `draft_integration_code`.") - @action(action_name="add_example") async def add_example(self, message): + logger.warning('called add_example') manager = FileContentsManager() content = message.content + logger.warning(content) write_integration(manager, content) self.agent.fetch_specs() self.agent.initialize_adhoc() self.agent.add_context(f"A new example has been added to `{content.get('slug')}.`") - From 539ca7a6ed3297e9ed17b42d6db37f3c09d8427f Mon Sep 17 00:00:00 2001 From: Matthew Printz Date: Fri, 20 Jun 2025 11:28:06 -0600 Subject: [PATCH 07/21] Switch biome to default to notebook view --- src/biome/app.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/biome/app.py b/src/biome/app.py index 2e907c9..7d3b03d 100644 --- a/src/biome/app.py +++ b/src/biome/app.py @@ -7,12 +7,12 @@ class BiomeApp(BeakerApp): name = "Biome" pages = { - 'chat': { - "title": "Biome chat", - "default": True, - }, "notebook": { "title": "Biome notebook", + "default": True, + }, + 'chat': { + "title": "Biome chat", }, "dev": { "title": "Dev interface", From be8b2cf96d0edb5bd37355d33db6cc59214ebedb Mon Sep 17 00:00:00 2001 From: Matthew Printz Date: Fri, 20 Jun 2025 11:34:58 -0600 Subject: [PATCH 08/21] Refactor to match Datasource->Integration name change --- src/biome/agent.py | 2 +- src/biome/context.py | 29 ++++++----------------------- 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/src/biome/agent.py b/src/biome/agent.py index cf61f97..d923265 100644 --- a/src/biome/agent.py +++ b/src/biome/agent.py @@ -278,7 +278,7 @@ async def add_integration(self, integration (str): The name of the target data source or API that will be added. description (str): A plain text description of what the data source is based on your knowledge of what the user is asking for, combined with their description if their description is relevant, or, if you do not know about the target data source. If the user does not provide any information, rely on what you know. Target a paragraph in length. schema_location (str): A URL or local filepath to fetch an OpenAPI schema from. If the user does not provide one, ask them for the URL or local filepath to the schema. - base_url (str): The base URL for the datasource that will be used for making OpenAPI calls. If the user does not provide one, ask them for the base URL of the API. + base_url (str): The base URL for the integration that will be used for making OpenAPI calls. If the user does not provide one, ask them for the base URL of the API. Returns: str: Message indicating success or failure of adding the integration. """ diff --git a/src/biome/context.py b/src/biome/context.py index f5b9389..21e2664 100644 --- a/src/biome/context.py +++ b/src/biome/context.py @@ -12,7 +12,7 @@ from beaker_kernel.lib.context import BeakerContext, action from beaker_kernel.subkernels.python import PythonSubkernel -from beaker_kernel.lib.types import Datasource, DatasourceAttachment +from beaker_kernel.lib.types import Integration, IntegrationAttachment from .agent import BiomeAgent from .integration import create_folder_structure_for_integration, get_integration_folder, write_integration @@ -52,12 +52,12 @@ async def setup(self, context_info=None, parent_header=None): }) await self.execute(command) - async def get_datasources(self) -> list[Datasource]: + async def get_integrations(self) -> list[Integration]: """ - fetch all of the adhoc-api datasources to pass to beaker. + fetch all of the adhoc-api integrations to pass to beaker. """ - # get list of keys not inherent to a datasource for the user-files category + # get list of keys not inherent to a integrations for the user-files category attached_files = {} for (_, spec) in self.agent.raw_specs: attached_files[spec["name"]] = [] @@ -85,7 +85,7 @@ async def get_datasources(self) -> list[Datasource]: spec[attachment_key].strip() ).strip().replace("documentation/", "") - attached_files[spec["name"]].append(DatasourceAttachment( + attached_files[spec["name"]].append(IntegrationAttachment( name=attachment_key, filepath=filepath_raw, content=None, @@ -95,7 +95,7 @@ async def get_datasources(self) -> list[Datasource]: # manually load examples return [ - Datasource( + Integration( slug=spec["slug"], url=str(yaml_location), name=spec["name"], @@ -107,23 +107,6 @@ async def get_datasources(self) -> list[Datasource]: for (yaml_location, spec) in self.agent.raw_specs ] - # frontend can request where to upload files to given a specific integration - async def get_integration_root(self, message): - content = message.content - return str( - Path(os.environ.get("BIOME_INTEGRATIONS_DIR", "/")) - / get_integration_folder(content.get("integration")) - ) - - # handles a case of uploading a new file to a temporary, unsaved datasource - # that way the frontend can upload directly rather than sending it in an action message - async def create_integration_folders_for_upload(self, message): - manager = FileContentsManager() - content = message.content - create_folder_structure_for_integration(manager, content.get("integration")) - return True - - async def save_integration(self, message): logger.warning('called save_integration') manager = FileContentsManager() From c0fdf370caa239fd91b7203fd3d285b09f3dc727 Mon Sep 17 00:00:00 2001 From: Matthew Printz Date: Mon, 23 Jun 2025 11:36:59 -0600 Subject: [PATCH 09/21] Checkpoint on work --- .../data}/census-acs/2023-1yr-api-changes.csv | 0 .../data}/census-ahs/census_ahs_codebook.csv | 0 .../data}/census-ahs/survey_1999.csv | 0 .../data}/census-ahs/survey_2001.csv | 0 .../data}/census-ahs/survey_2003.csv | 0 .../data}/census-ahs/survey_2005.csv | 0 .../data}/census-ahs/survey_2007.csv | 0 .../data}/census-ahs/survey_2009.csv | 0 .../data}/census-ahs/survey_2011.csv | 0 .../data}/census-ahs/survey_2013.csv | 0 .../data}/census-ahs/survey_2015.csv | 0 .../data}/census-ahs/survey_2017.csv | 0 .../data}/census-ahs/survey_2019.csv | 0 .../data}/census-ahs/survey_2021.csv | 0 .../data}/census-ahs/survey_2023.csv | 0 .../data}/census-nsch/2016e_topical.sas7bdat | 0 .../data}/census-nsch/2017e_topical.sas7bdat | 0 .../data}/census-nsch/2018e_topical.sas7bdat | 0 .../data}/census-nsch/2019e_topical.sas7bdat | 0 .../data}/census-nsch/2020e_topical.sas7bdat | 0 .../data}/census-nsch/2021e_topical.sas7bdat | 0 .../data}/census-nsch/2022e_topical.sas7bdat | 0 .../data}/census-nsch/2023e_topical.sas7bdat | 0 .../census-nsch/nsch_dictionary_codebook.csv | 0 ...-asthma-prevalence-by-county-2015_2022.csv | 0 .../epa-tri/EPA_TRI_Toxics_2014_2023.csv | 0 .../adhoc_data/data}/fda-gras/GRASNotices.csv | 0 .../nhanes_dietary/1999_2000_DEMOGRAPHICS.xpt | 0 .../1999_2000_DRXFMT_food_codes.xpt | 0 .../1999_2000_DRXIFF_individual_foods.xpt | 0 .../nhanes_dietary/2005_2006_DEMOGRAPHICS.xpt | 0 .../2005_2006_DR1IFF_individual_foods.xpt | 0 .../2005_2006_DRXFCD_food_codes.xpt | 0 .../2005_2006_DRXMCD_modification_codes.xpt | 0 .../nhanes_dietary/2009-2010_DEMOGRAPHICS.xpt | 0 .../2009_2010_DR1IFF_individual_foods.xpt | 0 .../2009_2010_DRXFCD_food_codes.xpt | 0 .../2009_2010_DRXMCD_modification_codes.xpt | 0 .../nhanes_dietary/2015_2016_DEMOGRAPHICS.xpt | 0 .../2015_2016_DR1IFF_individual_foods.xpt | 0 .../2015_2016_DRXFCD_food_codes.xpt | 0 .../nhanes_dietary/2017_2020_DEMOGRAPHICS.xpt | 0 .../2017_2020_DR1IFF_individual_foods.xpt | 0 .../2017_2020_DRXFCD_food_codes.xpt | 0 .../nhanes_dietary/2021_2023_DEMOGRAPHICS.xpt | 0 .../2021_2023_DR1IFF_individual_foods.xpt | 0 .../2021_2023_DRXFCD_food_codes.xpt | 0 .../EPest.county.estimates.2000.txt | 0 .../EPest.county.estimates.2005.txt | 0 .../EPest.county.estimates.2010.txt | 0 .../EPest_county_estimates_2013_2017_v2.txt | 0 .../EPest_county_estimates_2019.txt | 0 .../{ => adhoc_data}/datasources/ahs/api.yaml | 0 .../datasources/ahs/documentation/docs.md | 0 .../ahs/documentation/examples.yaml | 0 .../{ => adhoc_data}/datasources/aqs/api.yaml | 0 .../datasources/aqs/documentation/aqs.json | 0 .../aqs/documentation/examples.yaml | 0 .../datasources/cbioportal/api.yaml | 0 .../cbioportal/documentation/cbioportal.json | 0 .../cbioportal/documentation/examples.yaml | 0 .../{ => adhoc_data}/datasources/cda/api.yaml | 0 .../datasources/cda/documentation/cda.yaml | 0 .../cda/documentation/examples.yaml | 0 .../datasources/cdc_tracking_network/api.yaml | 0 .../documentation/api_examples.md | 0 .../documentation/examples.yaml | 0 .../documentation/user_guide.md | 0 .../datasources/census_acs/api.yaml | 0 .../census_acs/documentation/census_sdk.md | 0 .../census_acs/documentation/census_web.md | 0 .../census_acs/documentation/examples.yaml | 0 .../chis_california_asthma/api.yaml | 0 .../documentation/examples.yaml | 0 .../datasources/epa_tri/api.yaml | 0 .../epa_tri/documentation/examples.yaml | 0 .../datasources/faers/api.yaml | 0 .../faers/documentation/examples.yaml | 0 .../documentation/faers_fields_reference.csv | 0 .../faers/documentation/faers_web.md | 0 .../{ => adhoc_data}/datasources/gdc/api.yaml | 0 .../gdc/documentation/examples.yaml | 0 .../datasources/gdc/documentation/facets.txt | 0 .../datasources/gdc/documentation/gdc.md | 0 .../gdc/documentation/gdc_mappings.json | 0 .../datasources/gras/api.yaml | 0 .../gras/documentation/examples.yaml | 0 .../{ => adhoc_data}/datasources/hpa/api.yaml | 0 .../hpa/documentation/examples.yaml | 0 .../datasources/hpa/documentation/hpa_docs.md | 0 .../{ => adhoc_data}/datasources/idc/api.yaml | 0 .../idc/documentation/examples.yaml | 0 .../datasources/idc/documentation/idc.md | 0 .../datasources/indra/api.yaml | 0 .../indra/documentation/examples.yaml | 0 .../indra/documentation/indra.json | 0 .../datasources/netrias/api.yaml | 0 .../netrias/documentation/examples.yaml | 0 .../netrias/documentation/netrias.json | 0 .../datasources/nhanes_dietary/api.yaml | 0 .../nhanes_dietary/documentation/docs.md | 0 .../documentation/examples.yaml | 0 .../datasources/nsch/api.yaml | 0 .../nsch/documentation/examples.yaml | 0 .../{ => adhoc_data}/datasources/pdc/api.yaml | 0 .../pdc/documentation/examples.yaml | 0 .../pdc/documentation/pdc_schema.graphql | 0 .../datasources/synapse/api.yaml | 0 .../documentation/SynapseOpenApiSpec.json | 0 .../synapse/documentation/examples.yaml | 0 .../documentation/nf_programmatic_webdocs.md | 0 .../synapse/documentation/web_docs.md | 0 .../datasources/usda_foodcentral/api.yaml | 0 .../documentation/examples.yaml | 0 .../documentation/foodcentral_openapi.json | 0 .../datasources/usgs_pesticide/api.yaml | 0 .../documentation/examples.yaml | 0 .../datasources/waterservices_usgs/api.yaml | 0 .../documentation/examples.yaml | 0 .../documentation/site_types.md | 0 .../documentation/usgs_waterservices_web.md | 0 .../instructions/README.CLINVAR.QUERIES.md | 0 .../instructions/README.GTEX.QUERIES.md | 0 .../instructions/README.REFSEQ.QUERIES.md | 0 .../{ => adhoc_data}/prompts/agent_prompt.md | 0 .../prompts/consult_integration_docs.md | 0 .../prompts/draft_integration_code.md | 0 src/biome/agent.py | 38 +++++++++---------- src/biome/context.py | 29 ++++++++++++-- 129 files changed, 45 insertions(+), 22 deletions(-) rename {data => src/biome/adhoc_data/data}/census-acs/2023-1yr-api-changes.csv (100%) rename {data => src/biome/adhoc_data/data}/census-ahs/census_ahs_codebook.csv (100%) rename {data => src/biome/adhoc_data/data}/census-ahs/survey_1999.csv (100%) rename {data => src/biome/adhoc_data/data}/census-ahs/survey_2001.csv (100%) rename {data => src/biome/adhoc_data/data}/census-ahs/survey_2003.csv (100%) rename {data => src/biome/adhoc_data/data}/census-ahs/survey_2005.csv (100%) rename {data => src/biome/adhoc_data/data}/census-ahs/survey_2007.csv (100%) rename {data => src/biome/adhoc_data/data}/census-ahs/survey_2009.csv (100%) rename {data => src/biome/adhoc_data/data}/census-ahs/survey_2011.csv (100%) rename {data => src/biome/adhoc_data/data}/census-ahs/survey_2013.csv (100%) rename {data => src/biome/adhoc_data/data}/census-ahs/survey_2015.csv (100%) rename {data => src/biome/adhoc_data/data}/census-ahs/survey_2017.csv (100%) rename {data => src/biome/adhoc_data/data}/census-ahs/survey_2019.csv (100%) rename {data => src/biome/adhoc_data/data}/census-ahs/survey_2021.csv (100%) rename {data => src/biome/adhoc_data/data}/census-ahs/survey_2023.csv (100%) rename {data => src/biome/adhoc_data/data}/census-nsch/2016e_topical.sas7bdat (100%) rename {data => src/biome/adhoc_data/data}/census-nsch/2017e_topical.sas7bdat (100%) rename {data => src/biome/adhoc_data/data}/census-nsch/2018e_topical.sas7bdat (100%) rename {data => src/biome/adhoc_data/data}/census-nsch/2019e_topical.sas7bdat (100%) rename {data => src/biome/adhoc_data/data}/census-nsch/2020e_topical.sas7bdat (100%) rename {data => src/biome/adhoc_data/data}/census-nsch/2021e_topical.sas7bdat (100%) rename {data => src/biome/adhoc_data/data}/census-nsch/2022e_topical.sas7bdat (100%) rename {data => src/biome/adhoc_data/data}/census-nsch/2023e_topical.sas7bdat (100%) rename {data => src/biome/adhoc_data/data}/census-nsch/nsch_dictionary_codebook.csv (100%) rename {data => src/biome/adhoc_data/data}/chis_california_asthma/current-asthma-prevalence-by-county-2015_2022.csv (100%) rename {data => src/biome/adhoc_data/data}/epa-tri/EPA_TRI_Toxics_2014_2023.csv (100%) rename {data => src/biome/adhoc_data/data}/fda-gras/GRASNotices.csv (100%) rename {data => src/biome/adhoc_data/data}/nhanes_dietary/1999_2000_DEMOGRAPHICS.xpt (100%) rename {data => src/biome/adhoc_data/data}/nhanes_dietary/1999_2000_DRXFMT_food_codes.xpt (100%) rename {data => src/biome/adhoc_data/data}/nhanes_dietary/1999_2000_DRXIFF_individual_foods.xpt (100%) rename {data => src/biome/adhoc_data/data}/nhanes_dietary/2005_2006_DEMOGRAPHICS.xpt (100%) rename {data => src/biome/adhoc_data/data}/nhanes_dietary/2005_2006_DR1IFF_individual_foods.xpt (100%) rename {data => src/biome/adhoc_data/data}/nhanes_dietary/2005_2006_DRXFCD_food_codes.xpt (100%) rename {data => src/biome/adhoc_data/data}/nhanes_dietary/2005_2006_DRXMCD_modification_codes.xpt (100%) rename {data => src/biome/adhoc_data/data}/nhanes_dietary/2009-2010_DEMOGRAPHICS.xpt (100%) rename {data => src/biome/adhoc_data/data}/nhanes_dietary/2009_2010_DR1IFF_individual_foods.xpt (100%) rename {data => src/biome/adhoc_data/data}/nhanes_dietary/2009_2010_DRXFCD_food_codes.xpt (100%) rename {data => src/biome/adhoc_data/data}/nhanes_dietary/2009_2010_DRXMCD_modification_codes.xpt (100%) rename {data => src/biome/adhoc_data/data}/nhanes_dietary/2015_2016_DEMOGRAPHICS.xpt (100%) rename {data => src/biome/adhoc_data/data}/nhanes_dietary/2015_2016_DR1IFF_individual_foods.xpt (100%) rename {data => src/biome/adhoc_data/data}/nhanes_dietary/2015_2016_DRXFCD_food_codes.xpt (100%) rename {data => src/biome/adhoc_data/data}/nhanes_dietary/2017_2020_DEMOGRAPHICS.xpt (100%) rename {data => src/biome/adhoc_data/data}/nhanes_dietary/2017_2020_DR1IFF_individual_foods.xpt (100%) rename {data => src/biome/adhoc_data/data}/nhanes_dietary/2017_2020_DRXFCD_food_codes.xpt (100%) rename {data => src/biome/adhoc_data/data}/nhanes_dietary/2021_2023_DEMOGRAPHICS.xpt (100%) rename {data => src/biome/adhoc_data/data}/nhanes_dietary/2021_2023_DR1IFF_individual_foods.xpt (100%) rename {data => src/biome/adhoc_data/data}/nhanes_dietary/2021_2023_DRXFCD_food_codes.xpt (100%) rename {data => src/biome/adhoc_data/data}/usgs_pesticide/EPest.county.estimates.2000.txt (100%) rename {data => src/biome/adhoc_data/data}/usgs_pesticide/EPest.county.estimates.2005.txt (100%) rename {data => src/biome/adhoc_data/data}/usgs_pesticide/EPest.county.estimates.2010.txt (100%) rename {data => src/biome/adhoc_data/data}/usgs_pesticide/EPest_county_estimates_2013_2017_v2.txt (100%) rename {data => src/biome/adhoc_data/data}/usgs_pesticide/EPest_county_estimates_2019.txt (100%) rename src/biome/{ => adhoc_data}/datasources/ahs/api.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/ahs/documentation/docs.md (100%) rename src/biome/{ => adhoc_data}/datasources/ahs/documentation/examples.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/aqs/api.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/aqs/documentation/aqs.json (100%) rename src/biome/{ => adhoc_data}/datasources/aqs/documentation/examples.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/cbioportal/api.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/cbioportal/documentation/cbioportal.json (100%) rename src/biome/{ => adhoc_data}/datasources/cbioportal/documentation/examples.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/cda/api.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/cda/documentation/cda.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/cda/documentation/examples.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/cdc_tracking_network/api.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/cdc_tracking_network/documentation/api_examples.md (100%) rename src/biome/{ => adhoc_data}/datasources/cdc_tracking_network/documentation/examples.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/cdc_tracking_network/documentation/user_guide.md (100%) rename src/biome/{ => adhoc_data}/datasources/census_acs/api.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/census_acs/documentation/census_sdk.md (100%) rename src/biome/{ => adhoc_data}/datasources/census_acs/documentation/census_web.md (100%) rename src/biome/{ => adhoc_data}/datasources/census_acs/documentation/examples.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/chis_california_asthma/api.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/chis_california_asthma/documentation/examples.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/epa_tri/api.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/epa_tri/documentation/examples.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/faers/api.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/faers/documentation/examples.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/faers/documentation/faers_fields_reference.csv (100%) rename src/biome/{ => adhoc_data}/datasources/faers/documentation/faers_web.md (100%) rename src/biome/{ => adhoc_data}/datasources/gdc/api.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/gdc/documentation/examples.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/gdc/documentation/facets.txt (100%) rename src/biome/{ => adhoc_data}/datasources/gdc/documentation/gdc.md (100%) rename src/biome/{ => adhoc_data}/datasources/gdc/documentation/gdc_mappings.json (100%) rename src/biome/{ => adhoc_data}/datasources/gras/api.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/gras/documentation/examples.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/hpa/api.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/hpa/documentation/examples.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/hpa/documentation/hpa_docs.md (100%) rename src/biome/{ => adhoc_data}/datasources/idc/api.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/idc/documentation/examples.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/idc/documentation/idc.md (100%) rename src/biome/{ => adhoc_data}/datasources/indra/api.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/indra/documentation/examples.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/indra/documentation/indra.json (100%) rename src/biome/{ => adhoc_data}/datasources/netrias/api.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/netrias/documentation/examples.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/netrias/documentation/netrias.json (100%) rename src/biome/{ => adhoc_data}/datasources/nhanes_dietary/api.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/nhanes_dietary/documentation/docs.md (100%) rename src/biome/{ => adhoc_data}/datasources/nhanes_dietary/documentation/examples.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/nsch/api.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/nsch/documentation/examples.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/pdc/api.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/pdc/documentation/examples.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/pdc/documentation/pdc_schema.graphql (100%) rename src/biome/{ => adhoc_data}/datasources/synapse/api.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/synapse/documentation/SynapseOpenApiSpec.json (100%) rename src/biome/{ => adhoc_data}/datasources/synapse/documentation/examples.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/synapse/documentation/nf_programmatic_webdocs.md (100%) rename src/biome/{ => adhoc_data}/datasources/synapse/documentation/web_docs.md (100%) rename src/biome/{ => adhoc_data}/datasources/usda_foodcentral/api.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/usda_foodcentral/documentation/examples.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/usda_foodcentral/documentation/foodcentral_openapi.json (100%) rename src/biome/{ => adhoc_data}/datasources/usgs_pesticide/api.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/usgs_pesticide/documentation/examples.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/waterservices_usgs/api.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/waterservices_usgs/documentation/examples.yaml (100%) rename src/biome/{ => adhoc_data}/datasources/waterservices_usgs/documentation/site_types.md (100%) rename src/biome/{ => adhoc_data}/datasources/waterservices_usgs/documentation/usgs_waterservices_web.md (100%) rename src/biome/{ => adhoc_data}/instructions/README.CLINVAR.QUERIES.md (100%) rename src/biome/{ => adhoc_data}/instructions/README.GTEX.QUERIES.md (100%) rename src/biome/{ => adhoc_data}/instructions/README.REFSEQ.QUERIES.md (100%) rename src/biome/{ => adhoc_data}/prompts/agent_prompt.md (100%) rename src/biome/{ => adhoc_data}/prompts/consult_integration_docs.md (100%) rename src/biome/{ => adhoc_data}/prompts/draft_integration_code.md (100%) diff --git a/data/census-acs/2023-1yr-api-changes.csv b/src/biome/adhoc_data/data/census-acs/2023-1yr-api-changes.csv similarity index 100% rename from data/census-acs/2023-1yr-api-changes.csv rename to src/biome/adhoc_data/data/census-acs/2023-1yr-api-changes.csv diff --git a/data/census-ahs/census_ahs_codebook.csv b/src/biome/adhoc_data/data/census-ahs/census_ahs_codebook.csv similarity index 100% rename from data/census-ahs/census_ahs_codebook.csv rename to src/biome/adhoc_data/data/census-ahs/census_ahs_codebook.csv diff --git a/data/census-ahs/survey_1999.csv b/src/biome/adhoc_data/data/census-ahs/survey_1999.csv similarity index 100% rename from data/census-ahs/survey_1999.csv rename to src/biome/adhoc_data/data/census-ahs/survey_1999.csv diff --git a/data/census-ahs/survey_2001.csv b/src/biome/adhoc_data/data/census-ahs/survey_2001.csv similarity index 100% rename from data/census-ahs/survey_2001.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2001.csv diff --git a/data/census-ahs/survey_2003.csv b/src/biome/adhoc_data/data/census-ahs/survey_2003.csv similarity index 100% rename from data/census-ahs/survey_2003.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2003.csv diff --git a/data/census-ahs/survey_2005.csv b/src/biome/adhoc_data/data/census-ahs/survey_2005.csv similarity index 100% rename from data/census-ahs/survey_2005.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2005.csv diff --git a/data/census-ahs/survey_2007.csv b/src/biome/adhoc_data/data/census-ahs/survey_2007.csv similarity index 100% rename from data/census-ahs/survey_2007.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2007.csv diff --git a/data/census-ahs/survey_2009.csv b/src/biome/adhoc_data/data/census-ahs/survey_2009.csv similarity index 100% rename from data/census-ahs/survey_2009.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2009.csv diff --git a/data/census-ahs/survey_2011.csv b/src/biome/adhoc_data/data/census-ahs/survey_2011.csv similarity index 100% rename from data/census-ahs/survey_2011.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2011.csv diff --git a/data/census-ahs/survey_2013.csv b/src/biome/adhoc_data/data/census-ahs/survey_2013.csv similarity index 100% rename from data/census-ahs/survey_2013.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2013.csv diff --git a/data/census-ahs/survey_2015.csv b/src/biome/adhoc_data/data/census-ahs/survey_2015.csv similarity index 100% rename from data/census-ahs/survey_2015.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2015.csv diff --git a/data/census-ahs/survey_2017.csv b/src/biome/adhoc_data/data/census-ahs/survey_2017.csv similarity index 100% rename from data/census-ahs/survey_2017.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2017.csv diff --git a/data/census-ahs/survey_2019.csv b/src/biome/adhoc_data/data/census-ahs/survey_2019.csv similarity index 100% rename from data/census-ahs/survey_2019.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2019.csv diff --git a/data/census-ahs/survey_2021.csv b/src/biome/adhoc_data/data/census-ahs/survey_2021.csv similarity index 100% rename from data/census-ahs/survey_2021.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2021.csv diff --git a/data/census-ahs/survey_2023.csv b/src/biome/adhoc_data/data/census-ahs/survey_2023.csv similarity index 100% rename from data/census-ahs/survey_2023.csv rename to src/biome/adhoc_data/data/census-ahs/survey_2023.csv diff --git a/data/census-nsch/2016e_topical.sas7bdat b/src/biome/adhoc_data/data/census-nsch/2016e_topical.sas7bdat similarity index 100% rename from data/census-nsch/2016e_topical.sas7bdat rename to src/biome/adhoc_data/data/census-nsch/2016e_topical.sas7bdat diff --git a/data/census-nsch/2017e_topical.sas7bdat b/src/biome/adhoc_data/data/census-nsch/2017e_topical.sas7bdat similarity index 100% rename from data/census-nsch/2017e_topical.sas7bdat rename to src/biome/adhoc_data/data/census-nsch/2017e_topical.sas7bdat diff --git a/data/census-nsch/2018e_topical.sas7bdat b/src/biome/adhoc_data/data/census-nsch/2018e_topical.sas7bdat similarity index 100% rename from data/census-nsch/2018e_topical.sas7bdat rename to src/biome/adhoc_data/data/census-nsch/2018e_topical.sas7bdat diff --git a/data/census-nsch/2019e_topical.sas7bdat b/src/biome/adhoc_data/data/census-nsch/2019e_topical.sas7bdat similarity index 100% rename from data/census-nsch/2019e_topical.sas7bdat rename to src/biome/adhoc_data/data/census-nsch/2019e_topical.sas7bdat diff --git a/data/census-nsch/2020e_topical.sas7bdat b/src/biome/adhoc_data/data/census-nsch/2020e_topical.sas7bdat similarity index 100% rename from data/census-nsch/2020e_topical.sas7bdat rename to src/biome/adhoc_data/data/census-nsch/2020e_topical.sas7bdat diff --git a/data/census-nsch/2021e_topical.sas7bdat b/src/biome/adhoc_data/data/census-nsch/2021e_topical.sas7bdat similarity index 100% rename from data/census-nsch/2021e_topical.sas7bdat rename to src/biome/adhoc_data/data/census-nsch/2021e_topical.sas7bdat diff --git a/data/census-nsch/2022e_topical.sas7bdat b/src/biome/adhoc_data/data/census-nsch/2022e_topical.sas7bdat similarity index 100% rename from data/census-nsch/2022e_topical.sas7bdat rename to src/biome/adhoc_data/data/census-nsch/2022e_topical.sas7bdat diff --git a/data/census-nsch/2023e_topical.sas7bdat b/src/biome/adhoc_data/data/census-nsch/2023e_topical.sas7bdat similarity index 100% rename from data/census-nsch/2023e_topical.sas7bdat rename to src/biome/adhoc_data/data/census-nsch/2023e_topical.sas7bdat diff --git a/data/census-nsch/nsch_dictionary_codebook.csv b/src/biome/adhoc_data/data/census-nsch/nsch_dictionary_codebook.csv similarity index 100% rename from data/census-nsch/nsch_dictionary_codebook.csv rename to src/biome/adhoc_data/data/census-nsch/nsch_dictionary_codebook.csv diff --git a/data/chis_california_asthma/current-asthma-prevalence-by-county-2015_2022.csv b/src/biome/adhoc_data/data/chis_california_asthma/current-asthma-prevalence-by-county-2015_2022.csv similarity index 100% rename from data/chis_california_asthma/current-asthma-prevalence-by-county-2015_2022.csv rename to src/biome/adhoc_data/data/chis_california_asthma/current-asthma-prevalence-by-county-2015_2022.csv diff --git a/data/epa-tri/EPA_TRI_Toxics_2014_2023.csv b/src/biome/adhoc_data/data/epa-tri/EPA_TRI_Toxics_2014_2023.csv similarity index 100% rename from data/epa-tri/EPA_TRI_Toxics_2014_2023.csv rename to src/biome/adhoc_data/data/epa-tri/EPA_TRI_Toxics_2014_2023.csv diff --git a/data/fda-gras/GRASNotices.csv b/src/biome/adhoc_data/data/fda-gras/GRASNotices.csv similarity index 100% rename from data/fda-gras/GRASNotices.csv rename to src/biome/adhoc_data/data/fda-gras/GRASNotices.csv diff --git a/data/nhanes_dietary/1999_2000_DEMOGRAPHICS.xpt b/src/biome/adhoc_data/data/nhanes_dietary/1999_2000_DEMOGRAPHICS.xpt similarity index 100% rename from data/nhanes_dietary/1999_2000_DEMOGRAPHICS.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/1999_2000_DEMOGRAPHICS.xpt diff --git a/data/nhanes_dietary/1999_2000_DRXFMT_food_codes.xpt b/src/biome/adhoc_data/data/nhanes_dietary/1999_2000_DRXFMT_food_codes.xpt similarity index 100% rename from data/nhanes_dietary/1999_2000_DRXFMT_food_codes.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/1999_2000_DRXFMT_food_codes.xpt diff --git a/data/nhanes_dietary/1999_2000_DRXIFF_individual_foods.xpt b/src/biome/adhoc_data/data/nhanes_dietary/1999_2000_DRXIFF_individual_foods.xpt similarity index 100% rename from data/nhanes_dietary/1999_2000_DRXIFF_individual_foods.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/1999_2000_DRXIFF_individual_foods.xpt diff --git a/data/nhanes_dietary/2005_2006_DEMOGRAPHICS.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2005_2006_DEMOGRAPHICS.xpt similarity index 100% rename from data/nhanes_dietary/2005_2006_DEMOGRAPHICS.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2005_2006_DEMOGRAPHICS.xpt diff --git a/data/nhanes_dietary/2005_2006_DR1IFF_individual_foods.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2005_2006_DR1IFF_individual_foods.xpt similarity index 100% rename from data/nhanes_dietary/2005_2006_DR1IFF_individual_foods.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2005_2006_DR1IFF_individual_foods.xpt diff --git a/data/nhanes_dietary/2005_2006_DRXFCD_food_codes.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2005_2006_DRXFCD_food_codes.xpt similarity index 100% rename from data/nhanes_dietary/2005_2006_DRXFCD_food_codes.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2005_2006_DRXFCD_food_codes.xpt diff --git a/data/nhanes_dietary/2005_2006_DRXMCD_modification_codes.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2005_2006_DRXMCD_modification_codes.xpt similarity index 100% rename from data/nhanes_dietary/2005_2006_DRXMCD_modification_codes.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2005_2006_DRXMCD_modification_codes.xpt diff --git a/data/nhanes_dietary/2009-2010_DEMOGRAPHICS.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2009-2010_DEMOGRAPHICS.xpt similarity index 100% rename from data/nhanes_dietary/2009-2010_DEMOGRAPHICS.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2009-2010_DEMOGRAPHICS.xpt diff --git a/data/nhanes_dietary/2009_2010_DR1IFF_individual_foods.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2009_2010_DR1IFF_individual_foods.xpt similarity index 100% rename from data/nhanes_dietary/2009_2010_DR1IFF_individual_foods.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2009_2010_DR1IFF_individual_foods.xpt diff --git a/data/nhanes_dietary/2009_2010_DRXFCD_food_codes.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2009_2010_DRXFCD_food_codes.xpt similarity index 100% rename from data/nhanes_dietary/2009_2010_DRXFCD_food_codes.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2009_2010_DRXFCD_food_codes.xpt diff --git a/data/nhanes_dietary/2009_2010_DRXMCD_modification_codes.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2009_2010_DRXMCD_modification_codes.xpt similarity index 100% rename from data/nhanes_dietary/2009_2010_DRXMCD_modification_codes.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2009_2010_DRXMCD_modification_codes.xpt diff --git a/data/nhanes_dietary/2015_2016_DEMOGRAPHICS.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2015_2016_DEMOGRAPHICS.xpt similarity index 100% rename from data/nhanes_dietary/2015_2016_DEMOGRAPHICS.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2015_2016_DEMOGRAPHICS.xpt diff --git a/data/nhanes_dietary/2015_2016_DR1IFF_individual_foods.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2015_2016_DR1IFF_individual_foods.xpt similarity index 100% rename from data/nhanes_dietary/2015_2016_DR1IFF_individual_foods.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2015_2016_DR1IFF_individual_foods.xpt diff --git a/data/nhanes_dietary/2015_2016_DRXFCD_food_codes.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2015_2016_DRXFCD_food_codes.xpt similarity index 100% rename from data/nhanes_dietary/2015_2016_DRXFCD_food_codes.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2015_2016_DRXFCD_food_codes.xpt diff --git a/data/nhanes_dietary/2017_2020_DEMOGRAPHICS.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2017_2020_DEMOGRAPHICS.xpt similarity index 100% rename from data/nhanes_dietary/2017_2020_DEMOGRAPHICS.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2017_2020_DEMOGRAPHICS.xpt diff --git a/data/nhanes_dietary/2017_2020_DR1IFF_individual_foods.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2017_2020_DR1IFF_individual_foods.xpt similarity index 100% rename from data/nhanes_dietary/2017_2020_DR1IFF_individual_foods.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2017_2020_DR1IFF_individual_foods.xpt diff --git a/data/nhanes_dietary/2017_2020_DRXFCD_food_codes.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2017_2020_DRXFCD_food_codes.xpt similarity index 100% rename from data/nhanes_dietary/2017_2020_DRXFCD_food_codes.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2017_2020_DRXFCD_food_codes.xpt diff --git a/data/nhanes_dietary/2021_2023_DEMOGRAPHICS.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2021_2023_DEMOGRAPHICS.xpt similarity index 100% rename from data/nhanes_dietary/2021_2023_DEMOGRAPHICS.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2021_2023_DEMOGRAPHICS.xpt diff --git a/data/nhanes_dietary/2021_2023_DR1IFF_individual_foods.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2021_2023_DR1IFF_individual_foods.xpt similarity index 100% rename from data/nhanes_dietary/2021_2023_DR1IFF_individual_foods.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2021_2023_DR1IFF_individual_foods.xpt diff --git a/data/nhanes_dietary/2021_2023_DRXFCD_food_codes.xpt b/src/biome/adhoc_data/data/nhanes_dietary/2021_2023_DRXFCD_food_codes.xpt similarity index 100% rename from data/nhanes_dietary/2021_2023_DRXFCD_food_codes.xpt rename to src/biome/adhoc_data/data/nhanes_dietary/2021_2023_DRXFCD_food_codes.xpt diff --git a/data/usgs_pesticide/EPest.county.estimates.2000.txt b/src/biome/adhoc_data/data/usgs_pesticide/EPest.county.estimates.2000.txt similarity index 100% rename from data/usgs_pesticide/EPest.county.estimates.2000.txt rename to src/biome/adhoc_data/data/usgs_pesticide/EPest.county.estimates.2000.txt diff --git a/data/usgs_pesticide/EPest.county.estimates.2005.txt b/src/biome/adhoc_data/data/usgs_pesticide/EPest.county.estimates.2005.txt similarity index 100% rename from data/usgs_pesticide/EPest.county.estimates.2005.txt rename to src/biome/adhoc_data/data/usgs_pesticide/EPest.county.estimates.2005.txt diff --git a/data/usgs_pesticide/EPest.county.estimates.2010.txt b/src/biome/adhoc_data/data/usgs_pesticide/EPest.county.estimates.2010.txt similarity index 100% rename from data/usgs_pesticide/EPest.county.estimates.2010.txt rename to src/biome/adhoc_data/data/usgs_pesticide/EPest.county.estimates.2010.txt diff --git a/data/usgs_pesticide/EPest_county_estimates_2013_2017_v2.txt b/src/biome/adhoc_data/data/usgs_pesticide/EPest_county_estimates_2013_2017_v2.txt similarity index 100% rename from data/usgs_pesticide/EPest_county_estimates_2013_2017_v2.txt rename to src/biome/adhoc_data/data/usgs_pesticide/EPest_county_estimates_2013_2017_v2.txt diff --git a/data/usgs_pesticide/EPest_county_estimates_2019.txt b/src/biome/adhoc_data/data/usgs_pesticide/EPest_county_estimates_2019.txt similarity index 100% rename from data/usgs_pesticide/EPest_county_estimates_2019.txt rename to src/biome/adhoc_data/data/usgs_pesticide/EPest_county_estimates_2019.txt diff --git a/src/biome/datasources/ahs/api.yaml b/src/biome/adhoc_data/datasources/ahs/api.yaml similarity index 100% rename from src/biome/datasources/ahs/api.yaml rename to src/biome/adhoc_data/datasources/ahs/api.yaml diff --git a/src/biome/datasources/ahs/documentation/docs.md b/src/biome/adhoc_data/datasources/ahs/documentation/docs.md similarity index 100% rename from src/biome/datasources/ahs/documentation/docs.md rename to src/biome/adhoc_data/datasources/ahs/documentation/docs.md diff --git a/src/biome/datasources/ahs/documentation/examples.yaml b/src/biome/adhoc_data/datasources/ahs/documentation/examples.yaml similarity index 100% rename from src/biome/datasources/ahs/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/ahs/documentation/examples.yaml diff --git a/src/biome/datasources/aqs/api.yaml b/src/biome/adhoc_data/datasources/aqs/api.yaml similarity index 100% rename from src/biome/datasources/aqs/api.yaml rename to src/biome/adhoc_data/datasources/aqs/api.yaml diff --git a/src/biome/datasources/aqs/documentation/aqs.json b/src/biome/adhoc_data/datasources/aqs/documentation/aqs.json similarity index 100% rename from src/biome/datasources/aqs/documentation/aqs.json rename to src/biome/adhoc_data/datasources/aqs/documentation/aqs.json diff --git a/src/biome/datasources/aqs/documentation/examples.yaml b/src/biome/adhoc_data/datasources/aqs/documentation/examples.yaml similarity index 100% rename from src/biome/datasources/aqs/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/aqs/documentation/examples.yaml diff --git a/src/biome/datasources/cbioportal/api.yaml b/src/biome/adhoc_data/datasources/cbioportal/api.yaml similarity index 100% rename from src/biome/datasources/cbioportal/api.yaml rename to src/biome/adhoc_data/datasources/cbioportal/api.yaml diff --git a/src/biome/datasources/cbioportal/documentation/cbioportal.json b/src/biome/adhoc_data/datasources/cbioportal/documentation/cbioportal.json similarity index 100% rename from src/biome/datasources/cbioportal/documentation/cbioportal.json rename to src/biome/adhoc_data/datasources/cbioportal/documentation/cbioportal.json diff --git a/src/biome/datasources/cbioportal/documentation/examples.yaml b/src/biome/adhoc_data/datasources/cbioportal/documentation/examples.yaml similarity index 100% rename from src/biome/datasources/cbioportal/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/cbioportal/documentation/examples.yaml diff --git a/src/biome/datasources/cda/api.yaml b/src/biome/adhoc_data/datasources/cda/api.yaml similarity index 100% rename from src/biome/datasources/cda/api.yaml rename to src/biome/adhoc_data/datasources/cda/api.yaml diff --git a/src/biome/datasources/cda/documentation/cda.yaml b/src/biome/adhoc_data/datasources/cda/documentation/cda.yaml similarity index 100% rename from src/biome/datasources/cda/documentation/cda.yaml rename to src/biome/adhoc_data/datasources/cda/documentation/cda.yaml diff --git a/src/biome/datasources/cda/documentation/examples.yaml b/src/biome/adhoc_data/datasources/cda/documentation/examples.yaml similarity index 100% rename from src/biome/datasources/cda/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/cda/documentation/examples.yaml diff --git a/src/biome/datasources/cdc_tracking_network/api.yaml b/src/biome/adhoc_data/datasources/cdc_tracking_network/api.yaml similarity index 100% rename from src/biome/datasources/cdc_tracking_network/api.yaml rename to src/biome/adhoc_data/datasources/cdc_tracking_network/api.yaml diff --git a/src/biome/datasources/cdc_tracking_network/documentation/api_examples.md b/src/biome/adhoc_data/datasources/cdc_tracking_network/documentation/api_examples.md similarity index 100% rename from src/biome/datasources/cdc_tracking_network/documentation/api_examples.md rename to src/biome/adhoc_data/datasources/cdc_tracking_network/documentation/api_examples.md diff --git a/src/biome/datasources/cdc_tracking_network/documentation/examples.yaml b/src/biome/adhoc_data/datasources/cdc_tracking_network/documentation/examples.yaml similarity index 100% rename from src/biome/datasources/cdc_tracking_network/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/cdc_tracking_network/documentation/examples.yaml diff --git a/src/biome/datasources/cdc_tracking_network/documentation/user_guide.md b/src/biome/adhoc_data/datasources/cdc_tracking_network/documentation/user_guide.md similarity index 100% rename from src/biome/datasources/cdc_tracking_network/documentation/user_guide.md rename to src/biome/adhoc_data/datasources/cdc_tracking_network/documentation/user_guide.md diff --git a/src/biome/datasources/census_acs/api.yaml b/src/biome/adhoc_data/datasources/census_acs/api.yaml similarity index 100% rename from src/biome/datasources/census_acs/api.yaml rename to src/biome/adhoc_data/datasources/census_acs/api.yaml diff --git a/src/biome/datasources/census_acs/documentation/census_sdk.md b/src/biome/adhoc_data/datasources/census_acs/documentation/census_sdk.md similarity index 100% rename from src/biome/datasources/census_acs/documentation/census_sdk.md rename to src/biome/adhoc_data/datasources/census_acs/documentation/census_sdk.md diff --git a/src/biome/datasources/census_acs/documentation/census_web.md b/src/biome/adhoc_data/datasources/census_acs/documentation/census_web.md similarity index 100% rename from src/biome/datasources/census_acs/documentation/census_web.md rename to src/biome/adhoc_data/datasources/census_acs/documentation/census_web.md diff --git a/src/biome/datasources/census_acs/documentation/examples.yaml b/src/biome/adhoc_data/datasources/census_acs/documentation/examples.yaml similarity index 100% rename from src/biome/datasources/census_acs/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/census_acs/documentation/examples.yaml diff --git a/src/biome/datasources/chis_california_asthma/api.yaml b/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml similarity index 100% rename from src/biome/datasources/chis_california_asthma/api.yaml rename to src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml diff --git a/src/biome/datasources/chis_california_asthma/documentation/examples.yaml b/src/biome/adhoc_data/datasources/chis_california_asthma/documentation/examples.yaml similarity index 100% rename from src/biome/datasources/chis_california_asthma/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/chis_california_asthma/documentation/examples.yaml diff --git a/src/biome/datasources/epa_tri/api.yaml b/src/biome/adhoc_data/datasources/epa_tri/api.yaml similarity index 100% rename from src/biome/datasources/epa_tri/api.yaml rename to src/biome/adhoc_data/datasources/epa_tri/api.yaml diff --git a/src/biome/datasources/epa_tri/documentation/examples.yaml b/src/biome/adhoc_data/datasources/epa_tri/documentation/examples.yaml similarity index 100% rename from src/biome/datasources/epa_tri/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/epa_tri/documentation/examples.yaml diff --git a/src/biome/datasources/faers/api.yaml b/src/biome/adhoc_data/datasources/faers/api.yaml similarity index 100% rename from src/biome/datasources/faers/api.yaml rename to src/biome/adhoc_data/datasources/faers/api.yaml diff --git a/src/biome/datasources/faers/documentation/examples.yaml b/src/biome/adhoc_data/datasources/faers/documentation/examples.yaml similarity index 100% rename from src/biome/datasources/faers/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/faers/documentation/examples.yaml diff --git a/src/biome/datasources/faers/documentation/faers_fields_reference.csv b/src/biome/adhoc_data/datasources/faers/documentation/faers_fields_reference.csv similarity index 100% rename from src/biome/datasources/faers/documentation/faers_fields_reference.csv rename to src/biome/adhoc_data/datasources/faers/documentation/faers_fields_reference.csv diff --git a/src/biome/datasources/faers/documentation/faers_web.md b/src/biome/adhoc_data/datasources/faers/documentation/faers_web.md similarity index 100% rename from src/biome/datasources/faers/documentation/faers_web.md rename to src/biome/adhoc_data/datasources/faers/documentation/faers_web.md diff --git a/src/biome/datasources/gdc/api.yaml b/src/biome/adhoc_data/datasources/gdc/api.yaml similarity index 100% rename from src/biome/datasources/gdc/api.yaml rename to src/biome/adhoc_data/datasources/gdc/api.yaml diff --git a/src/biome/datasources/gdc/documentation/examples.yaml b/src/biome/adhoc_data/datasources/gdc/documentation/examples.yaml similarity index 100% rename from src/biome/datasources/gdc/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/gdc/documentation/examples.yaml diff --git a/src/biome/datasources/gdc/documentation/facets.txt b/src/biome/adhoc_data/datasources/gdc/documentation/facets.txt similarity index 100% rename from src/biome/datasources/gdc/documentation/facets.txt rename to src/biome/adhoc_data/datasources/gdc/documentation/facets.txt diff --git a/src/biome/datasources/gdc/documentation/gdc.md b/src/biome/adhoc_data/datasources/gdc/documentation/gdc.md similarity index 100% rename from src/biome/datasources/gdc/documentation/gdc.md rename to src/biome/adhoc_data/datasources/gdc/documentation/gdc.md diff --git a/src/biome/datasources/gdc/documentation/gdc_mappings.json b/src/biome/adhoc_data/datasources/gdc/documentation/gdc_mappings.json similarity index 100% rename from src/biome/datasources/gdc/documentation/gdc_mappings.json rename to src/biome/adhoc_data/datasources/gdc/documentation/gdc_mappings.json diff --git a/src/biome/datasources/gras/api.yaml b/src/biome/adhoc_data/datasources/gras/api.yaml similarity index 100% rename from src/biome/datasources/gras/api.yaml rename to src/biome/adhoc_data/datasources/gras/api.yaml diff --git a/src/biome/datasources/gras/documentation/examples.yaml b/src/biome/adhoc_data/datasources/gras/documentation/examples.yaml similarity index 100% rename from src/biome/datasources/gras/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/gras/documentation/examples.yaml diff --git a/src/biome/datasources/hpa/api.yaml b/src/biome/adhoc_data/datasources/hpa/api.yaml similarity index 100% rename from src/biome/datasources/hpa/api.yaml rename to src/biome/adhoc_data/datasources/hpa/api.yaml diff --git a/src/biome/datasources/hpa/documentation/examples.yaml b/src/biome/adhoc_data/datasources/hpa/documentation/examples.yaml similarity index 100% rename from src/biome/datasources/hpa/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/hpa/documentation/examples.yaml diff --git a/src/biome/datasources/hpa/documentation/hpa_docs.md b/src/biome/adhoc_data/datasources/hpa/documentation/hpa_docs.md similarity index 100% rename from src/biome/datasources/hpa/documentation/hpa_docs.md rename to src/biome/adhoc_data/datasources/hpa/documentation/hpa_docs.md diff --git a/src/biome/datasources/idc/api.yaml b/src/biome/adhoc_data/datasources/idc/api.yaml similarity index 100% rename from src/biome/datasources/idc/api.yaml rename to src/biome/adhoc_data/datasources/idc/api.yaml diff --git a/src/biome/datasources/idc/documentation/examples.yaml b/src/biome/adhoc_data/datasources/idc/documentation/examples.yaml similarity index 100% rename from src/biome/datasources/idc/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/idc/documentation/examples.yaml diff --git a/src/biome/datasources/idc/documentation/idc.md b/src/biome/adhoc_data/datasources/idc/documentation/idc.md similarity index 100% rename from src/biome/datasources/idc/documentation/idc.md rename to src/biome/adhoc_data/datasources/idc/documentation/idc.md diff --git a/src/biome/datasources/indra/api.yaml b/src/biome/adhoc_data/datasources/indra/api.yaml similarity index 100% rename from src/biome/datasources/indra/api.yaml rename to src/biome/adhoc_data/datasources/indra/api.yaml diff --git a/src/biome/datasources/indra/documentation/examples.yaml b/src/biome/adhoc_data/datasources/indra/documentation/examples.yaml similarity index 100% rename from src/biome/datasources/indra/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/indra/documentation/examples.yaml diff --git a/src/biome/datasources/indra/documentation/indra.json b/src/biome/adhoc_data/datasources/indra/documentation/indra.json similarity index 100% rename from src/biome/datasources/indra/documentation/indra.json rename to src/biome/adhoc_data/datasources/indra/documentation/indra.json diff --git a/src/biome/datasources/netrias/api.yaml b/src/biome/adhoc_data/datasources/netrias/api.yaml similarity index 100% rename from src/biome/datasources/netrias/api.yaml rename to src/biome/adhoc_data/datasources/netrias/api.yaml diff --git a/src/biome/datasources/netrias/documentation/examples.yaml b/src/biome/adhoc_data/datasources/netrias/documentation/examples.yaml similarity index 100% rename from src/biome/datasources/netrias/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/netrias/documentation/examples.yaml diff --git a/src/biome/datasources/netrias/documentation/netrias.json b/src/biome/adhoc_data/datasources/netrias/documentation/netrias.json similarity index 100% rename from src/biome/datasources/netrias/documentation/netrias.json rename to src/biome/adhoc_data/datasources/netrias/documentation/netrias.json diff --git a/src/biome/datasources/nhanes_dietary/api.yaml b/src/biome/adhoc_data/datasources/nhanes_dietary/api.yaml similarity index 100% rename from src/biome/datasources/nhanes_dietary/api.yaml rename to src/biome/adhoc_data/datasources/nhanes_dietary/api.yaml diff --git a/src/biome/datasources/nhanes_dietary/documentation/docs.md b/src/biome/adhoc_data/datasources/nhanes_dietary/documentation/docs.md similarity index 100% rename from src/biome/datasources/nhanes_dietary/documentation/docs.md rename to src/biome/adhoc_data/datasources/nhanes_dietary/documentation/docs.md diff --git a/src/biome/datasources/nhanes_dietary/documentation/examples.yaml b/src/biome/adhoc_data/datasources/nhanes_dietary/documentation/examples.yaml similarity index 100% rename from src/biome/datasources/nhanes_dietary/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/nhanes_dietary/documentation/examples.yaml diff --git a/src/biome/datasources/nsch/api.yaml b/src/biome/adhoc_data/datasources/nsch/api.yaml similarity index 100% rename from src/biome/datasources/nsch/api.yaml rename to src/biome/adhoc_data/datasources/nsch/api.yaml diff --git a/src/biome/datasources/nsch/documentation/examples.yaml b/src/biome/adhoc_data/datasources/nsch/documentation/examples.yaml similarity index 100% rename from src/biome/datasources/nsch/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/nsch/documentation/examples.yaml diff --git a/src/biome/datasources/pdc/api.yaml b/src/biome/adhoc_data/datasources/pdc/api.yaml similarity index 100% rename from src/biome/datasources/pdc/api.yaml rename to src/biome/adhoc_data/datasources/pdc/api.yaml diff --git a/src/biome/datasources/pdc/documentation/examples.yaml b/src/biome/adhoc_data/datasources/pdc/documentation/examples.yaml similarity index 100% rename from src/biome/datasources/pdc/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/pdc/documentation/examples.yaml diff --git a/src/biome/datasources/pdc/documentation/pdc_schema.graphql b/src/biome/adhoc_data/datasources/pdc/documentation/pdc_schema.graphql similarity index 100% rename from src/biome/datasources/pdc/documentation/pdc_schema.graphql rename to src/biome/adhoc_data/datasources/pdc/documentation/pdc_schema.graphql diff --git a/src/biome/datasources/synapse/api.yaml b/src/biome/adhoc_data/datasources/synapse/api.yaml similarity index 100% rename from src/biome/datasources/synapse/api.yaml rename to src/biome/adhoc_data/datasources/synapse/api.yaml diff --git a/src/biome/datasources/synapse/documentation/SynapseOpenApiSpec.json b/src/biome/adhoc_data/datasources/synapse/documentation/SynapseOpenApiSpec.json similarity index 100% rename from src/biome/datasources/synapse/documentation/SynapseOpenApiSpec.json rename to src/biome/adhoc_data/datasources/synapse/documentation/SynapseOpenApiSpec.json diff --git a/src/biome/datasources/synapse/documentation/examples.yaml b/src/biome/adhoc_data/datasources/synapse/documentation/examples.yaml similarity index 100% rename from src/biome/datasources/synapse/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/synapse/documentation/examples.yaml diff --git a/src/biome/datasources/synapse/documentation/nf_programmatic_webdocs.md b/src/biome/adhoc_data/datasources/synapse/documentation/nf_programmatic_webdocs.md similarity index 100% rename from src/biome/datasources/synapse/documentation/nf_programmatic_webdocs.md rename to src/biome/adhoc_data/datasources/synapse/documentation/nf_programmatic_webdocs.md diff --git a/src/biome/datasources/synapse/documentation/web_docs.md b/src/biome/adhoc_data/datasources/synapse/documentation/web_docs.md similarity index 100% rename from src/biome/datasources/synapse/documentation/web_docs.md rename to src/biome/adhoc_data/datasources/synapse/documentation/web_docs.md diff --git a/src/biome/datasources/usda_foodcentral/api.yaml b/src/biome/adhoc_data/datasources/usda_foodcentral/api.yaml similarity index 100% rename from src/biome/datasources/usda_foodcentral/api.yaml rename to src/biome/adhoc_data/datasources/usda_foodcentral/api.yaml diff --git a/src/biome/datasources/usda_foodcentral/documentation/examples.yaml b/src/biome/adhoc_data/datasources/usda_foodcentral/documentation/examples.yaml similarity index 100% rename from src/biome/datasources/usda_foodcentral/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/usda_foodcentral/documentation/examples.yaml diff --git a/src/biome/datasources/usda_foodcentral/documentation/foodcentral_openapi.json b/src/biome/adhoc_data/datasources/usda_foodcentral/documentation/foodcentral_openapi.json similarity index 100% rename from src/biome/datasources/usda_foodcentral/documentation/foodcentral_openapi.json rename to src/biome/adhoc_data/datasources/usda_foodcentral/documentation/foodcentral_openapi.json diff --git a/src/biome/datasources/usgs_pesticide/api.yaml b/src/biome/adhoc_data/datasources/usgs_pesticide/api.yaml similarity index 100% rename from src/biome/datasources/usgs_pesticide/api.yaml rename to src/biome/adhoc_data/datasources/usgs_pesticide/api.yaml diff --git a/src/biome/datasources/usgs_pesticide/documentation/examples.yaml b/src/biome/adhoc_data/datasources/usgs_pesticide/documentation/examples.yaml similarity index 100% rename from src/biome/datasources/usgs_pesticide/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/usgs_pesticide/documentation/examples.yaml diff --git a/src/biome/datasources/waterservices_usgs/api.yaml b/src/biome/adhoc_data/datasources/waterservices_usgs/api.yaml similarity index 100% rename from src/biome/datasources/waterservices_usgs/api.yaml rename to src/biome/adhoc_data/datasources/waterservices_usgs/api.yaml diff --git a/src/biome/datasources/waterservices_usgs/documentation/examples.yaml b/src/biome/adhoc_data/datasources/waterservices_usgs/documentation/examples.yaml similarity index 100% rename from src/biome/datasources/waterservices_usgs/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/waterservices_usgs/documentation/examples.yaml diff --git a/src/biome/datasources/waterservices_usgs/documentation/site_types.md b/src/biome/adhoc_data/datasources/waterservices_usgs/documentation/site_types.md similarity index 100% rename from src/biome/datasources/waterservices_usgs/documentation/site_types.md rename to src/biome/adhoc_data/datasources/waterservices_usgs/documentation/site_types.md diff --git a/src/biome/datasources/waterservices_usgs/documentation/usgs_waterservices_web.md b/src/biome/adhoc_data/datasources/waterservices_usgs/documentation/usgs_waterservices_web.md similarity index 100% rename from src/biome/datasources/waterservices_usgs/documentation/usgs_waterservices_web.md rename to src/biome/adhoc_data/datasources/waterservices_usgs/documentation/usgs_waterservices_web.md diff --git a/src/biome/instructions/README.CLINVAR.QUERIES.md b/src/biome/adhoc_data/instructions/README.CLINVAR.QUERIES.md similarity index 100% rename from src/biome/instructions/README.CLINVAR.QUERIES.md rename to src/biome/adhoc_data/instructions/README.CLINVAR.QUERIES.md diff --git a/src/biome/instructions/README.GTEX.QUERIES.md b/src/biome/adhoc_data/instructions/README.GTEX.QUERIES.md similarity index 100% rename from src/biome/instructions/README.GTEX.QUERIES.md rename to src/biome/adhoc_data/instructions/README.GTEX.QUERIES.md diff --git a/src/biome/instructions/README.REFSEQ.QUERIES.md b/src/biome/adhoc_data/instructions/README.REFSEQ.QUERIES.md similarity index 100% rename from src/biome/instructions/README.REFSEQ.QUERIES.md rename to src/biome/adhoc_data/instructions/README.REFSEQ.QUERIES.md diff --git a/src/biome/prompts/agent_prompt.md b/src/biome/adhoc_data/prompts/agent_prompt.md similarity index 100% rename from src/biome/prompts/agent_prompt.md rename to src/biome/adhoc_data/prompts/agent_prompt.md diff --git a/src/biome/prompts/consult_integration_docs.md b/src/biome/adhoc_data/prompts/consult_integration_docs.md similarity index 100% rename from src/biome/prompts/consult_integration_docs.md rename to src/biome/adhoc_data/prompts/consult_integration_docs.md diff --git a/src/biome/prompts/draft_integration_code.md b/src/biome/adhoc_data/prompts/draft_integration_code.md similarity index 100% rename from src/biome/prompts/draft_integration_code.md rename to src/biome/adhoc_data/prompts/draft_integration_code.md diff --git a/src/biome/agent.py b/src/biome/agent.py index d923265..72c55d6 100644 --- a/src/biome/agent.py +++ b/src/biome/agent.py @@ -39,7 +39,7 @@ def debug(self, message): # Load docstrings at module level def load_docstring(filename): - root_folder = Path(__file__).resolve().parent + root_folder = Path(__file__).resolve().parent / 'adhoc_data' with open(os.path.join(root_folder, 'prompts', filename), 'r') as f: return f.read() @@ -57,8 +57,8 @@ def ignore_tags(loader, tag_suffix, node): yaml.add_multi_constructor('', ignore_tags, yaml.SafeLoader) -DRAFT_INTEGRATION_CODE_DOC = load_docstring('draft_integration_code.md') -CONSULT_INTEGRATIONS_DOCS_DOC = load_docstring('consult_integration_docs.md') +# DRAFT_INTEGRATION_CODE_DOC = load_docstring('draft_integration_code.md') +# CONSULT_INTEGRATIONS_DOCS_DOC = load_docstring('consult_integration_docs.md') class BiomeAgent(BeakerAgent): """ @@ -71,25 +71,25 @@ class BiomeAgent(BeakerAgent): def __init__(self, context: BaseContext = None, tools: list = None, **kwargs): logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO')) self.logger = MessageLogger(self.log, logger) + super().__init__(context, tools, **kwargs) - self.root_folder = Path(__file__).resolve().parent - self.fetch_specs() + # self.root_folder = Path(__file__).resolve().parent + # self.fetch_specs() - instructions_dir = Path(os.path.join(self.root_folder, 'instructions')) - self.instructions = "\n".join( - file.read_text() - for file in instructions_dir.iterdir() if file.is_file() - ) + # instructions_dir = Path(os.path.join(self.root_folder, 'instructions')) + # self.instructions = "\n".join( + # file.read_text() + # for file in instructions_dir.iterdir() if file.is_file() + # ) - super().__init__(context, tools, **kwargs) - self.initialize_adhoc() - - # Load prompt files and set the Agent context - prompts_dir = os.path.join(self.root_folder, 'prompts') - with open(os.path.join(prompts_dir, 'agent_prompt.md'), 'r') as f: - template = f.read() - self.__doc__ = template.format(api_list=self.integration_list, instructions=self.instructions) - self.add_context(self.__doc__) + # self.initialize_adhoc() + + # # Load prompt files and set the Agent context + # prompts_dir = os.path.join(self.root_folder, 'prompts') + # with open(os.path.join(prompts_dir, 'agent_prompt.md'), 'r') as f: + # template = f.read() + # self.__doc__ = template.format(api_list=self.integration_list, instructions=self.instructions) + # self.add_context(self.__doc__) def fetch_specs(self): integration_root = os.path.join(self.root_folder, INTEGRATIONS_FOLDER) diff --git a/src/biome/context.py b/src/biome/context.py index 21e2664..e62195f 100644 --- a/src/biome/context.py +++ b/src/biome/context.py @@ -13,6 +13,8 @@ from beaker_kernel.lib.context import BeakerContext, action from beaker_kernel.subkernels.python import PythonSubkernel from beaker_kernel.lib.types import Integration, IntegrationAttachment +from beaker_kernel.lib.integrations.base import BaseIntegrationProvider +from beaker_kernel.lib.integrations.adhoc import AdhocIntegrationProvider from .agent import BiomeAgent from .integration import create_folder_structure_for_integration, get_integration_folder, write_integration @@ -24,6 +26,9 @@ logger = logging.getLogger(__name__) +# TODO: Change this for better autodiscovery, etc +ADHOC_DIR_PATH = (Path(__file__).parent / "adhoc_data") + class BiomeContext(BeakerContext): @@ -31,7 +36,24 @@ class BiomeContext(BeakerContext): agent_cls: "BaseAgent" = BiomeAgent def __init__(self, beaker_kernel: "LLMKernel", config: Dict[str, Any]): + + from adhoc_api.uaii import gpt_41, o3_mini, claude_37_sonnet, gemini_15_pro + ttl_seconds = 1800 + drafter_config_gemini = {**gemini_15_pro, 'ttl_seconds': ttl_seconds, 'api_key': os.environ.get("GEMINI_API_KEY", "")} + drafter_config_anthropic = {**claude_37_sonnet, 'api_key': os.environ.get("ANTHROPIC_API_KEY")} + curator_config = {**o3_mini, 'api_key': os.environ.get("OPENAI_API_KEY")} + gpt_41_config = {**gpt_41, 'api_key': os.environ.get("OPENAI_API_KEY")} + super().__init__(beaker_kernel, self.agent_cls, config) + # Initialize adhoc integration + self.integrations.append(AdhocIntegrationProvider.from_file_structure( + adhoc_path=ADHOC_DIR_PATH, + drafter_config=[gpt_41_config, drafter_config_anthropic, drafter_config_gemini], + curator_config=curator_config, + contextualizer_config=gpt_41_config, + logger=logger, + )) + if not isinstance(self.subkernel, PythonSubkernel): raise ValueError("This context is only valid for Python.") @@ -59,7 +81,8 @@ async def get_integrations(self) -> list[Integration]: # get list of keys not inherent to a integrations for the user-files category attached_files = {} - for (_, spec) in self.agent.raw_specs: + # for (_, spec) in self.agent.raw_specs: + for spec in self.integrations[0].list_integrations(): attached_files[spec["name"]] = [] for attachment_key in [ key for key in spec.keys() if key not in [ @@ -97,14 +120,14 @@ async def get_integrations(self) -> list[Integration]: return [ Integration( slug=spec["slug"], - url=str(yaml_location), + url=str("yaml_location"), name=spec["name"], description=spec.get("description"), source=spec.get("documentation").replace("!fill", ""), attached_files=attached_files[spec["name"]], examples=spec.get("loaded_examples", []) ) - for (yaml_location, spec) in self.agent.raw_specs + for spec in self.integrations[0].list_integrations() ] async def save_integration(self, message): From 483dcbf70f05517fba7d9b357ec6e428e133c8d7 Mon Sep 17 00:00:00 2001 From: Matthew Printz Date: Mon, 23 Jun 2025 14:05:43 -0600 Subject: [PATCH 10/21] Fixes for end-to-end --- src/biome/agent.py | 50 ++++++++++++++++++++++---------------------- src/biome/context.py | 6 +++--- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/biome/agent.py b/src/biome/agent.py index 72c55d6..4b81a0f 100644 --- a/src/biome/agent.py +++ b/src/biome/agent.py @@ -173,31 +173,31 @@ def log(self, event_type: str, content = None) -> None: content=content ) - @tool() - @with_docstring('draft_integration_code.md') - async def draft_integration_code(self, integration: str, goal: str, agent: AgentRef, loop: LoopControllerRef, react_context: ReactContextRef) -> str: - logger.info(f"using integration: {integration}") - try: - code = self.api.use_api(integration, goal) - return f"Here is the code the drafter created to use the API to accomplish the goal: \n\n```\n{code}\n```" - except Exception as e: - if self.api is None: - return "Do not attempt to fix this result: there is no API key for the agent that creates the request. Inform the user that they need to specify GEMINI_API_KEY and consider this a successful tool invocation." - logger.error(str(e)) - return f"An error occurred while using the API. The error was: {str(e)}. Please try again with a different goal." - - @tool() - @with_docstring('consult_integration_docs.md') - async def consult_integration_docs(self, integration: str, query: str, agent: AgentRef, loop: LoopControllerRef, react_context: ReactContextRef) -> str: - logger.info(f"asking integration: {integration}") - try: - results = self.api.ask_api(integration, query) - return f"Here is the information I found about how to use the API: \n{results}" - except Exception as e: - if self.api is None: - return "Do not attempt to fix this result: there is no API for the agent that creates the request. Inform the user that they need to specify GEMINI_API_KEY and consider this a successful tool invocation." - logger.error(str(e)) - return f"An error occurred while asking the API. The error was: {str(e)}. Please try again with a different question." + # @tool() + # @with_docstring('draft_integration_code.md') + # async def draft_integration_code(self, integration: str, goal: str, agent: AgentRef, loop: LoopControllerRef, react_context: ReactContextRef) -> str: + # logger.info(f"using integration: {integration}") + # try: + # code = self.api.use_api(integration, goal) + # return f"Here is the code the drafter created to use the API to accomplish the goal: \n\n```\n{code}\n```" + # except Exception as e: + # if self.api is None: + # return "Do not attempt to fix this result: there is no API key for the agent that creates the request. Inform the user that they need to specify GEMINI_API_KEY and consider this a successful tool invocation." + # logger.error(str(e)) + # return f"An error occurred while using the API. The error was: {str(e)}. Please try again with a different goal." + + # @tool() + # @with_docstring('consult_integration_docs.md') + # async def consult_integration_docs(self, integration: str, query: str, agent: AgentRef, loop: LoopControllerRef, react_context: ReactContextRef) -> str: + # logger.info(f"asking integration: {integration}") + # try: + # results = self.api.ask_api(integration, query) + # return f"Here is the information I found about how to use the API: \n{results}" + # except Exception as e: + # if self.api is None: + # return "Do not attempt to fix this result: there is no API for the agent that creates the request. Inform the user that they need to specify GEMINI_API_KEY and consider this a successful tool invocation." + # logger.error(str(e)) + # return f"An error occurred while asking the API. The error was: {str(e)}. Please try again with a different question." @tool() diff --git a/src/biome/context.py b/src/biome/context.py index e62195f..b93db57 100644 --- a/src/biome/context.py +++ b/src/biome/context.py @@ -44,15 +44,15 @@ def __init__(self, beaker_kernel: "LLMKernel", config: Dict[str, Any]): curator_config = {**o3_mini, 'api_key': os.environ.get("OPENAI_API_KEY")} gpt_41_config = {**gpt_41, 'api_key': os.environ.get("OPENAI_API_KEY")} - super().__init__(beaker_kernel, self.agent_cls, config) # Initialize adhoc integration - self.integrations.append(AdhocIntegrationProvider.from_file_structure( + adhoc_integration = AdhocIntegrationProvider.from_file_structure( adhoc_path=ADHOC_DIR_PATH, drafter_config=[gpt_41_config, drafter_config_anthropic, drafter_config_gemini], curator_config=curator_config, contextualizer_config=gpt_41_config, logger=logger, - )) + ) + super().__init__(beaker_kernel, self.agent_cls, config, integrations=[adhoc_integration]) if not isinstance(self.subkernel, PythonSubkernel): raise ValueError("This context is only valid for Python.") From 52022bab2e1e580471c213fd71a0b004c371608a Mon Sep 17 00:00:00 2001 From: Satchel Baldwin Date: Thu, 26 Jun 2025 12:25:10 -0500 Subject: [PATCH 11/21] UNFINISHED (includes test stubs): remove adhoc handling from biome in favor of beaker --- src/biome/agent.py | 295 ++++++++++----------------------------- src/biome/context.py | 123 +++++----------- src/biome/integration.py | 152 -------------------- 3 files changed, 108 insertions(+), 462 deletions(-) delete mode 100644 src/biome/integration.py diff --git a/src/biome/agent.py b/src/biome/agent.py index 4b81a0f..70f4f40 100644 --- a/src/biome/agent.py +++ b/src/biome/agent.py @@ -1,22 +1,14 @@ -import json import logging -import re import requests -from time import sleep -import asyncio import os from archytas.tool_utils import AgentRef, LoopControllerRef, ReactContextRef, tool -from typing import Any, List +from typing import List from beaker_kernel.lib.agent import BeakerAgent from beaker_kernel.lib.context import BaseContext -import yaml from pathlib import Path -from adhoc_api.tool import AdhocApi, ensure_name_slug_compatibility -from adhoc_api.loader import load_yaml_api -from adhoc_api.uaii import gpt_41, o3_mini, claude_37_sonnet, gemini_15_pro logger = logging.getLogger(__name__) @@ -51,15 +43,6 @@ def decorator(func): return func return decorator -# yaml loader that ignores !tag directives for getting raw yaml text from datasources -def ignore_tags(loader, tag_suffix, node): - return tag_suffix + ' ' + node.value -yaml.add_multi_constructor('', ignore_tags, yaml.SafeLoader) - - -# DRAFT_INTEGRATION_CODE_DOC = load_docstring('draft_integration_code.md') -# CONSULT_INTEGRATIONS_DOCS_DOC = load_docstring('consult_integration_docs.md') - class BiomeAgent(BeakerAgent): """ You are the Biome Agent, a chat assistant that helps users with biomedical research tasks. @@ -73,133 +56,6 @@ def __init__(self, context: BaseContext = None, tools: list = None, **kwargs): self.logger = MessageLogger(self.log, logger) super().__init__(context, tools, **kwargs) - # self.root_folder = Path(__file__).resolve().parent - # self.fetch_specs() - - # instructions_dir = Path(os.path.join(self.root_folder, 'instructions')) - # self.instructions = "\n".join( - # file.read_text() - # for file in instructions_dir.iterdir() if file.is_file() - # ) - - # self.initialize_adhoc() - - # # Load prompt files and set the Agent context - # prompts_dir = os.path.join(self.root_folder, 'prompts') - # with open(os.path.join(prompts_dir, 'agent_prompt.md'), 'r') as f: - # template = f.read() - # self.__doc__ = template.format(api_list=self.integration_list, instructions=self.instructions) - # self.add_context(self.__doc__) - - def fetch_specs(self): - integration_root = os.path.join(self.root_folder, INTEGRATIONS_FOLDER) - - data_dir_raw = os.environ.get("BIOME_DATA_DIR", "./data") - try: - data_dir = Path(data_dir_raw).resolve(strict=True) - logger.info(f"Using data_dir: {data_dir}") - except OSError as e: - data_dir = '' - logger.error(f"Failed to set biome data dir: {data_dir_raw} does not exist: {e}") - self.data_dir = data_dir - - # Get API specs and directories in one pass - self.integration_specs = [] - self.raw_specs = [] # no interpolation - self.integration_directories = {} - for integration_dir in os.listdir(integration_root): - if integration_dir == '.ipynb_checkpoints': - os.rmdir(os.path.join(integration_root, integration_dir)) - - integration_full_path = os.path.join(integration_root, integration_dir) - if os.path.isdir(integration_full_path): - integration_yaml = Path(os.path.join(integration_full_path, 'api.yaml')) - if not integration_yaml.is_file(): - logger.warning(f"Ignoring malformed API: {integration_yaml}") - continue - - api_spec = load_yaml_api(integration_yaml) - raw_contents = integration_yaml.read_text() - raw_spec = yaml.safe_load(raw_contents) - - # Replace {DATASET_FILES_BASE_PATH} with data_dir path; { and {{ to reduce mental overhead - api_spec['documentation'] = api_spec['documentation'].replace('{DATASET_FILES_BASE_PATH}', str(data_dir)) - api_spec['documentation'] = api_spec['documentation'].replace('{{DATASET_FILES_BASE_PATH}}', str(data_dir)) - - if 'examples' in api_spec and isinstance(api_spec['examples'], list): - for example in api_spec['examples']: - if 'code' in example and isinstance(example['code'], str): - example['code'] = example['code'].replace('{{DATASET_FILES_BASE_PATH}}', str(data_dir)) - example['code'] = example['code'].replace('{DATASET_FILES_BASE_PATH}', str(data_dir)) - - try: - ensure_name_slug_compatibility(raw_spec) - # add the loaded examples in too, since we want that tag parsed but also the raw text as well - raw_spec['loaded_examples'] = api_spec.get('examples', []) - self.raw_specs.append((os.path.join(integration_dir, 'api.yaml'), raw_spec)) - except Exception as e: - logger.error(f"Failed to load integration `{integration_yaml}` from raw yaml: {e}") - - ensure_name_slug_compatibility(api_spec) - self.integration_specs.append(api_spec) - self.integration_directories[api_spec['slug']] = integration_dir - - def initialize_adhoc(self): - # Note: not all providers support ttl_seconds - ttl_seconds = 1800 - drafter_config_gemini = {**gemini_15_pro, 'ttl_seconds': ttl_seconds, 'api_key': os.environ.get("GEMINI_API_KEY", "")} - drafter_config_anthropic = {**claude_37_sonnet, 'api_key': os.environ.get("ANTHROPIC_API_KEY")} - curator_config = {**o3_mini, 'api_key': os.environ.get("OPENAI_API_KEY")} - gpt_41_config = {**gpt_41, 'api_key': os.environ.get("OPENAI_API_KEY")} - specs = self.integration_specs - - try: - self.api = AdhocApi( - apis=specs, - drafter_config=[gpt_41_config, drafter_config_anthropic, drafter_config_gemini], - curator_config=curator_config, - contextualizer_config=gpt_41_config, - logger=self.logger, - ) - except ValueError as e: - self.add_context(f"The datasources failed to load for this reason: {str(e)}. Please inform the user immediately.") - self.api = None - - self.integration_list = [spec['slug'] for spec in specs] - - def log(self, event_type: str, content = None) -> None: - self.context.beaker_kernel.log( - event_type=f"agent_{event_type}", - content=content - ) - - # @tool() - # @with_docstring('draft_integration_code.md') - # async def draft_integration_code(self, integration: str, goal: str, agent: AgentRef, loop: LoopControllerRef, react_context: ReactContextRef) -> str: - # logger.info(f"using integration: {integration}") - # try: - # code = self.api.use_api(integration, goal) - # return f"Here is the code the drafter created to use the API to accomplish the goal: \n\n```\n{code}\n```" - # except Exception as e: - # if self.api is None: - # return "Do not attempt to fix this result: there is no API key for the agent that creates the request. Inform the user that they need to specify GEMINI_API_KEY and consider this a successful tool invocation." - # logger.error(str(e)) - # return f"An error occurred while using the API. The error was: {str(e)}. Please try again with a different goal." - - # @tool() - # @with_docstring('consult_integration_docs.md') - # async def consult_integration_docs(self, integration: str, query: str, agent: AgentRef, loop: LoopControllerRef, react_context: ReactContextRef) -> str: - # logger.info(f"asking integration: {integration}") - # try: - # results = self.api.ask_api(integration, query) - # return f"Here is the information I found about how to use the API: \n{results}" - # except Exception as e: - # if self.api is None: - # return "Do not attempt to fix this result: there is no API for the agent that creates the request. Inform the user that they need to specify GEMINI_API_KEY and consider this a successful tool invocation." - # logger.error(str(e)) - # return f"An error occurred while asking the API. The error was: {str(e)}. Please try again with a different question." - - @tool() async def drs_uri_info(self, uris: List[str]) -> List[dict]: """ @@ -234,88 +90,77 @@ async def drs_uri_info(self, uris: List[str]) -> List[dict]: return responses - @tool() - async def add_example(self, integration: str, code: str, query: str, notes: str = None) -> str: - """ - Add a successful code example to the integration's examples.yaml documentation file. - This tool should be used after successfully completing a task with an integration to capture the working code for future reference. - - The API names must match one of the names in the agent's integration list. - - Args: - integration (str): The name of the integration the example is for - code (str): The working, successful code to add as an example - query (str): A brief description of what the example demonstrates - notes (str, optional): Additional notes about the example, such as implementation details - - Returns: - str: Message indicating success or failure of adding the example - """ - if integration not in self.integration_list: - raise ValueError(f"Error: the API name must match one of the names in the {self.integration_list}. The API name provided was {api}.") - - self.context.beaker_kernel.send_response( - "iopub", "add_example", content={ - "integration": integration, - "code": code, - "query": query, - "notes": notes - } - ) - return "Successfully added example." - - @tool() - async def add_integration(self, - integration: str, - description: str, - base_url: str, - schema_location: str) -> str: - """ - Adds an integration to the list of supported integrations usable within Biome. - This will be added to the API and data source list. - - Args: - integration (str): The name of the target data source or API that will be added. - description (str): A plain text description of what the data source is based on your knowledge of what the user is asking for, combined with their description if their description is relevant, or, if you do not know about the target data source. If the user does not provide any information, rely on what you know. Target a paragraph in length. - schema_location (str): A URL or local filepath to fetch an OpenAPI schema from. If the user does not provide one, ask them for the URL or local filepath to the schema. - base_url (str): The base URL for the integration that will be used for making OpenAPI calls. If the user does not provide one, ask them for the base URL of the API. - Returns: - str: Message indicating success or failure of adding the integration. - """ - - try: - if schema_location.startswith('http'): - response = requests.get(schema_location) - if response.status_code != 200: - return f'Failed to get OpenAPI schema: {response.status_code}' - schema = response.content.decode("utf-8") - else: - with open(schema_location, 'r') as f: - schema = f.read() - except Exception as e: - return f'Failed to get OpenAPI schema: {e}' - - # calls save_integration in context.py as an action after finishing - self.context.beaker_kernel.send_response( - "iopub", "add_integration", content={ - "integration": integration, - "description": description, - "base_url": base_url, - "schema": schema - } - ) - return f"Added integration `{integration}`." - - @tool() - async def get_available_integrations(self) -> list: - """ - Get list of integrations that the agent is designed to interact with. + # @tool() + # async def add_example(self, integration: str, code: str, query: str, notes: str = None) -> str: + # """ + # Add a successful code example to the integration's examples.yaml documentation file. + # This tool should be used after successfully completing a task with an integration to capture the working code for future reference. + + # The API names must match one of the names in the agent's integration list. + + # Args: + # integration (str): The name of the integration the example is for + # code (str): The working, successful code to add as an example + # query (str): A brief description of what the example demonstrates + # notes (str, optional): Additional notes about the example, such as implementation details + + # Returns: + # str: Message indicating success or failure of adding the example + # """ + # if integration not in self.integration_list: + # raise ValueError(f"Error: the API name must match one of the names in the {self.integration_list}. The API name provided was {api}.") + + # self.context.beaker_kernel.send_response( + # "iopub", "add_example", content={ + # "integration": integration, + # "code": code, + # "query": query, + # "notes": notes + # } + # ) + # return "Successfully added example." - Returns: - list: The list of available integrations. - """ - return self.integration_list + # @tool() + # async def add_integration(self, + # integration: str, + # description: str, + # base_url: str, + # schema_location: str) -> str: + # """ + # Adds an integration to the list of supported integrations usable within Biome. + # This will be added to the API and data source list. + + # Args: + # integration (str): The name of the target data source or API that will be added. + # description (str): A plain text description of what the data source is based on your knowledge of what the user is asking for, combined with their description if their description is relevant, or, if you do not know about the target data source. If the user does not provide any information, rely on what you know. Target a paragraph in length. + # schema_location (str): A URL or local filepath to fetch an OpenAPI schema from. If the user does not provide one, ask them for the URL or local filepath to the schema. + # base_url (str): The base URL for the integration that will be used for making OpenAPI calls. If the user does not provide one, ask them for the base URL of the API. + # Returns: + # str: Message indicating success or failure of adding the integration. + # """ + # try: + # if schema_location.startswith('http'): + # response = requests.get(schema_location) + # if response.status_code != 200: + # return f'Failed to get OpenAPI schema: {response.status_code}' + # schema = response.content.decode("utf-8") + # else: + # with open(schema_location, 'r') as f: + # schema = f.read() + # except Exception as e: + # return f'Failed to get OpenAPI schema: {e}' + + # # calls save_integration in context.py as an action after finishing + # self.context.beaker_kernel.send_response( + # "iopub", "add_integration", content={ + # "integration": integration, + # "description": description, + # "base_url": base_url, + # "schema": schema + # } + # ) + # return f"Added integration `{integration}`." @tool() async def extract_pdf(self, pdf_path: str, agent: AgentRef) -> str: diff --git a/src/biome/context.py b/src/biome/context.py index b93db57..21ca7a6 100644 --- a/src/biome/context.py +++ b/src/biome/context.py @@ -1,12 +1,7 @@ +from dataclasses import asdict from typing import TYPE_CHECKING, Any, Dict import os -import re import logging -import json - -from jupyter_server.services.contents.filemanager import ( - FileContentsManager, -) from pathlib import Path @@ -17,7 +12,6 @@ from beaker_kernel.lib.integrations.adhoc import AdhocIntegrationProvider from .agent import BiomeAgent -from .integration import create_folder_structure_for_integration, get_integration_folder, write_integration if TYPE_CHECKING: from beaker_kernel.kernel import LLMKernel @@ -30,13 +24,40 @@ ADHOC_DIR_PATH = (Path(__file__).parent / "adhoc_data") -class BiomeContext(BeakerContext): +class TestIntegrationProvider(BaseIntegrationProvider): + integrations: list[Integration] + def __init__(self): + self.integrations = [ + Integration( + "test_1", + "Test Integration 1", + "First Test Integration" + ), + Integration( + "test_2", + "Test Integration 2", + "Second Test Integration" + ), + ] + super().__init__(display_name="Biome Second Test Integration") + def list_integrations(self): + return [asdict(i) for i in self.integrations] + def get_integration(self, integration_id): + pass + def add_integration(self, **payload): + pass + def add_resource(self, integration_id, **payload): + pass + def list_resources(self, integration_id, resource_type=None): + pass + def get_resource(self, integration_id, resource_id): + pass +class BiomeContext(BeakerContext): SLUG = "biome" agent_cls: "BaseAgent" = BiomeAgent def __init__(self, beaker_kernel: "LLMKernel", config: Dict[str, Any]): - from adhoc_api.uaii import gpt_41, o3_mini, claude_37_sonnet, gemini_15_pro ttl_seconds = 1800 drafter_config_gemini = {**gemini_15_pro, 'ttl_seconds': ttl_seconds, 'api_key': os.environ.get("GEMINI_API_KEY", "")} @@ -51,8 +72,15 @@ def __init__(self, beaker_kernel: "LLMKernel", config: Dict[str, Any]): curator_config=curator_config, contextualizer_config=gpt_41_config, logger=logger, + display_name="Biome Specialist Agents" + ) + test_integration = TestIntegrationProvider() + super().__init__( + beaker_kernel, + self.agent_cls, + config, + integrations=[adhoc_integration, test_integration] ) - super().__init__(beaker_kernel, self.agent_cls, config, integrations=[adhoc_integration]) if not isinstance(self.subkernel, PythonSubkernel): raise ValueError("This context is only valid for Python.") @@ -73,78 +101,3 @@ async def setup(self, context_info=None, parent_header=None): "netrias_api_key": os.environ.get("NETRIAS_KEY"), }) await self.execute(command) - - async def get_integrations(self) -> list[Integration]: - """ - fetch all of the adhoc-api integrations to pass to beaker. - """ - - # get list of keys not inherent to a integrations for the user-files category - attached_files = {} - # for (_, spec) in self.agent.raw_specs: - for spec in self.integrations[0].list_integrations(): - attached_files[spec["name"]] = [] - for attachment_key in [ - key for key in spec.keys() if key not in [ - "name", - "slug", - "description", - "cache_key", - "documentation", - "examples", - "cache_body", - "loaded_examples" - ] - ]: - if not isinstance(spec[attachment_key], str): - logger.warning(f"warning: key {attachment_key} on spec {spec['name']} is of type {type(spec[attachment_key])} and not str. ignoring and continuing") - continue - - # trim yaml tags since they will be readded at save time - # TODO: handle not-eliding documentation/ - filepath_raw = re.sub( - r"!load_[a-zA-Z]+", - "", - spec[attachment_key].strip() - ).strip().replace("documentation/", "") - - attached_files[spec["name"]].append(IntegrationAttachment( - name=attachment_key, - filepath=filepath_raw, - content=None, - is_empty_file=False - )) - - # manually load examples - - return [ - Integration( - slug=spec["slug"], - url=str("yaml_location"), - name=spec["name"], - description=spec.get("description"), - source=spec.get("documentation").replace("!fill", ""), - attached_files=attached_files[spec["name"]], - examples=spec.get("loaded_examples", []) - ) - for spec in self.integrations[0].list_integrations() - ] - - async def save_integration(self, message): - logger.warning('called save_integration') - manager = FileContentsManager() - content = message.content - write_integration(manager, content) - self.agent.fetch_specs() - self.agent.initialize_adhoc() - self.agent.add_context(f"A new integration has been added: `{content.get('slug')}`. You may now use this with `draft_integration_code`.") - - async def add_example(self, message): - logger.warning('called add_example') - manager = FileContentsManager() - content = message.content - logger.warning(content) - write_integration(manager, content) - self.agent.fetch_specs() - self.agent.initialize_adhoc() - self.agent.add_context(f"A new example has been added to `{content.get('slug')}.`") diff --git a/src/biome/integration.py b/src/biome/integration.py deleted file mode 100644 index bf4a862..0000000 --- a/src/biome/integration.py +++ /dev/null @@ -1,152 +0,0 @@ -import yaml -from pathlib import Path -import os -import logging - -logger = logging.getLogger(__name__) - -def str_presenter(dumper, data): - if len(data.splitlines()) > 1: # check for multiline string - return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") - return dumper.represent_scalar("tag:yaml.org,2002:str", data) -yaml.add_representer(str, str_presenter) -yaml.representer.SafeRepresenter.add_representer(str, str_presenter) - -# add dumpers but not loaders, since we need to get raw specifications elsewhere to give to beaker - -class LoadYamlTag(yaml.YAMLObject): - yaml_tag = "!load_yaml" - def __init__(self, payload): - self.payload = payload - def __repr__(self): - return f"LoadYamlTag({self.payload})" - @classmethod - def from_yaml(cls, loader, node): - return LoadYamlTag(node.value) - @classmethod - def to_yaml(cls, dumper, data): - logger.warning(msg="load_yaml dump") - return dumper.represent_scalar(cls.yaml_tag, data.payload) - -# yaml.SafeLoader.add_constructor("!load_yaml", LoadYamlTag.from_yaml) -yaml.SafeDumper.add_multi_representer(LoadYamlTag, LoadYamlTag.to_yaml) - - -class LoadTextTag(yaml.YAMLObject): - yaml_tag = "!load_txt" - def __init__(self, payload): - self.payload = payload - def __repr__(self): - return f"LoadTextTag({self.payload})" - @classmethod - def from_yaml(cls, loader, node): - return LoadTextTag(node.value) - @classmethod - def to_yaml(cls, dumper, data): - return dumper.represent_scalar(cls.yaml_tag, data.payload) - -# yaml.SafeLoader.add_constructor("!load_txt", LoadTextTag.from_yaml) -yaml.SafeDumper.add_multi_representer(LoadTextTag, LoadTextTag.to_yaml) - - -class FillTag(yaml.YAMLObject): - yaml_tag = "!fill" - def __init__(self, payload): - self.payload = payload - def __repr__(self): - return f"FillTag({self.payload})" - @classmethod - def from_yaml(cls, loader, node): - return FillTag(node.value) - @classmethod - def to_yaml(cls, dumper, data): - return dumper.represent_scalar(cls.yaml_tag, data.payload, style="|") - -# yaml.SafeLoader.add_constructor("!fill", FillTag.from_yaml) -yaml.SafeDumper.add_multi_representer(FillTag, FillTag.to_yaml) - - -def create_file_model(content: str): - return { - "type": "file", - "format": "text", - "content": content - } - - -def create_directory_model(): - return { - "type": "directory", - "format": "json", - "mimetype": None - } - - -def get_integration_slug(integration): - return integration.get("slug", integration.get("name", "").lower().replace(" ", "_")) - - -def get_integration_folder(integration): - url: str = integration.get("url", "") - if url == "": - return get_integration_slug(integration) - elif url.endswith("api.yaml"): - return url[:-(len("api.yaml"))] - else: - return url - - -def get_integration_base_path(integration): - return (Path(os.environ.get("BIOME_INTEGRATIONS_DIR", "./")) - / get_integration_folder(integration)).resolve() - - -def create_folder_structure_for_integration(manager, integration): - base_path = get_integration_base_path(integration) - if not manager.dir_exists(str(base_path)): - manager.save(create_directory_model(), str(base_path)) - documentation_path = base_path / "documentation" - if not manager.dir_exists(str(documentation_path)): - manager.save(create_directory_model(), str(documentation_path)) - - -def format_integration(integration): - saved_fields = { - "name": integration.get("name"), - "slug": get_integration_slug(integration), - "cache_key": f"integration_{get_integration_slug(integration)}", - "examples": LoadYamlTag("documentation/examples.yaml"), - "description": integration.get("description"), - "documentation": FillTag(integration.get("source")) - } - for file in integration.get("attached_files", []): - saved_fields[file.get("name")] = LoadTextTag( - f"documentation/{file.get('filepath')}" - ) - return yaml.safe_dump(saved_fields) - - -def write_integration(manager, integration): - base_path = get_integration_base_path(integration) - create_folder_structure_for_integration(manager, integration) - - example_path = base_path / "documentation" / "examples.yaml" - manager.save( - create_file_model(yaml.safe_dump(integration.get("examples", ""))), - str(example_path) - ) - - integration_yaml_path = base_path / "api.yaml" - manager.save( - create_file_model(format_integration(integration)), - str(integration_yaml_path) - ) - - for file in integration.get("attached_files", []): - # files that are not empty content have yet to be saved -- uploading can be handled by frontend. - # so if a tool creates a new file, it needs to be written here. - if (content := file.get("content", None)) is not None: - manager.save( - create_file_model(content), - str(base_path / "documentation" / file.get('filepath', 'invalid_filepath')) - ) From 438bd83ff3246b975b0bec228c136a03ff692b9f Mon Sep 17 00:00:00 2001 From: Satchel Baldwin Date: Thu, 26 Jun 2025 16:56:51 -0500 Subject: [PATCH 12/21] change to match the baseintegrationprovider now that mutable is different --- src/biome/context.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/biome/context.py b/src/biome/context.py index 21ca7a6..158d072 100644 --- a/src/biome/context.py +++ b/src/biome/context.py @@ -44,10 +44,6 @@ def list_integrations(self): return [asdict(i) for i in self.integrations] def get_integration(self, integration_id): pass - def add_integration(self, **payload): - pass - def add_resource(self, integration_id, **payload): - pass def list_resources(self, integration_id, resource_type=None): pass def get_resource(self, integration_id, resource_id): From 7d9cbfd1c4e45d926481c37ad06cc64a499d40f7 Mon Sep 17 00:00:00 2001 From: Satchel Baldwin Date: Thu, 26 Jun 2025 16:57:05 -0500 Subject: [PATCH 13/21] change yaml to new format --- src/biome/adhoc_data/datasources/ahs/api.yaml | 35 +++--- .../{documentation => attachments}/docs.md | 0 .../ahs/{documentation => }/examples.yaml | 0 src/biome/adhoc_data/datasources/aqs/api.yaml | 103 +++++++++--------- .../{documentation => attachments}/aqs.json | 0 .../aqs/{documentation => }/examples.yaml | 0 .../datasources/cbioportal/api.yaml | 16 ++- .../cbioportal.json | 0 .../{documentation => }/examples.yaml | 0 src/biome/adhoc_data/datasources/cda/api.yaml | 10 +- .../{documentation => attachments}/cda.yaml | 0 .../cda/{documentation => }/examples.yaml | 0 .../datasources/cdc_tracking_network/api.yaml | 33 +++--- .../api_examples.md | 0 .../user_guide.md | 0 .../{documentation => }/examples.yaml | 0 .../datasources/census_acs/api.yaml | 31 +++--- .../census_sdk.md | 0 .../census_web.md | 0 .../{documentation => }/examples.yaml | 0 .../chis_california_asthma/api.yaml | 31 +++--- .../{documentation => }/examples.yaml | 0 .../adhoc_data/datasources/epa_tri/api.yaml | 37 +++---- .../epa_tri/{documentation => }/examples.yaml | 0 .../adhoc_data/datasources/faers/api.yaml | 23 ++-- .../faers_fields_reference.csv | 0 .../faers_web.md | 0 .../faers/{documentation => }/examples.yaml | 0 src/biome/adhoc_data/datasources/gdc/api.yaml | 34 +++--- .../{documentation => attachments}/facets.txt | 0 .../gdc/{documentation => attachments}/gdc.md | 0 .../gdc_mappings.json | 0 .../gdc/{documentation => }/examples.yaml | 0 .../adhoc_data/datasources/gras/api.yaml | 25 ++--- .../gras/{documentation => }/examples.yaml | 0 src/biome/adhoc_data/datasources/hpa/api.yaml | 53 ++++----- .../hpa_docs.md | 0 .../hpa/{documentation => }/examples.yaml | 0 src/biome/adhoc_data/datasources/idc/api.yaml | 15 +-- .../idc/{documentation => attachments}/idc.md | 0 .../idc/{documentation => }/examples.yaml | 0 .../adhoc_data/datasources/indra/api.yaml | 44 ++++---- .../{documentation => attachments}/indra.json | 0 .../indra/{documentation => }/examples.yaml | 0 .../adhoc_data/datasources/netrias/api.yaml | 55 +++++----- .../netrias.json | 0 .../netrias/{documentation => }/examples.yaml | 0 .../datasources/nhanes_dietary/api.yaml | 79 +++++++------- .../{documentation => attachments}/docs.md | 0 .../{documentation => }/examples.yaml | 0 .../adhoc_data/datasources/nsch/api.yaml | 55 +++++----- .../nsch/{documentation => }/examples.yaml | 0 src/biome/adhoc_data/datasources/pdc/api.yaml | 21 ++-- .../pdc_schema.graphql | 0 .../pdc/{documentation => }/examples.yaml | 0 .../adhoc_data/datasources/synapse/api.yaml | 38 +++---- .../SynapseOpenApiSpec.json | 0 .../nf_programmatic_webdocs.md | 0 .../web_docs.md | 0 .../synapse/{documentation => }/examples.yaml | 0 .../datasources/usda_foodcentral/api.yaml | 27 ++--- .../foodcentral_openapi.json | 0 .../{documentation => }/examples.yaml | 0 .../datasources/usgs_pesticide/api.yaml | 47 ++++---- .../{documentation => }/examples.yaml | 0 .../datasources/waterservices_usgs/api.yaml | 23 ++-- .../site_types.md | 0 .../usgs_waterservices_web.md | 0 .../{documentation => }/examples.yaml | 0 69 files changed, 422 insertions(+), 413 deletions(-) rename src/biome/adhoc_data/datasources/ahs/{documentation => attachments}/docs.md (100%) rename src/biome/adhoc_data/datasources/ahs/{documentation => }/examples.yaml (100%) rename src/biome/adhoc_data/datasources/aqs/{documentation => attachments}/aqs.json (100%) rename src/biome/adhoc_data/datasources/aqs/{documentation => }/examples.yaml (100%) rename src/biome/adhoc_data/datasources/cbioportal/{documentation => attachments}/cbioportal.json (100%) rename src/biome/adhoc_data/datasources/cbioportal/{documentation => }/examples.yaml (100%) rename src/biome/adhoc_data/datasources/cda/{documentation => attachments}/cda.yaml (100%) rename src/biome/adhoc_data/datasources/cda/{documentation => }/examples.yaml (100%) rename src/biome/adhoc_data/datasources/cdc_tracking_network/{documentation => attachments}/api_examples.md (100%) rename src/biome/adhoc_data/datasources/cdc_tracking_network/{documentation => attachments}/user_guide.md (100%) rename src/biome/adhoc_data/datasources/cdc_tracking_network/{documentation => }/examples.yaml (100%) rename src/biome/adhoc_data/datasources/census_acs/{documentation => attachments}/census_sdk.md (100%) rename src/biome/adhoc_data/datasources/census_acs/{documentation => attachments}/census_web.md (100%) rename src/biome/adhoc_data/datasources/census_acs/{documentation => }/examples.yaml (100%) rename src/biome/adhoc_data/datasources/chis_california_asthma/{documentation => }/examples.yaml (100%) rename src/biome/adhoc_data/datasources/epa_tri/{documentation => }/examples.yaml (100%) rename src/biome/adhoc_data/datasources/faers/{documentation => attachments}/faers_fields_reference.csv (100%) rename src/biome/adhoc_data/datasources/faers/{documentation => attachments}/faers_web.md (100%) rename src/biome/adhoc_data/datasources/faers/{documentation => }/examples.yaml (100%) rename src/biome/adhoc_data/datasources/gdc/{documentation => attachments}/facets.txt (100%) rename src/biome/adhoc_data/datasources/gdc/{documentation => attachments}/gdc.md (100%) rename src/biome/adhoc_data/datasources/gdc/{documentation => attachments}/gdc_mappings.json (100%) rename src/biome/adhoc_data/datasources/gdc/{documentation => }/examples.yaml (100%) rename src/biome/adhoc_data/datasources/gras/{documentation => }/examples.yaml (100%) rename src/biome/adhoc_data/datasources/hpa/{documentation => attachments}/hpa_docs.md (100%) rename src/biome/adhoc_data/datasources/hpa/{documentation => }/examples.yaml (100%) rename src/biome/adhoc_data/datasources/idc/{documentation => attachments}/idc.md (100%) rename src/biome/adhoc_data/datasources/idc/{documentation => }/examples.yaml (100%) rename src/biome/adhoc_data/datasources/indra/{documentation => attachments}/indra.json (100%) rename src/biome/adhoc_data/datasources/indra/{documentation => }/examples.yaml (100%) rename src/biome/adhoc_data/datasources/netrias/{documentation => attachments}/netrias.json (100%) rename src/biome/adhoc_data/datasources/netrias/{documentation => }/examples.yaml (100%) rename src/biome/adhoc_data/datasources/nhanes_dietary/{documentation => attachments}/docs.md (100%) rename src/biome/adhoc_data/datasources/nhanes_dietary/{documentation => }/examples.yaml (100%) rename src/biome/adhoc_data/datasources/nsch/{documentation => }/examples.yaml (100%) rename src/biome/adhoc_data/datasources/pdc/{documentation => attachments}/pdc_schema.graphql (100%) rename src/biome/adhoc_data/datasources/pdc/{documentation => }/examples.yaml (100%) rename src/biome/adhoc_data/datasources/synapse/{documentation => attachments}/SynapseOpenApiSpec.json (100%) rename src/biome/adhoc_data/datasources/synapse/{documentation => attachments}/nf_programmatic_webdocs.md (100%) rename src/biome/adhoc_data/datasources/synapse/{documentation => attachments}/web_docs.md (100%) rename src/biome/adhoc_data/datasources/synapse/{documentation => }/examples.yaml (100%) rename src/biome/adhoc_data/datasources/usda_foodcentral/{documentation => attachments}/foodcentral_openapi.json (100%) rename src/biome/adhoc_data/datasources/usda_foodcentral/{documentation => }/examples.yaml (100%) rename src/biome/adhoc_data/datasources/usgs_pesticide/{documentation => }/examples.yaml (100%) rename src/biome/adhoc_data/datasources/waterservices_usgs/{documentation => attachments}/site_types.md (100%) rename src/biome/adhoc_data/datasources/waterservices_usgs/{documentation => attachments}/usgs_waterservices_web.md (100%) rename src/biome/adhoc_data/datasources/waterservices_usgs/{documentation => }/examples.yaml (100%) diff --git a/src/biome/adhoc_data/datasources/ahs/api.yaml b/src/biome/adhoc_data/datasources/ahs/api.yaml index 006ddd8..9130a81 100644 --- a/src/biome/adhoc_data/datasources/ahs/api.yaml +++ b/src/biome/adhoc_data/datasources/ahs/api.yaml @@ -1,22 +1,23 @@ name: "Census American Housing Survey (AHS)" -cache_key: api_assistant_census_ahs_files +integration_type: dataset + description: | You have access to knowledge to all the Census Housing survey data, every 2 years from the year 1999 to 2023. You can use this API for answering questions on american housing, specifically their housing conditions (mold, water problems, etc), which can be used to correlate to health conditions, or other purposes. -more_instructions: !load_txt documentation/docs.md -examples: !load_yaml documentation/examples.yaml -cache_body: - default: true -documentation: !fill | + +attachments: + more_instructions: docs.md + +prompt: | You have access to a file-based Census American Housing Survey - (National Public Use Files CSV) from every 2 years from 1999 to 2023, under the `{{DATASET_FILES_BASE_PATH}}/census-ahs` directory. + (National Public Use Files CSV) from every 2 years from 1999 to 2023, under the `${DATASET_FILES_BASE_PATH}/census-ahs` directory. The housing files are located and named per year as follows: - - {{DATASET_FILES_BASE_PATH}}/census-ahs/survey_2023.csv - - {{DATASET_FILES_BASE_PATH}}/census-ahs/survey_2021.csv + - ${DATASET_FILES_BASE_PATH}/census-ahs/survey_2023.csv + - ${DATASET_FILES_BASE_PATH}/census-ahs/survey_2021.csv - ... - - {{DATASET_FILES_BASE_PATH}}/census-ahs/survey_1999.csv + - ${DATASET_FILES_BASE_PATH}/census-ahs/survey_1999.csv the more recent onces are at the metropolitan area level, while older ones are only avaiable at the national level. Use whichever year files you need, depending on the user question. @@ -26,17 +27,17 @@ documentation: !fill | In particular, we have the household data file, which means we have the OMB13CBSA as a constantly availablecolumn to identify geographic area of survey( State-based Metropolitan and Micropolitan Statistical Areas). Pre-2015 have more geographical - column/information. Don't try to use state/county/etc if those are not present on + column/information. Don't try to use state/county/etc if those are not present on the census housing file for that year. - + Try to alreay have planned which columns you need for each, in order to process less columns/excess data since - merging both datasets will result in more than 1000 columns. Plan it out, select relevant columns + merging both datasets will result in more than 1000 columns. Plan it out, select relevant columns (list the column names once filtered to knkow which are available), then merge the datasets with those columns selected. Once you identify columns of interest available, you can use this csv codebook dictionary that explains what each column code means: - - {{DATASET_FILES_BASE_PATH}}/census-ahs/census_ahs_codebook.csv + - ${DATASET_FILES_BASE_PATH}/census-ahs/census_ahs_codebook.csv from the codebook,you can use the "Variable" column to match the variable you have access to (say "OMB13CBSA") and use the column "Response Codes" to get the meaning of the codes or the numeric code range values, for other types of variables. @@ -58,7 +59,7 @@ documentation: !fill | The dataset csv files are a bit big, in the sense that they're in wide format and contain too many columns. When planning what you need, select a subset - of the columns needed, including the geo/housing-characteristics columns for your analysis, + of the columns needed, including the geo/housing-characteristics columns for your analysis, but don't include the thousands of columns. # Overview of the American Housing Survey (AHS) @@ -114,6 +115,6 @@ documentation: !fill | Read the CSV fiels with index_col=False, like so: ``` - codebook = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/census-ahs/census_ahs_codebook.csv', dtype=str, index_col=False) + codebook = pd.read_csv('${DATASET_FILES_BASE_PATH}/census-ahs/census_ahs_codebook.csv', dtype=str, index_col=False) ``` - If you don't add the index_col=False, the first column values will disappear and the whole dataframe will be shifted! \ No newline at end of file + If you don't add the index_col=False, the first column values will disappear and the whole dataframe will be shifted! diff --git a/src/biome/adhoc_data/datasources/ahs/documentation/docs.md b/src/biome/adhoc_data/datasources/ahs/attachments/docs.md similarity index 100% rename from src/biome/adhoc_data/datasources/ahs/documentation/docs.md rename to src/biome/adhoc_data/datasources/ahs/attachments/docs.md diff --git a/src/biome/adhoc_data/datasources/ahs/documentation/examples.yaml b/src/biome/adhoc_data/datasources/ahs/examples.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/ahs/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/ahs/examples.yaml diff --git a/src/biome/adhoc_data/datasources/aqs/api.yaml b/src/biome/adhoc_data/datasources/aqs/api.yaml index 001b694..e8580b3 100644 --- a/src/biome/adhoc_data/datasources/aqs/api.yaml +++ b/src/biome/adhoc_data/datasources/aqs/api.yaml @@ -1,29 +1,30 @@ name: "EPA Air Quality System" -cache_key: api_assistant_aqs_json +integration_type: api + description: | - This API is the primary place to obtain row-level data - from the EPA's Air Quality System (AQS) database. AQS contains - ambient air sample data collected by state, local, tribal, and federal + This API is the primary place to obtain row-level data + from the EPA's Air Quality System (AQS) database. AQS contains + ambient air sample data collected by state, local, tribal, and federal air pollution control agencies from thousands of monitors around the nation. - It also contains meteorological data, descriptive information about + It also contains meteorological data, descriptive information about each monitoring station (including its geographic location and its operator), and information about the quality of the samples. Note, AQS does not contain - real-time air quality data (it can take 6 months from the time + real-time air quality data (it can take 6 months from the time data is collected until it is in AQS). For real-time data, please use the AirNow API. -raw_documentation: !load_txt documentation/aqs.json -examples: !load_yaml documentation/examples.yaml -cache_body: - default: true -documentation: !fill | + +attachments: + raw_documentation: aqs.json + +prompt: | # OpenAPI Spec JSON: - {raw_documentation} + ${raw_documentation} # Additional Instructions: - - This API is through OpenAPI Spec (Swagger). You will be provided the schema. + + This API is through OpenAPI Spec (Swagger). You will be provided the schema. All requests go to the following URL: https://aqs.epa.gov/data/api - That is the base URL for all requests. + That is the base URL for all requests. To use the AQS API always retrieve the email/key credentials from environment variables: - email: os.environ.get("API_EPA_AQS_EMAIL") @@ -31,7 +32,7 @@ documentation: !fill | Do not use placeholder auth values from the OpenAPI spec nor prompt the user for these values, just use the environment variables. Ensure you know the fields that come back from the OpenAPI spec in order to process the AQS data. - + Daily/yearly data can only be accessed one year at a time. If working for that in year/decade ranges, prefer to use the annualData endpoints, as the daily data may be too large to fit into context. If the user requests data for a year range, you will need to make multiple requests to the API- potentially sampling in between, or creating averages, in order to not make requests for every single year in the range. The earliest sample in the data set is from 1957. Year 1980 marks the beginning of nationally consistent operational and quality assurance procedures. @@ -43,7 +44,7 @@ documentation: !fill | AQS has a nominal quarterly reporting deadline: data must be reported by 90 days after the end of the calendar quarter in which they are collected. If you need to inspect the data of a JSON response before processing, and you're dealing with annual data, or multi-daily data, or - for a geographic region that could yield too much data to process (or fit into an LLM context window), + for a geographic region that could yield too much data to process (or fit into an LLM context window), better use this endpoint: ``` @@ -77,17 +78,17 @@ documentation: !fill | Here is a sample of the output of a Raw Data request with one data point: ``` - {{ + { "Header": [ - {{ + { "status": "success", "request_time": "2018-06-13T07:45:01-04:00", "url": "https://...", "rows": 1 - }} + } ], "Body": [ - {{ + { "state_code": "01", "county_code": "073", "site_number": "0023", @@ -113,9 +114,9 @@ documentation: !fill | "county_name": "Jefferson", "date_of_last_change": "2017-05-30", "cbsa_code": "13820" - }} + } ] - }} + } ``` # Error Handling and Status Codes @@ -124,37 +125,35 @@ documentation: !fill | If the API is not able to parse your request it will return a status code of 400 and only the header, which will include an array of error messages instead of a row count. For example: ``` - {{ - "Header": [ - {{ - "status": "Failed", - "request_time": "2018-06-13T08:05:46.588-04:00", - "url": "https://...", - "error": [ - "value is missing or the value is empty: param" - ] - }} - ], - "Body": [] - }} + { + "Header": [{ + "status": "Failed", + "request_time": "2018-06-13T08:05:46.588-04:00", + "url": "https://...", + "error": [ + "value is missing or the value is empty: param" + ] + }], + "Body": [] + } ``` - If the API is able to parse your request but no data matches your selections, it will return the - JSON header and an HTTP status of 200. The row count will be zero, + If the API is able to parse your request but no data matches your selections, it will return the + JSON header and an HTTP status of 200. The row count will be zero, the status will be a message that no data matched your selection criteria, and the body will be empty. ``` - {{ + { "Header": [ - {{ + { "status": "No data matched your selection", "request_time": "2018-06-13T10:15:42-04:00", "url": "https://...", "rows": 0 - }} + } ], "Body": [] - }} + } ``` # Request Limits and Terms of Service @@ -237,28 +236,28 @@ documentation: !fill | - Data Volume Management: Harris County has numerous monitoring stations (we found 24,532 daily ozone records for 2022 alone), requiring efficient data aggregation strategies. - Parameter Identification: The API requires specific parameter codes rather than names, necessitating lookup of these codes before querying data. - The EPA AQS API provides rich, detailed air quality data that can be invaluable + The EPA AQS API provides rich, detailed air quality data that can be invaluable for researching environmental factors affecting childhood asthma (and others). - By combining this data with health outcome data from other sources, researchers - can develop comprehensive analyses of how air quality impacts respiratory health + By combining this data with health outcome data from other sources, researchers + can develop comprehensive analyses of how air quality impacts respiratory health in specific geographic areas like a county or region. - The API's structure allows for flexible querying by location, time period, and + The API's structure allows for flexible querying by location, time period, and specific pollutants, making it a powerful tool for environmental health research. - However, users should be aware of specific parameter requirements and prepare + However, users should be aware of specific parameter requirements and prepare for handling large volumes of data when working with densely monitored areas. Note: If/when using the `metaData/isAvailable` endpoint to check if the API is available, use this sample response as guide: ``` - {{ + { "Header": [ - {{ + { "status": "API service is up and running healthy. Status: connection_pool: size: 5, connections: 1, in use: 1, waiting_in_queue: 0", "request_time": "2025-03-21T08:21:41-04:00", "url": "http://aqs.epa.gov/api/metaData/isAvailable?email={{email}}.com&key={{key}}" - }} + } ], "Data": [] - }} + } ``` - Never simulate this API- if it is not available ask the user how to proceed. \ No newline at end of file + Never simulate this API- if it is not available ask the user how to proceed. diff --git a/src/biome/adhoc_data/datasources/aqs/documentation/aqs.json b/src/biome/adhoc_data/datasources/aqs/attachments/aqs.json similarity index 100% rename from src/biome/adhoc_data/datasources/aqs/documentation/aqs.json rename to src/biome/adhoc_data/datasources/aqs/attachments/aqs.json diff --git a/src/biome/adhoc_data/datasources/aqs/documentation/examples.yaml b/src/biome/adhoc_data/datasources/aqs/examples.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/aqs/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/aqs/examples.yaml diff --git a/src/biome/adhoc_data/datasources/cbioportal/api.yaml b/src/biome/adhoc_data/datasources/cbioportal/api.yaml index 274a220..883e42d 100644 --- a/src/biome/adhoc_data/datasources/cbioportal/api.yaml +++ b/src/biome/adhoc_data/datasources/cbioportal/api.yaml @@ -1,17 +1,15 @@ - name: cbioportal slug: cbioportal -cache_key: api_assistant_cbioportal -examples: !load_yaml documentation/examples.yaml +integration_type: api description: | The cBioPortal for Cancer Genomics is an open-access, open-source resource for interactive exploration of multidimensional cancer genomics data sets. The goal of cBioPortal is to significantly lower the barriers between complex genomic data and cancer researchers by providing rapid, intuitive, and high-quality access to molecular profiles and clinical attributes from large-scale cancer genomics projects, and therefore to empower researchers to translate these rich data sets into biologic insights and clinical applications. -cbioportal: !load_txt documentation/cbioportal.json - +attachments: + cbioportal: cbioportal.json -documentation: !fill | - This API is through Open API Spec (Swagger). You will be provided the schema below. +prompt: | + This API is through Open API Spec (Swagger). You will be provided the schema below. All requests go to the following URL: https://www.cbioportal.org/ - - {cbioportal} + + ${cbioportal} diff --git a/src/biome/adhoc_data/datasources/cbioportal/documentation/cbioportal.json b/src/biome/adhoc_data/datasources/cbioportal/attachments/cbioportal.json similarity index 100% rename from src/biome/adhoc_data/datasources/cbioportal/documentation/cbioportal.json rename to src/biome/adhoc_data/datasources/cbioportal/attachments/cbioportal.json diff --git a/src/biome/adhoc_data/datasources/cbioportal/documentation/examples.yaml b/src/biome/adhoc_data/datasources/cbioportal/examples.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/cbioportal/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/cbioportal/examples.yaml diff --git a/src/biome/adhoc_data/datasources/cda/api.yaml b/src/biome/adhoc_data/datasources/cda/api.yaml index 964187b..a2dab67 100644 --- a/src/biome/adhoc_data/datasources/cda/api.yaml +++ b/src/biome/adhoc_data/datasources/cda/api.yaml @@ -1,4 +1,5 @@ name: "Cancer Data Aggregator" +integration_type: api description: | The Cancer Data Aggregator (CDA) is a service of the National Cancer Institutes' (NCI) Cancer Research Data Commons. We pull metadata for thousands of studies hosted at multiple data repositories across NCI, and make it available for search from a single tool so researchers can more easily find and reuse existing cancer research data. In between pulling and publishing, we thoroughly clean, harmonize, and cross-reference the metadata so you can easily do things like find subjects that have participated in multiple studies, discover data from a disease that was originally described in different ways at each repository, and compile all the data from your favorite program such as CPTAC - no matter where it ended up. @@ -53,11 +54,10 @@ description: | ## Data Standards Services (DSS) The DSS provides us with harmonized values mapped to the data sources above. In our current release, DSS has provided values for: ethnicity, file_format, morphology, primary_diagnosis, race, species, therapeutic_agent, source_material_type (cancer/normal), treatment_type, and vital_status. -raw_documentation: !load_txt documentation/cda.yaml -examples: !load_yaml documentation/examples.yaml -cache_key: api_assistant_cda_yaml -documentation: !fill | - {raw_documentation} +attachments: + raw_documentation: cda.yaml +prompt: | + ${raw_documentation} # Additional Instructions: diff --git a/src/biome/adhoc_data/datasources/cda/documentation/cda.yaml b/src/biome/adhoc_data/datasources/cda/attachments/cda.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/cda/documentation/cda.yaml rename to src/biome/adhoc_data/datasources/cda/attachments/cda.yaml diff --git a/src/biome/adhoc_data/datasources/cda/documentation/examples.yaml b/src/biome/adhoc_data/datasources/cda/examples.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/cda/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/cda/examples.yaml diff --git a/src/biome/adhoc_data/datasources/cdc_tracking_network/api.yaml b/src/biome/adhoc_data/datasources/cdc_tracking_network/api.yaml index fedfafe..047028a 100644 --- a/src/biome/adhoc_data/datasources/cdc_tracking_network/api.yaml +++ b/src/biome/adhoc_data/datasources/cdc_tracking_network/api.yaml @@ -1,37 +1,38 @@ name: "CDC Tracking Network" -cache_key: api_assistant_cdc_tracking_network_json +integration_type: api + description: | - The National Environmental Public Health Tracking Network (Tracking Network) - brings together health data and environmental data from national, state, county, - and city sources and provides supporting information to make the data easier + The National Environmental Public Health Tracking Network (Tracking Network) + brings together health data and environmental data from national, state, county, + and city sources and provides supporting information to make the data easier to understand. - The Tracking Network has data and information on environments and hazards, + The Tracking Network has data and information on environments and hazards, health effects, and population health. This resource includes childhood emergency department visits and hospitalizations at the county level. It provides: - Age-stratified data including children under 18 - County-level resolution - Information on asthma, and other conditions -user_guide: !load_txt documentation/user_guide.md -examples: !load_yaml documentation/examples.yaml -api_examples: !load_txt documentation/api_examples.md -cache_body: - default: true -documentation: !fill | - The Tracking Program collects, integrates, analyzes, and disseminates non-infectious disease, - environmental, and socio-economic data from various sources. + +attachments: + user_guide: user_guide.md + api_examples: api_examples.md + +prompt: | + The Tracking Program collects, integrates, analyzes, and disseminates non-infectious disease, + environmental, and socio-economic data from various sources. These include a collective of national, state, and local partners. This is an HTTP API, where the usage is free but rate-limiting is applied heavily when not using an API KEY. You will need an API Key. It is available in the environment variable: - apiToken: os.environ.get("API_CDC_TRACKING_NETWORK") - To fully understand specifics about what data is or is not available, + To fully understand specifics about what data is or is not available, you must run code against the API to see what features are available at what level of geography. - {user_guide} + ${user_guide} Never simulate this API- if it is not available ask the user how to proceed. The list of content areas is available at: - {api_examples} \ No newline at end of file + ${api_examples} diff --git a/src/biome/adhoc_data/datasources/cdc_tracking_network/documentation/api_examples.md b/src/biome/adhoc_data/datasources/cdc_tracking_network/attachments/api_examples.md similarity index 100% rename from src/biome/adhoc_data/datasources/cdc_tracking_network/documentation/api_examples.md rename to src/biome/adhoc_data/datasources/cdc_tracking_network/attachments/api_examples.md diff --git a/src/biome/adhoc_data/datasources/cdc_tracking_network/documentation/user_guide.md b/src/biome/adhoc_data/datasources/cdc_tracking_network/attachments/user_guide.md similarity index 100% rename from src/biome/adhoc_data/datasources/cdc_tracking_network/documentation/user_guide.md rename to src/biome/adhoc_data/datasources/cdc_tracking_network/attachments/user_guide.md diff --git a/src/biome/adhoc_data/datasources/cdc_tracking_network/documentation/examples.yaml b/src/biome/adhoc_data/datasources/cdc_tracking_network/examples.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/cdc_tracking_network/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/cdc_tracking_network/examples.yaml diff --git a/src/biome/adhoc_data/datasources/census_acs/api.yaml b/src/biome/adhoc_data/datasources/census_acs/api.yaml index 200fca7..2f8610f 100644 --- a/src/biome/adhoc_data/datasources/census_acs/api.yaml +++ b/src/biome/adhoc_data/datasources/census_acs/api.yaml @@ -1,21 +1,22 @@ name: "Census ACS (American Community Survey) and SF1" -cache_key: api_assistant_census_acs_json +integration_type: api + description: | - The Census ACS (American Community Survey) and SF1 (Decennial Census) API returns data that has been collected from + The Census ACS (American Community Survey) and SF1 (Decennial Census) API returns data that has been collected from the Census Bureau, a database that contains information on the population of the United States. Use this to control for socioeconomic factors (or for other purposes). -web_documentation: !load_txt documentation/census_web.md -sdk_documentation: !load_txt documentation/census_sdk.md -examples: !load_yaml documentation/examples.yaml -cache_body: - default: true -documentation: !fill | - The American Community Survey (ACS) is an ongoing survey that provides data every + +attachments: + web_documentation: census_web.md + sdk_documentation: census_sdk.md + +prompt: | + The American Community Survey (ACS) is an ongoing survey that provides data every year—giving communities the current information they need to make important decisions. - The ACS covers a broad range of topics about social, economic, housing, and demographic - characteristics of the U.S. population. + The ACS covers a broad range of topics about social, economic, housing, and demographic + characteristics of the U.S. population. - {web_documentation} + ${web_documentation} You will need an API Key to use the Census ACS API. It is available in the environment variable: @@ -26,11 +27,11 @@ documentation: !fill | Data ranges from 2005 to 2023. - {sdk_documentation} + ${sdk_documentation} ## For 1-year datasets when loading as file (may or may not apply when using python libraries...) ### Variable Changes - Variables, and the values they represent, may change over time. Use the file in `{{DATASET_FILES_BASE_PATH}}/census-acs/2023-1yr-api-changes.csv` as a guide for which variables have changed from the prior year for 2023 ACS 1-Year Detailed Tables, Data Profiles and Subject Tables. See below for a description of each change type. + Variables, and the values they represent, may change over time. Use the file in `${DATASET_FILES_BASE_PATH}/census-acs/2023-1yr-api-changes.csv` as a guide for which variables have changed from the prior year for 2023 ACS 1-Year Detailed Tables, Data Profiles and Subject Tables. See below for a description of each change type. No Change - The variable has not changed from the prior year (most variables). Updated - That variable has changed from the prior year and a matching variable for the current year has been found. - No Match - The variable has changed from the prior year and no matching or comparable variable has been found. \ No newline at end of file + No Match - The variable has changed from the prior year and no matching or comparable variable has been found. diff --git a/src/biome/adhoc_data/datasources/census_acs/documentation/census_sdk.md b/src/biome/adhoc_data/datasources/census_acs/attachments/census_sdk.md similarity index 100% rename from src/biome/adhoc_data/datasources/census_acs/documentation/census_sdk.md rename to src/biome/adhoc_data/datasources/census_acs/attachments/census_sdk.md diff --git a/src/biome/adhoc_data/datasources/census_acs/documentation/census_web.md b/src/biome/adhoc_data/datasources/census_acs/attachments/census_web.md similarity index 100% rename from src/biome/adhoc_data/datasources/census_acs/documentation/census_web.md rename to src/biome/adhoc_data/datasources/census_acs/attachments/census_web.md diff --git a/src/biome/adhoc_data/datasources/census_acs/documentation/examples.yaml b/src/biome/adhoc_data/datasources/census_acs/examples.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/census_acs/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/census_acs/examples.yaml diff --git a/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml b/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml index 5b95c21..0ce30e0 100644 --- a/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml +++ b/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml @@ -1,32 +1,31 @@ name: "CHIS California Asthma" -cache_key: api_assistant_chis_california_asthma_files +integration_type: dataset + description: | - This dataset contains current asthma prevalence, the estimated percentage of - Californians who have ever been diagnosed with asthma by a health care provider - AND report they still have asthma and/or had an asthma episode or attack within - the past 12 months, statewide and by county. The data are stratified by age group + This dataset contains current asthma prevalence, the estimated percentage of + Californians who have ever been diagnosed with asthma by a health care provider + AND report they still have asthma and/or had an asthma episode or attack within + the past 12 months, statewide and by county. The data are stratified by age group (all ages, 0-17, 18+, 0-4, 5-17, 18-64, 65+) and reported for 2-year periods. -examples: !load_yaml documentation/examples.yaml -cache_body: - default: true -documentation: !fill | + +prompt: | You have access to a file-based California CHIS Data-Current Asthma Prevalence by County 2015-2022 file, - located at `{{DATASET_FILES_BASE_PATH}}/chis_california_asthma/current-asthma-prevalence-by-county-2015_2022.csv`. + located at `${DATASET_FILES_BASE_PATH}/chis_california_asthma/current-asthma-prevalence-by-county-2015_2022.csv`. open it like so: ```python import pandas as pd - df = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/chis_california_asthma/current-asthma-prevalence-by-county-2015_2022.csv', encoding='latin1') + df = pd.read_csv('${DATASET_FILES_BASE_PATH}/chis_california_asthma/current-asthma-prevalence-by-county-2015_2022.csv', encoding='latin1') ``` - This dataset contains current asthma prevalence, the estimated percentage of - Californians who have ever been diagnosed with asthma by a health care provider - AND report they still have asthma and/or had an asthma episode or attack within - the past 12 months, statewide and by county. The data are stratified by age group + This dataset contains current asthma prevalence, the estimated percentage of + Californians who have ever been diagnosed with asthma by a health care provider + AND report they still have asthma and/or had an asthma episode or attack within + the past 12 months, statewide and by county. The data are stratified by age group (all ages, 0-17, 18+, 0-4, 5-17, 18-64, 65+) and reported for 2-year periods. The columns are: "COUNTY", "YEARS", "STRATA", "AGE GROUP", "CURRENT PREVALENCE", "95% CONFIDENCE INTERVAL", "COUNTIES GROUPED", "COMMENT" "Agre Group" possible values are: "All Ages", "0-17 years", "18+ years", "0-4 years", "5-17 years", "18-64 years", "65+ years" - You can use this API to answer questions on the current asthma prevalence in California by county and age group. \ No newline at end of file + You can use this API to answer questions on the current asthma prevalence in California by county and age group. diff --git a/src/biome/adhoc_data/datasources/chis_california_asthma/documentation/examples.yaml b/src/biome/adhoc_data/datasources/chis_california_asthma/examples.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/chis_california_asthma/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/chis_california_asthma/examples.yaml diff --git a/src/biome/adhoc_data/datasources/epa_tri/api.yaml b/src/biome/adhoc_data/datasources/epa_tri/api.yaml index 47ce7c8..5a47df7 100644 --- a/src/biome/adhoc_data/datasources/epa_tri/api.yaml +++ b/src/biome/adhoc_data/datasources/epa_tri/api.yaml @@ -1,20 +1,19 @@ name: "EPA Toxic Release Inventory (TRI)" -cache_key: api_assistant_epa_tri_files +integration_type: dataset + description: | - The Toxics Release Inventory (TRI) is a resource for learning about toxic chemical - releases and pollution prevention activities reported by industrial and federal - facilities. TRI data support informed decision-making by communities, + The Toxics Release Inventory (TRI) is a resource for learning about toxic chemical + releases and pollution prevention activities reported by industrial and federal + facilities. TRI data support informed decision-making by communities, government agencies, companies, and others. -examples: !load_yaml documentation/examples.yaml -cache_body: - default: true -documentation: !fill | - You have access to a file-based EPA Toxic Release Inventory (TRI) data, from 2014 to 2023, at the `{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv` file. + +prompt: | + You have access to a file-based EPA Toxic Release Inventory (TRI) data, from 2014 to 2023, at the `${DATASET_FILES_BASE_PATH}/epa-tri/EPA_TRI_Toxics_2014_2023.csv` file. The report is just one file- there's a year column, and then a list of chemicals released, and the amount released. It also contains geospatial information such as latitude, longitude, state/zip, and even EPA region. # Overview of the EPA TRI Dataset - TRI tracks the waste management of certain toxic chemicals that may pose a threat to human health and the environment. U.S. facilities in different industry sectors must report annually how much of each chemical they release into the environment and/or managed through recycling, energy recovery and treatment, as well as any practices implemented to prevent or reduce the generation of chemical waste. + TRI tracks the waste management of certain toxic chemicals that may pose a threat to human health and the environment. U.S. facilities in different industry sectors must report annually how much of each chemical they release into the environment and/or managed through recycling, energy recovery and treatment, as well as any practices implemented to prevent or reduce the generation of chemical waste. Contains information about toxic chemical releases reported by industrial and federal facilities; key features of this dataset include: - Facility information: Names, IDs, geographic coordinates, and locations @@ -31,7 +30,7 @@ documentation: !fill | There are currently 799 individually listed chemicals and 33 chemical categories covered by the TRI Program. Facilities that manufacture, process or otherwise use these chemicals in amounts above established levels must submit annual reporting forms for each chemical. Note that the TRI chemical list doesn't include all toxic chemicals used in the U.S. The Toxics Release Inventory (TRI) is a resource for learning about toxic chemical releases and pollution prevention activities reported by industrial and federal facilities. TRI data support informed decision-making by communities, government agencies, companies, and others. - + ## Data Dictionary: - "TRI Facility Name": Name of the facility reporting chemical releases - "TRI Facility ID": Unique identifier for the facility @@ -52,25 +51,25 @@ documentation: !fill | 1. When loading the data, ensure you specify the index_col=False parameter, since pandas is inferring the wrong index column and shifting the data without it: ```python import pandas as pd - tri_data = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv', index_col=False) + tri_data = pd.read_csv('${DATASET_FILES_BASE_PATH}/epa-tri/EPA_TRI_Toxics_2014_2023.csv', index_col=False) ``` - + 2. Common data analysis tasks: - Filter by year: `tri_data[tri_data['Year'] == 2023]` - Filter by chemical: `tri_data[tri_data['Chemical'] == 'Lead']` - Filter by state: `tri_data[tri_data['State'] == 'California']` - Find top polluters: `tri_data.sort_values('Releases (lb)', ascending=False).head(10)` - Analyze trends over time: `tri_data.groupby('Year')['Releases (lb)'].sum().plot()` - + 3. Geospatial analysis: - Create a map of facilities: Use latitude/longitude with libraries like folium or geopandas - Regional analysis: Group by EPA Region or State to compare pollution levels - + 4. Data quality considerations: - Check for missing values: `tri_data.isnull().sum()` - Some chemicals may have different reporting thresholds - RSEI Hazard scores provide context on relative toxicity beyond just release amounts - + Since this dataset contains a column for EPA Region, we can use this when relating to other more anonymous data sources that only use these properties- but do store latitude/longitude to correlate with the other data sources that have a higher resolution. @@ -86,8 +85,8 @@ documentation: !fill | The TRI Program is also different because the data it collects are: - annual, collected each July and made publicly available online; - - multimedia, reflecting chemical emissions to air, water and land; and - - broad, encompassing source reduction and other pollution prevention practices. + - multimedia, reflecting chemical emissions to air, water and land; and + - broad, encompassing source reduction and other pollution prevention practices. Numerous EPA programs use TRI data and information to: @@ -97,4 +96,4 @@ documentation: !fill | - improve data quality across EPA; and - develop program priorities and projects. - Under various environmental laws (identified in the figure below), other EPA programs also collect information about regulated chemicals. Users looking for information not available in the TRI can check the databases associated with these other programs. The Clean Air Act's National Emissions Inventory (NEI), for example, can be used to find estimates of air releases for facilities that do not report to the TRI or for mobile sources such as cars, which are not covered by the TRI. \ No newline at end of file + Under various environmental laws (identified in the figure below), other EPA programs also collect information about regulated chemicals. Users looking for information not available in the TRI can check the databases associated with these other programs. The Clean Air Act's National Emissions Inventory (NEI), for example, can be used to find estimates of air releases for facilities that do not report to the TRI or for mobile sources such as cars, which are not covered by the TRI. diff --git a/src/biome/adhoc_data/datasources/epa_tri/documentation/examples.yaml b/src/biome/adhoc_data/datasources/epa_tri/examples.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/epa_tri/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/epa_tri/examples.yaml diff --git a/src/biome/adhoc_data/datasources/faers/api.yaml b/src/biome/adhoc_data/datasources/faers/api.yaml index 81f7b8d..4ecb685 100644 --- a/src/biome/adhoc_data/datasources/faers/api.yaml +++ b/src/biome/adhoc_data/datasources/faers/api.yaml @@ -1,17 +1,18 @@ name: "FDA drug adverse event FAERS API" -cache_key: api_assistant_fda_openfda_faers_json +integration_type: api + description: | Drug Adverse Event Overview - The openFDA drug adverse event API returns data that has been collected from - the FDA Adverse Event Reporting System (FAERS), a database that contains + The openFDA drug adverse event API returns data that has been collected from + the FDA Adverse Event Reporting System (FAERS), a database that contains information on adverse event and medication error reports submitted to FDA. -web_documentation: !load_txt documentation/faers_web.md -fields_reference: !load_txt documentation/faers_fields_reference.csv -examples: !load_yaml documentation/examples.yaml -cache_body: - default: true -documentation: !fill | - {web_documentation} + +attachments: + web_documentation: faers_web.md + fields_reference: faers_fields_reference.csv + +prompt: | + ${web_documentation} # Additional Instructions: @@ -20,4 +21,4 @@ documentation: !fill | Do not use placeholder auth values from the OpenAPI spec nor prompt the user for these values, just use the environment variable. # Searchable Fields Reference: - {fields_reference} \ No newline at end of file + ${fields_reference} diff --git a/src/biome/adhoc_data/datasources/faers/documentation/faers_fields_reference.csv b/src/biome/adhoc_data/datasources/faers/attachments/faers_fields_reference.csv similarity index 100% rename from src/biome/adhoc_data/datasources/faers/documentation/faers_fields_reference.csv rename to src/biome/adhoc_data/datasources/faers/attachments/faers_fields_reference.csv diff --git a/src/biome/adhoc_data/datasources/faers/documentation/faers_web.md b/src/biome/adhoc_data/datasources/faers/attachments/faers_web.md similarity index 100% rename from src/biome/adhoc_data/datasources/faers/documentation/faers_web.md rename to src/biome/adhoc_data/datasources/faers/attachments/faers_web.md diff --git a/src/biome/adhoc_data/datasources/faers/documentation/examples.yaml b/src/biome/adhoc_data/datasources/faers/examples.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/faers/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/faers/examples.yaml diff --git a/src/biome/adhoc_data/datasources/gdc/api.yaml b/src/biome/adhoc_data/datasources/gdc/api.yaml index d5e8b3f..9322700 100644 --- a/src/biome/adhoc_data/datasources/gdc/api.yaml +++ b/src/biome/adhoc_data/datasources/gdc/api.yaml @@ -1,29 +1,31 @@ name: Genomics Data Commons +integration_type: api + description: | - The NCI's Genomic Data Commons (GDC) provides the cancer research community with a repository and computational + The NCI's Genomic Data Commons (GDC) provides the cancer research community with a repository and computational platform for cancer researchers who need to understand cancer, its clinical progression, and response to therapy. - The GDC supports several cancer genome programs at the NCI Center for Cancer Genomics (CCG), + The GDC supports several cancer genome programs at the NCI Center for Cancer Genomics (CCG), including The Cancer Genome Atlas (TCGA) and Therapeutically Applicable Research to Generate Effective Treatments (TARGET). -raw_documentation: !load_txt documentation/gdc.md -facets: !load_txt documentation/facets.txt -mappings: !load_txt documentation/gdc_mappings.json -examples: !load_yaml documentation/examples.yaml -cache_key: api_assistant_gdc_faceted -documentation: !fill | +attachments: + raw_documentation: gdc.md + facets: facets.txt + mappings: gdc_mappings.json + +prompt: | # GDC API Documentation - Below you will find the GDC API documentation. Note that + Below you will find the GDC API documentation. Note that there are several endpoints available--some are discussed in the `Search and Retrieval` section, and others are discussed in the `Data Analysis` section. When a user is searching for cases related to gene mutations you should use the `/ssm_occurrences` endpoint. - {raw_documentation} + ${raw_documentation} # Additional Instructions: - - You will be given python code to query the GDC API. + + You will be given python code to query the GDC API. When querying GDC, change the `format` from "TSV" to "JSON" @@ -34,7 +36,7 @@ documentation: !fill | This mapping is a dictionary where the keys are the endpoints and the values are the fields that are available for that endpoint. ``` - {mappings} + ${mappings} ``` These are the fields that are available for filtering and also the fields that can be returned. You can use @@ -43,7 +45,7 @@ documentation: !fill | A list of fields and their respective choices/facets are as follows: ``` - {facets} + ${facets} ``` Note this list of facets may be incomplete as it changes over time. Remember to consult it @@ -52,5 +54,5 @@ documentation: !fill | and getting no results, you should consider just using `*paragangliomas*`. Be sure to let the user know that you are using the wildcard operator so it might not be as specific as they would like. - If a user asks you to filter based on the type of cancer, it is often productive to filter on - disease type, primary diagnosis, or primary site. \ No newline at end of file + If a user asks you to filter based on the type of cancer, it is often productive to filter on + disease type, primary diagnosis, or primary site. diff --git a/src/biome/adhoc_data/datasources/gdc/documentation/facets.txt b/src/biome/adhoc_data/datasources/gdc/attachments/facets.txt similarity index 100% rename from src/biome/adhoc_data/datasources/gdc/documentation/facets.txt rename to src/biome/adhoc_data/datasources/gdc/attachments/facets.txt diff --git a/src/biome/adhoc_data/datasources/gdc/documentation/gdc.md b/src/biome/adhoc_data/datasources/gdc/attachments/gdc.md similarity index 100% rename from src/biome/adhoc_data/datasources/gdc/documentation/gdc.md rename to src/biome/adhoc_data/datasources/gdc/attachments/gdc.md diff --git a/src/biome/adhoc_data/datasources/gdc/documentation/gdc_mappings.json b/src/biome/adhoc_data/datasources/gdc/attachments/gdc_mappings.json similarity index 100% rename from src/biome/adhoc_data/datasources/gdc/documentation/gdc_mappings.json rename to src/biome/adhoc_data/datasources/gdc/attachments/gdc_mappings.json diff --git a/src/biome/adhoc_data/datasources/gdc/documentation/examples.yaml b/src/biome/adhoc_data/datasources/gdc/examples.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/gdc/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/gdc/examples.yaml diff --git a/src/biome/adhoc_data/datasources/gras/api.yaml b/src/biome/adhoc_data/datasources/gras/api.yaml index 490b3cb..3d0cdf0 100644 --- a/src/biome/adhoc_data/datasources/gras/api.yaml +++ b/src/biome/adhoc_data/datasources/gras/api.yaml @@ -1,15 +1,14 @@ -name: "FDA Generally Recognized as Safe (GRAS) API" -cache_key: api_assistant_gras_csv -description: > - The FDA Generally Recognized as Safe (GRAS) API is a csv file (export) data with notices filed from the FDA GRAS database from years 1998 to 2024. +name: "FDA Generally Recognized as Safe (GRAS)" +integration_type: dataset + +description: | + The FDA Generally Recognized as Safe (GRAS) is a csv file (export) data with notices filed from the FDA GRAS database from years 1998 to 2024. You may use this tool to check for food additives introductions or changes. Use this tool to answer questions such as: In which year was Pea fiber food additive/preservative introduced into the US food supply? -cache_body: - default: true -examples: !load_yaml documentation/examples.yaml -documentation: !fill | - You have access to the FDA Generally Recognized as Safe (GRAS) API, which is a csv file data with notices filed from the FDA GRAS database from years 1998 to 2024. + +prompt: | + You have access to the FDA Generally Recognized as Safe (GRAS), which is a csv file data with notices filed from the FDA GRAS database from years 1998 to 2024. You may use this to check for food additives introductions, and correlate with other data sources for analysis. "GRAS" is an acronym for the phrase Generally Recognized As Safe. @@ -19,15 +18,15 @@ documentation: !fill | # Additional Instructions: Use the raw csv notices file to answer questions about GRAS substances. Load the file from: - {{DATASET_FILES_BASE_PATH}}/fda-gras/GRASNotices.csv + ${DATASET_FILES_BASE_PATH}/fda-gras/GRASNotices.csv These are the columns available (separated by semicolons, some newlines/spaces): - GRAS Notice (GRN) No. ; Substance; Intended Use; Basis; Notifier; Notifier Address; + GRAS Notice (GRN) No. ; Substance; Intended Use; Basis; Notifier; Notifier Address; Date of filing; GRN Part 1; GRN Part 2; GRN Part 3; GRN Part 4; GRN Part 5; GRN Part 6; GRN Part 7; Date of closure; Date of correction letter; FDA's Letter; Date additional correspondence; Additional correspondence; Date additinoal correspondence 2; Additional correspondence 2; Date additional correspondence 3; Additional correspondence 3; Date additional correspondence 4; Additional correspondence 4; - Resubmission; Resubmitted; Notes; Related submission + Resubmission; Resubmitted; Notes; Related submission The Gras Notice GRN No is wrapped by spredsheet-software-like function.. Example: @@ -39,4 +38,4 @@ documentation: !fill | If the "FDA's Letter" column contains " FDA has no questions" it means the FDA did not challenge the notice, closest thing to "approved". Id the "FDA's Letter" column contains "FDA has questions" or "Pending", or similar, it means the FDA has questions about the notice, and the notice is not approved yet. If the "FDA's Letter" column contains "Notice does not provide a basis for a GRAS determination"- it means it challenged the notice, and the notice was not approved. - Use the stats of the "FDA's Letter" column to know if a food additive is actually approved to be introduced into the US food supply or not. \ No newline at end of file + Use the stats of the "FDA's Letter" column to know if a food additive is actually approved to be introduced into the US food supply or not. diff --git a/src/biome/adhoc_data/datasources/gras/documentation/examples.yaml b/src/biome/adhoc_data/datasources/gras/examples.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/gras/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/gras/examples.yaml diff --git a/src/biome/adhoc_data/datasources/hpa/api.yaml b/src/biome/adhoc_data/datasources/hpa/api.yaml index db61b20..c620d50 100644 --- a/src/biome/adhoc_data/datasources/hpa/api.yaml +++ b/src/biome/adhoc_data/datasources/hpa/api.yaml @@ -1,39 +1,42 @@ name: "Human Protein Atlas" +integration_type: api + description: | - The Human Protein Atlas is a Swedish-based program initiated in 2003 with the aim to map all - the human proteins in cells, tissues, and organs using an integration of various omics - technologies, including antibody-based imaging, mass spectrometry-based proteomics, - transcriptomics, and systems biology. All the data in the knowledge resource is open access - to allow scientists both in academia and industry to freely access the data for exploration + The Human Protein Atlas is a Swedish-based program initiated in 2003 with the aim to map all + the human proteins in cells, tissues, and organs using an integration of various omics + technologies, including antibody-based imaging, mass spectrometry-based proteomics, + transcriptomics, and systems biology. All the data in the knowledge resource is open access + to allow scientists both in academia and industry to freely access the data for exploration of the human proteome. - The Human Protein Atlas consists of eight separate resources, each focusing on a particular + The Human Protein Atlas consists of eight separate resources, each focusing on a particular aspect of the genome-wide analysis of the human proteins: - - The Tissue resource, showing the distribution of the proteins across all major tissues and + - The Tissue resource, showing the distribution of the proteins across all major tissues and organs in the human body - - The Brain resource, exploring the distribution of proteins in various regions of the - mammalian brain - - The Single Cell resource, showing expression of protein-coding genes in immune cells and + - The Brain resource, exploring the distribution of proteins in various regions of the + mammalian brain + - The Single Cell resource, showing expression of protein-coding genes in immune cells and human single cell types based on bulk and single cell RNA-seq - The Subcellular resource, showing the subcellular localization of proteins in single cells - - The Cancer resource, showing the impact of protein levels for the survival of patients + - The Cancer resource, showing the impact of protein levels for the survival of patients with cancer - - The Blood resource, describing proteins detected in blood and showing protein levels in + - The Blood resource, describing proteins detected in blood and showing protein levels in blood in patients with different diseases - - The Cell line resource, showing expression of protein-coding genes in human cancer cell + - The Cell line resource, showing expression of protein-coding genes in human cancer cell lines - - The Structure & Interaction resource, showing predicted 3D structures and exploring + - The Structure & Interaction resource, showing predicted 3D structures and exploring protein-coding genes in the context of protein-protein and metabolic interaction networks. - - The Human Protein Atlas program has already contributed to several thousands of publications - in the field of human biology and disease and is selected by the organization ELIXIR as a - European core resource due to its fundamental importance for a wider life science community. - In addition the Human Protein Atlas has been appointed Global Core Biodata Resource (GCBR) - by the Global Biodata Coalition. The Human Protein Atlas consortium is mainly funded by the + + The Human Protein Atlas program has already contributed to several thousands of publications + in the field of human biology and disease and is selected by the organization ELIXIR as a + European core resource due to its fundamental importance for a wider life science community. + In addition the Human Protein Atlas has been appointed Global Core Biodata Resource (GCBR) + by the Global Biodata Coalition. The Human Protein Atlas consortium is mainly funded by the Knut and Alice Wallenberg Foundation. -raw_documentation: !load_txt documentation/hpa_docs.md -examples: !load_yaml documentation/examples.yaml -cache_key: "api_assistant_pdc_graphql" -documentation: !fill | - {raw_documentation} \ No newline at end of file + +attachments: + raw_documentation: hpa_docs.md + +prompt: | + ${raw_documentation} diff --git a/src/biome/adhoc_data/datasources/hpa/documentation/hpa_docs.md b/src/biome/adhoc_data/datasources/hpa/attachments/hpa_docs.md similarity index 100% rename from src/biome/adhoc_data/datasources/hpa/documentation/hpa_docs.md rename to src/biome/adhoc_data/datasources/hpa/attachments/hpa_docs.md diff --git a/src/biome/adhoc_data/datasources/hpa/documentation/examples.yaml b/src/biome/adhoc_data/datasources/hpa/examples.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/hpa/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/hpa/examples.yaml diff --git a/src/biome/adhoc_data/datasources/idc/api.yaml b/src/biome/adhoc_data/datasources/idc/api.yaml index 9e5c0a5..fafd982 100644 --- a/src/biome/adhoc_data/datasources/idc/api.yaml +++ b/src/biome/adhoc_data/datasources/idc/api.yaml @@ -8,16 +8,17 @@ description: | - Commercial-friendly: >95% of the data in IDC is covered by the permissive CC-BY license, which allows commercial reuse (small subset of data is covered by the CC-NC license); each file in IDC is tagged with the license to make it easier for you to understand and follow the rules - Cloud-based: All of the data in IDC is available from both Google and AWS public buckets: fast and free to download, no out-of-cloud egress fees - Harmonized: All of the images and image-derived data in IDC is harmonized into standard DICOM representation -raw_documentation: !load_txt documentation/idc.md -examples: !load_yaml documentation/examples.yaml -cache_key: "api_assistant_idc_yaml" -documentation: !fill | - {raw_documentation} + +attachments: + raw_documentation: idc.md + +prompt: | + ${raw_documentation} # Additional Instructions: - + Be sure to import and instantiate the client for the Imaging Data Commons API: ```python from idc_index import index client = index.IDCClient() - ``` \ No newline at end of file + ``` diff --git a/src/biome/adhoc_data/datasources/idc/documentation/idc.md b/src/biome/adhoc_data/datasources/idc/attachments/idc.md similarity index 100% rename from src/biome/adhoc_data/datasources/idc/documentation/idc.md rename to src/biome/adhoc_data/datasources/idc/attachments/idc.md diff --git a/src/biome/adhoc_data/datasources/idc/documentation/examples.yaml b/src/biome/adhoc_data/datasources/idc/examples.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/idc/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/idc/examples.yaml diff --git a/src/biome/adhoc_data/datasources/indra/api.yaml b/src/biome/adhoc_data/datasources/indra/api.yaml index a005c89..7996527 100644 --- a/src/biome/adhoc_data/datasources/indra/api.yaml +++ b/src/biome/adhoc_data/datasources/indra/api.yaml @@ -1,38 +1,40 @@ name: INDRA Context Graph Extension (CoGEx) -description: > - INDRA CoGEx (Context Graph Extension) is an automatically assembled biomedical knowledge graph which integrates causal mechanisms from INDRA - with non-causal contextual relations including properties, ontology, and data. +integration_type: api +description: | + INDRA CoGEx (Context Graph Extension) is an automatically assembled biomedical knowledge graph which integrates causal mechanisms from INDRA + with non-causal contextual relations including properties, ontology, and data. - The INDRA CoGEx Query API is a RESTful service designed to streamline complex queries within the biomedical domain, - powered by the Contextual Graph Explorer (CoGEx) framework. It provides an interface to retrieve detailed insights - by integrating and analyzing data from diverse biomedical sources. Users can execute queries to uncover relationships - such as diseases associated with specific clinical trials, drugs linked to particular side effects, or other intricate - connections in biomedical knowledge. By leveraging its contextual graph-based approach, the API enables researchers, - data scientists, and healthcare professionals to explore and analyze rich biomedical data efficiently, supporting - advancements in drug discovery, clinical decision-making, and research. With its robust architecture, the INDRA CoGEx + The INDRA CoGEx Query API is a RESTful service designed to streamline complex queries within the biomedical domain, + powered by the Contextual Graph Explorer (CoGEx) framework. It provides an interface to retrieve detailed insights + by integrating and analyzing data from diverse biomedical sources. Users can execute queries to uncover relationships + such as diseases associated with specific clinical trials, drugs linked to particular side effects, or other intricate + connections in biomedical knowledge. By leveraging its contextual graph-based approach, the API enables researchers, + data scientists, and healthcare professionals to explore and analyze rich biomedical data efficiently, supporting + advancements in drug discovery, clinical decision-making, and research. With its robust architecture, the INDRA CoGEx Query API facilitates the discovery of meaningful patterns and supports informed decision-making in complex biomedical scenarios. -raw_documentation: !load_txt documentation/indra.json -examples: !load_yaml documentation/examples.yaml -cache_key: "api_assistant_indra_json" -documentation: !fill | - {raw_documentation} + +attachments: + raw_documentation: indra.json + +prompt: | + ${raw_documentation} # Additional Instructions: - + You should make use of the python requests library to interact with the API. Note that the base URL of the API is 'https://discovery.indra.bio/api/' - + For example, here is an example request: - + curl -X 'POST' \ 'https://discovery.indra.bio/api/get_go_terms_for_gene' \ -H 'accept: application/json' \ -H 'Content-Type: application/json' \ - -d '{{ + -d '{ "gene": [ "HGNC", "9896" ], "include_indirect": true - }}' - ``` \ No newline at end of file + }' + ``` diff --git a/src/biome/adhoc_data/datasources/indra/documentation/indra.json b/src/biome/adhoc_data/datasources/indra/attachments/indra.json similarity index 100% rename from src/biome/adhoc_data/datasources/indra/documentation/indra.json rename to src/biome/adhoc_data/datasources/indra/attachments/indra.json diff --git a/src/biome/adhoc_data/datasources/indra/documentation/examples.yaml b/src/biome/adhoc_data/datasources/indra/examples.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/indra/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/indra/examples.yaml diff --git a/src/biome/adhoc_data/datasources/netrias/api.yaml b/src/biome/adhoc_data/datasources/netrias/api.yaml index 17f362e..0d494c1 100644 --- a/src/biome/adhoc_data/datasources/netrias/api.yaml +++ b/src/biome/adhoc_data/datasources/netrias/api.yaml @@ -1,42 +1,43 @@ -name: Netrias Harmonization API -cache_key: api_assistant_netrias +name: Netrias Harmonization API +integration_type: api + description: | - The Netrias Harmonization REST API leverages AI-driven harmonization approaches to standardize - cancer-related terminologies from the Cancer Data Service (CDS), managed by the Center for + The Netrias Harmonization REST API leverages AI-driven harmonization approaches to standardize + cancer-related terminologies from the Cancer Data Service (CDS), managed by the Center for Biomedical Informatics and Information Technology (CBIIT) at NIH. - This API is designed to process and harmonize textual variations of terms, mapping them to - standardized terms defined in the CDS data model ontology (version 5.0.5). By utilizing - advanced machine learning techniques, the service enhances data consistency, improves - interoperability, and streamlines standardization for cancer research and biomedical + This API is designed to process and harmonize textual variations of terms, mapping them to + standardized terms defined in the CDS data model ontology (version 5.0.5). By utilizing + advanced machine learning techniques, the service enhances data consistency, improves + interoperability, and streamlines standardization for cancer research and biomedical informatics applications. - The API enables automated term reconciliation, ensuring uniform terminology across datasets - for more reliable data integration and analysis. The current intended application is for - harmonization of terms in study metadata from cancer researchers delivering research data + The API enables automated term reconciliation, ensuring uniform terminology across datasets + for more reliable data integration and analysis. The current intended application is for + harmonization of terms in study metadata from cancer researchers delivering research data to the Cancer Research Data Commons. -raw_documentation: !load_txt documentation/netrias.json -examples: !load_yaml documentation/examples.yaml -cache_body: - default: true -documentation: !fill | - {raw_documentation} + +attachments: + raw_documentation: netrias.json + +prompt: | + ${raw_documentation} # Additional Instructions: - - The REST API is accessible at https://apieval.netriaslabs.cloud/. + + The REST API is accessible at https://apieval.netriaslabs.cloud/. It uses API Keys to track access. It is just applied as an additional HTTP header `x-api-key: ` - + **IMPORTANT**: the API Key is available in the environment variable `NETRIAS_KEY`. - For CDS standard terms, commonly referred to as Common Data Elements or CDEs, with large numbers of permissible values, - Netrias builds fine tuned LLMs from synthetically generated training data. If a CDE has a small number of permissible values, - we apply in-context learning approaches where we craft a prompt to feed an LLM API chat endpoint (e.g. OpenAI) requesting it - generate harmonizations with no or few examples. When used as a tool, this may lead to significant variations in response + For CDS standard terms, commonly referred to as Common Data Elements or CDEs, with large numbers of permissible values, + Netrias builds fine tuned LLMs from synthetically generated training data. If a CDE has a small number of permissible values, + we apply in-context learning approaches where we craft a prompt to feed an LLM API chat endpoint (e.g. OpenAI) requesting it + generate harmonizations with no or few examples. When used as a tool, this may lead to significant variations in response latency as the fine tuned models may take some time to load into a GPU, tokenize the input, and infer the intended response. - The REST API also supports the harmonization of terms from ontologies supporting Sage Bionetworks' data repositories focused - on rare diseases and cancers. In a demonstration integration of the API with Sage's Synapse chatbot application the teams + The REST API also supports the harmonization of terms from ontologies supporting Sage Bionetworks' data repositories focused + on rare diseases and cancers. In a demonstration integration of the API with Sage's Synapse chatbot application the teams focused on four types of variations. Here are some examples: 1. Typos @@ -44,7 +45,7 @@ documentation: !fill | b. whole exme squencing → whole exome sequencing 2. Exact synonyms - a. NF-1 → Neurofibromatosis type 1 + a. NF-1 → Neurofibromatosis type 1 b. WEX → whole exome sequencing 3. Narrow synonyms diff --git a/src/biome/adhoc_data/datasources/netrias/documentation/netrias.json b/src/biome/adhoc_data/datasources/netrias/attachments/netrias.json similarity index 100% rename from src/biome/adhoc_data/datasources/netrias/documentation/netrias.json rename to src/biome/adhoc_data/datasources/netrias/attachments/netrias.json diff --git a/src/biome/adhoc_data/datasources/netrias/documentation/examples.yaml b/src/biome/adhoc_data/datasources/netrias/examples.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/netrias/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/netrias/examples.yaml diff --git a/src/biome/adhoc_data/datasources/nhanes_dietary/api.yaml b/src/biome/adhoc_data/datasources/nhanes_dietary/api.yaml index 739d9ac..569c102 100644 --- a/src/biome/adhoc_data/datasources/nhanes_dietary/api.yaml +++ b/src/biome/adhoc_data/datasources/nhanes_dietary/api.yaml @@ -1,66 +1,67 @@ name: "NHANES Dietary Data" -cache_key: api_assistant_nhanes_dietary_files +integration_type: dataset + description: | API for accessing and analyzing NHANES dietary data, including food consumption patterns and processed food analysis across different survey cycles from 1999 to 2023, with demographic information. -web_docs: !load_txt documentation/docs.md -examples: !load_yaml documentation/examples.yaml -cache_body: - default: true -documentation: !fill | + +attachments: + web_docs: docs.md + +prompt: | You have access to NHANES Dietary Data files from multiple survey cycles, including dietary intake and demographic information: - + # 1999-2000 NHANES Cycle - - {{DATASET_FILES_BASE_PATH}}/nhanes_dietary/1999_2000_DEMOGRAPHICS.xpt - Demographic information for participants - - {{DATASET_FILES_BASE_PATH}}/nhanes_dietary/1999_2000_DRXFMT_food_codes.xpt - Food code format file with descriptions - - {{DATASET_FILES_BASE_PATH}}/nhanes_dietary/1999_2000_DRXIFF_individual_foods.xpt - Individual foods consumption data - + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/1999_2000_DEMOGRAPHICS.xpt - Demographic information for participants + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/1999_2000_DRXFMT_food_codes.xpt - Food code format file with descriptions + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/1999_2000_DRXIFF_individual_foods.xpt - Individual foods consumption data + # 2005-2006 NHANES Cycle - - {{DATASET_FILES_BASE_PATH}}/nhanes_dietary/2005_2006_DEMOGRAPHICS.xpt - Demographic information for participants - - {{DATASET_FILES_BASE_PATH}}/nhanes_dietary/2005_2006_DR1IFF_individual_foods.xpt - Individual foods consumption data - - {{DATASET_FILES_BASE_PATH}}/nhanes_dietary/2005_2006_DRXFCD_food_codes.xpt - Food code format file with descriptions - - {{DATASET_FILES_BASE_PATH}}/nhanes_dietary/2005_2006_DRXMCD_modification_codes.xpt - Modification codes for dietary items - + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2005_2006_DEMOGRAPHICS.xpt - Demographic information for participants + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2005_2006_DR1IFF_individual_foods.xpt - Individual foods consumption data + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2005_2006_DRXFCD_food_codes.xpt - Food code format file with descriptions + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2005_2006_DRXMCD_modification_codes.xpt - Modification codes for dietary items + # 2009-2010 NHANES Cycle - - {{DATASET_FILES_BASE_PATH}}/nhanes_dietary/2009-2010_DEMOGRAPHICS.xpt - Demographic information for participants - - {{DATASET_FILES_BASE_PATH}}/nhanes_dietary/2009_2010_DR1IFF_individual_foods.xpt - Individual foods consumption data - - {{DATASET_FILES_BASE_PATH}}/nhanes_dietary/2009_2010_DRXFCD_food_codes.xpt - Food code format file with descriptions - - {{DATASET_FILES_BASE_PATH}}/nhanes_dietary/2009_2010_DRXMCD_modification_codes.xpt - Modification codes for dietary items - + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2009-2010_DEMOGRAPHICS.xpt - Demographic information for participants + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2009_2010_DR1IFF_individual_foods.xpt - Individual foods consumption data + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2009_2010_DRXFCD_food_codes.xpt - Food code format file with descriptions + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2009_2010_DRXMCD_modification_codes.xpt - Modification codes for dietary items + # 2015-2016 NHANES Cycle - - {{DATASET_FILES_BASE_PATH}}/nhanes_dietary/2015_2016_DEMOGRAPHICS.xpt - Demographic information for participants - - {{DATASET_FILES_BASE_PATH}}/nhanes_dietary/2015_2016_DR1IFF_individual_foods.xpt - Individual foods consumption data - - {{DATASET_FILES_BASE_PATH}}/nhanes_dietary/2015_2016_DRXFCD_food_codes.xpt - Food code format file with descriptions - + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2015_2016_DEMOGRAPHICS.xpt - Demographic information for participants + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2015_2016_DR1IFF_individual_foods.xpt - Individual foods consumption data + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2015_2016_DRXFCD_food_codes.xpt - Food code format file with descriptions + # 2017-2020 NHANES Cycle - - {{DATASET_FILES_BASE_PATH}}/nhanes_dietary/2017_2020_DEMOGRAPHICS.xpt - Demographic information for participants - - {{DATASET_FILES_BASE_PATH}}/nhanes_dietary/2017_2020_DR1IFF_individual_foods.xpt - Individual foods consumption data - - {{DATASET_FILES_BASE_PATH}}/nhanes_dietary/2017_2020_DRXFCD_food_codes.xpt - Food code format file with descriptions - + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2017_2020_DEMOGRAPHICS.xpt - Demographic information for participants + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2017_2020_DR1IFF_individual_foods.xpt - Individual foods consumption data + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2017_2020_DRXFCD_food_codes.xpt - Food code format file with descriptions + # 2021-2023 NHANES Cycle - - {{DATASET_FILES_BASE_PATH}}/nhanes_dietary/2021_2023_DEMOGRAPHICS.xpt - Demographic information for participants - - {{DATASET_FILES_BASE_PATH}}/nhanes_dietary/2021_2023_DR1IFF_individual_foods.xpt - Individual foods consumption data - - {{DATASET_FILES_BASE_PATH}}/nhanes_dietary/2021_2023_DRXFCD_food_codes.xpt - Food code format file with descriptions - + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2021_2023_DEMOGRAPHICS.xpt - Demographic information for participants + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2021_2023_DR1IFF_individual_foods.xpt - Individual foods consumption data + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2021_2023_DRXFCD_food_codes.xpt - Food code format file with descriptions + Key variables: - SEQN: Respondent sequence number (unique identifier for participants) - use this to link demographic data with dietary data - DRDIFDCD/DR1FDCD: Food code that links to food descriptions in the format file - DR1ILINE: Line number for individual food items - + Demographic files contain: - Age, gender, race/ethnicity, education, income, and other sociodemographic variables - Geographic information (limited to census region in public files) - + The food codes files (DRXFMT/DRXFCD) contain: - Food codes and their text descriptions - + The individual foods files (DRXIFF/DR1IFF) contain detailed consumption data for each food item consumed by participants during the 24-hour recall period. - + The modification codes files (DRXMCD) contain information about modifications made to standard food items (e.g., fat removed from meat, salt added to vegetables). - + Note: The demographic files can be linked to the dietary files using the SEQN variable to analyze dietary patterns by demographic characteristics such as age, gender, race/ethnicity, and socioeconomic status. - {web_docs} \ No newline at end of file + ${web_docs} diff --git a/src/biome/adhoc_data/datasources/nhanes_dietary/documentation/docs.md b/src/biome/adhoc_data/datasources/nhanes_dietary/attachments/docs.md similarity index 100% rename from src/biome/adhoc_data/datasources/nhanes_dietary/documentation/docs.md rename to src/biome/adhoc_data/datasources/nhanes_dietary/attachments/docs.md diff --git a/src/biome/adhoc_data/datasources/nhanes_dietary/documentation/examples.yaml b/src/biome/adhoc_data/datasources/nhanes_dietary/examples.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/nhanes_dietary/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/nhanes_dietary/examples.yaml diff --git a/src/biome/adhoc_data/datasources/nsch/api.yaml b/src/biome/adhoc_data/datasources/nsch/api.yaml index 794eb00..9275255 100644 --- a/src/biome/adhoc_data/datasources/nsch/api.yaml +++ b/src/biome/adhoc_data/datasources/nsch/api.yaml @@ -1,48 +1,47 @@ name: "National Survey of Children's Health (NSCH)" -cache_key: api_assistant_nsch_files +integration_type: dataset + description: | - You have access to knowledge from the National Survey of Children's Health (NSCH) data, available every year from - 2016 to 2023. This dataset API can help answer questions about children's health, healthcare access, family functioning, + You have access to knowledge from the National Survey of Children's Health (NSCH) data, available every year from + 2016 to 2023. This dataset API can help answer questions about children's health, healthcare access, family functioning, neighborhood characteristics, and social determinants of health for children aged 0-17 years in the United States. -examples: !load_yaml documentation/examples.yaml -cache_body: - default: true -documentation: !fill | + +prompt: | You have access to file-based National Survey of Children's Health (NSCH) data - from 2016 to 2023, located under the `{{DATASET_FILES_BASE_PATH}}/census-nsch` directory. + from 2016 to 2023, located under the `${DATASET_FILES_BASE_PATH}/census-nsch` directory. The NSCH files are located and named per year as follows: - - {{DATASET_FILES_BASE_PATH}}/census-nsch/2023e_topical.sas7bdat - - {{DATASET_FILES_BASE_PATH}}/census-nsch/2022e_topical.sas7bdat - - {{DATASET_FILES_BASE_PATH}}/census-nsch/2021e_topical.sas7bdat - - {{DATASET_FILES_BASE_PATH}}/census-nsch/2020e_topical.sas7bdat - - {{DATASET_FILES_BASE_PATH}}/census-nsch/2019e_topical.sas7bdat - - {{DATASET_FILES_BASE_PATH}}/census-nsch/2018e_topical.sas7bdat - - {{DATASET_FILES_BASE_PATH}}/census-nsch/2017e_topical.sas7bdat - - {{DATASET_FILES_BASE_PATH}}/census-nsch/2016e_topical.sas7bdat - - Codebook: {{DATASET_FILES_BASE_PATH}}/census-nsch/nsch_dictionary_codebook.csv - - The NSCH is a national survey that collects information on the health and well-being of children ages 0-17 years - in the United States and the factors that may relate to their well-being. The survey includes data on physical and + - ${DATASET_FILES_BASE_PATH}/census-nsch/2023e_topical.sas7bdat + - ${DATASET_FILES_BASE_PATH}/census-nsch/2022e_topical.sas7bdat + - ${DATASET_FILES_BASE_PATH}/census-nsch/2021e_topical.sas7bdat + - ${DATASET_FILES_BASE_PATH}/census-nsch/2020e_topical.sas7bdat + - ${DATASET_FILES_BASE_PATH}/census-nsch/2019e_topical.sas7bdat + - ${DATASET_FILES_BASE_PATH}/census-nsch/2018e_topical.sas7bdat + - ${DATASET_FILES_BASE_PATH}/census-nsch/2017e_topical.sas7bdat + - ${DATASET_FILES_BASE_PATH}/census-nsch/2016e_topical.sas7bdat + - Codebook: ${DATASET_FILES_BASE_PATH}/census-nsch/nsch_dictionary_codebook.csv + + The NSCH is a national survey that collects information on the health and well-being of children ages 0-17 years + in the United States and the factors that may relate to their well-being. The survey includes data on physical and mental health, access to quality health care, and the child's family, neighborhood, school, and social context. - The NSCH is also designed to assess the prevalence and impact of special health care needs among children in the US and + The NSCH is also designed to assess the prevalence and impact of special health care needs among children in the US and explores the extent to which children with special health care needs (CSHCN) have medical homes, adequate health insurance, access to needed services, and adequate care coordination. Other topics may include functional difficulties, transition services, - shared decision-making, and satisfaction with care. + shared decision-making, and satisfaction with care. Information is collected from parents or caregivers who know about the child's health. It also examines the physical and emotional health of children ages 0-17 years of age. - These factors include access to - and quality of - health care, family interactions, parental health, + These factors include access to - and quality of - health care, family interactions, parental health, neighborhood characteristics, as well as school and after-school experiences (example: child screen time). To read these SAS7BDAT files with pandas, use: ```python import pandas as pd - filename = "{{DATASET_FILES_BASE_PATH}}/census-nsch/2023e_topical.sas7bdat" + filename = "${DATASET_FILES_BASE_PATH}/census-nsch/2023e_topical.sas7bdat" df = pd.read_sas(filename, format="sas7bdat") ``` - + The NSCH datasets include state identifiers (FIPSST) and other geographic information that can be used for regional analysis. Check which geographic variables are available in each year's dataset before attempting to use them. @@ -51,7 +50,7 @@ documentation: !fill | 1. Variable names may change slightly between survey years 2. Many variables use coded responses (e.g. subset for examples: 1=Yes||2=No, 1 = Never served in the military||2 = Only on active duty for training in the Reserves You'll need to use the nsch codebook to decode these variable response codes- - The codebook is available at {{DATASET_FILES_BASE_PATH}}/census-nsch/nsch_dictionary_codebook.csv + The codebook is available at ${DATASET_FILES_BASE_PATH}/census-nsch/nsch_dictionary_codebook.csv You may open it with pandas and find the "Variable" name, the "Question", "Description", and its "Response Code"s, etc. The Response Code options/alternatives per column are separated by a double pipe (||),such as "1=Yes||2=No" In the codebook, the Question itself is under the "Question" column @@ -75,7 +74,7 @@ documentation: !fill | Open it with pandas like so: ``` - codebook = pd.read_csv("{{DATASET_FILES_BASE_PATH}}/census-nsch/nsch_dictionary_codebook.csv", index_col=False) # ensure index_col=False, otherwise the dataframe will be shifted! + codebook = pd.read_csv("${DATASET_FILES_BASE_PATH}/census-nsch/nsch_dictionary_codebook.csv", index_col=False) # ensure index_col=False, otherwise the dataframe will be shifted! ``` The NSCH is not an API but a collection of dataset files from 2016 to 2023 in SAS7BDAT format. These datasets provide rich information on: @@ -111,4 +110,4 @@ documentation: !fill | - FOODSIT: Food situation in household - TENURE: Housing tenure (owned/rented) - Environmental factors are primarily self-reported by parents/caregivers. \ No newline at end of file + Environmental factors are primarily self-reported by parents/caregivers. diff --git a/src/biome/adhoc_data/datasources/nsch/documentation/examples.yaml b/src/biome/adhoc_data/datasources/nsch/examples.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/nsch/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/nsch/examples.yaml diff --git a/src/biome/adhoc_data/datasources/pdc/api.yaml b/src/biome/adhoc_data/datasources/pdc/api.yaml index f14408f..e627593 100644 --- a/src/biome/adhoc_data/datasources/pdc/api.yaml +++ b/src/biome/adhoc_data/datasources/pdc/api.yaml @@ -4,22 +4,23 @@ description: | The PDC was developed to advance our understanding of how proteins help to shape the risk, diagnosis, development, progression, and treatment of cancer. In-depth analysis of proteomic data allows us to study both how and why cancer develops and to devise ways of personalizing treatment for patients using precision medicine. The PDC is one of several repositories within the NCI Cancer Research Data Commons (CRDC), a secure cloud-based infrastructure featuring diverse data sets and innovative analytic tools – all designed to advance data-driven scientific discovery. The CRDC enables researchers to link proteomic data with other data sets (e.g., genomic and imaging data) and to submit, collect, analyze, store, and share data throughout the cancer data ecosystem. Access to highly curated and standardized biospecimen, clinical and proteomic data with direct integration of accompanying data resources (genomic and medical image datasets). - - Uses: + + Uses: * Intuitive interface to filter, query, search, visualize and download the data and metadata. * A common data harmonization pipeline to uniformly analyze all PDC data and provide advanced visualization of the quantitative information. * Cloud based (Amazon Web Services) infrastructure facilitates interoperability with AWS based data analysis tools and platforms natively. * Application programming interface (API) provides cloud-agnostic data access and allows third parties to extend the functionality beyond the PDC. * A highly structured workspace that serves as a private user data store and also data submission portal. * Distributes controlled access data, such as the patient-specific protein fasta sequence databases, with dbGaP authorization and eRA Commons authentication. -raw_documentation: !load_txt documentation/pdc_schema.graphql -examples: !load_yaml documentation/examples.yaml -cache_key: "api_assistant_pdc_graphql" -documentation: !fill | - {raw_documentation} + +attachments: + raw_documentation: pdc_schema.graphql + +prompt: | + ${raw_documentation} # Additional Instructions: - - This API is through GraphQL. You will be provided the schema. + + This API is through GraphQL. You will be provided the schema. All requests go to the following URL: https://pdc.cancer.gov/graphql - Make all GraphQL requests to that URL. \ No newline at end of file + Make all GraphQL requests to that URL. diff --git a/src/biome/adhoc_data/datasources/pdc/documentation/pdc_schema.graphql b/src/biome/adhoc_data/datasources/pdc/attachments/pdc_schema.graphql similarity index 100% rename from src/biome/adhoc_data/datasources/pdc/documentation/pdc_schema.graphql rename to src/biome/adhoc_data/datasources/pdc/attachments/pdc_schema.graphql diff --git a/src/biome/adhoc_data/datasources/pdc/documentation/examples.yaml b/src/biome/adhoc_data/datasources/pdc/examples.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/pdc/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/pdc/examples.yaml diff --git a/src/biome/adhoc_data/datasources/synapse/api.yaml b/src/biome/adhoc_data/datasources/synapse/api.yaml index c09f4b1..1345670 100644 --- a/src/biome/adhoc_data/datasources/synapse/api.yaml +++ b/src/biome/adhoc_data/datasources/synapse/api.yaml @@ -1,31 +1,31 @@ name: "NF Synapse" -cache_key: api_assistant_synapse +integration_type: api description: | The NF Synapse API is a RESTful API that allows you to interact with the Neurofibromatosis Data Portal. It additionally contains a Python client that can be used to interact with the API (on the package `synapseclient`, already installed in the environment). The code drafter has access both to the openapi spec, the web docs, and the python client documentation. -raw_documentation: !load_txt documentation/SynapseOpenApiSpec.json -web_docs: !load_txt documentation/web_docs.md -examples: !load_yaml documentation/examples.yaml -python_client: !load_txt documentation/nf_programmatic_webdocs.md -cache_body: - default: true -documentation: !fill | + +attachments: + raw_documentation: SynapseOpenApiSpec.json + web_docs: web_docs.md + python_client: nf_programmatic_webdocs.md + +prompt: | # OpenAPI Spec JSON: - {raw_documentation} + ${raw_documentation} # Additional Instructions: - - This API is through OpenAPI Spec. You have been provided the schema. + + This API is through OpenAPI Spec. You have been provided the schema. All requests go to the following URL: https://repo-prod.prod.sagebase.org - That is the base URL for all requests. + That is the base URL for all requests. To use the Synapse API always retrieve the auth header token from the environment variable: - bearer token: os.environ.get("API_SYNAPSE") Do not use placeholder auth values from the OpenAPI spec nor prompt the user for these values, just use the environment variables. Ensure you know the fields that come back from the OpenAPI spec in order to process the Synapse data. - + Never simulate this API- if it is not available ask the user how to proceed. You will mostly have to use the /repo/v1 and /file/v1 paths. @@ -41,12 +41,12 @@ documentation: !fill | You only have read/view/download assess/permissions- don't try to create or upload data- the openapi spec may have create/upload endpoints but do not use them. # More Web Docs: - {web_docs} + ${web_docs} # Example issuing a POST to search for `High throughput analysis of pNF cell lines` (omitting auth bearer token header; eg Authorization: Bearer ) ``` POST https://repo-prod.prod.sagebase.org/repo/v1/search - {{ + { "queryTerm": [ "Blakeley" ], @@ -55,11 +55,11 @@ documentation: !fill | "returnFields": [], "start": 0, "size": "10" - }} + } ``` # Python Client - {python_client} + ${python_client} # About the NF synapse subdomain @@ -80,6 +80,6 @@ documentation: !fill | downloads_dir = Path(__file__).parent.parent / "downloads" / "synapse" - syn = synapseclient.Synapse(cache_root_dir=downloads_dir) + syn = synapseclient.Synapse(cache_root_dir=downloads_dir) syn.login(authToken=os.environ.get("API_SYNAPSE")) # pass in the authToken on python client login! - ``` \ No newline at end of file + ``` diff --git a/src/biome/adhoc_data/datasources/synapse/documentation/SynapseOpenApiSpec.json b/src/biome/adhoc_data/datasources/synapse/attachments/SynapseOpenApiSpec.json similarity index 100% rename from src/biome/adhoc_data/datasources/synapse/documentation/SynapseOpenApiSpec.json rename to src/biome/adhoc_data/datasources/synapse/attachments/SynapseOpenApiSpec.json diff --git a/src/biome/adhoc_data/datasources/synapse/documentation/nf_programmatic_webdocs.md b/src/biome/adhoc_data/datasources/synapse/attachments/nf_programmatic_webdocs.md similarity index 100% rename from src/biome/adhoc_data/datasources/synapse/documentation/nf_programmatic_webdocs.md rename to src/biome/adhoc_data/datasources/synapse/attachments/nf_programmatic_webdocs.md diff --git a/src/biome/adhoc_data/datasources/synapse/documentation/web_docs.md b/src/biome/adhoc_data/datasources/synapse/attachments/web_docs.md similarity index 100% rename from src/biome/adhoc_data/datasources/synapse/documentation/web_docs.md rename to src/biome/adhoc_data/datasources/synapse/attachments/web_docs.md diff --git a/src/biome/adhoc_data/datasources/synapse/documentation/examples.yaml b/src/biome/adhoc_data/datasources/synapse/examples.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/synapse/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/synapse/examples.yaml diff --git a/src/biome/adhoc_data/datasources/usda_foodcentral/api.yaml b/src/biome/adhoc_data/datasources/usda_foodcentral/api.yaml index 600aeb6..a525845 100644 --- a/src/biome/adhoc_data/datasources/usda_foodcentral/api.yaml +++ b/src/biome/adhoc_data/datasources/usda_foodcentral/api.yaml @@ -1,24 +1,25 @@ name: "USDA FoodData Central API" -cache_key: api_assistant_usda_foodcentral_json +integration_type: api + description: | This openapi is the primary place to obtain data from the USDA's FoodData Central (nutrition) database. It is intended to assist in incorporating nutrient data into any analysis we need. -raw_documentation: !load_txt documentation/foodcentral_openapi.json -examples: !load_yaml documentation/examples.yaml -cache_body: - default: true -documentation: !fill | + +attachments: + raw_documentation: foodcentral_openapi.json + +prompt: | This API is the primary place to obtain data from the USDA's FoodData Central database. # OpenAPI Spec JSON: - {raw_documentation} + ${raw_documentation} # Additional Instructions: - - This API is through OpenAPI Spec (Swagger). You will be provided the schema. + + This API is through OpenAPI Spec (Swagger). You will be provided the schema. All requests go to the following URL: https://api.nal.usda.gov/fdc/v1/ - That is the base URL for all requests. + That is the base URL for all requests. To use the USDA FoodData Central API always retrieve the `api_key` query parameter from environment variables: - api_key: os.environ.get("API_USDA_FDC") @@ -61,10 +62,10 @@ documentation: !fill | Note: The "dataType" parameter values need to be specified as an array. ## Data Type Comparison - ### Foundation Foods + ### Foundation Foods Data and metadata on individual samples of commodity/commodity-derived minimally processed foods with insights into variability - ### Experimental Foods + ### Experimental Foods Data on food published in peer-reviewed journals supported by or in collaboration with USDA ### Food and Nutrient Database for Dietary Studies (FNDDS) @@ -74,4 +75,4 @@ documentation: !fill | Data from labels of national and international branded foods collected by a public-private partnership ### SR Legacy - Historic data on food components including nutrients derived from analyses, calculations, and published literature \ No newline at end of file + Historic data on food components including nutrients derived from analyses, calculations, and published literature diff --git a/src/biome/adhoc_data/datasources/usda_foodcentral/documentation/foodcentral_openapi.json b/src/biome/adhoc_data/datasources/usda_foodcentral/attachments/foodcentral_openapi.json similarity index 100% rename from src/biome/adhoc_data/datasources/usda_foodcentral/documentation/foodcentral_openapi.json rename to src/biome/adhoc_data/datasources/usda_foodcentral/attachments/foodcentral_openapi.json diff --git a/src/biome/adhoc_data/datasources/usda_foodcentral/documentation/examples.yaml b/src/biome/adhoc_data/datasources/usda_foodcentral/examples.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/usda_foodcentral/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/usda_foodcentral/examples.yaml diff --git a/src/biome/adhoc_data/datasources/usgs_pesticide/api.yaml b/src/biome/adhoc_data/datasources/usgs_pesticide/api.yaml index 2671151..4c8483b 100644 --- a/src/biome/adhoc_data/datasources/usgs_pesticide/api.yaml +++ b/src/biome/adhoc_data/datasources/usgs_pesticide/api.yaml @@ -1,45 +1,44 @@ name: "USGS Pesticide National Synthesis Project" -cache_key: api_assistant_usgs_pesticide_files +integration_type: dataset + description: | The USGS Pesticide National Synthesis Project contains annual agricultural pesticide use data by county, on a 2-5 year interval, from 2000 to 2019. -examples: !load_yaml documentation/examples.yaml -cache_body: - default: true -documentation: !fill | + +prompt: | You have access to a file-based USGS Pesticide National Synthesis Project data, for annual agricultural pesticide use by county and pesticide compound. - We've added files on a 2-5 year interval, from 2000 to 2019, under the `{{DATASET_FILES_BASE_PATH}}/usgs_pesticide` directory. + We've added files on a 2-5 year interval, from 2000 to 2019, under the `${DATASET_FILES_BASE_PATH}/usgs_pesticide` directory. The exact files are located in: - - {{DATASET_FILES_BASE_PATH}}/usgs_pesticide/EPest.county.estimates.2000.txt - - {{DATASET_FILES_BASE_PATH}}/usgs_pesticide/EPest.county.estimates.2005.txt - - {{DATASET_FILES_BASE_PATH}}/usgs_pesticide/EPest.county.estimates.2010.txt - - {{DATASET_FILES_BASE_PATH}}/usgs_pesticide/EPest_county_estimates_2013_2017_v2.txt - - {{DATASET_FILES_BASE_PATH}}/usgs_pesticide/EPest_county_estimates_2019.txt + - ${DATASET_FILES_BASE_PATH}/usgs_pesticide/EPest.county.estimates.2000.txt + - ${DATASET_FILES_BASE_PATH}/usgs_pesticide/EPest.county.estimates.2005.txt + - ${DATASET_FILES_BASE_PATH}/usgs_pesticide/EPest.county.estimates.2010.txt + - ${DATASET_FILES_BASE_PATH}/usgs_pesticide/EPest_county_estimates_2013_2017_v2.txt + - ${DATASET_FILES_BASE_PATH}/usgs_pesticide/EPest_county_estimates_2019.txt Files use FIPS states and county codes, and values are tab-delimited. The files are actually yearly from 1992-2012, but we've added a subset. You could download more from that range at: https://water.usgs.gov/nawqa/pnsp/usage/maps/county-level/PesticideUseEstimates/EPest.county.estimates..txt - The USGS agricultural pesticide-use estimates are supported by funding from the USGS National Water Quality Program - for the purpose of better understanding pesticides in freshwater and their impact - on water availability nationwide. Final annual pesticide-use estimates, for approximately 400 compounds, + The USGS agricultural pesticide-use estimates are supported by funding from the USGS National Water Quality Program + for the purpose of better understanding pesticides in freshwater and their impact + on water availability nationwide. Final annual pesticide-use estimates, for approximately 400 compounds, from 2018-22 will be published in 2025 (not available yet) - After that, preliminary estimates will be published annually and later updated + After that, preliminary estimates will be published annually and later updated with final estimates once the USDA Census of Agriculture is released (every five years). The total number of pesticides included in the analysis will fluctuate annually. - Beginning 2015, the provider of the surveyed pesticide data used to derive the - county-level use estimates discontinued making estimates for seed treatment - application of pesticides because of complexity and uncertainty. + Beginning 2015, the provider of the surveyed pesticide data used to derive the + county-level use estimates discontinued making estimates for seed treatment + application of pesticides because of complexity and uncertainty. Pesticide use estimates prior to 2015 include estimates with seed treatment application. - The columns are as follows: - - COMPOUND - - YEAR - - STATE_FIPS_CODE - - COUNTY_FIPS_CODE - - EPEST_LOW_KG + The columns are as follows: + - COMPOUND + - YEAR + - STATE_FIPS_CODE + - COUNTY_FIPS_CODE + - EPEST_LOW_KG - EPEST_HIGH_KG You may use the python `us` library to convert FIPS codes to state and county names, diff --git a/src/biome/adhoc_data/datasources/usgs_pesticide/documentation/examples.yaml b/src/biome/adhoc_data/datasources/usgs_pesticide/examples.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/usgs_pesticide/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/usgs_pesticide/examples.yaml diff --git a/src/biome/adhoc_data/datasources/waterservices_usgs/api.yaml b/src/biome/adhoc_data/datasources/waterservices_usgs/api.yaml index ce8e8fd..4535e58 100644 --- a/src/biome/adhoc_data/datasources/waterservices_usgs/api.yaml +++ b/src/biome/adhoc_data/datasources/waterservices_usgs/api.yaml @@ -1,19 +1,20 @@ name: "USGS Water Services API" -cache_key: api_assistant_usgs_waterservices_json +integration_type: api + description: | - You can use this service to retrieve daily statistical data about the hundreds of + You can use this service to retrieve daily statistical data about the hundreds of thousands of hydrologic sites served by the USGS. Especially crucial to analyze water quality to correlate with other data sources. -web_documentation: !load_txt documentation/usgs_waterservices_web.md -site_types: !load_txt documentation/site_types.md -examples: !load_yaml documentation/examples.yaml -cache_body: - default: true -documentation: !fill | - You can use this service to retrieve daily statistical data about the hundreds of + +attachments: + web_documentation: usgs_waterservices_web.md + site_types: site_types.md + +prompt: | + You can use this service to retrieve daily statistical data about the hundreds of thousands of hydrologic sites served by the USGS. Various output formats available, prefer JSON. - {web_documentation} + ${web_documentation} ### Site Types (siteType) minor filter codes - {site_types} \ No newline at end of file + ${site_types} diff --git a/src/biome/adhoc_data/datasources/waterservices_usgs/documentation/site_types.md b/src/biome/adhoc_data/datasources/waterservices_usgs/attachments/site_types.md similarity index 100% rename from src/biome/adhoc_data/datasources/waterservices_usgs/documentation/site_types.md rename to src/biome/adhoc_data/datasources/waterservices_usgs/attachments/site_types.md diff --git a/src/biome/adhoc_data/datasources/waterservices_usgs/documentation/usgs_waterservices_web.md b/src/biome/adhoc_data/datasources/waterservices_usgs/attachments/usgs_waterservices_web.md similarity index 100% rename from src/biome/adhoc_data/datasources/waterservices_usgs/documentation/usgs_waterservices_web.md rename to src/biome/adhoc_data/datasources/waterservices_usgs/attachments/usgs_waterservices_web.md diff --git a/src/biome/adhoc_data/datasources/waterservices_usgs/documentation/examples.yaml b/src/biome/adhoc_data/datasources/waterservices_usgs/examples.yaml similarity index 100% rename from src/biome/adhoc_data/datasources/waterservices_usgs/documentation/examples.yaml rename to src/biome/adhoc_data/datasources/waterservices_usgs/examples.yaml From 9ca293c923c99c738aa9fffad5e2d0015fc53657 Mon Sep 17 00:00:00 2001 From: Satchel Baldwin Date: Fri, 27 Jun 2025 10:26:42 -0500 Subject: [PATCH 14/21] change adhoc init based on beaker-kernel changes --- src/biome/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/biome/context.py b/src/biome/context.py index 158d072..5af6c64 100644 --- a/src/biome/context.py +++ b/src/biome/context.py @@ -62,7 +62,7 @@ def __init__(self, beaker_kernel: "LLMKernel", config: Dict[str, Any]): gpt_41_config = {**gpt_41, 'api_key': os.environ.get("OPENAI_API_KEY")} # Initialize adhoc integration - adhoc_integration = AdhocIntegrationProvider.from_file_structure( + adhoc_integration = AdhocIntegrationProvider( adhoc_path=ADHOC_DIR_PATH, drafter_config=[gpt_41_config, drafter_config_anthropic, drafter_config_gemini], curator_config=curator_config, From 2b55eba089125e22e73f07a32ceb96d022c08fef Mon Sep 17 00:00:00 2001 From: Satchel Baldwin Date: Tue, 1 Jul 2025 15:13:35 -0500 Subject: [PATCH 15/21] add: tag all resources by uuid, treat examples as resources, move to resources as a generic property instead of attachments --- src/biome/adhoc_data/datasources/ahs/api.yaml | 547 ++++++++--- .../adhoc_data/datasources/ahs/examples.yaml | 498 ---------- src/biome/adhoc_data/datasources/aqs/api.yaml | 739 ++++++++++----- .../adhoc_data/datasources/aqs/examples.yaml | 563 ----------- .../datasources/cbioportal/api.yaml | 642 ++++++++++++- .../datasources/cbioportal/examples.yaml | 589 ------------ src/biome/adhoc_data/datasources/cda/api.yaml | 186 ++-- .../adhoc_data/datasources/cda/examples.yaml | 0 .../datasources/cdc_tracking_network/api.yaml | 80 +- .../cdc_tracking_network/examples.yaml | 0 .../datasources/census_acs/api.yaml | 101 +- .../datasources/census_acs/examples.yaml | 52 - .../chis_california_asthma/api.yaml | 53 +- .../chis_california_asthma/examples.yaml | 17 - .../adhoc_data/datasources/epa_tri/api.yaml | 542 +++++++++-- .../datasources/epa_tri/examples.yaml | 462 --------- .../adhoc_data/datasources/faers/api.yaml | 75 +- .../datasources/faers/examples.yaml | 33 - src/biome/adhoc_data/datasources/gdc/api.yaml | 321 ++++++- .../adhoc_data/datasources/gdc/examples.yaml | 262 ----- .../adhoc_data/datasources/gras/api.yaml | 92 +- .../adhoc_data/datasources/gras/examples.yaml | 23 - src/biome/adhoc_data/datasources/hpa/api.yaml | 126 ++- .../adhoc_data/datasources/hpa/examples.yaml | 37 - src/biome/adhoc_data/datasources/idc/api.yaml | 359 ++++++- .../adhoc_data/datasources/idc/examples.yaml | 444 --------- .../adhoc_data/datasources/indra/api.yaml | 265 +++++- .../datasources/indra/examples.yaml | 175 ---- .../adhoc_data/datasources/netrias/api.yaml | 129 +-- .../datasources/netrias/examples.yaml | 62 -- .../datasources/nhanes_dietary/api.yaml | 133 +-- .../datasources/nhanes_dietary/examples.yaml | 0 .../adhoc_data/datasources/nsch/api.yaml | 394 +++++--- .../adhoc_data/datasources/nsch/examples.yaml | 279 ------ src/biome/adhoc_data/datasources/pdc/api.yaml | 897 +++++++++++++++++- .../adhoc_data/datasources/pdc/examples.yaml | 826 ---------------- .../adhoc_data/datasources/synapse/api.yaml | 190 ++-- .../datasources/synapse/examples.yaml | 11 - .../datasources/usda_foodcentral/api.yaml | 181 ++-- .../usda_foodcentral/examples.yaml | 21 - .../datasources/usgs_pesticide/api.yaml | 82 +- .../datasources/usgs_pesticide/examples.yaml | 0 .../datasources/waterservices_usgs/api.yaml | 60 +- .../waterservices_usgs/examples.yaml | 32 - src/biome/context.py | 4 +- 45 files changed, 4910 insertions(+), 5674 deletions(-) delete mode 100644 src/biome/adhoc_data/datasources/ahs/examples.yaml delete mode 100644 src/biome/adhoc_data/datasources/aqs/examples.yaml delete mode 100644 src/biome/adhoc_data/datasources/cbioportal/examples.yaml delete mode 100644 src/biome/adhoc_data/datasources/cda/examples.yaml delete mode 100644 src/biome/adhoc_data/datasources/cdc_tracking_network/examples.yaml delete mode 100644 src/biome/adhoc_data/datasources/census_acs/examples.yaml delete mode 100644 src/biome/adhoc_data/datasources/chis_california_asthma/examples.yaml delete mode 100644 src/biome/adhoc_data/datasources/epa_tri/examples.yaml delete mode 100644 src/biome/adhoc_data/datasources/faers/examples.yaml delete mode 100644 src/biome/adhoc_data/datasources/gdc/examples.yaml delete mode 100644 src/biome/adhoc_data/datasources/gras/examples.yaml delete mode 100644 src/biome/adhoc_data/datasources/hpa/examples.yaml delete mode 100644 src/biome/adhoc_data/datasources/idc/examples.yaml delete mode 100644 src/biome/adhoc_data/datasources/indra/examples.yaml delete mode 100644 src/biome/adhoc_data/datasources/netrias/examples.yaml delete mode 100644 src/biome/adhoc_data/datasources/nhanes_dietary/examples.yaml delete mode 100644 src/biome/adhoc_data/datasources/nsch/examples.yaml delete mode 100644 src/biome/adhoc_data/datasources/pdc/examples.yaml delete mode 100644 src/biome/adhoc_data/datasources/synapse/examples.yaml delete mode 100644 src/biome/adhoc_data/datasources/usda_foodcentral/examples.yaml delete mode 100644 src/biome/adhoc_data/datasources/usgs_pesticide/examples.yaml delete mode 100644 src/biome/adhoc_data/datasources/waterservices_usgs/examples.yaml diff --git a/src/biome/adhoc_data/datasources/ahs/api.yaml b/src/biome/adhoc_data/datasources/ahs/api.yaml index 9130a81..65d1591 100644 --- a/src/biome/adhoc_data/datasources/ahs/api.yaml +++ b/src/biome/adhoc_data/datasources/ahs/api.yaml @@ -1,120 +1,441 @@ -name: "Census American Housing Survey (AHS)" -integration_type: dataset - description: | You have access to knowledge to all the Census Housing survey data, every 2 years from the year 1999 to 2023. You can use this API for answering questions on american housing, specifically their housing conditions (mold, water problems, etc), which can be used to correlate to health conditions, or other purposes. +integration_type: dataset +name: Census American Housing Survey (AHS) +prompt: | + You have access to a file-based Census American Housing Survey + (National Public Use Files CSV) from every 2 years from 1999 to 2023, under the `${DATASET_FILES_BASE_PATH}/census-ahs` directory. + The housing files are located and named per year as follows: + - ${DATASET_FILES_BASE_PATH}/census-ahs/survey_2023.csv + - ${DATASET_FILES_BASE_PATH}/census-ahs/survey_2021.csv + - ... + - ${DATASET_FILES_BASE_PATH}/census-ahs/survey_1999.csv + the more recent onces are at the metropolitan area level, while older ones + are only avaiable at the national level. Use whichever year files you need, depending + on the user question. -attachments: - more_instructions: docs.md + {more_instructions} -prompt: | - You have access to a file-based Census American Housing Survey - (National Public Use Files CSV) from every 2 years from 1999 to 2023, under the `${DATASET_FILES_BASE_PATH}/census-ahs` directory. - The housing files are located and named per year as follows: - - ${DATASET_FILES_BASE_PATH}/census-ahs/survey_2023.csv - - ${DATASET_FILES_BASE_PATH}/census-ahs/survey_2021.csv - - ... - - ${DATASET_FILES_BASE_PATH}/census-ahs/survey_1999.csv - the more recent onces are at the metropolitan area level, while older ones - are only avaiable at the national level. Use whichever year files you need, depending - on the user question. - - {more_instructions} - - In particular, we have the household data file, which means we - have the OMB13CBSA as a constantly availablecolumn to identify geographic area of survey( - State-based Metropolitan and Micropolitan Statistical Areas). Pre-2015 have more geographical - column/information. Don't try to use state/county/etc if those are not present on - the census housing file for that year. - - Try to alreay have planned which columns you need for each, in order to process less columns/excess data since - merging both datasets will result in more than 1000 columns. Plan it out, select relevant columns - (list the column names once filtered to knkow which are available), - then merge the datasets with those columns selected. - - Once you identify columns of interest available, you can use this csv codebook dictionary that explains what each column code means: - - - ${DATASET_FILES_BASE_PATH}/census-ahs/census_ahs_codebook.csv - - from the codebook,you can use the "Variable" column to match the variable you have access to (say "OMB13CBSA") and use the column "Response Codes" to get - the meaning of the codes or the numeric code range values, for other types of variables. - - you may open it and use it at your discretion- usually better once you have some columns of interest - in order to select a subset of the codebook, since it is pretty big and confusing. - - Don't try to find the codebook columns without checking if they are present, since - the codebook is very exhaustive and contains all the columns or other types of files. - - You can also use knowledge on the usual column codenames from the census to answer questions, but ensure - to check the columns names with pandas (python lib) and _always_ check if the column - is present in the dataset itself instead of assuming it's present. - - Instead of assuming a specific column exist, or if you wish to match with a specific column pattern, - try to compare substrings instead. For example, if you need data for mold, try - to check for the 'mold' substring in the column names, since these csv files sometimes append - soe prefixes. - - The dataset csv files are a bit big, in the sense that they're in - wide format and contain too many columns. When planning what you need, select a subset - of the columns needed, including the geo/housing-characteristics columns for your analysis, - but don't include the thousands of columns. - - # Overview of the American Housing Survey (AHS) - The American Housing Survey (AHS) is a comprehensive national housing survey conducted by the U.S. Census Bureau. It provides detailed information about housing conditions, household characteristics, and neighborhood features across the United States. - - ## Key Dataset Characteristics - - ### Temporal Coverage - - Biennial surveys from 1999 to 2023 - - Data is collected every two years, allowing for trend analysis - - Most recent survey available is from 2023 - - ### Geospatial Resolution - - Metropolitan area level using Core-Based Statistical Areas (CBSA codes) - - Recent surveys (post-2015) use 2013 OMB CBSA codes - - Older surveys (pre-2015) use different geographic identifiers (SMSA codes) - - County-level data is limited, especially in recent surveys - - Harris County/Houston Data: Available in older surveys (1999-2007) using SMSA code 3360, but not found in more recent surveys (2015-2023) - - ### Housing Condition Variables - - Mold: Presence of mold in different areas of the home (basement, bathroom, bedroom, kitchen, living room) - - Water Leaks: Inside and outside water leaks, including specific sources (roof, plumbing, basement) - - Pests: Presence of rodents (rats, mice) and insects - - Air Quality: Indoor air quality ratings - - HVAC Systems: Heating and cooling system types and adequacy - - ### Health-Related Variables - - Asthma: Recent surveys (2015, 2023) include questions about household members diagnosed with asthma - - Emergency Room Visits: Data on ER visits due to asthma - - Asthma Medication: Some surveys include information about asthma medication use - - ### Household Composition - - Number of persons in household - - Presence of children (various age groups) - - Number of adults and elders - - ### Limitations and Challenges - #### Geographic Specificity: - - Recent surveys (2015-2023) do not contain Houston/Harris County specific data - - Only metropolitan area level data is available in recent surveys - - #### Data Consistency: - - Variable definitions and coding schemes change across survey years - - Some variables are only available in certain years - - #### Direct Causality: - - The survey provides correlational data but cannot establish causal relationships between housing conditions and health outcomes - - #### Sample Size: - - When filtering for specific conditions, sample sizes can become small, limiting statistical power - - # Additional Instructions: - - Read the CSV fiels with index_col=False, like so: - ``` - codebook = pd.read_csv('${DATASET_FILES_BASE_PATH}/census-ahs/census_ahs_codebook.csv', dtype=str, index_col=False) - ``` - If you don't add the index_col=False, the first column values will disappear and the whole dataframe will be shifted! + In particular, we have the household data file, which means we + have the OMB13CBSA as a constantly availablecolumn to identify geographic area of survey( + State-based Metropolitan and Micropolitan Statistical Areas). Pre-2015 have more geographical + column/information. Don't try to use state/county/etc if those are not present on + the census housing file for that year. + + Try to alreay have planned which columns you need for each, in order to process less columns/excess data since + merging both datasets will result in more than 1000 columns. Plan it out, select relevant columns + (list the column names once filtered to knkow which are available), + then merge the datasets with those columns selected. + + Once you identify columns of interest available, you can use this csv codebook dictionary that explains what each column code means: + + - ${DATASET_FILES_BASE_PATH}/census-ahs/census_ahs_codebook.csv + + from the codebook,you can use the "Variable" column to match the variable you have access to (say "OMB13CBSA") and use the column "Response Codes" to get + the meaning of the codes or the numeric code range values, for other types of variables. + + you may open it and use it at your discretion- usually better once you have some columns of interest + in order to select a subset of the codebook, since it is pretty big and confusing. + + Don't try to find the codebook columns without checking if they are present, since + the codebook is very exhaustive and contains all the columns or other types of files. + + You can also use knowledge on the usual column codenames from the census to answer questions, but ensure + to check the columns names with pandas (python lib) and _always_ check if the column + is present in the dataset itself instead of assuming it's present. + + Instead of assuming a specific column exist, or if you wish to match with a specific column pattern, + try to compare substrings instead. For example, if you need data for mold, try + to check for the 'mold' substring in the column names, since these csv files sometimes append + soe prefixes. + + The dataset csv files are a bit big, in the sense that they're in + wide format and contain too many columns. When planning what you need, select a subset + of the columns needed, including the geo/housing-characteristics columns for your analysis, + but don't include the thousands of columns. + + # Overview of the American Housing Survey (AHS) + The American Housing Survey (AHS) is a comprehensive national housing survey conducted by the U.S. Census Bureau. It provides detailed information about housing conditions, household characteristics, and neighborhood features across the United States. + + ## Key Dataset Characteristics + + ### Temporal Coverage + - Biennial surveys from 1999 to 2023 + - Data is collected every two years, allowing for trend analysis + - Most recent survey available is from 2023 + + ### Geospatial Resolution + - Metropolitan area level using Core-Based Statistical Areas (CBSA codes) + - Recent surveys (post-2015) use 2013 OMB CBSA codes + - Older surveys (pre-2015) use different geographic identifiers (SMSA codes) + - County-level data is limited, especially in recent surveys + - Harris County/Houston Data: Available in older surveys (1999-2007) using SMSA code 3360, but not found in more recent surveys (2015-2023) + + ### Housing Condition Variables + - Mold: Presence of mold in different areas of the home (basement, bathroom, bedroom, kitchen, living room) + - Water Leaks: Inside and outside water leaks, including specific sources (roof, plumbing, basement) + - Pests: Presence of rodents (rats, mice) and insects + - Air Quality: Indoor air quality ratings + - HVAC Systems: Heating and cooling system types and adequacy + + ### Health-Related Variables + - Asthma: Recent surveys (2015, 2023) include questions about household members diagnosed with asthma + - Emergency Room Visits: Data on ER visits due to asthma + - Asthma Medication: Some surveys include information about asthma medication use + + ### Household Composition + - Number of persons in household + - Presence of children (various age groups) + - Number of adults and elders + + ### Limitations and Challenges + #### Geographic Specificity: + - Recent surveys (2015-2023) do not contain Houston/Harris County specific data + - Only metropolitan area level data is available in recent surveys + + #### Data Consistency: + - Variable definitions and coding schemes change across survey years + - Some variables are only available in certain years + + #### Direct Causality: + - The survey provides correlational data but cannot establish causal relationships between housing conditions and health outcomes + + #### Sample Size: + - When filtering for specific conditions, sample sizes can become small, limiting statistical power + + # Additional Instructions: + + Read the CSV fiels with index_col=False, like so: + ``` + codebook = pd.read_csv('${DATASET_FILES_BASE_PATH}/census-ahs/census_ahs_codebook.csv', dtype=str, index_col=False) + ``` + If you don't add the index_col=False, the first column values will disappear and the whole dataframe will be shifted! +resources: + 3a5b7607-80c5-44ae-a240-a9a388f78afd: + filepath: docs.md + integration: census_american_housing_survey_(ahs) + name: more_instructions + resource_id: 3a5b7607-80c5-44ae-a240-a9a388f78afd + resource_type: file + 5025a179-2b90-46ab-ab7e-deef3aa8b129: + code: | + import pandas as pd + + def get_code_name(code): + # Read codebook to get metro area meanings + # ensure to set index_col=False, since the codebook has no index column + codebook = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/census-ahs/census_ahs_codebook.csv', index_col=False) + + # Find the row for OMB13CBSA + ombs_row = codebook[codebook['Variable'] == 'OMB13CBSA'] + + response_codes_text = ombs_row['Response Codes'].iloc[0] # Get the string value from the Series + code_pairs = response_codes_text.split("||") + + code_pairs_dict = {k.strip(): v.strip() for k, v in (code_pair.split(":") for code_pair in code_pairs)} + + return code_pairs_dict.get(code) + integration: census_american_housing_survey_(ahs) + notes: null + query: |- + Use the codebook to get the meaning of the codes for the OMB13CBSA variable, matching one code to the meaning by creating a dictionary of code:meaning pairs and returning the statistical metro area name for a given code + resource_id: 5025a179-2b90-46ab-ab7e-deef3aa8b129 + resource_type: example + 569a9b73-d61a-4b9a-8ee1-9c85c1637e1a: + code: "import pandas as pd\nimport numpy as np\n\n# Load the AHS dataset (2023)\ + \ with asthma-related variables\nfile_path = '{{DATASET_FILES_BASE_PATH}}/census-ahs/survey_2023.csv'\n\ + \n# Define variables of interest\nasthma_vars = ['HHLDASTHMA'] # Household\ + \ member ever diagnosed with asthma\nhousing_vars = [\n 'MOLDBASEM', 'MOLDBATH',\ + \ 'MOLDBEDRM', 'MOLDKITCH', 'MOLDLROOM', 'MOLDOTHER', # Mold\n 'LEAKI',\ + \ 'LEAKO' # Water\ + \ leaks\n]\nchild_vars = ['NUMOLDKIDS', 'NUMYNGKIDS']\ngeo_vars = ['OMB13CBSA']\n\ + \n# Combine all variables of interest\ncols_of_interest = asthma_vars + housing_vars\ + \ + child_vars + geo_vars\n\n# Load a sample of the data\nsample_size = 10000\ + \ # Adjust based on memory constraints\nahs_sample = pd.read_csv(file_path,\ + \ usecols=cols_of_interest, nrows=sample_size)\n\n# Clean the data by removing\ + \ quotes from string values\nfor col in ahs_sample.columns:\n if ahs_sample[col].dtype\ + \ == 'object':\n ahs_sample[col] = ahs_sample[col].str.replace(\"'\"\ + , \"\")\n\n# Create a binary variable for presence of children\nahs_sample['has_children']\ + \ = ((ahs_sample['NUMOLDKIDS'] > 0) | (ahs_sample['NUMYNGKIDS'] > 0)).astype(int)\n\ + \n# Convert asthma variable to binary (Yes/No)\nahs_sample['asthma'] = ahs_sample['HHLDASTHMA'].apply(\n\ + \ lambda x: 1 if x == '1' else (0 if x == '2' else np.nan)\n)\n\n# Convert\ + \ housing condition variables to binary (Yes/No)\nfor col in housing_vars:\n\ + \ ahs_sample[col + '_binary'] = ahs_sample[col].apply(\n lambda x:\ + \ 1 if x == '1' else (0 if x == '2' else np.nan)\n )\n\n# Analyze relationship\ + \ between housing conditions and asthma\nprint(\"Relationship between asthma\ + \ and housing conditions:\")\nfor col in housing_vars:\n binary_col = col\ + \ + '_binary'\n valid_data = ahs_sample.dropna(subset=['asthma', binary_col])\n\ + \ \n if len(valid_data) > 0:\n with_asthma = valid_data[valid_data['asthma']\ + \ == 1][binary_col].mean() * 100\n without_asthma = valid_data[valid_data['asthma']\ + \ == 0][binary_col].mean() * 100\n \n if not np.isnan(with_asthma)\ + \ and not np.isnan(without_asthma):\n print(f\"\\n{col}:\")\n \ + \ print(f\" - Homes with asthma: {with_asthma:.1f}%\")\n \ + \ print(f\" - Homes without asthma: {without_asthma:.1f}%\")\n \ + \ print(f\" - Difference: {with_asthma - without_asthma:.1f} percentage points\"\ + )\n\n# Calculate the prevalence of asthma in households with and without children\n\ + asthma_by_children = ahs_sample.dropna(subset=['asthma', 'has_children']).groupby('has_children')['asthma'].mean()\ + \ * 100\nprint(\"\\nAsthma Prevalence by Presence of Children:\")\nprint(f\"\ + Households with children: {asthma_by_children.get(1, 'N/A'):.1f}%\")\nprint(f\"\ + Households without children: {asthma_by_children.get(0, 'N/A'):.1f}%\")\nprint(f\"\ + Difference: {asthma_by_children.get(1, 0) - asthma_by_children.get(0, 0):.1f}\ + \ percentage points\")\n" + integration: census_american_housing_survey_(ahs) + notes: null + query: |- + Analyzing the relationship between housing conditions, asthma, and the presence of children using the 2023 American Housing Survey data. This code calculates and compares the prevalence of various housing conditions (mold, water leaks) in homes with and without reported asthma cases, and also examines the prevalence of asthma in households with and without children. + resource_id: 569a9b73-d61a-4b9a-8ee1-9c85c1637e1a + resource_type: example + 80f97f58-e289-4afe-9959-37212def3ce6: + code: "import pandas as pd\nimport numpy as np\n\n# Load the 2007 AHS dataset\ + \ with Houston data\nfile_path = '{{DATASET_FILES_BASE_PATH}}/census-ahs/survey_2007.csv'\n\ + \n# Define Houston SMSA code\nhouston_smsa = '3360'\n\n# Define housing condition\ + \ variables based on our exploration\nhousing_vars = [\n 'LEAK', 'ILEAK',\ + \ # Water leaks\n 'RATS', 'MICE' \ + \ # Pests\n]\n\n# Define household size\ + \ variable\nhousehold_var = 'PER'\n\n# Combine all columns of interest\ncols_of_interest\ + \ = ['SMSA'] + housing_vars + [household_var]\n\n# Load the Houston data\nhouston_data\ + \ = pd.read_csv(file_path, usecols=cols_of_interest)\n\n# Clean the data by\ + \ removing quotes if needed\nfor col in houston_data.columns:\n if houston_data[col].dtype\ + \ == 'object':\n houston_data[col] = houston_data[col].str.replace(\"\ + '\", \"\")\n\n# Filter for Houston\nhouston_data = houston_data[houston_data['SMSA']\ + \ == houston_smsa]\n\nprint(f\"Loaded {len(houston_data)} Houston records from\ + \ 2007 survey\")\n\n# Create a binary variable for households likely to have\ + \ children (3+ persons)\n# This is a proxy since we don't have direct child\ + \ indicators\nhouston_data['likely_has_children'] = houston_data[household_var].apply(\n\ + \ lambda x: 1 if x not in ['-6', '-9'] and int(x) >= 3 else 0\n)\n\nprint(f\"\ + \\nHouseholds likely to have children (3+ persons): {houston_data['likely_has_children'].sum()}\ + \ ({houston_data['likely_has_children'].mean()*100:.1f}% of Houston sample)\"\ + )\n\n# Convert housing condition variables to binary (Yes/No)\nfor col in housing_vars:\n\ + \ houston_data[col + '_binary'] = houston_data[col].apply(\n lambda\ + \ x: 1 if x == '1' else (0 if x == '2' else np.nan)\n )\n\n# Analyze housing\ + \ conditions\nprint(\"\\nPrevalence of housing conditions in Houston (2007):\"\ + )\nfor col in housing_vars:\n binary_col = col + '_binary'\n valid_data\ + \ = houston_data.dropna(subset=[binary_col])\n if len(valid_data) > 0:\n\ + \ yes_pct = valid_data[binary_col].mean() * 100\n print(f\"{col}:\ + \ {yes_pct:.1f}% reported 'Yes' (based on {len(valid_data)} valid responses)\"\ + )\n\n# Analyze relationship between household size and housing conditions\n\ + print(\"\\nRelationship between household size and housing conditions in Houston\ + \ (2007):\")\nfor col in housing_vars:\n binary_col = col + '_binary'\n \ + \ valid_data = houston_data.dropna(subset=[binary_col, 'likely_has_children'])\n\ + \ \n if len(valid_data) > 0:\n with_children = valid_data[valid_data['likely_has_children']\ + \ == 1][binary_col].mean() * 100\n without_children = valid_data[valid_data['likely_has_children']\ + \ == 0][binary_col].mean() * 100\n \n if not np.isnan(with_children)\ + \ and not np.isnan(without_children):\n print(f\"\\n{col}:\")\n \ + \ print(f\" - Households with 3+ persons: {with_children:.1f}%\"\ + )\n print(f\" - Households with 1-2 persons: {without_children:.1f}%\"\ + )\n print(f\" - Difference: {with_children - without_children:.1f}\ + \ percentage points\")\n" + integration: census_american_housing_survey_(ahs) + notes: null + query: |- + Analyzing housing conditions in Houston, TX using the 2007 American Housing Survey data. This code identifies Houston records using the SMSA code, analyzes the prevalence of water leaks and pest problems, and compares these conditions between larger households (3+ persons, likely to have children) and smaller households (1-2 persons). + resource_id: 80f97f58-e289-4afe-9959-37212def3ce6 + resource_type: example + ad42a806-0e81-4e63-97cf-2f7aad6a1a35: + code: | + import pandas as pd + + # Read codebook to get metro area meanings + # ensure to set index_col=False, since the codebook has no index column + codebook = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/census-ahs/census_ahs_codebook.csv', index_col=False) + + # Find the row for OMB13CBSA + ombs_row = codebook[codebook['Variable'] == 'OMB13CBSA'] + + # Return the Response Codes column + ombs_row['Response Codes'] + integration: census_american_housing_survey_(ahs) + notes: null + query: What is the meaning of the codes for the OMB13CBSA variable? + resource_id: ad42a806-0e81-4e63-97cf-2f7aad6a1a35 + resource_type: example + badf3293-de67-4c9c-944e-6dbbb8b609d6: + code: "import pandas as pd\nimport numpy as np\n\n# Load the codebook to find\ + \ information about variables\ncodebook_path = '{{DATASET_FILES_BASE_PATH}}/census-ahs/census_ahs_codebook.csv'\n\ + codebook = pd.read_csv(codebook_path, dtype=str, index_col=False)\n\n# Define\ + \ search terms related to asthma and respiratory health\nasthma_terms = [\n\ + \ 'asthma', 'allerg', 'respir', 'breath', 'lung', 'air', 'ventil', \n \ + \ 'humid', 'mold', 'mildew', 'dust', 'smoke', 'pest', 'roach', 'rodent', \n\ + \ 'insect', 'heat', 'cool', 'toxic', 'pollut', 'chemical', 'lead'\n]\n\n\ + # Search for variables related to these terms\nasthma_vars = []\nfor term in\ + \ asthma_terms:\n term_vars = codebook[\n (codebook['Description'].str.contains(term,\ + \ case=False, na=False)) |\n (codebook['Question Text'].str.contains(term,\ + \ case=False, na=False)) |\n (codebook['Variable'].str.contains(term,\ + \ case=False, na=False))\n ]\n if not term_vars.empty:\n for _,\ + \ row in term_vars.iterrows():\n var_info = {\n 'Variable':\ + \ row['Variable'],\n 'Description': row['Description'] if 'Description'\ + \ in row and not pd.isna(row['Description']) else 'No description',\n \ + \ 'Question_Text': row['Question Text'] if 'Question Text' in row and\ + \ not pd.isna(row['Question Text']) else 'No question text',\n \ + \ 'Survey_Years': row['Survey Years'] if 'Survey Years' in row and not pd.isna(row['Survey\ + \ Years']) else 'Unknown',\n 'Response_Codes': row['Response\ + \ Codes'] if 'Response Codes' in row and not pd.isna(row['Response Codes'])\ + \ else 'No codes'\n }\n # Check if this variable is already\ + \ in our list\n if not any(v['Variable'] == var_info['Variable']\ + \ for v in asthma_vars):\n asthma_vars.append(var_info)\n\n#\ + \ Create a DataFrame for easier viewing\nasthma_vars_df = pd.DataFrame(asthma_vars)\n\ + \n# Check if we have any variables specifically about asthma\nasthma_specific\ + \ = asthma_vars_df[\n asthma_vars_df['Description'].str.contains('asthma',\ + \ case=False, na=False) |\n asthma_vars_df['Question_Text'].str.contains('asthma',\ + \ case=False, na=False)\n]\n\nprint(f\"Found {len(asthma_vars_df)} variables\ + \ potentially related to asthma and respiratory health\")\nprint(f\"Found {len(asthma_specific)}\ + \ variables specifically mentioning asthma\")\n\n# Display the asthma-specific\ + \ variables if any\nif not asthma_specific.empty:\n print(\"\\nVariables\ + \ specifically mentioning asthma:\")\n for _, row in asthma_specific.head(5).iterrows():\ + \ # Show first 5 for brevity\n print(f\"\\nVariable: {row['Variable']}\"\ + )\n print(f\"Description: {row['Description']}\")\n print(f\"\ + Question Text: {row['Question_Text']}\")\n print(f\"Survey Years: {row['Survey_Years']}\"\ + )\n\n# Check for variables related to indoor air quality\nair_quality = asthma_vars_df[\n\ + \ asthma_vars_df['Description'].str.contains('air quality|ventil|humid',\ + \ case=False, na=False) |\n asthma_vars_df['Question_Text'].str.contains('air\ + \ quality|ventil|humid', case=False, na=False)\n]\n\nprint(f\"\\nFound {len(air_quality)}\ + \ variables related to indoor air quality\")\n\n# Check for variables related\ + \ to pests\npest_vars = asthma_vars_df[\n asthma_vars_df['Description'].str.contains('pest|roach|rodent|insect|rat|mice|mouse',\ + \ case=False, na=False) |\n asthma_vars_df['Question_Text'].str.contains('pest|roach|rodent|insect|rat|mice|mouse',\ + \ case=False, na=False)\n]\n\nprint(f\"\\nFound {len(pest_vars)} variables related\ + \ to pests\")\n\n# Check for variables related to heating and cooling systems\n\ + hvac_vars = asthma_vars_df[\n asthma_vars_df['Description'].str.contains('heat|cool|air\ + \ condition|hvac', case=False, na=False) |\n asthma_vars_df['Question_Text'].str.contains('heat|cool|air\ + \ condition|hvac', case=False, na=False)\n]\n\nprint(f\"\\nFound {len(hvac_vars)}\ + \ variables related to heating and cooling systems\")" + integration: census_american_housing_survey_(ahs) + notes: null + query: |- + Exploring the American Housing Survey codebook to identify variables related to asthma, respiratory health, and housing conditions. This code searches the codebook for terms related to asthma, air quality, pests, and HVAC systems, and provides a summary of available variables that could be relevant for analyzing the relationship between housing conditions and respiratory health. + resource_id: badf3293-de67-4c9c-944e-6dbbb8b609d6 + resource_type: example + e72b358a-6a63-4d79-9ee5-e81b5d99af2b: + code: "import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\ + import seaborn as sns\nfrom matplotlib.ticker import PercentFormatter\n\n# Load\ + \ a sample of the AHS dataset (2023)\nfile_path = '{{DATASET_FILES_BASE_PATH}}/census-ahs/survey_2023.csv'\n\ + \n# Define columns of interest based on our exploration\ngeo_cols = ['OMB13CBSA']\n\ + mold_cols = ['MOLDBASEM', 'MOLDBATH', 'MOLDBEDRM', 'MOLDKITCH', 'MOLDLROOM',\ + \ 'MOLDOTHER']\nleak_cols = ['LEAKI', 'LEAKO']\nchild_cols = ['NUMOLDKIDS',\ + \ 'NUMYNGKIDS']\n\n# Combine all columns of interest\ncols_of_interest = geo_cols\ + \ + mold_cols + leak_cols + child_cols\n\n# Load a sample of the data with only\ + \ the columns we need\nsample_size = 10000 # Adjust based on memory constraints\n\ + ahs_sample = pd.read_csv(file_path, usecols=cols_of_interest, nrows=sample_size)\n\ + \n# Clean the data by removing quotes from string values\nfor col in ahs_sample.columns:\n\ + \ if ahs_sample[col].dtype == 'object':\n ahs_sample[col] = ahs_sample[col].str.replace(\"\ + '\", \"\")\n\n# Create a binary variable for presence of children\nahs_sample['has_children']\ + \ = ((ahs_sample['NUMOLDKIDS'] > 0) | (ahs_sample['NUMYNGKIDS'] > 0)).astype(int)\n\ + \n# Convert categorical variables to numeric for analysis\nfor col in mold_cols\ + \ + leak_cols:\n # Convert '1' (Yes) to 1, '2' (No) to 0, and others to NaN\n\ + \ ahs_sample[col] = ahs_sample[col].apply(\n lambda x: 1 if x == '1'\ + \ else (0 if x == '2' else np.nan)\n )\n\n# Create a more descriptive mapping\ + \ for the variables\nvariable_descriptions = {\n 'MOLDBASEM': 'Mold in Basement',\n\ + \ 'MOLDBATH': 'Mold in Bathroom',\n 'MOLDBEDRM': 'Mold in Bedroom',\n\ + \ 'MOLDKITCH': 'Mold in Kitchen',\n 'MOLDLROOM': 'Mold in Living Room',\n\ + \ 'MOLDOTHER': 'Mold in Other Areas',\n 'LEAKI': 'Inside Water Leaks',\n\ + \ 'LEAKO': 'Outside Water Leaks'\n}\n\n# Prepare data for visualization\n\ + conditions = []\nwith_children_vals = []\nwithout_children_vals = []\nsample_sizes\ + \ = []\n\nfor condition in mold_cols + leak_cols:\n valid_data = ahs_sample.dropna(subset=[condition,\ + \ 'has_children'])\n \n if len(valid_data) > 0:\n with_children_val\ + \ = valid_data[valid_data['has_children'] == 1][condition].mean() * 100\n \ + \ without_children_val = valid_data[valid_data['has_children'] == 0][condition].mean()\ + \ * 100\n \n if not np.isnan(with_children_val) and not np.isnan(without_children_val):\n\ + \ conditions.append(variable_descriptions.get(condition, condition))\n\ + \ with_children_vals.append(with_children_val)\n without_children_vals.append(without_children_val)\n\ + \ sample_sizes.append(len(valid_data))\n\n# Create a DataFrame for\ + \ plotting\nplot_data = pd.DataFrame({\n 'Condition': conditions,\n 'With\ + \ Children (%)': with_children_vals,\n 'Without Children (%)': without_children_vals,\n\ + \ 'Sample Size': sample_sizes\n})\n\n# Sort by prevalence in homes with children\n\ + plot_data = plot_data.sort_values(by='With Children (%)', ascending=False)\n\ + \n# Create a horizontal bar chart\nplt.figure(figsize=(12, 10))\nsns.set_style(\"\ + whitegrid\")\n\n# Create the horizontal bar chart\nax = sns.barplot(\n x='With\ + \ Children (%)', \n y='Condition', \n data=plot_data, \n color='#3498db',\n\ + \ label='Homes with Children'\n)\n\nsns.barplot(\n x='Without Children\ + \ (%)', \n y='Condition', \n data=plot_data, \n color='#e74c3c',\n\ + \ label='Homes without Children'\n)\n\n# Add percentage labels to the bars\n\ + for i, (with_val, without_val) in enumerate(zip(plot_data['With Children (%)'],\ + \ plot_data['Without Children (%)'])):\n if with_val > without_val:\n \ + \ ax.text(with_val + 0.5, i, f'{with_val:.1f}%', va='center')\n ax.text(without_val\ + \ - 2.5, i, f'{without_val:.1f}%', va='center', color='white')\n else:\n\ + \ ax.text(with_val - 2.5, i, f'{with_val:.1f}%', va='center', color='white')\n\ + \ ax.text(without_val + 0.5, i, f'{without_val:.1f}%', va='center')\n\ + \n# Add sample size information to the y-axis labels\nlabels = [f\"{condition}\\\ + n(n={size:,})\" for condition, size in zip(plot_data['Condition'], plot_data['Sample\ + \ Size'])]\nax.set_yticklabels(labels)\n\n# Format the x-axis as percentages\n\ + ax.xaxis.set_major_formatter(PercentFormatter())\n\n# Add a title and labels\n\ + plt.title('Housing Conditions by Presence of Children\\nAmerican Housing Survey\ + \ (2023)', fontsize=16, pad=20)\nplt.xlabel('Prevalence (%)', fontsize=12)\n\ + plt.ylabel('Housing Condition', fontsize=12)\nplt.legend(title='Household Type',\ + \ loc='lower right')\n\n# Add a note about the data\nplt.figtext(0.5, 0.01,\ + \ \n \"Note: Data from 2023 American Housing Survey. Sample sizes\ + \ vary by condition.\\n\"\n \"Percentages represent the proportion\ + \ of households reporting each condition.\", \n ha='center', fontsize=10,\ + \ style='italic')\n\nplt.tight_layout(rect=[0, 0.03, 1, 0.97])\nplt.show()\n\ + \n# Create a summary table of the results\nsummary_table = plot_data.copy()\n\ + summary_table['Difference (percentage points)'] = summary_table['With Children\ + \ (%)'] - summary_table['Without Children (%)']\nsummary_table = summary_table.sort_values(by='Difference\ + \ (percentage points)', ascending=False)\n\nprint(\"Summary of Housing Conditions\ + \ by Presence of Children:\")\nprint(summary_table[['Condition', 'With Children\ + \ (%)', 'Without Children (%)', 'Difference (percentage points)', 'Sample Size']].to_string(index=False,\ + \ float_format=lambda x: f\"{x:.1f}\"))\n" + integration: census_american_housing_survey_(ahs) + notes: null + query: |- + Analyzing and visualizing the relationship between housing conditions (mold, water leaks) and the presence of children in households using the 2023 American Housing Survey data. This code creates a horizontal bar chart comparing the prevalence of various housing conditions in homes with and without children, and generates a summary table of the differences. + resource_id: e72b358a-6a63-4d79-9ee5-e81b5d99af2b + resource_type: example + ea7d2aec-2d76-4b95-ae85-177db08ff019: + code: "import pandas as pd\n\ndata_dir = \"{{DATASET_FILES_BASE_PATH}}/census-ahs\"\ + \n\ndef get_florida_mold_data():\n # Store results\n florida_mold_data\ + \ = []\n \n # Years to analyze\n years = range(2001, 2012, 2)\n \ + \ \n for year in years:\n try:\n # Read data file for year\n\ + \ file_path = os.path.join(data_dir, f'survey_{year}.csv')\n \ + \ \n df = pd.read_csv(file_path, index_col=False)\n \ + \ \n # Find mold-related columns\n mold_cols = [col\ + \ for col in df.columns if 'mold' in col.lower()]\n \n \ + \ # Find geographic identifier columns\n geo_cols = [col for col\ + \ in df.columns if col in ['SMSA', 'METRO3', 'OMB13CBSA', 'DIVISION']]\n \ + \ \n if len(mold_cols) == 0:\n continue\n\ + \ \n if len(geo_cols) == 0:\n continue\n\ + \ \n # Select relevant columns\n cols_to_use\ + \ = geo_cols + mold_cols\n df_subset = df[cols_to_use]\n \ + \ \n # Read codebook to identify Florida metro areas\n \ + \ codebook_path = os.path.join(data_dir, 'census_ahs_codebook.csv')\n \ + \ \n # ensure to pass index_col=False, else sometimes data\ + \ shifts\n codebook = pd.read_csv(codebook_path, index_col=False)\n\ + \ \n # Get Florida metro codes for the geographic identifier\ + \ being used\n geo_col = geo_cols[len(geo_cols) - 1]\n \ + \ \n geo_codes = codebook[codebook['Variable'] == geo_col]\n \ + \ if geo_codes.empty:\n continue\n \n\ + \ florida_codes = [code.split(':')[0].strip() \n \ + \ for code in geo_codes['Response Codes'].iloc[0].split('||')\n \ + \ if 'FL' in code.upper()]\n \n \ + \ # Filter for Florida metros using substring matching\n df_fl\ + \ = df_subset[df_subset[geo_col].apply(lambda x: any(code in str(x) for code\ + \ in florida_codes))]\n \n # Add year column\n \ + \ df_fl.loc[:, 'YEAR'] = year\n \n florida_mold_data.append(df_fl)\n\ + \ \n except Exception as e:\n import traceback\n\ + \ print(f\"Traceback: {traceback.format_exc()}\")\n continue\n\ + \ \n # Combine all years\n if florida_mold_data:\n combined_data\ + \ = pd.concat(florida_mold_data, ignore_index=True)\n \n # Replace\ + \ values in the 'MOLD' columns with number to string mappings\n # where\ + \ MOLD columns are columns that contain MOLD in the column name (substring match)\n\ + \ mold_cols = [col for col in combined_data.columns if 'mold' in col.lower()]\n\ + \ \n for col in mold_cols:\n def convert_value(x):\n\ + \ # Remove any extra quotes that might be present\n \ + \ # many times codes are wrapped in quotes..\n x_str =\ + \ str(x).strip().strip(\"'\\\"\")\n if x_str in ['Y', '1', '1.0']:\n\ + \ return \"Yes\"\n elif x_str in ['N', '2',\ + \ '2.0']:\n return \"No\"\n elif x_str in\ + \ ['-6', '-6.0', 'N/A', 'NA', '']:\n return \"N/A\"\n \ + \ else:\n return x\n \n \ + \ # Remember that this example converts to \"Yes\" / \"No\" / \"N/A\"\n \ + \ # if you plot it, you'll want to \"count\" these text ocurrences\n\ + \ # also remember that values soemtimes are wrapped in quotes, such\ + \ as \"'6'\",\n # that is why we strip them or sometimes compare\ + \ to substrings\n combined_data[col] = combined_data[col].apply(convert_value)\n\ + \ \n return combined_data\n else:\n return pd.DataFrame()\n" + integration: census_american_housing_survey_(ahs) + notes: null + query: |- + I wonder if there are any trends for metro areas in Florida, US in which mold presence increased between years 2001 and 2011? Can you get the data for matching mold presence? + resource_id: ea7d2aec-2d76-4b95-ae85-177db08ff019 + resource_type: example +slug: census_american_housing_survey_(ahs) diff --git a/src/biome/adhoc_data/datasources/ahs/examples.yaml b/src/biome/adhoc_data/datasources/ahs/examples.yaml deleted file mode 100644 index b9d3db0..0000000 --- a/src/biome/adhoc_data/datasources/ahs/examples.yaml +++ /dev/null @@ -1,498 +0,0 @@ -- query: "What is the meaning of the codes for the OMB13CBSA variable?" - code: | - import pandas as pd - - # Read codebook to get metro area meanings - # ensure to set index_col=False, since the codebook has no index column - codebook = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/census-ahs/census_ahs_codebook.csv', index_col=False) - - # Find the row for OMB13CBSA - ombs_row = codebook[codebook['Variable'] == 'OMB13CBSA'] - - # Return the Response Codes column - ombs_row['Response Codes'] - -- query: "Use the codebook to get the meaning of the codes for the OMB13CBSA variable, matching one code to the meaning by creating a dictionary of code:meaning pairs and returning the statistical metro area name for a given code" - code: | - import pandas as pd - - def get_code_name(code): - # Read codebook to get metro area meanings - # ensure to set index_col=False, since the codebook has no index column - codebook = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/census-ahs/census_ahs_codebook.csv', index_col=False) - - # Find the row for OMB13CBSA - ombs_row = codebook[codebook['Variable'] == 'OMB13CBSA'] - - response_codes_text = ombs_row['Response Codes'].iloc[0] # Get the string value from the Series - code_pairs = response_codes_text.split("||") - - code_pairs_dict = {k.strip(): v.strip() for k, v in (code_pair.split(":") for code_pair in code_pairs)} - - return code_pairs_dict.get(code) - - -- query: "I wonder if there are any trends for metro areas in Florida, US in which mold presence increased between years 2001 and 2011? Can you get the data for matching mold presence?" - code: | - import pandas as pd - - data_dir = "{{DATASET_FILES_BASE_PATH}}/census-ahs" - - def get_florida_mold_data(): - # Store results - florida_mold_data = [] - - # Years to analyze - years = range(2001, 2012, 2) - - for year in years: - try: - # Read data file for year - file_path = os.path.join(data_dir, f'survey_{year}.csv') - - df = pd.read_csv(file_path, index_col=False) - - # Find mold-related columns - mold_cols = [col for col in df.columns if 'mold' in col.lower()] - - # Find geographic identifier columns - geo_cols = [col for col in df.columns if col in ['SMSA', 'METRO3', 'OMB13CBSA', 'DIVISION']] - - if len(mold_cols) == 0: - continue - - if len(geo_cols) == 0: - continue - - # Select relevant columns - cols_to_use = geo_cols + mold_cols - df_subset = df[cols_to_use] - - # Read codebook to identify Florida metro areas - codebook_path = os.path.join(data_dir, 'census_ahs_codebook.csv') - - # ensure to pass index_col=False, else sometimes data shifts - codebook = pd.read_csv(codebook_path, index_col=False) - - # Get Florida metro codes for the geographic identifier being used - geo_col = geo_cols[len(geo_cols) - 1] - - geo_codes = codebook[codebook['Variable'] == geo_col] - if geo_codes.empty: - continue - - florida_codes = [code.split(':')[0].strip() - for code in geo_codes['Response Codes'].iloc[0].split('||') - if 'FL' in code.upper()] - - # Filter for Florida metros using substring matching - df_fl = df_subset[df_subset[geo_col].apply(lambda x: any(code in str(x) for code in florida_codes))] - - # Add year column - df_fl.loc[:, 'YEAR'] = year - - florida_mold_data.append(df_fl) - - except Exception as e: - import traceback - print(f"Traceback: {traceback.format_exc()}") - continue - - # Combine all years - if florida_mold_data: - combined_data = pd.concat(florida_mold_data, ignore_index=True) - - # Replace values in the 'MOLD' columns with number to string mappings - # where MOLD columns are columns that contain MOLD in the column name (substring match) - mold_cols = [col for col in combined_data.columns if 'mold' in col.lower()] - - for col in mold_cols: - def convert_value(x): - # Remove any extra quotes that might be present - # many times codes are wrapped in quotes.. - x_str = str(x).strip().strip("'\"") - if x_str in ['Y', '1', '1.0']: - return "Yes" - elif x_str in ['N', '2', '2.0']: - return "No" - elif x_str in ['-6', '-6.0', 'N/A', 'NA', '']: - return "N/A" - else: - return x - - # Remember that this example converts to "Yes" / "No" / "N/A" - # if you plot it, you'll want to "count" these text ocurrences - # also remember that values soemtimes are wrapped in quotes, such as "'6'", - # that is why we strip them or sometimes compare to substrings - combined_data[col] = combined_data[col].apply(convert_value) - - return combined_data - else: - return pd.DataFrame() - -- query: "Analyzing and visualizing the relationship between housing conditions (mold, water leaks) and the presence of children in households using the 2023 American Housing Survey data. This code creates a horizontal bar chart comparing the prevalence of various housing conditions in homes with and without children, and generates a summary table of the differences." - code: | - import pandas as pd - import numpy as np - import matplotlib.pyplot as plt - import seaborn as sns - from matplotlib.ticker import PercentFormatter - - # Load a sample of the AHS dataset (2023) - file_path = '{{DATASET_FILES_BASE_PATH}}/census-ahs/survey_2023.csv' - - # Define columns of interest based on our exploration - geo_cols = ['OMB13CBSA'] - mold_cols = ['MOLDBASEM', 'MOLDBATH', 'MOLDBEDRM', 'MOLDKITCH', 'MOLDLROOM', 'MOLDOTHER'] - leak_cols = ['LEAKI', 'LEAKO'] - child_cols = ['NUMOLDKIDS', 'NUMYNGKIDS'] - - # Combine all columns of interest - cols_of_interest = geo_cols + mold_cols + leak_cols + child_cols - - # Load a sample of the data with only the columns we need - sample_size = 10000 # Adjust based on memory constraints - ahs_sample = pd.read_csv(file_path, usecols=cols_of_interest, nrows=sample_size) - - # Clean the data by removing quotes from string values - for col in ahs_sample.columns: - if ahs_sample[col].dtype == 'object': - ahs_sample[col] = ahs_sample[col].str.replace("'", "") - - # Create a binary variable for presence of children - ahs_sample['has_children'] = ((ahs_sample['NUMOLDKIDS'] > 0) | (ahs_sample['NUMYNGKIDS'] > 0)).astype(int) - - # Convert categorical variables to numeric for analysis - for col in mold_cols + leak_cols: - # Convert '1' (Yes) to 1, '2' (No) to 0, and others to NaN - ahs_sample[col] = ahs_sample[col].apply( - lambda x: 1 if x == '1' else (0 if x == '2' else np.nan) - ) - - # Create a more descriptive mapping for the variables - variable_descriptions = { - 'MOLDBASEM': 'Mold in Basement', - 'MOLDBATH': 'Mold in Bathroom', - 'MOLDBEDRM': 'Mold in Bedroom', - 'MOLDKITCH': 'Mold in Kitchen', - 'MOLDLROOM': 'Mold in Living Room', - 'MOLDOTHER': 'Mold in Other Areas', - 'LEAKI': 'Inside Water Leaks', - 'LEAKO': 'Outside Water Leaks' - } - - # Prepare data for visualization - conditions = [] - with_children_vals = [] - without_children_vals = [] - sample_sizes = [] - - for condition in mold_cols + leak_cols: - valid_data = ahs_sample.dropna(subset=[condition, 'has_children']) - - if len(valid_data) > 0: - with_children_val = valid_data[valid_data['has_children'] == 1][condition].mean() * 100 - without_children_val = valid_data[valid_data['has_children'] == 0][condition].mean() * 100 - - if not np.isnan(with_children_val) and not np.isnan(without_children_val): - conditions.append(variable_descriptions.get(condition, condition)) - with_children_vals.append(with_children_val) - without_children_vals.append(without_children_val) - sample_sizes.append(len(valid_data)) - - # Create a DataFrame for plotting - plot_data = pd.DataFrame({ - 'Condition': conditions, - 'With Children (%)': with_children_vals, - 'Without Children (%)': without_children_vals, - 'Sample Size': sample_sizes - }) - - # Sort by prevalence in homes with children - plot_data = plot_data.sort_values(by='With Children (%)', ascending=False) - - # Create a horizontal bar chart - plt.figure(figsize=(12, 10)) - sns.set_style("whitegrid") - - # Create the horizontal bar chart - ax = sns.barplot( - x='With Children (%)', - y='Condition', - data=plot_data, - color='#3498db', - label='Homes with Children' - ) - - sns.barplot( - x='Without Children (%)', - y='Condition', - data=plot_data, - color='#e74c3c', - label='Homes without Children' - ) - - # Add percentage labels to the bars - for i, (with_val, without_val) in enumerate(zip(plot_data['With Children (%)'], plot_data['Without Children (%)'])): - if with_val > without_val: - ax.text(with_val + 0.5, i, f'{with_val:.1f}%', va='center') - ax.text(without_val - 2.5, i, f'{without_val:.1f}%', va='center', color='white') - else: - ax.text(with_val - 2.5, i, f'{with_val:.1f}%', va='center', color='white') - ax.text(without_val + 0.5, i, f'{without_val:.1f}%', va='center') - - # Add sample size information to the y-axis labels - labels = [f"{condition}\n(n={size:,})" for condition, size in zip(plot_data['Condition'], plot_data['Sample Size'])] - ax.set_yticklabels(labels) - - # Format the x-axis as percentages - ax.xaxis.set_major_formatter(PercentFormatter()) - - # Add a title and labels - plt.title('Housing Conditions by Presence of Children\nAmerican Housing Survey (2023)', fontsize=16, pad=20) - plt.xlabel('Prevalence (%)', fontsize=12) - plt.ylabel('Housing Condition', fontsize=12) - plt.legend(title='Household Type', loc='lower right') - - # Add a note about the data - plt.figtext(0.5, 0.01, - "Note: Data from 2023 American Housing Survey. Sample sizes vary by condition.\n" - "Percentages represent the proportion of households reporting each condition.", - ha='center', fontsize=10, style='italic') - - plt.tight_layout(rect=[0, 0.03, 1, 0.97]) - plt.show() - - # Create a summary table of the results - summary_table = plot_data.copy() - summary_table['Difference (percentage points)'] = summary_table['With Children (%)'] - summary_table['Without Children (%)'] - summary_table = summary_table.sort_values(by='Difference (percentage points)', ascending=False) - - print("Summary of Housing Conditions by Presence of Children:") - print(summary_table[['Condition', 'With Children (%)', 'Without Children (%)', 'Difference (percentage points)', 'Sample Size']].to_string(index=False, float_format=lambda x: f"{x:.1f}")) - - -- query: "Analyzing the relationship between housing conditions, asthma, and the presence of children using the 2023 American Housing Survey data. This code calculates and compares the prevalence of various housing conditions (mold, water leaks) in homes with and without reported asthma cases, and also examines the prevalence of asthma in households with and without children." - code: | - import pandas as pd - import numpy as np - - # Load the AHS dataset (2023) with asthma-related variables - file_path = '{{DATASET_FILES_BASE_PATH}}/census-ahs/survey_2023.csv' - - # Define variables of interest - asthma_vars = ['HHLDASTHMA'] # Household member ever diagnosed with asthma - housing_vars = [ - 'MOLDBASEM', 'MOLDBATH', 'MOLDBEDRM', 'MOLDKITCH', 'MOLDLROOM', 'MOLDOTHER', # Mold - 'LEAKI', 'LEAKO' # Water leaks - ] - child_vars = ['NUMOLDKIDS', 'NUMYNGKIDS'] - geo_vars = ['OMB13CBSA'] - - # Combine all variables of interest - cols_of_interest = asthma_vars + housing_vars + child_vars + geo_vars - - # Load a sample of the data - sample_size = 10000 # Adjust based on memory constraints - ahs_sample = pd.read_csv(file_path, usecols=cols_of_interest, nrows=sample_size) - - # Clean the data by removing quotes from string values - for col in ahs_sample.columns: - if ahs_sample[col].dtype == 'object': - ahs_sample[col] = ahs_sample[col].str.replace("'", "") - - # Create a binary variable for presence of children - ahs_sample['has_children'] = ((ahs_sample['NUMOLDKIDS'] > 0) | (ahs_sample['NUMYNGKIDS'] > 0)).astype(int) - - # Convert asthma variable to binary (Yes/No) - ahs_sample['asthma'] = ahs_sample['HHLDASTHMA'].apply( - lambda x: 1 if x == '1' else (0 if x == '2' else np.nan) - ) - - # Convert housing condition variables to binary (Yes/No) - for col in housing_vars: - ahs_sample[col + '_binary'] = ahs_sample[col].apply( - lambda x: 1 if x == '1' else (0 if x == '2' else np.nan) - ) - - # Analyze relationship between housing conditions and asthma - print("Relationship between asthma and housing conditions:") - for col in housing_vars: - binary_col = col + '_binary' - valid_data = ahs_sample.dropna(subset=['asthma', binary_col]) - - if len(valid_data) > 0: - with_asthma = valid_data[valid_data['asthma'] == 1][binary_col].mean() * 100 - without_asthma = valid_data[valid_data['asthma'] == 0][binary_col].mean() * 100 - - if not np.isnan(with_asthma) and not np.isnan(without_asthma): - print(f"\n{col}:") - print(f" - Homes with asthma: {with_asthma:.1f}%") - print(f" - Homes without asthma: {without_asthma:.1f}%") - print(f" - Difference: {with_asthma - without_asthma:.1f} percentage points") - - # Calculate the prevalence of asthma in households with and without children - asthma_by_children = ahs_sample.dropna(subset=['asthma', 'has_children']).groupby('has_children')['asthma'].mean() * 100 - print("\nAsthma Prevalence by Presence of Children:") - print(f"Households with children: {asthma_by_children.get(1, 'N/A'):.1f}%") - print(f"Households without children: {asthma_by_children.get(0, 'N/A'):.1f}%") - print(f"Difference: {asthma_by_children.get(1, 0) - asthma_by_children.get(0, 0):.1f} percentage points") - - -- query: "Analyzing housing conditions in Houston, TX using the 2007 American Housing Survey data. This code identifies Houston records using the SMSA code, analyzes the prevalence of water leaks and pest problems, and compares these conditions between larger households (3+ persons, likely to have children) and smaller households (1-2 persons)." - code: | - import pandas as pd - import numpy as np - - # Load the 2007 AHS dataset with Houston data - file_path = '{{DATASET_FILES_BASE_PATH}}/census-ahs/survey_2007.csv' - - # Define Houston SMSA code - houston_smsa = '3360' - - # Define housing condition variables based on our exploration - housing_vars = [ - 'LEAK', 'ILEAK', # Water leaks - 'RATS', 'MICE' # Pests - ] - - # Define household size variable - household_var = 'PER' - - # Combine all columns of interest - cols_of_interest = ['SMSA'] + housing_vars + [household_var] - - # Load the Houston data - houston_data = pd.read_csv(file_path, usecols=cols_of_interest) - - # Clean the data by removing quotes if needed - for col in houston_data.columns: - if houston_data[col].dtype == 'object': - houston_data[col] = houston_data[col].str.replace("'", "") - - # Filter for Houston - houston_data = houston_data[houston_data['SMSA'] == houston_smsa] - - print(f"Loaded {len(houston_data)} Houston records from 2007 survey") - - # Create a binary variable for households likely to have children (3+ persons) - # This is a proxy since we don't have direct child indicators - houston_data['likely_has_children'] = houston_data[household_var].apply( - lambda x: 1 if x not in ['-6', '-9'] and int(x) >= 3 else 0 - ) - - print(f"\nHouseholds likely to have children (3+ persons): {houston_data['likely_has_children'].sum()} ({houston_data['likely_has_children'].mean()*100:.1f}% of Houston sample)") - - # Convert housing condition variables to binary (Yes/No) - for col in housing_vars: - houston_data[col + '_binary'] = houston_data[col].apply( - lambda x: 1 if x == '1' else (0 if x == '2' else np.nan) - ) - - # Analyze housing conditions - print("\nPrevalence of housing conditions in Houston (2007):") - for col in housing_vars: - binary_col = col + '_binary' - valid_data = houston_data.dropna(subset=[binary_col]) - if len(valid_data) > 0: - yes_pct = valid_data[binary_col].mean() * 100 - print(f"{col}: {yes_pct:.1f}% reported 'Yes' (based on {len(valid_data)} valid responses)") - - # Analyze relationship between household size and housing conditions - print("\nRelationship between household size and housing conditions in Houston (2007):") - for col in housing_vars: - binary_col = col + '_binary' - valid_data = houston_data.dropna(subset=[binary_col, 'likely_has_children']) - - if len(valid_data) > 0: - with_children = valid_data[valid_data['likely_has_children'] == 1][binary_col].mean() * 100 - without_children = valid_data[valid_data['likely_has_children'] == 0][binary_col].mean() * 100 - - if not np.isnan(with_children) and not np.isnan(without_children): - print(f"\n{col}:") - print(f" - Households with 3+ persons: {with_children:.1f}%") - print(f" - Households with 1-2 persons: {without_children:.1f}%") - print(f" - Difference: {with_children - without_children:.1f} percentage points") - - -- query: "Exploring the American Housing Survey codebook to identify variables related to asthma, respiratory health, and housing conditions. This code searches the codebook for terms related to asthma, air quality, pests, and HVAC systems, and provides a summary of available variables that could be relevant for analyzing the relationship between housing conditions and respiratory health." - code: | - import pandas as pd - import numpy as np - - # Load the codebook to find information about variables - codebook_path = '{{DATASET_FILES_BASE_PATH}}/census-ahs/census_ahs_codebook.csv' - codebook = pd.read_csv(codebook_path, dtype=str, index_col=False) - - # Define search terms related to asthma and respiratory health - asthma_terms = [ - 'asthma', 'allerg', 'respir', 'breath', 'lung', 'air', 'ventil', - 'humid', 'mold', 'mildew', 'dust', 'smoke', 'pest', 'roach', 'rodent', - 'insect', 'heat', 'cool', 'toxic', 'pollut', 'chemical', 'lead' - ] - - # Search for variables related to these terms - asthma_vars = [] - for term in asthma_terms: - term_vars = codebook[ - (codebook['Description'].str.contains(term, case=False, na=False)) | - (codebook['Question Text'].str.contains(term, case=False, na=False)) | - (codebook['Variable'].str.contains(term, case=False, na=False)) - ] - if not term_vars.empty: - for _, row in term_vars.iterrows(): - var_info = { - 'Variable': row['Variable'], - 'Description': row['Description'] if 'Description' in row and not pd.isna(row['Description']) else 'No description', - 'Question_Text': row['Question Text'] if 'Question Text' in row and not pd.isna(row['Question Text']) else 'No question text', - 'Survey_Years': row['Survey Years'] if 'Survey Years' in row and not pd.isna(row['Survey Years']) else 'Unknown', - 'Response_Codes': row['Response Codes'] if 'Response Codes' in row and not pd.isna(row['Response Codes']) else 'No codes' - } - # Check if this variable is already in our list - if not any(v['Variable'] == var_info['Variable'] for v in asthma_vars): - asthma_vars.append(var_info) - - # Create a DataFrame for easier viewing - asthma_vars_df = pd.DataFrame(asthma_vars) - - # Check if we have any variables specifically about asthma - asthma_specific = asthma_vars_df[ - asthma_vars_df['Description'].str.contains('asthma', case=False, na=False) | - asthma_vars_df['Question_Text'].str.contains('asthma', case=False, na=False) - ] - - print(f"Found {len(asthma_vars_df)} variables potentially related to asthma and respiratory health") - print(f"Found {len(asthma_specific)} variables specifically mentioning asthma") - - # Display the asthma-specific variables if any - if not asthma_specific.empty: - print("\nVariables specifically mentioning asthma:") - for _, row in asthma_specific.head(5).iterrows(): # Show first 5 for brevity - print(f"\nVariable: {row['Variable']}") - print(f"Description: {row['Description']}") - print(f"Question Text: {row['Question_Text']}") - print(f"Survey Years: {row['Survey_Years']}") - - # Check for variables related to indoor air quality - air_quality = asthma_vars_df[ - asthma_vars_df['Description'].str.contains('air quality|ventil|humid', case=False, na=False) | - asthma_vars_df['Question_Text'].str.contains('air quality|ventil|humid', case=False, na=False) - ] - - print(f"\nFound {len(air_quality)} variables related to indoor air quality") - - # Check for variables related to pests - pest_vars = asthma_vars_df[ - asthma_vars_df['Description'].str.contains('pest|roach|rodent|insect|rat|mice|mouse', case=False, na=False) | - asthma_vars_df['Question_Text'].str.contains('pest|roach|rodent|insect|rat|mice|mouse', case=False, na=False) - ] - - print(f"\nFound {len(pest_vars)} variables related to pests") - - # Check for variables related to heating and cooling systems - hvac_vars = asthma_vars_df[ - asthma_vars_df['Description'].str.contains('heat|cool|air condition|hvac', case=False, na=False) | - asthma_vars_df['Question_Text'].str.contains('heat|cool|air condition|hvac', case=False, na=False) - ] - - print(f"\nFound {len(hvac_vars)} variables related to heating and cooling systems") \ No newline at end of file diff --git a/src/biome/adhoc_data/datasources/aqs/api.yaml b/src/biome/adhoc_data/datasources/aqs/api.yaml index e8580b3..7d87df5 100644 --- a/src/biome/adhoc_data/datasources/aqs/api.yaml +++ b/src/biome/adhoc_data/datasources/aqs/api.yaml @@ -1,263 +1,478 @@ -name: "EPA Air Quality System" -integration_type: api - description: | - This API is the primary place to obtain row-level data - from the EPA's Air Quality System (AQS) database. AQS contains - ambient air sample data collected by state, local, tribal, and federal - air pollution control agencies from thousands of monitors around the nation. - It also contains meteorological data, descriptive information about - each monitoring station (including its geographic location and its operator), - and information about the quality of the samples. Note, AQS does not contain - real-time air quality data (it can take 6 months from the time - data is collected until it is in AQS). For real-time data, please use - the AirNow API. - -attachments: - raw_documentation: aqs.json - -prompt: | - # OpenAPI Spec JSON: - ${raw_documentation} - - # Additional Instructions: - - This API is through OpenAPI Spec (Swagger). You will be provided the schema. - All requests go to the following URL: https://aqs.epa.gov/data/api - That is the base URL for all requests. - - To use the AQS API always retrieve the email/key credentials from environment variables: - - email: os.environ.get("API_EPA_AQS_EMAIL") - - api_key: os.environ.get("API_EPA_AQS") - Do not use placeholder auth values from the OpenAPI spec nor prompt the user for these values, just use the environment variables. - - Ensure you know the fields that come back from the OpenAPI spec in order to process the AQS data. - - Daily/yearly data can only be accessed one year at a time. If working for that in year/decade ranges, prefer to use the annualData endpoints, as the daily data may be too large to fit into context. - If the user requests data for a year range, you will need to make multiple requests to the API- potentially sampling in between, or creating averages, in order to not make requests for every single year in the range. - The earliest sample in the data set is from 1957. Year 1980 marks the beginning of nationally consistent operational and quality assurance procedures. - - Measured entities in the AQS data set are referred to as parameters (which includes pollutants/substances), and use integer codes specific to AQS. - - If a monitor is scheduled to collect data and does not it will contain `null` data. This is a placeholder to let EPA know that more data will not be forthcoming. AQS does place absolute limits on the values that can be submitted. - However, these are fairly liberal limits- for many parameters, negative values are acceptable. - AQS has a nominal quarterly reporting deadline: data must be reported by 90 days after the end of the calendar quarter in which they are collected. - - If you need to inspect the data of a JSON response before processing, and you're dealing with annual data, or multi-daily data, or - for a geographic region that could yield too much data to process (or fit into an LLM context window), - - better use this endpoint: - ``` - /metaData/fieldsByService?email={{email}}&key={{key}}&service=annualData - ``` - (example above is checking shape of annualData endpoint). It will provide fields documentation for each property in the response. - If you use `print(response_data)` on a huge JSON response, you will not be able to proceed- you can also try checking the length as string, - and if that's too long, inspect whatever amount of characters that would be enough. - That is, never do: - ``` - data = response.json() - print(data) - ``` - instead, either use fieldsByService endpoint as described earlier, or use the Example 3 below (in the #Examples section) to generate a schema and inspect the data. - - IMPORTANT: `parameter_name` is not a property in the respponse usually, check `parameter` or `parameter_code` in responses first instead! - - # Output Format for the AQS API - JSON - The only output format available from the services is JSON. This is in conformity with the 18f API standards for government. - - The JSON response has two top-level elements. A header and a body. The header contains information about the request and the body contains the data. - - The header contains the following elements: - - - `status`. "SUCCESS" if the request returned data. "FAILED" if the request could not be processed. "No data matched your selection" if the request was processed but resulted in no data. - - `request_time`. The date and time (at the location of the server) when the request was received. - - `url`. The original URL that was submitted to make the request. - - `row`. The number of rows of data in the body object. - - `errors`. Any errors encountered when attempting to respond to the request. - The body is an array with one object per data row. The contents of the object will vary based on the request made. (E.g. the columns of data returned are different for each query. - - Here is a sample of the output of a Raw Data request with one data point: - ``` - { - "Header": [ - { - "status": "success", - "request_time": "2018-06-13T07:45:01-04:00", - "url": "https://...", - "rows": 1 - } - ], - "Body": [ - { - "state_code": "01", - "county_code": "073", - "site_number": "0023", - "parameter_code": "88101", - "poc": 1.0, - "latitude": 33.0, - "longitude": -86.0, - "datum": "WGS84", - "parameter_name": "PM2.5 - Local Conditions", - "date_local": "2017-04-01", - "time_local": "00:00", - "date_gmt": "2017-04-01", - "time_gmt": "06:00", - "sample_measurement": 6.2, - "unit_of_measure": "Micrograms/cubic meter (LC)", - "detection_limit": 2.0, - "uncertainty": null, - "qualifiers": null, - "method_type": "FRM", - "method_code": "142", - "method_name": "BGI Models PQ200-VSCC or PQ200A-VSCC - Gravimetric", - "state_name": "Alabama", - "county_name": "Jefferson", - "date_of_last_change": "2017-05-30", - "cbsa_code": "13820" - } - ] - } - ``` - - # Error Handling and Status Codes - If the API is able to parse your request it will return the JSON described above and an HTTP status of 200. - - If the API is not able to parse your request it will return a status code of 400 and only the header, which will include an array of error messages instead of a row count. - For example: - ``` - { - "Header": [{ - "status": "Failed", - "request_time": "2018-06-13T08:05:46.588-04:00", - "url": "https://...", - "error": [ - "value is missing or the value is empty: param" - ] - }], - "Body": [] - } - ``` - - If the API is able to parse your request but no data matches your selections, it will return the - JSON header and an HTTP status of 200. The row count will be zero, - the status will be a message that no data matched your selection criteria, - and the body will be empty. - ``` - { - "Header": [ - { - "status": "No data matched your selection", - "request_time": "2018-06-13T10:15:42-04:00", - "url": "https://...", - "rows": 0 - } - ], - "Body": [] - } - ``` - - # Request Limits and Terms of Service - The API has the following limits imposed on request size: - - - Length of time. All services (except Monitor) must have the end date (edate field) be in the same year as the begin date (bdate field). - - Number of parameters. Most services allow for the selection of multiple parameter codes (param field). A maximum of 5 parameter codes may be listed in a single request. - - Please adhere to the following when using the API: - - - _Limit the size of queries_. Our database contains billions of values and you may request more than you intend. If you are unsure of the amount of data, start small and work your way up. We request that you limit queries to 1,000,000 rows of data each. You can use the "observation count" field on the annualData service to determine how much data exists for a time-parameter-geography combination. If you have any questions or need advice, please contact us. - - _Limit the frequency of queries_. Our system can process a limited load. If scripting requests, please wait for one request to complete before submitting another and do not make more than 10 requests per minute. Also, we request a pause of 5 seconds between requests and adjust accordingly for response time and size. - If you violate these terms, we may disable your account without notice (but we will be in contact via the email address provided). - - # Usage tips - This section contains suggestions for completing certain data related tasks and links to software tools using the API. - - - Determine if or how much data exists for a time-parameter-geography combination: - - Retrieve data using the annualData service. - - If no records are returned, we do not have the data. - - If records are returned, use the observation count to determine the temporal and geographic distribution of the data. - - - Monthly averages: - - AQS does not routinely calculate monthly aggregate statistics. - - If you need these, you must calculate them yourself. - - These can be calculated from the sample data or the daily data without loss of fidelity. - - - Determine a single value for a site with collocated monitors: - - Many sites will have collocated monitors - monitors collecting the same parameter at the same time. - - The API currently provides only monitor level values. (We anticipate adding site-level values at some point.) - - For some criteria pollutants (PM2.5, ozone, lead, and NO2), the regulations define procedures for defining a single site-level value. - - For other pollutants, determining a single site-level value is left to the investigator. - - # More Details - ## API Structure and Capabilities - - The EPA AQS API provides access to air quality data from monitoring stations across the US - - Data is available at multiple temporal resolutions: annual, quarterly, daily, and sample-level. - - Data can be queried at various geographic levels: state, county, site, CBSA, and bounding box - - The API returns data in JSON format with a standard structure - - Key pollutants relevant to asthma (ozone, PM2.5, NO2, SO2, CO) are available through the API - - Data can be used to analyze both acute exposures (daily data) and chronic exposures (annual data) - - Seasonal patterns in air quality can be correlated with seasonal patterns in asthma exacerbations - - The API enables spatial analysis to identify potential hotspots of poor air quality - - ## Data Integration Strategies - - Geographic alignment: Use census tract or ZIP code as common geographic units - - Temporal alignment: Monthly aggregation captures seasonal patterns while managing data volume - - Spatial interpolation can estimate air quality values between monitoring stations - - Population-weighted averages can be used when aggregating to larger geographic areas - - A standardized spatial grid can facilitate integration with other environmental datasets - - ## Key Parameters for Asthma Research - The API provides access to several air pollutants known to affect respiratory health: - ``` - Parameter Code Parameter Name Relevance to Asthma - 44201 Ozone Major trigger for asthma attacks - 88101 PM2.5 Can penetrate deep into lungs - 81102 PM10 Can irritate airways - 42602 Nitrogen Dioxide (NO2) Common in vehicle emissions - 42401 Sulfur Dioxide (SO2) Industrial emissions linked to respiratory issues - 42101 Carbon Monoxide (CO) Affects oxygen transport - ``` - - ## Key Endpoints and Parameters - The API offers several endpoint categories: - - - Monitoring Sites: monitors/byCounty - Returns information about monitoring stations - - Daily Data: dailyData/byCounty - Daily measurements for parameters - - Annual Data: annualData/byCounty - Yearly aggregated measurements - - Sample Data: sampleData/byCounty - Individual sample measurements - - Metadata: Various endpoints for parameter codes, classes, etc. - - All data requests require: - - Geographic parameters (state/county codes) - - Date parameters (bdate/edate in YYYYMMDD format) - - Parameter code for the pollutant - - # Implementation Challenges and Solutions - - Date Parameter Requirements: Despite documentation suggesting otherwise, all data endpoints require explicit date parameters (bdate and edate), not just the year parameter. - - Data Volume Management: Harris County has numerous monitoring stations (we found 24,532 daily ozone records for 2022 alone), requiring efficient data aggregation strategies. - - Parameter Identification: The API requires specific parameter codes rather than names, necessitating lookup of these codes before querying data. - - The EPA AQS API provides rich, detailed air quality data that can be invaluable - for researching environmental factors affecting childhood asthma (and others). - By combining this data with health outcome data from other sources, researchers - can develop comprehensive analyses of how air quality impacts respiratory health - in specific geographic areas like a county or region. - The API's structure allows for flexible querying by location, time period, and - specific pollutants, making it a powerful tool for environmental health research. - However, users should be aware of specific parameter requirements and prepare - for handling large volumes of data when working with densely monitored areas. - - Note: If/when using the `metaData/isAvailable` endpoint to check if the API is available, - use this sample response as guide: - ``` - { - "Header": [ - { - "status": "API service is up and running healthy. Status: connection_pool: size: 5, connections: 1, in use: 1, waiting_in_queue: 0", - "request_time": "2025-03-21T08:21:41-04:00", - "url": "http://aqs.epa.gov/api/metaData/isAvailable?email={{email}}.com&key={{key}}" - } - ], - "Data": [] - } - ``` - Never simulate this API- if it is not available ask the user how to proceed. + This API is the primary place to obtain row-level data + from the EPA's Air Quality System (AQS) database. AQS contains + ambient air sample data collected by state, local, tribal, and federal + air pollution control agencies from thousands of monitors around the nation. + It also contains meteorological data, descriptive information about + each monitoring station (including its geographic location and its operator), + and information about the quality of the samples. Note, AQS does not contain + real-time air quality data (it can take 6 months from the time + data is collected until it is in AQS). For real-time data, please use + the AirNow API. +integration_type: api +name: EPA Air Quality System +prompt: "# OpenAPI Spec JSON:\n${raw_documentation}\n\n# Additional Instructions:\n\ + \nThis API is through OpenAPI Spec (Swagger). You will be provided the schema.\n\ + All requests go to the following URL: https://aqs.epa.gov/data/api\nThat is the\ + \ base URL for all requests.\n\nTo use the AQS API always retrieve the email/key\ + \ credentials from environment variables:\n- email: os.environ.get(\"API_EPA_AQS_EMAIL\"\ + )\n- api_key: os.environ.get(\"API_EPA_AQS\")\nDo not use placeholder auth values\ + \ from the OpenAPI spec nor prompt the user for these values, just use the environment\ + \ variables.\n\nEnsure you know the fields that come back from the OpenAPI spec\ + \ in order to process the AQS data.\n\nDaily/yearly data can only be accessed one\ + \ year at a time. If working for that in year/decade ranges, prefer to use the annualData\ + \ endpoints, as the daily data may be too large to fit into context.\nIf the user\ + \ requests data for a year range, you will need to make multiple requests to the\ + \ API- potentially sampling in between, or creating averages, in order to not make\ + \ requests for every single year in the range.\nThe earliest sample in the data\ + \ set is from 1957. Year 1980 marks the beginning of nationally consistent operational\ + \ and quality assurance procedures.\n\nMeasured entities in the AQS data set are\ + \ referred to as parameters (which includes pollutants/substances), and use integer\ + \ codes specific to AQS.\n\nIf a monitor is scheduled to collect data and does not\ + \ it will contain `null` data. This is a placeholder to let EPA know that more data\ + \ will not be forthcoming. AQS does place absolute limits on the values that\ + \ can be submitted.\nHowever, these are fairly liberal limits- for many parameters,\ + \ negative values are acceptable.\nAQS has a nominal quarterly reporting deadline:\ + \ data must be reported by 90 days after the end of the calendar quarter in which\ + \ they are collected.\n\nIf you need to inspect the data of a JSON response before\ + \ processing, and you're dealing with annual data, or multi-daily data, or\nfor\ + \ a geographic region that could yield too much data to process (or fit into an\ + \ LLM context window),\n\nbetter use this endpoint:\n```\n/metaData/fieldsByService?email={{email}}&key={{key}}&service=annualData\n\ + ```\n(example above is checking shape of annualData endpoint). It will provide fields\ + \ documentation for each property in the response.\nIf you use `print(response_data)`\ + \ on a huge JSON response, you will not be able to proceed- you can also try checking\ + \ the length as string,\nand if that's too long, inspect whatever amount of characters\ + \ that would be enough.\nThat is, never do:\n```\ndata = response.json()\nprint(data)\n\ + ```\ninstead, either use fieldsByService endpoint as described earlier, or use the\ + \ Example 3 below (in the #Examples section) to generate a schema and inspect the\ + \ data.\n\nIMPORTANT: `parameter_name` is not a property in the respponse usually,\ + \ check `parameter` or `parameter_code` in responses first instead!\n\n# Output\ + \ Format for the AQS API - JSON\nThe only output format available from the services\ + \ is JSON. This is in conformity with the 18f API standards for government.\n\n\ + The JSON response has two top-level elements. A header and a body. The header contains\ + \ information about the request and the body contains the data.\n\nThe header contains\ + \ the following elements:\n\n- `status`. \"SUCCESS\" if the request returned data.\ + \ \"FAILED\" if the request could not be processed. \"No data matched your selection\"\ + \ if the request was processed but resulted in no data.\n- `request_time`. The date\ + \ and time (at the location of the server) when the request was received.\n- `url`.\ + \ The original URL that was submitted to make the request.\n- `row`. The number\ + \ of rows of data in the body object.\n- `errors`. Any errors encountered when attempting\ + \ to respond to the request.\nThe body is an array with one object per data row.\ + \ The contents of the object will vary based on the request made. (E.g. the columns\ + \ of data returned are different for each query.\n\nHere is a sample of the output\ + \ of a Raw Data request with one data point:\n```\n {\n \"Header\": [\n\ + \ {\n \"status\": \"success\",\n \"request_time\":\ + \ \"2018-06-13T07:45:01-04:00\",\n \"url\": \"https://...\",\n \ + \ \"rows\": 1\n }\n ],\n \"Body\": [\n {\n \ + \ \"state_code\": \"01\",\n \"county_code\": \"073\",\n \ + \ \"site_number\": \"0023\",\n \"parameter_code\": \"88101\",\n\ + \ \"poc\": 1.0,\n \"latitude\": 33.0,\n \"longitude\"\ + : -86.0,\n \"datum\": \"WGS84\",\n \"parameter_name\": \"\ + PM2.5 - Local Conditions\",\n \"date_local\": \"2017-04-01\",\n \ + \ \"time_local\": \"00:00\",\n \"date_gmt\": \"2017-04-01\",\n\ + \ \"time_gmt\": \"06:00\",\n \"sample_measurement\": 6.2,\n\ + \ \"unit_of_measure\": \"Micrograms/cubic meter (LC)\",\n \ + \ \"detection_limit\": 2.0,\n \"uncertainty\": null,\n \"\ + qualifiers\": null,\n \"method_type\": \"FRM\",\n \"method_code\"\ + : \"142\",\n \"method_name\": \"BGI Models PQ200-VSCC or PQ200A-VSCC\ + \ - Gravimetric\",\n \"state_name\": \"Alabama\",\n \"county_name\"\ + : \"Jefferson\",\n \"date_of_last_change\": \"2017-05-30\",\n \ + \ \"cbsa_code\": \"13820\"\n }\n ]\n }\n```\n\n# Error Handling\ + \ and Status Codes\nIf the API is able to parse your request it will return the\ + \ JSON described above and an HTTP status of 200.\n\nIf the API is not able to parse\ + \ your request it will return a status code of 400 and only the header, which will\ + \ include an array of error messages instead of a row count.\nFor example:\n```\n\ + \ {\n \"Header\": [{\n \"status\": \"Failed\",\n \ + \ \"request_time\": \"2018-06-13T08:05:46.588-04:00\",\n \"url\": \"\ + https://...\",\n \"error\": [\n \"value is missing or\ + \ the value is empty: param\"\n ]\n }],\n \"Body\"\ + : []\n }\n```\n\nIf the API is able to parse your request but no data matches\ + \ your selections, it will return the\nJSON header and an HTTP status of 200. The\ + \ row count will be zero,\nthe status will be a message that no data matched your\ + \ selection criteria,\nand the body will be empty.\n```\n{\n \"Header\": [\n\ + \ {\n \"status\": \"No data matched your selection\",\n \"request_time\"\ + : \"2018-06-13T10:15:42-04:00\",\n \"url\": \"https://...\",\n \"\ + rows\": 0\n }\n ],\n \"Body\": []\n}\n```\n\n# Request Limits and Terms\ + \ of Service\nThe API has the following limits imposed on request size:\n\n- Length\ + \ of time. All services (except Monitor) must have the end date (edate field) be\ + \ in the same year as the begin date (bdate field).\n- Number of parameters. Most\ + \ services allow for the selection of multiple parameter codes (param field). A\ + \ maximum of 5 parameter codes may be listed in a single request.\n\nPlease adhere\ + \ to the following when using the API:\n\n- _Limit the size of queries_. Our database\ + \ contains billions of values and you may request more than you intend. If you are\ + \ unsure of the amount of data, start small and work your way up. We request that\ + \ you limit queries to 1,000,000 rows of data each. You can use the \"observation\ + \ count\" field on the annualData service to determine how much data exists for\ + \ a time-parameter-geography combination. If you have any questions or need advice,\ + \ please contact us.\n- _Limit the frequency of queries_. Our system can process\ + \ a limited load. If scripting requests, please wait for one request to complete\ + \ before submitting another and do not make more than 10 requests per minute. Also,\ + \ we request a pause of 5 seconds between requests and adjust accordingly for response\ + \ time and size.\nIf you violate these terms, we may disable your account without\ + \ notice (but we will be in contact via the email address provided).\n\n# Usage\ + \ tips\nThis section contains suggestions for completing certain data related tasks\ + \ and links to software tools using the API.\n\n- Determine if or how much data\ + \ exists for a time-parameter-geography combination:\n - Retrieve data using\ + \ the annualData service.\n - If no records are returned, we do not have the\ + \ data.\n - If records are returned, use the observation count to determine the\ + \ temporal and geographic distribution of the data.\n\n- Monthly averages:\n \ + \ - AQS does not routinely calculate monthly aggregate statistics.\n - If you\ + \ need these, you must calculate them yourself.\n - These can be calculated from\ + \ the sample data or the daily data without loss of fidelity.\n\n- Determine a single\ + \ value for a site with collocated monitors:\n - Many sites will have collocated\ + \ monitors - monitors collecting the same parameter at the same time.\n - The\ + \ API currently provides only monitor level values. (We anticipate adding site-level\ + \ values at some point.)\n - For some criteria pollutants (PM2.5, ozone, lead,\ + \ and NO2), the regulations define procedures for defining a single site-level value.\n\ + \ - For other pollutants, determining a single site-level value is left to the\ + \ investigator.\n\n# More Details\n## API Structure and Capabilities\n- The EPA\ + \ AQS API provides access to air quality data from monitoring stations across the\ + \ US\n- Data is available at multiple temporal resolutions: annual, quarterly, daily,\ + \ and sample-level.\n- Data can be queried at various geographic levels: state,\ + \ county, site, CBSA, and bounding box\n- The API returns data in JSON format with\ + \ a standard structure\n- Key pollutants relevant to asthma (ozone, PM2.5, NO2,\ + \ SO2, CO) are available through the API\n- Data can be used to analyze both acute\ + \ exposures (daily data) and chronic exposures (annual data)\n- Seasonal patterns\ + \ in air quality can be correlated with seasonal patterns in asthma exacerbations\n\ + - The API enables spatial analysis to identify potential hotspots of poor air quality\n\ + \n## Data Integration Strategies\n- Geographic alignment: Use census tract or ZIP\ + \ code as common geographic units\n- Temporal alignment: Monthly aggregation captures\ + \ seasonal patterns while managing data volume\n- Spatial interpolation can estimate\ + \ air quality values between monitoring stations\n- Population-weighted averages\ + \ can be used when aggregating to larger geographic areas\n- A standardized spatial\ + \ grid can facilitate integration with other environmental datasets\n\n## Key Parameters\ + \ for Asthma Research\nThe API provides access to several air pollutants known to\ + \ affect respiratory health:\n```\nParameter Code\tParameter Name\tRelevance to\ + \ Asthma\n44201\tOzone\tMajor trigger for asthma attacks\n88101\tPM2.5\tCan penetrate\ + \ deep into lungs\n81102\tPM10\tCan irritate airways\n42602\tNitrogen Dioxide (NO2)\t\ + Common in vehicle emissions\n42401\tSulfur Dioxide (SO2)\tIndustrial emissions linked\ + \ to respiratory issues\n42101\tCarbon Monoxide\t(CO) Affects oxygen transport\n\ + ```\n\n## Key Endpoints and Parameters\nThe API offers several endpoint categories:\n\ + \n- Monitoring Sites: monitors/byCounty - Returns information about monitoring stations\n\ + - Daily Data: dailyData/byCounty - Daily measurements for parameters\n- Annual Data:\ + \ annualData/byCounty - Yearly aggregated measurements\n- Sample Data: sampleData/byCounty\ + \ - Individual sample measurements\n- Metadata: Various endpoints for parameter\ + \ codes, classes, etc.\n\nAll data requests require:\n- Geographic parameters (state/county\ + \ codes)\n- Date parameters (bdate/edate in YYYYMMDD format)\n- Parameter code for\ + \ the pollutant\n\n# Implementation Challenges and Solutions\n- Date Parameter Requirements:\ + \ Despite documentation suggesting otherwise, all data endpoints require explicit\ + \ date parameters (bdate and edate), not just the year parameter.\n- Data Volume\ + \ Management: Harris County has numerous monitoring stations (we found 24,532 daily\ + \ ozone records for 2022 alone), requiring efficient data aggregation strategies.\n\ + - Parameter Identification: The API requires specific parameter codes rather than\ + \ names, necessitating lookup of these codes before querying data.\n\nThe EPA AQS\ + \ API provides rich, detailed air quality data that can be invaluable\nfor researching\ + \ environmental factors affecting childhood asthma (and others).\nBy combining this\ + \ data with health outcome data from other sources, researchers\ncan develop comprehensive\ + \ analyses of how air quality impacts respiratory health\nin specific geographic\ + \ areas like a county or region.\nThe API's structure allows for flexible querying\ + \ by location, time period, and\nspecific pollutants, making it a powerful tool\ + \ for environmental health research.\nHowever, users should be aware of specific\ + \ parameter requirements and prepare\nfor handling large volumes of data when working\ + \ with densely monitored areas.\n\nNote: If/when using the `metaData/isAvailable`\ + \ endpoint to check if the API is available,\nuse this sample response as guide:\n\ + ```\n{\n \"Header\": [\n {\n \"status\": \"API service is up\ + \ and running healthy. Status: connection_pool: size: 5, connections: 1, in use:\ + \ 1, waiting_in_queue: 0\",\n \"request_time\": \"2025-03-21T08:21:41-04:00\"\ + ,\n \"url\": \"http://aqs.epa.gov/api/metaData/isAvailable?email={{email}}.com&key={{key}}\"\ + \n }\n ],\n \"Data\": []\n}\n```\nNever simulate this API- if it is\ + \ not available ask the user how to proceed.\n" +resources: + 00b2ebdf-94ff-475f-be91-a1f69e1b4403: + code: "import os\nimport requests\nimport pandas as pd\nfrom datetime import datetime\n\ + import calendar\n\n# API configuration\nbase_url = \"https://aqs.epa.gov/data/api\"\ + \nemail = os.environ.get(\"API_EPA_AQS_EMAIL\")\nkey = os.environ.get(\"API_EPA_AQS\"\ + )\n\n# Define parameters\nparameters = {\n 'Ozone': 44201,\n 'CO': 42101,\ + \ \n 'NO2': 42602,\n 'PM2.5': 88101\n}\n\n# Harris County, TX codes\n\ + state_code = '48'\ncounty_code = '201'\nyear = 2022\n\ndef get_monthly_data(param_code,\ + \ month):\n # Get number of days in month\n num_days = calendar.monthrange(year,\ + \ month)[1]\n \n params = {\n 'email': email,\n 'key': key,\n\ + \ 'param': param_code,\n 'bdate': f'{year}{month:02d}01',\n \ + \ 'edate': f'{year}{month:02d}{num_days}',\n 'state': state_code,\n\ + \ 'county': county_code\n }\n \n response = requests.get(f\"\ + {base_url}/dailyData/byCounty\", params=params)\n return response.json()\n\ + \n# Initialize empty list to store all data\nall_data = []\n\n# Loop through\ + \ each parameter and month\nfor param_name, param_code in parameters.items():\n\ + \ for month in range(1, 13):\n try:\n data = get_monthly_data(param_code,\ + \ month)\n \n if 'Data' in data:\n for\ + \ record in data['Data']:\n all_data.append({\n \ + \ 'date': record['date_local'],\n 'parameter':\ + \ param_name,\n 'value': record['arithmetic_mean'],\n\ + \ 'units': record['units_of_measure'],\n \ + \ 'site': record['site_number'],\n 'latitude':\ + \ record['latitude'],\n 'longitude': record['longitude']\n\ + \ })\n except Exception as e:\n print(f\"\ + Error fetching data for {param_name} in month {month}: {str(e)}\")\n\n# Convert\ + \ to DataFrame\ndf = pd.DataFrame(all_data)\n\n# Convert date string to datetime\n\ + df['date'] = pd.to_datetime(df['date'])\n\n# Sort by date and parameter\ndf\ + \ = df.sort_values(['date', 'parameter'])\n\n# Save to CSV\ndf.to_csv('harris_county_air_quality_2022.csv',\ + \ index=False)\n" + integration: epa_air_quality_system + notes: null + query: Retrieve daily air quality data for ozone, CO, NO2, and PM2.5 in Harris + County, TX for each month of the year 2022. + resource_id: 00b2ebdf-94ff-475f-be91-a1f69e1b4403 + resource_type: example + 07d42e21-2e7d-467a-878b-e0f7572f3b8d: + code: | + import requests + import os + + # Get API credentials from environment variables + email = os.environ.get("API_EPA_AQS_EMAIL") + key = os.environ.get("API_EPA_AQS") + + # Define the API endpoint and parameters + state_code = 12 # "Florida" + county_code = 117 # "Seminole" county + year = 2000 + # codes can be fetched from /list/parametersByClass?email={email}&key={key}&pc=ALL + # These are asthma related pollutant/parameter codes: + pollutant_codes = [ + '44201', # ozone + '81102', # PM10 + '81104', # PM2.5 + '42602', # Nitrogen dioxide (NO2) + '42401', # Sulfur dioxide + '42101', # Carbon monoxide + ] + + bdate = f"{year}0501" + edate = f"{year}0501" + + # Construct the API URL + param_string = ','.join(pollutant_codes) + + url = f'https://aqs.epa.gov/data/api/annualData/byCounty?email={email}&key={key}¶m={param_string}&bdate={bdate}&edate={edate}&state={state_code}&county={county_code}' + + # Make the API request + response = requests.get(url) + + data = None + # Check if the request was successful + if response.status_code == 200: + data = response.json() + + data + integration: epa_air_quality_system + notes: null + query: Query annual air pollution data for Seminole County, Florida for the year + 2000. Use pollutants relating to asthma. + resource_id: 07d42e21-2e7d-467a-878b-e0f7572f3b8d + resource_type: example + 14b7c79f-eec0-4fa0-b6bc-42a7b81c85c6: + code: "import os\nimport requests\nimport pandas as pd\nimport matplotlib.pyplot\ + \ as plt\nimport seaborn as sns\n\n# Get authentication details from environment\ + \ variables\nemail = os.environ.get(\"API_EPA_AQS_EMAIL\")\napi_key = os.environ.get(\"\ + API_EPA_AQS\")\n\n# Base URL for the EPA AQS API\nbase_url = \"https://aqs.epa.gov/data/api\"\ + \n\n# Function to make API requests\ndef get_aqs_data(endpoint, params=None):\n\ + \ \"\"\"\n Make a request to the EPA AQS API\n \n Parameters:\n\ + \ -----------\n endpoint : str\n The API endpoint to query\n \ + \ params : dict\n Additional parameters for the request\n \n\ + \ Returns:\n --------\n dict\n The JSON response from the API\n\ + \ \"\"\"\n # Add authentication parameters\n if params is None:\n \ + \ params = {}\n params['email'] = email\n params['key'] = api_key\n\ + \ \n # Make the request\n url = f\"{base_url}/{endpoint}\"\n response\ + \ = requests.get(url, params=params)\n \n # Check if the request was successful\n\ + \ if response.status_code == 200:\n return response.json()\n else:\n\ + \ print(f\"Error: {response.status_code}\")\n print(response.text)\n\ + \ return None\n\n# Get list of parameter classes\nclasses_response =\ + \ get_aqs_data(\"list/classes\")\n\n# Check if we got a valid response for classes\n\ + if classes_response and classes_response.get('Header', [{}])[0].get('status')\ + \ == 'Success':\n # Convert to DataFrame for easier viewing\n classes_data\ + \ = classes_response.get('Data', [])\n classes_df = pd.DataFrame(classes_data)\n\ + \ \n print(\"Parameter Classes:\")\n print(classes_df)\n \n #\ + \ Now get parameters for the CRITERIA class (most relevant to air quality standards)\n\ + \ criteria_params_response = get_aqs_data(\"list/parametersByClass\", {\"\ + pc\": \"CRITERIA\"})\n \n if criteria_params_response and criteria_params_response.get('Header',\ + \ [{}])[0].get('status') == 'Success':\n # Convert to DataFrame for easier\ + \ viewing\n criteria_params_data = criteria_params_response.get('Data',\ + \ [])\n criteria_params_df = pd.DataFrame(criteria_params_data)\n \ + \ \n print(f\"\\nFound {len(criteria_params_data)} criteria pollutant\ + \ parameters\")\n print(\"\\nCriteria pollutant parameters:\")\n \ + \ print(criteria_params_df)\n \n # Create a dictionary of key\ + \ parameter codes for asthma research\n asthma_param_codes = {\n \ + \ 'Ozone': '44201',\n 'PM2.5': '88101',\n 'PM10':\ + \ '81102',\n 'NO2': '42602',\n 'SO2': '42401',\n \ + \ 'CO': '42101'\n }\n \n print(\"\\nKey parameter\ + \ codes for asthma research:\")\n for pollutant, code in asthma_param_codes.items():\n\ + \ print(f\"{pollutant}: {code}\")\n else:\n print(\"Failed\ + \ to retrieve criteria parameters data\")\nelse:\n print(\"Failed to retrieve\ + \ parameter classes data\")\n" + integration: epa_air_quality_system + notes: null + query: |- + Retrieving and exploring parameter classes and codes for air quality research. This example demonstrates how to get a list of parameter classes, then retrieve specific parameters within the CRITERIA class, which contains the most important air pollutants for health research. It also identifies key parameter codes relevant to asthma research. + resource_id: 14b7c79f-eec0-4fa0-b6bc-42a7b81c85c6 + resource_type: example + 19afbed5-92dc-48af-9354-55a544e417ce: + code: "import requests\nimport os\n\n# Get API credentials from environment variables\n\ + email = os.environ.get(\"API_EPA_AQS_EMAIL\")\nkey = os.environ.get(\"API_EPA_AQS\"\ + )\n\n# Define parameters\nstate_code = \"12\" # Florida\ncounty_code = \"095\"\ + \ # Orange County\nparam_codes = [\n \"44201\", # Ozone\n \"42401\",\ + \ # SO2\n \"42101\", # CO \n \"42602\", # NO2\n \"88101\" # PM2.5\n\ + ]\n\n# Use annual data API endpoint since we want historical data\n# Sample\ + \ every 5 years from 1980-2020 to avoid too many API calls\nyears = range(1980,\ + \ 2021, 5)\n\n# Initialize list to store results\nall_data = []\n\n# Make API\ + \ requests for each year\nfor year in years:\n bdate = f\"{year}0101\"\n\ + \ edate = f\"{year}1231\"\n \n url = f\"https://aqs.epa.gov/data/api/annualData/byCounty\"\ + \n params = {\n \"email\": email,\n \"key\": key,\n \ + \ \"param\": \",\".join(param_codes),\n \"bdate\": bdate,\n \"\ + edate\": edate,\n \"state\": state_code,\n \"county\": county_code\n\ + \ }\n \n response = requests.get(url, params=params)\n \n if\ + \ response.status_code == 200:\n data = response.json()\n if \"\ + Data\" in data:\n all_data.extend(data[\"Data\"])\n else:\n\ + \ print(f\"No data found for year {year}\")\n else:\n print(f\"\ + Error {response.status_code}: {response.text}\")\n\n# Check the structure of\ + \ the collected data\nif all_data:\n print(all_data[0]) # Print the first\ + \ entry to see the structure\n df = pd.DataFrame(all_data)\n \n # Clean\ + \ up the DataFrame\n if not df.empty:\n # Select relevant columns\n\ + \ # parameter_name doesn't exist- use `parameter` or `parameter_code`:\n\ + \ cols = ['parameter', 'year', 'arithmetic_mean', \n \ + \ 'first_max_value', 'first_max_datetime',\n 'observation_count']\n\ + \ df = df[cols]\n \n # Sort by year and parameter\n \ + \ df = df.sort_values(['year', 'parameter'])\n \n # Convert\ + \ year to int\n df['year'] = df['year'].astype(int)\n \n \ + \ # Display first few rows\n print(df.head())\n else:\n print(\"\ + No data found for the specified parameters\")\nelse:\n print(\"No data collected.\"\ + )\n" + integration: epa_air_quality_system + notes: null + query: Collect historical air quality data from the EPA Air Quality System (AQS) + for Orange County, FL. + resource_id: 19afbed5-92dc-48af-9354-55a544e417ce + resource_type: example + 365697ce-eb50-444b-8a82-637a2a0131f4: + filepath: aqs.json + integration: epa_air_quality_system + name: raw_documentation + resource_id: 365697ce-eb50-444b-8a82-637a2a0131f4 + resource_type: file + 39430090-3680-4065-9bd1-7755ba712889: + code: "def get_pollutant_data(state_code, county_code, param_code, year, pollutant_name):\n\ + \ \"\"\"\n Get and process data for a specific pollutant\n \n Parameters:\n\ + \ -----------\n state_code : str\n Two-digit state code\n county_code\ + \ : str\n Three-digit county code\n param_code : str\n Parameter\ + \ code for the pollutant\n year : str\n Year to retrieve data for\n\ + \ pollutant_name : str\n Name of the pollutant (for display purposes)\n\ + \ \n Returns:\n --------\n pandas.DataFrame or None\n \ + \ DataFrame containing the processed data, or None if the request failed\n\ + \ \"\"\"\n print(f\"Retrieving daily {pollutant_name} data for Harris\ + \ County, TX for {year}...\")\n \n bdate = f\"{year}0101\" # January\ + \ 1st of the year\n edate = f\"{year}1231\" # December 31st of the year\n\ + \ \n data_response = get_aqs_data(\"dailyData/byCounty\", {\n \"\ + state\": state_code,\n \"county\": county_code,\n \"param\": param_code,\n\ + \ \"bdate\": bdate,\n \"edate\": edate\n })\n \n if data_response\ + \ and data_response.get('Header', [{}])[0].get('status') == 'Success':\n \ + \ data = data_response.get('Data', [])\n df = pd.DataFrame(data)\n\ + \ \n # Convert date_local to datetime\n df['date'] = pd.to_datetime(df['date_local'])\n\ + \ \n # Group by site and date to get daily averages for each site\n\ + \ site_daily_avg = df.groupby(['local_site_name', 'date'])['arithmetic_mean'].mean().reset_index()\n\ + \ \n # Get the overall daily average across all sites\n \ + \ daily_avg = site_daily_avg.groupby('date')['arithmetic_mean'].mean().reset_index()\n\ + \ \n # Rename the arithmetic_mean column to the pollutant name\ + \ for clarity\n daily_avg = daily_avg.rename(columns={'arithmetic_mean':\ + \ pollutant_name.lower()})\n \n return daily_avg, df['units_of_measure'].iloc[0]\n\ + \ else:\n print(f\"Failed to retrieve {pollutant_name} data\")\n \ + \ return None, None\n" + integration: epa_air_quality_system + notes: null + query: 'Here''s a reusable function for retrieving and processing data for multiple + pollutants:' + resource_id: 39430090-3680-4065-9bd1-7755ba712889 + resource_type: example + 4245dd8c-005b-4bfa-b50e-ba076b53ffbe: + code: "import requests\nfrom genson import SchemaBuilder\nimport os\n\n# Get API\ + \ credentials from environment variables\nemail = os.environ.get(\"API_EPA_AQS_EMAIL\"\ + )\nkey = os.environ.get(\"API_EPA_AQS\")\n\n# Define parameters\nstate_code\ + \ = \"48\" # Texas\ncounty_code = \"201\" # Harris County\nparam_codes = [\n\ + \ \"44201\", # Ozone\n \"42401\", # SO2\n \"42101\", # CO \n \ + \ \"42602\", # NO2\n \"88101\", # PM2.5\n \"81102\" # PM10\n]\n\n# Sample\ + \ one year to inspect the response structure\nyear = 2020\nbdate = f\"{year}0101\"\ + \nedate = f\"{year}1231\"\n\nurl = f\"https://aqs.epa.gov/data/api/annualData/byCounty\"\ + \nparams = {\n \"email\": email,\n \"key\": key,\n \"param\": \",\"\ + .join(param_codes),\n \"bdate\": bdate,\n \"edate\": edate,\n \"state\"\ + : state_code,\n \"county\": county_code\n}\n\nresponse = requests.get(url,\ + \ params=params)\nresponse_data = response.json()\n\n# annual data by county\ + \ can include a lot of data, especially if we specify a broad date range and\ + \ a county with a lot of data- lets generate a schema and inspect that instead\ + \ of inspecting the raw data\nbuilder = SchemaBuilder()\nbuilder.add_object(response_data)\n\ + schema = builder.to_schema()\n\nprint(schema[\"properties\"])\n" + integration: epa_air_quality_system + notes: null + query: |- + Inspect API response change by generating a schema from an API response so that we don't inspect the raw data and blow up the context window. + resource_id: 4245dd8c-005b-4bfa-b50e-ba076b53ffbe + resource_type: example + e643cf2e-66f1-428e-8f2c-2626c4c59192: + code: "import os\nimport requests\nimport pandas as pd\nimport matplotlib.pyplot\ + \ as plt\nimport seaborn as sns\nfrom datetime import datetime\n\n# Set the\ + \ style for our plots\nsns.set(style=\"whitegrid\")\nplt.rcParams['figure.figsize']\ + \ = (12, 6)\n\n# Get authentication details from environment variables\nemail\ + \ = os.environ.get(\"API_EPA_AQS_EMAIL\")\napi_key = os.environ.get(\"API_EPA_AQS\"\ + )\n\n# Base URL for the EPA AQS API\nbase_url = \"https://aqs.epa.gov/data/api\"\ + \n\n# Function to make API requests\ndef get_aqs_data(endpoint, params=None):\n\ + \ \"\"\"\n Make a request to the EPA AQS API\n \n Parameters:\n\ + \ -----------\n endpoint : str\n The API endpoint to query\n \ + \ params : dict\n Additional parameters for the request\n \n\ + \ Returns:\n --------\n dict\n The JSON response from the API\n\ + \ \"\"\"\n # Add authentication parameters\n if params is None:\n \ + \ params = {}\n params['email'] = email\n params['key'] = api_key\n\ + \ \n # Make the request\n url = f\"{base_url}/{endpoint}\"\n response\ + \ = requests.get(url, params=params)\n \n # Check if the request was successful\n\ + \ if response.status_code == 200:\n return response.json()\n else:\n\ + \ print(f\"Error: {response.status_code}\")\n print(response.text)\n\ + \ return None\n\n# Function to get and process data for a specific pollutant\n\ + def get_pollutant_data(state_code, county_code, param_code, year, pollutant_name):\n\ + \ \"\"\"\n Get and process data for a specific pollutant\n \n Parameters:\n\ + \ -----------\n state_code : str\n Two-digit state code\n county_code\ + \ : str\n Three-digit county code\n param_code : str\n Parameter\ + \ code for the pollutant\n year : str\n Year to retrieve data for\n\ + \ pollutant_name : str\n Name of the pollutant (for display purposes)\n\ + \ \n Returns:\n --------\n pandas.DataFrame or None\n \ + \ DataFrame containing the processed data, or None if the request failed\n\ + \ \"\"\"\n print(f\"Retrieving daily {pollutant_name} data for {year}...\"\ + )\n \n bdate = f\"{year}0101\" # January 1st of the year\n edate =\ + \ f\"{year}1231\" # December 31st of the year\n \n data_response = get_aqs_data(\"\ + dailyData/byCounty\", {\n \"state\": state_code,\n \"county\"\ + : county_code,\n \"param\": param_code,\n \"bdate\": bdate,\n\ + \ \"edate\": edate\n })\n \n if data_response and data_response.get('Header',\ + \ [{}])[0].get('status') == 'Success':\n data = data_response.get('Data',\ + \ [])\n df = pd.DataFrame(data)\n \n print(f\"\\nFound\ + \ {len(data)} daily {pollutant_name} records for {year}\")\n \n \ + \ # Convert date_local to datetime\n df['date'] = pd.to_datetime(df['date_local'])\n\ + \ \n # Extract month and day of week for analysis\n df['month']\ + \ = df['date'].dt.month\n df['month_name'] = df['date'].dt.month_name()\n\ + \ \n # Group by site and date to get daily averages for each site\n\ + \ site_daily_avg = df.groupby(['local_site_name', 'date'])['arithmetic_mean'].mean().reset_index()\n\ + \ \n # Get the overall daily average across all sites\n \ + \ daily_avg = site_daily_avg.groupby('date')['arithmetic_mean'].mean().reset_index()\n\ + \ \n # Rename the arithmetic_mean column to the pollutant name\ + \ for clarity\n daily_avg = daily_avg.rename(columns={'arithmetic_mean':\ + \ pollutant_name.lower()})\n \n return daily_avg, df['units_of_measure'].iloc[0]\n\ + \ else:\n print(f\"Failed to retrieve {pollutant_name} data\")\n \ + \ return None, None\n\n# Set up parameters\nstate_code = \"48\" # Texas\n\ + county_code = \"201\" # Harris County\nyear = \"2022\"\n\n# Define pollutants\ + \ to retrieve\npollutants = {\n 'Ozone': '44201',\n 'PM2.5': '88101',\n\ + \ 'NO2': '42602',\n 'SO2': '42401'\n}\n\n# Get data for each pollutant\n\ + pollutant_data = {}\nunits = {}\n\nfor pollutant_name, param_code in pollutants.items():\n\ + \ daily_avg, unit = get_pollutant_data(state_code, county_code, param_code,\ + \ year, pollutant_name)\n if daily_avg is not None:\n pollutant_data[pollutant_name.lower()]\ + \ = daily_avg\n units[pollutant_name.lower()] = unit\n\n# Merge all pollutant\ + \ data on date\nmerged_df = None\n\nfor pollutant_name, daily_avg in pollutant_data.items():\n\ + \ if merged_df is None:\n merged_df = daily_avg\n else:\n \ + \ merged_df = pd.merge(merged_df, daily_avg[['date', pollutant_name]], on='date',\ + \ how='outer')\n\n# Calculate correlations between pollutants\nif merged_df\ + \ is not None:\n print(\"\\nCorrelations between pollutants:\")\n correlation_matrix\ + \ = merged_df.drop('date', axis=1).corr()\n print(correlation_matrix)\n \ + \ \n # Plot correlation heatmap\n plt.figure(figsize=(10, 8))\n sns.heatmap(correlation_matrix,\ + \ annot=True, cmap='coolwarm', vmin=-1, vmax=1, center=0)\n plt.title(f'Correlation\ + \ Matrix of Air Pollutants ({year})', fontsize=16)\n plt.tight_layout()\n\ + \ plt.show()\n" + integration: epa_air_quality_system + notes: null + query: |- + Multi-pollutant correlation analysis for air quality research. This example demonstrates how to retrieve data for multiple pollutants (Ozone, PM2.5, NO2, and SO2), process the data to get daily averages, merge the datasets, and analyze correlations between different pollutants. It includes a visualization of the correlation matrix using a heatmap. + resource_id: e643cf2e-66f1-428e-8f2c-2626c4c59192 + resource_type: example +slug: epa_air_quality_system diff --git a/src/biome/adhoc_data/datasources/aqs/examples.yaml b/src/biome/adhoc_data/datasources/aqs/examples.yaml deleted file mode 100644 index 5a63b3d..0000000 --- a/src/biome/adhoc_data/datasources/aqs/examples.yaml +++ /dev/null @@ -1,563 +0,0 @@ -- query: "Query annual air pollution data for Seminole County, Florida for the year 2000. Use pollutants relating to asthma." - code: | - import requests - import os - - # Get API credentials from environment variables - email = os.environ.get("API_EPA_AQS_EMAIL") - key = os.environ.get("API_EPA_AQS") - - # Define the API endpoint and parameters - state_code = 12 # "Florida" - county_code = 117 # "Seminole" county - year = 2000 - # codes can be fetched from /list/parametersByClass?email={email}&key={key}&pc=ALL - # These are asthma related pollutant/parameter codes: - pollutant_codes = [ - '44201', # ozone - '81102', # PM10 - '81104', # PM2.5 - '42602', # Nitrogen dioxide (NO2) - '42401', # Sulfur dioxide - '42101', # Carbon monoxide - ] - - bdate = f"{year}0501" - edate = f"{year}0501" - - # Construct the API URL - param_string = ','.join(pollutant_codes) - - url = f'https://aqs.epa.gov/data/api/annualData/byCounty?email={email}&key={key}¶m={param_string}&bdate={bdate}&edate={edate}&state={state_code}&county={county_code}' - - # Make the API request - response = requests.get(url) - - data = None - # Check if the request was successful - if response.status_code == 200: - data = response.json() - - data - -- query: "Collect historical air quality data from the EPA Air Quality System (AQS) for Orange County, FL." - code: | - import requests - import os - - # Get API credentials from environment variables - email = os.environ.get("API_EPA_AQS_EMAIL") - key = os.environ.get("API_EPA_AQS") - - # Define parameters - state_code = "12" # Florida - county_code = "095" # Orange County - param_codes = [ - "44201", # Ozone - "42401", # SO2 - "42101", # CO - "42602", # NO2 - "88101" # PM2.5 - ] - - # Use annual data API endpoint since we want historical data - # Sample every 5 years from 1980-2020 to avoid too many API calls - years = range(1980, 2021, 5) - - # Initialize list to store results - all_data = [] - - # Make API requests for each year - for year in years: - bdate = f"{year}0101" - edate = f"{year}1231" - - url = f"https://aqs.epa.gov/data/api/annualData/byCounty" - params = { - "email": email, - "key": key, - "param": ",".join(param_codes), - "bdate": bdate, - "edate": edate, - "state": state_code, - "county": county_code - } - - response = requests.get(url, params=params) - - if response.status_code == 200: - data = response.json() - if "Data" in data: - all_data.extend(data["Data"]) - else: - print(f"No data found for year {year}") - else: - print(f"Error {response.status_code}: {response.text}") - - # Check the structure of the collected data - if all_data: - print(all_data[0]) # Print the first entry to see the structure - df = pd.DataFrame(all_data) - - # Clean up the DataFrame - if not df.empty: - # Select relevant columns - # parameter_name doesn't exist- use `parameter` or `parameter_code`: - cols = ['parameter', 'year', 'arithmetic_mean', - 'first_max_value', 'first_max_datetime', - 'observation_count'] - df = df[cols] - - # Sort by year and parameter - df = df.sort_values(['year', 'parameter']) - - # Convert year to int - df['year'] = df['year'].astype(int) - - # Display first few rows - print(df.head()) - else: - print("No data found for the specified parameters") - else: - print("No data collected.") - -- query: "Inspect API response change by generating a schema from an API response so that we don't inspect the raw data and blow up the context window." - code: | - import requests - from genson import SchemaBuilder - import os - - # Get API credentials from environment variables - email = os.environ.get("API_EPA_AQS_EMAIL") - key = os.environ.get("API_EPA_AQS") - - # Define parameters - state_code = "48" # Texas - county_code = "201" # Harris County - param_codes = [ - "44201", # Ozone - "42401", # SO2 - "42101", # CO - "42602", # NO2 - "88101", # PM2.5 - "81102" # PM10 - ] - - # Sample one year to inspect the response structure - year = 2020 - bdate = f"{year}0101" - edate = f"{year}1231" - - url = f"https://aqs.epa.gov/data/api/annualData/byCounty" - params = { - "email": email, - "key": key, - "param": ",".join(param_codes), - "bdate": bdate, - "edate": edate, - "state": state_code, - "county": county_code - } - - response = requests.get(url, params=params) - response_data = response.json() - - # annual data by county can include a lot of data, especially if we specify a broad date range and a county with a lot of data- lets generate a schema and inspect that instead of inspecting the raw data - builder = SchemaBuilder() - builder.add_object(response_data) - schema = builder.to_schema() - - print(schema["properties"]) - -- query: "Retrieve daily air quality data for ozone, CO, NO2, and PM2.5 in Harris County, TX for each month of the year 2022." - code: | - import os - import requests - import pandas as pd - from datetime import datetime - import calendar - - # API configuration - base_url = "https://aqs.epa.gov/data/api" - email = os.environ.get("API_EPA_AQS_EMAIL") - key = os.environ.get("API_EPA_AQS") - - # Define parameters - parameters = { - 'Ozone': 44201, - 'CO': 42101, - 'NO2': 42602, - 'PM2.5': 88101 - } - - # Harris County, TX codes - state_code = '48' - county_code = '201' - year = 2022 - - def get_monthly_data(param_code, month): - # Get number of days in month - num_days = calendar.monthrange(year, month)[1] - - params = { - 'email': email, - 'key': key, - 'param': param_code, - 'bdate': f'{year}{month:02d}01', - 'edate': f'{year}{month:02d}{num_days}', - 'state': state_code, - 'county': county_code - } - - response = requests.get(f"{base_url}/dailyData/byCounty", params=params) - return response.json() - - # Initialize empty list to store all data - all_data = [] - - # Loop through each parameter and month - for param_name, param_code in parameters.items(): - for month in range(1, 13): - try: - data = get_monthly_data(param_code, month) - - if 'Data' in data: - for record in data['Data']: - all_data.append({ - 'date': record['date_local'], - 'parameter': param_name, - 'value': record['arithmetic_mean'], - 'units': record['units_of_measure'], - 'site': record['site_number'], - 'latitude': record['latitude'], - 'longitude': record['longitude'] - }) - except Exception as e: - print(f"Error fetching data for {param_name} in month {month}: {str(e)}") - - # Convert to DataFrame - df = pd.DataFrame(all_data) - - # Convert date string to datetime - df['date'] = pd.to_datetime(df['date']) - - # Sort by date and parameter - df = df.sort_values(['date', 'parameter']) - - # Save to CSV - df.to_csv('harris_county_air_quality_2022.csv', index=False) - - -- query: "Here's a reusable function for retrieving and processing data for multiple pollutants:" - code: | - def get_pollutant_data(state_code, county_code, param_code, year, pollutant_name): - """ - Get and process data for a specific pollutant - - Parameters: - ----------- - state_code : str - Two-digit state code - county_code : str - Three-digit county code - param_code : str - Parameter code for the pollutant - year : str - Year to retrieve data for - pollutant_name : str - Name of the pollutant (for display purposes) - - Returns: - -------- - pandas.DataFrame or None - DataFrame containing the processed data, or None if the request failed - """ - print(f"Retrieving daily {pollutant_name} data for Harris County, TX for {year}...") - - bdate = f"{year}0101" # January 1st of the year - edate = f"{year}1231" # December 31st of the year - - data_response = get_aqs_data("dailyData/byCounty", { - "state": state_code, - "county": county_code, - "param": param_code, - "bdate": bdate, - "edate": edate - }) - - if data_response and data_response.get('Header', [{}])[0].get('status') == 'Success': - data = data_response.get('Data', []) - df = pd.DataFrame(data) - - # Convert date_local to datetime - df['date'] = pd.to_datetime(df['date_local']) - - # Group by site and date to get daily averages for each site - site_daily_avg = df.groupby(['local_site_name', 'date'])['arithmetic_mean'].mean().reset_index() - - # Get the overall daily average across all sites - daily_avg = site_daily_avg.groupby('date')['arithmetic_mean'].mean().reset_index() - - # Rename the arithmetic_mean column to the pollutant name for clarity - daily_avg = daily_avg.rename(columns={'arithmetic_mean': pollutant_name.lower()}) - - return daily_avg, df['units_of_measure'].iloc[0] - else: - print(f"Failed to retrieve {pollutant_name} data") - return None, None - -- query: "Retrieving and exploring parameter classes and codes for air quality research. This example demonstrates how to get a list of parameter classes, then retrieve specific parameters within the CRITERIA class, which contains the most important air pollutants for health research. It also identifies key parameter codes relevant to asthma research." - code: | - import os - import requests - import pandas as pd - import matplotlib.pyplot as plt - import seaborn as sns - - # Get authentication details from environment variables - email = os.environ.get("API_EPA_AQS_EMAIL") - api_key = os.environ.get("API_EPA_AQS") - - # Base URL for the EPA AQS API - base_url = "https://aqs.epa.gov/data/api" - - # Function to make API requests - def get_aqs_data(endpoint, params=None): - """ - Make a request to the EPA AQS API - - Parameters: - ----------- - endpoint : str - The API endpoint to query - params : dict - Additional parameters for the request - - Returns: - -------- - dict - The JSON response from the API - """ - # Add authentication parameters - if params is None: - params = {} - params['email'] = email - params['key'] = api_key - - # Make the request - url = f"{base_url}/{endpoint}" - response = requests.get(url, params=params) - - # Check if the request was successful - if response.status_code == 200: - return response.json() - else: - print(f"Error: {response.status_code}") - print(response.text) - return None - - # Get list of parameter classes - classes_response = get_aqs_data("list/classes") - - # Check if we got a valid response for classes - if classes_response and classes_response.get('Header', [{}])[0].get('status') == 'Success': - # Convert to DataFrame for easier viewing - classes_data = classes_response.get('Data', []) - classes_df = pd.DataFrame(classes_data) - - print("Parameter Classes:") - print(classes_df) - - # Now get parameters for the CRITERIA class (most relevant to air quality standards) - criteria_params_response = get_aqs_data("list/parametersByClass", {"pc": "CRITERIA"}) - - if criteria_params_response and criteria_params_response.get('Header', [{}])[0].get('status') == 'Success': - # Convert to DataFrame for easier viewing - criteria_params_data = criteria_params_response.get('Data', []) - criteria_params_df = pd.DataFrame(criteria_params_data) - - print(f"\nFound {len(criteria_params_data)} criteria pollutant parameters") - print("\nCriteria pollutant parameters:") - print(criteria_params_df) - - # Create a dictionary of key parameter codes for asthma research - asthma_param_codes = { - 'Ozone': '44201', - 'PM2.5': '88101', - 'PM10': '81102', - 'NO2': '42602', - 'SO2': '42401', - 'CO': '42101' - } - - print("\nKey parameter codes for asthma research:") - for pollutant, code in asthma_param_codes.items(): - print(f"{pollutant}: {code}") - else: - print("Failed to retrieve criteria parameters data") - else: - print("Failed to retrieve parameter classes data") - - -- query: "Multi-pollutant correlation analysis for air quality research. This example demonstrates how to retrieve data for multiple pollutants (Ozone, PM2.5, NO2, and SO2), process the data to get daily averages, merge the datasets, and analyze correlations between different pollutants. It includes a visualization of the correlation matrix using a heatmap." - code: | - import os - import requests - import pandas as pd - import matplotlib.pyplot as plt - import seaborn as sns - from datetime import datetime - - # Set the style for our plots - sns.set(style="whitegrid") - plt.rcParams['figure.figsize'] = (12, 6) - - # Get authentication details from environment variables - email = os.environ.get("API_EPA_AQS_EMAIL") - api_key = os.environ.get("API_EPA_AQS") - - # Base URL for the EPA AQS API - base_url = "https://aqs.epa.gov/data/api" - - # Function to make API requests - def get_aqs_data(endpoint, params=None): - """ - Make a request to the EPA AQS API - - Parameters: - ----------- - endpoint : str - The API endpoint to query - params : dict - Additional parameters for the request - - Returns: - -------- - dict - The JSON response from the API - """ - # Add authentication parameters - if params is None: - params = {} - params['email'] = email - params['key'] = api_key - - # Make the request - url = f"{base_url}/{endpoint}" - response = requests.get(url, params=params) - - # Check if the request was successful - if response.status_code == 200: - return response.json() - else: - print(f"Error: {response.status_code}") - print(response.text) - return None - - # Function to get and process data for a specific pollutant - def get_pollutant_data(state_code, county_code, param_code, year, pollutant_name): - """ - Get and process data for a specific pollutant - - Parameters: - ----------- - state_code : str - Two-digit state code - county_code : str - Three-digit county code - param_code : str - Parameter code for the pollutant - year : str - Year to retrieve data for - pollutant_name : str - Name of the pollutant (for display purposes) - - Returns: - -------- - pandas.DataFrame or None - DataFrame containing the processed data, or None if the request failed - """ - print(f"Retrieving daily {pollutant_name} data for {year}...") - - bdate = f"{year}0101" # January 1st of the year - edate = f"{year}1231" # December 31st of the year - - data_response = get_aqs_data("dailyData/byCounty", { - "state": state_code, - "county": county_code, - "param": param_code, - "bdate": bdate, - "edate": edate - }) - - if data_response and data_response.get('Header', [{}])[0].get('status') == 'Success': - data = data_response.get('Data', []) - df = pd.DataFrame(data) - - print(f"\nFound {len(data)} daily {pollutant_name} records for {year}") - - # Convert date_local to datetime - df['date'] = pd.to_datetime(df['date_local']) - - # Extract month and day of week for analysis - df['month'] = df['date'].dt.month - df['month_name'] = df['date'].dt.month_name() - - # Group by site and date to get daily averages for each site - site_daily_avg = df.groupby(['local_site_name', 'date'])['arithmetic_mean'].mean().reset_index() - - # Get the overall daily average across all sites - daily_avg = site_daily_avg.groupby('date')['arithmetic_mean'].mean().reset_index() - - # Rename the arithmetic_mean column to the pollutant name for clarity - daily_avg = daily_avg.rename(columns={'arithmetic_mean': pollutant_name.lower()}) - - return daily_avg, df['units_of_measure'].iloc[0] - else: - print(f"Failed to retrieve {pollutant_name} data") - return None, None - - # Set up parameters - state_code = "48" # Texas - county_code = "201" # Harris County - year = "2022" - - # Define pollutants to retrieve - pollutants = { - 'Ozone': '44201', - 'PM2.5': '88101', - 'NO2': '42602', - 'SO2': '42401' - } - - # Get data for each pollutant - pollutant_data = {} - units = {} - - for pollutant_name, param_code in pollutants.items(): - daily_avg, unit = get_pollutant_data(state_code, county_code, param_code, year, pollutant_name) - if daily_avg is not None: - pollutant_data[pollutant_name.lower()] = daily_avg - units[pollutant_name.lower()] = unit - - # Merge all pollutant data on date - merged_df = None - - for pollutant_name, daily_avg in pollutant_data.items(): - if merged_df is None: - merged_df = daily_avg - else: - merged_df = pd.merge(merged_df, daily_avg[['date', pollutant_name]], on='date', how='outer') - - # Calculate correlations between pollutants - if merged_df is not None: - print("\nCorrelations between pollutants:") - correlation_matrix = merged_df.drop('date', axis=1).corr() - print(correlation_matrix) - - # Plot correlation heatmap - plt.figure(figsize=(10, 8)) - sns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', vmin=-1, vmax=1, center=0) - plt.title(f'Correlation Matrix of Air Pollutants ({year})', fontsize=16) - plt.tight_layout() - plt.show() diff --git a/src/biome/adhoc_data/datasources/cbioportal/api.yaml b/src/biome/adhoc_data/datasources/cbioportal/api.yaml index 883e42d..0d3f270 100644 --- a/src/biome/adhoc_data/datasources/cbioportal/api.yaml +++ b/src/biome/adhoc_data/datasources/cbioportal/api.yaml @@ -1,15 +1,637 @@ -name: cbioportal -slug: cbioportal +description: | + The cBioPortal for Cancer Genomics is an open-access, open-source resource for interactive exploration of multidimensional cancer genomics data sets. The goal of cBioPortal is to significantly lower the barriers between complex genomic data and cancer researchers by providing rapid, intuitive, and high-quality access to molecular profiles and clinical attributes from large-scale cancer genomics projects, and therefore to empower researchers to translate these rich data sets into biologic insights and clinical applications. integration_type: api +name: cbioportal +prompt: | + This API is through Open API Spec (Swagger). You will be provided the schema below. + All requests go to the following URL: https://www.cbioportal.org/ -description: | - The cBioPortal for Cancer Genomics is an open-access, open-source resource for interactive exploration of multidimensional cancer genomics data sets. The goal of cBioPortal is to significantly lower the barriers between complex genomic data and cancer researchers by providing rapid, intuitive, and high-quality access to molecular profiles and clinical attributes from large-scale cancer genomics projects, and therefore to empower researchers to translate these rich data sets into biologic insights and clinical applications. + ${cbioportal} +resources: + 21897f3f-9451-4338-b2b4-8ae3faeafc20: + code: | + import requests + import pandas as pd -attachments: - cbioportal: cbioportal.json + def fetch_aml_studies(): + # Base URL for cBioPortal API + base_url = "https://www.cbioportal.org/api" -prompt: | - This API is through Open API Spec (Swagger). You will be provided the schema below. - All requests go to the following URL: https://www.cbioportal.org/ + # Get all studies + response = requests.get(f"{base_url}/studies") + + if response.status_code != 200: + raise Exception(f"API request failed with status code {response.status_code}") + + # Get all studies as JSON + studies = response.json() + + # Filter for AML related studies + aml_studies = [] + for study in studies: + # Convert all fields to lowercase for case-insensitive search + study_id = study.get('studyId', '').lower() + name = study.get('name', '').lower() + cancer_type = study.get('cancerTypeId', '').lower() + description = study.get('description', '').lower() + + # Check if 'aml' appears in any of the relevant fields + if any('aml' in field for field in [study_id, name, cancer_type, description]): + aml_studies.append({ + 'studyId': study['studyId'], + 'name': study['name'], + 'description': study.get('description', 'N/A'), + 'cancerTypeId': study.get('cancerTypeId', 'N/A'), + 'sampleCount': study.get('allSampleCount', 0), + 'status': study.get('status', 'N/A'), + 'publicStudy': study.get('publicStudy', False) + }) + + # Convert to DataFrame for better display + df = pd.DataFrame(aml_studies) + + # Sort by sample count descending + df = df.sort_values('sampleCount', ascending=False) + + return df + + # Execute the function + aml_studies_df = fetch_aml_studies() + print(f"Found {len(aml_studies_df)} AML-related studies") + print(aml_studies_df) + integration: cbioportal + notes: null + query: Fetch and filter studies related to Acute Myeloid Leukemia (AML) from cBioPortal + API. + resource_id: 21897f3f-9451-4338-b2b4-8ae3faeafc20 + resource_type: example + 3449a203-e65b-4a8d-91cb-593ccdbef58c: + code: | + import requests + import pandas as pd + import time + + # Configuration + BASE_URL = "https://www.cbioportal.org/api" + + # Define studies and their RNA-seq profiles + STUDY_PROFILES = { + 'aml_target_gdc': { + 'name': 'TARGET-AML (GDC)', + 'profile': 'aml_target_gdc_mrna_seq_tpm_Zscores', + 'type': 'TPM' + }, + 'aml_ohsu_2022': { + 'name': 'OHSU AML 2022', + 'profile': 'aml_ohsu_2022_mrna_median_Zscores', + 'type': 'RPKM' + } + # ... other studies ... + } + + # Define STAT5 genes with their Entrez IDs + STAT5_GENES = { + 6776: 'STAT5A', + 6777: 'STAT5B' + } + + # Function to get RNA-seq samples for a study + def get_rna_seq_samples(study_id): + """Get list of sample IDs that have RNA-seq data for a study.""" + url = f"{BASE_URL}/studies/{study_id}/sample-lists" + response = requests.get(url) + if response.status_code == 200: + sample_lists = response.json() + # Find RNA-seq sample list + rna_list = next((sl['sampleListId'] for sl in sample_lists + if sl['category'] == 'all_cases_with_mrna_rnaseq_data'), None) + if rna_list: + # Get sample IDs from the list + url = f"{BASE_URL}/sample-lists/{rna_list}/sample-ids" + response = requests.get(url) + if response.status_code == 200: + return response.json() + return None + + # Function to get expression data + def get_expression_data(profile_id, gene_ids, sample_ids): + """Fetch expression z-scores for specific genes and samples.""" + url = f"{BASE_URL}/molecular-profiles/{profile_id}/molecular-data/fetch" + # IMPORTANT: Must provide sample IDs explicitly + data = { + "sampleIds": sample_ids, # List of specific sample IDs + "entrezGeneIds": gene_ids # List of Entrez gene IDs + } + response = requests.post(url, json=data) + if response.status_code == 200: + return response.json() + return None + + # Process each study + all_study_data = [] + + for study_id, study_info in STUDY_PROFILES.items(): + print(f"\nProcessing {study_info['name']}...") + + # First get the sample IDs for the study + sample_ids = get_rna_seq_samples(study_id) + + if sample_ids: + print(f"Found {len(sample_ids)} RNA-seq samples") + + # Process samples in chunks to avoid large requests + chunk_size = 100 + study_data = [] + + for i in range(0, len(sample_ids), chunk_size): + chunk = sample_ids[i:i + chunk_size] + print(f"Fetching data for samples {i+1}-{i+len(chunk)}...") + + # Get expression data for this chunk of samples + expression_data = get_expression_data( + study_info['profile'], + list(STAT5_GENES.keys()), + chunk + ) + + if expression_data: + study_data.extend(expression_data) + time.sleep(0.2) # Small delay between requests + + if study_data: + # Convert to DataFrame + df = pd.DataFrame(study_data) + + # Add metadata + df['gene_symbol'] = df['entrezGeneId'].map(STAT5_GENES) + df['study_id'] = study_id + df['study_name'] = study_info['name'] + df['measurement_type'] = study_info['type'] + + # Select and rename columns + df = df[[ + 'study_id', 'study_name', 'measurement_type', + 'gene_symbol', 'sampleId', 'value' + ]] + df.columns = [ + 'study_id', 'study_name', 'measurement_type', + 'gene', 'sample_id', 'zscore' + ] + + all_study_data.append(df) + + # Combine all data + if all_study_data: + # Create long format + combined_df = pd.concat(all_study_data, ignore_index=True) + + # Create wide format (samples as rows, genes as columns) + combined_wide = combined_df.pivot_table( + index=['study_id', 'study_name', 'measurement_type', 'sample_id'], + columns='gene', + values='zscore' + ).reset_index() + + # Save both versions + stat5_all_studies = combined_df # Long format + stat5_all_studies_wide = combined_wide # Wide format + + # Example of the resulting data structure: + print("\nLong format example (first few rows):") + print(stat5_all_studies.head()) + + print("\nWide format example (first few rows):") + print(stat5_all_studies_wide.head()) + integration: cbioportal + notes: | + This example demonstrates: + 1. How to properly fetch sample IDs for each study + 2. How to use these sample IDs when requesting molecular data + 3. How to handle large requests by processing samples in chunks + 4. How to combine data from multiple studies into a single dataframe + 5. How to create both long and wide format versions of the data + + Key points: + - Must provide explicit sample IDs in the molecular data request + - Z-scores are pre-calculated by cBioPortal for each study + - Different studies may use different measurement types (TPM, RPKM, RSEM) + - Processing in chunks helps avoid timeouts with large datasets + query: Example of how to fetch RNA-seq z-scores for STAT5A and STAT5B across multiple + AML studies + resource_id: 3449a203-e65b-4a8d-91cb-593ccdbef58c + resource_type: example + 52adfa23-1b07-462e-968b-d09ed12848f0: + filepath: cbioportal.json + integration: cbioportal + name: cbioportal + resource_id: 52adfa23-1b07-462e-968b-d09ed12848f0 + resource_type: file + 61d5403b-63fe-4f11-9e0d-04c5aec310df: + code: | + import requests + import pandas as pd + from collections import Counter + import numpy as np + + # Base URL for cBioPortal API + base_url = "https://www.cbioportal.org/api" + study_id = "aml_target_gdc" # Example study ID + + # Get mutations using the sample list ID + mutations_url = f"{base_url}/molecular-profiles/{study_id}_mutations/mutations?sampleListId={study_id}_all" + mutations_response = requests.get(mutations_url) + + if mutations_response.status_code == 200: + mutations_data = mutations_response.json() + mutations_df = pd.DataFrame(mutations_data) + + # Get gene information for the entrez IDs + entrez_ids = [int(x) for x in mutations_df['entrezGeneId'].unique()] # Convert to regular Python integers + genes_url = f"{base_url}/genes/fetch" + genes_response = requests.post(genes_url, json=entrez_ids) + + if genes_response.status_code == 200: + genes_data = genes_response.json() + gene_map = {gene['entrezGeneId']: gene['hugoGeneSymbol'] for gene in genes_data} + + # Add gene symbols to mutations dataframe + mutations_df['gene_symbol'] = mutations_df['entrezGeneId'].map(gene_map) + + # Print summary statistics + print(f"Total mutations found: {len(mutations_df)}") + print(f"Number of affected genes: {len(mutations_df['gene_symbol'].unique())}") + + print("\nMost frequently mutated genes:") + print(mutations_df['gene_symbol'].value_counts().head(10)) + + print("\nMutation types distribution:") + print(mutations_df['mutationType'].value_counts()) + + print("\nSample of mutations (first 5 rows):") + display_cols = ['gene_symbol', 'sampleId', 'mutationType', 'proteinChange', 'chr', 'startPosition', 'variantType'] + print(mutations_df[display_cols].head()) + + # Calculate sample statistics + samples_with_mutations = len(mutations_df['sampleId'].unique()) + print(f"\nNumber of samples with mutations: {samples_with_mutations}") + + avg_mutations_per_sample = len(mutations_df) / samples_with_mutations + print(f"Average mutations per sample: {avg_mutations_per_sample:.2f}") + else: + print(f"Error fetching mutations: {mutations_response.status_code}") + print("Response:", mutations_response.text) + integration: cbioportal + notes: | + It shows: + 1. How to properly construct the API endpoint URL with sample list ID + 2. How to fetch mutation data for a study + 3. How to get gene symbol information using entrez IDs + 4. How to create a comprehensive mutation analysis including: + - Total mutation counts + - Gene frequency analysis + - Mutation type distribution + - Sample statistics + The example uses the TARGET AML (GDC) study but can be modified for any study by changing the study_id. + query: This example demonstrates how to fetch and analyze mutation data for a + specific study in cBioPortal + resource_id: 61d5403b-63fe-4f11-9e0d-04c5aec310df + resource_type: example + 7f738f92-b1d2-463b-826f-db0dc1331e56: + code: | + import requests + import pandas as pd + from typing import List, Dict + + def get_molecular_profiles(study_ids: List[str]) -> Dict[str, pd.DataFrame]: + """ + Retrieve molecular profiles for multiple studies from cBioPortal. + + Args: + study_ids: List of cBioPortal study IDs + + Returns: + Dictionary mapping study IDs to DataFrames containing their molecular profiles + """ + # Base URL for cBioPortal's web API + base_url = "https://www.cbioportal.org/api" + + # Dictionary to store results + results = {} - ${cbioportal} + print("Fetching molecular profiles...") + for study_id in study_ids: + print(f"\nStudy: {study_id}") + print("-" * 40) + + # Get molecular profiles for this study + response = requests.get(f"{base_url}/studies/{study_id}/molecular-profiles") + + if response.status_code == 200: + profiles = response.json() + + # Create a summary DataFrame + profile_data = [] + for profile in profiles: + profile_data.append({ + 'molecularProfileId': profile['molecularProfileId'], + 'name': profile['name'], + 'datatype': profile['datatype'], + 'molecularAlterationType': profile['molecularAlterationType'], + 'description': profile.get('description', 'N/A') + }) + + if profile_data: + df = pd.DataFrame(profile_data) + results[study_id] = df + print(f"Found {len(df)} molecular profiles") + + # Display summary of profile types + print("\nProfile types:") + type_summary = df.groupby('molecularAlterationType')['molecularProfileId'].count() + print(type_summary) + else: + print("No molecular profiles found") + results[study_id] = pd.DataFrame() + else: + print(f"Error accessing study: {response.status_code}") + results[study_id] = pd.DataFrame() + + return results + + # Example usage with AML studies + aml_studies = [ + 'aml_target_2018_pub', + 'aml_ohsu_2018', + 'aml_ohsu_2022', + 'laml_tcga', + 'laml_tcga_pan_can_atlas_2018', + 'aml_target_gdc', + 'mnm_washu_2016' + ] + + # Get molecular profiles for all studies + profiles_by_study = get_molecular_profiles(aml_studies) + + # Example of how to work with the results - get RNA expression profiles for first study + first_study = list(profiles_by_study.keys())[0] + if not profiles_by_study[first_study].empty: + rna_profiles = profiles_by_study[first_study][ + profiles_by_study[first_study]['molecularAlterationType'] == 'MRNA_EXPRESSION' + ] + print(f"\nRNA expression profiles for {first_study}:") + print(rna_profiles[['molecularProfileId', 'name', 'datatype']].to_string()) + integration: cbioportal + notes: null + query: |- + Example showing how to retrieve and analyze the molecular profiles available for multiple studies from cBioPortal. The code demonstrates how to fetch metadata on the molecular profiles, organize them into DataFrames, and analyze specific information about each profile like the types of profiles available such as RNA expression. + resource_id: 7f738f92-b1d2-463b-826f-db0dc1331e56 + resource_type: example + 84612cb9-2439-4e07-96ae-edfac09374e9: + code: | + import requests + import pandas as pd + import json + + # Base URL + BASE_URL = "https://www.cbioportal.org/api" + + # List of AML studies + AML_STUDIES = [ + 'aml_target_2018_pub', + 'aml_ohsu_2018', + 'aml_ohsu_2022', + 'laml_tcga', + 'laml_tcga_pan_can_atlas_2018', + 'laml_tcga_gdac', + 'aml_target_gdc', + 'washu_pdi_2016', + 'laml_tcga_pub' + ] + + # Function to get mutations for a study + def get_study_mutations(study_id): + print(f"\nProcessing study: {study_id}") + + # Get the molecular profile ID + molecular_profile_id = f"{study_id}_mutations" + + # Get the sample list ID + sample_list_id = f"{study_id}_all" + + # Fetch mutations + mutations_response = requests.get( + f"{BASE_URL}/molecular-profiles/{molecular_profile_id}/mutations", + params={"sampleListId": sample_list_id, "projection": "DETAILED"} + ) + + if mutations_response.status_code == 200: + mutations = mutations_response.json() + print(f"Found {len(mutations)} mutations") + + # Convert to DataFrame + mutations_df = pd.DataFrame(mutations) + + # Extract gene symbols from the nested dictionary + mutations_df['gene_symbol'] = mutations_df['gene'].apply(lambda x: x['hugoGeneSymbol']) + + # Create a more focused DataFrame with key columns + focused_df = mutations_df[[ + 'gene_symbol', + 'sampleId', + 'proteinChange', + 'mutationType', + 'chr', + 'startPosition', + 'endPosition', + 'referenceAllele', + 'variantAllele' + ]] + + return focused_df + else: + print(f"Error fetching mutations: {mutations_response.text}") + return None + + # Process all studies + all_mutations_dfs = [] + + for study_id in AML_STUDIES: + study_df = get_study_mutations(study_id) + if study_df is not None: + study_df['study_id'] = study_id # Add study ID column + all_mutations_dfs.append(study_df) + + # Combine all mutations + if all_mutations_dfs: + combined_df = pd.concat(all_mutations_dfs, ignore_index=True) + + print("\nOverall mutation data summary:") + print(f"Total number of mutations: {len(combined_df)}") + print(f"Number of unique genes: {combined_df['gene_symbol'].nunique()}") + print(f"Number of unique samples: {combined_df['sampleId'].nunique()}") + + print("\nTop 20 most frequently mutated genes across all studies:") + print(combined_df['gene_symbol'].value_counts().head(20)) + + print("\nDistribution of mutation types:") + print(combined_df['mutationType'].value_counts()) + + print("\nNumber of mutations by study:") + print(combined_df['study_id'].value_counts()) + + # Save to CSV + combined_df.to_csv('all_aml_mutations.csv', index=False) + integration: cbioportal + notes: null + query: |- + Retrieve and analyze mutation data from multiple AML studies, including data processing and summary statistics. This example demonstrates how to fetch mutations across multiple studies, handle the nested JSON response, and create a comprehensive mutation dataset with key information. + resource_id: 84612cb9-2439-4e07-96ae-edfac09374e9 + resource_type: example + a1958b77-329b-4c77-a1b6-8cbdefd56eea: + code: | + import requests + + url = "https://www.cbioportal.org/api/studies" + response = requests.get(url) + + if response.status_code == 200: + studies = response.json() + colorectal_studies = [study for study in studies if study['cancerTypeId'].lower() in ['coadread', 'coad', 'read']] + print(f"Found {len(colorectal_studies)} colorectal cancer studies.") + print("\nStudy Details:") + for study in colorectal_studies: + print(f"\nName: {study['name']}") + print(f"Study ID: {study['studyId']}") + print(f"Description: {study['description']}") + print(f"Cancer Type ID: {study['cancerTypeId']}") + print("-" * 80) + integration: cbioportal + notes: null + query: |- + Query and display all colorectal cancer studies from cBioPortal. This example searches for studies with cancer type IDs 'coadread', 'coad', and 'read' (representing colorectal adenocarcinoma, colon adenocarcinoma, and rectal adenocarcinoma respectively). For each study, it displays the name, study ID, description, and cancer type ID. + resource_id: a1958b77-329b-4c77-a1b6-8cbdefd56eea + resource_type: example + c68b74b2-c97c-4cf5-b8b4-7e514db2db65: + code: | + import requests + import pandas as pd + + # cBioPortal API base URL + base_url = "https://www.cbioportal.org/api" + + # Endpoint to fetch studies + endpoint = "/studies" + + # Parameters for the query + params = {"cancerTypeId": "coadread"} + + # Make the API request + response = requests.get(base_url + endpoint, params=params) + + # Check for successful response + response.raise_for_status() + + # Parse the JSON response + studies = response.json() + + # Create a pandas DataFrame from the results + df = pd.DataFrame(studies) + + # Print the DataFrame + print(df) + integration: cbioportal + notes: null + query: Query for studies related to colorectal cancer using the cancerTypeId 'coadread'. + resource_id: c68b74b2-c97c-4cf5-b8b4-7e514db2db65 + resource_type: example + d54d8188-81f5-4fff-a641-230480369fe4: + code: | + import requests + import json + + # Get samples for the study + study_id = "coadread_genentech" + samples_url = f"https://www.cbioportal.org/api/studies/{study_id}/samples" + samples_response = requests.get(samples_url) + + if samples_response.status_code == 200: + samples = json.loads(samples_response.content) + sample_ids = [sample['sampleId'] for sample in samples] + print(f"Found {len(sample_ids)} samples") + + # Get mutation data + mutations_url = f"https://www.cbioportal.org/api/molecular-profiles/{study_id}_mutations/mutations/fetch" + + # Create the filter with sample IDs + data = { + "sampleIds": sample_ids + } + + # Make the POST request + mutations_response = requests.post(mutations_url, json=data) + + if mutations_response.status_code == 200: + mutations = json.loads(mutations_response.content) + print(f"\nRetrieved {len(mutations)} mutations") + + # Focus on key cancer genes + key_genes = { + 'APC': 324, # EntrezGeneID for APC + 'TP53': 7157, # EntrezGeneID for TP53 + 'KRAS': 3845, # EntrezGeneID for KRAS + 'PIK3CA': 5290 # EntrezGeneID for PIK3CA + } + + # Analyze mutations for these genes + gene_details = {} + for gene_symbol, gene_id in key_genes.items(): + gene_mutations = [m for m in mutations if m['entrezGeneId'] == gene_id] + + # Count mutation types + mutation_types = {} + protein_changes = [] + + for mutation in gene_mutations: + mut_type = mutation['mutationType'] + if mut_type in mutation_types: + mutation_types[mut_type] += 1 + else: + mutation_types[mut_type] = 1 + + if mutation['proteinChange']: + protein_changes.append(mutation['proteinChange']) + + gene_details[gene_symbol] = { + 'total_mutations': len(gene_mutations), + 'mutation_types': mutation_types, + 'protein_changes': protein_changes + } + + # Print detailed analysis + print("\nDetailed Analysis of Key Cancer Genes:\n") + for gene, details in gene_details.items(): + print(f"\n{gene} Analysis:") + print(f"Total mutations: {details['total_mutations']}") + + print("Mutation types:") + for mut_type, count in details['mutation_types'].items(): + print(f" - {mut_type}: {count}") + + print("Protein changes (top 5):") + for change in details['protein_changes'][:5]: + print(f" - {change}") + if len(details['protein_changes']) > 5: + print(f" ... and {len(details['protein_changes'])-5} more changes") + integration: cbioportal + notes: | + The example demonstrates how to: + 1. Get sample IDs for a study + 2. Fetch mutation data using the molecular profiles endpoint + 3. Filter and analyze mutations for specific genes + 4. Count mutation types and protein changes + The code provides a detailed breakdown of mutation types and frequencies for each gene. + query: |- + Query mutation data from cBioPortal for a specific study (Genentech colorectal cancer study) and analyze mutations in key cancer genes (APC, TP53, KRAS, PIK3CA) + resource_id: d54d8188-81f5-4fff-a641-230480369fe4 + resource_type: example +slug: cbioportal diff --git a/src/biome/adhoc_data/datasources/cbioportal/examples.yaml b/src/biome/adhoc_data/datasources/cbioportal/examples.yaml deleted file mode 100644 index 4de70d7..0000000 --- a/src/biome/adhoc_data/datasources/cbioportal/examples.yaml +++ /dev/null @@ -1,589 +0,0 @@ -- query: "Query and display all colorectal cancer studies from cBioPortal. This example searches for studies with cancer type IDs 'coadread', 'coad', and 'read' (representing colorectal adenocarcinoma, colon adenocarcinoma, and rectal adenocarcinoma respectively). For each study, it displays the name, study ID, description, and cancer type ID." - code: | - import requests - - url = "https://www.cbioportal.org/api/studies" - response = requests.get(url) - - if response.status_code == 200: - studies = response.json() - colorectal_studies = [study for study in studies if study['cancerTypeId'].lower() in ['coadread', 'coad', 'read']] - print(f"Found {len(colorectal_studies)} colorectal cancer studies.") - print("\nStudy Details:") - for study in colorectal_studies: - print(f"\nName: {study['name']}") - print(f"Study ID: {study['studyId']}") - print(f"Description: {study['description']}") - print(f"Cancer Type ID: {study['cancerTypeId']}") - print("-" * 80) - -- query: "Query mutation data from cBioPortal for a specific study (Genentech colorectal cancer study) and analyze mutations in key cancer genes (APC, TP53, KRAS, PIK3CA)" - notes: | - The example demonstrates how to: - 1. Get sample IDs for a study - 2. Fetch mutation data using the molecular profiles endpoint - 3. Filter and analyze mutations for specific genes - 4. Count mutation types and protein changes - The code provides a detailed breakdown of mutation types and frequencies for each gene. - code: | - import requests - import json - - # Get samples for the study - study_id = "coadread_genentech" - samples_url = f"https://www.cbioportal.org/api/studies/{study_id}/samples" - samples_response = requests.get(samples_url) - - if samples_response.status_code == 200: - samples = json.loads(samples_response.content) - sample_ids = [sample['sampleId'] for sample in samples] - print(f"Found {len(sample_ids)} samples") - - # Get mutation data - mutations_url = f"https://www.cbioportal.org/api/molecular-profiles/{study_id}_mutations/mutations/fetch" - - # Create the filter with sample IDs - data = { - "sampleIds": sample_ids - } - - # Make the POST request - mutations_response = requests.post(mutations_url, json=data) - - if mutations_response.status_code == 200: - mutations = json.loads(mutations_response.content) - print(f"\nRetrieved {len(mutations)} mutations") - - # Focus on key cancer genes - key_genes = { - 'APC': 324, # EntrezGeneID for APC - 'TP53': 7157, # EntrezGeneID for TP53 - 'KRAS': 3845, # EntrezGeneID for KRAS - 'PIK3CA': 5290 # EntrezGeneID for PIK3CA - } - - # Analyze mutations for these genes - gene_details = {} - for gene_symbol, gene_id in key_genes.items(): - gene_mutations = [m for m in mutations if m['entrezGeneId'] == gene_id] - - # Count mutation types - mutation_types = {} - protein_changes = [] - - for mutation in gene_mutations: - mut_type = mutation['mutationType'] - if mut_type in mutation_types: - mutation_types[mut_type] += 1 - else: - mutation_types[mut_type] = 1 - - if mutation['proteinChange']: - protein_changes.append(mutation['proteinChange']) - - gene_details[gene_symbol] = { - 'total_mutations': len(gene_mutations), - 'mutation_types': mutation_types, - 'protein_changes': protein_changes - } - - # Print detailed analysis - print("\nDetailed Analysis of Key Cancer Genes:\n") - for gene, details in gene_details.items(): - print(f"\n{gene} Analysis:") - print(f"Total mutations: {details['total_mutations']}") - - print("Mutation types:") - for mut_type, count in details['mutation_types'].items(): - print(f" - {mut_type}: {count}") - - print("Protein changes (top 5):") - for change in details['protein_changes'][:5]: - print(f" - {change}") - if len(details['protein_changes']) > 5: - print(f" ... and {len(details['protein_changes'])-5} more changes") - - -- query: "Query for studies related to colorectal cancer using the cancerTypeId 'coadread'." - code: | - import requests - import pandas as pd - - # cBioPortal API base URL - base_url = "https://www.cbioportal.org/api" - - # Endpoint to fetch studies - endpoint = "/studies" - - # Parameters for the query - params = {"cancerTypeId": "coadread"} - - # Make the API request - response = requests.get(base_url + endpoint, params=params) - - # Check for successful response - response.raise_for_status() - - # Parse the JSON response - studies = response.json() - - # Create a pandas DataFrame from the results - df = pd.DataFrame(studies) - - # Print the DataFrame - print(df) - - -- query: "Fetch and filter studies related to Acute Myeloid Leukemia (AML) from cBioPortal API." - code: | - import requests - import pandas as pd - - def fetch_aml_studies(): - # Base URL for cBioPortal API - base_url = "https://www.cbioportal.org/api" - - # Get all studies - response = requests.get(f"{base_url}/studies") - - if response.status_code != 200: - raise Exception(f"API request failed with status code {response.status_code}") - - # Get all studies as JSON - studies = response.json() - - # Filter for AML related studies - aml_studies = [] - for study in studies: - # Convert all fields to lowercase for case-insensitive search - study_id = study.get('studyId', '').lower() - name = study.get('name', '').lower() - cancer_type = study.get('cancerTypeId', '').lower() - description = study.get('description', '').lower() - - # Check if 'aml' appears in any of the relevant fields - if any('aml' in field for field in [study_id, name, cancer_type, description]): - aml_studies.append({ - 'studyId': study['studyId'], - 'name': study['name'], - 'description': study.get('description', 'N/A'), - 'cancerTypeId': study.get('cancerTypeId', 'N/A'), - 'sampleCount': study.get('allSampleCount', 0), - 'status': study.get('status', 'N/A'), - 'publicStudy': study.get('publicStudy', False) - }) - - # Convert to DataFrame for better display - df = pd.DataFrame(aml_studies) - - # Sort by sample count descending - df = df.sort_values('sampleCount', ascending=False) - - return df - - # Execute the function - aml_studies_df = fetch_aml_studies() - print(f"Found {len(aml_studies_df)} AML-related studies") - print(aml_studies_df) - - -- query: "This example demonstrates how to fetch and analyze mutation data for a specific study in cBioPortal" - notes: | - It shows: - 1. How to properly construct the API endpoint URL with sample list ID - 2. How to fetch mutation data for a study - 3. How to get gene symbol information using entrez IDs - 4. How to create a comprehensive mutation analysis including: - - Total mutation counts - - Gene frequency analysis - - Mutation type distribution - - Sample statistics - The example uses the TARGET AML (GDC) study but can be modified for any study by changing the study_id. - code: | - import requests - import pandas as pd - from collections import Counter - import numpy as np - - # Base URL for cBioPortal API - base_url = "https://www.cbioportal.org/api" - study_id = "aml_target_gdc" # Example study ID - - # Get mutations using the sample list ID - mutations_url = f"{base_url}/molecular-profiles/{study_id}_mutations/mutations?sampleListId={study_id}_all" - mutations_response = requests.get(mutations_url) - - if mutations_response.status_code == 200: - mutations_data = mutations_response.json() - mutations_df = pd.DataFrame(mutations_data) - - # Get gene information for the entrez IDs - entrez_ids = [int(x) for x in mutations_df['entrezGeneId'].unique()] # Convert to regular Python integers - genes_url = f"{base_url}/genes/fetch" - genes_response = requests.post(genes_url, json=entrez_ids) - - if genes_response.status_code == 200: - genes_data = genes_response.json() - gene_map = {gene['entrezGeneId']: gene['hugoGeneSymbol'] for gene in genes_data} - - # Add gene symbols to mutations dataframe - mutations_df['gene_symbol'] = mutations_df['entrezGeneId'].map(gene_map) - - # Print summary statistics - print(f"Total mutations found: {len(mutations_df)}") - print(f"Number of affected genes: {len(mutations_df['gene_symbol'].unique())}") - - print("\nMost frequently mutated genes:") - print(mutations_df['gene_symbol'].value_counts().head(10)) - - print("\nMutation types distribution:") - print(mutations_df['mutationType'].value_counts()) - - print("\nSample of mutations (first 5 rows):") - display_cols = ['gene_symbol', 'sampleId', 'mutationType', 'proteinChange', 'chr', 'startPosition', 'variantType'] - print(mutations_df[display_cols].head()) - - # Calculate sample statistics - samples_with_mutations = len(mutations_df['sampleId'].unique()) - print(f"\nNumber of samples with mutations: {samples_with_mutations}") - - avg_mutations_per_sample = len(mutations_df) / samples_with_mutations - print(f"Average mutations per sample: {avg_mutations_per_sample:.2f}") - else: - print(f"Error fetching mutations: {mutations_response.status_code}") - print("Response:", mutations_response.text) - - -- query: "Retrieve and analyze mutation data from multiple AML studies, including data processing and summary statistics. This example demonstrates how to fetch mutations across multiple studies, handle the nested JSON response, and create a comprehensive mutation dataset with key information." - code: | - import requests - import pandas as pd - import json - - # Base URL - BASE_URL = "https://www.cbioportal.org/api" - - # List of AML studies - AML_STUDIES = [ - 'aml_target_2018_pub', - 'aml_ohsu_2018', - 'aml_ohsu_2022', - 'laml_tcga', - 'laml_tcga_pan_can_atlas_2018', - 'laml_tcga_gdac', - 'aml_target_gdc', - 'washu_pdi_2016', - 'laml_tcga_pub' - ] - - # Function to get mutations for a study - def get_study_mutations(study_id): - print(f"\nProcessing study: {study_id}") - - # Get the molecular profile ID - molecular_profile_id = f"{study_id}_mutations" - - # Get the sample list ID - sample_list_id = f"{study_id}_all" - - # Fetch mutations - mutations_response = requests.get( - f"{BASE_URL}/molecular-profiles/{molecular_profile_id}/mutations", - params={"sampleListId": sample_list_id, "projection": "DETAILED"} - ) - - if mutations_response.status_code == 200: - mutations = mutations_response.json() - print(f"Found {len(mutations)} mutations") - - # Convert to DataFrame - mutations_df = pd.DataFrame(mutations) - - # Extract gene symbols from the nested dictionary - mutations_df['gene_symbol'] = mutations_df['gene'].apply(lambda x: x['hugoGeneSymbol']) - - # Create a more focused DataFrame with key columns - focused_df = mutations_df[[ - 'gene_symbol', - 'sampleId', - 'proteinChange', - 'mutationType', - 'chr', - 'startPosition', - 'endPosition', - 'referenceAllele', - 'variantAllele' - ]] - - return focused_df - else: - print(f"Error fetching mutations: {mutations_response.text}") - return None - - # Process all studies - all_mutations_dfs = [] - - for study_id in AML_STUDIES: - study_df = get_study_mutations(study_id) - if study_df is not None: - study_df['study_id'] = study_id # Add study ID column - all_mutations_dfs.append(study_df) - - # Combine all mutations - if all_mutations_dfs: - combined_df = pd.concat(all_mutations_dfs, ignore_index=True) - - print("\nOverall mutation data summary:") - print(f"Total number of mutations: {len(combined_df)}") - print(f"Number of unique genes: {combined_df['gene_symbol'].nunique()}") - print(f"Number of unique samples: {combined_df['sampleId'].nunique()}") - - print("\nTop 20 most frequently mutated genes across all studies:") - print(combined_df['gene_symbol'].value_counts().head(20)) - - print("\nDistribution of mutation types:") - print(combined_df['mutationType'].value_counts()) - - print("\nNumber of mutations by study:") - print(combined_df['study_id'].value_counts()) - - # Save to CSV - combined_df.to_csv('all_aml_mutations.csv', index=False) - - -- query: "Example showing how to retrieve and analyze the molecular profiles available for multiple studies from cBioPortal. The code demonstrates how to fetch metadata on the molecular profiles, organize them into DataFrames, and analyze specific information about each profile like the types of profiles available such as RNA expression." - code: | - import requests - import pandas as pd - from typing import List, Dict - - def get_molecular_profiles(study_ids: List[str]) -> Dict[str, pd.DataFrame]: - """ - Retrieve molecular profiles for multiple studies from cBioPortal. - - Args: - study_ids: List of cBioPortal study IDs - - Returns: - Dictionary mapping study IDs to DataFrames containing their molecular profiles - """ - # Base URL for cBioPortal's web API - base_url = "https://www.cbioportal.org/api" - - # Dictionary to store results - results = {} - - print("Fetching molecular profiles...") - for study_id in study_ids: - print(f"\nStudy: {study_id}") - print("-" * 40) - - # Get molecular profiles for this study - response = requests.get(f"{base_url}/studies/{study_id}/molecular-profiles") - - if response.status_code == 200: - profiles = response.json() - - # Create a summary DataFrame - profile_data = [] - for profile in profiles: - profile_data.append({ - 'molecularProfileId': profile['molecularProfileId'], - 'name': profile['name'], - 'datatype': profile['datatype'], - 'molecularAlterationType': profile['molecularAlterationType'], - 'description': profile.get('description', 'N/A') - }) - - if profile_data: - df = pd.DataFrame(profile_data) - results[study_id] = df - print(f"Found {len(df)} molecular profiles") - - # Display summary of profile types - print("\nProfile types:") - type_summary = df.groupby('molecularAlterationType')['molecularProfileId'].count() - print(type_summary) - else: - print("No molecular profiles found") - results[study_id] = pd.DataFrame() - else: - print(f"Error accessing study: {response.status_code}") - results[study_id] = pd.DataFrame() - - return results - - # Example usage with AML studies - aml_studies = [ - 'aml_target_2018_pub', - 'aml_ohsu_2018', - 'aml_ohsu_2022', - 'laml_tcga', - 'laml_tcga_pan_can_atlas_2018', - 'aml_target_gdc', - 'mnm_washu_2016' - ] - - # Get molecular profiles for all studies - profiles_by_study = get_molecular_profiles(aml_studies) - - # Example of how to work with the results - get RNA expression profiles for first study - first_study = list(profiles_by_study.keys())[0] - if not profiles_by_study[first_study].empty: - rna_profiles = profiles_by_study[first_study][ - profiles_by_study[first_study]['molecularAlterationType'] == 'MRNA_EXPRESSION' - ] - print(f"\nRNA expression profiles for {first_study}:") - print(rna_profiles[['molecularProfileId', 'name', 'datatype']].to_string()) - - -- query: "Example of how to fetch RNA-seq z-scores for STAT5A and STAT5B across multiple AML studies" - notes: | - This example demonstrates: - 1. How to properly fetch sample IDs for each study - 2. How to use these sample IDs when requesting molecular data - 3. How to handle large requests by processing samples in chunks - 4. How to combine data from multiple studies into a single dataframe - 5. How to create both long and wide format versions of the data - - Key points: - - Must provide explicit sample IDs in the molecular data request - - Z-scores are pre-calculated by cBioPortal for each study - - Different studies may use different measurement types (TPM, RPKM, RSEM) - - Processing in chunks helps avoid timeouts with large datasets - code: | - import requests - import pandas as pd - import time - - # Configuration - BASE_URL = "https://www.cbioportal.org/api" - - # Define studies and their RNA-seq profiles - STUDY_PROFILES = { - 'aml_target_gdc': { - 'name': 'TARGET-AML (GDC)', - 'profile': 'aml_target_gdc_mrna_seq_tpm_Zscores', - 'type': 'TPM' - }, - 'aml_ohsu_2022': { - 'name': 'OHSU AML 2022', - 'profile': 'aml_ohsu_2022_mrna_median_Zscores', - 'type': 'RPKM' - } - # ... other studies ... - } - - # Define STAT5 genes with their Entrez IDs - STAT5_GENES = { - 6776: 'STAT5A', - 6777: 'STAT5B' - } - - # Function to get RNA-seq samples for a study - def get_rna_seq_samples(study_id): - """Get list of sample IDs that have RNA-seq data for a study.""" - url = f"{BASE_URL}/studies/{study_id}/sample-lists" - response = requests.get(url) - if response.status_code == 200: - sample_lists = response.json() - # Find RNA-seq sample list - rna_list = next((sl['sampleListId'] for sl in sample_lists - if sl['category'] == 'all_cases_with_mrna_rnaseq_data'), None) - if rna_list: - # Get sample IDs from the list - url = f"{BASE_URL}/sample-lists/{rna_list}/sample-ids" - response = requests.get(url) - if response.status_code == 200: - return response.json() - return None - - # Function to get expression data - def get_expression_data(profile_id, gene_ids, sample_ids): - """Fetch expression z-scores for specific genes and samples.""" - url = f"{BASE_URL}/molecular-profiles/{profile_id}/molecular-data/fetch" - # IMPORTANT: Must provide sample IDs explicitly - data = { - "sampleIds": sample_ids, # List of specific sample IDs - "entrezGeneIds": gene_ids # List of Entrez gene IDs - } - response = requests.post(url, json=data) - if response.status_code == 200: - return response.json() - return None - - # Process each study - all_study_data = [] - - for study_id, study_info in STUDY_PROFILES.items(): - print(f"\nProcessing {study_info['name']}...") - - # First get the sample IDs for the study - sample_ids = get_rna_seq_samples(study_id) - - if sample_ids: - print(f"Found {len(sample_ids)} RNA-seq samples") - - # Process samples in chunks to avoid large requests - chunk_size = 100 - study_data = [] - - for i in range(0, len(sample_ids), chunk_size): - chunk = sample_ids[i:i + chunk_size] - print(f"Fetching data for samples {i+1}-{i+len(chunk)}...") - - # Get expression data for this chunk of samples - expression_data = get_expression_data( - study_info['profile'], - list(STAT5_GENES.keys()), - chunk - ) - - if expression_data: - study_data.extend(expression_data) - time.sleep(0.2) # Small delay between requests - - if study_data: - # Convert to DataFrame - df = pd.DataFrame(study_data) - - # Add metadata - df['gene_symbol'] = df['entrezGeneId'].map(STAT5_GENES) - df['study_id'] = study_id - df['study_name'] = study_info['name'] - df['measurement_type'] = study_info['type'] - - # Select and rename columns - df = df[[ - 'study_id', 'study_name', 'measurement_type', - 'gene_symbol', 'sampleId', 'value' - ]] - df.columns = [ - 'study_id', 'study_name', 'measurement_type', - 'gene', 'sample_id', 'zscore' - ] - - all_study_data.append(df) - - # Combine all data - if all_study_data: - # Create long format - combined_df = pd.concat(all_study_data, ignore_index=True) - - # Create wide format (samples as rows, genes as columns) - combined_wide = combined_df.pivot_table( - index=['study_id', 'study_name', 'measurement_type', 'sample_id'], - columns='gene', - values='zscore' - ).reset_index() - - # Save both versions - stat5_all_studies = combined_df # Long format - stat5_all_studies_wide = combined_wide # Wide format - - # Example of the resulting data structure: - print("\nLong format example (first few rows):") - print(stat5_all_studies.head()) - - print("\nWide format example (first few rows):") - print(stat5_all_studies_wide.head()) diff --git a/src/biome/adhoc_data/datasources/cda/api.yaml b/src/biome/adhoc_data/datasources/cda/api.yaml index a2dab67..a4785a2 100644 --- a/src/biome/adhoc_data/datasources/cda/api.yaml +++ b/src/biome/adhoc_data/datasources/cda/api.yaml @@ -1,72 +1,118 @@ -name: "Cancer Data Aggregator" +description: "The Cancer Data Aggregator (CDA) is a service of the National Cancer\ + \ Institutes' (NCI) Cancer Research Data Commons. We pull metadata for thousands\ + \ of studies hosted at multiple data repositories across NCI, and make it available\ + \ for search from a single tool so researchers can more easily find and reuse existing\ + \ cancer research data. In between pulling and publishing, we thoroughly clean,\ + \ harmonize, and cross-reference the metadata so you can easily do things like find\ + \ subjects that have participated in multiple studies, discover data from a disease\ + \ that was originally described in different ways at each repository, and compile\ + \ all the data from your favorite program such as CPTAC - no matter where it ended\ + \ up.\n\n# Conceptual Overview\nYou can think of the CDA as a really, really enormous\ + \ spreadsheet full of data. To search this enormous spreadsheet, you'd want to select\ + \ columns that have data you're interested in, and then filter the rows to only\ + \ the values you care about.\n\nCDA data comes from six sources:\n- The Proteomic\ + \ Data Commons (PDC)\n- The Genomic Data Commons (GDC)\n- The Imaging Data Commons\ + \ (IDC)\n- The Cancer Data Service (CDS)\n- The Integrated Canine Data Commons (ICDC)\n\ + - The ISB Cancer Gateway in the Cloud (ISB-CGC)\n\nThe CDA makes this data searchable\ + \ in five main endpoints:\n- subject: A patient entity captures the study-independent\ + \ metadata for research subjects. Human research subjects are usually not traceable\ + \ to a particular person to protect the subjects privacy.\n- researchsubject: A\ + \ research subject is the entity of interest in a specific research study or project,\ + \ typically a human being or an animal, but can also be a device, group of humans\ + \ or animals, or a tissue sample. Human research subjects are usually not traceable\ + \ to a particular person to protect the subjects privacy. This entity plays the\ + \ role of the case_id in existing data. A subject who participates in 3 studies\ + \ will have 3 researchsubject IDs.\n- specimen: Any material taken as a sample from\ + \ a biological entity (living or dead), or from a physical object or the environment.\ + \ Specimens are usually collected as an example of their kind, often for use in\ + \ some investigation.\n- mutation: Molecular data about specific mutations, currently\ + \ limited to the TCGA-READ project from GDC.\n- file: A unit of data about subjects,\ + \ researchsubjects, specimens, or their associated information.\n\nand two endpoints\ + \ that offer deeper information about data in the researchsubject endpoint:\n- diagnosis:\ + \ A collection of characteristics that describe an abnormal condition of the body\ + \ as assessed at a point in time. May be used to capture information about neoplastic\ + \ and non-neoplastic conditions.\n- treatment: Represent medication administration\ + \ or other treatment types.\n\nIf the CDA works like a giant spreadsheet, the endpoints\ + \ are sets of specific columns that always go together. Any metadata field can be\ + \ searched from any endpoint, the only difference between search types is what type\ + \ of data is returned by default.\n\nIf you are looking to build a cohort of distinct\ + \ individuals who meet some criteria, you would search by subject, and the result\ + \ will be a table of information with one row per subject. If you are looking for\ + \ biosamples that can be ordered or a specific format of information (e.g. histological\ + \ slides) start with specimen. This search will return a table of information with\ + \ one row per biosample. If you want to build a cohort, but are particularly interested\ + \ in studies rather than the participants, search by researchsubject.\n\n# Data\ + \ Sources\n## Genomics Data Commons\nThe Genomic Data Commons (GDC) is a cancer\ + \ knowledge network that supports hosting, standardization, and analysis of genomic,\ + \ clinical, and biospecimen data from cancer research programs. The GDC harmonizes\ + \ raw sequencing data, identifies and applies state-of-the-art bioinformatics methods\ + \ for generating mutation calls, structural variants and other high-level data,\ + \ and provides scalable downloads and web-based analysis tools. Because of the personal\ + \ nature of genomic data, some genomic data in the GDC may be controlled access,\ + \ requiring eRA Commons authentication and dbGaP authorization to access the data.\n\ + \n## Proteomic Data Commons\nThe Proteomic Data Commons (PDC) was developed to advance\ + \ understanding of how proteins help to shape the risk, diagnosis, development,\ + \ progression, and treatment of cancer. In-depth analysis of proteomic data allows\ + \ the study of both how and why cancer develops and informs ways of tailoring treatment\ + \ for individual patients using precision medicine. All proteomic data in the PDC\ + \ are open access and, with appropriate attribution, can be included in publications.\n\ + \n## Imaging Data Commons\nNCI Imaging Data Commons (IDC) is a cloud-based repository\ + \ of publicly available cancer imaging data co-located with the analysis and exploration\ + \ tools and resources. IDC is a node within the broader NCI Cancer Research Data\ + \ Commons (CRDC) infrastructure that provides secure access to a large, comprehensive,\ + \ and expanding collection of cancer research data.\n\nAll data hosted by IDC is\ + \ available publicly. The current content of IDC is populated using the radiology\ + \ collections from The Cancer Imaging Archive (TCIA), as well as data collected\ + \ by other major NCI initiatives, such as TCGA, CPTAC, NLST and HTAN. IDC does not\ + \ perform de-identification of images but accepts data de-identified by TCIA or\ + \ other Data Coordinating Centers that are approved by NCI Security.\n\n## Cancer\ + \ Data Services\nThe CDS provides data storage and sharing capabilities for NCI-funded\ + \ studies that fall under the following categories: \u2022 Studies with data that\ + \ do not match an existing CRDC data commons \u2022 Studies with data that do not\ + \ fit current data type criteria and/or the minimum metadata standards for a CRDC\ + \ data commons.\n\nCDS currently hosts a variety of data types from NCI projects\ + \ such as the Human Tumor Atlas Network (HTAN), Division of Cancer Control and Population\ + \ Sciences (DCCPS), and Childhood Cancer Data Initiative (CCDI) as well as data\ + \ from independent research projects. The CDS is home to both open and controlled\ + \ access data.\n\n## Integrated Canine Data Commons\nThe Integrated Canine Data\ + \ Commons (ICDC) is a cloud-based repository of spontaneously-arising canine cancer\ + \ data. ICDC was established to further research on human cancers by enabling comparative\ + \ analysis with canine cancer. The data in the ICDC is sourced from multiple different\ + \ programs and projects; all focused on canine subjects. The data is harmonized\ + \ into an integrated data model and then made available to the research community.\n\ + \n## ISB Cancer Gateway in the Cloud\nThe ISB Cancer Gateway in the Cloud (ISB-CGC)\ + \ is one of three National Cancer Institute (NCI) Cloud Resources tasked with enabling\ + \ researchers to combine cancer data and cloud computation. The ISB-CGC cloud resource\ + \ hosts data from a variety of sources such as HTAN and TCGA, CPTAC, and TARGET\ + \ from the GDC and PDC in Google BigQuery columnar data tables. This includes file,\ + \ case, clinical, and open access derived data that can be accessed both programmatically\ + \ and through interactive web applications, eliminating the need to download and\ + \ store large data sets.\n\n## Data Standards Services (DSS)\nThe DSS provides us\ + \ with harmonized values mapped to the data sources above. In our current release,\ + \ DSS has provided values for: ethnicity, file_format, morphology, primary_diagnosis,\ + \ race, species, therapeutic_agent, source_material_type (cancer/normal), treatment_type,\ + \ and vital_status.\n" integration_type: api -description: | - The Cancer Data Aggregator (CDA) is a service of the National Cancer Institutes' (NCI) Cancer Research Data Commons. We pull metadata for thousands of studies hosted at multiple data repositories across NCI, and make it available for search from a single tool so researchers can more easily find and reuse existing cancer research data. In between pulling and publishing, we thoroughly clean, harmonize, and cross-reference the metadata so you can easily do things like find subjects that have participated in multiple studies, discover data from a disease that was originally described in different ways at each repository, and compile all the data from your favorite program such as CPTAC - no matter where it ended up. - - # Conceptual Overview - You can think of the CDA as a really, really enormous spreadsheet full of data. To search this enormous spreadsheet, you'd want to select columns that have data you're interested in, and then filter the rows to only the values you care about. - - CDA data comes from six sources: - - The Proteomic Data Commons (PDC) - - The Genomic Data Commons (GDC) - - The Imaging Data Commons (IDC) - - The Cancer Data Service (CDS) - - The Integrated Canine Data Commons (ICDC) - - The ISB Cancer Gateway in the Cloud (ISB-CGC) - - The CDA makes this data searchable in five main endpoints: - - subject: A patient entity captures the study-independent metadata for research subjects. Human research subjects are usually not traceable to a particular person to protect the subjects privacy. - - researchsubject: A research subject is the entity of interest in a specific research study or project, typically a human being or an animal, but can also be a device, group of humans or animals, or a tissue sample. Human research subjects are usually not traceable to a particular person to protect the subjects privacy. This entity plays the role of the case_id in existing data. A subject who participates in 3 studies will have 3 researchsubject IDs. - - specimen: Any material taken as a sample from a biological entity (living or dead), or from a physical object or the environment. Specimens are usually collected as an example of their kind, often for use in some investigation. - - mutation: Molecular data about specific mutations, currently limited to the TCGA-READ project from GDC. - - file: A unit of data about subjects, researchsubjects, specimens, or their associated information. - - and two endpoints that offer deeper information about data in the researchsubject endpoint: - - diagnosis: A collection of characteristics that describe an abnormal condition of the body as assessed at a point in time. May be used to capture information about neoplastic and non-neoplastic conditions. - - treatment: Represent medication administration or other treatment types. - - If the CDA works like a giant spreadsheet, the endpoints are sets of specific columns that always go together. Any metadata field can be searched from any endpoint, the only difference between search types is what type of data is returned by default. - - If you are looking to build a cohort of distinct individuals who meet some criteria, you would search by subject, and the result will be a table of information with one row per subject. If you are looking for biosamples that can be ordered or a specific format of information (e.g. histological slides) start with specimen. This search will return a table of information with one row per biosample. If you want to build a cohort, but are particularly interested in studies rather than the participants, search by researchsubject. - - # Data Sources - ## Genomics Data Commons - The Genomic Data Commons (GDC) is a cancer knowledge network that supports hosting, standardization, and analysis of genomic, clinical, and biospecimen data from cancer research programs. The GDC harmonizes raw sequencing data, identifies and applies state-of-the-art bioinformatics methods for generating mutation calls, structural variants and other high-level data, and provides scalable downloads and web-based analysis tools. Because of the personal nature of genomic data, some genomic data in the GDC may be controlled access, requiring eRA Commons authentication and dbGaP authorization to access the data. - - ## Proteomic Data Commons - The Proteomic Data Commons (PDC) was developed to advance understanding of how proteins help to shape the risk, diagnosis, development, progression, and treatment of cancer. In-depth analysis of proteomic data allows the study of both how and why cancer develops and informs ways of tailoring treatment for individual patients using precision medicine. All proteomic data in the PDC are open access and, with appropriate attribution, can be included in publications. - - ## Imaging Data Commons - NCI Imaging Data Commons (IDC) is a cloud-based repository of publicly available cancer imaging data co-located with the analysis and exploration tools and resources. IDC is a node within the broader NCI Cancer Research Data Commons (CRDC) infrastructure that provides secure access to a large, comprehensive, and expanding collection of cancer research data. - - All data hosted by IDC is available publicly. The current content of IDC is populated using the radiology collections from The Cancer Imaging Archive (TCIA), as well as data collected by other major NCI initiatives, such as TCGA, CPTAC, NLST and HTAN. IDC does not perform de-identification of images but accepts data de-identified by TCIA or other Data Coordinating Centers that are approved by NCI Security. - - ## Cancer Data Services - The CDS provides data storage and sharing capabilities for NCI-funded studies that fall under the following categories: • Studies with data that do not match an existing CRDC data commons • Studies with data that do not fit current data type criteria and/or the minimum metadata standards for a CRDC data commons. - - CDS currently hosts a variety of data types from NCI projects such as the Human Tumor Atlas Network (HTAN), Division of Cancer Control and Population Sciences (DCCPS), and Childhood Cancer Data Initiative (CCDI) as well as data from independent research projects. The CDS is home to both open and controlled access data. - - ## Integrated Canine Data Commons - The Integrated Canine Data Commons (ICDC) is a cloud-based repository of spontaneously-arising canine cancer data. ICDC was established to further research on human cancers by enabling comparative analysis with canine cancer. The data in the ICDC is sourced from multiple different programs and projects; all focused on canine subjects. The data is harmonized into an integrated data model and then made available to the research community. - - ## ISB Cancer Gateway in the Cloud - The ISB Cancer Gateway in the Cloud (ISB-CGC) is one of three National Cancer Institute (NCI) Cloud Resources tasked with enabling researchers to combine cancer data and cloud computation. The ISB-CGC cloud resource hosts data from a variety of sources such as HTAN and TCGA, CPTAC, and TARGET from the GDC and PDC in Google BigQuery columnar data tables. This includes file, case, clinical, and open access derived data that can be accessed both programmatically and through interactive web applications, eliminating the need to download and store large data sets. - - ## Data Standards Services (DSS) - The DSS provides us with harmonized values mapped to the data sources above. In our current release, DSS has provided values for: ethnicity, file_format, morphology, primary_diagnosis, race, species, therapeutic_agent, source_material_type (cancer/normal), treatment_type, and vital_status. -attachments: - raw_documentation: cda.yaml +name: Cancer Data Aggregator prompt: | - ${raw_documentation} - - # Additional Instructions: - - Typically metadata retrieved from CDA will locate the raw data files via Data Repository Service (DRS) URIs. - DRS URIs are not to the data itself, but will return metadata about the location of the data. - The location metadata (which should include the actual data download URL) can be retrieved by using the drs_uri_info tool if it is available, - or by querying "https://nci-crdc.datacommons.io/ga4gh/drs/v1/objects/" directly. - - All data in CDA should be publicly accessible without any special permissions or credentials. - - You should make use of the python requests library to interact with the API. - Note that the base URL of the API is 'https://cda.datacommons.cancer.gov/' + ${raw_documentation} + + # Additional Instructions: + + Typically metadata retrieved from CDA will locate the raw data files via Data Repository Service (DRS) URIs. + DRS URIs are not to the data itself, but will return metadata about the location of the data. + The location metadata (which should include the actual data download URL) can be retrieved by using the drs_uri_info tool if it is available, + or by querying "https://nci-crdc.datacommons.io/ga4gh/drs/v1/objects/" directly. + + All data in CDA should be publicly accessible without any special permissions or credentials. + + You should make use of the python requests library to interact with the API. + Note that the base URL of the API is 'https://cda.datacommons.cancer.gov/' +resources: + 5361ce45-01fb-4c16-aa22-31d5397a0bd1: + filepath: cda.yaml + integration: cancer_data_aggregator + name: raw_documentation + resource_id: 5361ce45-01fb-4c16-aa22-31d5397a0bd1 + resource_type: file +slug: cancer_data_aggregator diff --git a/src/biome/adhoc_data/datasources/cda/examples.yaml b/src/biome/adhoc_data/datasources/cda/examples.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/src/biome/adhoc_data/datasources/cdc_tracking_network/api.yaml b/src/biome/adhoc_data/datasources/cdc_tracking_network/api.yaml index 047028a..b61c9a2 100644 --- a/src/biome/adhoc_data/datasources/cdc_tracking_network/api.yaml +++ b/src/biome/adhoc_data/datasources/cdc_tracking_network/api.yaml @@ -1,38 +1,46 @@ -name: "CDC Tracking Network" -integration_type: api - description: | - The National Environmental Public Health Tracking Network (Tracking Network) - brings together health data and environmental data from national, state, county, - and city sources and provides supporting information to make the data easier - to understand. - The Tracking Network has data and information on environments and hazards, - health effects, and population health. - This resource includes childhood emergency department visits and hospitalizations at the county level. It provides: - - Age-stratified data including children under 18 - - County-level resolution - - Information on asthma, and other conditions - -attachments: - user_guide: user_guide.md - api_examples: api_examples.md - + The National Environmental Public Health Tracking Network (Tracking Network) + brings together health data and environmental data from national, state, county, + and city sources and provides supporting information to make the data easier + to understand. + The Tracking Network has data and information on environments and hazards, + health effects, and population health. + This resource includes childhood emergency department visits and hospitalizations at the county level. It provides: + - Age-stratified data including children under 18 + - County-level resolution + - Information on asthma, and other conditions +integration_type: api +name: CDC Tracking Network prompt: | - The Tracking Program collects, integrates, analyzes, and disseminates non-infectious disease, - environmental, and socio-economic data from various sources. - These include a collective of national, state, and local partners. - - This is an HTTP API, where the usage is free but rate-limiting is applied heavily when not using an API KEY. - You will need an API Key. It is available in the environment variable: - - apiToken: os.environ.get("API_CDC_TRACKING_NETWORK") - - To fully understand specifics about what data is or is not available, - you must run code against the API to see what features are available at - what level of geography. - - ${user_guide} - - Never simulate this API- if it is not available ask the user how to proceed. - - The list of content areas is available at: - ${api_examples} + The Tracking Program collects, integrates, analyzes, and disseminates non-infectious disease, + environmental, and socio-economic data from various sources. + These include a collective of national, state, and local partners. + + This is an HTTP API, where the usage is free but rate-limiting is applied heavily when not using an API KEY. + You will need an API Key. It is available in the environment variable: + - apiToken: os.environ.get("API_CDC_TRACKING_NETWORK") + + To fully understand specifics about what data is or is not available, + you must run code against the API to see what features are available at + what level of geography. + + ${user_guide} + + Never simulate this API- if it is not available ask the user how to proceed. + + The list of content areas is available at: + ${api_examples} +resources: + 5b62b4c6-536c-4a8d-8e4b-bd295ac8ed72: + filepath: api_examples.md + integration: cdc_tracking_network + name: api_examples + resource_id: 5b62b4c6-536c-4a8d-8e4b-bd295ac8ed72 + resource_type: file + 7c5e6353-7e1f-4639-91ee-9dc568887a69: + filepath: user_guide.md + integration: cdc_tracking_network + name: user_guide + resource_id: 7c5e6353-7e1f-4639-91ee-9dc568887a69 + resource_type: file +slug: cdc_tracking_network diff --git a/src/biome/adhoc_data/datasources/cdc_tracking_network/examples.yaml b/src/biome/adhoc_data/datasources/cdc_tracking_network/examples.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/src/biome/adhoc_data/datasources/census_acs/api.yaml b/src/biome/adhoc_data/datasources/census_acs/api.yaml index 2f8610f..d55c9c3 100644 --- a/src/biome/adhoc_data/datasources/census_acs/api.yaml +++ b/src/biome/adhoc_data/datasources/census_acs/api.yaml @@ -1,37 +1,72 @@ -name: "Census ACS (American Community Survey) and SF1" -integration_type: api - description: | The Census ACS (American Community Survey) and SF1 (Decennial Census) API returns data that has been collected from the Census Bureau, a database that contains information on the population of the United States. Use this to control for socioeconomic factors (or for other purposes). - -attachments: - web_documentation: census_web.md - sdk_documentation: census_sdk.md - -prompt: | - The American Community Survey (ACS) is an ongoing survey that provides data every - year—giving communities the current information they need to make important decisions. - The ACS covers a broad range of topics about social, economic, housing, and demographic - characteristics of the U.S. population. - - ${web_documentation} - - You will need an API Key to use the Census ACS API. - It is available in the environment variable: - - api_key: os.environ.get("API_CENSUS") - - You have an api key, so you have access to the census pypi package, and also direct access to the census web api. - Prefer the pypi package if you can, it's easier to use. - - Data ranges from 2005 to 2023. - - ${sdk_documentation} - - ## For 1-year datasets when loading as file (may or may not apply when using python libraries...) - ### Variable Changes - Variables, and the values they represent, may change over time. Use the file in `${DATASET_FILES_BASE_PATH}/census-acs/2023-1yr-api-changes.csv` as a guide for which variables have changed from the prior year for 2023 ACS 1-Year Detailed Tables, Data Profiles and Subject Tables. See below for a description of each change type. - No Change - The variable has not changed from the prior year (most variables). - Updated - That variable has changed from the prior year and a matching variable for the current year has been found. - No Match - The variable has changed from the prior year and no matching or comparable variable has been found. +integration_type: api +name: Census ACS (American Community Survey) and SF1 +prompt: "The American Community Survey (ACS) is an ongoing survey that provides data\ + \ every\nyear\u2014giving communities the current information they need to make\ + \ important decisions.\nThe ACS covers a broad range of topics about social, economic,\ + \ housing, and demographic\ncharacteristics of the U.S. population.\n\n${web_documentation}\n\ + \nYou will need an API Key to use the Census ACS API.\nIt is available in the environment\ + \ variable:\n- api_key: os.environ.get(\"API_CENSUS\")\n\nYou have an api key, so\ + \ you have access to the census pypi package, and also direct access to the census\ + \ web api.\nPrefer the pypi package if you can, it's easier to use.\n\nData ranges\ + \ from 2005 to 2023.\n\n${sdk_documentation}\n\n## For 1-year datasets when loading\ + \ as file (may or may not apply when using python libraries...)\n### Variable Changes\n\ + Variables, and the values they represent, may change over time. Use the file in\ + \ `${DATASET_FILES_BASE_PATH}/census-acs/2023-1yr-api-changes.csv` as a guide for\ + \ which variables have changed from the prior year for 2023 ACS 1-Year Detailed\ + \ Tables, Data Profiles and Subject Tables. See below for a description of each\ + \ change type.\nNo Change - The variable has not changed from the prior year (most\ + \ variables).\nUpdated - That variable has changed from the prior year and a matching\ + \ variable for the current year has been found.\nNo Match - The variable has changed\ + \ from the prior year and no matching or comparable variable has been found.\n" +resources: + 76813161-e3dd-400e-b1db-c6534c3fb484: + filepath: census_web.md + integration: census_acs_(american_community_survey)_and_sf1 + name: web_documentation + resource_id: 76813161-e3dd-400e-b1db-c6534c3fb484 + resource_type: file + 93d4da90-fe43-4121-8e1b-d9d677637143: + code: "import pandas as pd\nfrom census import Census\nfrom us import states\n\ + import matplotlib.pyplot as plt\n\nc = Census(os.environ.get(\"API_CENSUS\"\ + ))\n\n# Get ACS data for Harris County census tracts\nharris_ses = c.acs5.get(\n\ + \ ('NAME', 'B19013_001E', 'B17001_002E', 'B25064_001E', 'B01003_001E'),\n\ + \ {'for': 'tract:*', 'in': f'state:{states.TX.fips} county:201'}\n)\n\n#\ + \ Convert to DataFrame\nharris_ses_df = pd.DataFrame(harris_ses)\nharris_ses_df.columns\ + \ = ['Name', 'Median_Income', 'Poverty_Count', \n 'Median_Rent',\ + \ 'Population', 'state', 'county', 'tract']\n\n# Calculate poverty rate\nharris_ses_df['Poverty_Rate']\ + \ = harris_ses_df['Poverty_Count'] / harris_ses_df['Population'] * 100\n" + integration: census_acs_(american_community_survey)_and_sf1 + notes: null + query: Calculate the poverty rate in Harris County, Texas- Descriptive analysis + resource_id: 93d4da90-fe43-4121-8e1b-d9d677637143 + resource_type: example + 9a1dd209-4e65-49af-a5cc-2f9d73cd8d4b: + code: "import pandas as pd\nfrom census import Census\nfrom us import states\n\ + import matplotlib.pyplot as plt\n\nc = Census(os.environ.get(\"API_CENSUS\"\ + ))\n\n# Get ACS data for Harris County census tracts\nharris_ses = c.acs5.get(\n\ + \ ('NAME', 'B19013_001E', 'B17001_002E', 'B25064_001E', 'B01003_001E'),\n\ + \ {'for': 'tract:*', 'in': f'state:{states.TX.fips} county:201'}\n)\n\n#\ + \ Convert to DataFrame\nharris_ses_df = pd.DataFrame(harris_ses)\nharris_ses_df.columns\ + \ = ['Name', 'Median_Income', 'Poverty_Count', \n 'Median_Rent',\ + \ 'Population', 'state', 'county', 'tract']\n\n# Group census tracts by income\ + \ quartiles\nharris_ses_df['Income_Quartile'] = pd.qcut(harris_ses_df['Median_Income'],\ + \ 4, labels=False)\n\n# Analyze environmental exposures by income quartile\n\ + exposure_by_ses = pd.merge(environmental_data, harris_ses_df, on='tract')\n\n\ + # Calculate average exposures by income quartile\nquartile_summary = exposure_by_ses.groupby('Income_Quartile')['PM25_Annual',\ + \ 'Ozone_Days_Exceeded', 'TRI_Releases'].mean()" + integration: census_acs_(american_community_survey)_and_sf1 + notes: null + query: Calculate the poverty rate in Harris County, Texas- Stratified analysis + resource_id: 9a1dd209-4e65-49af-a5cc-2f9d73cd8d4b + resource_type: example + cb8c091d-ff82-46d7-bdea-893aa5b0c6c5: + filepath: census_sdk.md + integration: census_acs_(american_community_survey)_and_sf1 + name: sdk_documentation + resource_id: cb8c091d-ff82-46d7-bdea-893aa5b0c6c5 + resource_type: file +slug: census_acs_(american_community_survey)_and_sf1 diff --git a/src/biome/adhoc_data/datasources/census_acs/examples.yaml b/src/biome/adhoc_data/datasources/census_acs/examples.yaml deleted file mode 100644 index e25d9a9..0000000 --- a/src/biome/adhoc_data/datasources/census_acs/examples.yaml +++ /dev/null @@ -1,52 +0,0 @@ - -- query: "Calculate the poverty rate in Harris County, Texas- Descriptive analysis" - code: | - import pandas as pd - from census import Census - from us import states - import matplotlib.pyplot as plt - - c = Census(os.environ.get("API_CENSUS")) - - # Get ACS data for Harris County census tracts - harris_ses = c.acs5.get( - ('NAME', 'B19013_001E', 'B17001_002E', 'B25064_001E', 'B01003_001E'), - {'for': 'tract:*', 'in': f'state:{states.TX.fips} county:201'} - ) - - # Convert to DataFrame - harris_ses_df = pd.DataFrame(harris_ses) - harris_ses_df.columns = ['Name', 'Median_Income', 'Poverty_Count', - 'Median_Rent', 'Population', 'state', 'county', 'tract'] - - # Calculate poverty rate - harris_ses_df['Poverty_Rate'] = harris_ses_df['Poverty_Count'] / harris_ses_df['Population'] * 100 - -- query: "Calculate the poverty rate in Harris County, Texas- Stratified analysis" - code: | - import pandas as pd - from census import Census - from us import states - import matplotlib.pyplot as plt - - c = Census(os.environ.get("API_CENSUS")) - - # Get ACS data for Harris County census tracts - harris_ses = c.acs5.get( - ('NAME', 'B19013_001E', 'B17001_002E', 'B25064_001E', 'B01003_001E'), - {'for': 'tract:*', 'in': f'state:{states.TX.fips} county:201'} - ) - - # Convert to DataFrame - harris_ses_df = pd.DataFrame(harris_ses) - harris_ses_df.columns = ['Name', 'Median_Income', 'Poverty_Count', - 'Median_Rent', 'Population', 'state', 'county', 'tract'] - - # Group census tracts by income quartiles - harris_ses_df['Income_Quartile'] = pd.qcut(harris_ses_df['Median_Income'], 4, labels=False) - - # Analyze environmental exposures by income quartile - exposure_by_ses = pd.merge(environmental_data, harris_ses_df, on='tract') - - # Calculate average exposures by income quartile - quartile_summary = exposure_by_ses.groupby('Income_Quartile')['PM25_Annual', 'Ozone_Days_Exceeded', 'TRI_Releases'].mean() \ No newline at end of file diff --git a/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml b/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml index 0ce30e0..8b9d5fb 100644 --- a/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml +++ b/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml @@ -1,31 +1,46 @@ -name: "CHIS California Asthma" -integration_type: dataset - description: | This dataset contains current asthma prevalence, the estimated percentage of Californians who have ever been diagnosed with asthma by a health care provider AND report they still have asthma and/or had an asthma episode or attack within the past 12 months, statewide and by county. The data are stratified by age group (all ages, 0-17, 18+, 0-4, 5-17, 18-64, 65+) and reported for 2-year periods. - +integration_type: dataset +name: CHIS California Asthma prompt: | - You have access to a file-based California CHIS Data-Current Asthma Prevalence by County 2015-2022 file, - located at `${DATASET_FILES_BASE_PATH}/chis_california_asthma/current-asthma-prevalence-by-county-2015_2022.csv`. + You have access to a file-based California CHIS Data-Current Asthma Prevalence by County 2015-2022 file, + located at `${DATASET_FILES_BASE_PATH}/chis_california_asthma/current-asthma-prevalence-by-county-2015_2022.csv`. - open it like so: - ```python - import pandas as pd - df = pd.read_csv('${DATASET_FILES_BASE_PATH}/chis_california_asthma/current-asthma-prevalence-by-county-2015_2022.csv', encoding='latin1') - ``` + open it like so: + ```python + import pandas as pd + df = pd.read_csv('${DATASET_FILES_BASE_PATH}/chis_california_asthma/current-asthma-prevalence-by-county-2015_2022.csv', encoding='latin1') + ``` - This dataset contains current asthma prevalence, the estimated percentage of - Californians who have ever been diagnosed with asthma by a health care provider - AND report they still have asthma and/or had an asthma episode or attack within - the past 12 months, statewide and by county. The data are stratified by age group - (all ages, 0-17, 18+, 0-4, 5-17, 18-64, 65+) and reported for 2-year periods. + This dataset contains current asthma prevalence, the estimated percentage of + Californians who have ever been diagnosed with asthma by a health care provider + AND report they still have asthma and/or had an asthma episode or attack within + the past 12 months, statewide and by county. The data are stratified by age group + (all ages, 0-17, 18+, 0-4, 5-17, 18-64, 65+) and reported for 2-year periods. - The columns are: "COUNTY", "YEARS", "STRATA", "AGE GROUP", "CURRENT PREVALENCE", "95% CONFIDENCE INTERVAL", "COUNTIES GROUPED", "COMMENT" + The columns are: "COUNTY", "YEARS", "STRATA", "AGE GROUP", "CURRENT PREVALENCE", "95% CONFIDENCE INTERVAL", "COUNTIES GROUPED", "COMMENT" - "Agre Group" possible values are: "All Ages", "0-17 years", "18+ years", "0-4 years", "5-17 years", "18-64 years", "65+ years" + "Agre Group" possible values are: "All Ages", "0-17 years", "18+ years", "0-4 years", "5-17 years", "18-64 years", "65+ years" - You can use this API to answer questions on the current asthma prevalence in California by county and age group. + You can use this API to answer questions on the current asthma prevalence in California by county and age group. +resources: + ca7ff323-7bef-489d-9d2c-760628f40825: + code: "import pandas as pd\nimport os\n\n# Define the file path using the correct\ + \ filename\nfile_path = os.path.join('{{DATASET_FILES_BASE_PATH}}', 'chis_california_asthma',\ + \ 'current-asthma-prevalence-by-county-2015_2022.csv')\n\ndf = pd.read_csv(file_path,\ + \ encoding='latin1') # latin1 encoding has worked before, not specifying it\ + \ hasn't\n\n# Check the column names to make sure we're using the right ones\n\ + print(\"Column names:\", df.columns.tolist())\n\n# Display a sample of the data\ + \ to understand its structure\nprint(\"\\nSample data:\")\nprint(df.head(2))\n\ + \ " + integration: chis_california_asthma + notes: null + query: Open the california chis asthma dataset and display columns/first 2 rows + to understand the structure + resource_id: ca7ff323-7bef-489d-9d2c-760628f40825 + resource_type: example +slug: chis_california_asthma diff --git a/src/biome/adhoc_data/datasources/chis_california_asthma/examples.yaml b/src/biome/adhoc_data/datasources/chis_california_asthma/examples.yaml deleted file mode 100644 index 67ba5fa..0000000 --- a/src/biome/adhoc_data/datasources/chis_california_asthma/examples.yaml +++ /dev/null @@ -1,17 +0,0 @@ -- query: "Open the california chis asthma dataset and display columns/first 2 rows to understand the structure" - code: | - import pandas as pd - import os - - # Define the file path using the correct filename - file_path = os.path.join('{{DATASET_FILES_BASE_PATH}}', 'chis_california_asthma', 'current-asthma-prevalence-by-county-2015_2022.csv') - - df = pd.read_csv(file_path, encoding='latin1') # latin1 encoding has worked before, not specifying it hasn't - - # Check the column names to make sure we're using the right ones - print("Column names:", df.columns.tolist()) - - # Display a sample of the data to understand its structure - print("\nSample data:") - print(df.head(2)) - \ No newline at end of file diff --git a/src/biome/adhoc_data/datasources/epa_tri/api.yaml b/src/biome/adhoc_data/datasources/epa_tri/api.yaml index 5a47df7..f84a8a6 100644 --- a/src/biome/adhoc_data/datasources/epa_tri/api.yaml +++ b/src/biome/adhoc_data/datasources/epa_tri/api.yaml @@ -1,99 +1,455 @@ -name: "EPA Toxic Release Inventory (TRI)" -integration_type: dataset - description: | The Toxics Release Inventory (TRI) is a resource for learning about toxic chemical releases and pollution prevention activities reported by industrial and federal facilities. TRI data support informed decision-making by communities, government agencies, companies, and others. - +integration_type: dataset +name: EPA Toxic Release Inventory (TRI) prompt: | - You have access to a file-based EPA Toxic Release Inventory (TRI) data, from 2014 to 2023, at the `${DATASET_FILES_BASE_PATH}/epa-tri/EPA_TRI_Toxics_2014_2023.csv` file. - The report is just one file- there's a year column, and then a list of chemicals released, and the amount released. - It also contains geospatial information such as latitude, longitude, state/zip, and even EPA region. - - # Overview of the EPA TRI Dataset - TRI tracks the waste management of certain toxic chemicals that may pose a threat to human health and the environment. U.S. facilities in different industry sectors must report annually how much of each chemical they release into the environment and/or managed through recycling, energy recovery and treatment, as well as any practices implemented to prevent or reduce the generation of chemical waste. - - Contains information about toxic chemical releases reported by industrial and federal facilities; key features of this dataset include: - - Facility information: Names, IDs, geographic coordinates, and locations - - Chemical data: Chemical names and release amounts (in pounds) - - Geographic information: Latitude, longitude, state, county, EPA region, ZIP codes, and Census Block Groups - - Risk metrics: RSEI (Risk-Screening Environmental Indicators) Hazard scores - - Temporal coverage: 10 years of data (2014-2023) - - In general, chemicals covered by the TRI Program are those that cause: - - Cancer or other chronic human health effects - - Significant adverse acute human health effects - - Significant adverse environmental effects - - There are currently 799 individually listed chemicals and 33 chemical categories covered by the TRI Program. Facilities that manufacture, process or otherwise use these chemicals in amounts above established levels must submit annual reporting forms for each chemical. Note that the TRI chemical list doesn't include all toxic chemicals used in the U.S. - - The Toxics Release Inventory (TRI) is a resource for learning about toxic chemical releases and pollution prevention activities reported by industrial and federal facilities. TRI data support informed decision-making by communities, government agencies, companies, and others. - - ## Data Dictionary: - - "TRI Facility Name": Name of the facility reporting chemical releases - - "TRI Facility ID": Unique identifier for the facility - - "Year": Reporting year (2014-2023) - - "Census Block Group": Geographic identifier for census block group - - "ZIP Code": Postal code of the facility - - "City": City where the facility is located - - "County": County where the facility is located- format: "COUNTY-NAME-ONLY, STATE 2-DIGIT CODE" (value in all caps) - - "State": State where the facility is located - - "EPA Region": EPA administrative region (numbered 1-10) - - "Latitude"/"Longitude": Geographic coordinates of the facility - - "Chemical": Name of the toxic chemical being reported - - "Releases (lb)": Total amount of chemical released to the environment in pounds - - "Waste Managed (lb)": Total amount of chemical waste managed in pounds - - "RSEI Hazard": Risk-Screening Environmental Indicators score, which measures the relative hazard of chemicals - - ## Working with the Data: - 1. When loading the data, ensure you specify the index_col=False parameter, since pandas is inferring the wrong index column and shifting the data without it: - ```python - import pandas as pd - tri_data = pd.read_csv('${DATASET_FILES_BASE_PATH}/epa-tri/EPA_TRI_Toxics_2014_2023.csv', index_col=False) - ``` - - 2. Common data analysis tasks: - - Filter by year: `tri_data[tri_data['Year'] == 2023]` - - Filter by chemical: `tri_data[tri_data['Chemical'] == 'Lead']` - - Filter by state: `tri_data[tri_data['State'] == 'California']` - - Find top polluters: `tri_data.sort_values('Releases (lb)', ascending=False).head(10)` - - Analyze trends over time: `tri_data.groupby('Year')['Releases (lb)'].sum().plot()` - - 3. Geospatial analysis: - - Create a map of facilities: Use latitude/longitude with libraries like folium or geopandas - - Regional analysis: Group by EPA Region or State to compare pollution levels - - 4. Data quality considerations: - - Check for missing values: `tri_data.isnull().sum()` - - Some chemicals may have different reporting thresholds - - RSEI Hazard scores provide context on relative toxicity beyond just release amounts - - Since this dataset contains a column for EPA Region, we can use this when relating to other more anonymous data sources - that only use these properties- but do store latitude/longitude to correlate with the other data sources that have a higher resolution. - - When checking for respiratory chemicals, always include the CAS number, not just the name- either that or check that the - non-CAS-included name you use is a substring of the value in the Chemical column when trying to make a match. - Example Chemical column values: Manganese compounds (N450), Ammonia (7664-41-7), Nitrate compounds (water dissociable) (N511) - - # Web Documentation - - ## What industries are included in the TRI? - Facilities that report to TRI are typically larger facilities involved in manufacturing, metal mining, electric power generation, chemical manufacturing and hazardous waste treatment. Not all industry sectors are covered by the TRI Program, and not all facilities in covered sectors are required to report to TRI. - - The TRI Program is also different because the data it collects are: - - - annual, collected each July and made publicly available online; - - multimedia, reflecting chemical emissions to air, water and land; and - - broad, encompassing source reduction and other pollution prevention practices. - - Numerous EPA programs use TRI data and information to: - - - provide a more complete picture of environmental performance at facilities and corporations; - - identify facilities that are potentially out of compliance with regulations or operating permits; - - support technical analysis for regulation; - - improve data quality across EPA; and - - develop program priorities and projects. - - Under various environmental laws (identified in the figure below), other EPA programs also collect information about regulated chemicals. Users looking for information not available in the TRI can check the databases associated with these other programs. The Clean Air Act's National Emissions Inventory (NEI), for example, can be used to find estimates of air releases for facilities that do not report to the TRI or for mobile sources such as cars, which are not covered by the TRI. + You have access to a file-based EPA Toxic Release Inventory (TRI) data, from 2014 to 2023, at the `${DATASET_FILES_BASE_PATH}/epa-tri/EPA_TRI_Toxics_2014_2023.csv` file. + The report is just one file- there's a year column, and then a list of chemicals released, and the amount released. + It also contains geospatial information such as latitude, longitude, state/zip, and even EPA region. + + # Overview of the EPA TRI Dataset + TRI tracks the waste management of certain toxic chemicals that may pose a threat to human health and the environment. U.S. facilities in different industry sectors must report annually how much of each chemical they release into the environment and/or managed through recycling, energy recovery and treatment, as well as any practices implemented to prevent or reduce the generation of chemical waste. + + Contains information about toxic chemical releases reported by industrial and federal facilities; key features of this dataset include: + - Facility information: Names, IDs, geographic coordinates, and locations + - Chemical data: Chemical names and release amounts (in pounds) + - Geographic information: Latitude, longitude, state, county, EPA region, ZIP codes, and Census Block Groups + - Risk metrics: RSEI (Risk-Screening Environmental Indicators) Hazard scores + - Temporal coverage: 10 years of data (2014-2023) + + In general, chemicals covered by the TRI Program are those that cause: + - Cancer or other chronic human health effects + - Significant adverse acute human health effects + - Significant adverse environmental effects + + There are currently 799 individually listed chemicals and 33 chemical categories covered by the TRI Program. Facilities that manufacture, process or otherwise use these chemicals in amounts above established levels must submit annual reporting forms for each chemical. Note that the TRI chemical list doesn't include all toxic chemicals used in the U.S. + + The Toxics Release Inventory (TRI) is a resource for learning about toxic chemical releases and pollution prevention activities reported by industrial and federal facilities. TRI data support informed decision-making by communities, government agencies, companies, and others. + + ## Data Dictionary: + - "TRI Facility Name": Name of the facility reporting chemical releases + - "TRI Facility ID": Unique identifier for the facility + - "Year": Reporting year (2014-2023) + - "Census Block Group": Geographic identifier for census block group + - "ZIP Code": Postal code of the facility + - "City": City where the facility is located + - "County": County where the facility is located- format: "COUNTY-NAME-ONLY, STATE 2-DIGIT CODE" (value in all caps) + - "State": State where the facility is located + - "EPA Region": EPA administrative region (numbered 1-10) + - "Latitude"/"Longitude": Geographic coordinates of the facility + - "Chemical": Name of the toxic chemical being reported + - "Releases (lb)": Total amount of chemical released to the environment in pounds + - "Waste Managed (lb)": Total amount of chemical waste managed in pounds + - "RSEI Hazard": Risk-Screening Environmental Indicators score, which measures the relative hazard of chemicals + + ## Working with the Data: + 1. When loading the data, ensure you specify the index_col=False parameter, since pandas is inferring the wrong index column and shifting the data without it: + ```python + import pandas as pd + tri_data = pd.read_csv('${DATASET_FILES_BASE_PATH}/epa-tri/EPA_TRI_Toxics_2014_2023.csv', index_col=False) + ``` + + 2. Common data analysis tasks: + - Filter by year: `tri_data[tri_data['Year'] == 2023]` + - Filter by chemical: `tri_data[tri_data['Chemical'] == 'Lead']` + - Filter by state: `tri_data[tri_data['State'] == 'California']` + - Find top polluters: `tri_data.sort_values('Releases (lb)', ascending=False).head(10)` + - Analyze trends over time: `tri_data.groupby('Year')['Releases (lb)'].sum().plot()` + + 3. Geospatial analysis: + - Create a map of facilities: Use latitude/longitude with libraries like folium or geopandas + - Regional analysis: Group by EPA Region or State to compare pollution levels + + 4. Data quality considerations: + - Check for missing values: `tri_data.isnull().sum()` + - Some chemicals may have different reporting thresholds + - RSEI Hazard scores provide context on relative toxicity beyond just release amounts + + Since this dataset contains a column for EPA Region, we can use this when relating to other more anonymous data sources + that only use these properties- but do store latitude/longitude to correlate with the other data sources that have a higher resolution. + + When checking for respiratory chemicals, always include the CAS number, not just the name- either that or check that the + non-CAS-included name you use is a substring of the value in the Chemical column when trying to make a match. + Example Chemical column values: Manganese compounds (N450), Ammonia (7664-41-7), Nitrate compounds (water dissociable) (N511) + + # Web Documentation + + ## What industries are included in the TRI? + Facilities that report to TRI are typically larger facilities involved in manufacturing, metal mining, electric power generation, chemical manufacturing and hazardous waste treatment. Not all industry sectors are covered by the TRI Program, and not all facilities in covered sectors are required to report to TRI. + + The TRI Program is also different because the data it collects are: + + - annual, collected each July and made publicly available online; + - multimedia, reflecting chemical emissions to air, water and land; and + - broad, encompassing source reduction and other pollution prevention practices. + + Numerous EPA programs use TRI data and information to: + + - provide a more complete picture of environmental performance at facilities and corporations; + - identify facilities that are potentially out of compliance with regulations or operating permits; + - support technical analysis for regulation; + - improve data quality across EPA; and + - develop program priorities and projects. + + Under various environmental laws (identified in the figure below), other EPA programs also collect information about regulated chemicals. Users looking for information not available in the TRI can check the databases associated with these other programs. The Clean Air Act's National Emissions Inventory (NEI), for example, can be used to find estimates of air releases for facilities that do not report to the TRI or for mobile sources such as cars, which are not covered by the TRI. +resources: + 125d85a4-0a2a-4306-b0d3-cc3c9451bb8e: + code: "import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\n\ + \n# Load and prepare data (assuming tri_data is already loaded and cleaned)\n\ + file_path = '{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv'\n\ + tri_data = pd.read_csv(file_path, low_memory=False)\n\n# Fix the numeric columns\ + \ that might be stored as strings with commas\nfor col in ['Releases (lb)',\ + \ 'Waste Managed (lb)', 'RSEI Hazard']:\n if tri_data[col].dtype == 'object':\n\ + \ # Remove commas and convert to float\n tri_data[col] = tri_data[col].str.replace(',',\ + \ '').astype(float)\n\n# Filter for a specific county\ncounty_data = tri_data[(tri_data['County']\ + \ == 'HARRIS, TX') | \n (tri_data['County'] == 'HARRIS COUNTY,\ + \ TX')]\n\n# Analyze data by ZIP code\nzip_summary = county_data.groupby('ZIP\ + \ Code').agg({\n 'TRI Facility Name': 'nunique',\n 'Releases (lb)': 'sum',\n\ + \ 'RSEI Hazard': 'sum'\n}).reset_index()\n\nzip_summary.columns = ['ZIP Code',\ + \ 'Number of Facilities', 'Total Releases (lb)', 'Total RSEI Hazard']\nzip_summary\ + \ = zip_summary.sort_values('Total Releases (lb)', ascending=False)\n\nprint(\"\ + Top 10 ZIP Codes by Total Releases:\")\nfor i, (zipcode, facilities, releases,\ + \ hazard) in enumerate(zip_summary.head(10).values, 1):\n print(f\"{i}. ZIP\ + \ {zipcode}: {facilities} facilities, {releases:,.2f} lb, RSEI Hazard: {hazard:,.2f}\"\ + )\n\n# Create a bar chart of top 10 ZIP codes by releases\nplt.figure(figsize=(12,\ + \ 6))\ntop_zips = zip_summary.head(10)\nbars = plt.bar(top_zips['ZIP Code'].astype(str),\ + \ top_zips['Total Releases (lb)'] / 1e6)\nplt.title('Top 10 ZIP Codes by Total\ + \ Toxic Releases')\nplt.xlabel('ZIP Code')\nplt.ylabel('Total Releases (Million\ + \ lb)')\nplt.xticks(rotation=45)\nplt.grid(True, axis='y', linestyle='--', alpha=0.7)\n\ + \n# Add value labels on top of bars\nfor bar in bars:\n height = bar.get_height()\n\ + \ plt.text(bar.get_x() + bar.get_width()/2., height + 0.5,\n f'{height:.1f}M',\ + \ ha='center', va='bottom')\n\nplt.tight_layout()\nplt.savefig('top_zips_by_releases.png')\n\ + plt.show()\n" + integration: epa_toxic_release_inventory_(tri) + notes: null + query: |- + Analyzing and visualizing toxic releases by ZIP code. This example shows how to aggregate TRI data by geographic units (ZIP codes), calculate total releases and hazard scores, and create a bar chart visualization of the top areas with highest toxic releases. + resource_id: 125d85a4-0a2a-4306-b0d3-cc3c9451bb8e + resource_type: example + 25907f4c-47dc-4c98-8ca4-584e6cd8556c: + code: | + import pandas as pd + + # Always the TRI csv with index_col=False param, since pandas is inferring the wrong index column and shifting the data without it + df = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv', dtype=str, index_col=False) + + # Filter for any entries containing 'Orange' in the County column + # sample name without the county suffix, all uppercase and with correct spacing + # In general, check contains/case=False instead of exact matches + orange_county = df[df['County'].str.contains('ORANGE, FL', case=False, na=False)] + + # Display results + print(f"Found {len(orange_county)} toxic release records containing 'ORANGE, FL' in the County name.") + print("\nSample of the data:") + print(orange_county.head()) + integration: epa_toxic_release_inventory_(tri) + notes: null + query: Open the TRI data file with index_col=False, since otherwise columns are + shifted, and find data for Orange County, FL. + resource_id: 25907f4c-47dc-4c98-8ca4-584e6cd8556c + resource_type: example + 42c5c855-df3e-4b20-9144-5083a691ce5f: + code: | + import pandas as pd + + # Load the TRI data + tri_data = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv', index_col=False) + + # Define respiratory/asthma related chemicals + # Updated to match the actual format in the dataset (with CAS numbers) + respiratory_chemicals = [ + 'Ammonia (7664-41-7)', + 'Chlorine (7782-50-5)', + 'Sulfuric acid (7664-93-9)', + 'Hydrochloric acid (7647-01-0)', + 'Nitric acid (7697-37-2)', + 'Ozone', # Keep checking if this has a CAS number format + 'Toluene (108-88-3)', + 'Benzene (71-43-2)', + 'Formaldehyde (50-00-0)', + 'Methanol (67-56-1)' + ] + + # Filter data for: + # - Year 2022 + # - Harris County (case-insensitive match) + # - Respiratory chemicals + respiratory_incidents = tri_data[ + (tri_data['Year'] == 2022) & + (tri_data['County'].str.upper() == 'HARRIS, TX') & + (tri_data['Chemical'].isin(respiratory_chemicals)) + ] + + # Convert 'Releases (lb)' to numeric, coercing errors to NaN since ascending= on sort_values messes with it + respiratory_incidents['Releases (lb)'] = pd.to_numeric(respiratory_incidents['Releases (lb)'], errors='coerce') + + # Sort by release amount + respiratory_incidents = respiratory_incidents.sort_values('Releases (lb)', ascending=False) + + # Select relevant columns + result = respiratory_incidents[[ + 'TRI Facility Name', + 'Chemical', + 'Releases (lb)', + 'RSEI Hazard', + 'Latitude', + 'Longitude' + ]] + + result + integration: epa_toxic_release_inventory_(tri) + notes: null + query: |- + Capture any asthma-respiratory related hazard incident from the TRI- EPA's Toxic Release Inventory data- you have access to this api. FOr now focus on problems in 2022, in the county of Harris, TX. + resource_id: 42c5c855-df3e-4b20-9144-5083a691ce5f + resource_type: example + 494e3a73-da61-4132-a169-56880ce5ea60: + code: "import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\n\ + \n# Load and prepare data (assuming tri_data is already loaded and cleaned)\n\ + file_path = '{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv'\n\ + tri_data = pd.read_csv(file_path, low_memory=False)\n\n# Fix the numeric columns\ + \ that might be stored as strings with commas\nfor col in ['Releases (lb)',\ + \ 'Waste Managed (lb)', 'RSEI Hazard']:\n if tri_data[col].dtype == 'object':\n\ + \ # Remove commas and convert to float\n tri_data[col] = tri_data[col].str.replace(',',\ + \ '').astype(float)\n\n# Filter for a specific county\ncounty_data = tri_data[(tri_data['County']\ + \ == 'HARRIS, TX') | \n (tri_data['County'] == 'HARRIS COUNTY,\ + \ TX')]\n\n# List of chemicals known to potentially trigger or worsen a specific\ + \ health condition (e.g., asthma)\ncondition_related_chemicals = [\n 'Ammonia',\ + \ 'Chlorine', 'Formaldehyde', 'Sulfuric acid', 'Hydrochloric acid',\n 'Nitrogen\ + \ oxides', 'Sulfur dioxide', 'Particulate matter', 'Isocyanates',\n 'Toluene\ + \ diisocyanate', 'Phthalates', 'Volatile organic compounds', 'VOCs',\n 'Ozone',\ + \ 'Acrolein', 'Benzene', 'Styrene', 'Xylene', 'Toluene'\n]\n\n# Create a function\ + \ to check if a chemical is related to the health condition\ndef is_condition_related(chemical_name):\n\ + \ for chem in condition_related_chemicals:\n if chem.lower() in chemical_name.lower():\n\ + \ return True\n return False\n\n# Add a column to identify condition-related\ + \ chemicals\ncounty_data['Condition_Related'] = county_data['Chemical'].apply(is_condition_related)\n\ + \n# Calculate total condition-related releases\ncondition_releases = county_data[county_data['Condition_Related']]['Releases\ + \ (lb)'].sum()\ntotal_releases = county_data['Releases (lb)'].sum()\ncondition_percentage\ + \ = (condition_releases / total_releases) * 100\n\nprint(f\"Total toxic releases:\ + \ {total_releases:,.2f} lb\")\nprint(f\"Condition-related chemical releases:\ + \ {condition_releases:,.2f} lb\")\nprint(f\"Percentage of condition-related\ + \ releases: {condition_percentage:.2f}%\")\n\n# Analyze trends over time for\ + \ condition-related chemicals\nyearly_condition = county_data[county_data['Condition_Related']].groupby('Year')['Releases\ + \ (lb)'].sum()\nyearly_total = county_data.groupby('Year')['Releases (lb)'].sum()\n\ + yearly_percentage = (yearly_condition / yearly_total) * 100\n\n# Create a time\ + \ series plot\nplt.figure(figsize=(12, 6))\nplt.plot(yearly_condition.index,\ + \ yearly_condition.values / 1e6, 'o-', color='red', \n label='Condition-related\ + \ Releases')\nplt.plot(yearly_total.index, yearly_total.values / 1e6, 'o-',\ + \ color='blue', \n label='Total Releases')\nplt.title('Yearly Trends\ + \ in Toxic Releases')\nplt.xlabel('Year')\nplt.ylabel('Releases (Million lb)')\n\ + plt.legend()\nplt.grid(True, linestyle='--', alpha=0.7)\nplt.tight_layout()\n\ + plt.savefig('yearly_condition_trends.png')\nplt.show()\n" + integration: epa_toxic_release_inventory_(tri) + notes: null + query: |- + Identifying and analyzing chemicals related to specific health conditions. This example demonstrates how to filter TRI data for chemicals associated with a particular health condition (like asthma), calculate their proportion of total releases, and analyze trends over time. This approach can be adapted for various health conditions by modifying the list of relevant chemicals. + resource_id: 494e3a73-da61-4132-a169-56880ce5ea60 + resource_type: example + 7dcb643d-1565-415e-99c7-04c0c48e6366: + code: "import pandas as pd\nimport folium\nfrom folium.plugins import MarkerCluster\n\ + import numpy as np\n\n# Load and prepare data (assuming tri_data is already\ + \ loaded and cleaned)\nfile_path = '{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv'\n\ + tri_data = pd.read_csv(file_path, low_memory=False)\n\n# Fix the numeric columns\ + \ that might be stored as strings with commas\nfor col in ['Releases (lb)',\ + \ 'Waste Managed (lb)', 'RSEI Hazard']:\n if tri_data[col].dtype == 'object':\n\ + \ # Remove commas and convert to float\n tri_data[col] = tri_data[col].str.replace(',',\ + \ '').astype(float)\n\n# Filter for a specific county\ncounty_data = tri_data[(tri_data['County']\ + \ == 'HARRIS, TX') | \n (tri_data['County'] == 'HARRIS COUNTY,\ + \ TX')]\n\n# Create a summary of facilities\nfacility_summary = county_data.groupby(['TRI\ + \ Facility Name', 'Latitude', 'Longitude']).agg({\n 'Releases (lb)': 'sum',\n\ + \ 'RSEI Hazard': 'sum',\n 'Chemical': 'nunique',\n 'Year': 'nunique'\n\ + }).reset_index()\n\nfacility_summary.columns = ['Facility Name', 'Latitude',\ + \ 'Longitude', 'Total Releases (lb)', \n 'Total RSEI\ + \ Hazard', 'Number of Chemicals', 'Years Reported']\n\n# Sort by total releases\n\ + facility_summary = facility_summary.sort_values('Total Releases (lb)', ascending=False)\n\ + \n# Create an interactive map\n# Center the map on the county\ncounty_center\ + \ = [29.7604, -95.3698] # Houston coordinates as center\nm = folium.Map(location=county_center,\ + \ zoom_start=10, tiles='CartoDB positron')\n\n# Add a marker cluster for better\ + \ visualization\nmarker_cluster = MarkerCluster().add_to(m)\n\n# Add markers\ + \ for each facility\nfor i, row in facility_summary.iterrows():\n # Scale\ + \ the circle size based on the log of releases (to make it more visible)\n \ + \ radius = np.log1p(row['Total Releases (lb)']) * 0.5\n \n # Create\ + \ popup content\n popup_content = f\"\"\"\n {row['Facility Name']}
\n\ + \ Total Releases: {row['Total Releases (lb)']:,.2f} lb
\n RSEI Hazard:\ + \ {row['Total RSEI Hazard']:,.2f}
\n Number of Chemicals: {row['Number\ + \ of Chemicals']}
\n Years Reported: {row['Years Reported']}\n \"\"\ + \"\n \n # Add a circle marker\n folium.CircleMarker(\n location=[row['Latitude'],\ + \ row['Longitude']],\n radius=radius,\n popup=folium.Popup(popup_content,\ + \ max_width=300),\n color='red',\n fill=True,\n fill_color='red',\n\ + \ fill_opacity=0.6,\n opacity=0.8\n ).add_to(marker_cluster)\n\ + \n# Save the map\nmap_file = 'facilities_map.html'\nm.save(map_file)\nprint(f\"\ + Interactive map saved to: {map_file}\")\n" + integration: epa_toxic_release_inventory_(tri) + notes: null + query: |- + Creating an interactive map of toxic release facilities. This example shows how to use the folium library to generate an interactive web map displaying the locations of facilities, with circle sizes proportional to release amounts and popup information showing detailed facility data. This visualization helps identify spatial patterns and hotspots of toxic releases. + resource_id: 7dcb643d-1565-415e-99c7-04c0c48e6366 + resource_type: example + 848b37b3-e119-42e7-a85d-b8a11e58ae19: + code: | + import pandas as pd + + # Load the TRI data + tri_data = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv', index_col=False) + + # List of respiratory-related chemicals + respiratory_chemicals = [ + 'Ammonia', + 'Chlorine', + 'Sulfuric acid', + 'Hydrochloric acid', + 'Hydrogen fluoride', + 'Nitric acid', + 'Ozone', + 'Phosgene', + 'Toluene diisocyanate', + 'Methylene diphenyl diisocyanate' + ] + + # Filter the data + respiratory_incidents = tri_data[ + (tri_data['Year'] == 2022) & + (tri_data['County'] == 'HARRIS, TX') & + (tri_data['Chemical'].str.contains('|'.join(respiratory_chemicals), case=False, na=False)) + ] + + # Sort by release amount + respiratory_incidents = respiratory_incidents.sort_values('Releases (lb)', ascending=False) + + # Select relevant columns + result = respiratory_incidents[[ + 'TRI Facility Name', + 'Chemical', + 'Releases (lb)', + 'RSEI Hazard' + ]] + + print(result) + integration: epa_toxic_release_inventory_(tri) + notes: null + query: |- + Query to retrieve asthma-respiratory related hazard incidents from the EPA's TRI for the year 2022 in Harris County, TX. + resource_id: 848b37b3-e119-42e7-a85d-b8a11e58ae19 + resource_type: example + 8de5ce51-3ff8-4ab3-a347-311b499acf7a: + code: "import pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as\ + \ sns\nimport numpy as np\nfrom scipy.stats import pearsonr\n\n# Load and prepare\ + \ data (assuming tri_data is already loaded and cleaned)\nfile_path = '{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv'\n\ + tri_data = pd.read_csv(file_path, low_memory=False)\n\n# Fix the numeric columns\ + \ that might be stored as strings with commas\nfor col in ['Releases (lb)',\ + \ 'Waste Managed (lb)', 'RSEI Hazard']:\n if tri_data[col].dtype == 'object':\n\ + \ # Remove commas and convert to float\n tri_data[col] = tri_data[col].str.replace(',',\ + \ '').astype(float)\n\n# Filter for a specific county\ncounty_data = tri_data[(tri_data['County']\ + \ == 'HARRIS, TX') | \n (tri_data['County'] == 'HARRIS COUNTY,\ + \ TX')]\n\n# Analyze data by Census Block Group\ncensus_summary = county_data.groupby('Census\ + \ Block Group').agg({\n 'TRI Facility Name': 'nunique',\n 'Releases (lb)':\ + \ 'sum',\n 'RSEI Hazard': 'sum',\n 'Chemical': 'nunique'\n}).reset_index()\n\ + \ncensus_summary.columns = ['Census Block Group', 'Number of Facilities', \n\ + \ 'Total Releases (lb)', 'Total RSEI Hazard', 'Number\ + \ of Chemicals']\n\n# Create a scatter plot of total releases vs RSEI Hazard\ + \ by Census Block Group\nplt.figure(figsize=(10, 8))\nplt.scatter(\n np.log10(census_summary['Total\ + \ Releases (lb)']), \n np.log10(census_summary['Total RSEI Hazard'] + 1),\ + \ # Add 1 to avoid log(0)\n alpha=0.7,\n s=50,\n c='blue'\n)\nplt.title('Relationship\ + \ Between Total Releases and RSEI Hazard Score (Log Scale)')\nplt.xlabel('Log10(Total\ + \ Releases in lb)')\nplt.ylabel('Log10(RSEI Hazard Score)')\nplt.grid(True,\ + \ linestyle='--', alpha=0.7)\n\n# Add a trend line\nz = np.polyfit(np.log10(census_summary['Total\ + \ Releases (lb)']), \n np.log10(census_summary['Total RSEI Hazard']\ + \ + 1), 1)\np = np.poly1d(z)\nplt.plot(np.log10(census_summary['Total Releases\ + \ (lb)']), \n p(np.log10(census_summary['Total Releases (lb)'])), \n\ + \ \"r--\", linewidth=2)\n\n# Add correlation coefficient\ncorr, _ = pearsonr(np.log10(census_summary['Total\ + \ Releases (lb)']), \n np.log10(census_summary['Total RSEI Hazard']\ + \ + 1))\nplt.annotate(f\"Correlation: {corr:.2f}\", xy=(0.05, 0.95), xycoords='axes\ + \ fraction', \n fontsize=12, bbox=dict(boxstyle=\"round,pad=0.3\"\ + , fc=\"white\", ec=\"gray\", alpha=0.8))\n\nplt.tight_layout()\nplt.savefig('releases_vs_hazard.png')\n\ + plt.show()" + integration: epa_toxic_release_inventory_(tri) + notes: null + query: |- + Analyzing the relationship between release amounts and hazard scores. This example demonstrates how to investigate the correlation between the quantity of toxic releases and their associated RSEI Hazard scores using scatter plots, trend lines, and correlation statistics. This analysis helps understand whether larger releases necessarily correspond to higher health risks. + resource_id: 8de5ce51-3ff8-4ab3-a347-311b499acf7a + resource_type: example + 9ec78574-65e7-4a7c-bb6b-7752a2cdf5a7: + code: "import pandas as pd\n\n# Load the data\nfile_path = '{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv'\n\ + tri_data = pd.read_csv(file_path, low_memory=False)\n\n# Fix the numeric columns\ + \ that might be stored as strings with commas\nfor col in ['Releases (lb)',\ + \ 'Waste Managed (lb)', 'RSEI Hazard']:\n if tri_data[col].dtype == 'object':\n\ + \ # Remove commas and convert to float\n tri_data[col] = tri_data[col].str.replace(',',\ + \ '').astype(float)\n\n# Filter for a specific county\nharris_data = tri_data[(tri_data['County']\ + \ == 'HARRIS, TX') | \n (tri_data['County'] == 'HARRIS COUNTY,\ + \ TX')]\n\n# Create a summary of facilities\nfacility_summary = harris_data.groupby(['TRI\ + \ Facility Name', 'Latitude', 'Longitude']).agg({\n 'Releases (lb)': 'sum',\n\ + \ 'RSEI Hazard': 'sum',\n 'Chemical': 'nunique',\n 'Year': 'nunique'\n\ + }).reset_index()\n\nfacility_summary.columns = ['Facility Name', 'Latitude',\ + \ 'Longitude', 'Total Releases (lb)', \n 'Total RSEI\ + \ Hazard', 'Number of Chemicals', 'Years Reported']\n\n# Sort by total releases\n\ + facility_summary = facility_summary.sort_values('Total Releases (lb)', ascending=False)\n\ + \nprint(\"Summary of Facilities:\")\nprint(f\"Total number of facilities: {len(facility_summary)}\"\ + )\nprint(f\"Total toxic releases: {facility_summary['Total Releases (lb)'].sum():,.2f}\ + \ lb\")\nprint(f\"Total RSEI Hazard score: {facility_summary['Total RSEI Hazard'].sum():,.2f}\"\ + )\n\n# Print top 10 facilities by releases\nprint(\"\\nTop 10 Facilities by\ + \ Total Releases:\")\nfor i, (name, lat, lon, releases, hazard, chemicals, years)\ + \ in enumerate(facility_summary.head(10).values, 1):\n print(f\"{i}. {name}:\ + \ {releases:,.2f} lb, RSEI Hazard: {hazard:,.2f}, Chemicals: {chemicals}\")\n" + integration: epa_toxic_release_inventory_(tri) + notes: null + query: |- + Basic loading, cleaning, and summarizing of EPA TRI data for a specific county. This example demonstrates how to handle the comma-separated numeric values in the dataset, filter for a specific geographic area, and create a summary of facilities with their total releases, hazard scores, and chemical counts. + resource_id: 9ec78574-65e7-4a7c-bb6b-7752a2cdf5a7 + resource_type: example + e2bd90f9-24a6-41e5-a3ba-1aca807d8990: + code: | + import pandas as pd + + # Load the data + tri_data = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv', index_col=False) + + print("Sample of County values:") + print(tri_data['County'].unique()[:10]) # Print first 10 unique county values + + print("\nSample of Chemical values containing 'Ammonia':") + ammonia_chemicals = [chem for chem in tri_data['Chemical'].unique() if 'Ammonia' in str(chem)] + print(ammonia_chemicals) + + # Try a more flexible approach + print("\nTrying a more flexible filter:") + filtered_data = tri_data[ + tri_data['Chemical'].str.contains('Ammonia', case=False, na=False) + ] + + print(f"Found {len(filtered_data)} records for any Ammonia") + + # Now try to find Harris county with a flexible approach + print("\nLooking for Harris county:") + harris_data = tri_data[ + tri_data['County'].str.contains('Harris', case=False, na=False) + ] + + print(f"Found {len(harris_data)} records for Harris county") + print("Sample county values in Harris results:") + print(harris_data['County'].unique()) + + # Now try the combined filter with the correct formats + print("\nTrying combined filter with formats from the dataset:") + final_filtered = tri_data[ + tri_data['County'].str.contains('Harris', case=False, na=False) & + tri_data['Chemical'].str.contains('Ammonia', case=False, na=False) + ] + + print(f"Found {len(final_filtered)} records for Ammonia in Harris county") + if len(final_filtered) > 0: + # Select relevant columns and sort by release amount + result = final_filtered[['TRI Facility Name', 'Chemical', 'Releases (lb)', 'RSEI Hazard', 'Latitude', 'Longitude']] + result['Releases (lb)'] = pd.to_numeric(result['Releases (lb)'], errors='coerce') + result = result.sort_values('Releases (lb)', ascending=False) + print("\nResults:") + print(result.head()) + integration: epa_toxic_release_inventory_(tri) + notes: null + query: |- + When not finding results for a combined query, try a more systematic approach by finding items and counting length by one by one, in order to discover if there is matching data, but the query is not returning any results. This is to discover amonia references for the county of 'HARRIS, TX'. + resource_id: e2bd90f9-24a6-41e5-a3ba-1aca807d8990 + resource_type: example +slug: epa_toxic_release_inventory_(tri) diff --git a/src/biome/adhoc_data/datasources/epa_tri/examples.yaml b/src/biome/adhoc_data/datasources/epa_tri/examples.yaml deleted file mode 100644 index 8753d11..0000000 --- a/src/biome/adhoc_data/datasources/epa_tri/examples.yaml +++ /dev/null @@ -1,462 +0,0 @@ -- query: "Open the TRI data file with index_col=False, since otherwise columns are shifted, and find data for Orange County, FL." - code: | - import pandas as pd - - # Always the TRI csv with index_col=False param, since pandas is inferring the wrong index column and shifting the data without it - df = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv', dtype=str, index_col=False) - - # Filter for any entries containing 'Orange' in the County column - # sample name without the county suffix, all uppercase and with correct spacing - # In general, check contains/case=False instead of exact matches - orange_county = df[df['County'].str.contains('ORANGE, FL', case=False, na=False)] - - # Display results - print(f"Found {len(orange_county)} toxic release records containing 'ORANGE, FL' in the County name.") - print("\nSample of the data:") - print(orange_county.head()) - -- query: "Capture any asthma-respiratory related hazard incident from the TRI- EPA's Toxic Release Inventory data- you have access to this api. FOr now focus on problems in 2022, in the county of Harris, TX." - code: | - import pandas as pd - - # Load the TRI data - tri_data = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv', index_col=False) - - # Define respiratory/asthma related chemicals - # Updated to match the actual format in the dataset (with CAS numbers) - respiratory_chemicals = [ - 'Ammonia (7664-41-7)', - 'Chlorine (7782-50-5)', - 'Sulfuric acid (7664-93-9)', - 'Hydrochloric acid (7647-01-0)', - 'Nitric acid (7697-37-2)', - 'Ozone', # Keep checking if this has a CAS number format - 'Toluene (108-88-3)', - 'Benzene (71-43-2)', - 'Formaldehyde (50-00-0)', - 'Methanol (67-56-1)' - ] - - # Filter data for: - # - Year 2022 - # - Harris County (case-insensitive match) - # - Respiratory chemicals - respiratory_incidents = tri_data[ - (tri_data['Year'] == 2022) & - (tri_data['County'].str.upper() == 'HARRIS, TX') & - (tri_data['Chemical'].isin(respiratory_chemicals)) - ] - - # Convert 'Releases (lb)' to numeric, coercing errors to NaN since ascending= on sort_values messes with it - respiratory_incidents['Releases (lb)'] = pd.to_numeric(respiratory_incidents['Releases (lb)'], errors='coerce') - - # Sort by release amount - respiratory_incidents = respiratory_incidents.sort_values('Releases (lb)', ascending=False) - - # Select relevant columns - result = respiratory_incidents[[ - 'TRI Facility Name', - 'Chemical', - 'Releases (lb)', - 'RSEI Hazard', - 'Latitude', - 'Longitude' - ]] - - result - -- query: "When not finding results for a combined query, try a more systematic approach by finding items and counting length by one by one, in order to discover if there is matching data, but the query is not returning any results. This is to discover amonia references for the county of 'HARRIS, TX'." - code: | - import pandas as pd - - # Load the data - tri_data = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv', index_col=False) - - print("Sample of County values:") - print(tri_data['County'].unique()[:10]) # Print first 10 unique county values - - print("\nSample of Chemical values containing 'Ammonia':") - ammonia_chemicals = [chem for chem in tri_data['Chemical'].unique() if 'Ammonia' in str(chem)] - print(ammonia_chemicals) - - # Try a more flexible approach - print("\nTrying a more flexible filter:") - filtered_data = tri_data[ - tri_data['Chemical'].str.contains('Ammonia', case=False, na=False) - ] - - print(f"Found {len(filtered_data)} records for any Ammonia") - - # Now try to find Harris county with a flexible approach - print("\nLooking for Harris county:") - harris_data = tri_data[ - tri_data['County'].str.contains('Harris', case=False, na=False) - ] - - print(f"Found {len(harris_data)} records for Harris county") - print("Sample county values in Harris results:") - print(harris_data['County'].unique()) - - # Now try the combined filter with the correct formats - print("\nTrying combined filter with formats from the dataset:") - final_filtered = tri_data[ - tri_data['County'].str.contains('Harris', case=False, na=False) & - tri_data['Chemical'].str.contains('Ammonia', case=False, na=False) - ] - - print(f"Found {len(final_filtered)} records for Ammonia in Harris county") - if len(final_filtered) > 0: - # Select relevant columns and sort by release amount - result = final_filtered[['TRI Facility Name', 'Chemical', 'Releases (lb)', 'RSEI Hazard', 'Latitude', 'Longitude']] - result['Releases (lb)'] = pd.to_numeric(result['Releases (lb)'], errors='coerce') - result = result.sort_values('Releases (lb)', ascending=False) - print("\nResults:") - print(result.head()) - -- query: "Query to retrieve asthma-respiratory related hazard incidents from the EPA's TRI for the year 2022 in Harris County, TX." - code: | - import pandas as pd - - # Load the TRI data - tri_data = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv', index_col=False) - - # List of respiratory-related chemicals - respiratory_chemicals = [ - 'Ammonia', - 'Chlorine', - 'Sulfuric acid', - 'Hydrochloric acid', - 'Hydrogen fluoride', - 'Nitric acid', - 'Ozone', - 'Phosgene', - 'Toluene diisocyanate', - 'Methylene diphenyl diisocyanate' - ] - - # Filter the data - respiratory_incidents = tri_data[ - (tri_data['Year'] == 2022) & - (tri_data['County'] == 'HARRIS, TX') & - (tri_data['Chemical'].str.contains('|'.join(respiratory_chemicals), case=False, na=False)) - ] - - # Sort by release amount - respiratory_incidents = respiratory_incidents.sort_values('Releases (lb)', ascending=False) - - # Select relevant columns - result = respiratory_incidents[[ - 'TRI Facility Name', - 'Chemical', - 'Releases (lb)', - 'RSEI Hazard' - ]] - - print(result) - - -- query: "Basic loading, cleaning, and summarizing of EPA TRI data for a specific county. This example demonstrates how to handle the comma-separated numeric values in the dataset, filter for a specific geographic area, and create a summary of facilities with their total releases, hazard scores, and chemical counts." - code: | - import pandas as pd - - # Load the data - file_path = '{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv' - tri_data = pd.read_csv(file_path, low_memory=False) - - # Fix the numeric columns that might be stored as strings with commas - for col in ['Releases (lb)', 'Waste Managed (lb)', 'RSEI Hazard']: - if tri_data[col].dtype == 'object': - # Remove commas and convert to float - tri_data[col] = tri_data[col].str.replace(',', '').astype(float) - - # Filter for a specific county - harris_data = tri_data[(tri_data['County'] == 'HARRIS, TX') | - (tri_data['County'] == 'HARRIS COUNTY, TX')] - - # Create a summary of facilities - facility_summary = harris_data.groupby(['TRI Facility Name', 'Latitude', 'Longitude']).agg({ - 'Releases (lb)': 'sum', - 'RSEI Hazard': 'sum', - 'Chemical': 'nunique', - 'Year': 'nunique' - }).reset_index() - - facility_summary.columns = ['Facility Name', 'Latitude', 'Longitude', 'Total Releases (lb)', - 'Total RSEI Hazard', 'Number of Chemicals', 'Years Reported'] - - # Sort by total releases - facility_summary = facility_summary.sort_values('Total Releases (lb)', ascending=False) - - print("Summary of Facilities:") - print(f"Total number of facilities: {len(facility_summary)}") - print(f"Total toxic releases: {facility_summary['Total Releases (lb)'].sum():,.2f} lb") - print(f"Total RSEI Hazard score: {facility_summary['Total RSEI Hazard'].sum():,.2f}") - - # Print top 10 facilities by releases - print("\nTop 10 Facilities by Total Releases:") - for i, (name, lat, lon, releases, hazard, chemicals, years) in enumerate(facility_summary.head(10).values, 1): - print(f"{i}. {name}: {releases:,.2f} lb, RSEI Hazard: {hazard:,.2f}, Chemicals: {chemicals}") - - -- query: "Analyzing and visualizing toxic releases by ZIP code. This example shows how to aggregate TRI data by geographic units (ZIP codes), calculate total releases and hazard scores, and create a bar chart visualization of the top areas with highest toxic releases." - code: | - import pandas as pd - import matplotlib.pyplot as plt - import numpy as np - - # Load and prepare data (assuming tri_data is already loaded and cleaned) - file_path = '{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv' - tri_data = pd.read_csv(file_path, low_memory=False) - - # Fix the numeric columns that might be stored as strings with commas - for col in ['Releases (lb)', 'Waste Managed (lb)', 'RSEI Hazard']: - if tri_data[col].dtype == 'object': - # Remove commas and convert to float - tri_data[col] = tri_data[col].str.replace(',', '').astype(float) - - # Filter for a specific county - county_data = tri_data[(tri_data['County'] == 'HARRIS, TX') | - (tri_data['County'] == 'HARRIS COUNTY, TX')] - - # Analyze data by ZIP code - zip_summary = county_data.groupby('ZIP Code').agg({ - 'TRI Facility Name': 'nunique', - 'Releases (lb)': 'sum', - 'RSEI Hazard': 'sum' - }).reset_index() - - zip_summary.columns = ['ZIP Code', 'Number of Facilities', 'Total Releases (lb)', 'Total RSEI Hazard'] - zip_summary = zip_summary.sort_values('Total Releases (lb)', ascending=False) - - print("Top 10 ZIP Codes by Total Releases:") - for i, (zipcode, facilities, releases, hazard) in enumerate(zip_summary.head(10).values, 1): - print(f"{i}. ZIP {zipcode}: {facilities} facilities, {releases:,.2f} lb, RSEI Hazard: {hazard:,.2f}") - - # Create a bar chart of top 10 ZIP codes by releases - plt.figure(figsize=(12, 6)) - top_zips = zip_summary.head(10) - bars = plt.bar(top_zips['ZIP Code'].astype(str), top_zips['Total Releases (lb)'] / 1e6) - plt.title('Top 10 ZIP Codes by Total Toxic Releases') - plt.xlabel('ZIP Code') - plt.ylabel('Total Releases (Million lb)') - plt.xticks(rotation=45) - plt.grid(True, axis='y', linestyle='--', alpha=0.7) - - # Add value labels on top of bars - for bar in bars: - height = bar.get_height() - plt.text(bar.get_x() + bar.get_width()/2., height + 0.5, - f'{height:.1f}M', ha='center', va='bottom') - - plt.tight_layout() - plt.savefig('top_zips_by_releases.png') - plt.show() - - -- query: "Identifying and analyzing chemicals related to specific health conditions. This example demonstrates how to filter TRI data for chemicals associated with a particular health condition (like asthma), calculate their proportion of total releases, and analyze trends over time. This approach can be adapted for various health conditions by modifying the list of relevant chemicals." - code: | - import pandas as pd - import matplotlib.pyplot as plt - import numpy as np - - # Load and prepare data (assuming tri_data is already loaded and cleaned) - file_path = '{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv' - tri_data = pd.read_csv(file_path, low_memory=False) - - # Fix the numeric columns that might be stored as strings with commas - for col in ['Releases (lb)', 'Waste Managed (lb)', 'RSEI Hazard']: - if tri_data[col].dtype == 'object': - # Remove commas and convert to float - tri_data[col] = tri_data[col].str.replace(',', '').astype(float) - - # Filter for a specific county - county_data = tri_data[(tri_data['County'] == 'HARRIS, TX') | - (tri_data['County'] == 'HARRIS COUNTY, TX')] - - # List of chemicals known to potentially trigger or worsen a specific health condition (e.g., asthma) - condition_related_chemicals = [ - 'Ammonia', 'Chlorine', 'Formaldehyde', 'Sulfuric acid', 'Hydrochloric acid', - 'Nitrogen oxides', 'Sulfur dioxide', 'Particulate matter', 'Isocyanates', - 'Toluene diisocyanate', 'Phthalates', 'Volatile organic compounds', 'VOCs', - 'Ozone', 'Acrolein', 'Benzene', 'Styrene', 'Xylene', 'Toluene' - ] - - # Create a function to check if a chemical is related to the health condition - def is_condition_related(chemical_name): - for chem in condition_related_chemicals: - if chem.lower() in chemical_name.lower(): - return True - return False - - # Add a column to identify condition-related chemicals - county_data['Condition_Related'] = county_data['Chemical'].apply(is_condition_related) - - # Calculate total condition-related releases - condition_releases = county_data[county_data['Condition_Related']]['Releases (lb)'].sum() - total_releases = county_data['Releases (lb)'].sum() - condition_percentage = (condition_releases / total_releases) * 100 - - print(f"Total toxic releases: {total_releases:,.2f} lb") - print(f"Condition-related chemical releases: {condition_releases:,.2f} lb") - print(f"Percentage of condition-related releases: {condition_percentage:.2f}%") - - # Analyze trends over time for condition-related chemicals - yearly_condition = county_data[county_data['Condition_Related']].groupby('Year')['Releases (lb)'].sum() - yearly_total = county_data.groupby('Year')['Releases (lb)'].sum() - yearly_percentage = (yearly_condition / yearly_total) * 100 - - # Create a time series plot - plt.figure(figsize=(12, 6)) - plt.plot(yearly_condition.index, yearly_condition.values / 1e6, 'o-', color='red', - label='Condition-related Releases') - plt.plot(yearly_total.index, yearly_total.values / 1e6, 'o-', color='blue', - label='Total Releases') - plt.title('Yearly Trends in Toxic Releases') - plt.xlabel('Year') - plt.ylabel('Releases (Million lb)') - plt.legend() - plt.grid(True, linestyle='--', alpha=0.7) - plt.tight_layout() - plt.savefig('yearly_condition_trends.png') - plt.show() - - -- query: "Creating an interactive map of toxic release facilities. This example shows how to use the folium library to generate an interactive web map displaying the locations of facilities, with circle sizes proportional to release amounts and popup information showing detailed facility data. This visualization helps identify spatial patterns and hotspots of toxic releases." - code: | - import pandas as pd - import folium - from folium.plugins import MarkerCluster - import numpy as np - - # Load and prepare data (assuming tri_data is already loaded and cleaned) - file_path = '{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv' - tri_data = pd.read_csv(file_path, low_memory=False) - - # Fix the numeric columns that might be stored as strings with commas - for col in ['Releases (lb)', 'Waste Managed (lb)', 'RSEI Hazard']: - if tri_data[col].dtype == 'object': - # Remove commas and convert to float - tri_data[col] = tri_data[col].str.replace(',', '').astype(float) - - # Filter for a specific county - county_data = tri_data[(tri_data['County'] == 'HARRIS, TX') | - (tri_data['County'] == 'HARRIS COUNTY, TX')] - - # Create a summary of facilities - facility_summary = county_data.groupby(['TRI Facility Name', 'Latitude', 'Longitude']).agg({ - 'Releases (lb)': 'sum', - 'RSEI Hazard': 'sum', - 'Chemical': 'nunique', - 'Year': 'nunique' - }).reset_index() - - facility_summary.columns = ['Facility Name', 'Latitude', 'Longitude', 'Total Releases (lb)', - 'Total RSEI Hazard', 'Number of Chemicals', 'Years Reported'] - - # Sort by total releases - facility_summary = facility_summary.sort_values('Total Releases (lb)', ascending=False) - - # Create an interactive map - # Center the map on the county - county_center = [29.7604, -95.3698] # Houston coordinates as center - m = folium.Map(location=county_center, zoom_start=10, tiles='CartoDB positron') - - # Add a marker cluster for better visualization - marker_cluster = MarkerCluster().add_to(m) - - # Add markers for each facility - for i, row in facility_summary.iterrows(): - # Scale the circle size based on the log of releases (to make it more visible) - radius = np.log1p(row['Total Releases (lb)']) * 0.5 - - # Create popup content - popup_content = f""" - {row['Facility Name']}
- Total Releases: {row['Total Releases (lb)']:,.2f} lb
- RSEI Hazard: {row['Total RSEI Hazard']:,.2f}
- Number of Chemicals: {row['Number of Chemicals']}
- Years Reported: {row['Years Reported']} - """ - - # Add a circle marker - folium.CircleMarker( - location=[row['Latitude'], row['Longitude']], - radius=radius, - popup=folium.Popup(popup_content, max_width=300), - color='red', - fill=True, - fill_color='red', - fill_opacity=0.6, - opacity=0.8 - ).add_to(marker_cluster) - - # Save the map - map_file = 'facilities_map.html' - m.save(map_file) - print(f"Interactive map saved to: {map_file}") - - -- query: "Analyzing the relationship between release amounts and hazard scores. This example demonstrates how to investigate the correlation between the quantity of toxic releases and their associated RSEI Hazard scores using scatter plots, trend lines, and correlation statistics. This analysis helps understand whether larger releases necessarily correspond to higher health risks." - code: | - import pandas as pd - import matplotlib.pyplot as plt - import seaborn as sns - import numpy as np - from scipy.stats import pearsonr - - # Load and prepare data (assuming tri_data is already loaded and cleaned) - file_path = '{{DATASET_FILES_BASE_PATH}}/epa-tri/EPA_TRI_Toxics_2014_2023.csv' - tri_data = pd.read_csv(file_path, low_memory=False) - - # Fix the numeric columns that might be stored as strings with commas - for col in ['Releases (lb)', 'Waste Managed (lb)', 'RSEI Hazard']: - if tri_data[col].dtype == 'object': - # Remove commas and convert to float - tri_data[col] = tri_data[col].str.replace(',', '').astype(float) - - # Filter for a specific county - county_data = tri_data[(tri_data['County'] == 'HARRIS, TX') | - (tri_data['County'] == 'HARRIS COUNTY, TX')] - - # Analyze data by Census Block Group - census_summary = county_data.groupby('Census Block Group').agg({ - 'TRI Facility Name': 'nunique', - 'Releases (lb)': 'sum', - 'RSEI Hazard': 'sum', - 'Chemical': 'nunique' - }).reset_index() - - census_summary.columns = ['Census Block Group', 'Number of Facilities', - 'Total Releases (lb)', 'Total RSEI Hazard', 'Number of Chemicals'] - - # Create a scatter plot of total releases vs RSEI Hazard by Census Block Group - plt.figure(figsize=(10, 8)) - plt.scatter( - np.log10(census_summary['Total Releases (lb)']), - np.log10(census_summary['Total RSEI Hazard'] + 1), # Add 1 to avoid log(0) - alpha=0.7, - s=50, - c='blue' - ) - plt.title('Relationship Between Total Releases and RSEI Hazard Score (Log Scale)') - plt.xlabel('Log10(Total Releases in lb)') - plt.ylabel('Log10(RSEI Hazard Score)') - plt.grid(True, linestyle='--', alpha=0.7) - - # Add a trend line - z = np.polyfit(np.log10(census_summary['Total Releases (lb)']), - np.log10(census_summary['Total RSEI Hazard'] + 1), 1) - p = np.poly1d(z) - plt.plot(np.log10(census_summary['Total Releases (lb)']), - p(np.log10(census_summary['Total Releases (lb)'])), - "r--", linewidth=2) - - # Add correlation coefficient - corr, _ = pearsonr(np.log10(census_summary['Total Releases (lb)']), - np.log10(census_summary['Total RSEI Hazard'] + 1)) - plt.annotate(f"Correlation: {corr:.2f}", xy=(0.05, 0.95), xycoords='axes fraction', - fontsize=12, bbox=dict(boxstyle="round,pad=0.3", fc="white", ec="gray", alpha=0.8)) - - plt.tight_layout() - plt.savefig('releases_vs_hazard.png') - plt.show() \ No newline at end of file diff --git a/src/biome/adhoc_data/datasources/faers/api.yaml b/src/biome/adhoc_data/datasources/faers/api.yaml index 4ecb685..92d4aae 100644 --- a/src/biome/adhoc_data/datasources/faers/api.yaml +++ b/src/biome/adhoc_data/datasources/faers/api.yaml @@ -1,24 +1,71 @@ -name: "FDA drug adverse event FAERS API" -integration_type: api - description: | Drug Adverse Event Overview The openFDA drug adverse event API returns data that has been collected from the FDA Adverse Event Reporting System (FAERS), a database that contains information on adverse event and medication error reports submitted to FDA. +integration_type: api +name: FDA drug adverse event FAERS API +prompt: | + ${web_documentation} -attachments: - web_documentation: faers_web.md - fields_reference: faers_fields_reference.csv + # Additional Instructions: -prompt: | - ${web_documentation} + You will need an API Key to use any openFDA APIs. It is available in the environment variable: + - API Key: os.environ.get("API_OPENFDA") + Do not use placeholder auth values from the OpenAPI spec nor prompt the user for these values, just use the environment variable. + + # Searchable Fields Reference: + ${fields_reference} +resources: + 66215565-2562-455a-aedc-80cdb06f6c1a: + filepath: faers_fields_reference.csv + integration: fda_drug_adverse_event_faers_api + name: fields_reference + resource_id: 66215565-2562-455a-aedc-80cdb06f6c1a + resource_type: file + a02272d5-d7da-4070-942e-750f5f5f9088: + code: |- + import requests + import os + import pandas as pd + + # Base URL for the API + base_url = "https://api.fda.gov/drug/event.json?api_key=" + os.environ.get('API_OPENFDA') + + # Query parameters + params = { + 'count': 'patient.drug.medicinalproduct.exact', + 'limit': 100 # Get top 100 medications by report count + } + + # Make API request + response = requests.get(base_url, params=params) + + # Convert to DataFrame + results = response.json()['results'] + df = pd.DataFrame(results, columns=['term', 'count']) - # Additional Instructions: + # Sort by count in descending order + df = df.sort_values('count', ascending=False) - You will need an API Key to use any openFDA APIs. It is available in the environment variable: - - API Key: os.environ.get("API_OPENFDA") - Do not use placeholder auth values from the OpenAPI spec nor prompt the user for these values, just use the environment variable. + # Display top medications with highest adverse event counts + print("Top medications by adverse event count:") + print(df.head(10)) - # Searchable Fields Reference: - ${fields_reference} + # Get medication with highest count + top_med = df.iloc[0] + print(f"\nMedication with highest number of adverse events:") + print(f"{top_med['term']}: {top_med['count']} reports") + integration: fda_drug_adverse_event_faers_api + notes: null + query: Can you tell me which medication have the highest adverse effects reported + from all of the available? + resource_id: a02272d5-d7da-4070-942e-750f5f5f9088 + resource_type: example + f9afc1aa-371f-4318-9fb1-cd5030db39fe: + filepath: faers_web.md + integration: fda_drug_adverse_event_faers_api + name: web_documentation + resource_id: f9afc1aa-371f-4318-9fb1-cd5030db39fe + resource_type: file +slug: fda_drug_adverse_event_faers_api diff --git a/src/biome/adhoc_data/datasources/faers/examples.yaml b/src/biome/adhoc_data/datasources/faers/examples.yaml deleted file mode 100644 index 13a019f..0000000 --- a/src/biome/adhoc_data/datasources/faers/examples.yaml +++ /dev/null @@ -1,33 +0,0 @@ -- query: "Can you tell me which medication have the highest adverse effects reported from all of the available?" - code: | - import requests - import os - import pandas as pd - - # Base URL for the API - base_url = "https://api.fda.gov/drug/event.json?api_key=" + os.environ.get('API_OPENFDA') - - # Query parameters - params = { - 'count': 'patient.drug.medicinalproduct.exact', - 'limit': 100 # Get top 100 medications by report count - } - - # Make API request - response = requests.get(base_url, params=params) - - # Convert to DataFrame - results = response.json()['results'] - df = pd.DataFrame(results, columns=['term', 'count']) - - # Sort by count in descending order - df = df.sort_values('count', ascending=False) - - # Display top medications with highest adverse event counts - print("Top medications by adverse event count:") - print(df.head(10)) - - # Get medication with highest count - top_med = df.iloc[0] - print(f"\nMedication with highest number of adverse events:") - print(f"{top_med['term']}: {top_med['count']} reports") \ No newline at end of file diff --git a/src/biome/adhoc_data/datasources/gdc/api.yaml b/src/biome/adhoc_data/datasources/gdc/api.yaml index 9322700..4923562 100644 --- a/src/biome/adhoc_data/datasources/gdc/api.yaml +++ b/src/biome/adhoc_data/datasources/gdc/api.yaml @@ -1,58 +1,299 @@ -name: Genomics Data Commons +description: | + The NCI's Genomic Data Commons (GDC) provides the cancer research community with a repository and computational + platform for cancer researchers who need to understand cancer, its clinical progression, and response to therapy. + The GDC supports several cancer genome programs at the NCI Center for Cancer Genomics (CCG), + including The Cancer Genome Atlas (TCGA) and Therapeutically Applicable Research to Generate Effective Treatments (TARGET). integration_type: api +name: Genomics Data Commons +prompt: | + # GDC API Documentation -description: | - The NCI's Genomic Data Commons (GDC) provides the cancer research community with a repository and computational - platform for cancer researchers who need to understand cancer, its clinical progression, and response to therapy. - The GDC supports several cancer genome programs at the NCI Center for Cancer Genomics (CCG), - including The Cancer Genome Atlas (TCGA) and Therapeutically Applicable Research to Generate Effective Treatments (TARGET). + Below you will find the GDC API documentation. Note that + there are several endpoints available--some are discussed in the `Search and Retrieval` + section, and others are discussed in the `Data Analysis` section. -attachments: - raw_documentation: gdc.md - facets: facets.txt - mappings: gdc_mappings.json + When a user is searching for cases related to gene mutations you should use the `/ssm_occurrences` endpoint. -prompt: | - # GDC API Documentation + ${raw_documentation} + + # Additional Instructions: + + You will be given python code to query the GDC API. + + When querying GDC, change the `format` from "TSV" to "JSON" + + If you download a file by file-id, the output will be TSV. + Save the file with the proper extension. + + When filtering you should rely on the fields provided for a given endpoint in the following mapping. + This mapping is a dictionary where the keys are the endpoints and the values are the fields that are available for that endpoint. + + ``` + ${mappings} + ``` + + These are the fields that are available for filtering and also the fields that can be returned. You can use + the facets to filter the fields that are available for a given endpoint. + + A list of fields and their respective choices/facets are as follows: + + ``` + ${facets} + ``` + + Note this list of facets may be incomplete as it changes over time. Remember to consult it + when you are filtering. If you are unsure about options available for a given field, you should consider + just using the wildcard operator. For example, if you are looking for cases for "paragangliomas and glomus tumors" + and getting no results, you should consider just using `*paragangliomas*`. Be sure to let the user know that you are using + the wildcard operator so it might not be as specific as they would like. + + If a user asks you to filter based on the type of cancer, it is often productive to filter on + disease type, primary diagnosis, or primary site. +resources: + 4b9845c7-d61b-440d-880e-b3d2b315a415: + filepath: facets.txt + integration: genomics_data_commons + name: facets + resource_id: 4b9845c7-d61b-440d-880e-b3d2b315a415 + resource_type: file + 56f47a1d-b966-4669-8ffd-c3b9de388afd: + code: | + import requests + import json + import pandas as pd + from pandas import json_normalize + + # Define the fields to be returned + fields = [ + "case_id", + ] + + # Initialize pagination variables + all_cases = [] + current_page = 1 + page_size = 1000 + + while True: + # Construct the request parameters + params = { + "fields": ",".join(fields), + "format": "JSON", + "size": page_size, + "from": (current_page - 1) * page_size + } + + # Send the request + response = requests.get(endpoint, params=params) + + # Check for successful response + if response.status_code == 200: + data = response.json() + cases = data["data"]["hits"] + all_cases.extend(cases) + if len(cases) < page_size: + break + current_page += 1 + else: + print(f"Error: {response.status_code} - {response.text}") + break + + # Convert the results to a DataFrame + all_cases_df = pd.DataFrame(json_normalize(all_cases)) + print(f"There are {len(all_cases_df)} cases in GDC in total") + # Display the DataFrame + all_cases_df.head() + integration: genomics_data_commons + notes: null + query: |- + Get all cases from GDC with no filters. Fetch only the case ids and paginate through the results to fetch all available cases. Store the results as a dataframe + resource_id: 56f47a1d-b966-4669-8ffd-c3b9de388afd + resource_type: example + 57f85658-1659-44e4-a9b8-ec9eb41f7aae: + code: | + import pandas as pd + from pandas import json_normalize - Below you will find the GDC API documentation. Note that - there are several endpoints available--some are discussed in the `Search and Retrieval` - section, and others are discussed in the `Data Analysis` section. + import requests + import json - When a user is searching for cases related to gene mutations you should use the `/ssm_occurrences` endpoint. + # Define the endpoint and filters + endpoint = "https://api.gdc.cancer.gov/ssm_occurrences" + filters = { + "op": "and", + "content": [ + { + "op": "=", + "content": { + "field": "case.disease_type", + "value": "*myeloid leukemia*" + } + }, + { + "op": "in", + "content": { + "field": "ssm.consequence.transcript.gene.symbol", + "value": ["JAK2"] + } + } + ] + } - ${raw_documentation} + # Define the fields to be returned + fields = [ + "ssm_id", + "ssm.consequence.transcript.gene.symbol", + "ssm.mutation_type", + "ssm.genomic_dna_change", + "ssm.consequence.transcript.aa_change", + "ssm.consequence.transcript.consequence_type", + "case.project.project_id", + "case.submitter_id", + "case.case_id", + "case.diagnoses.primary_diagnosis" + ] - # Additional Instructions: + # Construct the request parameters + params = { + "filters": json.dumps(filters), + "fields": ",".join(fields), + "format": "JSON", + "size": "1000" + } - You will be given python code to query the GDC API. + # Send the request + response = requests.get(endpoint, params=params) + print(f"total hits: {response.json()['data']['pagination']['total']}") + all_ssms = response.json()['data']['hits'] - When querying GDC, change the `format` from "TSV" to "JSON" + ssms = pd.DataFrame(json_normalize(all_ssms)) + ssms.head() + integration: genomics_data_commons + notes: null + query: Query the GDC for cases of myeloid leukemia with a JAK2 somatic mutation + and return the results as a pandas dataframe. + resource_id: 57f85658-1659-44e4-a9b8-ec9eb41f7aae + resource_type: example + 74735957-03a3-4706-aa40-f8bf200553cc: + filepath: gdc.md + integration: genomics_data_commons + name: raw_documentation + resource_id: 74735957-03a3-4706-aa40-f8bf200553cc + resource_type: file + 7a94f76a-0854-41a3-bb97-69a4dcbcaad9: + code: "import pandas as pd\nfrom pandas import json_normalize\n\nimport requests\n\ + import json\n\n# Define the endpoint and filters\nendpoint = \"https://api.gdc.cancer.gov/ssm_occurrences\"\ + \nfilters = {\n \"op\": \"and\",\n \"content\": [\n {\n \ + \ \"op\": \"in\",\n \"content\": {\n \"field\"\ + : \"case.primary_site\",\n \"value\": [\"Bronchus and lung\"\ + ]\n }\n },\n {\n \"op\": \"in\",\n \ + \ \"content\": {\n \"field\": \"case.demographic.gender\"\ + ,\n \"value\": [\"male\"]\n }\n },\n \ + \ {\n \"op\": \"<\",\n \"content\": {\n \ + \ \"field\": \"case.diagnoses.age_at_diagnosis\",\n \"value\"\ + : 16436 # 45 years in days\n }\n },\n {\n \ + \ \"op\": \"in\",\n \"content\": {\n \"field\":\ + \ \"case.exposures.tobacco_smoking_status\",\n \"value\": [\"\ + Lifelong Non-smoker\"]\n }\n }\n ]\n}\n\n# Define the fields\ + \ to be returned\nfields = [\n \"ssm_id\",\n \"ssm.consequence.transcript.gene.symbol\"\ + ,\n \"ssm.mutation_type\",\n \"ssm.genomic_dna_change\",\n \"ssm.consequence.transcript.aa_change\"\ + ,\n \"ssm.consequence.transcript.consequence_type\",\n \"case.project.project_id\"\ + ,\n \"case.submitter_id\",\n \"case.case_id\",\n \"case.diagnoses.primary_diagnosis\"\ + ,\n \"case.primary_site\",\n \"case.demographic.gender\",\n \"case.diagnoses.age_at_diagnosis\"\ + ,\n \"case.exposures.tobacco_smoking_status\"\n]\n\n# Construct the request\ + \ parameters\nparams = {\n \"filters\": json.dumps(filters),\n \"fields\"\ + : \",\".join(fields),\n \"format\": \"JSON\",\n \"size\": \"1000\"\n}\n\ + \n# Send the request\nresponse = requests.get(endpoint, params=params)\n\n#\ + \ Check for successful response\nif response.status_code == 200:\n data =\ + \ response.json()\n # Extract and display the mutation data\n mutations\ + \ = data[\"data\"][\"hits\"]\n print(f\"Found {len(mutations)} mutations\ + \ in lung cancer cases for males under 45 who never smoked:\")\n lung_mutation_df\ + \ = pd.DataFrame(json_normalize(mutations))\nelse:\n print(f\"Error: {response.status_code}\ + \ - {response.text}\")\n \nlung_mutation_df.head()" + integration: genomics_data_commons + notes: null + query: |- + Get simple somatic mutation occurrences (cases) from GDC where the primary site is bronchus and lung, the gender is male, the age at diagnosis is less than 45 years old, and the patient is a lifelong non-smoker. Return the results as a pandas dataframe. + resource_id: 7a94f76a-0854-41a3-bb97-69a4dcbcaad9 + resource_type: example + 9f1cc9a9-6acd-4797-b050-73afbaeb74c3: + code: | + import pandas as pd + from pandas import json_normalize - If you download a file by file-id, the output will be TSV. - Save the file with the proper extension. + import requests + import json - When filtering you should rely on the fields provided for a given endpoint in the following mapping. - This mapping is a dictionary where the keys are the endpoints and the values are the fields that are available for that endpoint. + # Define the endpoint and filters + endpoint = "https://api.gdc.cancer.gov/cases" + filters = { + "op": "=", + "content": { + "field": "disease_type", + "value": "*myeloid leukemia*" + } + } - ``` - ${mappings} - ``` + # Define the fields to be returned + fields = [ + "submitter_id", + "case_id", + "primary_site", + "disease_type", + "diagnoses.age_at_diagnosis", + "diagnoses.primary_diagnosis", + "demographic.gender", + "exposures.tobacco_smoking_status", + "files.file_id", + "files.file_name", + "files.data_type", + "files.experimental_strategy" + ] - These are the fields that are available for filtering and also the fields that can be returned. You can use - the facets to filter the fields that are available for a given endpoint. + # Initialize pagination variables + all_cases = [] + current_page = 1 + page_size = 1000 - A list of fields and their respective choices/facets are as follows: + while True: + # Construct the request parameters + params = { + "filters": json.dumps(filters), + "fields": ",".join(fields), + "format": "JSON", + "size": page_size, + "from": (current_page - 1) * page_size + } - ``` - ${facets} - ``` + # Send the request + response = requests.get(endpoint, params=params) - Note this list of facets may be incomplete as it changes over time. Remember to consult it - when you are filtering. If you are unsure about options available for a given field, you should consider - just using the wildcard operator. For example, if you are looking for cases for "paragangliomas and glomus tumors" - and getting no results, you should consider just using `*paragangliomas*`. Be sure to let the user know that you are using - the wildcard operator so it might not be as specific as they would like. + # Check for successful response + if response.status_code == 200: + data = response.json() + cases = data["data"]["hits"] + all_cases.extend(cases) + if len(cases) < page_size: + break + current_page += 1 + else: + print(f"Error: {response.status_code} - {response.text}") + break - If a user asks you to filter based on the type of cancer, it is often productive to filter on - disease type, primary diagnosis, or primary site. + # Convert the results to a DataFrame + cases_df = pd.DataFrame(json_normalize(all_cases)) + print(f"There are {len(cases_df)} cases in GDC for AML") + # Display the DataFrame + cases_df.head() + integration: genomics_data_commons + notes: null + query: |- + Get all cases where disease type is myeloid leukemia from GDC and return the results as a pandas dataframe. Paginate through the results to fetch all available cases + resource_id: 9f1cc9a9-6acd-4797-b050-73afbaeb74c3 + resource_type: example + a96216e1-8a2b-4fab-90e8-565d1db62fbe: + filepath: gdc_mappings.json + integration: genomics_data_commons + name: mappings + resource_id: a96216e1-8a2b-4fab-90e8-565d1db62fbe + resource_type: file +slug: genomics_data_commons diff --git a/src/biome/adhoc_data/datasources/gdc/examples.yaml b/src/biome/adhoc_data/datasources/gdc/examples.yaml deleted file mode 100644 index de9bb75..0000000 --- a/src/biome/adhoc_data/datasources/gdc/examples.yaml +++ /dev/null @@ -1,262 +0,0 @@ -- query: "Query the GDC for cases of myeloid leukemia with a JAK2 somatic mutation and return the results as a pandas dataframe." - code: | - import pandas as pd - from pandas import json_normalize - - import requests - import json - - # Define the endpoint and filters - endpoint = "https://api.gdc.cancer.gov/ssm_occurrences" - filters = { - "op": "and", - "content": [ - { - "op": "=", - "content": { - "field": "case.disease_type", - "value": "*myeloid leukemia*" - } - }, - { - "op": "in", - "content": { - "field": "ssm.consequence.transcript.gene.symbol", - "value": ["JAK2"] - } - } - ] - } - - # Define the fields to be returned - fields = [ - "ssm_id", - "ssm.consequence.transcript.gene.symbol", - "ssm.mutation_type", - "ssm.genomic_dna_change", - "ssm.consequence.transcript.aa_change", - "ssm.consequence.transcript.consequence_type", - "case.project.project_id", - "case.submitter_id", - "case.case_id", - "case.diagnoses.primary_diagnosis" - ] - - # Construct the request parameters - params = { - "filters": json.dumps(filters), - "fields": ",".join(fields), - "format": "JSON", - "size": "1000" - } - - # Send the request - response = requests.get(endpoint, params=params) - print(f"total hits: {response.json()['data']['pagination']['total']}") - all_ssms = response.json()['data']['hits'] - - ssms = pd.DataFrame(json_normalize(all_ssms)) - ssms.head() - - -- query: "Get all cases where disease type is myeloid leukemia from GDC and return the results as a pandas dataframe. Paginate through the results to fetch all available cases" - code: | - import pandas as pd - from pandas import json_normalize - - import requests - import json - - # Define the endpoint and filters - endpoint = "https://api.gdc.cancer.gov/cases" - filters = { - "op": "=", - "content": { - "field": "disease_type", - "value": "*myeloid leukemia*" - } - } - - # Define the fields to be returned - fields = [ - "submitter_id", - "case_id", - "primary_site", - "disease_type", - "diagnoses.age_at_diagnosis", - "diagnoses.primary_diagnosis", - "demographic.gender", - "exposures.tobacco_smoking_status", - "files.file_id", - "files.file_name", - "files.data_type", - "files.experimental_strategy" - ] - - # Initialize pagination variables - all_cases = [] - current_page = 1 - page_size = 1000 - - while True: - # Construct the request parameters - params = { - "filters": json.dumps(filters), - "fields": ",".join(fields), - "format": "JSON", - "size": page_size, - "from": (current_page - 1) * page_size - } - - # Send the request - response = requests.get(endpoint, params=params) - - # Check for successful response - if response.status_code == 200: - data = response.json() - cases = data["data"]["hits"] - all_cases.extend(cases) - if len(cases) < page_size: - break - current_page += 1 - else: - print(f"Error: {response.status_code} - {response.text}") - break - - # Convert the results to a DataFrame - cases_df = pd.DataFrame(json_normalize(all_cases)) - print(f"There are {len(cases_df)} cases in GDC for AML") - # Display the DataFrame - cases_df.head() - -- query: "Get all cases from GDC with no filters. Fetch only the case ids and paginate through the results to fetch all available cases. Store the results as a dataframe" - code: | - import requests - import json - import pandas as pd - from pandas import json_normalize - - # Define the fields to be returned - fields = [ - "case_id", - ] - - # Initialize pagination variables - all_cases = [] - current_page = 1 - page_size = 1000 - - while True: - # Construct the request parameters - params = { - "fields": ",".join(fields), - "format": "JSON", - "size": page_size, - "from": (current_page - 1) * page_size - } - - # Send the request - response = requests.get(endpoint, params=params) - - # Check for successful response - if response.status_code == 200: - data = response.json() - cases = data["data"]["hits"] - all_cases.extend(cases) - if len(cases) < page_size: - break - current_page += 1 - else: - print(f"Error: {response.status_code} - {response.text}") - break - - # Convert the results to a DataFrame - all_cases_df = pd.DataFrame(json_normalize(all_cases)) - print(f"There are {len(all_cases_df)} cases in GDC in total") - # Display the DataFrame - all_cases_df.head() - -- query: "Get simple somatic mutation occurrences (cases) from GDC where the primary site is bronchus and lung, the gender is male, the age at diagnosis is less than 45 years old, and the patient is a lifelong non-smoker. Return the results as a pandas dataframe." - code: | - import pandas as pd - from pandas import json_normalize - - import requests - import json - - # Define the endpoint and filters - endpoint = "https://api.gdc.cancer.gov/ssm_occurrences" - filters = { - "op": "and", - "content": [ - { - "op": "in", - "content": { - "field": "case.primary_site", - "value": ["Bronchus and lung"] - } - }, - { - "op": "in", - "content": { - "field": "case.demographic.gender", - "value": ["male"] - } - }, - { - "op": "<", - "content": { - "field": "case.diagnoses.age_at_diagnosis", - "value": 16436 # 45 years in days - } - }, - { - "op": "in", - "content": { - "field": "case.exposures.tobacco_smoking_status", - "value": ["Lifelong Non-smoker"] - } - } - ] - } - - # Define the fields to be returned - fields = [ - "ssm_id", - "ssm.consequence.transcript.gene.symbol", - "ssm.mutation_type", - "ssm.genomic_dna_change", - "ssm.consequence.transcript.aa_change", - "ssm.consequence.transcript.consequence_type", - "case.project.project_id", - "case.submitter_id", - "case.case_id", - "case.diagnoses.primary_diagnosis", - "case.primary_site", - "case.demographic.gender", - "case.diagnoses.age_at_diagnosis", - "case.exposures.tobacco_smoking_status" - ] - - # Construct the request parameters - params = { - "filters": json.dumps(filters), - "fields": ",".join(fields), - "format": "JSON", - "size": "1000" - } - - # Send the request - response = requests.get(endpoint, params=params) - - # Check for successful response - if response.status_code == 200: - data = response.json() - # Extract and display the mutation data - mutations = data["data"]["hits"] - print(f"Found {len(mutations)} mutations in lung cancer cases for males under 45 who never smoked:") - lung_mutation_df = pd.DataFrame(json_normalize(mutations)) - else: - print(f"Error: {response.status_code} - {response.text}") - - lung_mutation_df.head() \ No newline at end of file diff --git a/src/biome/adhoc_data/datasources/gras/api.yaml b/src/biome/adhoc_data/datasources/gras/api.yaml index 3d0cdf0..574b318 100644 --- a/src/biome/adhoc_data/datasources/gras/api.yaml +++ b/src/biome/adhoc_data/datasources/gras/api.yaml @@ -1,41 +1,65 @@ -name: "FDA Generally Recognized as Safe (GRAS)" -integration_type: dataset - description: | - The FDA Generally Recognized as Safe (GRAS) is a csv file (export) data with notices filed from the FDA GRAS database from years 1998 to 2024. - You may use this tool to check for food additives introductions or changes. - Use this tool to answer questions such as: - In which year was Pea fiber food additive/preservative introduced into the US food supply? - -prompt: | - You have access to the FDA Generally Recognized as Safe (GRAS), which is a csv file data with notices filed from the FDA GRAS database from years 1998 to 2024. - You may use this to check for food additives introductions, and correlate with other data sources for analysis. - - "GRAS" is an acronym for the phrase Generally Recognized As Safe. - Under sections 201(s) and 409 of the Federal Food, Drug, and Cosmetic Act, - any substance that is intentionally added to food is a food additive. + The FDA Generally Recognized as Safe (GRAS) is a csv file (export) data with notices filed from the FDA GRAS database from years 1998 to 2024. + You may use this tool to check for food additives introductions or changes. + Use this tool to answer questions such as: + In which year was Pea fiber food additive/preservative introduced into the US food supply? +integration_type: dataset +name: FDA Generally Recognized as Safe (GRAS) +prompt: "You have access to the FDA Generally Recognized as Safe (GRAS), which is\ + \ a csv file data with notices filed from the FDA GRAS database from years 1998\ + \ to 2024.\nYou may use this to check for food additives introductions, and correlate\ + \ with other data sources for analysis.\n\n\"GRAS\" is an acronym for the phrase\ + \ Generally Recognized As Safe.\nUnder sections 201(s) and 409 of the Federal Food,\ + \ Drug, and Cosmetic Act,\nany substance that is intentionally added to food is\ + \ a food additive.\n\n# Additional Instructions:\nUse the raw csv notices file to\ + \ answer questions about GRAS substances. Load the file from:\n\n${DATASET_FILES_BASE_PATH}/fda-gras/GRASNotices.csv\n\ + \nThese are the columns available (separated by semicolons, some newlines/spaces):\n\ + GRAS Notice (GRN) No.\t; Substance; Intended Use; Basis; Notifier;\tNotifier Address;\n\ + Date of filing;\tGRN Part 1;\tGRN Part 2;\tGRN Part 3;\tGRN Part 4;\tGRN Part 5;\t\ + GRN Part 6;\tGRN Part 7;\nDate of closure; Date of correction letter;\tFDA's Letter;\t\ + Date additional correspondence;\tAdditional correspondence;\nDate additinoal correspondence\ + \ 2;\tAdditional correspondence 2; Date additional correspondence 3;\nAdditional\ + \ correspondence 3; Date additional correspondence 4; Additional correspondence\ + \ 4;\nResubmission; Resubmitted; Notes;\tRelated submission\n\nThe Gras Notice GRN\ + \ No is wrapped by spredsheet-software-like function..\nExample:\n=T(\"1\") for\ + \ \"GRAS Notice (GRN) No.\" = 1\n\nThe file which was downloaded from FDA GRAS Notices;\ + \ http://www.hfpappexternal.fda.gov/scripts/fdcc/?set=GRASNotices;\nFile last updated\ + \ by FDA on 3/3/2025.\n\nIf the \"FDA's Letter\" column contains \" FDA has no questions\"\ + \ it means the FDA did not challenge the notice, closest thing to \"approved\".\n\ + Id the \"FDA's Letter\" column contains \"FDA has questions\" or \"Pending\", or\ + \ similar, it means the FDA has questions about the notice, and the notice is not\ + \ approved yet.\nIf the \"FDA's Letter\" column contains \"Notice does not provide\ + \ a basis for a GRAS determination\"- it means it challenged the notice, and the\ + \ notice was not approved.\nUse the stats of the \"FDA's Letter\" column to know\ + \ if a food additive is actually approved to be introduced into the US food supply\ + \ or not.\n" +resources: + fd30b48e-e83b-41a2-a969-49ebc58da112: + code: |- + import pandas as pd - # Additional Instructions: - Use the raw csv notices file to answer questions about GRAS substances. Load the file from: + # Load GRAS notices data + df = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/fda-gras/GRASNotices.csv', dtype=str, index_col=False) - ${DATASET_FILES_BASE_PATH}/fda-gras/GRASNotices.csv + # print head + print(f"First 5 rows of GRAS notices:") + print(df.head()) - These are the columns available (separated by semicolons, some newlines/spaces): - GRAS Notice (GRN) No. ; Substance; Intended Use; Basis; Notifier; Notifier Address; - Date of filing; GRN Part 1; GRN Part 2; GRN Part 3; GRN Part 4; GRN Part 5; GRN Part 6; GRN Part 7; - Date of closure; Date of correction letter; FDA's Letter; Date additional correspondence; Additional correspondence; - Date additinoal correspondence 2; Additional correspondence 2; Date additional correspondence 3; - Additional correspondence 3; Date additional correspondence 4; Additional correspondence 4; - Resubmission; Resubmitted; Notes; Related submission + # Search for notices containing Dioctyl sodium sulfosuccinate (case insensitive) + search_term = 'Dioctyl sodium sulfosuccinate' - The Gras Notice GRN No is wrapped by spredsheet-software-like function.. - Example: - =T("1") for "GRAS Notice (GRN) No." = 1 + print(f"Searching for notices containing {search_term}...") - The file which was downloaded from FDA GRAS Notices; http://www.hfpappexternal.fda.gov/scripts/fdcc/?set=GRASNotices; - File last updated by FDA on 3/3/2025. + matches = df[df['Substance'].str.contains(search_term, case=False, na=False)] - If the "FDA's Letter" column contains " FDA has no questions" it means the FDA did not challenge the notice, closest thing to "approved". - Id the "FDA's Letter" column contains "FDA has questions" or "Pending", or similar, it means the FDA has questions about the notice, and the notice is not approved yet. - If the "FDA's Letter" column contains "Notice does not provide a basis for a GRAS determination"- it means it challenged the notice, and the notice was not approved. - Use the stats of the "FDA's Letter" column to know if a food additive is actually approved to be introduced into the US food supply or not. + # Display relevant columns + if len(matches) > 0: + print(matches[['GRAS Notice (GRN) No.', 'Substance', 'Date of filing', "FDA's Letter"]]) + else: + print(f"No GRAS notices found for {search_term}") + integration: fda_generally_recognized_as_safe_(gras) + notes: null + query: Is there any FDA GRAS notice for Dioctyl sodium sulfosuccinate as an additive? + resource_id: fd30b48e-e83b-41a2-a969-49ebc58da112 + resource_type: example +slug: fda_generally_recognized_as_safe_(gras) diff --git a/src/biome/adhoc_data/datasources/gras/examples.yaml b/src/biome/adhoc_data/datasources/gras/examples.yaml deleted file mode 100644 index 8aedf0e..0000000 --- a/src/biome/adhoc_data/datasources/gras/examples.yaml +++ /dev/null @@ -1,23 +0,0 @@ -- query: "Is there any FDA GRAS notice for Dioctyl sodium sulfosuccinate as an additive?" - code: | - import pandas as pd - - # Load GRAS notices data - df = pd.read_csv('{{DATASET_FILES_BASE_PATH}}/fda-gras/GRASNotices.csv', dtype=str, index_col=False) - - # print head - print(f"First 5 rows of GRAS notices:") - print(df.head()) - - # Search for notices containing Dioctyl sodium sulfosuccinate (case insensitive) - search_term = 'Dioctyl sodium sulfosuccinate' - - print(f"Searching for notices containing {search_term}...") - - matches = df[df['Substance'].str.contains(search_term, case=False, na=False)] - - # Display relevant columns - if len(matches) > 0: - print(matches[['GRAS Notice (GRN) No.', 'Substance', 'Date of filing', "FDA's Letter"]]) - else: - print(f"No GRAS notices found for {search_term}") \ No newline at end of file diff --git a/src/biome/adhoc_data/datasources/hpa/api.yaml b/src/biome/adhoc_data/datasources/hpa/api.yaml index c620d50..f153fe2 100644 --- a/src/biome/adhoc_data/datasources/hpa/api.yaml +++ b/src/biome/adhoc_data/datasources/hpa/api.yaml @@ -1,42 +1,88 @@ -name: "Human Protein Atlas" +description: | + The Human Protein Atlas is a Swedish-based program initiated in 2003 with the aim to map all + the human proteins in cells, tissues, and organs using an integration of various omics + technologies, including antibody-based imaging, mass spectrometry-based proteomics, + transcriptomics, and systems biology. All the data in the knowledge resource is open access + to allow scientists both in academia and industry to freely access the data for exploration + of the human proteome. + + The Human Protein Atlas consists of eight separate resources, each focusing on a particular + aspect of the genome-wide analysis of the human proteins: + + - The Tissue resource, showing the distribution of the proteins across all major tissues and + organs in the human body + - The Brain resource, exploring the distribution of proteins in various regions of the + mammalian brain + - The Single Cell resource, showing expression of protein-coding genes in immune cells and + human single cell types based on bulk and single cell RNA-seq + - The Subcellular resource, showing the subcellular localization of proteins in single cells + - The Cancer resource, showing the impact of protein levels for the survival of patients + with cancer + - The Blood resource, describing proteins detected in blood and showing protein levels in + blood in patients with different diseases + - The Cell line resource, showing expression of protein-coding genes in human cancer cell + lines + - The Structure & Interaction resource, showing predicted 3D structures and exploring + protein-coding genes in the context of protein-protein and metabolic interaction networks. + + The Human Protein Atlas program has already contributed to several thousands of publications + in the field of human biology and disease and is selected by the organization ELIXIR as a + European core resource due to its fundamental importance for a wider life science community. + In addition the Human Protein Atlas has been appointed Global Core Biodata Resource (GCBR) + by the Global Biodata Coalition. The Human Protein Atlas consortium is mainly funded by the + Knut and Alice Wallenberg Foundation. integration_type: api +name: Human Protein Atlas +prompt: '${raw_documentation} -description: | - The Human Protein Atlas is a Swedish-based program initiated in 2003 with the aim to map all - the human proteins in cells, tissues, and organs using an integration of various omics - technologies, including antibody-based imaging, mass spectrometry-based proteomics, - transcriptomics, and systems biology. All the data in the knowledge resource is open access - to allow scientists both in academia and industry to freely access the data for exploration - of the human proteome. - - The Human Protein Atlas consists of eight separate resources, each focusing on a particular - aspect of the genome-wide analysis of the human proteins: - - - The Tissue resource, showing the distribution of the proteins across all major tissues and - organs in the human body - - The Brain resource, exploring the distribution of proteins in various regions of the - mammalian brain - - The Single Cell resource, showing expression of protein-coding genes in immune cells and - human single cell types based on bulk and single cell RNA-seq - - The Subcellular resource, showing the subcellular localization of proteins in single cells - - The Cancer resource, showing the impact of protein levels for the survival of patients - with cancer - - The Blood resource, describing proteins detected in blood and showing protein levels in - blood in patients with different diseases - - The Cell line resource, showing expression of protein-coding genes in human cancer cell - lines - - The Structure & Interaction resource, showing predicted 3D structures and exploring - protein-coding genes in the context of protein-protein and metabolic interaction networks. - - The Human Protein Atlas program has already contributed to several thousands of publications - in the field of human biology and disease and is selected by the organization ELIXIR as a - European core resource due to its fundamental importance for a wider life science community. - In addition the Human Protein Atlas has been appointed Global Core Biodata Resource (GCBR) - by the Global Biodata Coalition. The Human Protein Atlas consortium is mainly funded by the - Knut and Alice Wallenberg Foundation. - -attachments: - raw_documentation: hpa_docs.md - -prompt: | - ${raw_documentation} + ' +resources: + 04c53147-cc0a-42f5-8e1d-6427a595948b: + filepath: hpa_docs.md + integration: human_protein_atlas + name: raw_documentation + resource_id: 04c53147-cc0a-42f5-8e1d-6427a595948b + resource_type: file + 92183e68-5e56-4487-a92f-8a1163b4e178: + code: | + import requests + import json + + # Query JAK2 data from HPA API + jak2_id = "ENSG00000096968" + url = f"https://www.proteinatlas.org/{jak2_id}.json" + + # Get the data + response = requests.get(url) + data = response.json() + + # Print basic gene information + print(f"Gene: {data.get('Gene', 'N/A')}") + print(f"Description: {data.get('Gene description', 'N/A')}") + print("-" * 80) + + # RNA Expression Summary + print("\nRNA Expression Summary:") + print(f"Tissue specificity: {data.get('RNA tissue specificity', 'N/A')}") + print(f"Tissue distribution: {data.get('RNA tissue distribution', 'N/A')}") + print(f"Tissue specificity score: {data.get('RNA tissue specificity score', 'N/A')}") + + # Blood cell specific information + print("\nRNA Blood Cell Expression:") + print(f"Blood cell specificity: {data.get('RNA blood cell specificity', 'N/A')}") + print(f"Blood cell distribution: {data.get('RNA blood cell distribution', 'N/A')}") + print(f"Blood cell specificity score: {data.get('RNA blood cell specificity score', 'N/A')}") + + # Protein Expression Summary + print("\nProtein Expression Summary:") + print(f"Reliability (Immunohistochemistry): {data.get('Reliability (IH)', 'N/A')}") + print(f"Subcellular main location: {data.get('Subcellular main location', 'N/A')}") + if 'Subcellular additional location' in data: + print(f"Subcellular additional location: {data.get('Subcellular additional location', 'N/A')}") + integration: human_protein_atlas + notes: null + query: |- + Query the Human Protein Atlas API for RNA and protein expression summary of a gene (JAK2) using its Ensembl ID. The example demonstrates how to retrieve and display tissue specificity, distribution, and subcellular localization information. + resource_id: 92183e68-5e56-4487-a92f-8a1163b4e178 + resource_type: example +slug: human_protein_atlas diff --git a/src/biome/adhoc_data/datasources/hpa/examples.yaml b/src/biome/adhoc_data/datasources/hpa/examples.yaml deleted file mode 100644 index 9de4459..0000000 --- a/src/biome/adhoc_data/datasources/hpa/examples.yaml +++ /dev/null @@ -1,37 +0,0 @@ - -- query: "Query the Human Protein Atlas API for RNA and protein expression summary of a gene (JAK2) using its Ensembl ID. The example demonstrates how to retrieve and display tissue specificity, distribution, and subcellular localization information." - code: | - import requests - import json - - # Query JAK2 data from HPA API - jak2_id = "ENSG00000096968" - url = f"https://www.proteinatlas.org/{jak2_id}.json" - - # Get the data - response = requests.get(url) - data = response.json() - - # Print basic gene information - print(f"Gene: {data.get('Gene', 'N/A')}") - print(f"Description: {data.get('Gene description', 'N/A')}") - print("-" * 80) - - # RNA Expression Summary - print("\nRNA Expression Summary:") - print(f"Tissue specificity: {data.get('RNA tissue specificity', 'N/A')}") - print(f"Tissue distribution: {data.get('RNA tissue distribution', 'N/A')}") - print(f"Tissue specificity score: {data.get('RNA tissue specificity score', 'N/A')}") - - # Blood cell specific information - print("\nRNA Blood Cell Expression:") - print(f"Blood cell specificity: {data.get('RNA blood cell specificity', 'N/A')}") - print(f"Blood cell distribution: {data.get('RNA blood cell distribution', 'N/A')}") - print(f"Blood cell specificity score: {data.get('RNA blood cell specificity score', 'N/A')}") - - # Protein Expression Summary - print("\nProtein Expression Summary:") - print(f"Reliability (Immunohistochemistry): {data.get('Reliability (IH)', 'N/A')}") - print(f"Subcellular main location: {data.get('Subcellular main location', 'N/A')}") - if 'Subcellular additional location' in data: - print(f"Subcellular additional location: {data.get('Subcellular additional location', 'N/A')}") diff --git a/src/biome/adhoc_data/datasources/idc/api.yaml b/src/biome/adhoc_data/datasources/idc/api.yaml index fafd982..61e721f 100644 --- a/src/biome/adhoc_data/datasources/idc/api.yaml +++ b/src/biome/adhoc_data/datasources/idc/api.yaml @@ -1,24 +1,347 @@ -name: "Imaging Data Commons" description: | - NCI Imaging Data Commons (IDC) is a cloud-based environment containing publicly available cancer imaging data co-located with the analysis and exploration tools and resources. + NCI Imaging Data Commons (IDC) is a cloud-based environment containing publicly available cancer imaging data co-located with the analysis and exploration tools and resources. - Features: - - >65 TB: IDC contains radiology, brightfield (H&E) and fluorescence slide microscopy images, along with image-derived data (annotations, segmentations, quantitative measurements) and accompanying clinical data - - Free: All of the data in IDC is publicly available: no registration, no access requests - - Commercial-friendly: >95% of the data in IDC is covered by the permissive CC-BY license, which allows commercial reuse (small subset of data is covered by the CC-NC license); each file in IDC is tagged with the license to make it easier for you to understand and follow the rules - - Cloud-based: All of the data in IDC is available from both Google and AWS public buckets: fast and free to download, no out-of-cloud egress fees - - Harmonized: All of the images and image-derived data in IDC is harmonized into standard DICOM representation + Features: + - >65 TB: IDC contains radiology, brightfield (H&E) and fluorescence slide microscopy images, along with image-derived data (annotations, segmentations, quantitative measurements) and accompanying clinical data + - Free: All of the data in IDC is publicly available: no registration, no access requests + - Commercial-friendly: >95% of the data in IDC is covered by the permissive CC-BY license, which allows commercial reuse (small subset of data is covered by the CC-NC license); each file in IDC is tagged with the license to make it easier for you to understand and follow the rules + - Cloud-based: All of the data in IDC is available from both Google and AWS public buckets: fast and free to download, no out-of-cloud egress fees + - Harmonized: All of the images and image-derived data in IDC is harmonized into standard DICOM representation +integration_type: api +name: Imaging Data Commons +prompt: | + ${raw_documentation} -attachments: - raw_documentation: idc.md + # Additional Instructions: -prompt: | - ${raw_documentation} + Be sure to import and instantiate the client for the Imaging Data Commons API: + ```python + from idc_index import index + client = index.IDCClient() + ``` +resources: + 090cdd92-54e0-4a30-82ab-a2e9c67b50c7: + code: "# Query to list all studies in the 'cmb_aml' collection\nquery_all_studies\ + \ = \"\"\"\nSELECT DISTINCT StudyDescription\nFROM \n index\nWHERE \n \ + \ collection_id = 'cmb_aml'\n\"\"\"\n\ntry:\n df_all_studies = client.sql_query(query_all_studies)\n\ + \ print(\"All studies in the 'cmb_aml' collection:\")\n print(df_all_studies)\n\ + except Exception as e:\n print(f\"An error occurred: {str(e)}\")\n" + integration: imaging_data_commons + notes: null + query: List all studies for a given collection + resource_id: 090cdd92-54e0-4a30-82ab-a2e9c67b50c7 + resource_type: example + 1be1fe03-4bb2-4240-a95e-96efb9c1d4d8: + code: | + from idc_index import index + import pandas as pd + + client = index.IDCClient() + + # Query for slide microscopy data + query = """ + SELECT DISTINCT + collection_id, + PatientID, + SeriesDescription, + StudyDescription, + BodyPartExamined, + SeriesDate + FROM + index + WHERE + Modality = 'SM' + """ + + try: + df = client.sql_query(query) + print("Slide Microscopy Data Available:") + print("\nCollections with slide microscopy data:") + print(df['collection_id'].unique()) + print("\nSample of the data:") + print(df.head()) + print("\nTotal number of records:", len(df)) + except Exception as e: + print(f"An error occurred: {str(e)}") + integration: imaging_data_commons + notes: null + query: Get slide microscopy data + resource_id: 1be1fe03-4bb2-4240-a95e-96efb9c1d4d8 + resource_type: example + 25d73a5f-adb2-495c-9588-8237bb39883d: + code: "from idc_index import index\nimport pandas as pd\n\nclient = index.IDCClient()\n\ + \n# Query for all available fields for AML slides\nquery = \"\"\"\nSELECT *\n\ + FROM\n index\nWHERE\n Modality = 'SM'\n AND collection_id IN ('cptac_aml',\ + \ 'cmb_aml')\n AND SeriesDescription LIKE '%bone marrow%'\nORDER BY\n \ + \ PatientID, SeriesDate\n\"\"\"\n\ntry:\n df = client.sql_query(query)\n\ + \ print(\"AML Bone Marrow Slide Data:\")\n print(\"\\nTotal number of\ + \ bone marrow slides:\", len(df))\n\n print(\"\\nAvailable columns (metadata\ + \ fields):\")\n print(df.columns.tolist())\n\n # Look for any columns\ + \ that might contain annotations or clinical data\n clinical_cols = [col\ + \ for col in df.columns if any(term in col.lower() \n \ + \ for term in ['annotation', 'clinical', 'blast',\ + \ 'cell', \n 'pathology',\ + \ 'diagnosis', 'grade', 'stage'])]\n print(\"\\nPotentially relevant clinical/annotation\ + \ columns:\")\n print(clinical_cols)\n\n # Get the source DOIs which might\ + \ lead to additional clinical data\n print(\"\\nUnique source DOIs (for finding\ + \ additional clinical data):\")\n print(df['source_DOI'].unique())\n\nexcept\ + \ Exception as e:\n print(f\"An error occurred: {str(e)}\")\n\n# Let's also\ + \ check if there are any study-level annotations\nquery_study = \"\"\"\nSELECT\ + \ DISTINCT\n StudyDescription,\n collection_id,\n source_DOI,\n \ + \ license_short_name\nFROM\n index\nWHERE\n collection_id IN ('cptac_aml',\ + \ 'cmb_aml')\n\"\"\"\n\ntry:\n df_study = client.sql_query(query_study)\n\ + \ print(\"\\nStudy-level information:\")\n print(df_study)\nexcept Exception\ + \ as e:\n print(f\"An error occurred querying study data: {str(e)}\")\n" + integration: imaging_data_commons + notes: null + query: Get additional information for microscopy data for a specific collection + resource_id: 25d73a5f-adb2-495c-9588-8237bb39883d + resource_type: example + 3078cb17-9102-4e54-a381-d0b546032cef: + code: "from idc_index import index\nimport pandas as pd\n\nclient = index.IDCClient()\n\ + \n# Query to find collection names containing 'aml'\nquery = \"\"\"\nSELECT\ + \ DISTINCT collection_id\nFROM \n index\nWHERE \n LOWER(collection_id)\ + \ LIKE '%aml%'\n\"\"\"\n\ntry:\n df_collections = client.sql_query(query)\n\ + \ print(\"Collections with 'aml' in their name:\")\n print(df_collections)\n\ + except Exception as e:\n print(f\"An error occurred: {str(e)}\")\n" + integration: imaging_data_commons + notes: null + query: Find collections in the Imaging Data Commons with 'aml' in their name, + likely related to Acute Myeloid Leukemia. + resource_id: 3078cb17-9102-4e54-a381-d0b546032cef + resource_type: example + 49f63b7d-8e99-4e88-ac9d-86db0d831a8f: + code: | + from idc_index import index + import pandas as pd + + client = index.IDCClient() + + # First, let's see what columns are available + query_columns = """ + SELECT * FROM index LIMIT 1 + """ + + try: + df = client.sql_query(query_columns) + print("Available columns in the index:") + print(df.columns.tolist()) + except Exception as e: + print(f"An error occurred: {str(e)}") + integration: imaging_data_commons + notes: null + query: Get columns available in IDC index + resource_id: 49f63b7d-8e99-4e88-ac9d-86db0d831a8f + resource_type: example + 4adff70f-bc39-4728-a215-b8f7908ff5d0: + code: | + import pandas as pd + from idc_index import index + + client = index.IDCClient() + + # Query for the desired images + query = """ + SELECT series_aws_url + FROM index + WHERE collection_id = 'cmb_aml' + AND StudyDescription = 'XR_Biopsy_BoneMarrow' + """ + df = client.sql_query(query) + + # Create a download manifest using the current directory + with open('download_manifest.txt', 'w') as f: + for url in df['series_aws_url']: + f.write(f'cp {url} ./\n') + + # Execute the download + s5cmd_binary = client.s5cmdPath + !{s5cmd_binary} --no-sign-request --endpoint-url https://s3.amazonaws.com run download_manifest.txt + integration: imaging_data_commons + notes: null + query: Download bone marrow biopsy images from the 'cmb_aml' collection for a + specific study using IDC's public S3 bucket. + resource_id: 4adff70f-bc39-4728-a215-b8f7908ff5d0 + resource_type: example + 51d2dd36-412e-492e-b52f-37b60219dd8c: + code: "from idc_index import index\nimport pandas as pd\n\n# Initialize the IDC\ + \ client\nclient = index.IDCClient()\n\n# Query to find studies within the 'cptac_aml'\ + \ collection\nquery = \"\"\"\nSELECT DISTINCT StudyDescription\nFROM \n index\n\ + WHERE \n collection_id = 'cptac_aml'\n\"\"\"\n\ntry:\n # Execute the query\n\ + \ df_studies = client.sql_query(query)\n print(\"Studies in 'cptac_aml':\"\ + )\n print(df_studies)\nexcept Exception as e:\n print(f\"An error occurred:\ + \ {str(e)}\")\n" + integration: imaging_data_commons + notes: null + query: Fetch studies for a given collection in the Imaging Data Commons (IDC), + specifically for the 'cptac_aml' collection. + resource_id: 51d2dd36-412e-492e-b52f-37b60219dd8c + resource_type: example + 590305c8-9e62-49a5-a28f-8cc74df03667: + code: | + import pandas as pd + from idc_index import index + + client = index.IDCClient() + + query = """ + SELECT series_aws_url + FROM index + WHERE collection_id = 'cmb_aml' + AND StudyDescription = 'XR_Biopsy_BoneMarrow' + """ + df = client.sql_query(query) + print(df.head()) + integration: imaging_data_commons + notes: null + query: Retrieve imagery URLs for the 'XR_Biopsy_BoneMarrow' study in the 'cmb_aml' + collection from IDC. + resource_id: 590305c8-9e62-49a5-a28f-8cc74df03667 + resource_type: example + a155ea7d-f42f-4546-8d4c-b1135994ea47: + filepath: idc.md + integration: imaging_data_commons + name: raw_documentation + resource_id: a155ea7d-f42f-4546-8d4c-b1135994ea47 + resource_type: file + bbfbce09-abfe-4565-a50b-6afc299df2d3: + code: | + import pydicom + import matplotlib.pyplot as plt + + # Load the DICOM file + filename = '73f18e49-8699-4c1f-90ad-5db94b6b5602.dcm' + ds = pydicom.dcmread(filename) + + # Display the image + plt.imshow(ds.pixel_array, cmap=plt.cm.gray) + plt.title('Bone Marrow Biopsy Image') + plt.axis('off') + plt.show() + integration: imaging_data_commons + notes: null + query: Basic plot of dicom image from file + resource_id: bbfbce09-abfe-4565-a50b-6afc299df2d3 + resource_type: example + c19b78ab-29cb-4be3-8069-1ef22302dfb2: + code: "import sys\nimport subprocess\n\n# Install required package\nsubprocess.check_call([sys.executable,\ + \ \"-m\", \"pip\", \"install\", \"pydicom\"])\n\nimport boto3\nfrom botocore\ + \ import UNSIGNED\nfrom botocore.config import Config\nimport os\nimport tempfile\n\ + import pydicom\nfrom PIL import Image\nimport numpy as np\nimport matplotlib.pyplot\ + \ as plt\n\n# Create S3 client\ns3 = boto3.client('s3', config=Config(signature_version=UNSIGNED))\n\ + \n# Create temporary directory\ntemp_dir = tempfile.mkdtemp()\n\n# Let's try\ + \ the smallest DICOM file first\nfile_key = \"d53e7fc3-8ef9-4de5-8059-47b21a67eb4f/40006642-6ee5-44ca-bc5f-e1cc2bd7ee70.dcm\"\ + \nlocal_file = os.path.join(temp_dir, \"slide.dcm\")\n\ntry:\n print(f\"\ + Downloading file: {file_key}\")\n s3.download_file('idc-open-data', file_key,\ + \ local_file)\n print(\"Download successful!\")\n\n # Try to read the\ + \ DICOM file\n print(\"\\nReading DICOM file...\")\n ds = pydicom.dcmread(local_file)\n\ + \n print(\"\\nDICOM metadata:\")\n print(f\"Patient ID: {ds.PatientID}\"\ + )\n print(f\"Modality: {ds.Modality}\")\n print(f\"Image Type: {ds.ImageType}\"\ + )\n\n # Convert to image and display\n if hasattr(ds, 'pixel_array'):\n\ + \ print(\"\\nConverting to image...\")\n pixel_array = ds.pixel_array\n\ + \n # Normalize the pixel values\n if pixel_array.dtype != np.uint8:\n\ + \ pixel_array = ((pixel_array - pixel_array.min()) * 255.0 / \n \ + \ (pixel_array.max() - pixel_array.min())).astype(np.uint8)\n\ + \n # Create image\n img = Image.fromarray(pixel_array)\n\n \ + \ # Display\n plt.figure(figsize=(15, 10))\n plt.imshow(img,\ + \ cmap='gray')\n plt.axis('off')\n plt.title(f\"DICOM Image -\ + \ Patient: {ds.PatientID}\")\n plt.show()\n else:\n print(\"\ + No pixel data found in the DICOM file\")\n\nexcept Exception as e:\n print(f\"\ + An error occurred: {str(e)}\")\nfinally:\n # Clean up\n if os.path.exists(local_file):\n\ + \ os.remove(local_file)\n os.rmdir(temp_dir)\n" + integration: imaging_data_commons + notes: null + query: Download and visualize a slide microscopy image with pydicom and matplotlib + resource_id: c19b78ab-29cb-4be3-8069-1ef22302dfb2 + resource_type: example + dc0ce97c-46a2-406b-a582-a51be49367c9: + code: "from idc_index import index\nimport pandas as pd\n\nclient = index.IDCClient()\n\ + \n# Query to get download URLs for bone marrow slides\nquery = \"\"\"\nSELECT\n\ + \ PatientID,\n SeriesDescription,\n series_aws_url,\n series_size_MB\n\ + FROM\n index\nWHERE\n Modality = 'SM'\n AND collection_id IN ('cptac_aml',\ + \ 'cmb_aml')\n AND SeriesDescription LIKE '%bone marrow%'\nLIMIT 1\n\"\"\"\ + \n\ntry:\n df = client.sql_query(query)\n print(\"Sample slide information:\"\ + )\n print(df)\n \n if not df.empty:\n url = df['series_aws_url'].iloc[0]\n\ + \ size_mb = df['series_size_MB'].iloc[0]\n print(f\"\\nFile size:\ + \ {size_mb:.2f} MB\")\n print(f\"Download URL: {url}\")\nexcept Exception\ + \ as e:\n print(f\"An error occurred: {str(e)}\")\n" + integration: imaging_data_commons + notes: null + query: Query to get download URLs for bone marrow slides + resource_id: dc0ce97c-46a2-406b-a582-a51be49367c9 + resource_type: example + e0279959-d257-438e-8513-fb40391082c1: + code: "from idc_index import index\nimport pandas as pd\n\nclient = index.IDCClient()\n\ + \n# Query for AML-specific slide microscopy data\nquery = \"\"\"\nSELECT DISTINCT\n\ + \ collection_id,\n PatientID,\n SeriesDescription,\n StudyDescription,\n\ + \ BodyPartExamined,\n SeriesDate,\n Manufacturer,\n ManufacturerModelName,\n\ + \ SeriesNumber,\n instanceCount\nFROM\n index\nWHERE\n Modality\ + \ = 'SM'\n AND collection_id IN ('cptac_aml', 'cmb_aml')\nORDER BY\n PatientID,\ + \ SeriesDate\n\"\"\"\n\ntry:\n df = client.sql_query(query)\n print(\"\ + AML Slide Microscopy Data:\")\n print(\"\\nTotal number of records:\", len(df))\n\ + \n print(\"\\nUnique patients:\", len(df['PatientID'].unique()))\n\n print(\"\ + \\nTypes of series descriptions (slide types):\")\n print(df['SeriesDescription'].unique())\n\ + \n print(\"\\nSample of the data:\")\n print(df.head(10))\n\n # Save\ + \ to a CSV file for further analysis\n df.to_csv('aml_pathology_data.csv',\ + \ index=False)\n print(\"\\nData has been saved to 'aml_pathology_data.csv'\"\ + )\n\nexcept Exception as e:\n print(f\"An error occurred: {str(e)}\") \n" + integration: imaging_data_commons + notes: null + query: Get slide microscopy data for a specific collection + resource_id: e0279959-d257-438e-8513-fb40391082c1 + resource_type: example + f5ffd878-c1c9-46e9-b83c-6fd54fe975a5: + code: | + import boto3 + from botocore import UNSIGNED + from botocore.config import Config + import os + import tempfile + + # Function to list contents of an S3 path + def list_s3_contents(bucket, prefix): + s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED)) + try: + result = s3.list_objects_v2(Bucket=bucket, Prefix=prefix) + if 'Contents' in result: + return result['Contents'] + return [] + except Exception as e: + print(f"Error listing S3 contents: {str(e)}") + return [] + + # Parse S3 URL and list contents + s3_url = "s3://idc-open-data/d53e7fc3-8ef9-4de5-8059-47b21a67eb4f" + bucket = s3_url.split('/')[2] + prefix = '/'.join(s3_url.split('/')[3:]) + + print(f"Checking contents of bucket: {bucket}") + print(f"With prefix: {prefix}") + + contents = list_s3_contents(bucket, prefix) + print("\nFound files:") + for item in contents: + print(f"- {item['Key']} ({item['Size']/1024/1024:.2f} MB)") + + # Let's also check if we can get any pre-signed URLs or public access URLs + s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED)) + try: + # Try to get the bucket location + location = s3.get_bucket_location(Bucket=bucket) + print(f"\nBucket location: {location}") - # Additional Instructions: + # Try to get bucket policy + try: + policy = s3.get_bucket_policy(Bucket=bucket) + print("\nBucket policy:", policy) + except Exception as e: + print("\nCouldn't get bucket policy:", str(e)) - Be sure to import and instantiate the client for the Imaging Data Commons API: - ```python - from idc_index import index - client = index.IDCClient() - ``` + except Exception as e: + print(f"\nError getting bucket information: {str(e)}") + integration: imaging_data_commons + notes: null + query: Get list of files in an S3 bucket for a specific collection on IDC + resource_id: f5ffd878-c1c9-46e9-b83c-6fd54fe975a5 + resource_type: example +slug: imaging_data_commons diff --git a/src/biome/adhoc_data/datasources/idc/examples.yaml b/src/biome/adhoc_data/datasources/idc/examples.yaml deleted file mode 100644 index 65ff8df..0000000 --- a/src/biome/adhoc_data/datasources/idc/examples.yaml +++ /dev/null @@ -1,444 +0,0 @@ -- query: "Get columns available in IDC index" - code: | - from idc_index import index - import pandas as pd - - client = index.IDCClient() - - # First, let's see what columns are available - query_columns = """ - SELECT * FROM index LIMIT 1 - """ - - try: - df = client.sql_query(query_columns) - print("Available columns in the index:") - print(df.columns.tolist()) - except Exception as e: - print(f"An error occurred: {str(e)}") - -- query: "Get slide microscopy data" - code: | - from idc_index import index - import pandas as pd - - client = index.IDCClient() - - # Query for slide microscopy data - query = """ - SELECT DISTINCT - collection_id, - PatientID, - SeriesDescription, - StudyDescription, - BodyPartExamined, - SeriesDate - FROM - index - WHERE - Modality = 'SM' - """ - - try: - df = client.sql_query(query) - print("Slide Microscopy Data Available:") - print("\nCollections with slide microscopy data:") - print(df['collection_id'].unique()) - print("\nSample of the data:") - print(df.head()) - print("\nTotal number of records:", len(df)) - except Exception as e: - print(f"An error occurred: {str(e)}") - -- query: "Get slide microscopy data for a specific collection" - code: | - from idc_index import index - import pandas as pd - - client = index.IDCClient() - - # Query for AML-specific slide microscopy data - query = """ - SELECT DISTINCT - collection_id, - PatientID, - SeriesDescription, - StudyDescription, - BodyPartExamined, - SeriesDate, - Manufacturer, - ManufacturerModelName, - SeriesNumber, - instanceCount - FROM - index - WHERE - Modality = 'SM' - AND collection_id IN ('cptac_aml', 'cmb_aml') - ORDER BY - PatientID, SeriesDate - """ - - try: - df = client.sql_query(query) - print("AML Slide Microscopy Data:") - print("\nTotal number of records:", len(df)) - - print("\nUnique patients:", len(df['PatientID'].unique())) - - print("\nTypes of series descriptions (slide types):") - print(df['SeriesDescription'].unique()) - - print("\nSample of the data:") - print(df.head(10)) - - # Save to a CSV file for further analysis - df.to_csv('aml_pathology_data.csv', index=False) - print("\nData has been saved to 'aml_pathology_data.csv'") - - except Exception as e: - print(f"An error occurred: {str(e)}") - -- query: "Get additional information for microscopy data for a specific collection" - code: | - from idc_index import index - import pandas as pd - - client = index.IDCClient() - - # Query for all available fields for AML slides - query = """ - SELECT * - FROM - index - WHERE - Modality = 'SM' - AND collection_id IN ('cptac_aml', 'cmb_aml') - AND SeriesDescription LIKE '%bone marrow%' - ORDER BY - PatientID, SeriesDate - """ - - try: - df = client.sql_query(query) - print("AML Bone Marrow Slide Data:") - print("\nTotal number of bone marrow slides:", len(df)) - - print("\nAvailable columns (metadata fields):") - print(df.columns.tolist()) - - # Look for any columns that might contain annotations or clinical data - clinical_cols = [col for col in df.columns if any(term in col.lower() - for term in ['annotation', 'clinical', 'blast', 'cell', - 'pathology', 'diagnosis', 'grade', 'stage'])] - print("\nPotentially relevant clinical/annotation columns:") - print(clinical_cols) - - # Get the source DOIs which might lead to additional clinical data - print("\nUnique source DOIs (for finding additional clinical data):") - print(df['source_DOI'].unique()) - - except Exception as e: - print(f"An error occurred: {str(e)}") - - # Let's also check if there are any study-level annotations - query_study = """ - SELECT DISTINCT - StudyDescription, - collection_id, - source_DOI, - license_short_name - FROM - index - WHERE - collection_id IN ('cptac_aml', 'cmb_aml') - """ - - try: - df_study = client.sql_query(query_study) - print("\nStudy-level information:") - print(df_study) - except Exception as e: - print(f"An error occurred querying study data: {str(e)}") - -- query: "Query to get download URLs for bone marrow slides" - code: | - from idc_index import index - import pandas as pd - - client = index.IDCClient() - - # Query to get download URLs for bone marrow slides - query = """ - SELECT - PatientID, - SeriesDescription, - series_aws_url, - series_size_MB - FROM - index - WHERE - Modality = 'SM' - AND collection_id IN ('cptac_aml', 'cmb_aml') - AND SeriesDescription LIKE '%bone marrow%' - LIMIT 1 - """ - - try: - df = client.sql_query(query) - print("Sample slide information:") - print(df) - - if not df.empty: - url = df['series_aws_url'].iloc[0] - size_mb = df['series_size_MB'].iloc[0] - print(f"\nFile size: {size_mb:.2f} MB") - print(f"Download URL: {url}") - except Exception as e: - print(f"An error occurred: {str(e)}") - -- query: "Get list of files in an S3 bucket for a specific collection on IDC" - code: | - import boto3 - from botocore import UNSIGNED - from botocore.config import Config - import os - import tempfile - - # Function to list contents of an S3 path - def list_s3_contents(bucket, prefix): - s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED)) - try: - result = s3.list_objects_v2(Bucket=bucket, Prefix=prefix) - if 'Contents' in result: - return result['Contents'] - return [] - except Exception as e: - print(f"Error listing S3 contents: {str(e)}") - return [] - - # Parse S3 URL and list contents - s3_url = "s3://idc-open-data/d53e7fc3-8ef9-4de5-8059-47b21a67eb4f" - bucket = s3_url.split('/')[2] - prefix = '/'.join(s3_url.split('/')[3:]) - - print(f"Checking contents of bucket: {bucket}") - print(f"With prefix: {prefix}") - - contents = list_s3_contents(bucket, prefix) - print("\nFound files:") - for item in contents: - print(f"- {item['Key']} ({item['Size']/1024/1024:.2f} MB)") - - # Let's also check if we can get any pre-signed URLs or public access URLs - s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED)) - try: - # Try to get the bucket location - location = s3.get_bucket_location(Bucket=bucket) - print(f"\nBucket location: {location}") - - # Try to get bucket policy - try: - policy = s3.get_bucket_policy(Bucket=bucket) - print("\nBucket policy:", policy) - except Exception as e: - print("\nCouldn't get bucket policy:", str(e)) - - except Exception as e: - print(f"\nError getting bucket information: {str(e)}") - -- query: "Download and visualize a slide microscopy image with pydicom and matplotlib" - code: | - import sys - import subprocess - - # Install required package - subprocess.check_call([sys.executable, "-m", "pip", "install", "pydicom"]) - - import boto3 - from botocore import UNSIGNED - from botocore.config import Config - import os - import tempfile - import pydicom - from PIL import Image - import numpy as np - import matplotlib.pyplot as plt - - # Create S3 client - s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED)) - - # Create temporary directory - temp_dir = tempfile.mkdtemp() - - # Let's try the smallest DICOM file first - file_key = "d53e7fc3-8ef9-4de5-8059-47b21a67eb4f/40006642-6ee5-44ca-bc5f-e1cc2bd7ee70.dcm" - local_file = os.path.join(temp_dir, "slide.dcm") - - try: - print(f"Downloading file: {file_key}") - s3.download_file('idc-open-data', file_key, local_file) - print("Download successful!") - - # Try to read the DICOM file - print("\nReading DICOM file...") - ds = pydicom.dcmread(local_file) - - print("\nDICOM metadata:") - print(f"Patient ID: {ds.PatientID}") - print(f"Modality: {ds.Modality}") - print(f"Image Type: {ds.ImageType}") - - # Convert to image and display - if hasattr(ds, 'pixel_array'): - print("\nConverting to image...") - pixel_array = ds.pixel_array - - # Normalize the pixel values - if pixel_array.dtype != np.uint8: - pixel_array = ((pixel_array - pixel_array.min()) * 255.0 / - (pixel_array.max() - pixel_array.min())).astype(np.uint8) - - # Create image - img = Image.fromarray(pixel_array) - - # Display - plt.figure(figsize=(15, 10)) - plt.imshow(img, cmap='gray') - plt.axis('off') - plt.title(f"DICOM Image - Patient: {ds.PatientID}") - plt.show() - else: - print("No pixel data found in the DICOM file") - - except Exception as e: - print(f"An error occurred: {str(e)}") - finally: - # Clean up - if os.path.exists(local_file): - os.remove(local_file) - os.rmdir(temp_dir) - - -- query: "Find collections in the Imaging Data Commons with 'aml' in their name, likely related to Acute Myeloid Leukemia." - code: | - from idc_index import index - import pandas as pd - - client = index.IDCClient() - - # Query to find collection names containing 'aml' - query = """ - SELECT DISTINCT collection_id - FROM - index - WHERE - LOWER(collection_id) LIKE '%aml%' - """ - - try: - df_collections = client.sql_query(query) - print("Collections with 'aml' in their name:") - print(df_collections) - except Exception as e: - print(f"An error occurred: {str(e)}") - -- query: "Download bone marrow biopsy images from the 'cmb_aml' collection for a specific study using IDC's public S3 bucket." - code: | - import pandas as pd - from idc_index import index - - client = index.IDCClient() - - # Query for the desired images - query = """ - SELECT series_aws_url - FROM index - WHERE collection_id = 'cmb_aml' - AND StudyDescription = 'XR_Biopsy_BoneMarrow' - """ - df = client.sql_query(query) - - # Create a download manifest using the current directory - with open('download_manifest.txt', 'w') as f: - for url in df['series_aws_url']: - f.write(f'cp {url} ./\n') - - # Execute the download - s5cmd_binary = client.s5cmdPath - !{s5cmd_binary} --no-sign-request --endpoint-url https://s3.amazonaws.com run download_manifest.txt - -- query: "Basic plot of dicom image from file" - code: | - import pydicom - import matplotlib.pyplot as plt - - # Load the DICOM file - filename = '73f18e49-8699-4c1f-90ad-5db94b6b5602.dcm' - ds = pydicom.dcmread(filename) - - # Display the image - plt.imshow(ds.pixel_array, cmap=plt.cm.gray) - plt.title('Bone Marrow Biopsy Image') - plt.axis('off') - plt.show() - -- query: "List all studies for a given collection" - code: | - # Query to list all studies in the 'cmb_aml' collection - query_all_studies = """ - SELECT DISTINCT StudyDescription - FROM - index - WHERE - collection_id = 'cmb_aml' - """ - - try: - df_all_studies = client.sql_query(query_all_studies) - print("All studies in the 'cmb_aml' collection:") - print(df_all_studies) - except Exception as e: - print(f"An error occurred: {str(e)}") - -- query: "Retrieve imagery URLs for the 'XR_Biopsy_BoneMarrow' study in the 'cmb_aml' collection from IDC." - code: | - import pandas as pd - from idc_index import index - - client = index.IDCClient() - - query = """ - SELECT series_aws_url - FROM index - WHERE collection_id = 'cmb_aml' - AND StudyDescription = 'XR_Biopsy_BoneMarrow' - """ - df = client.sql_query(query) - print(df.head()) - -- query: "Fetch studies for a given collection in the Imaging Data Commons (IDC), specifically for the 'cptac_aml' collection." - code: | - from idc_index import index - import pandas as pd - - # Initialize the IDC client - client = index.IDCClient() - - # Query to find studies within the 'cptac_aml' collection - query = """ - SELECT DISTINCT StudyDescription - FROM - index - WHERE - collection_id = 'cptac_aml' - """ - - try: - # Execute the query - df_studies = client.sql_query(query) - print("Studies in 'cptac_aml':") - print(df_studies) - except Exception as e: - print(f"An error occurred: {str(e)}") - \ No newline at end of file diff --git a/src/biome/adhoc_data/datasources/indra/api.yaml b/src/biome/adhoc_data/datasources/indra/api.yaml index 7996527..c03c8e9 100644 --- a/src/biome/adhoc_data/datasources/indra/api.yaml +++ b/src/biome/adhoc_data/datasources/indra/api.yaml @@ -1,40 +1,233 @@ -name: INDRA Context Graph Extension (CoGEx) -integration_type: api description: | - INDRA CoGEx (Context Graph Extension) is an automatically assembled biomedical knowledge graph which integrates causal mechanisms from INDRA - with non-causal contextual relations including properties, ontology, and data. + INDRA CoGEx (Context Graph Extension) is an automatically assembled biomedical knowledge graph which integrates causal mechanisms from INDRA + with non-causal contextual relations including properties, ontology, and data. - The INDRA CoGEx Query API is a RESTful service designed to streamline complex queries within the biomedical domain, - powered by the Contextual Graph Explorer (CoGEx) framework. It provides an interface to retrieve detailed insights - by integrating and analyzing data from diverse biomedical sources. Users can execute queries to uncover relationships - such as diseases associated with specific clinical trials, drugs linked to particular side effects, or other intricate - connections in biomedical knowledge. By leveraging its contextual graph-based approach, the API enables researchers, - data scientists, and healthcare professionals to explore and analyze rich biomedical data efficiently, supporting - advancements in drug discovery, clinical decision-making, and research. With its robust architecture, the INDRA CoGEx - Query API facilitates the discovery of meaningful patterns and supports informed decision-making in complex biomedical scenarios. + The INDRA CoGEx Query API is a RESTful service designed to streamline complex queries within the biomedical domain, + powered by the Contextual Graph Explorer (CoGEx) framework. It provides an interface to retrieve detailed insights + by integrating and analyzing data from diverse biomedical sources. Users can execute queries to uncover relationships + such as diseases associated with specific clinical trials, drugs linked to particular side effects, or other intricate + connections in biomedical knowledge. By leveraging its contextual graph-based approach, the API enables researchers, + data scientists, and healthcare professionals to explore and analyze rich biomedical data efficiently, supporting + advancements in drug discovery, clinical decision-making, and research. With its robust architecture, the INDRA CoGEx + Query API facilitates the discovery of meaningful patterns and supports informed decision-making in complex biomedical scenarios. +integration_type: api +name: INDRA Context Graph Extension (CoGEx) +prompt: | + ${raw_documentation} -attachments: - raw_documentation: indra.json + # Additional Instructions: -prompt: | - ${raw_documentation} - - # Additional Instructions: - - You should make use of the python requests library to interact with the API. - Note that the base URL of the API is 'https://discovery.indra.bio/api/' - - For example, here is an example request: - - curl -X 'POST' \ - 'https://discovery.indra.bio/api/get_go_terms_for_gene' \ - -H 'accept: application/json' \ - -H 'Content-Type: application/json' \ - -d '{ - "gene": [ - "HGNC", - "9896" - ], - "include_indirect": true - }' - ``` + You should make use of the python requests library to interact with the API. + Note that the base URL of the API is 'https://discovery.indra.bio/api/' + + For example, here is an example request: + + curl -X 'POST' \ + 'https://discovery.indra.bio/api/get_go_terms_for_gene' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "gene": [ + "HGNC", + "9896" + ], + "include_indirect": true + }' + ``` +resources: + b7c4d45f-287b-47e2-aa9b-5ee5bae07353: + code: | + import requests + + base_url = "https://discovery.indra.bio/api/" + + # Get pathways for JAK2 + jak2_payload = { + "gene": ["HGNC", "6192"] # JAK2 HGNC ID + } + + # Get JAK2 pathways + jak2_pathways_response = requests.post( + base_url + "get_pathways_for_gene", + json=jak2_payload + ) + + print("JAK2 Pathways Response:") + print(jak2_pathways_response.json()) + + # Get shared pathways with common partner genes + partner_genes = [ + ["HGNC", "6192"], # JAK2 + ["HGNC", "3603"], # STAT3 + ["HGNC", "6193"], # JAK3 + ["HGNC", "3236"] # EGFR + ] + + shared_payload = { + "genes": partner_genes + } + + # Get shared pathways + shared_pathways_response = requests.post( + base_url + "get_shared_pathways_for_genes", + json=shared_payload + ) + + print("\nShared Pathways Response:") + print(shared_pathways_response.json()) + integration: indra_context_graph_extension_(cogex) + notes: null + query: |- + Perform a gene pathway analysis for JAK2 to find associated pathways and shared pathways with other genes like STAT3, JAK3, and EGFR. + resource_id: b7c4d45f-287b-47e2-aa9b-5ee5bae07353 + resource_type: example + c8f8db77-c0bb-4ee2-9525-925707496b5a: + code: | + import requests + import json + + def get_mesh_literature(): + # API base URL + base_url = "https://discovery.indra.bio/api/" + + # Get papers with evidence for AML MeSH term + payload = { + "mesh_term": ["MESH", "D015470"], + "include_child_terms": True + } + + # Make request to get evidences + response = requests.post( + base_url + "get_evidences_for_mesh", + json=payload, + headers={"Content-Type": "application/json"} + ) + + if response.status_code == 200: + # Process and return results + results = response.json() + + # Convert results to a more readable format + evidence_by_statement = {} + for stmt_hash, evidences in results.items(): + evidence_by_statement[stmt_hash] = { + 'pmids': list(set([ev['text_refs'].get('PMID') for ev in evidences if ev.get('text_refs', {}).get('PMID')])), + 'evidence_count': len(evidences) + } + + return evidence_by_statement + else: + print(f"Error: {response.status_code}") + return None + + if __name__ == "__main__": + evidence_data = get_mesh_literature() + + if evidence_data: + # Print summary of findings + print(f"Found {len(evidence_data)} statements with evidence") + + # Get total evidence count + total_evidence = sum(data['evidence_count'] for data in evidence_data.values()) + print(f"Total evidence count: {total_evidence}") + + # Get unique PMIDs + all_pmids = set() + for data in evidence_data.values(): + all_pmids.update(data['pmids']) + print(f"Total unique PMIDs: {len(all_pmids)}") + integration: indra_context_graph_extension_(cogex) + notes: null + query: Retrieve literature evidence related to AML using MeSH term queries, including + child terms, and summarize the findings. + resource_id: c8f8db77-c0bb-4ee2-9525-925707496b5a + resource_type: example + e57fbdf9-732a-407a-8469-921076b69054: + code: | + import requests + + def analyze_jak2(): + # API base URL + base_url = "https://discovery.indra.bio/api/" + + # JAK2 HGNC ID + jak2_node = ["HGNC", "6192"] + + # Get GO terms + go_terms_payload = { + "gene": jak2_node, + "include_indirect": True + } + go_response = requests.post( + base_url + "get_go_terms_for_gene", + json=go_terms_payload + ) + + # Get pathways + pathways_payload = { + "gene": jak2_node + } + pathways_response = requests.post( + base_url + "get_pathways_for_gene", + json=pathways_payload + ) + + # Check if kinase + kinase_payload = { + "genes": ["JAK2"] + } + kinase_response = requests.post( + base_url + "is_kinase", + json=kinase_payload + ) + + # Check if phosphatase + phosphatase_payload = { + "genes": ["JAK2"] + } + phosphatase_response = requests.post( + base_url + "is_phosphatase", + json=phosphatase_payload + ) + + # Check if transcription factor + tf_payload = { + "genes": ["JAK2"] + } + tf_response = requests.post( + base_url + "is_transcription_factor", + json=tf_payload + ) + + # Compile and return results + results = { + "go_terms": go_response.json(), + "pathways": pathways_response.json(), + "is_kinase": kinase_response.json().get("JAK2", False), + "is_phosphatase": phosphatase_response.json().get("JAK2", False), + "is_transcription_factor": tf_response.json().get("JAK2", False) + } + + return results + + if __name__ == "__main__": + results = analyze_jak2() + print(f"Analysis Results for JAK2:") + print(f"Number of GO terms: {len(results['go_terms'])}") + print(f"Number of pathways: {len(results['pathways'])}") + print(f"Is kinase: {results['is_kinase']}") + print(f"Is phosphatase: {results['is_phosphatase']}") + print(f"Is transcription factor: {results['is_transcription_factor']}") + integration: indra_context_graph_extension_(cogex) + notes: null + query: |- + This example demonstrates how to analyze the JAK2 gene using the INDRA CoGEx API to retrieve GO terms, pathway associations, and check if it is a kinase, phosphatase, or transcription factor. + resource_id: e57fbdf9-732a-407a-8469-921076b69054 + resource_type: example + f49c8aa7-727e-444e-a3a2-314a5f0d6631: + filepath: indra.json + integration: indra_context_graph_extension_(cogex) + name: raw_documentation + resource_id: f49c8aa7-727e-444e-a3a2-314a5f0d6631 + resource_type: file +slug: indra_context_graph_extension_(cogex) diff --git a/src/biome/adhoc_data/datasources/indra/examples.yaml b/src/biome/adhoc_data/datasources/indra/examples.yaml deleted file mode 100644 index 546a8a2..0000000 --- a/src/biome/adhoc_data/datasources/indra/examples.yaml +++ /dev/null @@ -1,175 +0,0 @@ -- query: "Retrieve literature evidence related to AML using MeSH term queries, including child terms, and summarize the findings." - code: | - import requests - import json - - def get_mesh_literature(): - # API base URL - base_url = "https://discovery.indra.bio/api/" - - # Get papers with evidence for AML MeSH term - payload = { - "mesh_term": ["MESH", "D015470"], - "include_child_terms": True - } - - # Make request to get evidences - response = requests.post( - base_url + "get_evidences_for_mesh", - json=payload, - headers={"Content-Type": "application/json"} - ) - - if response.status_code == 200: - # Process and return results - results = response.json() - - # Convert results to a more readable format - evidence_by_statement = {} - for stmt_hash, evidences in results.items(): - evidence_by_statement[stmt_hash] = { - 'pmids': list(set([ev['text_refs'].get('PMID') for ev in evidences if ev.get('text_refs', {}).get('PMID')])), - 'evidence_count': len(evidences) - } - - return evidence_by_statement - else: - print(f"Error: {response.status_code}") - return None - - if __name__ == "__main__": - evidence_data = get_mesh_literature() - - if evidence_data: - # Print summary of findings - print(f"Found {len(evidence_data)} statements with evidence") - - # Get total evidence count - total_evidence = sum(data['evidence_count'] for data in evidence_data.values()) - print(f"Total evidence count: {total_evidence}") - - # Get unique PMIDs - all_pmids = set() - for data in evidence_data.values(): - all_pmids.update(data['pmids']) - print(f"Total unique PMIDs: {len(all_pmids)}") - - -- query: "Perform a gene pathway analysis for JAK2 to find associated pathways and shared pathways with other genes like STAT3, JAK3, and EGFR." - code: | - import requests - - base_url = "https://discovery.indra.bio/api/" - - # Get pathways for JAK2 - jak2_payload = { - "gene": ["HGNC", "6192"] # JAK2 HGNC ID - } - - # Get JAK2 pathways - jak2_pathways_response = requests.post( - base_url + "get_pathways_for_gene", - json=jak2_payload - ) - - print("JAK2 Pathways Response:") - print(jak2_pathways_response.json()) - - # Get shared pathways with common partner genes - partner_genes = [ - ["HGNC", "6192"], # JAK2 - ["HGNC", "3603"], # STAT3 - ["HGNC", "6193"], # JAK3 - ["HGNC", "3236"] # EGFR - ] - - shared_payload = { - "genes": partner_genes - } - - # Get shared pathways - shared_pathways_response = requests.post( - base_url + "get_shared_pathways_for_genes", - json=shared_payload - ) - - print("\nShared Pathways Response:") - print(shared_pathways_response.json()) - - -- query: "This example demonstrates how to analyze the JAK2 gene using the INDRA CoGEx API to retrieve GO terms, pathway associations, and check if it is a kinase, phosphatase, or transcription factor." - code: | - import requests - - def analyze_jak2(): - # API base URL - base_url = "https://discovery.indra.bio/api/" - - # JAK2 HGNC ID - jak2_node = ["HGNC", "6192"] - - # Get GO terms - go_terms_payload = { - "gene": jak2_node, - "include_indirect": True - } - go_response = requests.post( - base_url + "get_go_terms_for_gene", - json=go_terms_payload - ) - - # Get pathways - pathways_payload = { - "gene": jak2_node - } - pathways_response = requests.post( - base_url + "get_pathways_for_gene", - json=pathways_payload - ) - - # Check if kinase - kinase_payload = { - "genes": ["JAK2"] - } - kinase_response = requests.post( - base_url + "is_kinase", - json=kinase_payload - ) - - # Check if phosphatase - phosphatase_payload = { - "genes": ["JAK2"] - } - phosphatase_response = requests.post( - base_url + "is_phosphatase", - json=phosphatase_payload - ) - - # Check if transcription factor - tf_payload = { - "genes": ["JAK2"] - } - tf_response = requests.post( - base_url + "is_transcription_factor", - json=tf_payload - ) - - # Compile and return results - results = { - "go_terms": go_response.json(), - "pathways": pathways_response.json(), - "is_kinase": kinase_response.json().get("JAK2", False), - "is_phosphatase": phosphatase_response.json().get("JAK2", False), - "is_transcription_factor": tf_response.json().get("JAK2", False) - } - - return results - - if __name__ == "__main__": - results = analyze_jak2() - print(f"Analysis Results for JAK2:") - print(f"Number of GO terms: {len(results['go_terms'])}") - print(f"Number of pathways: {len(results['pathways'])}") - print(f"Is kinase: {results['is_kinase']}") - print(f"Is phosphatase: {results['is_phosphatase']}") - print(f"Is transcription factor: {results['is_transcription_factor']}") diff --git a/src/biome/adhoc_data/datasources/netrias/api.yaml b/src/biome/adhoc_data/datasources/netrias/api.yaml index 0d494c1..1f03f52 100644 --- a/src/biome/adhoc_data/datasources/netrias/api.yaml +++ b/src/biome/adhoc_data/datasources/netrias/api.yaml @@ -1,58 +1,79 @@ -name: Netrias Harmonization API -integration_type: api - description: | - The Netrias Harmonization REST API leverages AI-driven harmonization approaches to standardize - cancer-related terminologies from the Cancer Data Service (CDS), managed by the Center for - Biomedical Informatics and Information Technology (CBIIT) at NIH. - - This API is designed to process and harmonize textual variations of terms, mapping them to - standardized terms defined in the CDS data model ontology (version 5.0.5). By utilizing - advanced machine learning techniques, the service enhances data consistency, improves - interoperability, and streamlines standardization for cancer research and biomedical - informatics applications. - - The API enables automated term reconciliation, ensuring uniform terminology across datasets - for more reliable data integration and analysis. The current intended application is for - harmonization of terms in study metadata from cancer researchers delivering research data - to the Cancer Research Data Commons. - -attachments: - raw_documentation: netrias.json - -prompt: | - ${raw_documentation} - - # Additional Instructions: + The Netrias Harmonization REST API leverages AI-driven harmonization approaches to standardize + cancer-related terminologies from the Cancer Data Service (CDS), managed by the Center for + Biomedical Informatics and Information Technology (CBIIT) at NIH. - The REST API is accessible at https://apieval.netriaslabs.cloud/. - It uses API Keys to track access. It is just applied as an additional HTTP header `x-api-key: ` + This API is designed to process and harmonize textual variations of terms, mapping them to + standardized terms defined in the CDS data model ontology (version 5.0.5). By utilizing + advanced machine learning techniques, the service enhances data consistency, improves + interoperability, and streamlines standardization for cancer research and biomedical + informatics applications. - **IMPORTANT**: the API Key is available in the environment variable `NETRIAS_KEY`. - - For CDS standard terms, commonly referred to as Common Data Elements or CDEs, with large numbers of permissible values, - Netrias builds fine tuned LLMs from synthetically generated training data. If a CDE has a small number of permissible values, - we apply in-context learning approaches where we craft a prompt to feed an LLM API chat endpoint (e.g. OpenAI) requesting it - generate harmonizations with no or few examples. When used as a tool, this may lead to significant variations in response - latency as the fine tuned models may take some time to load into a GPU, tokenize the input, and infer the intended response. - - The REST API also supports the harmonization of terms from ontologies supporting Sage Bionetworks' data repositories focused - on rare diseases and cancers. In a demonstration integration of the API with Sage's Synapse chatbot application the teams - focused on four types of variations. Here are some examples: - - 1. Typos - a. neurofibromatisis type ! → Neurofibromatosis type 1 - b. whole exme squencing → whole exome sequencing - - 2. Exact synonyms - a. NF-1 → Neurofibromatosis type 1 - b. WEX → whole exome sequencing - - 3. Narrow synonyms - a. Watson Syndrome → Neurofibromatosis type 1 - b. clinical whole-exome sequencing → whole exome sequencing - - 4. Broad synonyms - a. NF → Neurofibromatosis type 1 - b. DNA sequencing → whole exome sequencing + The API enables automated term reconciliation, ensuring uniform terminology across datasets + for more reliable data integration and analysis. The current intended application is for + harmonization of terms in study metadata from cancer researchers delivering research data + to the Cancer Research Data Commons. +integration_type: api +name: Netrias Harmonization API +prompt: "${raw_documentation}\n\n# Additional Instructions:\n\nThe REST API is accessible\ + \ at https://apieval.netriaslabs.cloud/.\nIt uses API Keys to track access. It is\ + \ just applied as an additional HTTP header `x-api-key: `\n\n**IMPORTANT**:\ + \ the API Key is available in the environment variable `NETRIAS_KEY`.\n\nFor CDS\ + \ standard terms, commonly referred to as Common Data Elements or CDEs, with large\ + \ numbers of permissible values,\nNetrias builds fine tuned LLMs from synthetically\ + \ generated training data. If a CDE has a small number of permissible values,\n\ + we apply in-context learning approaches where we craft a prompt to feed an LLM API\ + \ chat endpoint (e.g. OpenAI) requesting it\ngenerate harmonizations with no or\ + \ few examples. When used as a tool, this may lead to significant variations in\ + \ response\nlatency as the fine tuned models may take some time to load into a GPU,\ + \ tokenize the input, and infer the intended response.\n\nThe REST API also supports\ + \ the harmonization of terms from ontologies supporting Sage Bionetworks' data repositories\ + \ focused\non rare diseases and cancers. In a demonstration integration of the API\ + \ with Sage's Synapse chatbot application the teams\nfocused on four types of variations.\ + \ Here are some examples:\n\n1. Typos\n a. neurofibromatisis type ! \u2192 Neurofibromatosis\ + \ type 1\n b. whole exme squencing \u2192 whole exome sequencing\n\n2. Exact\ + \ synonyms\n a. NF-1 \u2192 Neurofibromatosis type 1\n b. WEX \u2192 whole\ + \ exome sequencing\n\n3. Narrow synonyms\n a. Watson Syndrome \u2192 Neurofibromatosis\ + \ type 1\n b. clinical whole-exome sequencing \u2192 whole exome sequencing\n\ + \n4. Broad synonyms\n a. NF \u2192 Neurofibromatosis type 1\n b. DNA sequencing\ + \ \u2192 whole exome sequencing\n" +resources: + 2030d84d-840e-44a1-89d4-f027e2158a31: + code: "import requests\nimport os\nimport json\nimport difflib\n\n# Get API key\ + \ from environment variable\napi_key = os.environ.get(\"NETRIAS_KEY\")\n\n#\ + \ Set up the request headers\nheaders = {\n \"x-api-key\": api_key,\n \ + \ \"Content-Type\": \"application/json\"\n}\n\n# Base URL for the API\nbase_url\ + \ = \"https://apieval.netriaslabs.cloud\"\n\n# The drug term to harmonize\n\ + drug_term = \"zorubicin_9012\"\n\n# Get the standards to find therapeutic agents\n\ + standards_url = f\"{base_url}/v1/standards\"\nstandards_response = requests.get(standards_url,\ + \ headers=headers)\n\nif standards_response.status_code == 200:\n standards\ + \ = standards_response.json()\n \n # Look for therapeutic agents in the\ + \ DataHub standards\n if \"datahub\" in standards and \"therapeutic_agents\"\ + \ in standards[\"datahub\"]:\n therapeutic_agents = standards[\"datahub\"\ + ][\"therapeutic_agents\"]\n \n # Find closest matches using difflib\n\ + \ matches = difflib.get_close_matches(drug_term, therapeutic_agents,\ + \ n=5, cutoff=0.5)\n \n print(f\"Found closest matches for '{drug_term}':\"\ + )\n for match in matches:\n print(f\" {match}\")\n \ + \ \n if matches:\n print(f\"\\nBest harmonization result:\ + \ '{matches[0]}'\")\n else:\n print(f\"No close matches found\ + \ for '{drug_term}'\")\n else:\n print(\"Could not find therapeutic_agents\ + \ in DataHub standards\")\nelse:\n print(f\"Error getting standards: {standards_response.status_code}\"\ + )\n print(standards_response.text)\n" + integration: netrias_harmonization_api + notes: |- + This example demonstrates how to harmonize a drug term that includes a non-standard suffix (e.g., "zorubicin_9012") by: + 1. Retrieving the therapeutic_agents standard from the DataHub + 2. Using Python's difflib to find the closest matches + 3. Identifying "Zorubicin" as the standard term + The approach uses string similarity matching when the direct API harmonization doesn't return results. This is useful for handling terms with identifiers, suffixes, or other non-standard additions that aren't in the controlled vocabulary. + query: Harmonizing a drug term with a non-standard suffix + resource_id: 2030d84d-840e-44a1-89d4-f027e2158a31 + resource_type: example + 8242a126-8ef1-48de-aa94-e0ccc7e58a28: + filepath: netrias.json + integration: netrias_harmonization_api + name: raw_documentation + resource_id: 8242a126-8ef1-48de-aa94-e0ccc7e58a28 + resource_type: file +slug: netrias_harmonization_api diff --git a/src/biome/adhoc_data/datasources/netrias/examples.yaml b/src/biome/adhoc_data/datasources/netrias/examples.yaml deleted file mode 100644 index 304c4b4..0000000 --- a/src/biome/adhoc_data/datasources/netrias/examples.yaml +++ /dev/null @@ -1,62 +0,0 @@ -- query: Harmonizing a drug term with a non-standard suffix - code: | - import requests - import os - import json - import difflib - - # Get API key from environment variable - api_key = os.environ.get("NETRIAS_KEY") - - # Set up the request headers - headers = { - "x-api-key": api_key, - "Content-Type": "application/json" - } - - # Base URL for the API - base_url = "https://apieval.netriaslabs.cloud" - - # The drug term to harmonize - drug_term = "zorubicin_9012" - - # Get the standards to find therapeutic agents - standards_url = f"{base_url}/v1/standards" - standards_response = requests.get(standards_url, headers=headers) - - if standards_response.status_code == 200: - standards = standards_response.json() - - # Look for therapeutic agents in the DataHub standards - if "datahub" in standards and "therapeutic_agents" in standards["datahub"]: - therapeutic_agents = standards["datahub"]["therapeutic_agents"] - - # Find closest matches using difflib - matches = difflib.get_close_matches(drug_term, therapeutic_agents, n=5, cutoff=0.5) - - print(f"Found closest matches for '{drug_term}':") - for match in matches: - print(f" {match}") - - if matches: - print(f"\nBest harmonization result: '{matches[0]}'") - else: - print(f"No close matches found for '{drug_term}'") - else: - print("Could not find therapeutic_agents in DataHub standards") - else: - print(f"Error getting standards: {standards_response.status_code}") - print(standards_response.text) - notes: 'This example demonstrates how to harmonize a drug term that includes a non-standard - suffix (e.g., "zorubicin_9012") by: - - 1. Retrieving the therapeutic_agents standard from the DataHub - - 2. Using Python''s difflib to find the closest matches - - 3. Identifying "Zorubicin" as the standard term - - - The approach uses string similarity matching when the direct API harmonization - doesn''t return results. This is useful for handling terms with identifiers, suffixes, - or other non-standard additions that aren''t in the controlled vocabulary.' diff --git a/src/biome/adhoc_data/datasources/nhanes_dietary/api.yaml b/src/biome/adhoc_data/datasources/nhanes_dietary/api.yaml index 569c102..2b1f870 100644 --- a/src/biome/adhoc_data/datasources/nhanes_dietary/api.yaml +++ b/src/biome/adhoc_data/datasources/nhanes_dietary/api.yaml @@ -1,67 +1,70 @@ -name: "NHANES Dietary Data" -integration_type: dataset - description: | - API for accessing and analyzing NHANES dietary data, including food consumption patterns - and processed food analysis across different survey cycles from 1999 to 2023, with demographic information. - -attachments: - web_docs: docs.md - + API for accessing and analyzing NHANES dietary data, including food consumption patterns + and processed food analysis across different survey cycles from 1999 to 2023, with demographic information. +integration_type: dataset +name: NHANES Dietary Data prompt: | - You have access to NHANES Dietary Data files from multiple survey cycles, including dietary intake and demographic information: - - # 1999-2000 NHANES Cycle - - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/1999_2000_DEMOGRAPHICS.xpt - Demographic information for participants - - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/1999_2000_DRXFMT_food_codes.xpt - Food code format file with descriptions - - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/1999_2000_DRXIFF_individual_foods.xpt - Individual foods consumption data - - # 2005-2006 NHANES Cycle - - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2005_2006_DEMOGRAPHICS.xpt - Demographic information for participants - - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2005_2006_DR1IFF_individual_foods.xpt - Individual foods consumption data - - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2005_2006_DRXFCD_food_codes.xpt - Food code format file with descriptions - - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2005_2006_DRXMCD_modification_codes.xpt - Modification codes for dietary items - - # 2009-2010 NHANES Cycle - - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2009-2010_DEMOGRAPHICS.xpt - Demographic information for participants - - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2009_2010_DR1IFF_individual_foods.xpt - Individual foods consumption data - - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2009_2010_DRXFCD_food_codes.xpt - Food code format file with descriptions - - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2009_2010_DRXMCD_modification_codes.xpt - Modification codes for dietary items - - # 2015-2016 NHANES Cycle - - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2015_2016_DEMOGRAPHICS.xpt - Demographic information for participants - - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2015_2016_DR1IFF_individual_foods.xpt - Individual foods consumption data - - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2015_2016_DRXFCD_food_codes.xpt - Food code format file with descriptions - - # 2017-2020 NHANES Cycle - - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2017_2020_DEMOGRAPHICS.xpt - Demographic information for participants - - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2017_2020_DR1IFF_individual_foods.xpt - Individual foods consumption data - - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2017_2020_DRXFCD_food_codes.xpt - Food code format file with descriptions - - # 2021-2023 NHANES Cycle - - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2021_2023_DEMOGRAPHICS.xpt - Demographic information for participants - - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2021_2023_DR1IFF_individual_foods.xpt - Individual foods consumption data - - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2021_2023_DRXFCD_food_codes.xpt - Food code format file with descriptions - - Key variables: - - SEQN: Respondent sequence number (unique identifier for participants) - use this to link demographic data with dietary data - - DRDIFDCD/DR1FDCD: Food code that links to food descriptions in the format file - - DR1ILINE: Line number for individual food items - - Demographic files contain: - - Age, gender, race/ethnicity, education, income, and other sociodemographic variables - - Geographic information (limited to census region in public files) - - The food codes files (DRXFMT/DRXFCD) contain: - - Food codes and their text descriptions - - The individual foods files (DRXIFF/DR1IFF) contain detailed consumption data for each food item - consumed by participants during the 24-hour recall period. - - The modification codes files (DRXMCD) contain information about modifications made to standard - food items (e.g., fat removed from meat, salt added to vegetables). - - Note: The demographic files can be linked to the dietary files using the SEQN variable to analyze - dietary patterns by demographic characteristics such as age, gender, race/ethnicity, and socioeconomic status. - - ${web_docs} + You have access to NHANES Dietary Data files from multiple survey cycles, including dietary intake and demographic information: + + # 1999-2000 NHANES Cycle + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/1999_2000_DEMOGRAPHICS.xpt - Demographic information for participants + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/1999_2000_DRXFMT_food_codes.xpt - Food code format file with descriptions + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/1999_2000_DRXIFF_individual_foods.xpt - Individual foods consumption data + + # 2005-2006 NHANES Cycle + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2005_2006_DEMOGRAPHICS.xpt - Demographic information for participants + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2005_2006_DR1IFF_individual_foods.xpt - Individual foods consumption data + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2005_2006_DRXFCD_food_codes.xpt - Food code format file with descriptions + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2005_2006_DRXMCD_modification_codes.xpt - Modification codes for dietary items + + # 2009-2010 NHANES Cycle + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2009-2010_DEMOGRAPHICS.xpt - Demographic information for participants + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2009_2010_DR1IFF_individual_foods.xpt - Individual foods consumption data + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2009_2010_DRXFCD_food_codes.xpt - Food code format file with descriptions + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2009_2010_DRXMCD_modification_codes.xpt - Modification codes for dietary items + + # 2015-2016 NHANES Cycle + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2015_2016_DEMOGRAPHICS.xpt - Demographic information for participants + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2015_2016_DR1IFF_individual_foods.xpt - Individual foods consumption data + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2015_2016_DRXFCD_food_codes.xpt - Food code format file with descriptions + + # 2017-2020 NHANES Cycle + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2017_2020_DEMOGRAPHICS.xpt - Demographic information for participants + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2017_2020_DR1IFF_individual_foods.xpt - Individual foods consumption data + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2017_2020_DRXFCD_food_codes.xpt - Food code format file with descriptions + + # 2021-2023 NHANES Cycle + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2021_2023_DEMOGRAPHICS.xpt - Demographic information for participants + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2021_2023_DR1IFF_individual_foods.xpt - Individual foods consumption data + - ${DATASET_FILES_BASE_PATH}/nhanes_dietary/2021_2023_DRXFCD_food_codes.xpt - Food code format file with descriptions + + Key variables: + - SEQN: Respondent sequence number (unique identifier for participants) - use this to link demographic data with dietary data + - DRDIFDCD/DR1FDCD: Food code that links to food descriptions in the format file + - DR1ILINE: Line number for individual food items + + Demographic files contain: + - Age, gender, race/ethnicity, education, income, and other sociodemographic variables + - Geographic information (limited to census region in public files) + + The food codes files (DRXFMT/DRXFCD) contain: + - Food codes and their text descriptions + + The individual foods files (DRXIFF/DR1IFF) contain detailed consumption data for each food item + consumed by participants during the 24-hour recall period. + + The modification codes files (DRXMCD) contain information about modifications made to standard + food items (e.g., fat removed from meat, salt added to vegetables). + + Note: The demographic files can be linked to the dietary files using the SEQN variable to analyze + dietary patterns by demographic characteristics such as age, gender, race/ethnicity, and socioeconomic status. + + ${web_docs} +resources: + 19e60e0b-82e2-4af2-b6f2-212287b43277: + filepath: docs.md + integration: nhanes_dietary_data + name: web_docs + resource_id: 19e60e0b-82e2-4af2-b6f2-212287b43277 + resource_type: file +slug: nhanes_dietary_data diff --git a/src/biome/adhoc_data/datasources/nhanes_dietary/examples.yaml b/src/biome/adhoc_data/datasources/nhanes_dietary/examples.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/src/biome/adhoc_data/datasources/nsch/api.yaml b/src/biome/adhoc_data/datasources/nsch/api.yaml index 9275255..ace4c8e 100644 --- a/src/biome/adhoc_data/datasources/nsch/api.yaml +++ b/src/biome/adhoc_data/datasources/nsch/api.yaml @@ -1,113 +1,287 @@ -name: "National Survey of Children's Health (NSCH)" +description: | + You have access to knowledge from the National Survey of Children's Health (NSCH) data, available every year from + 2016 to 2023. This dataset API can help answer questions about children's health, healthcare access, family functioning, + neighborhood characteristics, and social determinants of health for children aged 0-17 years in the United States. integration_type: dataset +name: National Survey of Children's Health (NSCH) +prompt: | + You have access to file-based National Survey of Children's Health (NSCH) data + from 2016 to 2023, located under the `${DATASET_FILES_BASE_PATH}/census-nsch` directory. -description: | - You have access to knowledge from the National Survey of Children's Health (NSCH) data, available every year from - 2016 to 2023. This dataset API can help answer questions about children's health, healthcare access, family functioning, - neighborhood characteristics, and social determinants of health for children aged 0-17 years in the United States. + The NSCH files are located and named per year as follows: + - ${DATASET_FILES_BASE_PATH}/census-nsch/2023e_topical.sas7bdat + - ${DATASET_FILES_BASE_PATH}/census-nsch/2022e_topical.sas7bdat + - ${DATASET_FILES_BASE_PATH}/census-nsch/2021e_topical.sas7bdat + - ${DATASET_FILES_BASE_PATH}/census-nsch/2020e_topical.sas7bdat + - ${DATASET_FILES_BASE_PATH}/census-nsch/2019e_topical.sas7bdat + - ${DATASET_FILES_BASE_PATH}/census-nsch/2018e_topical.sas7bdat + - ${DATASET_FILES_BASE_PATH}/census-nsch/2017e_topical.sas7bdat + - ${DATASET_FILES_BASE_PATH}/census-nsch/2016e_topical.sas7bdat + - Codebook: ${DATASET_FILES_BASE_PATH}/census-nsch/nsch_dictionary_codebook.csv -prompt: | - You have access to file-based National Survey of Children's Health (NSCH) data - from 2016 to 2023, located under the `${DATASET_FILES_BASE_PATH}/census-nsch` directory. - - The NSCH files are located and named per year as follows: - - ${DATASET_FILES_BASE_PATH}/census-nsch/2023e_topical.sas7bdat - - ${DATASET_FILES_BASE_PATH}/census-nsch/2022e_topical.sas7bdat - - ${DATASET_FILES_BASE_PATH}/census-nsch/2021e_topical.sas7bdat - - ${DATASET_FILES_BASE_PATH}/census-nsch/2020e_topical.sas7bdat - - ${DATASET_FILES_BASE_PATH}/census-nsch/2019e_topical.sas7bdat - - ${DATASET_FILES_BASE_PATH}/census-nsch/2018e_topical.sas7bdat - - ${DATASET_FILES_BASE_PATH}/census-nsch/2017e_topical.sas7bdat - - ${DATASET_FILES_BASE_PATH}/census-nsch/2016e_topical.sas7bdat - - Codebook: ${DATASET_FILES_BASE_PATH}/census-nsch/nsch_dictionary_codebook.csv - - The NSCH is a national survey that collects information on the health and well-being of children ages 0-17 years - in the United States and the factors that may relate to their well-being. The survey includes data on physical and - mental health, access to quality health care, and the child's family, neighborhood, school, and social context. - - The NSCH is also designed to assess the prevalence and impact of special health care needs among children in the US and - explores the extent to which children with special health care needs (CSHCN) have medical homes, - adequate health insurance, access to needed services, and adequate care coordination. - Other topics may include functional difficulties, transition services, - shared decision-making, and satisfaction with care. - Information is collected from parents or caregivers who know about the child's health. - It also examines the physical and emotional health of children ages 0-17 years of age. - These factors include access to - and quality of - health care, family interactions, parental health, - neighborhood characteristics, as well as school and after-school experiences (example: child screen time). - - To read these SAS7BDAT files with pandas, use: - ```python - import pandas as pd - filename = "${DATASET_FILES_BASE_PATH}/census-nsch/2023e_topical.sas7bdat" - df = pd.read_sas(filename, format="sas7bdat") - ``` - - The NSCH datasets include state identifiers (FIPSST) and other geographic information that can be used - for regional analysis. Check which geographic variables are available in each year's dataset before - attempting to use them. - - When working with NSCH data, be aware that: - 1. Variable names may change slightly between survey years - 2. Many variables use coded responses (e.g. subset for examples: 1=Yes||2=No, 1 = Never served in the military||2 = Only on active duty for training in the Reserves - You'll need to use the nsch codebook to decode these variable response codes- - The codebook is available at ${DATASET_FILES_BASE_PATH}/census-nsch/nsch_dictionary_codebook.csv - You may open it with pandas and find the "Variable" name, the "Question", "Description", and its "Response Code"s, etc. - The Response Code options/alternatives per column are separated by a double pipe (||),such as "1=Yes||2=No" - In the codebook, the Question itself is under the "Question" column - - You'll see mentions of "screener" and "topical" in the code book- some screener properties are present in the topical files, - which makes it confusing, unfortunately. - - Codebook columns: - ['Variable', # => variable code/name - 'Source', # => source, a question category from "Topical", "Screener", "Operational" - 'Topic', - 'Survey Years', => available in which file years (eg if 2016, 2017), only check for that variable in the corresponding files, may not be available in files for other years - 'Response Code', => 1=Yes||2=No, etc - 'Description', => description of the question/variable - 'Question', => Question text as a human would ask it - 'Type', - 'Universe', - 'Availability', - 'Imputation Strategy', - 'User Notes'] - - Open it with pandas like so: - ``` - codebook = pd.read_csv("${DATASET_FILES_BASE_PATH}/census-nsch/nsch_dictionary_codebook.csv", index_col=False) # ensure index_col=False, otherwise the dataframe will be shifted! - ``` - - The NSCH is not an API but a collection of dataset files from 2016 to 2023 in SAS7BDAT format. These datasets provide rich information on: - - Physical and mental health conditions (asthma, allergies, ADHD, etc.) - - Access to healthcare - - Family, neighborhood, and social context - - Housing and environmental factors - - Insurance coverage and care coordination - - The geographic resolution for this dataset is at the state level- it only contains state FIPS codes, not county FIPS codes. - The FIPS Code column is called "FIPSST" and is available in all years. - - Instead of assuming a specific column or variable exists, or if you wish to match with a specific column pattern, - try to compare substrings instead if an exact match is not found. For example, if you need data related to asthma, try - to check for the 'asth' substring in the column names. Another example: children age may be under SC_AGE_YEARS (not CHILDREN_AGE), as another example. - Use the codebook, filter out variables or meanings, don't just assume a random color code column of, say, K6Q65A, exists - since it may have existed in previous or new years- the surveys change and the codes aren't obvious. - - # Key Variables for Health Conditions Analysis - ## Asthma and Allergies - - K2Q40A: Asthma diagnosis (1=Yes, 2=No) - - ALLERGIES: Allergies diagnosis (1=Yes, 2=No) - - ALLERGIES_CURR: Current allergies (1=Yes, 2=No) - ## Other Health Conditions - - K2Q31A: ADHD diagnosis - - K2Q32A: Depression diagnosis - - K2Q33A: Anxiety diagnosis - - DIABETES: Diabetes diagnosis - - AUTOIMMUNE: Autoimmune disease diagnosis - ## Housing and Environmental Factors - - K9Q41: Smoke inside home - - ACE1: Difficulty covering basics like food or housing - - FOODSIT: Food situation in household - - TENURE: Housing tenure (owned/rented) - - Environmental factors are primarily self-reported by parents/caregivers. + The NSCH is a national survey that collects information on the health and well-being of children ages 0-17 years + in the United States and the factors that may relate to their well-being. The survey includes data on physical and + mental health, access to quality health care, and the child's family, neighborhood, school, and social context. + + The NSCH is also designed to assess the prevalence and impact of special health care needs among children in the US and + explores the extent to which children with special health care needs (CSHCN) have medical homes, + adequate health insurance, access to needed services, and adequate care coordination. + Other topics may include functional difficulties, transition services, + shared decision-making, and satisfaction with care. + Information is collected from parents or caregivers who know about the child's health. + It also examines the physical and emotional health of children ages 0-17 years of age. + These factors include access to - and quality of - health care, family interactions, parental health, + neighborhood characteristics, as well as school and after-school experiences (example: child screen time). + + To read these SAS7BDAT files with pandas, use: + ```python + import pandas as pd + filename = "${DATASET_FILES_BASE_PATH}/census-nsch/2023e_topical.sas7bdat" + df = pd.read_sas(filename, format="sas7bdat") + ``` + + The NSCH datasets include state identifiers (FIPSST) and other geographic information that can be used + for regional analysis. Check which geographic variables are available in each year's dataset before + attempting to use them. + + When working with NSCH data, be aware that: + 1. Variable names may change slightly between survey years + 2. Many variables use coded responses (e.g. subset for examples: 1=Yes||2=No, 1 = Never served in the military||2 = Only on active duty for training in the Reserves + You'll need to use the nsch codebook to decode these variable response codes- + The codebook is available at ${DATASET_FILES_BASE_PATH}/census-nsch/nsch_dictionary_codebook.csv + You may open it with pandas and find the "Variable" name, the "Question", "Description", and its "Response Code"s, etc. + The Response Code options/alternatives per column are separated by a double pipe (||),such as "1=Yes||2=No" + In the codebook, the Question itself is under the "Question" column + + You'll see mentions of "screener" and "topical" in the code book- some screener properties are present in the topical files, + which makes it confusing, unfortunately. + + Codebook columns: + ['Variable', # => variable code/name + 'Source', # => source, a question category from "Topical", "Screener", "Operational" + 'Topic', + 'Survey Years', => available in which file years (eg if 2016, 2017), only check for that variable in the corresponding files, may not be available in files for other years + 'Response Code', => 1=Yes||2=No, etc + 'Description', => description of the question/variable + 'Question', => Question text as a human would ask it + 'Type', + 'Universe', + 'Availability', + 'Imputation Strategy', + 'User Notes'] + + Open it with pandas like so: + ``` + codebook = pd.read_csv("${DATASET_FILES_BASE_PATH}/census-nsch/nsch_dictionary_codebook.csv", index_col=False) # ensure index_col=False, otherwise the dataframe will be shifted! + ``` + + The NSCH is not an API but a collection of dataset files from 2016 to 2023 in SAS7BDAT format. These datasets provide rich information on: + - Physical and mental health conditions (asthma, allergies, ADHD, etc.) + - Access to healthcare + - Family, neighborhood, and social context + - Housing and environmental factors + - Insurance coverage and care coordination + + The geographic resolution for this dataset is at the state level- it only contains state FIPS codes, not county FIPS codes. + The FIPS Code column is called "FIPSST" and is available in all years. + + Instead of assuming a specific column or variable exists, or if you wish to match with a specific column pattern, + try to compare substrings instead if an exact match is not found. For example, if you need data related to asthma, try + to check for the 'asth' substring in the column names. Another example: children age may be under SC_AGE_YEARS (not CHILDREN_AGE), as another example. + Use the codebook, filter out variables or meanings, don't just assume a random color code column of, say, K6Q65A, exists + since it may have existed in previous or new years- the surveys change and the codes aren't obvious. + + # Key Variables for Health Conditions Analysis + ## Asthma and Allergies + - K2Q40A: Asthma diagnosis (1=Yes, 2=No) + - ALLERGIES: Allergies diagnosis (1=Yes, 2=No) + - ALLERGIES_CURR: Current allergies (1=Yes, 2=No) + ## Other Health Conditions + - K2Q31A: ADHD diagnosis + - K2Q32A: Depression diagnosis + - K2Q33A: Anxiety diagnosis + - DIABETES: Diabetes diagnosis + - AUTOIMMUNE: Autoimmune disease diagnosis + ## Housing and Environmental Factors + - K9Q41: Smoke inside home + - ACE1: Difficulty covering basics like food or housing + - FOODSIT: Food situation in household + - TENURE: Housing tenure (owned/rented) + + Environmental factors are primarily self-reported by parents/caregivers. +resources: + 118cc8be-f400-40e0-bac7-b10745d888fb: + code: "import pandas as pd\nimport numpy as np\n\n# Read the codebook\ncodebook\ + \ = pd.read_csv(\"{{DATASET_FILES_BASE_PATH}}/census-nsch/nsch_dictionary_codebook.csv\"\ + , index_col=False)\n\n# Read 2023 data \ndf = pd.read_sas(\"{{DATASET_FILES_BASE_PATH}}/census-nsch/2023e_topical.sas7bdat\"\ + , format=\"sas7bdat\")\n\n# Find condition-related variables from codebook\n\ + conditions = codebook[\n (codebook['Variable'].str.contains('adhd|autism|asth',\ + \ case=False, na=False)) &\n (codebook['Source'] == 'Topical')\n][['Variable',\ + \ 'Question', 'Response Code']]\n\n# Get condition variables that exist in our\ + \ dataset\ncondition_vars = [var for var in conditions['Variable'] if var in\ + \ df.columns]\n\n# Calculate prevalence for each condition\nresults = []\nfor\ + \ var in condition_vars:\n condition_info = conditions[conditions['Variable']\ + \ == var].iloc[0]\n \n # Get counts and percentages\n counts = df[var].value_counts()\n\ + \ total = counts.sum()\n percentages = (counts / total * 100).round(2)\n\ + \ \n # Add to results\n results.append({\n 'Condition': condition_info['Question'],\n\ + \ 'Total_Responses': total,\n 'Yes_Count': counts.get(1, 0), \ + \ # Assuming 1 = Yes\n 'Percentage': percentages.get(1, 0)\n })\n\n\ + # Create results dataframe\nresults_df = pd.DataFrame(results)\nresults_df =\ + \ results_df.sort_values('Percentage', ascending=False)\n\nprint(results_df)\n" + integration: national_survey_of_children's_health_(nsch) + notes: null + query: Retrieve health conditions reported in the National Survey of Children's + Health, including ADHD, autism, and asthma. + resource_id: 118cc8be-f400-40e0-bac7-b10745d888fb + resource_type: example + 4cdaa736-1ca8-4158-8d3f-60de680da7f3: + code: "import pandas as pd\n\n# Read codebook with explicit parameters\n\n# Ensure\ + \ index_col=FALSE, if not first column values disappear and the whole dataframe\ + \ is shifted!\ncodebook = pd.read_csv(\"{{DATASET_FILES_BASE_PATH}}/census-nsch/nsch_dictionary_codebook.csv\"\ + , index_col=False) \n# Display the first few rows\nprint(codebook.head())\n\ + \n# and columns if needed\nprint(codebook.columns.tolist())\n" + integration: national_survey_of_children's_health_(nsch) + notes: null + query: Check the codebook columns head and available columns + resource_id: 4cdaa736-1ca8-4158-8d3f-60de680da7f3 + resource_type: example + c9fb9487-5e03-4706-b0b9-3b051e61abab: + code: | + import pandas as pd + + df_2023 = pd.read_sas("{{DATASET_FILES_BASE_PATH}}/census-nsch/2023e_topical.sas7bdat", format="sas7bdat") + + existing_cols = df_2023.columns.tolist() + print(existing_cols) + integration: national_survey_of_children's_health_(nsch) + notes: null + query: Check the existing columns in the dataset to see if the screentime variable + and which geo columns are included. + resource_id: c9fb9487-5e03-4706-b0b9-3b051e61abab + resource_type: example + e2681d9f-b1b5-441b-9deb-bd203d9de6f4: + code: "import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\ + import seaborn as sns\nimport os\nfrom matplotlib.ticker import PercentFormatter\n\ + \n# Set the path to the NSCH data directory\nnsch_dir = \"{{DATASET_FILES_BASE_PATH}}/census-nsch/\"\ + \n\n# Function to load and prepare NSCH data for a specific year\ndef load_nsch_data(year):\n\ + \ # Load the dataset\n file_path = os.path.join(nsch_dir, f\"{year}e_topical.sas7bdat\"\ + )\n df = pd.read_sas(file_path)\n \n # Add year column\n df['year']\ + \ = year\n \n # Clean state FIPS codes\n df['FIPSST_clean'] = df['FIPSST'].astype(str).str.strip(\"\ + b'\").str.strip(\"'\")\n \n # Create binary health condition variables\n\ + \ df['has_asthma'] = np.where(df['K2Q40A'] == 1, 1, 0)\n \n # Create\ + \ binary environmental factor variables if they exist in the dataset\n if\ + \ 'K9Q41' in df.columns:\n df['smoke_inside'] = np.where(df['K9Q41']\ + \ == 1, 1, 0)\n \n if 'ACE1' in df.columns:\n df['housing_insecurity']\ + \ = np.where(df['ACE1'] >= 3, 1, 0)\n \n if 'FOODSIT' in df.columns:\n\ + \ df['food_insecurity'] = np.where(df['FOODSIT'] >= 3, 1, 0)\n \n\ + \ if 'TENURE' in df.columns:\n df['rented_housing'] = np.where(df['TENURE']\ + \ == 3, 1, 0)\n \n return df\n\n# Load data for all available years (2016-2023)\n\ + years = range(2016, 2024)\nnsch_data_by_year = {}\nfor year in years:\n try:\n\ + \ nsch_data_by_year[year] = load_nsch_data(year)\n print(f\"Loaded\ + \ NSCH data for {year}: {len(nsch_data_by_year[year])} records\")\n except\ + \ Exception as e:\n print(f\"Error loading data for {year}: {e}\")\n\n\ + # Create a mapping of FIPS codes to state names\nfips_to_state = {\n '01':\ + \ 'Alabama', '02': 'Alaska', '04': 'Arizona', '05': 'Arkansas', '06': 'California',\n\ + \ '08': 'Colorado', '09': 'Connecticut', '10': 'Delaware', '11': 'District\ + \ of Columbia',\n '12': 'Florida', '13': 'Georgia', '15': 'Hawaii', '16':\ + \ 'Idaho', '17': 'Illinois',\n '18': 'Indiana', '19': 'Iowa', '20': 'Kansas',\ + \ '21': 'Kentucky', '22': 'Louisiana',\n '23': 'Maine', '24': 'Maryland',\ + \ '25': 'Massachusetts', '26': 'Michigan', '27': 'Minnesota',\n '28': 'Mississippi',\ + \ '29': 'Missouri', '30': 'Montana', '31': 'Nebraska', '32': 'Nevada',\n \ + \ '33': 'New Hampshire', '34': 'New Jersey', '35': 'New Mexico', '36': 'New\ + \ York',\n '37': 'North Carolina', '38': 'North Dakota', '39': 'Ohio', '40':\ + \ 'Oklahoma', '41': 'Oregon',\n '42': 'Pennsylvania', '44': 'Rhode Island',\ + \ '45': 'South Carolina', '46': 'South Dakota',\n '47': 'Tennessee', '48':\ + \ 'Texas', '49': 'Utah', '50': 'Vermont', '51': 'Virginia',\n '53': 'Washington',\ + \ '54': 'West Virginia', '55': 'Wisconsin', '56': 'Wyoming'\n}\n\n# 1. Analyze\ + \ national trends in asthma prevalence over time\nasthma_trends = []\nfor year,\ + \ df in nsch_data_by_year.items():\n asthma_rate = df['has_asthma'].mean()\n\ + \ asthma_trends.append({'Year': year, 'Asthma Rate': asthma_rate})\n\nasthma_trends_df\ + \ = pd.DataFrame(asthma_trends)\n\n# Visualize national asthma trends\nplt.figure(figsize=(10,\ + \ 6))\nsns.lineplot(x='Year', y='Asthma Rate', data=asthma_trends_df, marker='o',\ + \ linewidth=2)\nplt.title('National Childhood Asthma Prevalence Trends (2016-2023)',\ + \ fontsize=16)\nplt.xlabel('Year', fontsize=12)\nplt.ylabel('Proportion of Children\ + \ with Asthma', fontsize=12)\nplt.grid(True, linestyle='--', alpha=0.7)\nplt.gca().yaxis.set_major_formatter(PercentFormatter(1.0))\n\ + plt.tight_layout()\nplt.savefig('national_asthma_trends.png')\nplt.close()\n\ + \n# 2. Analyze state-level asthma trends for selected states\n# Select a few\ + \ states for comparison (including Texas)\nselected_states = ['Texas', 'California',\ + \ 'New York', 'Florida', 'Illinois']\nselected_fips = [k for k, v in fips_to_state.items()\ + \ if v in selected_states]\n\nstate_trends = []\nfor year, df in nsch_data_by_year.items():\n\ + \ # Add state names\n df['State'] = df['FIPSST_clean'].map(fips_to_state)\n\ + \ \n # Calculate asthma rates by state\n state_asthma = df.groupby('State')['has_asthma'].mean().reset_index()\n\ + \ state_asthma['Year'] = year\n \n # Filter for selected states\n \ + \ selected_state_data = state_asthma[state_asthma['State'].isin(selected_states)]\n\ + \ state_trends.append(selected_state_data)\n\nstate_trends_df = pd.concat(state_trends)\n\ + \n# Visualize state-level asthma trends\nplt.figure(figsize=(12, 7))\nsns.lineplot(x='Year',\ + \ y='has_asthma', hue='State', data=state_trends_df, marker='o', linewidth=2)\n\ + plt.title('Childhood Asthma Prevalence Trends by State (2016-2023)', fontsize=16)\n\ + plt.xlabel('Year', fontsize=12)\nplt.ylabel('Proportion of Children with Asthma',\ + \ fontsize=12)\nplt.grid(True, linestyle='--', alpha=0.7)\nplt.gca().yaxis.set_major_formatter(PercentFormatter(1.0))\n\ + plt.legend(title='State', bbox_to_anchor=(1.05, 1), loc='upper left')\nplt.tight_layout()\n\ + plt.savefig('state_asthma_trends.png')\nplt.close()\n\n# 3. Analyze the relationship\ + \ between environmental factors and asthma over time\n# Focus on smoke exposure\ + \ inside the home\nsmoke_asthma_trends = []\nfor year, df in nsch_data_by_year.items():\n\ + \ if 'smoke_inside' in df.columns:\n # Calculate asthma rates by smoke\ + \ exposure\n smoke_yes_rate = df[df['smoke_inside'] == 1]['has_asthma'].mean()\n\ + \ smoke_no_rate = df[df['smoke_inside'] == 0]['has_asthma'].mean()\n\ + \ \n smoke_asthma_trends.append({\n 'Year': year,\n\ + \ 'Smoke Exposure': 'Yes',\n 'Asthma Rate': smoke_yes_rate\n\ + \ })\n smoke_asthma_trends.append({\n 'Year': year,\n\ + \ 'Smoke Exposure': 'No',\n 'Asthma Rate': smoke_no_rate\n\ + \ })\n\nsmoke_asthma_df = pd.DataFrame(smoke_asthma_trends)\n\n# Visualize\ + \ the relationship between smoke exposure and asthma over time\nplt.figure(figsize=(12,\ + \ 7))\nsns.lineplot(x='Year', y='Asthma Rate', hue='Smoke Exposure', data=smoke_asthma_df,\ + \ marker='o', linewidth=2)\nplt.title('Childhood Asthma Rates by Smoke Exposure\ + \ Inside Home (2016-2023)', fontsize=16)\nplt.xlabel('Year', fontsize=12)\n\ + plt.ylabel('Proportion of Children with Asthma', fontsize=12)\nplt.grid(True,\ + \ linestyle='--', alpha=0.7)\nplt.gca().yaxis.set_major_formatter(PercentFormatter(1.0))\n\ + plt.legend(title='Smoke Inside Home', bbox_to_anchor=(1.05, 1), loc='upper left')\n\ + plt.tight_layout()\nplt.savefig('smoke_asthma_trends.png')\nplt.close()\n\n\ + # 4. Analyze the combined effect of multiple environmental factors on asthma\n\ + # Create a risk score based on the number of environmental risk factors\n# For\ + \ the most recent year (2023)\nrecent_data = nsch_data_by_year[2023]\n\n# Create\ + \ a risk score (0-4) based on the number of environmental risk factors\nenv_factors\ + \ = ['smoke_inside', 'housing_insecurity', 'food_insecurity', 'rented_housing']\n\ + for factor in env_factors:\n if factor not in recent_data.columns:\n \ + \ recent_data[factor] = 0 # Default to 0 if the column doesn't exist\n\n\ + recent_data['env_risk_score'] = recent_data[env_factors].sum(axis=1)\n\n# Calculate\ + \ asthma rates by risk score\nrisk_asthma = recent_data.groupby('env_risk_score')['has_asthma'].mean().reset_index()\n\ + risk_asthma['env_risk_score'] = risk_asthma['env_risk_score'].astype(str)\n\n\ + # Visualize asthma rates by environmental risk score\nplt.figure(figsize=(10,\ + \ 6))\nsns.barplot(x='env_risk_score', y='has_asthma', data=risk_asthma, palette='viridis')\n\ + plt.title('Asthma Rates by Number of Environmental Risk Factors (2023)', fontsize=16)\n\ + plt.xlabel('Number of Environmental Risk Factors', fontsize=12)\nplt.ylabel('Proportion\ + \ of Children with Asthma', fontsize=12)\nplt.grid(True, linestyle='--', alpha=0.7,\ + \ axis='y')\nplt.gca().yaxis.set_major_formatter(PercentFormatter(1.0))\n\n\ + # Add the actual percentages as text on the bars\nfor i, p in enumerate(plt.gca().patches):\n\ + \ height = p.get_height()\n plt.text(p.get_x() + p.get_width()/2., height\ + \ + 0.005, f'{height:.1%}',\n ha='center', fontsize=12)\n\nplt.tight_layout()\n\ + plt.savefig('asthma_by_risk_score.png')\nplt.close()\n\n# 5. Create a summary\ + \ table of findings\nprint(\"Summary of Findings:\")\nprint(f\"1. National Asthma\ + \ Trend: The childhood asthma rate {'increased' if asthma_trends_df.iloc[-1]['Asthma\ + \ Rate'] > asthma_trends_df.iloc[0]['Asthma Rate'] else 'decreased'} from {asthma_trends_df.iloc[0]['Asthma\ + \ Rate']:.1%} in 2016 to {asthma_trends_df.iloc[-1]['Asthma Rate']:.1%} in 2023.\"\ + )\n\n# Calculate the average difference in asthma rates between children exposed\ + \ to smoke and those not exposed\navg_diff = smoke_asthma_df[smoke_asthma_df['Smoke\ + \ Exposure'] == 'Yes']['Asthma Rate'].mean() - smoke_asthma_df[smoke_asthma_df['Smoke\ + \ Exposure'] == 'No']['Asthma Rate'].mean()\nprint(f\"2. Smoke Exposure Impact:\ + \ Children exposed to smoke inside the home had on average {avg_diff:.1%} higher\ + \ asthma rates compared to those not exposed.\")\n\n# Calculate the ratio of\ + \ asthma rates for highest vs. lowest risk scores\nif len(risk_asthma) > 1:\n\ + \ highest_risk = risk_asthma['has_asthma'].max()\n lowest_risk = risk_asthma['has_asthma'].min()\n\ + \ risk_ratio = highest_risk / lowest_risk\n print(f\"3. Environmental\ + \ Risk Factors: Children with the highest number of environmental risk factors\ + \ were {risk_ratio:.1f}x more likely to have asthma compared to those with the\ + \ lowest number of risk factors.\")\n\n# Texas-specific findings\ntexas_data\ + \ = state_trends_df[state_trends_df['State'] == 'Texas']\ntexas_trend = 'increased'\ + \ if texas_data.iloc[-1]['has_asthma'] > texas_data.iloc[0]['has_asthma'] else\ + \ 'decreased'\nprint(f\"4. Texas Trends: The childhood asthma rate in Texas\ + \ {texas_trend} from {texas_data.iloc[0]['has_asthma']:.1%} in 2016 to {texas_data.iloc[-1]['has_asthma']:.1%}\ + \ in 2023.\")\n" + integration: national_survey_of_children's_health_(nsch) + notes: null + query: |- + Analyze trends in childhood asthma and environmental factors over time (2016-2023) using NSCH data. Load multiple years of data, harmonize variables across years, and analyze the relationship between asthma prevalence and environmental factors like smoke exposure, housing insecurity, and food insecurity. Include state-level comparisons and create a composite environmental risk score. Do so specifically for Texas. + resource_id: e2681d9f-b1b5-441b-9deb-bd203d9de6f4 + resource_type: example +slug: national_survey_of_children's_health_(nsch) diff --git a/src/biome/adhoc_data/datasources/nsch/examples.yaml b/src/biome/adhoc_data/datasources/nsch/examples.yaml deleted file mode 100644 index 17ea684..0000000 --- a/src/biome/adhoc_data/datasources/nsch/examples.yaml +++ /dev/null @@ -1,279 +0,0 @@ -- query: "Check the existing columns in the dataset to see if the screentime variable and which geo columns are included." - code: | - import pandas as pd - - df_2023 = pd.read_sas("{{DATASET_FILES_BASE_PATH}}/census-nsch/2023e_topical.sas7bdat", format="sas7bdat") - - existing_cols = df_2023.columns.tolist() - print(existing_cols) - - -- query: "Check the codebook columns head and available columns" - code: | - import pandas as pd - - # Read codebook with explicit parameters - - # Ensure index_col=FALSE, if not first column values disappear and the whole dataframe is shifted! - codebook = pd.read_csv("{{DATASET_FILES_BASE_PATH}}/census-nsch/nsch_dictionary_codebook.csv", index_col=False) - # Display the first few rows - print(codebook.head()) - - # and columns if needed - print(codebook.columns.tolist()) - -- query: "Retrieve health conditions reported in the National Survey of Children's Health, including ADHD, autism, and asthma." - code: | - import pandas as pd - import numpy as np - - # Read the codebook - codebook = pd.read_csv("{{DATASET_FILES_BASE_PATH}}/census-nsch/nsch_dictionary_codebook.csv", index_col=False) - - # Read 2023 data - df = pd.read_sas("{{DATASET_FILES_BASE_PATH}}/census-nsch/2023e_topical.sas7bdat", format="sas7bdat") - - # Find condition-related variables from codebook - conditions = codebook[ - (codebook['Variable'].str.contains('adhd|autism|asth', case=False, na=False)) & - (codebook['Source'] == 'Topical') - ][['Variable', 'Question', 'Response Code']] - - # Get condition variables that exist in our dataset - condition_vars = [var for var in conditions['Variable'] if var in df.columns] - - # Calculate prevalence for each condition - results = [] - for var in condition_vars: - condition_info = conditions[conditions['Variable'] == var].iloc[0] - - # Get counts and percentages - counts = df[var].value_counts() - total = counts.sum() - percentages = (counts / total * 100).round(2) - - # Add to results - results.append({ - 'Condition': condition_info['Question'], - 'Total_Responses': total, - 'Yes_Count': counts.get(1, 0), # Assuming 1 = Yes - 'Percentage': percentages.get(1, 0) - }) - - # Create results dataframe - results_df = pd.DataFrame(results) - results_df = results_df.sort_values('Percentage', ascending=False) - - print(results_df) - - -- query: "Analyze trends in childhood asthma and environmental factors over time (2016-2023) using NSCH data. Load multiple years of data, harmonize variables across years, and analyze the relationship between asthma prevalence and environmental factors like smoke exposure, housing insecurity, and food insecurity. Include state-level comparisons and create a composite environmental risk score. Do so specifically for Texas." - code: | - import pandas as pd - import numpy as np - import matplotlib.pyplot as plt - import seaborn as sns - import os - from matplotlib.ticker import PercentFormatter - - # Set the path to the NSCH data directory - nsch_dir = "{{DATASET_FILES_BASE_PATH}}/census-nsch/" - - # Function to load and prepare NSCH data for a specific year - def load_nsch_data(year): - # Load the dataset - file_path = os.path.join(nsch_dir, f"{year}e_topical.sas7bdat") - df = pd.read_sas(file_path) - - # Add year column - df['year'] = year - - # Clean state FIPS codes - df['FIPSST_clean'] = df['FIPSST'].astype(str).str.strip("b'").str.strip("'") - - # Create binary health condition variables - df['has_asthma'] = np.where(df['K2Q40A'] == 1, 1, 0) - - # Create binary environmental factor variables if they exist in the dataset - if 'K9Q41' in df.columns: - df['smoke_inside'] = np.where(df['K9Q41'] == 1, 1, 0) - - if 'ACE1' in df.columns: - df['housing_insecurity'] = np.where(df['ACE1'] >= 3, 1, 0) - - if 'FOODSIT' in df.columns: - df['food_insecurity'] = np.where(df['FOODSIT'] >= 3, 1, 0) - - if 'TENURE' in df.columns: - df['rented_housing'] = np.where(df['TENURE'] == 3, 1, 0) - - return df - - # Load data for all available years (2016-2023) - years = range(2016, 2024) - nsch_data_by_year = {} - for year in years: - try: - nsch_data_by_year[year] = load_nsch_data(year) - print(f"Loaded NSCH data for {year}: {len(nsch_data_by_year[year])} records") - except Exception as e: - print(f"Error loading data for {year}: {e}") - - # Create a mapping of FIPS codes to state names - fips_to_state = { - '01': 'Alabama', '02': 'Alaska', '04': 'Arizona', '05': 'Arkansas', '06': 'California', - '08': 'Colorado', '09': 'Connecticut', '10': 'Delaware', '11': 'District of Columbia', - '12': 'Florida', '13': 'Georgia', '15': 'Hawaii', '16': 'Idaho', '17': 'Illinois', - '18': 'Indiana', '19': 'Iowa', '20': 'Kansas', '21': 'Kentucky', '22': 'Louisiana', - '23': 'Maine', '24': 'Maryland', '25': 'Massachusetts', '26': 'Michigan', '27': 'Minnesota', - '28': 'Mississippi', '29': 'Missouri', '30': 'Montana', '31': 'Nebraska', '32': 'Nevada', - '33': 'New Hampshire', '34': 'New Jersey', '35': 'New Mexico', '36': 'New York', - '37': 'North Carolina', '38': 'North Dakota', '39': 'Ohio', '40': 'Oklahoma', '41': 'Oregon', - '42': 'Pennsylvania', '44': 'Rhode Island', '45': 'South Carolina', '46': 'South Dakota', - '47': 'Tennessee', '48': 'Texas', '49': 'Utah', '50': 'Vermont', '51': 'Virginia', - '53': 'Washington', '54': 'West Virginia', '55': 'Wisconsin', '56': 'Wyoming' - } - - # 1. Analyze national trends in asthma prevalence over time - asthma_trends = [] - for year, df in nsch_data_by_year.items(): - asthma_rate = df['has_asthma'].mean() - asthma_trends.append({'Year': year, 'Asthma Rate': asthma_rate}) - - asthma_trends_df = pd.DataFrame(asthma_trends) - - # Visualize national asthma trends - plt.figure(figsize=(10, 6)) - sns.lineplot(x='Year', y='Asthma Rate', data=asthma_trends_df, marker='o', linewidth=2) - plt.title('National Childhood Asthma Prevalence Trends (2016-2023)', fontsize=16) - plt.xlabel('Year', fontsize=12) - plt.ylabel('Proportion of Children with Asthma', fontsize=12) - plt.grid(True, linestyle='--', alpha=0.7) - plt.gca().yaxis.set_major_formatter(PercentFormatter(1.0)) - plt.tight_layout() - plt.savefig('national_asthma_trends.png') - plt.close() - - # 2. Analyze state-level asthma trends for selected states - # Select a few states for comparison (including Texas) - selected_states = ['Texas', 'California', 'New York', 'Florida', 'Illinois'] - selected_fips = [k for k, v in fips_to_state.items() if v in selected_states] - - state_trends = [] - for year, df in nsch_data_by_year.items(): - # Add state names - df['State'] = df['FIPSST_clean'].map(fips_to_state) - - # Calculate asthma rates by state - state_asthma = df.groupby('State')['has_asthma'].mean().reset_index() - state_asthma['Year'] = year - - # Filter for selected states - selected_state_data = state_asthma[state_asthma['State'].isin(selected_states)] - state_trends.append(selected_state_data) - - state_trends_df = pd.concat(state_trends) - - # Visualize state-level asthma trends - plt.figure(figsize=(12, 7)) - sns.lineplot(x='Year', y='has_asthma', hue='State', data=state_trends_df, marker='o', linewidth=2) - plt.title('Childhood Asthma Prevalence Trends by State (2016-2023)', fontsize=16) - plt.xlabel('Year', fontsize=12) - plt.ylabel('Proportion of Children with Asthma', fontsize=12) - plt.grid(True, linestyle='--', alpha=0.7) - plt.gca().yaxis.set_major_formatter(PercentFormatter(1.0)) - plt.legend(title='State', bbox_to_anchor=(1.05, 1), loc='upper left') - plt.tight_layout() - plt.savefig('state_asthma_trends.png') - plt.close() - - # 3. Analyze the relationship between environmental factors and asthma over time - # Focus on smoke exposure inside the home - smoke_asthma_trends = [] - for year, df in nsch_data_by_year.items(): - if 'smoke_inside' in df.columns: - # Calculate asthma rates by smoke exposure - smoke_yes_rate = df[df['smoke_inside'] == 1]['has_asthma'].mean() - smoke_no_rate = df[df['smoke_inside'] == 0]['has_asthma'].mean() - - smoke_asthma_trends.append({ - 'Year': year, - 'Smoke Exposure': 'Yes', - 'Asthma Rate': smoke_yes_rate - }) - smoke_asthma_trends.append({ - 'Year': year, - 'Smoke Exposure': 'No', - 'Asthma Rate': smoke_no_rate - }) - - smoke_asthma_df = pd.DataFrame(smoke_asthma_trends) - - # Visualize the relationship between smoke exposure and asthma over time - plt.figure(figsize=(12, 7)) - sns.lineplot(x='Year', y='Asthma Rate', hue='Smoke Exposure', data=smoke_asthma_df, marker='o', linewidth=2) - plt.title('Childhood Asthma Rates by Smoke Exposure Inside Home (2016-2023)', fontsize=16) - plt.xlabel('Year', fontsize=12) - plt.ylabel('Proportion of Children with Asthma', fontsize=12) - plt.grid(True, linestyle='--', alpha=0.7) - plt.gca().yaxis.set_major_formatter(PercentFormatter(1.0)) - plt.legend(title='Smoke Inside Home', bbox_to_anchor=(1.05, 1), loc='upper left') - plt.tight_layout() - plt.savefig('smoke_asthma_trends.png') - plt.close() - - # 4. Analyze the combined effect of multiple environmental factors on asthma - # Create a risk score based on the number of environmental risk factors - # For the most recent year (2023) - recent_data = nsch_data_by_year[2023] - - # Create a risk score (0-4) based on the number of environmental risk factors - env_factors = ['smoke_inside', 'housing_insecurity', 'food_insecurity', 'rented_housing'] - for factor in env_factors: - if factor not in recent_data.columns: - recent_data[factor] = 0 # Default to 0 if the column doesn't exist - - recent_data['env_risk_score'] = recent_data[env_factors].sum(axis=1) - - # Calculate asthma rates by risk score - risk_asthma = recent_data.groupby('env_risk_score')['has_asthma'].mean().reset_index() - risk_asthma['env_risk_score'] = risk_asthma['env_risk_score'].astype(str) - - # Visualize asthma rates by environmental risk score - plt.figure(figsize=(10, 6)) - sns.barplot(x='env_risk_score', y='has_asthma', data=risk_asthma, palette='viridis') - plt.title('Asthma Rates by Number of Environmental Risk Factors (2023)', fontsize=16) - plt.xlabel('Number of Environmental Risk Factors', fontsize=12) - plt.ylabel('Proportion of Children with Asthma', fontsize=12) - plt.grid(True, linestyle='--', alpha=0.7, axis='y') - plt.gca().yaxis.set_major_formatter(PercentFormatter(1.0)) - - # Add the actual percentages as text on the bars - for i, p in enumerate(plt.gca().patches): - height = p.get_height() - plt.text(p.get_x() + p.get_width()/2., height + 0.005, f'{height:.1%}', - ha='center', fontsize=12) - - plt.tight_layout() - plt.savefig('asthma_by_risk_score.png') - plt.close() - - # 5. Create a summary table of findings - print("Summary of Findings:") - print(f"1. National Asthma Trend: The childhood asthma rate {'increased' if asthma_trends_df.iloc[-1]['Asthma Rate'] > asthma_trends_df.iloc[0]['Asthma Rate'] else 'decreased'} from {asthma_trends_df.iloc[0]['Asthma Rate']:.1%} in 2016 to {asthma_trends_df.iloc[-1]['Asthma Rate']:.1%} in 2023.") - - # Calculate the average difference in asthma rates between children exposed to smoke and those not exposed - avg_diff = smoke_asthma_df[smoke_asthma_df['Smoke Exposure'] == 'Yes']['Asthma Rate'].mean() - smoke_asthma_df[smoke_asthma_df['Smoke Exposure'] == 'No']['Asthma Rate'].mean() - print(f"2. Smoke Exposure Impact: Children exposed to smoke inside the home had on average {avg_diff:.1%} higher asthma rates compared to those not exposed.") - - # Calculate the ratio of asthma rates for highest vs. lowest risk scores - if len(risk_asthma) > 1: - highest_risk = risk_asthma['has_asthma'].max() - lowest_risk = risk_asthma['has_asthma'].min() - risk_ratio = highest_risk / lowest_risk - print(f"3. Environmental Risk Factors: Children with the highest number of environmental risk factors were {risk_ratio:.1f}x more likely to have asthma compared to those with the lowest number of risk factors.") - - # Texas-specific findings - texas_data = state_trends_df[state_trends_df['State'] == 'Texas'] - texas_trend = 'increased' if texas_data.iloc[-1]['has_asthma'] > texas_data.iloc[0]['has_asthma'] else 'decreased' - print(f"4. Texas Trends: The childhood asthma rate in Texas {texas_trend} from {texas_data.iloc[0]['has_asthma']:.1%} in 2016 to {texas_data.iloc[-1]['has_asthma']:.1%} in 2023.") diff --git a/src/biome/adhoc_data/datasources/pdc/api.yaml b/src/biome/adhoc_data/datasources/pdc/api.yaml index e627593..641dd00 100644 --- a/src/biome/adhoc_data/datasources/pdc/api.yaml +++ b/src/biome/adhoc_data/datasources/pdc/api.yaml @@ -1,26 +1,877 @@ -name: "Proteomics Data Commons" -description: | - The objectives of the National Cancer Institute’s Proteomic Data Commons (PDC) are (1) to make cancer-related proteomic datasets easily accessible to the public, and (2) facilitate direct multiomics integration in support of precision medicine through interoperability with accompanying data resources (genomic and medical image datasets). - The PDC was developed to advance our understanding of how proteins help to shape the risk, diagnosis, development, progression, and treatment of cancer. In-depth analysis of proteomic data allows us to study both how and why cancer develops and to devise ways of personalizing treatment for patients using precision medicine. - The PDC is one of several repositories within the NCI Cancer Research Data Commons (CRDC), a secure cloud-based infrastructure featuring diverse data sets and innovative analytic tools – all designed to advance data-driven scientific discovery. The CRDC enables researchers to link proteomic data with other data sets (e.g., genomic and imaging data) and to submit, collect, analyze, store, and share data throughout the cancer data ecosystem. - Access to highly curated and standardized biospecimen, clinical and proteomic data with direct integration of accompanying data resources (genomic and medical image datasets). - - Uses: - * Intuitive interface to filter, query, search, visualize and download the data and metadata. - * A common data harmonization pipeline to uniformly analyze all PDC data and provide advanced visualization of the quantitative information. - * Cloud based (Amazon Web Services) infrastructure facilitates interoperability with AWS based data analysis tools and platforms natively. - * Application programming interface (API) provides cloud-agnostic data access and allows third parties to extend the functionality beyond the PDC. - * A highly structured workspace that serves as a private user data store and also data submission portal. - * Distributes controlled access data, such as the patient-specific protein fasta sequence databases, with dbGaP authorization and eRA Commons authentication. - -attachments: - raw_documentation: pdc_schema.graphql - +description: "The objectives of the National Cancer Institute\u2019s Proteomic Data\ + \ Commons (PDC) are (1) to make cancer-related proteomic datasets easily accessible\ + \ to the public, and (2) facilitate direct multiomics integration in support of\ + \ precision medicine through interoperability with accompanying data resources (genomic\ + \ and medical image datasets).\nThe PDC was developed to advance our understanding\ + \ of how proteins help to shape the risk, diagnosis, development, progression, and\ + \ treatment of cancer. In-depth analysis of proteomic data allows us to study both\ + \ how and why cancer develops and to devise ways of personalizing treatment for\ + \ patients using precision medicine.\nThe PDC is one of several repositories within\ + \ the NCI Cancer Research Data Commons (CRDC), a secure cloud-based infrastructure\ + \ featuring diverse data sets and innovative analytic tools \u2013 all designed\ + \ to advance data-driven scientific discovery. The CRDC enables researchers to link\ + \ proteomic data with other data sets (e.g., genomic and imaging data) and to submit,\ + \ collect, analyze, store, and share data throughout the cancer data ecosystem.\n\ + Access to highly curated and standardized biospecimen, clinical and proteomic data\ + \ with direct integration of accompanying data resources (genomic and medical image\ + \ datasets).\n\nUses:\n* Intuitive interface to filter, query, search, visualize\ + \ and download the data and metadata.\n* A common data harmonization pipeline to\ + \ uniformly analyze all PDC data and provide advanced visualization of the quantitative\ + \ information.\n* Cloud based (Amazon Web Services) infrastructure facilitates interoperability\ + \ with AWS based data analysis tools and platforms natively.\n* Application programming\ + \ interface (API) provides cloud-agnostic data access and allows third parties to\ + \ extend the functionality beyond the PDC.\n* A highly structured workspace that\ + \ serves as a private user data store and also data submission portal.\n* Distributes\ + \ controlled access data, such as the patient-specific protein fasta sequence databases,\ + \ with dbGaP authorization and eRA Commons authentication.\n" +integration_type: api +name: Proteomics Data Commons prompt: | - ${raw_documentation} + ${raw_documentation} + + # Additional Instructions: + + This API is through GraphQL. You will be provided the schema. + All requests go to the following URL: https://pdc.cancer.gov/graphql + Make all GraphQL requests to that URL. +resources: + 07112af0-449d-4d88-b914-acba9f2ae57b: + code: | + import requests + import json + + # Define the GraphQL endpoint + url = "https://pdc.cancer.gov/graphql" + + # Define the GraphQL query to get raw mass spec data for the Georgetown Lung Cancer Proteomics Study + query = ''' + { + getPaginatedUIFile( + study_name: "Georgetown Lung Cancer Proteomics Study", + data_category: "Raw Mass Spectra", + file_type: "Proprietary" + ) { + uiFiles { + file_id + file_name + file_type + file_size + md5sum + downloadable + } + } + } + ''' + + # Send the request + response = requests.post(url, json={'query': query}) + + # Check for successful response + if response.status_code == 200: + data = response.json() + file_info = data['data']['getPaginatedUIFile']['uiFiles'] + print("Raw mass spec data for the Georgetown Lung Cancer Proteomics Study:") + print(json.dumps(file_info, indent=2)) + else: + print(f"Error: {response.status_code} - {response.text}") + integration: proteomics_data_commons + notes: null + query: |- + Fetch raw mass spec data for a specific study using study name, data category, and file type in the Proteomics Data Commons (PDC). + resource_id: 07112af0-449d-4d88-b914-acba9f2ae57b + resource_type: example + 0a702f8c-8715-4e12-a8f9-ad32621dbfcf: + code: | + import requests + import json + + # Define the GraphQL endpoint + url = "https://pdc.cancer.gov/graphql" + + # Define the GraphQL query to get detailed information about a case + query = ''' + { + case(case_id: "9e8e8f51-d732-11ea-b1fd-0aad30af8a83") { + case_id + case_submitter_id + disease_type + primary_site + demographics { + gender + race + ethnicity + age_at_index + } + diagnoses { + diagnosis_id + diagnosis_submitter_id + tumor_stage + tumor_grade + } + samples { + sample_id + sample_submitter_id + sample_type + } + } + } + ''' + + # Send the request + response = requests.post(url, json={'query': query}) + + # Check for successful response + if response.status_code == 200: + data = response.json() + case_info = data['data']['case'] + print("Detailed information about the case:") + print(json.dumps(case_info, indent=2)) + else: + print(f"Error: {response.status_code} - {response.text}") + integration: proteomics_data_commons + notes: null + query: Fetch detailed information about a case using its case ID in the Proteomics + Data Commons (PDC). + resource_id: 0a702f8c-8715-4e12-a8f9-ad32621dbfcf + resource_type: example + 25f6b833-82fe-44fd-b8c8-bfbd6a7d2738: + code: | + import requests + import json + + # Define the GraphQL endpoint + url = "https://pdc.cancer.gov/graphql" + + # Define the GraphQL query to find studies with open standard data for processed mass spec + query = ''' + { + getPaginatedUIFile( + data_category: "Processed Mass Spectra", + file_type: "Open Standard", + offset: 0, + limit: 10 + ) { + uiFiles { + file_id + file_name + study_id + pdc_study_id + } + } + } + ''' + + # Send the request + response = requests.post(url, json={'query': query}) + + # Check for successful response + if response.status_code == 200: + data = response.json() + files = data['data']['getPaginatedUIFile']['uiFiles'] + print("Studies with open standard data for processed mass spec:") + print(json.dumps(files, indent=2)) + else: + print(f"Error: {response.status_code} - {response.text}") + integration: proteomics_data_commons + notes: null + query: |- + This example demonstrates how to query the Proteomics Data Commons for studies with open standard data for processed mass spec using the getPaginatedUIFile query. + resource_id: 25f6b833-82fe-44fd-b8c8-bfbd6a7d2738 + resource_type: example + 2af265a5-2548-4c2b-a3f8-cf853b2d4b0d: + code: | + import requests + import json + + url = 'https://pdc.cancer.gov/graphql' + + query = """ + query ($gene_name: String!, $offset: Int, $limit: Int) { + getPaginatedUIGeneAliquotSpectralCount(gene_name: $gene_name, offset: $offset, limit: $limit) { + total + uiGeneAliquotSpectralCounts { + aliquot_id + plex + experiment_type + spectral_count + distinct_peptide + unshared_peptide + precursor_area + log2_ratio + } + } + } + """ + + variables_stat5a = { + "gene_name": "STAT5A", + "offset": 0, + "limit": 10 # Retrieve the first 10 results + } + + variables_stat5b = { + "gene_name": "STAT5B", + "offset": 0, + "limit": 10 # Retrieve the first 10 results + } + + try: + response_stat5a = requests.post(url, json={'query': query, 'variables': variables_stat5a}) + response_stat5a.raise_for_status() + data_stat5a = response_stat5a.json() + + response_stat5b = requests.post(url, json={'query': query, 'variables': variables_stat5b}) + response_stat5b.raise_for_status() + data_stat5b = response_stat5b.json() + + print("STAT5A Protein Expression Data (First 10 Results):") + print(json.dumps(data_stat5a['data']['getPaginatedUIGeneAliquotSpectralCount']['uiGeneAliquotSpectralCounts'], indent=2)) + + print("\nSTAT5B Protein Expression Data (First 10 Results):") + print(json.dumps(data_stat5b['data']['getPaginatedUIGeneAliquotSpectralCount']['uiGeneAliquotSpectralCounts'], indent=2)) + + except requests.exceptions.RequestException as e: + print(f"An error occurred: {e}") + except json.JSONDecodeError as e: + print(f"Error decoding JSON response: {e}") + print("Response content:", response_stat5a.text) + print("Response content:", response_stat5b.text) + integration: proteomics_data_commons + notes: null + query: |- + Retrieve quantitative mass spectrometry data for STAT5A and STAT5B proteins using the getPaginatedUIGeneAliquotSpectralCount query. + resource_id: 2af265a5-2548-4c2b-a3f8-cf853b2d4b0d + resource_type: example + 75e500f5-31c0-4683-995b-26fc49d51426: + filepath: pdc_schema.graphql + integration: proteomics_data_commons + name: raw_documentation + resource_id: 75e500f5-31c0-4683-995b-26fc49d51426 + resource_type: file + 8708cae9-d62e-468d-9d78-3f61ac1913d6: + code: | + import requests + import json + import pandas as pd + + url = 'https://pdc.cancer.gov/graphql' + + # Updated query with correct schema + query = """ + query ($gene_name: String!) { + getPaginatedUIGeneAliquotSpectralCount( + gene_name: $gene_name, + offset: 0, + limit: 100 # Increased limit to get more samples + ) { + total + uiGeneAliquotSpectralCounts { + aliquot_id + plex + experiment_type + spectral_count + distinct_peptide + unshared_peptide + precursor_area + log2_ratio + study_submitter_id + } + } + } + """ + + def get_protein_data(gene_name): + response = requests.post(url, json={'query': query, 'variables': {'gene_name': gene_name}}) + response.raise_for_status() + data = response.json() + + if 'data' in data and 'getPaginatedUIGeneAliquotSpectralCount' in data['data']: + results = data['data']['getPaginatedUIGeneAliquotSpectralCount'] + samples = results['uiGeneAliquotSpectralCounts'] + + # Convert to DataFrame + df = pd.DataFrame(samples) + # Add gene name column + df['gene_name'] = gene_name + return df + return None + + try: + # Get data for both proteins + stat5a_df = get_protein_data('STAT5A') + stat5b_df = get_protein_data('STAT5B') + + # Combine the dataframes + combined_df = pd.concat([stat5a_df, stat5b_df], ignore_index=True) + + # Display basic information about the dataset + print("DataFrame Info:") + print(combined_df.info()) + + print("\nFirst few rows of the dataset:") + print(combined_df.head()) + + print("\nSummary statistics:") + print(combined_df.describe()) + + # Save the DataFrame for later use + combined_df.to_csv('stat5_protein_data.csv', index=False) + print("\nData has been saved to 'stat5_protein_data.csv'") + + except requests.exceptions.RequestException as e: + print(f"An error occurred: {e}") + if hasattr(e, 'response') and e.response is not None: + print("Response content:", e.response.text) + except json.JSONDecodeError as e: + print(f"Error decoding JSON response: {e}") + print("Response content:", response.text) + integration: proteomics_data_commons + notes: null + query: Load samples into a pandas dataframe + resource_id: 8708cae9-d62e-468d-9d78-3f61ac1913d6 + resource_type: example + 893e05e7-c0db-4482-aebe-1b09c9abfd4d: + code: | + import requests + import json + + url = 'https://pdc.cancer.gov/graphql' + study_id = 'PDC000478' # Using the PDC Study ID instead + + query = """ + query getStudyMetadata($pdc_study_id: String!) { + getPaginatedUIStudy(pdc_study_id: $pdc_study_id) { + uiStudies { + study_id + pdc_study_id + submitter_id_name + study_description + program_name + project_name + disease_type + primary_site + analytical_fraction + experiment_type + embargo_date + cases_count + aliquots_count + filesCount { + file_type + data_category + files_count + } + supplementaryFilesCount { + data_category + file_type + files_count + } + nonSupplementaryFilesCount { + data_category + file_type + files_count + } + contacts { + name + institution + email + url + } + versions { + number + } + } + } + } + """ + + variables = { + "pdc_study_id": study_id + } + + try: + response = requests.post(url, json={'query': query, 'variables': variables}) + response.raise_for_status() + data = response.json() + # Print a subset of the data + print(json.dumps(data['data']['getPaginatedUIStudy']['uiStudies'][0], indent=2)) + + except requests.exceptions.RequestException as e: + print(f"An error occurred: {e}") + if hasattr(e, 'response') and e.response is not None: + print("Response content:", e.response.text) + except json.JSONDecodeError as e: + print(f"Error decoding JSON response: {e}") + print("Response content:", response.text) + integration: proteomics_data_commons + notes: null + query: Fetch all available metadata for a study using its PDC Study ID. + resource_id: 893e05e7-c0db-4482-aebe-1b09c9abfd4d + resource_type: example + ab5e4368-3fd8-49f9-83f0-389cc6912cfd: + code: | + import requests + import json + + # Define the GraphQL endpoint + url = "https://pdc.cancer.gov/graphql" + + # Define the GraphQL query to filter for studies with primary site "Lung" + query_lung = ''' + { + getPaginatedUIStudy(primary_site: "Lung") { + uiStudies { + study_id + submitter_id_name + pdc_study_id + disease_type + primary_site + } + } + } + ''' + + # Define the GraphQL query to filter for studies with primary site "Bronchus and lung" + query_bronchus_and_lung = ''' + { + getPaginatedUIStudy(primary_site: "Bronchus and lung") { + uiStudies { + study_id + submitter_id_name + pdc_study_id + disease_type + primary_site + } + } + } + ''' + + # Send the request for "Lung" + response_lung = requests.post(url, json={'query': query_lung}) + + # Send the request for "Bronchus and lung" + response_bronchus_and_lung = requests.post(url, json={'query': query_bronchus_and_lung}) + + # Combine the results from both queries + lung_cancer_studies = [] + + if response_lung.status_code == 200: + data_lung = response_lung.json() + lung_cancer_studies.extend(data_lung['data']['getPaginatedUIStudy']['uiStudies']) + else: + print(f"Error: {response_lung.status_code} - {response_lung.text}") + + if response_bronchus_and_lung.status_code == 200: + data_bronchus_and_lung = response_bronchus_and_lung.json() + lung_cancer_studies.extend(data_bronchus_and_lung['data']['getPaginatedUIStudy']['uiStudies']) + else: + print(f"Error: {response_bronchus_and_lung.status_code} - {response_bronchus_and_lung.text}") + + # Remove duplicates + lung_cancer_studies = {study['study_id']: study for study in lung_cancer_studies}.values() + + # Print the number of studies found and the first 5 studies + print(f"Found {len(lung_cancer_studies)} studies with primary site 'Lung' or 'Bronchus and lung'.") + print(list(lung_cancer_studies)[:5]) + integration: proteomics_data_commons + notes: null + query: |- + This example demonstrates how to find lung cancer studies in the Proteomics Data Commons by querying for studies with primary sites 'Lung' and 'Bronchus and lung', and combining the results. + resource_id: ab5e4368-3fd8-49f9-83f0-389cc6912cfd + resource_type: example + b1057917-5943-4e63-a0e1-b1c2e12aa7a7: + code: | + import requests + import json + + url = 'https://pdc.cancer.gov/graphql' + + query = """ + { + getPaginatedUIStudy(disease_type: "Acute Myeloid Leukemia") { + uiStudies { + study_id + submitter_id_name + pdc_study_id + disease_type + primary_site + } + } + } + """ + + try: + response = requests.post(url, json={'query': query}) + response.raise_for_status() + + # Process the response data + data = response.json() + studies = data['data']['getPaginatedUIStudy']['uiStudies'] + + # Output the results + print(f"Found {len(studies)} studies related to Acute Myeloid Leukemia.") + for study in studies[:5]: # Print the first 5 studies + print(study) + + except requests.exceptions.RequestException as e: + print(f"An error occurred: {e}") + integration: proteomics_data_commons + notes: null + query: Query PDC to find studies with a specific disease type (in this case 'Acute + Myeloid Leukemia') + resource_id: b1057917-5943-4e63-a0e1-b1c2e12aa7a7 + resource_type: example + bf5e4934-19fe-4c99-b7c3-b3e37efd70a6: + code: | + import requests + import json + + # Define the GraphQL endpoint + url = "https://pdc.cancer.gov/graphql" + + # Define the GraphQL query to filter for studies with primary site "Lung" or "Bronchus and lung" + query_lung = ''' + { + getPaginatedUIStudy(primary_site: "Lung") { + uiStudies { + study_id + submitter_id_name + pdc_study_id + disease_type + primary_site + } + } + } + ''' + + query_bronchus_and_lung = ''' + { + getPaginatedUIStudy(primary_site: "Bronchus and lung") { + uiStudies { + study_id + submitter_id_name + pdc_study_id + disease_type + primary_site + } + } + } + ''' + + # Send the request for "Lung" + response_lung = requests.post(url, json={'query': query_lung}) + + # Send the request for "Bronchus and lung" + response_bronchus_and_lung = requests.post(url, json={'query': query_bronchus_and_lung}) + + lung_cancer_studies = [] + + # Check for successful response for "Lung" + if response_lung.status_code == 200: + data_lung = response_lung.json() + lung_cancer_studies.extend(data_lung['data']['getPaginatedUIStudy']['uiStudies']) + else: + print(f"Error: {response_lung.status_code} - {response_lung.text}") + + # Check for successful response for "Bronchus and lung" + if response_bronchus_and_lung.status_code == 200: + data_bronchus_and_lung = response_bronchus_and_lung.json() + lung_cancer_studies.extend(data_bronchus_and_lung['data']['getPaginatedUIStudy']['uiStudies']) + else: + print(f"Error: {response_bronchus_and_lung.status_code} - {response_bronchus_and_lung.text}") + + # Remove duplicates + lung_cancer_studies = {study['study_id']: study for study in lung_cancer_studies}.values() + + print(f"Found {len(lung_cancer_studies)} studies with primary site 'Lung' or 'Bronchus and lung'.") + # Display the first few studies + for study in list(lung_cancer_studies)[:5]: + print(study) + integration: proteomics_data_commons + notes: null + query: |- + Query the Proteomics Data Commons (PDC) to find studies with primary site 'Lung' or 'Bronchus and lung' using separate queries and combining results. + resource_id: bf5e4934-19fe-4c99-b7c3-b3e37efd70a6 + resource_type: example + c31ddab5-2539-4005-8b92-1512ec8d12fc: + code: | + import requests + import json + + # Define the GraphQL endpoint + url = "https://pdc.cancer.gov/graphql" + + # Define the GraphQL query to get open standard data for a study + query = ''' + { + getPaginatedUIFile( + study_name: "Georgetown Lung Cancer Proteomics Study", + file_type: "Open Standard" + ) { + uiFiles { + file_id + file_name + file_type + file_size + md5sum + downloadable + } + } + } + ''' + + # Send the request + response = requests.post(url, json={'query': query}) + + # Check for successful response + if response.status_code == 200: + data = response.json() + file_info = data['data']['getPaginatedUIFile']['uiFiles'] + print("Open standard data for the study:") + print(json.dumps(file_info, indent=2)) + else: + print(f"Error: {response.status_code} - {response.text}") + integration: proteomics_data_commons + notes: null + query: Retrieve open standard data files for a specific study using the Proteomics + Data Commons (PDC) API. + resource_id: c31ddab5-2539-4005-8b92-1512ec8d12fc + resource_type: example + c68f970e-4d39-4979-bcf2-43a823490773: + code: | + import requests + import json + + url = 'https://pdc.cancer.gov/graphql' + + # Corrected query structure + query = """ + query ($gene_name: String!) { + getPaginatedUIGeneAliquotSpectralCount( + gene_name: $gene_name, + offset: 0, + limit: 10 + ) { + total + uiGeneAliquotSpectralCounts { + aliquot_id + plex + experiment_type + spectral_count + distinct_peptide + unshared_peptide + log2_ratio + study_submitter_id + } + } + } + """ + + def print_results(data, gene_name): + if 'data' in data and 'getPaginatedUIGeneAliquotSpectralCount' in data['data']: + results = data['data']['getPaginatedUIGeneAliquotSpectralCount'] + total = results['total'] + samples = results['uiGeneAliquotSpectralCounts'] + + print(f"\n{gene_name} Results:") + print(f"Total samples available: {total}") + print("\nSample details (first 10 samples):") + + for i, sample in enumerate(samples): + print(f"\nSample {i+1}:") + print(f" Study ID: {sample['study_submitter_id']}") + print(f" Experiment Type: {sample['experiment_type']}") + print(f" Spectral Count: {sample['spectral_count']}") + print(f" Distinct Peptides: {sample['distinct_peptide']}") + print(f" Unshared Peptides: {sample['unshared_peptide']}") + print(f" Log2 Ratio: {sample['log2_ratio']}") + print(f" Plex: {sample['plex']}") + + try: + # Query STAT5A + response = requests.post(url, json={'query': query, 'variables': {'gene_name': 'STAT5A'}}) + response.raise_for_status() + stat5a_data = response.json() + print_results(stat5a_data, 'STAT5A') + + # Query STAT5B + response = requests.post(url, json={'query': query, 'variables': {'gene_name': 'STAT5B'}}) + response.raise_for_status() + stat5b_data = response.json() + print_results(stat5b_data, 'STAT5B') + + except requests.exceptions.RequestException as e: + print(f"An error occurred: {e}") + if hasattr(e.response, 'text'): + print("Response content:", e.response.text) + except json.JSONDecodeError as e: + print(f"Error decoding JSON response: {e}") + print("Response content:", response.text) + integration: proteomics_data_commons + notes: null + query: obtain quantitative data + resource_id: c68f970e-4d39-4979-bcf2-43a823490773 + resource_type: example + d119d4cc-347e-40bc-b3d3-f17f6e1ee16e: + code: | + import requests + import json + + # Define the GraphQL endpoint + url = "https://pdc.cancer.gov/graphql" + + # Define the GraphQL query to get cases for the Georgetown Lung Cancer Proteomics Study using study name + query = ''' + { + getPaginatedUICase(study_name: "Georgetown Lung Cancer Proteomics Study") { + uiCases { + case_id + case_submitter_id + disease_type + primary_site + } + } + } + ''' + + # Send the request for cases + response = requests.post(url, json={'query': query}) + + # Check for successful response for cases + if response.status_code == 200: + data_cases = response.json() + cases_info = data_cases['data']['getPaginatedUICase']['uiCases'] + print("Cases information for the Georgetown Lung Cancer Proteomics Study:") + print(json.dumps(cases_info, indent=2)) + else: + print(f"Error: {response.status_code} - {response.text}") + integration: proteomics_data_commons + notes: null + query: Fetch all cases based on a study name + resource_id: d119d4cc-347e-40bc-b3d3-f17f6e1ee16e + resource_type: example + d2fde53a-0ba6-4c00-acaf-e27d4218480b: + code: | + import requests + import json + + url = 'https://pdc.cancer.gov/graphql' + + # Query for STAT5B protein expression data + query = """ + query ($gene_name: String!) { + getPaginatedUIGeneAliquotSpectralCount(gene_name: $gene_name, offset: 0, limit: 10) { + total + uiGeneAliquotSpectralCounts { + aliquot_id + plex + experiment_type + spectral_count + distinct_peptide + unshared_peptide + precursor_area + log2_ratio + } + } + } + """ + + variables = { + "gene_name": "STAT5B" + } + + try: + response = requests.post(url, json={'query': query, 'variables': variables}) + response.raise_for_status() + data = response.json() + print("STAT5B Protein Expression Data:") + print(json.dumps(data, indent=2)) + except requests.exceptions.RequestException as e: + print(f"An error occurred: {e}") + except json.JSONDecodeError as e: + print(f"Error decoding JSON response: {e}") + print("Response content:", response.text) + integration: proteomics_data_commons + notes: null + query: obtain quantitative mass spec data on a specific protein (STAT5) + resource_id: d2fde53a-0ba6-4c00-acaf-e27d4218480b + resource_type: example + e5293d4f-52c8-414f-bfcd-3131b4000aa0: + code: | + import requests + import json + + # Define the GraphQL endpoint + url = "https://pdc.cancer.gov/graphql" + + # Define the GraphQL query to get the signed URL for the file + query = ''' + query FilesDataQuery($file_name: String!, $study_id: String!) { + uiFilesPerStudy(file_name: $file_name, study_id: $study_id) { + file_id + file_name + signedUrl { + url + } + } + } + ''' + + # Set the variables for the query + variables = { + "file_name": "Ctrl_1-set_1-label_113-frac_2-F9.wiff", + "study_id": "17d5bccf-d028-11ea-b1fd-0aad30af8a83" + } - # Additional Instructions: + # Send the request + response = requests.post(url, json={'query': query, 'variables': variables}) - This API is through GraphQL. You will be provided the schema. - All requests go to the following URL: https://pdc.cancer.gov/graphql - Make all GraphQL requests to that URL. + # Check for successful response + if response.status_code == 200: + data = response.json() + signed_url = data['data']['uiFilesPerStudy'][0]['signedUrl']['url'] + print(f"Signed URL for the file: {signed_url}") + else: + print(f"Error: {response.status_code} - {response.text}") + integration: proteomics_data_commons + notes: null + query: |- + Retrieve a signed URL for downloading a specific file (e.g. raw mass spec file) in a study using the Proteomics Data Commons (PDC) API. + resource_id: e5293d4f-52c8-414f-bfcd-3131b4000aa0 + resource_type: example + ed1fa5df-2d51-48ef-871b-b09abc5defc9: + code: "import requests\nimport json\nimport pandas as pd\nfrom pandas import json_normalize\n\ + \nurl = 'https://pdc.cancer.gov/graphql'\n\n# Define the GraphQL query\nquery\ + \ = \"\"\"\nquery FilteredClinicalDataPaginated($offset_value: Int, $limit_value:\ + \ Int, $source: String!) {\n getPaginatedUIClinical(\n offset: $offset_value,\n\ + \ limit: $limit_value,\n source: $source\n ) {\n total\n\ + \ uiClinical {\n case_id\n case_submitter_id\n\ + \ externalReferences {\n reference_resource_shortname\n\ + \ reference_entity_location\n }\n }\n \ + \ pagination {\n count\n total\n pages\n \ + \ size\n }\n }\n}\n\"\"\"\n\n# Set initial query variables\n\ + offset_value = 0\nlimit_value = 3993 # Fetch 1000 records per request\nsource\ + \ = \"PDC\"\n\n# Initialize list to store all case IDs\nall_case_ids = []\n\n\ + total_cases = 3993 # Total number of cases to fetch\n\nwhile offset_value <\ + \ total_cases:\n variables = {\n \"offset_value\": offset_value,\n\ + \ \"limit_value\": limit_value,\n \"source\": source\n }\n\n\ + \ # Make the API request\n response = requests.post(url, json={'query':\ + \ query, 'variables': variables})\n\n if response.status_code == 200:\n \ + \ data = response.json()\n \n # Extract case IDs\n \ + \ cases = data['data']['getPaginatedUIClinical']['uiClinical']\n all_case_ids.extend([case['case_id']\ + \ for case in cases])\n \n # Increment offset for next page\n\ + \ offset_value += limit_value\n else:\n print(f\"Error: {response.status_code}\"\ + )\n print(response.text)\n break\n\n# Print total number of case\ + \ IDs fetched\nprint(f\"Total cases fetched: {len(all_case_ids)}\")\n\npdc_cases_in_gdc\ + \ = []\n\nfor case in data['data']['getPaginatedUIClinical']['uiClinical']:\n\ + \ for exref in case['externalReferences']:\n if exref.get('reference_resource_shortname',\ + \ '') == 'GDC':\n case['gdc_case_id'] = exref['reference_entity_location'].split('cases/')[1].strip('\\\ + r')\n pdc_cases_in_gdc.append(case)\n\nprint(f\"Found {len(pdc_cases_in_gdc)}\ + \ cases that have external references to GDC.\")\n\npdc_in_gdc_df = pd.DataFrame(json_normalize(pdc_cases_in_gdc))\n\ + pdc_in_gdc_df.head()" + integration: proteomics_data_commons + notes: null + query: |- + Get all clinical data (cases) from PDC and filter for those that have external references to GDC. Return the results as a pandas dataframe. + resource_id: ed1fa5df-2d51-48ef-871b-b09abc5defc9 + resource_type: example +slug: proteomics_data_commons diff --git a/src/biome/adhoc_data/datasources/pdc/examples.yaml b/src/biome/adhoc_data/datasources/pdc/examples.yaml deleted file mode 100644 index 2b913c1..0000000 --- a/src/biome/adhoc_data/datasources/pdc/examples.yaml +++ /dev/null @@ -1,826 +0,0 @@ -- query: "obtain quantitative mass spec data on a specific protein (STAT5)" - code: | - import requests - import json - - url = 'https://pdc.cancer.gov/graphql' - - # Query for STAT5B protein expression data - query = """ - query ($gene_name: String!) { - getPaginatedUIGeneAliquotSpectralCount(gene_name: $gene_name, offset: 0, limit: 10) { - total - uiGeneAliquotSpectralCounts { - aliquot_id - plex - experiment_type - spectral_count - distinct_peptide - unshared_peptide - precursor_area - log2_ratio - } - } - } - """ - - variables = { - "gene_name": "STAT5B" - } - - try: - response = requests.post(url, json={'query': query, 'variables': variables}) - response.raise_for_status() - data = response.json() - print("STAT5B Protein Expression Data:") - print(json.dumps(data, indent=2)) - except requests.exceptions.RequestException as e: - print(f"An error occurred: {e}") - except json.JSONDecodeError as e: - print(f"Error decoding JSON response: {e}") - print("Response content:", response.text) - -- query: "obtain quantitative data" - code: | - import requests - import json - - url = 'https://pdc.cancer.gov/graphql' - - # Corrected query structure - query = """ - query ($gene_name: String!) { - getPaginatedUIGeneAliquotSpectralCount( - gene_name: $gene_name, - offset: 0, - limit: 10 - ) { - total - uiGeneAliquotSpectralCounts { - aliquot_id - plex - experiment_type - spectral_count - distinct_peptide - unshared_peptide - log2_ratio - study_submitter_id - } - } - } - """ - - def print_results(data, gene_name): - if 'data' in data and 'getPaginatedUIGeneAliquotSpectralCount' in data['data']: - results = data['data']['getPaginatedUIGeneAliquotSpectralCount'] - total = results['total'] - samples = results['uiGeneAliquotSpectralCounts'] - - print(f"\n{gene_name} Results:") - print(f"Total samples available: {total}") - print("\nSample details (first 10 samples):") - - for i, sample in enumerate(samples): - print(f"\nSample {i+1}:") - print(f" Study ID: {sample['study_submitter_id']}") - print(f" Experiment Type: {sample['experiment_type']}") - print(f" Spectral Count: {sample['spectral_count']}") - print(f" Distinct Peptides: {sample['distinct_peptide']}") - print(f" Unshared Peptides: {sample['unshared_peptide']}") - print(f" Log2 Ratio: {sample['log2_ratio']}") - print(f" Plex: {sample['plex']}") - - try: - # Query STAT5A - response = requests.post(url, json={'query': query, 'variables': {'gene_name': 'STAT5A'}}) - response.raise_for_status() - stat5a_data = response.json() - print_results(stat5a_data, 'STAT5A') - - # Query STAT5B - response = requests.post(url, json={'query': query, 'variables': {'gene_name': 'STAT5B'}}) - response.raise_for_status() - stat5b_data = response.json() - print_results(stat5b_data, 'STAT5B') - - except requests.exceptions.RequestException as e: - print(f"An error occurred: {e}") - if hasattr(e.response, 'text'): - print("Response content:", e.response.text) - except json.JSONDecodeError as e: - print(f"Error decoding JSON response: {e}") - print("Response content:", response.text) - -- query: "Load samples into a pandas dataframe" - code: | - import requests - import json - import pandas as pd - - url = 'https://pdc.cancer.gov/graphql' - - # Updated query with correct schema - query = """ - query ($gene_name: String!) { - getPaginatedUIGeneAliquotSpectralCount( - gene_name: $gene_name, - offset: 0, - limit: 100 # Increased limit to get more samples - ) { - total - uiGeneAliquotSpectralCounts { - aliquot_id - plex - experiment_type - spectral_count - distinct_peptide - unshared_peptide - precursor_area - log2_ratio - study_submitter_id - } - } - } - """ - - def get_protein_data(gene_name): - response = requests.post(url, json={'query': query, 'variables': {'gene_name': gene_name}}) - response.raise_for_status() - data = response.json() - - if 'data' in data and 'getPaginatedUIGeneAliquotSpectralCount' in data['data']: - results = data['data']['getPaginatedUIGeneAliquotSpectralCount'] - samples = results['uiGeneAliquotSpectralCounts'] - - # Convert to DataFrame - df = pd.DataFrame(samples) - # Add gene name column - df['gene_name'] = gene_name - return df - return None - - try: - # Get data for both proteins - stat5a_df = get_protein_data('STAT5A') - stat5b_df = get_protein_data('STAT5B') - - # Combine the dataframes - combined_df = pd.concat([stat5a_df, stat5b_df], ignore_index=True) - - # Display basic information about the dataset - print("DataFrame Info:") - print(combined_df.info()) - - print("\nFirst few rows of the dataset:") - print(combined_df.head()) - - print("\nSummary statistics:") - print(combined_df.describe()) - - # Save the DataFrame for later use - combined_df.to_csv('stat5_protein_data.csv', index=False) - print("\nData has been saved to 'stat5_protein_data.csv'") - - except requests.exceptions.RequestException as e: - print(f"An error occurred: {e}") - if hasattr(e, 'response') and e.response is not None: - print("Response content:", e.response.text) - except json.JSONDecodeError as e: - print(f"Error decoding JSON response: {e}") - print("Response content:", response.text) - -- query: "Query the Proteomics Data Commons (PDC) to find studies with primary site 'Lung' or 'Bronchus and lung' using separate queries and combining results." - code: | - import requests - import json - - # Define the GraphQL endpoint - url = "https://pdc.cancer.gov/graphql" - - # Define the GraphQL query to filter for studies with primary site "Lung" or "Bronchus and lung" - query_lung = ''' - { - getPaginatedUIStudy(primary_site: "Lung") { - uiStudies { - study_id - submitter_id_name - pdc_study_id - disease_type - primary_site - } - } - } - ''' - - query_bronchus_and_lung = ''' - { - getPaginatedUIStudy(primary_site: "Bronchus and lung") { - uiStudies { - study_id - submitter_id_name - pdc_study_id - disease_type - primary_site - } - } - } - ''' - - # Send the request for "Lung" - response_lung = requests.post(url, json={'query': query_lung}) - - # Send the request for "Bronchus and lung" - response_bronchus_and_lung = requests.post(url, json={'query': query_bronchus_and_lung}) - - lung_cancer_studies = [] - - # Check for successful response for "Lung" - if response_lung.status_code == 200: - data_lung = response_lung.json() - lung_cancer_studies.extend(data_lung['data']['getPaginatedUIStudy']['uiStudies']) - else: - print(f"Error: {response_lung.status_code} - {response_lung.text}") - - # Check for successful response for "Bronchus and lung" - if response_bronchus_and_lung.status_code == 200: - data_bronchus_and_lung = response_bronchus_and_lung.json() - lung_cancer_studies.extend(data_bronchus_and_lung['data']['getPaginatedUIStudy']['uiStudies']) - else: - print(f"Error: {response_bronchus_and_lung.status_code} - {response_bronchus_and_lung.text}") - - # Remove duplicates - lung_cancer_studies = {study['study_id']: study for study in lung_cancer_studies}.values() - - print(f"Found {len(lung_cancer_studies)} studies with primary site 'Lung' or 'Bronchus and lung'.") - # Display the first few studies - for study in list(lung_cancer_studies)[:5]: - print(study) - - -- query: "Fetch all cases based on a study name" - code: | - import requests - import json - - # Define the GraphQL endpoint - url = "https://pdc.cancer.gov/graphql" - - # Define the GraphQL query to get cases for the Georgetown Lung Cancer Proteomics Study using study name - query = ''' - { - getPaginatedUICase(study_name: "Georgetown Lung Cancer Proteomics Study") { - uiCases { - case_id - case_submitter_id - disease_type - primary_site - } - } - } - ''' - - # Send the request for cases - response = requests.post(url, json={'query': query}) - - # Check for successful response for cases - if response.status_code == 200: - data_cases = response.json() - cases_info = data_cases['data']['getPaginatedUICase']['uiCases'] - print("Cases information for the Georgetown Lung Cancer Proteomics Study:") - print(json.dumps(cases_info, indent=2)) - else: - print(f"Error: {response.status_code} - {response.text}") - - -- query: "Fetch detailed information about a case using its case ID in the Proteomics Data Commons (PDC)." - code: | - import requests - import json - - # Define the GraphQL endpoint - url = "https://pdc.cancer.gov/graphql" - - # Define the GraphQL query to get detailed information about a case - query = ''' - { - case(case_id: "9e8e8f51-d732-11ea-b1fd-0aad30af8a83") { - case_id - case_submitter_id - disease_type - primary_site - demographics { - gender - race - ethnicity - age_at_index - } - diagnoses { - diagnosis_id - diagnosis_submitter_id - tumor_stage - tumor_grade - } - samples { - sample_id - sample_submitter_id - sample_type - } - } - } - ''' - - # Send the request - response = requests.post(url, json={'query': query}) - - # Check for successful response - if response.status_code == 200: - data = response.json() - case_info = data['data']['case'] - print("Detailed information about the case:") - print(json.dumps(case_info, indent=2)) - else: - print(f"Error: {response.status_code} - {response.text}") - - -- query: "Fetch raw mass spec data for a specific study using study name, data category, and file type in the Proteomics Data Commons (PDC)." - code: | - import requests - import json - - # Define the GraphQL endpoint - url = "https://pdc.cancer.gov/graphql" - - # Define the GraphQL query to get raw mass spec data for the Georgetown Lung Cancer Proteomics Study - query = ''' - { - getPaginatedUIFile( - study_name: "Georgetown Lung Cancer Proteomics Study", - data_category: "Raw Mass Spectra", - file_type: "Proprietary" - ) { - uiFiles { - file_id - file_name - file_type - file_size - md5sum - downloadable - } - } - } - ''' - - # Send the request - response = requests.post(url, json={'query': query}) - - # Check for successful response - if response.status_code == 200: - data = response.json() - file_info = data['data']['getPaginatedUIFile']['uiFiles'] - print("Raw mass spec data for the Georgetown Lung Cancer Proteomics Study:") - print(json.dumps(file_info, indent=2)) - else: - print(f"Error: {response.status_code} - {response.text}") - - -- query: "Retrieve a signed URL for downloading a specific file (e.g. raw mass spec file) in a study using the Proteomics Data Commons (PDC) API." - code: | - import requests - import json - - # Define the GraphQL endpoint - url = "https://pdc.cancer.gov/graphql" - - # Define the GraphQL query to get the signed URL for the file - query = ''' - query FilesDataQuery($file_name: String!, $study_id: String!) { - uiFilesPerStudy(file_name: $file_name, study_id: $study_id) { - file_id - file_name - signedUrl { - url - } - } - } - ''' - - # Set the variables for the query - variables = { - "file_name": "Ctrl_1-set_1-label_113-frac_2-F9.wiff", - "study_id": "17d5bccf-d028-11ea-b1fd-0aad30af8a83" - } - - # Send the request - response = requests.post(url, json={'query': query, 'variables': variables}) - - # Check for successful response - if response.status_code == 200: - data = response.json() - signed_url = data['data']['uiFilesPerStudy'][0]['signedUrl']['url'] - print(f"Signed URL for the file: {signed_url}") - else: - print(f"Error: {response.status_code} - {response.text}") - - -- query: "Retrieve open standard data files for a specific study using the Proteomics Data Commons (PDC) API." - code: | - import requests - import json - - # Define the GraphQL endpoint - url = "https://pdc.cancer.gov/graphql" - - # Define the GraphQL query to get open standard data for a study - query = ''' - { - getPaginatedUIFile( - study_name: "Georgetown Lung Cancer Proteomics Study", - file_type: "Open Standard" - ) { - uiFiles { - file_id - file_name - file_type - file_size - md5sum - downloadable - } - } - } - ''' - - # Send the request - response = requests.post(url, json={'query': query}) - - # Check for successful response - if response.status_code == 200: - data = response.json() - file_info = data['data']['getPaginatedUIFile']['uiFiles'] - print("Open standard data for the study:") - print(json.dumps(file_info, indent=2)) - else: - print(f"Error: {response.status_code} - {response.text}") - - -- query: "This example demonstrates how to find lung cancer studies in the Proteomics Data Commons by querying for studies with primary sites 'Lung' and 'Bronchus and lung', and combining the results." - code: | - import requests - import json - - # Define the GraphQL endpoint - url = "https://pdc.cancer.gov/graphql" - - # Define the GraphQL query to filter for studies with primary site "Lung" - query_lung = ''' - { - getPaginatedUIStudy(primary_site: "Lung") { - uiStudies { - study_id - submitter_id_name - pdc_study_id - disease_type - primary_site - } - } - } - ''' - - # Define the GraphQL query to filter for studies with primary site "Bronchus and lung" - query_bronchus_and_lung = ''' - { - getPaginatedUIStudy(primary_site: "Bronchus and lung") { - uiStudies { - study_id - submitter_id_name - pdc_study_id - disease_type - primary_site - } - } - } - ''' - - # Send the request for "Lung" - response_lung = requests.post(url, json={'query': query_lung}) - - # Send the request for "Bronchus and lung" - response_bronchus_and_lung = requests.post(url, json={'query': query_bronchus_and_lung}) - - # Combine the results from both queries - lung_cancer_studies = [] - - if response_lung.status_code == 200: - data_lung = response_lung.json() - lung_cancer_studies.extend(data_lung['data']['getPaginatedUIStudy']['uiStudies']) - else: - print(f"Error: {response_lung.status_code} - {response_lung.text}") - - if response_bronchus_and_lung.status_code == 200: - data_bronchus_and_lung = response_bronchus_and_lung.json() - lung_cancer_studies.extend(data_bronchus_and_lung['data']['getPaginatedUIStudy']['uiStudies']) - else: - print(f"Error: {response_bronchus_and_lung.status_code} - {response_bronchus_and_lung.text}") - - # Remove duplicates - lung_cancer_studies = {study['study_id']: study for study in lung_cancer_studies}.values() - - # Print the number of studies found and the first 5 studies - print(f"Found {len(lung_cancer_studies)} studies with primary site 'Lung' or 'Bronchus and lung'.") - print(list(lung_cancer_studies)[:5]) - - -- query: "This example demonstrates how to query the Proteomics Data Commons for studies with open standard data for processed mass spec using the getPaginatedUIFile query." - code: | - import requests - import json - - # Define the GraphQL endpoint - url = "https://pdc.cancer.gov/graphql" - - # Define the GraphQL query to find studies with open standard data for processed mass spec - query = ''' - { - getPaginatedUIFile( - data_category: "Processed Mass Spectra", - file_type: "Open Standard", - offset: 0, - limit: 10 - ) { - uiFiles { - file_id - file_name - study_id - pdc_study_id - } - } - } - ''' - - # Send the request - response = requests.post(url, json={'query': query}) - - # Check for successful response - if response.status_code == 200: - data = response.json() - files = data['data']['getPaginatedUIFile']['uiFiles'] - print("Studies with open standard data for processed mass spec:") - print(json.dumps(files, indent=2)) - else: - print(f"Error: {response.status_code} - {response.text}") - -- query: "Query PDC to find studies with a specific disease type (in this case 'Acute Myeloid Leukemia')" - code: | - import requests - import json - - url = 'https://pdc.cancer.gov/graphql' - - query = """ - { - getPaginatedUIStudy(disease_type: "Acute Myeloid Leukemia") { - uiStudies { - study_id - submitter_id_name - pdc_study_id - disease_type - primary_site - } - } - } - """ - - try: - response = requests.post(url, json={'query': query}) - response.raise_for_status() - - # Process the response data - data = response.json() - studies = data['data']['getPaginatedUIStudy']['uiStudies'] - - # Output the results - print(f"Found {len(studies)} studies related to Acute Myeloid Leukemia.") - for study in studies[:5]: # Print the first 5 studies - print(study) - - except requests.exceptions.RequestException as e: - print(f"An error occurred: {e}") - -- query: "Fetch all available metadata for a study using its PDC Study ID." - code: | - import requests - import json - - url = 'https://pdc.cancer.gov/graphql' - study_id = 'PDC000478' # Using the PDC Study ID instead - - query = """ - query getStudyMetadata($pdc_study_id: String!) { - getPaginatedUIStudy(pdc_study_id: $pdc_study_id) { - uiStudies { - study_id - pdc_study_id - submitter_id_name - study_description - program_name - project_name - disease_type - primary_site - analytical_fraction - experiment_type - embargo_date - cases_count - aliquots_count - filesCount { - file_type - data_category - files_count - } - supplementaryFilesCount { - data_category - file_type - files_count - } - nonSupplementaryFilesCount { - data_category - file_type - files_count - } - contacts { - name - institution - email - url - } - versions { - number - } - } - } - } - """ - - variables = { - "pdc_study_id": study_id - } - - try: - response = requests.post(url, json={'query': query, 'variables': variables}) - response.raise_for_status() - data = response.json() - # Print a subset of the data - print(json.dumps(data['data']['getPaginatedUIStudy']['uiStudies'][0], indent=2)) - - except requests.exceptions.RequestException as e: - print(f"An error occurred: {e}") - if hasattr(e, 'response') and e.response is not None: - print("Response content:", e.response.text) - except json.JSONDecodeError as e: - print(f"Error decoding JSON response: {e}") - print("Response content:", response.text) - - -- query: "Retrieve quantitative mass spectrometry data for STAT5A and STAT5B proteins using the getPaginatedUIGeneAliquotSpectralCount query." - code: | - import requests - import json - - url = 'https://pdc.cancer.gov/graphql' - - query = """ - query ($gene_name: String!, $offset: Int, $limit: Int) { - getPaginatedUIGeneAliquotSpectralCount(gene_name: $gene_name, offset: $offset, limit: $limit) { - total - uiGeneAliquotSpectralCounts { - aliquot_id - plex - experiment_type - spectral_count - distinct_peptide - unshared_peptide - precursor_area - log2_ratio - } - } - } - """ - - variables_stat5a = { - "gene_name": "STAT5A", - "offset": 0, - "limit": 10 # Retrieve the first 10 results - } - - variables_stat5b = { - "gene_name": "STAT5B", - "offset": 0, - "limit": 10 # Retrieve the first 10 results - } - - try: - response_stat5a = requests.post(url, json={'query': query, 'variables': variables_stat5a}) - response_stat5a.raise_for_status() - data_stat5a = response_stat5a.json() - - response_stat5b = requests.post(url, json={'query': query, 'variables': variables_stat5b}) - response_stat5b.raise_for_status() - data_stat5b = response_stat5b.json() - - print("STAT5A Protein Expression Data (First 10 Results):") - print(json.dumps(data_stat5a['data']['getPaginatedUIGeneAliquotSpectralCount']['uiGeneAliquotSpectralCounts'], indent=2)) - - print("\nSTAT5B Protein Expression Data (First 10 Results):") - print(json.dumps(data_stat5b['data']['getPaginatedUIGeneAliquotSpectralCount']['uiGeneAliquotSpectralCounts'], indent=2)) - - except requests.exceptions.RequestException as e: - print(f"An error occurred: {e}") - except json.JSONDecodeError as e: - print(f"Error decoding JSON response: {e}") - print("Response content:", response_stat5a.text) - print("Response content:", response_stat5b.text) - - -- query: "Get all clinical data (cases) from PDC and filter for those that have external references to GDC. Return the results as a pandas dataframe." - code: | - import requests - import json - import pandas as pd - from pandas import json_normalize - - url = 'https://pdc.cancer.gov/graphql' - - # Define the GraphQL query - query = """ - query FilteredClinicalDataPaginated($offset_value: Int, $limit_value: Int, $source: String!) { - getPaginatedUIClinical( - offset: $offset_value, - limit: $limit_value, - source: $source - ) { - total - uiClinical { - case_id - case_submitter_id - externalReferences { - reference_resource_shortname - reference_entity_location - } - } - pagination { - count - total - pages - size - } - } - } - """ - - # Set initial query variables - offset_value = 0 - limit_value = 3993 # Fetch 1000 records per request - source = "PDC" - - # Initialize list to store all case IDs - all_case_ids = [] - - total_cases = 3993 # Total number of cases to fetch - - while offset_value < total_cases: - variables = { - "offset_value": offset_value, - "limit_value": limit_value, - "source": source - } - - # Make the API request - response = requests.post(url, json={'query': query, 'variables': variables}) - - if response.status_code == 200: - data = response.json() - - # Extract case IDs - cases = data['data']['getPaginatedUIClinical']['uiClinical'] - all_case_ids.extend([case['case_id'] for case in cases]) - - # Increment offset for next page - offset_value += limit_value - else: - print(f"Error: {response.status_code}") - print(response.text) - break - - # Print total number of case IDs fetched - print(f"Total cases fetched: {len(all_case_ids)}") - - pdc_cases_in_gdc = [] - - for case in data['data']['getPaginatedUIClinical']['uiClinical']: - for exref in case['externalReferences']: - if exref.get('reference_resource_shortname', '') == 'GDC': - case['gdc_case_id'] = exref['reference_entity_location'].split('cases/')[1].strip('\r') - pdc_cases_in_gdc.append(case) - - print(f"Found {len(pdc_cases_in_gdc)} cases that have external references to GDC.") - - pdc_in_gdc_df = pd.DataFrame(json_normalize(pdc_cases_in_gdc)) - pdc_in_gdc_df.head() \ No newline at end of file diff --git a/src/biome/adhoc_data/datasources/synapse/api.yaml b/src/biome/adhoc_data/datasources/synapse/api.yaml index 1345670..022d5ee 100644 --- a/src/biome/adhoc_data/datasources/synapse/api.yaml +++ b/src/biome/adhoc_data/datasources/synapse/api.yaml @@ -1,85 +1,109 @@ -name: "NF Synapse" -integration_type: api description: | - The NF Synapse API is a RESTful API that allows you to interact with the Neurofibromatosis Data Portal. - It additionally contains a Python client that can be used to interact with the API (on the package `synapseclient`, already installed in the environment). - The code drafter has access both to the openapi spec, the web docs, and the python client documentation. - -attachments: - raw_documentation: SynapseOpenApiSpec.json - web_docs: web_docs.md - python_client: nf_programmatic_webdocs.md - + The NF Synapse API is a RESTful API that allows you to interact with the Neurofibromatosis Data Portal. + It additionally contains a Python client that can be used to interact with the API (on the package `synapseclient`, already installed in the environment). + The code drafter has access both to the openapi spec, the web docs, and the python client documentation. +integration_type: api +name: NF Synapse prompt: | - # OpenAPI Spec JSON: - ${raw_documentation} - - # Additional Instructions: - - This API is through OpenAPI Spec. You have been provided the schema. - All requests go to the following URL: https://repo-prod.prod.sagebase.org - That is the base URL for all requests. - - To use the Synapse API always retrieve the auth header token from the environment variable: - - bearer token: os.environ.get("API_SYNAPSE") - Do not use placeholder auth values from the OpenAPI spec nor prompt the user for these values, just use the environment variables. - - Ensure you know the fields that come back from the OpenAPI spec in order to process the Synapse data. - - Never simulate this API- if it is not available ask the user how to proceed. - - You will mostly have to use the /repo/v1 and /file/v1 paths. - For example you can start search with: - POST /repo/v1/search - - and once you find a project or dataset you like, you can proceed to exploreother entities in - order to find a file / file handle and download it. - - An example file handle we found: syn4988822 with it, you can issue a download request. - You'll need to find the file ID first depending on your needs and the user query. - - You only have read/view/download assess/permissions- don't try to create or upload data- the openapi spec may have create/upload endpoints but do not use them. - - # More Web Docs: - ${web_docs} - - # Example issuing a POST to search for `High throughput analysis of pNF cell lines` (omitting auth bearer token header; eg Authorization: Bearer ) - ``` - POST https://repo-prod.prod.sagebase.org/repo/v1/search - { - "queryTerm": [ - "Blakeley" - ], - "rangeQuery": [], - "facetOptions": [], - "returnFields": [], - "start": 0, - "size": "10" - } - ``` - - # Python Client - ${python_client} - - # About the NF synapse subdomain - - NF Data Portal - A home for Neurofibromatosis research resources - - Files are data collected from from human samples, animal models, and cell lines from a variety of assays. - Datasets are curated collections of files organized to facilitate data sharing. - Initiatives are funder-organized programs, groups, consortia, cohorts, or awards, usually focused on a specific research area in neurofibromatosis. - Studies are hypothesis-driven projects with the goal of uncovering new knowledge about neurofibromatosis type 1, type 2, or schwannomatosis. - Papers are pre-prints and peer-reviewed articles produced by NF Data Portal studies - Tools include animal models, cell lines, genetic reagents, antibodies, and biobanks related to NF! - - Example of initializing the python client providing download dir and login token credentials: - ``` - import synapseclient, os - from pathlib import Path - - downloads_dir = Path(__file__).parent.parent / "downloads" / "synapse" - - syn = synapseclient.Synapse(cache_root_dir=downloads_dir) - syn.login(authToken=os.environ.get("API_SYNAPSE")) # pass in the authToken on python client login! - ``` + # OpenAPI Spec JSON: + ${raw_documentation} + + # Additional Instructions: + + This API is through OpenAPI Spec. You have been provided the schema. + All requests go to the following URL: https://repo-prod.prod.sagebase.org + That is the base URL for all requests. + + To use the Synapse API always retrieve the auth header token from the environment variable: + - bearer token: os.environ.get("API_SYNAPSE") + Do not use placeholder auth values from the OpenAPI spec nor prompt the user for these values, just use the environment variables. + + Ensure you know the fields that come back from the OpenAPI spec in order to process the Synapse data. + + Never simulate this API- if it is not available ask the user how to proceed. + + You will mostly have to use the /repo/v1 and /file/v1 paths. + For example you can start search with: + POST /repo/v1/search + + and once you find a project or dataset you like, you can proceed to exploreother entities in + order to find a file / file handle and download it. + + An example file handle we found: syn4988822 with it, you can issue a download request. + You'll need to find the file ID first depending on your needs and the user query. + + You only have read/view/download assess/permissions- don't try to create or upload data- the openapi spec may have create/upload endpoints but do not use them. + + # More Web Docs: + ${web_docs} + + # Example issuing a POST to search for `High throughput analysis of pNF cell lines` (omitting auth bearer token header; eg Authorization: Bearer ) + ``` + POST https://repo-prod.prod.sagebase.org/repo/v1/search + { + "queryTerm": [ + "Blakeley" + ], + "rangeQuery": [], + "facetOptions": [], + "returnFields": [], + "start": 0, + "size": "10" + } + ``` + + # Python Client + ${python_client} + + # About the NF synapse subdomain + + NF Data Portal + A home for Neurofibromatosis research resources + + Files are data collected from from human samples, animal models, and cell lines from a variety of assays. + Datasets are curated collections of files organized to facilitate data sharing. + Initiatives are funder-organized programs, groups, consortia, cohorts, or awards, usually focused on a specific research area in neurofibromatosis. + Studies are hypothesis-driven projects with the goal of uncovering new knowledge about neurofibromatosis type 1, type 2, or schwannomatosis. + Papers are pre-prints and peer-reviewed articles produced by NF Data Portal studies + Tools include animal models, cell lines, genetic reagents, antibodies, and biobanks related to NF! + + Example of initializing the python client providing download dir and login token credentials: + ``` + import synapseclient, os + from pathlib import Path + + downloads_dir = Path(__file__).parent.parent / "downloads" / "synapse" + + syn = synapseclient.Synapse(cache_root_dir=downloads_dir) + syn.login(authToken=os.environ.get("API_SYNAPSE")) # pass in the authToken on python client login! + ``` +resources: + 45324841-c013-4942-8b5f-db85856d7e48: + filepath: web_docs.md + integration: nf_synapse + name: web_docs + resource_id: 45324841-c013-4942-8b5f-db85856d7e48 + resource_type: file + 8d29393b-bc38-441f-aee1-88b60d1322b6: + filepath: SynapseOpenApiSpec.json + integration: nf_synapse + name: raw_documentation + resource_id: 8d29393b-bc38-441f-aee1-88b60d1322b6 + resource_type: file + 920140d9-ce8a-4cef-ad8c-418146706556: + code: "```\nimport synapseclient, os\nfrom pathlib import Path\n\ndownloads_dir\ + \ = Path(__file__).parent.parent / \"downloads\" / \"synapse\"\n\nsyn = synapseclient.Synapse(cache_root_dir=downloads_dir)\ + \ \nsyn.login(authToken=os.environ.get(\"API_SYNAPSE\")) # pass in the authToken\ + \ on python client login!\n```" + integration: nf_synapse + notes: null + query: Login to the Synapse API using the python client + resource_id: 920140d9-ce8a-4cef-ad8c-418146706556 + resource_type: example + b9801ff2-37bb-44d0-9f47-9ce3d18804b5: + filepath: nf_programmatic_webdocs.md + integration: nf_synapse + name: python_client + resource_id: b9801ff2-37bb-44d0-9f47-9ce3d18804b5 + resource_type: file +slug: nf_synapse diff --git a/src/biome/adhoc_data/datasources/synapse/examples.yaml b/src/biome/adhoc_data/datasources/synapse/examples.yaml deleted file mode 100644 index e003a91..0000000 --- a/src/biome/adhoc_data/datasources/synapse/examples.yaml +++ /dev/null @@ -1,11 +0,0 @@ -- query: "Login to the Synapse API using the python client" - code: | - ``` - import synapseclient, os - from pathlib import Path - - downloads_dir = Path(__file__).parent.parent / "downloads" / "synapse" - - syn = synapseclient.Synapse(cache_root_dir=downloads_dir) - syn.login(authToken=os.environ.get("API_SYNAPSE")) # pass in the authToken on python client login! - ``` \ No newline at end of file diff --git a/src/biome/adhoc_data/datasources/usda_foodcentral/api.yaml b/src/biome/adhoc_data/datasources/usda_foodcentral/api.yaml index a525845..0e3bbee 100644 --- a/src/biome/adhoc_data/datasources/usda_foodcentral/api.yaml +++ b/src/biome/adhoc_data/datasources/usda_foodcentral/api.yaml @@ -1,78 +1,107 @@ -name: "USDA FoodData Central API" -integration_type: api - description: | - This openapi is the primary place to obtain data from the USDA's FoodData Central (nutrition) database. - It is intended to assist in incorporating nutrient data into any analysis we need. - -attachments: - raw_documentation: foodcentral_openapi.json - + This openapi is the primary place to obtain data from the USDA's FoodData Central (nutrition) database. + It is intended to assist in incorporating nutrient data into any analysis we need. +integration_type: api +name: USDA FoodData Central API prompt: | - This API is the primary place to obtain data from the USDA's FoodData Central database. - - # OpenAPI Spec JSON: - - ${raw_documentation} - - # Additional Instructions: - - This API is through OpenAPI Spec (Swagger). You will be provided the schema. - All requests go to the following URL: https://api.nal.usda.gov/fdc/v1/ - That is the base URL for all requests. - - To use the USDA FoodData Central API always retrieve the `api_key` query parameter from environment variables: - - api_key: os.environ.get("API_USDA_FDC") - Do not use placeholder auth values from the OpenAPI spec nor other values from the documentation, only use the key from the environment variable! - anywhere you see YOUR_API_KEY or DEMO_KEY, replace with os.environ.get("API_USDA_FDC") - - The FoodData Central API provides REST access to FoodData Central (FDC). It is intended primarily to assist application developers wishing to incorporate nutrient data into their applications or websites. - - What's Available - The API provides two endpoints: the Food Search endpoint, which returns foods that match desired search criteria, and the Food Details endpoint, which returns details on a particular food. - - To take full advantage of the API, developers should familiarize themselves with the database by reading the database documentation available via links on Data Type Documentation. This documentation provides the detailed definitions and descriptions needed to understand the data elements referenced in the API documentation. - Note: The API that was available on the USDA Food Composition Databases Web site is no longer being updated and will be discontinued March 31, 2020. Users are encouraged to begin working with the new FoodData Central API system described on this page. This new API allows users to obtain Standard Reference (SR) Legacy data, provides the most current data from the USDA Global Branded Foods Database, and give users the ability to search for specific foods in Foundation Foods and the Food and Nutrient Database for Dietary Studies (FNDDS) 2019-2020. - - ## Sample Calls - - Note: These calls use DEMO_KEY for the API key, which can be used for initially exploring the API prior to signing up, but has much lower rate limits. See here for more info on uses and limitations of DEMO_KEY. - - GET REQUEST: - ``` - curl https://api.nal.usda.gov/fdc/v1/food/######?api_key=DEMO_KEY - ``` - The number (######) in the above sample must be a valid FoodData Central ID. - - ``` - curl https://api.nal.usda.gov/fdc/v1/foods/list?api_key=DEMO_KEY - curl https://api.nal.usda.gov/fdc/v1/foods/search?api_key=DEMO_KEY&query=Cheddar%20Cheese - ``` - - POST REQUEST: - ``` - curl -XPOST -H "Content-Type:application/json" -d '{{"pageSize":25}}' https://api.nal.usda.gov/fdc/v1/foods/list?api_key=DEMO_KEY - curl -XPOST -H "Content-Type:application/json" -d '{{"query":"Cheddar cheese"}}' https://api.nal.usda.gov/fdc/v1/foods/search?api_key=DEMO_KEY - curl -XPOST -H "Content-Type:application/json" -d "{{\"query\":\"Cheddar cheese\"}}" https://api.nal.usda.gov/fdc/v1/foods/search?api_key=DEMO_KEY - ``` - - ``` - curl -XPOST -H "Content-Type:application/json" -d '{{"query": "Cheddar cheese", "dataType": ["Branded"], "sortBy": "fdcId", "sortOrder": "desc"}}' https://api.nal.usda.gov/fdc/v1/foods/search?api_key=DEMO_KEY - ``` - Note: The "dataType" parameter values need to be specified as an array. - - ## Data Type Comparison - ### Foundation Foods - Data and metadata on individual samples of commodity/commodity-derived minimally processed foods with insights into variability - - ### Experimental Foods - Data on food published in peer-reviewed journals supported by or in collaboration with USDA - - ### Food and Nutrient Database for Dietary Studies (FNDDS) - Data on nutrients and portion weights for foods and beverages reported in What We Eat in America, NHANES - - ### Branded Foods - Data from labels of national and international branded foods collected by a public-private partnership - - ### SR Legacy - Historic data on food components including nutrients derived from analyses, calculations, and published literature + This API is the primary place to obtain data from the USDA's FoodData Central database. + + # OpenAPI Spec JSON: + + ${raw_documentation} + + # Additional Instructions: + + This API is through OpenAPI Spec (Swagger). You will be provided the schema. + All requests go to the following URL: https://api.nal.usda.gov/fdc/v1/ + That is the base URL for all requests. + + To use the USDA FoodData Central API always retrieve the `api_key` query parameter from environment variables: + - api_key: os.environ.get("API_USDA_FDC") + Do not use placeholder auth values from the OpenAPI spec nor other values from the documentation, only use the key from the environment variable! + anywhere you see YOUR_API_KEY or DEMO_KEY, replace with os.environ.get("API_USDA_FDC") + + The FoodData Central API provides REST access to FoodData Central (FDC). It is intended primarily to assist application developers wishing to incorporate nutrient data into their applications or websites. + + What's Available + The API provides two endpoints: the Food Search endpoint, which returns foods that match desired search criteria, and the Food Details endpoint, which returns details on a particular food. + + To take full advantage of the API, developers should familiarize themselves with the database by reading the database documentation available via links on Data Type Documentation. This documentation provides the detailed definitions and descriptions needed to understand the data elements referenced in the API documentation. + Note: The API that was available on the USDA Food Composition Databases Web site is no longer being updated and will be discontinued March 31, 2020. Users are encouraged to begin working with the new FoodData Central API system described on this page. This new API allows users to obtain Standard Reference (SR) Legacy data, provides the most current data from the USDA Global Branded Foods Database, and give users the ability to search for specific foods in Foundation Foods and the Food and Nutrient Database for Dietary Studies (FNDDS) 2019-2020. + + ## Sample Calls + + Note: These calls use DEMO_KEY for the API key, which can be used for initially exploring the API prior to signing up, but has much lower rate limits. See here for more info on uses and limitations of DEMO_KEY. + + GET REQUEST: + ``` + curl https://api.nal.usda.gov/fdc/v1/food/######?api_key=DEMO_KEY + ``` + The number (######) in the above sample must be a valid FoodData Central ID. + + ``` + curl https://api.nal.usda.gov/fdc/v1/foods/list?api_key=DEMO_KEY + curl https://api.nal.usda.gov/fdc/v1/foods/search?api_key=DEMO_KEY&query=Cheddar%20Cheese + ``` + + POST REQUEST: + ``` + curl -XPOST -H "Content-Type:application/json" -d '{{"pageSize":25}}' https://api.nal.usda.gov/fdc/v1/foods/list?api_key=DEMO_KEY + curl -XPOST -H "Content-Type:application/json" -d '{{"query":"Cheddar cheese"}}' https://api.nal.usda.gov/fdc/v1/foods/search?api_key=DEMO_KEY + curl -XPOST -H "Content-Type:application/json" -d "{{\"query\":\"Cheddar cheese\"}}" https://api.nal.usda.gov/fdc/v1/foods/search?api_key=DEMO_KEY + ``` + + ``` + curl -XPOST -H "Content-Type:application/json" -d '{{"query": "Cheddar cheese", "dataType": ["Branded"], "sortBy": "fdcId", "sortOrder": "desc"}}' https://api.nal.usda.gov/fdc/v1/foods/search?api_key=DEMO_KEY + ``` + Note: The "dataType" parameter values need to be specified as an array. + + ## Data Type Comparison + ### Foundation Foods + Data and metadata on individual samples of commodity/commodity-derived minimally processed foods with insights into variability + + ### Experimental Foods + Data on food published in peer-reviewed journals supported by or in collaboration with USDA + + ### Food and Nutrient Database for Dietary Studies (FNDDS) + Data on nutrients and portion weights for foods and beverages reported in What We Eat in America, NHANES + + ### Branded Foods + Data from labels of national and international branded foods collected by a public-private partnership + + ### SR Legacy + Historic data on food components including nutrients derived from analyses, calculations, and published literature +resources: + 0e410001-bac7-4cb7-9f32-b9064c64a4c1: + filepath: foodcentral_openapi.json + integration: usda_fooddata_central_api + name: raw_documentation + resource_id: 0e410001-bac7-4cb7-9f32-b9064c64a4c1 + resource_type: file + 4164944f-a6d4-452c-a290-3ab7bd2e9d9e: + code: |- + import requests + import os + + # Define the API endpoint and parameters + url = 'https://api.nal.usda.gov/fdc/v1/foods/search' + + api_key = os.environ.get("API_USDA_FDC") + + params = {'query': 'raw acerola juice', 'api_key': api_key} + + # Make the request to the USDA FoodData Central API + response = requests.get(url, params=params) + + # Check if the request was successful + if response.status_code == 200: + data = response.json() + # Extract the first food item from the results + food_item = data['foods'][0] + food_item + integration: usda_fooddata_central_api + notes: null + query: Search for nutrients for raw acerola juice + resource_id: 4164944f-a6d4-452c-a290-3ab7bd2e9d9e + resource_type: example +slug: usda_fooddata_central_api diff --git a/src/biome/adhoc_data/datasources/usda_foodcentral/examples.yaml b/src/biome/adhoc_data/datasources/usda_foodcentral/examples.yaml deleted file mode 100644 index f2d3a45..0000000 --- a/src/biome/adhoc_data/datasources/usda_foodcentral/examples.yaml +++ /dev/null @@ -1,21 +0,0 @@ -- query: "Search for nutrients for raw acerola juice" - code: | - import requests - import os - - # Define the API endpoint and parameters - url = 'https://api.nal.usda.gov/fdc/v1/foods/search' - - api_key = os.environ.get("API_USDA_FDC") - - params = {'query': 'raw acerola juice', 'api_key': api_key} - - # Make the request to the USDA FoodData Central API - response = requests.get(url, params=params) - - # Check if the request was successful - if response.status_code == 200: - data = response.json() - # Extract the first food item from the results - food_item = data['foods'][0] - food_item \ No newline at end of file diff --git a/src/biome/adhoc_data/datasources/usgs_pesticide/api.yaml b/src/biome/adhoc_data/datasources/usgs_pesticide/api.yaml index 4c8483b..2c1478f 100644 --- a/src/biome/adhoc_data/datasources/usgs_pesticide/api.yaml +++ b/src/biome/adhoc_data/datasources/usgs_pesticide/api.yaml @@ -1,45 +1,45 @@ -name: "USGS Pesticide National Synthesis Project" -integration_type: dataset - description: | The USGS Pesticide National Synthesis Project contains annual agricultural pesticide use data by county, on a 2-5 year interval, from 2000 to 2019. - +integration_type: dataset +name: USGS Pesticide National Synthesis Project prompt: | - You have access to a file-based USGS Pesticide National Synthesis Project data, for annual agricultural pesticide use by county and pesticide compound. - We've added files on a 2-5 year interval, from 2000 to 2019, under the `${DATASET_FILES_BASE_PATH}/usgs_pesticide` directory. - The exact files are located in: - - ${DATASET_FILES_BASE_PATH}/usgs_pesticide/EPest.county.estimates.2000.txt - - ${DATASET_FILES_BASE_PATH}/usgs_pesticide/EPest.county.estimates.2005.txt - - ${DATASET_FILES_BASE_PATH}/usgs_pesticide/EPest.county.estimates.2010.txt - - ${DATASET_FILES_BASE_PATH}/usgs_pesticide/EPest_county_estimates_2013_2017_v2.txt - - ${DATASET_FILES_BASE_PATH}/usgs_pesticide/EPest_county_estimates_2019.txt - - Files use FIPS states and county codes, and values are tab-delimited. - - The files are actually yearly from 1992-2012, but we've added a subset. You could download - more from that range at: https://water.usgs.gov/nawqa/pnsp/usage/maps/county-level/PesticideUseEstimates/EPest.county.estimates..txt - - The USGS agricultural pesticide-use estimates are supported by funding from the USGS National Water Quality Program - for the purpose of better understanding pesticides in freshwater and their impact - on water availability nationwide. Final annual pesticide-use estimates, for approximately 400 compounds, - from 2018-22 will be published in 2025 (not available yet) - After that, preliminary estimates will be published annually and later updated - with final estimates once the USDA Census of Agriculture is released (every five years). - The total number of pesticides included in the analysis will fluctuate annually. - - Beginning 2015, the provider of the surveyed pesticide data used to derive the - county-level use estimates discontinued making estimates for seed treatment - application of pesticides because of complexity and uncertainty. - Pesticide use estimates prior to 2015 include estimates with seed treatment application. - - The columns are as follows: - - COMPOUND - - YEAR - - STATE_FIPS_CODE - - COUNTY_FIPS_CODE - - EPEST_LOW_KG - - EPEST_HIGH_KG - - You may use the python `us` library to convert FIPS codes to state and county names, - and use pandas to read the tab-delimited text files. + You have access to a file-based USGS Pesticide National Synthesis Project data, for annual agricultural pesticide use by county and pesticide compound. + We've added files on a 2-5 year interval, from 2000 to 2019, under the `${DATASET_FILES_BASE_PATH}/usgs_pesticide` directory. + The exact files are located in: + - ${DATASET_FILES_BASE_PATH}/usgs_pesticide/EPest.county.estimates.2000.txt + - ${DATASET_FILES_BASE_PATH}/usgs_pesticide/EPest.county.estimates.2005.txt + - ${DATASET_FILES_BASE_PATH}/usgs_pesticide/EPest.county.estimates.2010.txt + - ${DATASET_FILES_BASE_PATH}/usgs_pesticide/EPest_county_estimates_2013_2017_v2.txt + - ${DATASET_FILES_BASE_PATH}/usgs_pesticide/EPest_county_estimates_2019.txt + + Files use FIPS states and county codes, and values are tab-delimited. + + The files are actually yearly from 1992-2012, but we've added a subset. You could download + more from that range at: https://water.usgs.gov/nawqa/pnsp/usage/maps/county-level/PesticideUseEstimates/EPest.county.estimates..txt + + The USGS agricultural pesticide-use estimates are supported by funding from the USGS National Water Quality Program + for the purpose of better understanding pesticides in freshwater and their impact + on water availability nationwide. Final annual pesticide-use estimates, for approximately 400 compounds, + from 2018-22 will be published in 2025 (not available yet) + After that, preliminary estimates will be published annually and later updated + with final estimates once the USDA Census of Agriculture is released (every five years). + The total number of pesticides included in the analysis will fluctuate annually. + + Beginning 2015, the provider of the surveyed pesticide data used to derive the + county-level use estimates discontinued making estimates for seed treatment + application of pesticides because of complexity and uncertainty. + Pesticide use estimates prior to 2015 include estimates with seed treatment application. + + The columns are as follows: + - COMPOUND + - YEAR + - STATE_FIPS_CODE + - COUNTY_FIPS_CODE + - EPEST_LOW_KG + - EPEST_HIGH_KG + + You may use the python `us` library to convert FIPS codes to state and county names, + and use pandas to read the tab-delimited text files. +resources: {} +slug: usgs_pesticide_national_synthesis_project diff --git a/src/biome/adhoc_data/datasources/usgs_pesticide/examples.yaml b/src/biome/adhoc_data/datasources/usgs_pesticide/examples.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/src/biome/adhoc_data/datasources/waterservices_usgs/api.yaml b/src/biome/adhoc_data/datasources/waterservices_usgs/api.yaml index 4535e58..ffa43a9 100644 --- a/src/biome/adhoc_data/datasources/waterservices_usgs/api.yaml +++ b/src/biome/adhoc_data/datasources/waterservices_usgs/api.yaml @@ -1,20 +1,48 @@ -name: "USGS Water Services API" -integration_type: api - description: | - You can use this service to retrieve daily statistical data about the hundreds of - thousands of hydrologic sites served by the USGS. Especially crucial to analyze - water quality to correlate with other data sources. - -attachments: - web_documentation: usgs_waterservices_web.md - site_types: site_types.md - + You can use this service to retrieve daily statistical data about the hundreds of + thousands of hydrologic sites served by the USGS. Especially crucial to analyze + water quality to correlate with other data sources. +integration_type: api +name: USGS Water Services API prompt: | - You can use this service to retrieve daily statistical data about the hundreds of - thousands of hydrologic sites served by the USGS. Various output formats available, prefer JSON. + You can use this service to retrieve daily statistical data about the hundreds of + thousands of hydrologic sites served by the USGS. Various output formats available, prefer JSON. - ${web_documentation} + ${web_documentation} - ### Site Types (siteType) minor filter codes - ${site_types} + ### Site Types (siteType) minor filter codes + ${site_types} +resources: + 4c84149a-8c62-40c1-aae9-9bae79c4c2fc: + filepath: usgs_waterservices_web.md + integration: usgs_water_services_api + name: web_documentation + resource_id: 4c84149a-8c62-40c1-aae9-9bae79c4c2fc + resource_type: file + 57592feb-0f49-4aea-8d78-570b3f674123: + code: "import requests\nfrom datetime import datetime, timedelta\nimport pandas\ + \ as pd\n\ndef get_water_quality_data():\n # Base URL for USGS water services\n\ + \ base_url = \"https://waterservices.usgs.gov/nwis/dv/\"\n \n # Calculate\ + \ date range (3 years from today)\n end_date = datetime.now()\n start_date\ + \ = end_date - timedelta(days=3*365)\n \n # Parameters to check common\ + \ water quality indicators\n params = [\n \"00300\",\"00400\",\"00095\"\ + ,\"00010\",\"00900\" # assorted for an example, could include more\n ]\n\ + \ \n # Build request parameters\n params = {\n 'format': 'json',\n\ + \ # 'stateCd': 12, # commented out beause 12 is already included in county\ + \ (only one of thie type of location filter allowed)\n 'countyCd': 12117,\ + \ # Seminole County FIPS code\n 'startDt': start_date.strftime('%Y-%m-%d'),\n\ + \ 'endDt': end_date.strftime('%Y-%m-%d'),\n 'parameterCd': ','.join(params),\n\ + \ 'siteStatus': 'active'\n }\n\n # Make request\n response =\ + \ requests.get(base_url, params=params)" + integration: usgs_water_services_api + notes: null + query: Get water quality data for Seminole County, Florida + resource_id: 57592feb-0f49-4aea-8d78-570b3f674123 + resource_type: example + ebee2534-6b90-4b45-8f05-03801d2e7a85: + filepath: site_types.md + integration: usgs_water_services_api + name: site_types + resource_id: ebee2534-6b90-4b45-8f05-03801d2e7a85 + resource_type: file +slug: usgs_water_services_api diff --git a/src/biome/adhoc_data/datasources/waterservices_usgs/examples.yaml b/src/biome/adhoc_data/datasources/waterservices_usgs/examples.yaml deleted file mode 100644 index a0786ca..0000000 --- a/src/biome/adhoc_data/datasources/waterservices_usgs/examples.yaml +++ /dev/null @@ -1,32 +0,0 @@ -- query: "Get water quality data for Seminole County, Florida" - code: | - import requests - from datetime import datetime, timedelta - import pandas as pd - - def get_water_quality_data(): - # Base URL for USGS water services - base_url = "https://waterservices.usgs.gov/nwis/dv/" - - # Calculate date range (3 years from today) - end_date = datetime.now() - start_date = end_date - timedelta(days=3*365) - - # Parameters to check common water quality indicators - params = [ - "00300","00400","00095","00010","00900" # assorted for an example, could include more - ] - - # Build request parameters - params = { - 'format': 'json', - # 'stateCd': 12, # commented out beause 12 is already included in county (only one of thie type of location filter allowed) - 'countyCd': 12117, # Seminole County FIPS code - 'startDt': start_date.strftime('%Y-%m-%d'), - 'endDt': end_date.strftime('%Y-%m-%d'), - 'parameterCd': ','.join(params), - 'siteStatus': 'active' - } - - # Make request - response = requests.get(base_url, params=params) \ No newline at end of file diff --git a/src/biome/context.py b/src/biome/context.py index 5af6c64..3b0f048 100644 --- a/src/biome/context.py +++ b/src/biome/context.py @@ -5,9 +5,9 @@ from pathlib import Path -from beaker_kernel.lib.context import BeakerContext, action +from beaker_kernel.lib.context import BeakerContext from beaker_kernel.subkernels.python import PythonSubkernel -from beaker_kernel.lib.types import Integration, IntegrationAttachment +from beaker_kernel.lib.types import Integration from beaker_kernel.lib.integrations.base import BaseIntegrationProvider from beaker_kernel.lib.integrations.adhoc import AdhocIntegrationProvider From 903d6a67f32a47bb9552c62237d7502f861e0524 Mon Sep 17 00:00:00 2001 From: Satchel Baldwin Date: Tue, 8 Jul 2025 23:47:25 -0500 Subject: [PATCH 16/21] change: move to uuids instead of name-based slugs --- src/biome/adhoc_data/datasources/ahs/api.yaml | 18 +++++----- src/biome/adhoc_data/datasources/aqs/api.yaml | 18 +++++----- .../datasources/cbioportal/api.yaml | 20 +++++------ src/biome/adhoc_data/datasources/cda/api.yaml | 4 +-- .../datasources/cdc_tracking_network/api.yaml | 6 ++-- .../datasources/census_acs/api.yaml | 10 +++--- .../chis_california_asthma/api.yaml | 4 +-- .../adhoc_data/datasources/epa_tri/api.yaml | 20 +++++------ .../adhoc_data/datasources/faers/api.yaml | 8 ++--- src/biome/adhoc_data/datasources/gdc/api.yaml | 16 ++++----- .../adhoc_data/datasources/gras/api.yaml | 4 +-- src/biome/adhoc_data/datasources/hpa/api.yaml | 6 ++-- src/biome/adhoc_data/datasources/idc/api.yaml | 30 ++++++++-------- .../adhoc_data/datasources/indra/api.yaml | 10 +++--- .../adhoc_data/datasources/netrias/api.yaml | 6 ++-- .../datasources/nhanes_dietary/api.yaml | 4 +-- .../adhoc_data/datasources/nsch/api.yaml | 10 +++--- src/biome/adhoc_data/datasources/pdc/api.yaml | 34 +++++++++---------- .../adhoc_data/datasources/synapse/api.yaml | 10 +++--- .../datasources/usda_foodcentral/api.yaml | 6 ++-- .../datasources/usgs_pesticide/api.yaml | 2 +- .../datasources/waterservices_usgs/api.yaml | 8 ++--- src/biome/context.py | 12 +++---- 23 files changed, 133 insertions(+), 133 deletions(-) diff --git a/src/biome/adhoc_data/datasources/ahs/api.yaml b/src/biome/adhoc_data/datasources/ahs/api.yaml index 65d1591..753f283 100644 --- a/src/biome/adhoc_data/datasources/ahs/api.yaml +++ b/src/biome/adhoc_data/datasources/ahs/api.yaml @@ -116,7 +116,7 @@ prompt: | resources: 3a5b7607-80c5-44ae-a240-a9a388f78afd: filepath: docs.md - integration: census_american_housing_survey_(ahs) + integration: d73e06c5-e41b-4f7c-8a11-0fe1b4e6dfeb name: more_instructions resource_id: 3a5b7607-80c5-44ae-a240-a9a388f78afd resource_type: file @@ -138,7 +138,7 @@ resources: code_pairs_dict = {k.strip(): v.strip() for k, v in (code_pair.split(":") for code_pair in code_pairs)} return code_pairs_dict.get(code) - integration: census_american_housing_survey_(ahs) + integration: d73e06c5-e41b-4f7c-8a11-0fe1b4e6dfeb notes: null query: |- Use the codebook to get the meaning of the codes for the OMB13CBSA variable, matching one code to the meaning by creating a dictionary of code:meaning pairs and returning the statistical metro area name for a given code @@ -182,7 +182,7 @@ resources: Households without children: {asthma_by_children.get(0, 'N/A'):.1f}%\")\nprint(f\"\ Difference: {asthma_by_children.get(1, 0) - asthma_by_children.get(0, 0):.1f}\ \ percentage points\")\n" - integration: census_american_housing_survey_(ahs) + integration: d73e06c5-e41b-4f7c-8a11-0fe1b4e6dfeb notes: null query: |- Analyzing the relationship between housing conditions, asthma, and the presence of children using the 2023 American Housing Survey data. This code calculates and compares the prevalence of various housing conditions (mold, water leaks) in homes with and without reported asthma cases, and also examines the prevalence of asthma in households with and without children. @@ -228,7 +228,7 @@ resources: )\n print(f\" - Households with 1-2 persons: {without_children:.1f}%\"\ )\n print(f\" - Difference: {with_children - without_children:.1f}\ \ percentage points\")\n" - integration: census_american_housing_survey_(ahs) + integration: d73e06c5-e41b-4f7c-8a11-0fe1b4e6dfeb notes: null query: |- Analyzing housing conditions in Houston, TX using the 2007 American Housing Survey data. This code identifies Houston records using the SMSA code, analyzes the prevalence of water leaks and pest problems, and compares these conditions between larger households (3+ persons, likely to have children) and smaller households (1-2 persons). @@ -247,7 +247,7 @@ resources: # Return the Response Codes column ombs_row['Response Codes'] - integration: census_american_housing_survey_(ahs) + integration: d73e06c5-e41b-4f7c-8a11-0fe1b4e6dfeb notes: null query: What is the meaning of the codes for the OMB13CBSA variable? resource_id: ad42a806-0e81-4e63-97cf-2f7aad6a1a35 @@ -301,7 +301,7 @@ resources: \ condition|hvac', case=False, na=False) |\n asthma_vars_df['Question_Text'].str.contains('heat|cool|air\ \ condition|hvac', case=False, na=False)\n]\n\nprint(f\"\\nFound {len(hvac_vars)}\ \ variables related to heating and cooling systems\")" - integration: census_american_housing_survey_(ahs) + integration: d73e06c5-e41b-4f7c-8a11-0fe1b4e6dfeb notes: null query: |- Exploring the American Housing Survey codebook to identify variables related to asthma, respiratory health, and housing conditions. This code searches the codebook for terms related to asthma, air quality, pests, and HVAC systems, and provides a summary of available variables that could be relevant for analyzing the relationship between housing conditions and respiratory health. @@ -375,7 +375,7 @@ resources: \ by Presence of Children:\")\nprint(summary_table[['Condition', 'With Children\ \ (%)', 'Without Children (%)', 'Difference (percentage points)', 'Sample Size']].to_string(index=False,\ \ float_format=lambda x: f\"{x:.1f}\"))\n" - integration: census_american_housing_survey_(ahs) + integration: d73e06c5-e41b-4f7c-8a11-0fe1b4e6dfeb notes: null query: |- Analyzing and visualizing the relationship between housing conditions (mold, water leaks) and the presence of children in households using the 2023 American Housing Survey data. This code creates a horizontal bar chart comparing the prevalence of various housing conditions in homes with and without children, and generates a summary table of the differences. @@ -432,10 +432,10 @@ resources: \ as \"'6'\",\n # that is why we strip them or sometimes compare\ \ to substrings\n combined_data[col] = combined_data[col].apply(convert_value)\n\ \ \n return combined_data\n else:\n return pd.DataFrame()\n" - integration: census_american_housing_survey_(ahs) + integration: d73e06c5-e41b-4f7c-8a11-0fe1b4e6dfeb notes: null query: |- I wonder if there are any trends for metro areas in Florida, US in which mold presence increased between years 2001 and 2011? Can you get the data for matching mold presence? resource_id: ea7d2aec-2d76-4b95-ae85-177db08ff019 resource_type: example -slug: census_american_housing_survey_(ahs) +slug: d73e06c5-e41b-4f7c-8a11-0fe1b4e6dfeb diff --git a/src/biome/adhoc_data/datasources/aqs/api.yaml b/src/biome/adhoc_data/datasources/aqs/api.yaml index 7d87df5..fc39e7a 100644 --- a/src/biome/adhoc_data/datasources/aqs/api.yaml +++ b/src/biome/adhoc_data/datasources/aqs/api.yaml @@ -207,7 +207,7 @@ resources: df['date'] = pd.to_datetime(df['date'])\n\n# Sort by date and parameter\ndf\ \ = df.sort_values(['date', 'parameter'])\n\n# Save to CSV\ndf.to_csv('harris_county_air_quality_2022.csv',\ \ index=False)\n" - integration: epa_air_quality_system + integration: 715e610b-28e4-4ae8-99b8-292db41dbd00 notes: null query: Retrieve daily air quality data for ozone, CO, NO2, and PM2.5 in Harris County, TX for each month of the year 2022. @@ -254,7 +254,7 @@ resources: data = response.json() data - integration: epa_air_quality_system + integration: 715e610b-28e4-4ae8-99b8-292db41dbd00 notes: null query: Query annual air pollution data for Seminole County, Florida for the year 2000. Use pollutants relating to asthma. @@ -299,7 +299,7 @@ resources: \ print(f\"{pollutant}: {code}\")\n else:\n print(\"Failed\ \ to retrieve criteria parameters data\")\nelse:\n print(\"Failed to retrieve\ \ parameter classes data\")\n" - integration: epa_air_quality_system + integration: 715e610b-28e4-4ae8-99b8-292db41dbd00 notes: null query: |- Retrieving and exploring parameter classes and codes for air quality research. This example demonstrates how to get a list of parameter classes, then retrieve specific parameters within the CRITERIA class, which contains the most important air pollutants for health research. It also identifies key parameter codes relevant to asthma research. @@ -336,7 +336,7 @@ resources: \ # Display first few rows\n print(df.head())\n else:\n print(\"\ No data found for the specified parameters\")\nelse:\n print(\"No data collected.\"\ )\n" - integration: epa_air_quality_system + integration: 715e610b-28e4-4ae8-99b8-292db41dbd00 notes: null query: Collect historical air quality data from the EPA Air Quality System (AQS) for Orange County, FL. @@ -344,7 +344,7 @@ resources: resource_type: example 365697ce-eb50-444b-8a82-637a2a0131f4: filepath: aqs.json - integration: epa_air_quality_system + integration: 715e610b-28e4-4ae8-99b8-292db41dbd00 name: raw_documentation resource_id: 365697ce-eb50-444b-8a82-637a2a0131f4 resource_type: file @@ -375,7 +375,7 @@ resources: \ pollutant_name.lower()})\n \n return daily_avg, df['units_of_measure'].iloc[0]\n\ \ else:\n print(f\"Failed to retrieve {pollutant_name} data\")\n \ \ return None, None\n" - integration: epa_air_quality_system + integration: 715e610b-28e4-4ae8-99b8-292db41dbd00 notes: null query: 'Here''s a reusable function for retrieving and processing data for multiple pollutants:' @@ -398,7 +398,7 @@ resources: \ a county with a lot of data- lets generate a schema and inspect that instead\ \ of inspecting the raw data\nbuilder = SchemaBuilder()\nbuilder.add_object(response_data)\n\ schema = builder.to_schema()\n\nprint(schema[\"properties\"])\n" - integration: epa_air_quality_system + integration: 715e610b-28e4-4ae8-99b8-292db41dbd00 notes: null query: |- Inspect API response change by generating a schema from an API response so that we don't inspect the raw data and blow up the context window. @@ -469,10 +469,10 @@ resources: \ annot=True, cmap='coolwarm', vmin=-1, vmax=1, center=0)\n plt.title(f'Correlation\ \ Matrix of Air Pollutants ({year})', fontsize=16)\n plt.tight_layout()\n\ \ plt.show()\n" - integration: epa_air_quality_system + integration: 715e610b-28e4-4ae8-99b8-292db41dbd00 notes: null query: |- Multi-pollutant correlation analysis for air quality research. This example demonstrates how to retrieve data for multiple pollutants (Ozone, PM2.5, NO2, and SO2), process the data to get daily averages, merge the datasets, and analyze correlations between different pollutants. It includes a visualization of the correlation matrix using a heatmap. resource_id: e643cf2e-66f1-428e-8f2c-2626c4c59192 resource_type: example -slug: epa_air_quality_system +slug: 715e610b-28e4-4ae8-99b8-292db41dbd00 diff --git a/src/biome/adhoc_data/datasources/cbioportal/api.yaml b/src/biome/adhoc_data/datasources/cbioportal/api.yaml index 0d3f270..b8455ed 100644 --- a/src/biome/adhoc_data/datasources/cbioportal/api.yaml +++ b/src/biome/adhoc_data/datasources/cbioportal/api.yaml @@ -59,7 +59,7 @@ resources: aml_studies_df = fetch_aml_studies() print(f"Found {len(aml_studies_df)} AML-related studies") print(aml_studies_df) - integration: cbioportal + integration: 26b8655b-4f68-4f70-bb30-5f1397dd201a notes: null query: Fetch and filter studies related to Acute Myeloid Leukemia (AML) from cBioPortal API. @@ -202,7 +202,7 @@ resources: print("\nWide format example (first few rows):") print(stat5_all_studies_wide.head()) - integration: cbioportal + integration: 26b8655b-4f68-4f70-bb30-5f1397dd201a notes: | This example demonstrates: 1. How to properly fetch sample IDs for each study @@ -222,7 +222,7 @@ resources: resource_type: example 52adfa23-1b07-462e-968b-d09ed12848f0: filepath: cbioportal.json - integration: cbioportal + integration: 26b8655b-4f68-4f70-bb30-5f1397dd201a name: cbioportal resource_id: 52adfa23-1b07-462e-968b-d09ed12848f0 resource_type: file @@ -280,7 +280,7 @@ resources: else: print(f"Error fetching mutations: {mutations_response.status_code}") print("Response:", mutations_response.text) - integration: cbioportal + integration: 26b8655b-4f68-4f70-bb30-5f1397dd201a notes: | It shows: 1. How to properly construct the API endpoint URL with sample list ID @@ -380,7 +380,7 @@ resources: ] print(f"\nRNA expression profiles for {first_study}:") print(rna_profiles[['molecularProfileId', 'name', 'datatype']].to_string()) - integration: cbioportal + integration: 26b8655b-4f68-4f70-bb30-5f1397dd201a notes: null query: |- Example showing how to retrieve and analyze the molecular profiles available for multiple studies from cBioPortal. The code demonstrates how to fetch metadata on the molecular profiles, organize them into DataFrames, and analyze specific information about each profile like the types of profiles available such as RNA expression. @@ -481,7 +481,7 @@ resources: # Save to CSV combined_df.to_csv('all_aml_mutations.csv', index=False) - integration: cbioportal + integration: 26b8655b-4f68-4f70-bb30-5f1397dd201a notes: null query: |- Retrieve and analyze mutation data from multiple AML studies, including data processing and summary statistics. This example demonstrates how to fetch mutations across multiple studies, handle the nested JSON response, and create a comprehensive mutation dataset with key information. @@ -505,7 +505,7 @@ resources: print(f"Description: {study['description']}") print(f"Cancer Type ID: {study['cancerTypeId']}") print("-" * 80) - integration: cbioportal + integration: 26b8655b-4f68-4f70-bb30-5f1397dd201a notes: null query: |- Query and display all colorectal cancer studies from cBioPortal. This example searches for studies with cancer type IDs 'coadread', 'coad', and 'read' (representing colorectal adenocarcinoma, colon adenocarcinoma, and rectal adenocarcinoma respectively). For each study, it displays the name, study ID, description, and cancer type ID. @@ -539,7 +539,7 @@ resources: # Print the DataFrame print(df) - integration: cbioportal + integration: 26b8655b-4f68-4f70-bb30-5f1397dd201a notes: null query: Query for studies related to colorectal cancer using the cancerTypeId 'coadread'. resource_id: c68b74b2-c97c-4cf5-b8b4-7e514db2db65 @@ -622,7 +622,7 @@ resources: print(f" - {change}") if len(details['protein_changes']) > 5: print(f" ... and {len(details['protein_changes'])-5} more changes") - integration: cbioportal + integration: 26b8655b-4f68-4f70-bb30-5f1397dd201a notes: | The example demonstrates how to: 1. Get sample IDs for a study @@ -634,4 +634,4 @@ resources: Query mutation data from cBioPortal for a specific study (Genentech colorectal cancer study) and analyze mutations in key cancer genes (APC, TP53, KRAS, PIK3CA) resource_id: d54d8188-81f5-4fff-a641-230480369fe4 resource_type: example -slug: cbioportal +slug: 26b8655b-4f68-4f70-bb30-5f1397dd201a diff --git a/src/biome/adhoc_data/datasources/cda/api.yaml b/src/biome/adhoc_data/datasources/cda/api.yaml index a4785a2..0ae4e95 100644 --- a/src/biome/adhoc_data/datasources/cda/api.yaml +++ b/src/biome/adhoc_data/datasources/cda/api.yaml @@ -111,8 +111,8 @@ prompt: | resources: 5361ce45-01fb-4c16-aa22-31d5397a0bd1: filepath: cda.yaml - integration: cancer_data_aggregator + integration: 16fba0e0-bf97-4d4d-8989-aad9ef934319 name: raw_documentation resource_id: 5361ce45-01fb-4c16-aa22-31d5397a0bd1 resource_type: file -slug: cancer_data_aggregator +slug: 16fba0e0-bf97-4d4d-8989-aad9ef934319 diff --git a/src/biome/adhoc_data/datasources/cdc_tracking_network/api.yaml b/src/biome/adhoc_data/datasources/cdc_tracking_network/api.yaml index b61c9a2..6315d41 100644 --- a/src/biome/adhoc_data/datasources/cdc_tracking_network/api.yaml +++ b/src/biome/adhoc_data/datasources/cdc_tracking_network/api.yaml @@ -33,14 +33,14 @@ prompt: | resources: 5b62b4c6-536c-4a8d-8e4b-bd295ac8ed72: filepath: api_examples.md - integration: cdc_tracking_network + integration: ba3db193-3d39-4793-a525-c569d31bd38b name: api_examples resource_id: 5b62b4c6-536c-4a8d-8e4b-bd295ac8ed72 resource_type: file 7c5e6353-7e1f-4639-91ee-9dc568887a69: filepath: user_guide.md - integration: cdc_tracking_network + integration: ba3db193-3d39-4793-a525-c569d31bd38b name: user_guide resource_id: 7c5e6353-7e1f-4639-91ee-9dc568887a69 resource_type: file -slug: cdc_tracking_network +slug: ba3db193-3d39-4793-a525-c569d31bd38b diff --git a/src/biome/adhoc_data/datasources/census_acs/api.yaml b/src/biome/adhoc_data/datasources/census_acs/api.yaml index d55c9c3..89bdba3 100644 --- a/src/biome/adhoc_data/datasources/census_acs/api.yaml +++ b/src/biome/adhoc_data/datasources/census_acs/api.yaml @@ -25,7 +25,7 @@ prompt: "The American Community Survey (ACS) is an ongoing survey that provides resources: 76813161-e3dd-400e-b1db-c6534c3fb484: filepath: census_web.md - integration: census_acs_(american_community_survey)_and_sf1 + integration: 50e33776-2cc4-4b05-8135-4ff03d5d42b1 name: web_documentation resource_id: 76813161-e3dd-400e-b1db-c6534c3fb484 resource_type: file @@ -39,7 +39,7 @@ resources: \ = ['Name', 'Median_Income', 'Poverty_Count', \n 'Median_Rent',\ \ 'Population', 'state', 'county', 'tract']\n\n# Calculate poverty rate\nharris_ses_df['Poverty_Rate']\ \ = harris_ses_df['Poverty_Count'] / harris_ses_df['Population'] * 100\n" - integration: census_acs_(american_community_survey)_and_sf1 + integration: 50e33776-2cc4-4b05-8135-4ff03d5d42b1 notes: null query: Calculate the poverty rate in Harris County, Texas- Descriptive analysis resource_id: 93d4da90-fe43-4121-8e1b-d9d677637143 @@ -58,15 +58,15 @@ resources: exposure_by_ses = pd.merge(environmental_data, harris_ses_df, on='tract')\n\n\ # Calculate average exposures by income quartile\nquartile_summary = exposure_by_ses.groupby('Income_Quartile')['PM25_Annual',\ \ 'Ozone_Days_Exceeded', 'TRI_Releases'].mean()" - integration: census_acs_(american_community_survey)_and_sf1 + integration: 50e33776-2cc4-4b05-8135-4ff03d5d42b1 notes: null query: Calculate the poverty rate in Harris County, Texas- Stratified analysis resource_id: 9a1dd209-4e65-49af-a5cc-2f9d73cd8d4b resource_type: example cb8c091d-ff82-46d7-bdea-893aa5b0c6c5: filepath: census_sdk.md - integration: census_acs_(american_community_survey)_and_sf1 + integration: 50e33776-2cc4-4b05-8135-4ff03d5d42b1 name: sdk_documentation resource_id: cb8c091d-ff82-46d7-bdea-893aa5b0c6c5 resource_type: file -slug: census_acs_(american_community_survey)_and_sf1 +slug: 50e33776-2cc4-4b05-8135-4ff03d5d42b1 diff --git a/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml b/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml index 8b9d5fb..f66ff83 100644 --- a/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml +++ b/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml @@ -37,10 +37,10 @@ resources: print(\"Column names:\", df.columns.tolist())\n\n# Display a sample of the data\ \ to understand its structure\nprint(\"\\nSample data:\")\nprint(df.head(2))\n\ \ " - integration: chis_california_asthma + integration: fd660ac8-f7cc-477a-bbcb-38df4d330711 notes: null query: Open the california chis asthma dataset and display columns/first 2 rows to understand the structure resource_id: ca7ff323-7bef-489d-9d2c-760628f40825 resource_type: example -slug: chis_california_asthma +slug: fd660ac8-f7cc-477a-bbcb-38df4d330711 diff --git a/src/biome/adhoc_data/datasources/epa_tri/api.yaml b/src/biome/adhoc_data/datasources/epa_tri/api.yaml index f84a8a6..dacfdc3 100644 --- a/src/biome/adhoc_data/datasources/epa_tri/api.yaml +++ b/src/biome/adhoc_data/datasources/epa_tri/api.yaml @@ -123,7 +123,7 @@ resources: \ plt.text(bar.get_x() + bar.get_width()/2., height + 0.5,\n f'{height:.1f}M',\ \ ha='center', va='bottom')\n\nplt.tight_layout()\nplt.savefig('top_zips_by_releases.png')\n\ plt.show()\n" - integration: epa_toxic_release_inventory_(tri) + integration: f9653810-18ac-4108-a1ea-7f7c2e31274a notes: null query: |- Analyzing and visualizing toxic releases by ZIP code. This example shows how to aggregate TRI data by geographic units (ZIP codes), calculate total releases and hazard scores, and create a bar chart visualization of the top areas with highest toxic releases. @@ -145,7 +145,7 @@ resources: print(f"Found {len(orange_county)} toxic release records containing 'ORANGE, FL' in the County name.") print("\nSample of the data:") print(orange_county.head()) - integration: epa_toxic_release_inventory_(tri) + integration: f9653810-18ac-4108-a1ea-7f7c2e31274a notes: null query: Open the TRI data file with index_col=False, since otherwise columns are shifted, and find data for Orange County, FL. @@ -200,7 +200,7 @@ resources: ]] result - integration: epa_toxic_release_inventory_(tri) + integration: f9653810-18ac-4108-a1ea-7f7c2e31274a notes: null query: |- Capture any asthma-respiratory related hazard incident from the TRI- EPA's Toxic Release Inventory data- you have access to this api. FOr now focus on problems in 2022, in the county of Harris, TX. @@ -242,7 +242,7 @@ resources: \ in Toxic Releases')\nplt.xlabel('Year')\nplt.ylabel('Releases (Million lb)')\n\ plt.legend()\nplt.grid(True, linestyle='--', alpha=0.7)\nplt.tight_layout()\n\ plt.savefig('yearly_condition_trends.png')\nplt.show()\n" - integration: epa_toxic_release_inventory_(tri) + integration: f9653810-18ac-4108-a1ea-7f7c2e31274a notes: null query: |- Identifying and analyzing chemicals related to specific health conditions. This example demonstrates how to filter TRI data for chemicals associated with a particular health condition (like asthma), calculate their proportion of total releases, and analyze trends over time. This approach can be adapted for various health conditions by modifying the list of relevant chemicals. @@ -282,7 +282,7 @@ resources: \ fill_opacity=0.6,\n opacity=0.8\n ).add_to(marker_cluster)\n\ \n# Save the map\nmap_file = 'facilities_map.html'\nm.save(map_file)\nprint(f\"\ Interactive map saved to: {map_file}\")\n" - integration: epa_toxic_release_inventory_(tri) + integration: f9653810-18ac-4108-a1ea-7f7c2e31274a notes: null query: |- Creating an interactive map of toxic release facilities. This example shows how to use the folium library to generate an interactive web map displaying the locations of facilities, with circle sizes proportional to release amounts and popup information showing detailed facility data. This visualization helps identify spatial patterns and hotspots of toxic releases. @@ -328,7 +328,7 @@ resources: ]] print(result) - integration: epa_toxic_release_inventory_(tri) + integration: f9653810-18ac-4108-a1ea-7f7c2e31274a notes: null query: |- Query to retrieve asthma-respiratory related hazard incidents from the EPA's TRI for the year 2022 in Harris County, TX. @@ -365,7 +365,7 @@ resources: \ fraction', \n fontsize=12, bbox=dict(boxstyle=\"round,pad=0.3\"\ , fc=\"white\", ec=\"gray\", alpha=0.8))\n\nplt.tight_layout()\nplt.savefig('releases_vs_hazard.png')\n\ plt.show()" - integration: epa_toxic_release_inventory_(tri) + integration: f9653810-18ac-4108-a1ea-7f7c2e31274a notes: null query: |- Analyzing the relationship between release amounts and hazard scores. This example demonstrates how to investigate the correlation between the quantity of toxic releases and their associated RSEI Hazard scores using scatter plots, trend lines, and correlation statistics. This analysis helps understand whether larger releases necessarily correspond to higher health risks. @@ -393,7 +393,7 @@ resources: \ Total Releases:\")\nfor i, (name, lat, lon, releases, hazard, chemicals, years)\ \ in enumerate(facility_summary.head(10).values, 1):\n print(f\"{i}. {name}:\ \ {releases:,.2f} lb, RSEI Hazard: {hazard:,.2f}, Chemicals: {chemicals}\")\n" - integration: epa_toxic_release_inventory_(tri) + integration: f9653810-18ac-4108-a1ea-7f7c2e31274a notes: null query: |- Basic loading, cleaning, and summarizing of EPA TRI data for a specific county. This example demonstrates how to handle the comma-separated numeric values in the dataset, filter for a specific geographic area, and create a summary of facilities with their total releases, hazard scores, and chemical counts. @@ -446,10 +446,10 @@ resources: result = result.sort_values('Releases (lb)', ascending=False) print("\nResults:") print(result.head()) - integration: epa_toxic_release_inventory_(tri) + integration: f9653810-18ac-4108-a1ea-7f7c2e31274a notes: null query: |- When not finding results for a combined query, try a more systematic approach by finding items and counting length by one by one, in order to discover if there is matching data, but the query is not returning any results. This is to discover amonia references for the county of 'HARRIS, TX'. resource_id: e2bd90f9-24a6-41e5-a3ba-1aca807d8990 resource_type: example -slug: epa_toxic_release_inventory_(tri) +slug: f9653810-18ac-4108-a1ea-7f7c2e31274a diff --git a/src/biome/adhoc_data/datasources/faers/api.yaml b/src/biome/adhoc_data/datasources/faers/api.yaml index 92d4aae..5ec76d7 100644 --- a/src/biome/adhoc_data/datasources/faers/api.yaml +++ b/src/biome/adhoc_data/datasources/faers/api.yaml @@ -19,7 +19,7 @@ prompt: | resources: 66215565-2562-455a-aedc-80cdb06f6c1a: filepath: faers_fields_reference.csv - integration: fda_drug_adverse_event_faers_api + integration: da3ecb53-dfc6-467d-a70d-43ab287e8254 name: fields_reference resource_id: 66215565-2562-455a-aedc-80cdb06f6c1a resource_type: file @@ -56,7 +56,7 @@ resources: top_med = df.iloc[0] print(f"\nMedication with highest number of adverse events:") print(f"{top_med['term']}: {top_med['count']} reports") - integration: fda_drug_adverse_event_faers_api + integration: da3ecb53-dfc6-467d-a70d-43ab287e8254 notes: null query: Can you tell me which medication have the highest adverse effects reported from all of the available? @@ -64,8 +64,8 @@ resources: resource_type: example f9afc1aa-371f-4318-9fb1-cd5030db39fe: filepath: faers_web.md - integration: fda_drug_adverse_event_faers_api + integration: da3ecb53-dfc6-467d-a70d-43ab287e8254 name: web_documentation resource_id: f9afc1aa-371f-4318-9fb1-cd5030db39fe resource_type: file -slug: fda_drug_adverse_event_faers_api +slug: da3ecb53-dfc6-467d-a70d-43ab287e8254 diff --git a/src/biome/adhoc_data/datasources/gdc/api.yaml b/src/biome/adhoc_data/datasources/gdc/api.yaml index 4923562..085d7c9 100644 --- a/src/biome/adhoc_data/datasources/gdc/api.yaml +++ b/src/biome/adhoc_data/datasources/gdc/api.yaml @@ -52,7 +52,7 @@ prompt: | resources: 4b9845c7-d61b-440d-880e-b3d2b315a415: filepath: facets.txt - integration: genomics_data_commons + integration: 7f4da82e-9df8-4235-82cc-00dcbceea9e8 name: facets resource_id: 4b9845c7-d61b-440d-880e-b3d2b315a415 resource_type: file @@ -102,7 +102,7 @@ resources: print(f"There are {len(all_cases_df)} cases in GDC in total") # Display the DataFrame all_cases_df.head() - integration: genomics_data_commons + integration: 7f4da82e-9df8-4235-82cc-00dcbceea9e8 notes: null query: |- Get all cases from GDC with no filters. Fetch only the case ids and paginate through the results to fetch all available cases. Store the results as a dataframe @@ -167,7 +167,7 @@ resources: ssms = pd.DataFrame(json_normalize(all_ssms)) ssms.head() - integration: genomics_data_commons + integration: 7f4da82e-9df8-4235-82cc-00dcbceea9e8 notes: null query: Query the GDC for cases of myeloid leukemia with a JAK2 somatic mutation and return the results as a pandas dataframe. @@ -175,7 +175,7 @@ resources: resource_type: example 74735957-03a3-4706-aa40-f8bf200553cc: filepath: gdc.md - integration: genomics_data_commons + integration: 7f4da82e-9df8-4235-82cc-00dcbceea9e8 name: raw_documentation resource_id: 74735957-03a3-4706-aa40-f8bf200553cc resource_type: file @@ -209,7 +209,7 @@ resources: \ in lung cancer cases for males under 45 who never smoked:\")\n lung_mutation_df\ \ = pd.DataFrame(json_normalize(mutations))\nelse:\n print(f\"Error: {response.status_code}\ \ - {response.text}\")\n \nlung_mutation_df.head()" - integration: genomics_data_commons + integration: 7f4da82e-9df8-4235-82cc-00dcbceea9e8 notes: null query: |- Get simple somatic mutation occurrences (cases) from GDC where the primary site is bronchus and lung, the gender is male, the age at diagnosis is less than 45 years old, and the patient is a lifelong non-smoker. Return the results as a pandas dataframe. @@ -284,7 +284,7 @@ resources: print(f"There are {len(cases_df)} cases in GDC for AML") # Display the DataFrame cases_df.head() - integration: genomics_data_commons + integration: 7f4da82e-9df8-4235-82cc-00dcbceea9e8 notes: null query: |- Get all cases where disease type is myeloid leukemia from GDC and return the results as a pandas dataframe. Paginate through the results to fetch all available cases @@ -292,8 +292,8 @@ resources: resource_type: example a96216e1-8a2b-4fab-90e8-565d1db62fbe: filepath: gdc_mappings.json - integration: genomics_data_commons + integration: 7f4da82e-9df8-4235-82cc-00dcbceea9e8 name: mappings resource_id: a96216e1-8a2b-4fab-90e8-565d1db62fbe resource_type: file -slug: genomics_data_commons +slug: 7f4da82e-9df8-4235-82cc-00dcbceea9e8 diff --git a/src/biome/adhoc_data/datasources/gras/api.yaml b/src/biome/adhoc_data/datasources/gras/api.yaml index 574b318..2548c11 100644 --- a/src/biome/adhoc_data/datasources/gras/api.yaml +++ b/src/biome/adhoc_data/datasources/gras/api.yaml @@ -57,9 +57,9 @@ resources: print(matches[['GRAS Notice (GRN) No.', 'Substance', 'Date of filing', "FDA's Letter"]]) else: print(f"No GRAS notices found for {search_term}") - integration: fda_generally_recognized_as_safe_(gras) + integration: 2a6f1149-8ff2-4c21-b528-cbf8b18bf30e notes: null query: Is there any FDA GRAS notice for Dioctyl sodium sulfosuccinate as an additive? resource_id: fd30b48e-e83b-41a2-a969-49ebc58da112 resource_type: example -slug: fda_generally_recognized_as_safe_(gras) +slug: 2a6f1149-8ff2-4c21-b528-cbf8b18bf30e diff --git a/src/biome/adhoc_data/datasources/hpa/api.yaml b/src/biome/adhoc_data/datasources/hpa/api.yaml index f153fe2..4a3a5af 100644 --- a/src/biome/adhoc_data/datasources/hpa/api.yaml +++ b/src/biome/adhoc_data/datasources/hpa/api.yaml @@ -39,7 +39,7 @@ prompt: '${raw_documentation} resources: 04c53147-cc0a-42f5-8e1d-6427a595948b: filepath: hpa_docs.md - integration: human_protein_atlas + integration: 6f995eeb-b836-4d32-8e4a-827e49d5ccf7 name: raw_documentation resource_id: 04c53147-cc0a-42f5-8e1d-6427a595948b resource_type: file @@ -79,10 +79,10 @@ resources: print(f"Subcellular main location: {data.get('Subcellular main location', 'N/A')}") if 'Subcellular additional location' in data: print(f"Subcellular additional location: {data.get('Subcellular additional location', 'N/A')}") - integration: human_protein_atlas + integration: 6f995eeb-b836-4d32-8e4a-827e49d5ccf7 notes: null query: |- Query the Human Protein Atlas API for RNA and protein expression summary of a gene (JAK2) using its Ensembl ID. The example demonstrates how to retrieve and display tissue specificity, distribution, and subcellular localization information. resource_id: 92183e68-5e56-4487-a92f-8a1163b4e178 resource_type: example -slug: human_protein_atlas +slug: 6f995eeb-b836-4d32-8e4a-827e49d5ccf7 diff --git a/src/biome/adhoc_data/datasources/idc/api.yaml b/src/biome/adhoc_data/datasources/idc/api.yaml index 61e721f..d64be59 100644 --- a/src/biome/adhoc_data/datasources/idc/api.yaml +++ b/src/biome/adhoc_data/datasources/idc/api.yaml @@ -26,7 +26,7 @@ resources: \ collection_id = 'cmb_aml'\n\"\"\"\n\ntry:\n df_all_studies = client.sql_query(query_all_studies)\n\ \ print(\"All studies in the 'cmb_aml' collection:\")\n print(df_all_studies)\n\ except Exception as e:\n print(f\"An error occurred: {str(e)}\")\n" - integration: imaging_data_commons + integration: 43858948-4b58-4312-b281-7a65e9c0d38d notes: null query: List all studies for a given collection resource_id: 090cdd92-54e0-4a30-82ab-a2e9c67b50c7 @@ -63,7 +63,7 @@ resources: print("\nTotal number of records:", len(df)) except Exception as e: print(f"An error occurred: {str(e)}") - integration: imaging_data_commons + integration: 43858948-4b58-4312-b281-7a65e9c0d38d notes: null query: Get slide microscopy data resource_id: 1be1fe03-4bb2-4240-a95e-96efb9c1d4d8 @@ -92,7 +92,7 @@ resources: \ 'cmb_aml')\n\"\"\"\n\ntry:\n df_study = client.sql_query(query_study)\n\ \ print(\"\\nStudy-level information:\")\n print(df_study)\nexcept Exception\ \ as e:\n print(f\"An error occurred querying study data: {str(e)}\")\n" - integration: imaging_data_commons + integration: 43858948-4b58-4312-b281-7a65e9c0d38d notes: null query: Get additional information for microscopy data for a specific collection resource_id: 25d73a5f-adb2-495c-9588-8237bb39883d @@ -104,7 +104,7 @@ resources: \ LIKE '%aml%'\n\"\"\"\n\ntry:\n df_collections = client.sql_query(query)\n\ \ print(\"Collections with 'aml' in their name:\")\n print(df_collections)\n\ except Exception as e:\n print(f\"An error occurred: {str(e)}\")\n" - integration: imaging_data_commons + integration: 43858948-4b58-4312-b281-7a65e9c0d38d notes: null query: Find collections in the Imaging Data Commons with 'aml' in their name, likely related to Acute Myeloid Leukemia. @@ -128,7 +128,7 @@ resources: print(df.columns.tolist()) except Exception as e: print(f"An error occurred: {str(e)}") - integration: imaging_data_commons + integration: 43858948-4b58-4312-b281-7a65e9c0d38d notes: null query: Get columns available in IDC index resource_id: 49f63b7d-8e99-4e88-ac9d-86db0d831a8f @@ -157,7 +157,7 @@ resources: # Execute the download s5cmd_binary = client.s5cmdPath !{s5cmd_binary} --no-sign-request --endpoint-url https://s3.amazonaws.com run download_manifest.txt - integration: imaging_data_commons + integration: 43858948-4b58-4312-b281-7a65e9c0d38d notes: null query: Download bone marrow biopsy images from the 'cmb_aml' collection for a specific study using IDC's public S3 bucket. @@ -171,7 +171,7 @@ resources: \ df_studies = client.sql_query(query)\n print(\"Studies in 'cptac_aml':\"\ )\n print(df_studies)\nexcept Exception as e:\n print(f\"An error occurred:\ \ {str(e)}\")\n" - integration: imaging_data_commons + integration: 43858948-4b58-4312-b281-7a65e9c0d38d notes: null query: Fetch studies for a given collection in the Imaging Data Commons (IDC), specifically for the 'cptac_aml' collection. @@ -192,7 +192,7 @@ resources: """ df = client.sql_query(query) print(df.head()) - integration: imaging_data_commons + integration: 43858948-4b58-4312-b281-7a65e9c0d38d notes: null query: Retrieve imagery URLs for the 'XR_Biopsy_BoneMarrow' study in the 'cmb_aml' collection from IDC. @@ -200,7 +200,7 @@ resources: resource_type: example a155ea7d-f42f-4546-8d4c-b1135994ea47: filepath: idc.md - integration: imaging_data_commons + integration: 43858948-4b58-4312-b281-7a65e9c0d38d name: raw_documentation resource_id: a155ea7d-f42f-4546-8d4c-b1135994ea47 resource_type: file @@ -218,7 +218,7 @@ resources: plt.title('Bone Marrow Biopsy Image') plt.axis('off') plt.show() - integration: imaging_data_commons + integration: 43858948-4b58-4312-b281-7a65e9c0d38d notes: null query: Basic plot of dicom image from file resource_id: bbfbce09-abfe-4565-a50b-6afc299df2d3 @@ -249,7 +249,7 @@ resources: No pixel data found in the DICOM file\")\n\nexcept Exception as e:\n print(f\"\ An error occurred: {str(e)}\")\nfinally:\n # Clean up\n if os.path.exists(local_file):\n\ \ os.remove(local_file)\n os.rmdir(temp_dir)\n" - integration: imaging_data_commons + integration: 43858948-4b58-4312-b281-7a65e9c0d38d notes: null query: Download and visualize a slide microscopy image with pydicom and matplotlib resource_id: c19b78ab-29cb-4be3-8069-1ef22302dfb2 @@ -265,7 +265,7 @@ resources: \ size_mb = df['series_size_MB'].iloc[0]\n print(f\"\\nFile size:\ \ {size_mb:.2f} MB\")\n print(f\"Download URL: {url}\")\nexcept Exception\ \ as e:\n print(f\"An error occurred: {str(e)}\")\n" - integration: imaging_data_commons + integration: 43858948-4b58-4312-b281-7a65e9c0d38d notes: null query: Query to get download URLs for bone marrow slides resource_id: dc0ce97c-46a2-406b-a582-a51be49367c9 @@ -285,7 +285,7 @@ resources: \ to a CSV file for further analysis\n df.to_csv('aml_pathology_data.csv',\ \ index=False)\n print(\"\\nData has been saved to 'aml_pathology_data.csv'\"\ )\n\nexcept Exception as e:\n print(f\"An error occurred: {str(e)}\") \n" - integration: imaging_data_commons + integration: 43858948-4b58-4312-b281-7a65e9c0d38d notes: null query: Get slide microscopy data for a specific collection resource_id: e0279959-d257-438e-8513-fb40391082c1 @@ -339,9 +339,9 @@ resources: except Exception as e: print(f"\nError getting bucket information: {str(e)}") - integration: imaging_data_commons + integration: 43858948-4b58-4312-b281-7a65e9c0d38d notes: null query: Get list of files in an S3 bucket for a specific collection on IDC resource_id: f5ffd878-c1c9-46e9-b83c-6fd54fe975a5 resource_type: example -slug: imaging_data_commons +slug: 43858948-4b58-4312-b281-7a65e9c0d38d diff --git a/src/biome/adhoc_data/datasources/indra/api.yaml b/src/biome/adhoc_data/datasources/indra/api.yaml index c03c8e9..0c14457 100644 --- a/src/biome/adhoc_data/datasources/indra/api.yaml +++ b/src/biome/adhoc_data/datasources/indra/api.yaml @@ -75,7 +75,7 @@ resources: print("\nShared Pathways Response:") print(shared_pathways_response.json()) - integration: indra_context_graph_extension_(cogex) + integration: c3befe4c-07ef-445b-8bfc-32debea298bf notes: null query: |- Perform a gene pathway analysis for JAK2 to find associated pathways and shared pathways with other genes like STAT3, JAK3, and EGFR. @@ -136,7 +136,7 @@ resources: for data in evidence_data.values(): all_pmids.update(data['pmids']) print(f"Total unique PMIDs: {len(all_pmids)}") - integration: indra_context_graph_extension_(cogex) + integration: c3befe4c-07ef-445b-8bfc-32debea298bf notes: null query: Retrieve literature evidence related to AML using MeSH term queries, including child terms, and summarize the findings. @@ -218,7 +218,7 @@ resources: print(f"Is kinase: {results['is_kinase']}") print(f"Is phosphatase: {results['is_phosphatase']}") print(f"Is transcription factor: {results['is_transcription_factor']}") - integration: indra_context_graph_extension_(cogex) + integration: c3befe4c-07ef-445b-8bfc-32debea298bf notes: null query: |- This example demonstrates how to analyze the JAK2 gene using the INDRA CoGEx API to retrieve GO terms, pathway associations, and check if it is a kinase, phosphatase, or transcription factor. @@ -226,8 +226,8 @@ resources: resource_type: example f49c8aa7-727e-444e-a3a2-314a5f0d6631: filepath: indra.json - integration: indra_context_graph_extension_(cogex) + integration: c3befe4c-07ef-445b-8bfc-32debea298bf name: raw_documentation resource_id: f49c8aa7-727e-444e-a3a2-314a5f0d6631 resource_type: file -slug: indra_context_graph_extension_(cogex) +slug: c3befe4c-07ef-445b-8bfc-32debea298bf diff --git a/src/biome/adhoc_data/datasources/netrias/api.yaml b/src/biome/adhoc_data/datasources/netrias/api.yaml index 1f03f52..f93ab8f 100644 --- a/src/biome/adhoc_data/datasources/netrias/api.yaml +++ b/src/biome/adhoc_data/datasources/netrias/api.yaml @@ -59,7 +59,7 @@ resources: \ for '{drug_term}'\")\n else:\n print(\"Could not find therapeutic_agents\ \ in DataHub standards\")\nelse:\n print(f\"Error getting standards: {standards_response.status_code}\"\ )\n print(standards_response.text)\n" - integration: netrias_harmonization_api + integration: 495ac407-7042-4eda-95ca-e10850e00081 notes: |- This example demonstrates how to harmonize a drug term that includes a non-standard suffix (e.g., "zorubicin_9012") by: 1. Retrieving the therapeutic_agents standard from the DataHub @@ -72,8 +72,8 @@ resources: resource_type: example 8242a126-8ef1-48de-aa94-e0ccc7e58a28: filepath: netrias.json - integration: netrias_harmonization_api + integration: 495ac407-7042-4eda-95ca-e10850e00081 name: raw_documentation resource_id: 8242a126-8ef1-48de-aa94-e0ccc7e58a28 resource_type: file -slug: netrias_harmonization_api +slug: 495ac407-7042-4eda-95ca-e10850e00081 diff --git a/src/biome/adhoc_data/datasources/nhanes_dietary/api.yaml b/src/biome/adhoc_data/datasources/nhanes_dietary/api.yaml index 2b1f870..5e6d2b7 100644 --- a/src/biome/adhoc_data/datasources/nhanes_dietary/api.yaml +++ b/src/biome/adhoc_data/datasources/nhanes_dietary/api.yaml @@ -63,8 +63,8 @@ prompt: | resources: 19e60e0b-82e2-4af2-b6f2-212287b43277: filepath: docs.md - integration: nhanes_dietary_data + integration: a37bcc79-b195-4ade-bd08-2d44d558facd name: web_docs resource_id: 19e60e0b-82e2-4af2-b6f2-212287b43277 resource_type: file -slug: nhanes_dietary_data +slug: a37bcc79-b195-4ade-bd08-2d44d558facd diff --git a/src/biome/adhoc_data/datasources/nsch/api.yaml b/src/biome/adhoc_data/datasources/nsch/api.yaml index ace4c8e..21a51e2 100644 --- a/src/biome/adhoc_data/datasources/nsch/api.yaml +++ b/src/biome/adhoc_data/datasources/nsch/api.yaml @@ -128,7 +128,7 @@ resources: \ # Assuming 1 = Yes\n 'Percentage': percentages.get(1, 0)\n })\n\n\ # Create results dataframe\nresults_df = pd.DataFrame(results)\nresults_df =\ \ results_df.sort_values('Percentage', ascending=False)\n\nprint(results_df)\n" - integration: national_survey_of_children's_health_(nsch) + integration: f7052ea8-d8f1-4203-b368-60fc23f02846 notes: null query: Retrieve health conditions reported in the National Survey of Children's Health, including ADHD, autism, and asthma. @@ -140,7 +140,7 @@ resources: \ is shifted!\ncodebook = pd.read_csv(\"{{DATASET_FILES_BASE_PATH}}/census-nsch/nsch_dictionary_codebook.csv\"\ , index_col=False) \n# Display the first few rows\nprint(codebook.head())\n\ \n# and columns if needed\nprint(codebook.columns.tolist())\n" - integration: national_survey_of_children's_health_(nsch) + integration: f7052ea8-d8f1-4203-b368-60fc23f02846 notes: null query: Check the codebook columns head and available columns resource_id: 4cdaa736-1ca8-4158-8d3f-60de680da7f3 @@ -153,7 +153,7 @@ resources: existing_cols = df_2023.columns.tolist() print(existing_cols) - integration: national_survey_of_children's_health_(nsch) + integration: f7052ea8-d8f1-4203-b368-60fc23f02846 notes: null query: Check the existing columns in the dataset to see if the screentime variable and which geo columns are included. @@ -278,10 +278,10 @@ resources: \ 'decreased'\nprint(f\"4. Texas Trends: The childhood asthma rate in Texas\ \ {texas_trend} from {texas_data.iloc[0]['has_asthma']:.1%} in 2016 to {texas_data.iloc[-1]['has_asthma']:.1%}\ \ in 2023.\")\n" - integration: national_survey_of_children's_health_(nsch) + integration: f7052ea8-d8f1-4203-b368-60fc23f02846 notes: null query: |- Analyze trends in childhood asthma and environmental factors over time (2016-2023) using NSCH data. Load multiple years of data, harmonize variables across years, and analyze the relationship between asthma prevalence and environmental factors like smoke exposure, housing insecurity, and food insecurity. Include state-level comparisons and create a composite environmental risk score. Do so specifically for Texas. resource_id: e2681d9f-b1b5-441b-9deb-bd203d9de6f4 resource_type: example -slug: national_survey_of_children's_health_(nsch) +slug: f7052ea8-d8f1-4203-b368-60fc23f02846 diff --git a/src/biome/adhoc_data/datasources/pdc/api.yaml b/src/biome/adhoc_data/datasources/pdc/api.yaml index 641dd00..65b0977 100644 --- a/src/biome/adhoc_data/datasources/pdc/api.yaml +++ b/src/biome/adhoc_data/datasources/pdc/api.yaml @@ -74,7 +74,7 @@ resources: print(json.dumps(file_info, indent=2)) else: print(f"Error: {response.status_code} - {response.text}") - integration: proteomics_data_commons + integration: bcaf77af-17e1-4811-9520-64d48e198988 notes: null query: |- Fetch raw mass spec data for a specific study using study name, data category, and file type in the Proteomics Data Commons (PDC). @@ -128,7 +128,7 @@ resources: print(json.dumps(case_info, indent=2)) else: print(f"Error: {response.status_code} - {response.text}") - integration: proteomics_data_commons + integration: bcaf77af-17e1-4811-9520-64d48e198988 notes: null query: Fetch detailed information about a case using its case ID in the Proteomics Data Commons (PDC). @@ -172,7 +172,7 @@ resources: print(json.dumps(files, indent=2)) else: print(f"Error: {response.status_code} - {response.text}") - integration: proteomics_data_commons + integration: bcaf77af-17e1-4811-9520-64d48e198988 notes: null query: |- This example demonstrates how to query the Proteomics Data Commons for studies with open standard data for processed mass spec using the getPaginatedUIFile query. @@ -236,7 +236,7 @@ resources: print(f"Error decoding JSON response: {e}") print("Response content:", response_stat5a.text) print("Response content:", response_stat5b.text) - integration: proteomics_data_commons + integration: bcaf77af-17e1-4811-9520-64d48e198988 notes: null query: |- Retrieve quantitative mass spectrometry data for STAT5A and STAT5B proteins using the getPaginatedUIGeneAliquotSpectralCount query. @@ -244,7 +244,7 @@ resources: resource_type: example 75e500f5-31c0-4683-995b-26fc49d51426: filepath: pdc_schema.graphql - integration: proteomics_data_commons + integration: bcaf77af-17e1-4811-9520-64d48e198988 name: raw_documentation resource_id: 75e500f5-31c0-4683-995b-26fc49d51426 resource_type: file @@ -325,7 +325,7 @@ resources: except json.JSONDecodeError as e: print(f"Error decoding JSON response: {e}") print("Response content:", response.text) - integration: proteomics_data_commons + integration: bcaf77af-17e1-4811-9520-64d48e198988 notes: null query: Load samples into a pandas dataframe resource_id: 8708cae9-d62e-468d-9d78-3f61ac1913d6 @@ -402,7 +402,7 @@ resources: except json.JSONDecodeError as e: print(f"Error decoding JSON response: {e}") print("Response content:", response.text) - integration: proteomics_data_commons + integration: bcaf77af-17e1-4811-9520-64d48e198988 notes: null query: Fetch all available metadata for a study using its PDC Study ID. resource_id: 893e05e7-c0db-4482-aebe-1b09c9abfd4d @@ -472,7 +472,7 @@ resources: # Print the number of studies found and the first 5 studies print(f"Found {len(lung_cancer_studies)} studies with primary site 'Lung' or 'Bronchus and lung'.") print(list(lung_cancer_studies)[:5]) - integration: proteomics_data_commons + integration: bcaf77af-17e1-4811-9520-64d48e198988 notes: null query: |- This example demonstrates how to find lung cancer studies in the Proteomics Data Commons by querying for studies with primary sites 'Lung' and 'Bronchus and lung', and combining the results. @@ -514,7 +514,7 @@ resources: except requests.exceptions.RequestException as e: print(f"An error occurred: {e}") - integration: proteomics_data_commons + integration: bcaf77af-17e1-4811-9520-64d48e198988 notes: null query: Query PDC to find studies with a specific disease type (in this case 'Acute Myeloid Leukemia') @@ -586,7 +586,7 @@ resources: # Display the first few studies for study in list(lung_cancer_studies)[:5]: print(study) - integration: proteomics_data_commons + integration: bcaf77af-17e1-4811-9520-64d48e198988 notes: null query: |- Query the Proteomics Data Commons (PDC) to find studies with primary site 'Lung' or 'Bronchus and lung' using separate queries and combining results. @@ -630,7 +630,7 @@ resources: print(json.dumps(file_info, indent=2)) else: print(f"Error: {response.status_code} - {response.text}") - integration: proteomics_data_commons + integration: bcaf77af-17e1-4811-9520-64d48e198988 notes: null query: Retrieve open standard data files for a specific study using the Proteomics Data Commons (PDC) API. @@ -706,7 +706,7 @@ resources: except json.JSONDecodeError as e: print(f"Error decoding JSON response: {e}") print("Response content:", response.text) - integration: proteomics_data_commons + integration: bcaf77af-17e1-4811-9520-64d48e198988 notes: null query: obtain quantitative data resource_id: c68f970e-4d39-4979-bcf2-43a823490773 @@ -744,7 +744,7 @@ resources: print(json.dumps(cases_info, indent=2)) else: print(f"Error: {response.status_code} - {response.text}") - integration: proteomics_data_commons + integration: bcaf77af-17e1-4811-9520-64d48e198988 notes: null query: Fetch all cases based on a study name resource_id: d119d4cc-347e-40bc-b3d3-f17f6e1ee16e @@ -790,7 +790,7 @@ resources: except json.JSONDecodeError as e: print(f"Error decoding JSON response: {e}") print("Response content:", response.text) - integration: proteomics_data_commons + integration: bcaf77af-17e1-4811-9520-64d48e198988 notes: null query: obtain quantitative mass spec data on a specific protein (STAT5) resource_id: d2fde53a-0ba6-4c00-acaf-e27d4218480b @@ -832,7 +832,7 @@ resources: print(f"Signed URL for the file: {signed_url}") else: print(f"Error: {response.status_code} - {response.text}") - integration: proteomics_data_commons + integration: bcaf77af-17e1-4811-9520-64d48e198988 notes: null query: |- Retrieve a signed URL for downloading a specific file (e.g. raw mass spec file) in a study using the Proteomics Data Commons (PDC) API. @@ -868,10 +868,10 @@ resources: r')\n pdc_cases_in_gdc.append(case)\n\nprint(f\"Found {len(pdc_cases_in_gdc)}\ \ cases that have external references to GDC.\")\n\npdc_in_gdc_df = pd.DataFrame(json_normalize(pdc_cases_in_gdc))\n\ pdc_in_gdc_df.head()" - integration: proteomics_data_commons + integration: bcaf77af-17e1-4811-9520-64d48e198988 notes: null query: |- Get all clinical data (cases) from PDC and filter for those that have external references to GDC. Return the results as a pandas dataframe. resource_id: ed1fa5df-2d51-48ef-871b-b09abc5defc9 resource_type: example -slug: proteomics_data_commons +slug: bcaf77af-17e1-4811-9520-64d48e198988 diff --git a/src/biome/adhoc_data/datasources/synapse/api.yaml b/src/biome/adhoc_data/datasources/synapse/api.yaml index 022d5ee..2d55848 100644 --- a/src/biome/adhoc_data/datasources/synapse/api.yaml +++ b/src/biome/adhoc_data/datasources/synapse/api.yaml @@ -80,13 +80,13 @@ prompt: | resources: 45324841-c013-4942-8b5f-db85856d7e48: filepath: web_docs.md - integration: nf_synapse + integration: c6b31ee9-05d7-48fe-a9b3-6840df604e9c name: web_docs resource_id: 45324841-c013-4942-8b5f-db85856d7e48 resource_type: file 8d29393b-bc38-441f-aee1-88b60d1322b6: filepath: SynapseOpenApiSpec.json - integration: nf_synapse + integration: c6b31ee9-05d7-48fe-a9b3-6840df604e9c name: raw_documentation resource_id: 8d29393b-bc38-441f-aee1-88b60d1322b6 resource_type: file @@ -95,15 +95,15 @@ resources: \ = Path(__file__).parent.parent / \"downloads\" / \"synapse\"\n\nsyn = synapseclient.Synapse(cache_root_dir=downloads_dir)\ \ \nsyn.login(authToken=os.environ.get(\"API_SYNAPSE\")) # pass in the authToken\ \ on python client login!\n```" - integration: nf_synapse + integration: c6b31ee9-05d7-48fe-a9b3-6840df604e9c notes: null query: Login to the Synapse API using the python client resource_id: 920140d9-ce8a-4cef-ad8c-418146706556 resource_type: example b9801ff2-37bb-44d0-9f47-9ce3d18804b5: filepath: nf_programmatic_webdocs.md - integration: nf_synapse + integration: c6b31ee9-05d7-48fe-a9b3-6840df604e9c name: python_client resource_id: b9801ff2-37bb-44d0-9f47-9ce3d18804b5 resource_type: file -slug: nf_synapse +slug: c6b31ee9-05d7-48fe-a9b3-6840df604e9c diff --git a/src/biome/adhoc_data/datasources/usda_foodcentral/api.yaml b/src/biome/adhoc_data/datasources/usda_foodcentral/api.yaml index 0e3bbee..3b0a732 100644 --- a/src/biome/adhoc_data/datasources/usda_foodcentral/api.yaml +++ b/src/biome/adhoc_data/datasources/usda_foodcentral/api.yaml @@ -74,7 +74,7 @@ prompt: | resources: 0e410001-bac7-4cb7-9f32-b9064c64a4c1: filepath: foodcentral_openapi.json - integration: usda_fooddata_central_api + integration: a77cc48a-579a-4145-b1bd-80ad50d482ec name: raw_documentation resource_id: 0e410001-bac7-4cb7-9f32-b9064c64a4c1 resource_type: file @@ -99,9 +99,9 @@ resources: # Extract the first food item from the results food_item = data['foods'][0] food_item - integration: usda_fooddata_central_api + integration: a77cc48a-579a-4145-b1bd-80ad50d482ec notes: null query: Search for nutrients for raw acerola juice resource_id: 4164944f-a6d4-452c-a290-3ab7bd2e9d9e resource_type: example -slug: usda_fooddata_central_api +slug: a77cc48a-579a-4145-b1bd-80ad50d482ec diff --git a/src/biome/adhoc_data/datasources/usgs_pesticide/api.yaml b/src/biome/adhoc_data/datasources/usgs_pesticide/api.yaml index 2c1478f..1a09213 100644 --- a/src/biome/adhoc_data/datasources/usgs_pesticide/api.yaml +++ b/src/biome/adhoc_data/datasources/usgs_pesticide/api.yaml @@ -42,4 +42,4 @@ prompt: | You may use the python `us` library to convert FIPS codes to state and county names, and use pandas to read the tab-delimited text files. resources: {} -slug: usgs_pesticide_national_synthesis_project +slug: 19c8907f-cd98-4a3e-8d72-fe466b20087e diff --git a/src/biome/adhoc_data/datasources/waterservices_usgs/api.yaml b/src/biome/adhoc_data/datasources/waterservices_usgs/api.yaml index ffa43a9..14eb51d 100644 --- a/src/biome/adhoc_data/datasources/waterservices_usgs/api.yaml +++ b/src/biome/adhoc_data/datasources/waterservices_usgs/api.yaml @@ -15,7 +15,7 @@ prompt: | resources: 4c84149a-8c62-40c1-aae9-9bae79c4c2fc: filepath: usgs_waterservices_web.md - integration: usgs_water_services_api + integration: c0deb2c9-90bf-476c-9ed0-c5f6070ebee8 name: web_documentation resource_id: 4c84149a-8c62-40c1-aae9-9bae79c4c2fc resource_type: file @@ -34,15 +34,15 @@ resources: \ 'endDt': end_date.strftime('%Y-%m-%d'),\n 'parameterCd': ','.join(params),\n\ \ 'siteStatus': 'active'\n }\n\n # Make request\n response =\ \ requests.get(base_url, params=params)" - integration: usgs_water_services_api + integration: c0deb2c9-90bf-476c-9ed0-c5f6070ebee8 notes: null query: Get water quality data for Seminole County, Florida resource_id: 57592feb-0f49-4aea-8d78-570b3f674123 resource_type: example ebee2534-6b90-4b45-8f05-03801d2e7a85: filepath: site_types.md - integration: usgs_water_services_api + integration: c0deb2c9-90bf-476c-9ed0-c5f6070ebee8 name: site_types resource_id: ebee2534-6b90-4b45-8f05-03801d2e7a85 resource_type: file -slug: usgs_water_services_api +slug: c0deb2c9-90bf-476c-9ed0-c5f6070ebee8 diff --git a/src/biome/context.py b/src/biome/context.py index 3b0f048..0bba8cf 100644 --- a/src/biome/context.py +++ b/src/biome/context.py @@ -29,14 +29,14 @@ class TestIntegrationProvider(BaseIntegrationProvider): def __init__(self): self.integrations = [ Integration( - "test_1", - "Test Integration 1", - "First Test Integration" + name="Test Integration 1", + description="First Test Integration", + provider="test" ), Integration( - "test_2", - "Test Integration 2", - "Second Test Integration" + name="Test Integration 2", + description="Second Test Integration", + provider="test" ), ] super().__init__(display_name="Biome Second Test Integration") From 4006a6591f3ab888230b925d0d5bf22d943155bf Mon Sep 17 00:00:00 2001 From: Satchel Baldwin Date: Wed, 9 Jul 2025 17:25:55 -0500 Subject: [PATCH 17/21] name integratiosn to specialist agents to match frontend editor in beaker --- src/biome/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/biome/context.py b/src/biome/context.py index 0bba8cf..8716c3d 100644 --- a/src/biome/context.py +++ b/src/biome/context.py @@ -68,7 +68,7 @@ def __init__(self, beaker_kernel: "LLMKernel", config: Dict[str, Any]): curator_config=curator_config, contextualizer_config=gpt_41_config, logger=logger, - display_name="Biome Specialist Agents" + display_name="Specialist Agents" ) test_integration = TestIntegrationProvider() super().__init__( From 42df4403f58faf7d3decba7686fd25e9aecc8ccf Mon Sep 17 00:00:00 2001 From: Satchel Baldwin Date: Mon, 14 Jul 2025 16:56:14 -0500 Subject: [PATCH 18/21] fix: new yaml format --- src/biome/adhoc_data/datasources/ahs/api.yaml | 241 ++++++------ src/biome/adhoc_data/datasources/aqs/api.yaml | 357 +++++++++--------- .../datasources/cbioportal/api.yaml | 37 +- src/biome/adhoc_data/datasources/cda/api.yaml | 25 +- .../datasources/cdc_tracking_network/api.yaml | 37 +- .../datasources/census_acs/api.yaml | 53 +-- .../chis_california_asthma/api.yaml | 43 ++- .../adhoc_data/datasources/epa_tri/api.yaml | 207 +++++----- .../adhoc_data/datasources/faers/api.yaml | 37 +- src/biome/adhoc_data/datasources/gdc/api.yaml | 111 +++--- .../adhoc_data/datasources/gras/api.yaml | 67 ++-- src/biome/adhoc_data/datasources/hpa/api.yaml | 19 +- src/biome/adhoc_data/datasources/idc/api.yaml | 57 +-- .../adhoc_data/datasources/indra/api.yaml | 61 +-- .../adhoc_data/datasources/netrias/api.yaml | 57 +-- .../datasources/nhanes_dietary/api.yaml | 25 +- .../adhoc_data/datasources/nsch/api.yaml | 227 +++++------ src/biome/adhoc_data/datasources/pdc/api.yaml | 57 +-- .../adhoc_data/datasources/synapse/api.yaml | 69 ++-- .../datasources/usda_foodcentral/api.yaml | 77 ++-- .../datasources/usgs_pesticide/api.yaml | 13 +- .../datasources/waterservices_usgs/api.yaml | 31 +- 22 files changed, 1009 insertions(+), 899 deletions(-) diff --git a/src/biome/adhoc_data/datasources/ahs/api.yaml b/src/biome/adhoc_data/datasources/ahs/api.yaml index 753f283..cf591d1 100644 --- a/src/biome/adhoc_data/datasources/ahs/api.yaml +++ b/src/biome/adhoc_data/datasources/ahs/api.yaml @@ -1,122 +1,17 @@ +datatype: dataset description: | You have access to knowledge to all the Census Housing survey data, every 2 years from the year 1999 to 2023. You can use this API for answering questions on american housing, specifically their housing conditions (mold, water problems, etc), which can be used to correlate to health conditions, or other purposes. -integration_type: dataset +img_url: null +last_updated: null name: Census American Housing Survey (AHS) -prompt: | - You have access to a file-based Census American Housing Survey - (National Public Use Files CSV) from every 2 years from 1999 to 2023, under the `${DATASET_FILES_BASE_PATH}/census-ahs` directory. - The housing files are located and named per year as follows: - - ${DATASET_FILES_BASE_PATH}/census-ahs/survey_2023.csv - - ${DATASET_FILES_BASE_PATH}/census-ahs/survey_2021.csv - - ... - - ${DATASET_FILES_BASE_PATH}/census-ahs/survey_1999.csv - the more recent onces are at the metropolitan area level, while older ones - are only avaiable at the national level. Use whichever year files you need, depending - on the user question. - - {more_instructions} - - In particular, we have the household data file, which means we - have the OMB13CBSA as a constantly availablecolumn to identify geographic area of survey( - State-based Metropolitan and Micropolitan Statistical Areas). Pre-2015 have more geographical - column/information. Don't try to use state/county/etc if those are not present on - the census housing file for that year. - - Try to alreay have planned which columns you need for each, in order to process less columns/excess data since - merging both datasets will result in more than 1000 columns. Plan it out, select relevant columns - (list the column names once filtered to knkow which are available), - then merge the datasets with those columns selected. - - Once you identify columns of interest available, you can use this csv codebook dictionary that explains what each column code means: - - - ${DATASET_FILES_BASE_PATH}/census-ahs/census_ahs_codebook.csv - - from the codebook,you can use the "Variable" column to match the variable you have access to (say "OMB13CBSA") and use the column "Response Codes" to get - the meaning of the codes or the numeric code range values, for other types of variables. - - you may open it and use it at your discretion- usually better once you have some columns of interest - in order to select a subset of the codebook, since it is pretty big and confusing. - - Don't try to find the codebook columns without checking if they are present, since - the codebook is very exhaustive and contains all the columns or other types of files. - - You can also use knowledge on the usual column codenames from the census to answer questions, but ensure - to check the columns names with pandas (python lib) and _always_ check if the column - is present in the dataset itself instead of assuming it's present. - - Instead of assuming a specific column exist, or if you wish to match with a specific column pattern, - try to compare substrings instead. For example, if you need data for mold, try - to check for the 'mold' substring in the column names, since these csv files sometimes append - soe prefixes. - - The dataset csv files are a bit big, in the sense that they're in - wide format and contain too many columns. When planning what you need, select a subset - of the columns needed, including the geo/housing-characteristics columns for your analysis, - but don't include the thousands of columns. - - # Overview of the American Housing Survey (AHS) - The American Housing Survey (AHS) is a comprehensive national housing survey conducted by the U.S. Census Bureau. It provides detailed information about housing conditions, household characteristics, and neighborhood features across the United States. - - ## Key Dataset Characteristics - - ### Temporal Coverage - - Biennial surveys from 1999 to 2023 - - Data is collected every two years, allowing for trend analysis - - Most recent survey available is from 2023 - - ### Geospatial Resolution - - Metropolitan area level using Core-Based Statistical Areas (CBSA codes) - - Recent surveys (post-2015) use 2013 OMB CBSA codes - - Older surveys (pre-2015) use different geographic identifiers (SMSA codes) - - County-level data is limited, especially in recent surveys - - Harris County/Houston Data: Available in older surveys (1999-2007) using SMSA code 3360, but not found in more recent surveys (2015-2023) - - ### Housing Condition Variables - - Mold: Presence of mold in different areas of the home (basement, bathroom, bedroom, kitchen, living room) - - Water Leaks: Inside and outside water leaks, including specific sources (roof, plumbing, basement) - - Pests: Presence of rodents (rats, mice) and insects - - Air Quality: Indoor air quality ratings - - HVAC Systems: Heating and cooling system types and adequacy - - ### Health-Related Variables - - Asthma: Recent surveys (2015, 2023) include questions about household members diagnosed with asthma - - Emergency Room Visits: Data on ER visits due to asthma - - Asthma Medication: Some surveys include information about asthma medication use - - ### Household Composition - - Number of persons in household - - Presence of children (various age groups) - - Number of adults and elders - - ### Limitations and Challenges - #### Geographic Specificity: - - Recent surveys (2015-2023) do not contain Houston/Harris County specific data - - Only metropolitan area level data is available in recent surveys - - #### Data Consistency: - - Variable definitions and coding schemes change across survey years - - Some variables are only available in certain years - - #### Direct Causality: - - The survey provides correlational data but cannot establish causal relationships between housing conditions and health outcomes - - #### Sample Size: - - When filtering for specific conditions, sample sizes can become small, limiting statistical power - - # Additional Instructions: - - Read the CSV fiels with index_col=False, like so: - ``` - codebook = pd.read_csv('${DATASET_FILES_BASE_PATH}/census-ahs/census_ahs_codebook.csv', dtype=str, index_col=False) - ``` - If you don't add the index_col=False, the first column values will disappear and the whole dataframe will be shifted! +provider: adhoc:specialist_agents resources: 3a5b7607-80c5-44ae-a240-a9a388f78afd: filepath: docs.md - integration: d73e06c5-e41b-4f7c-8a11-0fe1b4e6dfeb + integration: beb85142-d32f-42df-a438-b84829ddd4fd name: more_instructions resource_id: 3a5b7607-80c5-44ae-a240-a9a388f78afd resource_type: file @@ -138,7 +33,7 @@ resources: code_pairs_dict = {k.strip(): v.strip() for k, v in (code_pair.split(":") for code_pair in code_pairs)} return code_pairs_dict.get(code) - integration: d73e06c5-e41b-4f7c-8a11-0fe1b4e6dfeb + integration: beb85142-d32f-42df-a438-b84829ddd4fd notes: null query: |- Use the codebook to get the meaning of the codes for the OMB13CBSA variable, matching one code to the meaning by creating a dictionary of code:meaning pairs and returning the statistical metro area name for a given code @@ -182,7 +77,7 @@ resources: Households without children: {asthma_by_children.get(0, 'N/A'):.1f}%\")\nprint(f\"\ Difference: {asthma_by_children.get(1, 0) - asthma_by_children.get(0, 0):.1f}\ \ percentage points\")\n" - integration: d73e06c5-e41b-4f7c-8a11-0fe1b4e6dfeb + integration: beb85142-d32f-42df-a438-b84829ddd4fd notes: null query: |- Analyzing the relationship between housing conditions, asthma, and the presence of children using the 2023 American Housing Survey data. This code calculates and compares the prevalence of various housing conditions (mold, water leaks) in homes with and without reported asthma cases, and also examines the prevalence of asthma in households with and without children. @@ -228,7 +123,7 @@ resources: )\n print(f\" - Households with 1-2 persons: {without_children:.1f}%\"\ )\n print(f\" - Difference: {with_children - without_children:.1f}\ \ percentage points\")\n" - integration: d73e06c5-e41b-4f7c-8a11-0fe1b4e6dfeb + integration: beb85142-d32f-42df-a438-b84829ddd4fd notes: null query: |- Analyzing housing conditions in Houston, TX using the 2007 American Housing Survey data. This code identifies Houston records using the SMSA code, analyzes the prevalence of water leaks and pest problems, and compares these conditions between larger households (3+ persons, likely to have children) and smaller households (1-2 persons). @@ -247,7 +142,7 @@ resources: # Return the Response Codes column ombs_row['Response Codes'] - integration: d73e06c5-e41b-4f7c-8a11-0fe1b4e6dfeb + integration: beb85142-d32f-42df-a438-b84829ddd4fd notes: null query: What is the meaning of the codes for the OMB13CBSA variable? resource_id: ad42a806-0e81-4e63-97cf-2f7aad6a1a35 @@ -301,7 +196,7 @@ resources: \ condition|hvac', case=False, na=False) |\n asthma_vars_df['Question_Text'].str.contains('heat|cool|air\ \ condition|hvac', case=False, na=False)\n]\n\nprint(f\"\\nFound {len(hvac_vars)}\ \ variables related to heating and cooling systems\")" - integration: d73e06c5-e41b-4f7c-8a11-0fe1b4e6dfeb + integration: beb85142-d32f-42df-a438-b84829ddd4fd notes: null query: |- Exploring the American Housing Survey codebook to identify variables related to asthma, respiratory health, and housing conditions. This code searches the codebook for terms related to asthma, air quality, pests, and HVAC systems, and provides a summary of available variables that could be relevant for analyzing the relationship between housing conditions and respiratory health. @@ -375,7 +270,7 @@ resources: \ by Presence of Children:\")\nprint(summary_table[['Condition', 'With Children\ \ (%)', 'Without Children (%)', 'Difference (percentage points)', 'Sample Size']].to_string(index=False,\ \ float_format=lambda x: f\"{x:.1f}\"))\n" - integration: d73e06c5-e41b-4f7c-8a11-0fe1b4e6dfeb + integration: beb85142-d32f-42df-a438-b84829ddd4fd notes: null query: |- Analyzing and visualizing the relationship between housing conditions (mold, water leaks) and the presence of children in households using the 2023 American Housing Survey data. This code creates a horizontal bar chart comparing the prevalence of various housing conditions in homes with and without children, and generates a summary table of the differences. @@ -432,10 +327,120 @@ resources: \ as \"'6'\",\n # that is why we strip them or sometimes compare\ \ to substrings\n combined_data[col] = combined_data[col].apply(convert_value)\n\ \ \n return combined_data\n else:\n return pd.DataFrame()\n" - integration: d73e06c5-e41b-4f7c-8a11-0fe1b4e6dfeb + integration: beb85142-d32f-42df-a438-b84829ddd4fd notes: null query: |- I wonder if there are any trends for metro areas in Florida, US in which mold presence increased between years 2001 and 2011? Can you get the data for matching mold presence? resource_id: ea7d2aec-2d76-4b95-ae85-177db08ff019 resource_type: example -slug: d73e06c5-e41b-4f7c-8a11-0fe1b4e6dfeb +slug: adhocspecificationintegration_census_american_housing_survey__ahs_ +source: | + You have access to a file-based Census American Housing Survey + (National Public Use Files CSV) from every 2 years from 1999 to 2023, under the `${DATASET_FILES_BASE_PATH}/census-ahs` directory. + The housing files are located and named per year as follows: + - ${DATASET_FILES_BASE_PATH}/census-ahs/survey_2023.csv + - ${DATASET_FILES_BASE_PATH}/census-ahs/survey_2021.csv + - ... + - ${DATASET_FILES_BASE_PATH}/census-ahs/survey_1999.csv + the more recent onces are at the metropolitan area level, while older ones + are only avaiable at the national level. Use whichever year files you need, depending + on the user question. + + {more_instructions} + + In particular, we have the household data file, which means we + have the OMB13CBSA as a constantly availablecolumn to identify geographic area of survey( + State-based Metropolitan and Micropolitan Statistical Areas). Pre-2015 have more geographical + column/information. Don't try to use state/county/etc if those are not present on + the census housing file for that year. + + Try to alreay have planned which columns you need for each, in order to process less columns/excess data since + merging both datasets will result in more than 1000 columns. Plan it out, select relevant columns + (list the column names once filtered to knkow which are available), + then merge the datasets with those columns selected. + + Once you identify columns of interest available, you can use this csv codebook dictionary that explains what each column code means: + + - ${DATASET_FILES_BASE_PATH}/census-ahs/census_ahs_codebook.csv + + from the codebook,you can use the "Variable" column to match the variable you have access to (say "OMB13CBSA") and use the column "Response Codes" to get + the meaning of the codes or the numeric code range values, for other types of variables. + + you may open it and use it at your discretion- usually better once you have some columns of interest + in order to select a subset of the codebook, since it is pretty big and confusing. + + Don't try to find the codebook columns without checking if they are present, since + the codebook is very exhaustive and contains all the columns or other types of files. + + You can also use knowledge on the usual column codenames from the census to answer questions, but ensure + to check the columns names with pandas (python lib) and _always_ check if the column + is present in the dataset itself instead of assuming it's present. + + Instead of assuming a specific column exist, or if you wish to match with a specific column pattern, + try to compare substrings instead. For example, if you need data for mold, try + to check for the 'mold' substring in the column names, since these csv files sometimes append + soe prefixes. + + The dataset csv files are a bit big, in the sense that they're in + wide format and contain too many columns. When planning what you need, select a subset + of the columns needed, including the geo/housing-characteristics columns for your analysis, + but don't include the thousands of columns. + + # Overview of the American Housing Survey (AHS) + The American Housing Survey (AHS) is a comprehensive national housing survey conducted by the U.S. Census Bureau. It provides detailed information about housing conditions, household characteristics, and neighborhood features across the United States. + + ## Key Dataset Characteristics + + ### Temporal Coverage + - Biennial surveys from 1999 to 2023 + - Data is collected every two years, allowing for trend analysis + - Most recent survey available is from 2023 + + ### Geospatial Resolution + - Metropolitan area level using Core-Based Statistical Areas (CBSA codes) + - Recent surveys (post-2015) use 2013 OMB CBSA codes + - Older surveys (pre-2015) use different geographic identifiers (SMSA codes) + - County-level data is limited, especially in recent surveys + - Harris County/Houston Data: Available in older surveys (1999-2007) using SMSA code 3360, but not found in more recent surveys (2015-2023) + + ### Housing Condition Variables + - Mold: Presence of mold in different areas of the home (basement, bathroom, bedroom, kitchen, living room) + - Water Leaks: Inside and outside water leaks, including specific sources (roof, plumbing, basement) + - Pests: Presence of rodents (rats, mice) and insects + - Air Quality: Indoor air quality ratings + - HVAC Systems: Heating and cooling system types and adequacy + + ### Health-Related Variables + - Asthma: Recent surveys (2015, 2023) include questions about household members diagnosed with asthma + - Emergency Room Visits: Data on ER visits due to asthma + - Asthma Medication: Some surveys include information about asthma medication use + + ### Household Composition + - Number of persons in household + - Presence of children (various age groups) + - Number of adults and elders + + ### Limitations and Challenges + #### Geographic Specificity: + - Recent surveys (2015-2023) do not contain Houston/Harris County specific data + - Only metropolitan area level data is available in recent surveys + + #### Data Consistency: + - Variable definitions and coding schemes change across survey years + - Some variables are only available in certain years + + #### Direct Causality: + - The survey provides correlational data but cannot establish causal relationships between housing conditions and health outcomes + + #### Sample Size: + - When filtering for specific conditions, sample sizes can become small, limiting statistical power + + # Additional Instructions: + + Read the CSV fiels with index_col=False, like so: + ``` + codebook = pd.read_csv('${DATASET_FILES_BASE_PATH}/census-ahs/census_ahs_codebook.csv', dtype=str, index_col=False) + ``` + If you don't add the index_col=False, the first column values will disappear and the whole dataframe will be shifted! +url: null +uuid: beb85142-d32f-42df-a438-b84829ddd4fd diff --git a/src/biome/adhoc_data/datasources/aqs/api.yaml b/src/biome/adhoc_data/datasources/aqs/api.yaml index fc39e7a..26c1f2c 100644 --- a/src/biome/adhoc_data/datasources/aqs/api.yaml +++ b/src/biome/adhoc_data/datasources/aqs/api.yaml @@ -1,3 +1,4 @@ +datatype: api description: | This API is the primary place to obtain row-level data from the EPA's Air Quality System (AQS) database. AQS contains @@ -9,174 +10,10 @@ description: | real-time air quality data (it can take 6 months from the time data is collected until it is in AQS). For real-time data, please use the AirNow API. -integration_type: api +img_url: null +last_updated: null name: EPA Air Quality System -prompt: "# OpenAPI Spec JSON:\n${raw_documentation}\n\n# Additional Instructions:\n\ - \nThis API is through OpenAPI Spec (Swagger). You will be provided the schema.\n\ - All requests go to the following URL: https://aqs.epa.gov/data/api\nThat is the\ - \ base URL for all requests.\n\nTo use the AQS API always retrieve the email/key\ - \ credentials from environment variables:\n- email: os.environ.get(\"API_EPA_AQS_EMAIL\"\ - )\n- api_key: os.environ.get(\"API_EPA_AQS\")\nDo not use placeholder auth values\ - \ from the OpenAPI spec nor prompt the user for these values, just use the environment\ - \ variables.\n\nEnsure you know the fields that come back from the OpenAPI spec\ - \ in order to process the AQS data.\n\nDaily/yearly data can only be accessed one\ - \ year at a time. If working for that in year/decade ranges, prefer to use the annualData\ - \ endpoints, as the daily data may be too large to fit into context.\nIf the user\ - \ requests data for a year range, you will need to make multiple requests to the\ - \ API- potentially sampling in between, or creating averages, in order to not make\ - \ requests for every single year in the range.\nThe earliest sample in the data\ - \ set is from 1957. Year 1980 marks the beginning of nationally consistent operational\ - \ and quality assurance procedures.\n\nMeasured entities in the AQS data set are\ - \ referred to as parameters (which includes pollutants/substances), and use integer\ - \ codes specific to AQS.\n\nIf a monitor is scheduled to collect data and does not\ - \ it will contain `null` data. This is a placeholder to let EPA know that more data\ - \ will not be forthcoming. AQS does place absolute limits on the values that\ - \ can be submitted.\nHowever, these are fairly liberal limits- for many parameters,\ - \ negative values are acceptable.\nAQS has a nominal quarterly reporting deadline:\ - \ data must be reported by 90 days after the end of the calendar quarter in which\ - \ they are collected.\n\nIf you need to inspect the data of a JSON response before\ - \ processing, and you're dealing with annual data, or multi-daily data, or\nfor\ - \ a geographic region that could yield too much data to process (or fit into an\ - \ LLM context window),\n\nbetter use this endpoint:\n```\n/metaData/fieldsByService?email={{email}}&key={{key}}&service=annualData\n\ - ```\n(example above is checking shape of annualData endpoint). It will provide fields\ - \ documentation for each property in the response.\nIf you use `print(response_data)`\ - \ on a huge JSON response, you will not be able to proceed- you can also try checking\ - \ the length as string,\nand if that's too long, inspect whatever amount of characters\ - \ that would be enough.\nThat is, never do:\n```\ndata = response.json()\nprint(data)\n\ - ```\ninstead, either use fieldsByService endpoint as described earlier, or use the\ - \ Example 3 below (in the #Examples section) to generate a schema and inspect the\ - \ data.\n\nIMPORTANT: `parameter_name` is not a property in the respponse usually,\ - \ check `parameter` or `parameter_code` in responses first instead!\n\n# Output\ - \ Format for the AQS API - JSON\nThe only output format available from the services\ - \ is JSON. This is in conformity with the 18f API standards for government.\n\n\ - The JSON response has two top-level elements. A header and a body. The header contains\ - \ information about the request and the body contains the data.\n\nThe header contains\ - \ the following elements:\n\n- `status`. \"SUCCESS\" if the request returned data.\ - \ \"FAILED\" if the request could not be processed. \"No data matched your selection\"\ - \ if the request was processed but resulted in no data.\n- `request_time`. The date\ - \ and time (at the location of the server) when the request was received.\n- `url`.\ - \ The original URL that was submitted to make the request.\n- `row`. The number\ - \ of rows of data in the body object.\n- `errors`. Any errors encountered when attempting\ - \ to respond to the request.\nThe body is an array with one object per data row.\ - \ The contents of the object will vary based on the request made. (E.g. the columns\ - \ of data returned are different for each query.\n\nHere is a sample of the output\ - \ of a Raw Data request with one data point:\n```\n {\n \"Header\": [\n\ - \ {\n \"status\": \"success\",\n \"request_time\":\ - \ \"2018-06-13T07:45:01-04:00\",\n \"url\": \"https://...\",\n \ - \ \"rows\": 1\n }\n ],\n \"Body\": [\n {\n \ - \ \"state_code\": \"01\",\n \"county_code\": \"073\",\n \ - \ \"site_number\": \"0023\",\n \"parameter_code\": \"88101\",\n\ - \ \"poc\": 1.0,\n \"latitude\": 33.0,\n \"longitude\"\ - : -86.0,\n \"datum\": \"WGS84\",\n \"parameter_name\": \"\ - PM2.5 - Local Conditions\",\n \"date_local\": \"2017-04-01\",\n \ - \ \"time_local\": \"00:00\",\n \"date_gmt\": \"2017-04-01\",\n\ - \ \"time_gmt\": \"06:00\",\n \"sample_measurement\": 6.2,\n\ - \ \"unit_of_measure\": \"Micrograms/cubic meter (LC)\",\n \ - \ \"detection_limit\": 2.0,\n \"uncertainty\": null,\n \"\ - qualifiers\": null,\n \"method_type\": \"FRM\",\n \"method_code\"\ - : \"142\",\n \"method_name\": \"BGI Models PQ200-VSCC or PQ200A-VSCC\ - \ - Gravimetric\",\n \"state_name\": \"Alabama\",\n \"county_name\"\ - : \"Jefferson\",\n \"date_of_last_change\": \"2017-05-30\",\n \ - \ \"cbsa_code\": \"13820\"\n }\n ]\n }\n```\n\n# Error Handling\ - \ and Status Codes\nIf the API is able to parse your request it will return the\ - \ JSON described above and an HTTP status of 200.\n\nIf the API is not able to parse\ - \ your request it will return a status code of 400 and only the header, which will\ - \ include an array of error messages instead of a row count.\nFor example:\n```\n\ - \ {\n \"Header\": [{\n \"status\": \"Failed\",\n \ - \ \"request_time\": \"2018-06-13T08:05:46.588-04:00\",\n \"url\": \"\ - https://...\",\n \"error\": [\n \"value is missing or\ - \ the value is empty: param\"\n ]\n }],\n \"Body\"\ - : []\n }\n```\n\nIf the API is able to parse your request but no data matches\ - \ your selections, it will return the\nJSON header and an HTTP status of 200. The\ - \ row count will be zero,\nthe status will be a message that no data matched your\ - \ selection criteria,\nand the body will be empty.\n```\n{\n \"Header\": [\n\ - \ {\n \"status\": \"No data matched your selection\",\n \"request_time\"\ - : \"2018-06-13T10:15:42-04:00\",\n \"url\": \"https://...\",\n \"\ - rows\": 0\n }\n ],\n \"Body\": []\n}\n```\n\n# Request Limits and Terms\ - \ of Service\nThe API has the following limits imposed on request size:\n\n- Length\ - \ of time. All services (except Monitor) must have the end date (edate field) be\ - \ in the same year as the begin date (bdate field).\n- Number of parameters. Most\ - \ services allow for the selection of multiple parameter codes (param field). A\ - \ maximum of 5 parameter codes may be listed in a single request.\n\nPlease adhere\ - \ to the following when using the API:\n\n- _Limit the size of queries_. Our database\ - \ contains billions of values and you may request more than you intend. If you are\ - \ unsure of the amount of data, start small and work your way up. We request that\ - \ you limit queries to 1,000,000 rows of data each. You can use the \"observation\ - \ count\" field on the annualData service to determine how much data exists for\ - \ a time-parameter-geography combination. If you have any questions or need advice,\ - \ please contact us.\n- _Limit the frequency of queries_. Our system can process\ - \ a limited load. If scripting requests, please wait for one request to complete\ - \ before submitting another and do not make more than 10 requests per minute. Also,\ - \ we request a pause of 5 seconds between requests and adjust accordingly for response\ - \ time and size.\nIf you violate these terms, we may disable your account without\ - \ notice (but we will be in contact via the email address provided).\n\n# Usage\ - \ tips\nThis section contains suggestions for completing certain data related tasks\ - \ and links to software tools using the API.\n\n- Determine if or how much data\ - \ exists for a time-parameter-geography combination:\n - Retrieve data using\ - \ the annualData service.\n - If no records are returned, we do not have the\ - \ data.\n - If records are returned, use the observation count to determine the\ - \ temporal and geographic distribution of the data.\n\n- Monthly averages:\n \ - \ - AQS does not routinely calculate monthly aggregate statistics.\n - If you\ - \ need these, you must calculate them yourself.\n - These can be calculated from\ - \ the sample data or the daily data without loss of fidelity.\n\n- Determine a single\ - \ value for a site with collocated monitors:\n - Many sites will have collocated\ - \ monitors - monitors collecting the same parameter at the same time.\n - The\ - \ API currently provides only monitor level values. (We anticipate adding site-level\ - \ values at some point.)\n - For some criteria pollutants (PM2.5, ozone, lead,\ - \ and NO2), the regulations define procedures for defining a single site-level value.\n\ - \ - For other pollutants, determining a single site-level value is left to the\ - \ investigator.\n\n# More Details\n## API Structure and Capabilities\n- The EPA\ - \ AQS API provides access to air quality data from monitoring stations across the\ - \ US\n- Data is available at multiple temporal resolutions: annual, quarterly, daily,\ - \ and sample-level.\n- Data can be queried at various geographic levels: state,\ - \ county, site, CBSA, and bounding box\n- The API returns data in JSON format with\ - \ a standard structure\n- Key pollutants relevant to asthma (ozone, PM2.5, NO2,\ - \ SO2, CO) are available through the API\n- Data can be used to analyze both acute\ - \ exposures (daily data) and chronic exposures (annual data)\n- Seasonal patterns\ - \ in air quality can be correlated with seasonal patterns in asthma exacerbations\n\ - - The API enables spatial analysis to identify potential hotspots of poor air quality\n\ - \n## Data Integration Strategies\n- Geographic alignment: Use census tract or ZIP\ - \ code as common geographic units\n- Temporal alignment: Monthly aggregation captures\ - \ seasonal patterns while managing data volume\n- Spatial interpolation can estimate\ - \ air quality values between monitoring stations\n- Population-weighted averages\ - \ can be used when aggregating to larger geographic areas\n- A standardized spatial\ - \ grid can facilitate integration with other environmental datasets\n\n## Key Parameters\ - \ for Asthma Research\nThe API provides access to several air pollutants known to\ - \ affect respiratory health:\n```\nParameter Code\tParameter Name\tRelevance to\ - \ Asthma\n44201\tOzone\tMajor trigger for asthma attacks\n88101\tPM2.5\tCan penetrate\ - \ deep into lungs\n81102\tPM10\tCan irritate airways\n42602\tNitrogen Dioxide (NO2)\t\ - Common in vehicle emissions\n42401\tSulfur Dioxide (SO2)\tIndustrial emissions linked\ - \ to respiratory issues\n42101\tCarbon Monoxide\t(CO) Affects oxygen transport\n\ - ```\n\n## Key Endpoints and Parameters\nThe API offers several endpoint categories:\n\ - \n- Monitoring Sites: monitors/byCounty - Returns information about monitoring stations\n\ - - Daily Data: dailyData/byCounty - Daily measurements for parameters\n- Annual Data:\ - \ annualData/byCounty - Yearly aggregated measurements\n- Sample Data: sampleData/byCounty\ - \ - Individual sample measurements\n- Metadata: Various endpoints for parameter\ - \ codes, classes, etc.\n\nAll data requests require:\n- Geographic parameters (state/county\ - \ codes)\n- Date parameters (bdate/edate in YYYYMMDD format)\n- Parameter code for\ - \ the pollutant\n\n# Implementation Challenges and Solutions\n- Date Parameter Requirements:\ - \ Despite documentation suggesting otherwise, all data endpoints require explicit\ - \ date parameters (bdate and edate), not just the year parameter.\n- Data Volume\ - \ Management: Harris County has numerous monitoring stations (we found 24,532 daily\ - \ ozone records for 2022 alone), requiring efficient data aggregation strategies.\n\ - - Parameter Identification: The API requires specific parameter codes rather than\ - \ names, necessitating lookup of these codes before querying data.\n\nThe EPA AQS\ - \ API provides rich, detailed air quality data that can be invaluable\nfor researching\ - \ environmental factors affecting childhood asthma (and others).\nBy combining this\ - \ data with health outcome data from other sources, researchers\ncan develop comprehensive\ - \ analyses of how air quality impacts respiratory health\nin specific geographic\ - \ areas like a county or region.\nThe API's structure allows for flexible querying\ - \ by location, time period, and\nspecific pollutants, making it a powerful tool\ - \ for environmental health research.\nHowever, users should be aware of specific\ - \ parameter requirements and prepare\nfor handling large volumes of data when working\ - \ with densely monitored areas.\n\nNote: If/when using the `metaData/isAvailable`\ - \ endpoint to check if the API is available,\nuse this sample response as guide:\n\ - ```\n{\n \"Header\": [\n {\n \"status\": \"API service is up\ - \ and running healthy. Status: connection_pool: size: 5, connections: 1, in use:\ - \ 1, waiting_in_queue: 0\",\n \"request_time\": \"2025-03-21T08:21:41-04:00\"\ - ,\n \"url\": \"http://aqs.epa.gov/api/metaData/isAvailable?email={{email}}.com&key={{key}}\"\ - \n }\n ],\n \"Data\": []\n}\n```\nNever simulate this API- if it is\ - \ not available ask the user how to proceed.\n" +provider: adhoc:specialist_agents resources: 00b2ebdf-94ff-475f-be91-a1f69e1b4403: code: "import os\nimport requests\nimport pandas as pd\nfrom datetime import datetime\n\ @@ -207,7 +44,7 @@ resources: df['date'] = pd.to_datetime(df['date'])\n\n# Sort by date and parameter\ndf\ \ = df.sort_values(['date', 'parameter'])\n\n# Save to CSV\ndf.to_csv('harris_county_air_quality_2022.csv',\ \ index=False)\n" - integration: 715e610b-28e4-4ae8-99b8-292db41dbd00 + integration: 18e5974c-8cea-4fe7-88db-7c5779a17d5d notes: null query: Retrieve daily air quality data for ozone, CO, NO2, and PM2.5 in Harris County, TX for each month of the year 2022. @@ -254,7 +91,7 @@ resources: data = response.json() data - integration: 715e610b-28e4-4ae8-99b8-292db41dbd00 + integration: 18e5974c-8cea-4fe7-88db-7c5779a17d5d notes: null query: Query annual air pollution data for Seminole County, Florida for the year 2000. Use pollutants relating to asthma. @@ -299,7 +136,7 @@ resources: \ print(f\"{pollutant}: {code}\")\n else:\n print(\"Failed\ \ to retrieve criteria parameters data\")\nelse:\n print(\"Failed to retrieve\ \ parameter classes data\")\n" - integration: 715e610b-28e4-4ae8-99b8-292db41dbd00 + integration: 18e5974c-8cea-4fe7-88db-7c5779a17d5d notes: null query: |- Retrieving and exploring parameter classes and codes for air quality research. This example demonstrates how to get a list of parameter classes, then retrieve specific parameters within the CRITERIA class, which contains the most important air pollutants for health research. It also identifies key parameter codes relevant to asthma research. @@ -336,7 +173,7 @@ resources: \ # Display first few rows\n print(df.head())\n else:\n print(\"\ No data found for the specified parameters\")\nelse:\n print(\"No data collected.\"\ )\n" - integration: 715e610b-28e4-4ae8-99b8-292db41dbd00 + integration: 18e5974c-8cea-4fe7-88db-7c5779a17d5d notes: null query: Collect historical air quality data from the EPA Air Quality System (AQS) for Orange County, FL. @@ -344,7 +181,7 @@ resources: resource_type: example 365697ce-eb50-444b-8a82-637a2a0131f4: filepath: aqs.json - integration: 715e610b-28e4-4ae8-99b8-292db41dbd00 + integration: 18e5974c-8cea-4fe7-88db-7c5779a17d5d name: raw_documentation resource_id: 365697ce-eb50-444b-8a82-637a2a0131f4 resource_type: file @@ -375,7 +212,7 @@ resources: \ pollutant_name.lower()})\n \n return daily_avg, df['units_of_measure'].iloc[0]\n\ \ else:\n print(f\"Failed to retrieve {pollutant_name} data\")\n \ \ return None, None\n" - integration: 715e610b-28e4-4ae8-99b8-292db41dbd00 + integration: 18e5974c-8cea-4fe7-88db-7c5779a17d5d notes: null query: 'Here''s a reusable function for retrieving and processing data for multiple pollutants:' @@ -398,7 +235,7 @@ resources: \ a county with a lot of data- lets generate a schema and inspect that instead\ \ of inspecting the raw data\nbuilder = SchemaBuilder()\nbuilder.add_object(response_data)\n\ schema = builder.to_schema()\n\nprint(schema[\"properties\"])\n" - integration: 715e610b-28e4-4ae8-99b8-292db41dbd00 + integration: 18e5974c-8cea-4fe7-88db-7c5779a17d5d notes: null query: |- Inspect API response change by generating a schema from an API response so that we don't inspect the raw data and blow up the context window. @@ -469,10 +306,178 @@ resources: \ annot=True, cmap='coolwarm', vmin=-1, vmax=1, center=0)\n plt.title(f'Correlation\ \ Matrix of Air Pollutants ({year})', fontsize=16)\n plt.tight_layout()\n\ \ plt.show()\n" - integration: 715e610b-28e4-4ae8-99b8-292db41dbd00 + integration: 18e5974c-8cea-4fe7-88db-7c5779a17d5d notes: null query: |- Multi-pollutant correlation analysis for air quality research. This example demonstrates how to retrieve data for multiple pollutants (Ozone, PM2.5, NO2, and SO2), process the data to get daily averages, merge the datasets, and analyze correlations between different pollutants. It includes a visualization of the correlation matrix using a heatmap. resource_id: e643cf2e-66f1-428e-8f2c-2626c4c59192 resource_type: example -slug: 715e610b-28e4-4ae8-99b8-292db41dbd00 +slug: adhocspecificationintegration_epa_air_quality_system +source: "# OpenAPI Spec JSON:\n${raw_documentation}\n\n# Additional Instructions:\n\ + \nThis API is through OpenAPI Spec (Swagger). You will be provided the schema.\n\ + All requests go to the following URL: https://aqs.epa.gov/data/api\nThat is the\ + \ base URL for all requests.\n\nTo use the AQS API always retrieve the email/key\ + \ credentials from environment variables:\n- email: os.environ.get(\"API_EPA_AQS_EMAIL\"\ + )\n- api_key: os.environ.get(\"API_EPA_AQS\")\nDo not use placeholder auth values\ + \ from the OpenAPI spec nor prompt the user for these values, just use the environment\ + \ variables.\n\nEnsure you know the fields that come back from the OpenAPI spec\ + \ in order to process the AQS data.\n\nDaily/yearly data can only be accessed one\ + \ year at a time. If working for that in year/decade ranges, prefer to use the annualData\ + \ endpoints, as the daily data may be too large to fit into context.\nIf the user\ + \ requests data for a year range, you will need to make multiple requests to the\ + \ API- potentially sampling in between, or creating averages, in order to not make\ + \ requests for every single year in the range.\nThe earliest sample in the data\ + \ set is from 1957. Year 1980 marks the beginning of nationally consistent operational\ + \ and quality assurance procedures.\n\nMeasured entities in the AQS data set are\ + \ referred to as parameters (which includes pollutants/substances), and use integer\ + \ codes specific to AQS.\n\nIf a monitor is scheduled to collect data and does not\ + \ it will contain `null` data. This is a placeholder to let EPA know that more data\ + \ will not be forthcoming. AQS does place absolute limits on the values that\ + \ can be submitted.\nHowever, these are fairly liberal limits- for many parameters,\ + \ negative values are acceptable.\nAQS has a nominal quarterly reporting deadline:\ + \ data must be reported by 90 days after the end of the calendar quarter in which\ + \ they are collected.\n\nIf you need to inspect the data of a JSON response before\ + \ processing, and you're dealing with annual data, or multi-daily data, or\nfor\ + \ a geographic region that could yield too much data to process (or fit into an\ + \ LLM context window),\n\nbetter use this endpoint:\n```\n/metaData/fieldsByService?email={{email}}&key={{key}}&service=annualData\n\ + ```\n(example above is checking shape of annualData endpoint). It will provide fields\ + \ documentation for each property in the response.\nIf you use `print(response_data)`\ + \ on a huge JSON response, you will not be able to proceed- you can also try checking\ + \ the length as string,\nand if that's too long, inspect whatever amount of characters\ + \ that would be enough.\nThat is, never do:\n```\ndata = response.json()\nprint(data)\n\ + ```\ninstead, either use fieldsByService endpoint as described earlier, or use the\ + \ Example 3 below (in the #Examples section) to generate a schema and inspect the\ + \ data.\n\nIMPORTANT: `parameter_name` is not a property in the respponse usually,\ + \ check `parameter` or `parameter_code` in responses first instead!\n\n# Output\ + \ Format for the AQS API - JSON\nThe only output format available from the services\ + \ is JSON. This is in conformity with the 18f API standards for government.\n\n\ + The JSON response has two top-level elements. A header and a body. The header contains\ + \ information about the request and the body contains the data.\n\nThe header contains\ + \ the following elements:\n\n- `status`. \"SUCCESS\" if the request returned data.\ + \ \"FAILED\" if the request could not be processed. \"No data matched your selection\"\ + \ if the request was processed but resulted in no data.\n- `request_time`. The date\ + \ and time (at the location of the server) when the request was received.\n- `url`.\ + \ The original URL that was submitted to make the request.\n- `row`. The number\ + \ of rows of data in the body object.\n- `errors`. Any errors encountered when attempting\ + \ to respond to the request.\nThe body is an array with one object per data row.\ + \ The contents of the object will vary based on the request made. (E.g. the columns\ + \ of data returned are different for each query.\n\nHere is a sample of the output\ + \ of a Raw Data request with one data point:\n```\n {\n \"Header\": [\n\ + \ {\n \"status\": \"success\",\n \"request_time\":\ + \ \"2018-06-13T07:45:01-04:00\",\n \"url\": \"https://...\",\n \ + \ \"rows\": 1\n }\n ],\n \"Body\": [\n {\n \ + \ \"state_code\": \"01\",\n \"county_code\": \"073\",\n \ + \ \"site_number\": \"0023\",\n \"parameter_code\": \"88101\",\n\ + \ \"poc\": 1.0,\n \"latitude\": 33.0,\n \"longitude\"\ + : -86.0,\n \"datum\": \"WGS84\",\n \"parameter_name\": \"\ + PM2.5 - Local Conditions\",\n \"date_local\": \"2017-04-01\",\n \ + \ \"time_local\": \"00:00\",\n \"date_gmt\": \"2017-04-01\",\n\ + \ \"time_gmt\": \"06:00\",\n \"sample_measurement\": 6.2,\n\ + \ \"unit_of_measure\": \"Micrograms/cubic meter (LC)\",\n \ + \ \"detection_limit\": 2.0,\n \"uncertainty\": null,\n \"\ + qualifiers\": null,\n \"method_type\": \"FRM\",\n \"method_code\"\ + : \"142\",\n \"method_name\": \"BGI Models PQ200-VSCC or PQ200A-VSCC\ + \ - Gravimetric\",\n \"state_name\": \"Alabama\",\n \"county_name\"\ + : \"Jefferson\",\n \"date_of_last_change\": \"2017-05-30\",\n \ + \ \"cbsa_code\": \"13820\"\n }\n ]\n }\n```\n\n# Error Handling\ + \ and Status Codes\nIf the API is able to parse your request it will return the\ + \ JSON described above and an HTTP status of 200.\n\nIf the API is not able to parse\ + \ your request it will return a status code of 400 and only the header, which will\ + \ include an array of error messages instead of a row count.\nFor example:\n```\n\ + \ {\n \"Header\": [{\n \"status\": \"Failed\",\n \ + \ \"request_time\": \"2018-06-13T08:05:46.588-04:00\",\n \"url\": \"\ + https://...\",\n \"error\": [\n \"value is missing or\ + \ the value is empty: param\"\n ]\n }],\n \"Body\"\ + : []\n }\n```\n\nIf the API is able to parse your request but no data matches\ + \ your selections, it will return the\nJSON header and an HTTP status of 200. The\ + \ row count will be zero,\nthe status will be a message that no data matched your\ + \ selection criteria,\nand the body will be empty.\n```\n{\n \"Header\": [\n\ + \ {\n \"status\": \"No data matched your selection\",\n \"request_time\"\ + : \"2018-06-13T10:15:42-04:00\",\n \"url\": \"https://...\",\n \"\ + rows\": 0\n }\n ],\n \"Body\": []\n}\n```\n\n# Request Limits and Terms\ + \ of Service\nThe API has the following limits imposed on request size:\n\n- Length\ + \ of time. All services (except Monitor) must have the end date (edate field) be\ + \ in the same year as the begin date (bdate field).\n- Number of parameters. Most\ + \ services allow for the selection of multiple parameter codes (param field). A\ + \ maximum of 5 parameter codes may be listed in a single request.\n\nPlease adhere\ + \ to the following when using the API:\n\n- _Limit the size of queries_. Our database\ + \ contains billions of values and you may request more than you intend. If you are\ + \ unsure of the amount of data, start small and work your way up. We request that\ + \ you limit queries to 1,000,000 rows of data each. You can use the \"observation\ + \ count\" field on the annualData service to determine how much data exists for\ + \ a time-parameter-geography combination. If you have any questions or need advice,\ + \ please contact us.\n- _Limit the frequency of queries_. Our system can process\ + \ a limited load. If scripting requests, please wait for one request to complete\ + \ before submitting another and do not make more than 10 requests per minute. Also,\ + \ we request a pause of 5 seconds between requests and adjust accordingly for response\ + \ time and size.\nIf you violate these terms, we may disable your account without\ + \ notice (but we will be in contact via the email address provided).\n\n# Usage\ + \ tips\nThis section contains suggestions for completing certain data related tasks\ + \ and links to software tools using the API.\n\n- Determine if or how much data\ + \ exists for a time-parameter-geography combination:\n - Retrieve data using\ + \ the annualData service.\n - If no records are returned, we do not have the\ + \ data.\n - If records are returned, use the observation count to determine the\ + \ temporal and geographic distribution of the data.\n\n- Monthly averages:\n \ + \ - AQS does not routinely calculate monthly aggregate statistics.\n - If you\ + \ need these, you must calculate them yourself.\n - These can be calculated from\ + \ the sample data or the daily data without loss of fidelity.\n\n- Determine a single\ + \ value for a site with collocated monitors:\n - Many sites will have collocated\ + \ monitors - monitors collecting the same parameter at the same time.\n - The\ + \ API currently provides only monitor level values. (We anticipate adding site-level\ + \ values at some point.)\n - For some criteria pollutants (PM2.5, ozone, lead,\ + \ and NO2), the regulations define procedures for defining a single site-level value.\n\ + \ - For other pollutants, determining a single site-level value is left to the\ + \ investigator.\n\n# More Details\n## API Structure and Capabilities\n- The EPA\ + \ AQS API provides access to air quality data from monitoring stations across the\ + \ US\n- Data is available at multiple temporal resolutions: annual, quarterly, daily,\ + \ and sample-level.\n- Data can be queried at various geographic levels: state,\ + \ county, site, CBSA, and bounding box\n- The API returns data in JSON format with\ + \ a standard structure\n- Key pollutants relevant to asthma (ozone, PM2.5, NO2,\ + \ SO2, CO) are available through the API\n- Data can be used to analyze both acute\ + \ exposures (daily data) and chronic exposures (annual data)\n- Seasonal patterns\ + \ in air quality can be correlated with seasonal patterns in asthma exacerbations\n\ + - The API enables spatial analysis to identify potential hotspots of poor air quality\n\ + \n## Data Integration Strategies\n- Geographic alignment: Use census tract or ZIP\ + \ code as common geographic units\n- Temporal alignment: Monthly aggregation captures\ + \ seasonal patterns while managing data volume\n- Spatial interpolation can estimate\ + \ air quality values between monitoring stations\n- Population-weighted averages\ + \ can be used when aggregating to larger geographic areas\n- A standardized spatial\ + \ grid can facilitate integration with other environmental datasets\n\n## Key Parameters\ + \ for Asthma Research\nThe API provides access to several air pollutants known to\ + \ affect respiratory health:\n```\nParameter Code\tParameter Name\tRelevance to\ + \ Asthma\n44201\tOzone\tMajor trigger for asthma attacks\n88101\tPM2.5\tCan penetrate\ + \ deep into lungs\n81102\tPM10\tCan irritate airways\n42602\tNitrogen Dioxide (NO2)\t\ + Common in vehicle emissions\n42401\tSulfur Dioxide (SO2)\tIndustrial emissions linked\ + \ to respiratory issues\n42101\tCarbon Monoxide\t(CO) Affects oxygen transport\n\ + ```\n\n## Key Endpoints and Parameters\nThe API offers several endpoint categories:\n\ + \n- Monitoring Sites: monitors/byCounty - Returns information about monitoring stations\n\ + - Daily Data: dailyData/byCounty - Daily measurements for parameters\n- Annual Data:\ + \ annualData/byCounty - Yearly aggregated measurements\n- Sample Data: sampleData/byCounty\ + \ - Individual sample measurements\n- Metadata: Various endpoints for parameter\ + \ codes, classes, etc.\n\nAll data requests require:\n- Geographic parameters (state/county\ + \ codes)\n- Date parameters (bdate/edate in YYYYMMDD format)\n- Parameter code for\ + \ the pollutant\n\n# Implementation Challenges and Solutions\n- Date Parameter Requirements:\ + \ Despite documentation suggesting otherwise, all data endpoints require explicit\ + \ date parameters (bdate and edate), not just the year parameter.\n- Data Volume\ + \ Management: Harris County has numerous monitoring stations (we found 24,532 daily\ + \ ozone records for 2022 alone), requiring efficient data aggregation strategies.\n\ + - Parameter Identification: The API requires specific parameter codes rather than\ + \ names, necessitating lookup of these codes before querying data.\n\nThe EPA AQS\ + \ API provides rich, detailed air quality data that can be invaluable\nfor researching\ + \ environmental factors affecting childhood asthma (and others).\nBy combining this\ + \ data with health outcome data from other sources, researchers\ncan develop comprehensive\ + \ analyses of how air quality impacts respiratory health\nin specific geographic\ + \ areas like a county or region.\nThe API's structure allows for flexible querying\ + \ by location, time period, and\nspecific pollutants, making it a powerful tool\ + \ for environmental health research.\nHowever, users should be aware of specific\ + \ parameter requirements and prepare\nfor handling large volumes of data when working\ + \ with densely monitored areas.\n\nNote: If/when using the `metaData/isAvailable`\ + \ endpoint to check if the API is available,\nuse this sample response as guide:\n\ + ```\n{\n \"Header\": [\n {\n \"status\": \"API service is up\ + \ and running healthy. Status: connection_pool: size: 5, connections: 1, in use:\ + \ 1, waiting_in_queue: 0\",\n \"request_time\": \"2025-03-21T08:21:41-04:00\"\ + ,\n \"url\": \"http://aqs.epa.gov/api/metaData/isAvailable?email={{email}}.com&key={{key}}\"\ + \n }\n ],\n \"Data\": []\n}\n```\nNever simulate this API- if it is\ + \ not available ask the user how to proceed.\n" +url: null +uuid: 18e5974c-8cea-4fe7-88db-7c5779a17d5d diff --git a/src/biome/adhoc_data/datasources/cbioportal/api.yaml b/src/biome/adhoc_data/datasources/cbioportal/api.yaml index b8455ed..ab36919 100644 --- a/src/biome/adhoc_data/datasources/cbioportal/api.yaml +++ b/src/biome/adhoc_data/datasources/cbioportal/api.yaml @@ -1,12 +1,10 @@ +datatype: api description: | The cBioPortal for Cancer Genomics is an open-access, open-source resource for interactive exploration of multidimensional cancer genomics data sets. The goal of cBioPortal is to significantly lower the barriers between complex genomic data and cancer researchers by providing rapid, intuitive, and high-quality access to molecular profiles and clinical attributes from large-scale cancer genomics projects, and therefore to empower researchers to translate these rich data sets into biologic insights and clinical applications. -integration_type: api +img_url: null +last_updated: null name: cbioportal -prompt: | - This API is through Open API Spec (Swagger). You will be provided the schema below. - All requests go to the following URL: https://www.cbioportal.org/ - - ${cbioportal} +provider: adhoc:specialist_agents resources: 21897f3f-9451-4338-b2b4-8ae3faeafc20: code: | @@ -59,7 +57,7 @@ resources: aml_studies_df = fetch_aml_studies() print(f"Found {len(aml_studies_df)} AML-related studies") print(aml_studies_df) - integration: 26b8655b-4f68-4f70-bb30-5f1397dd201a + integration: 33e3119f-5504-4615-981e-55000c066e80 notes: null query: Fetch and filter studies related to Acute Myeloid Leukemia (AML) from cBioPortal API. @@ -202,7 +200,7 @@ resources: print("\nWide format example (first few rows):") print(stat5_all_studies_wide.head()) - integration: 26b8655b-4f68-4f70-bb30-5f1397dd201a + integration: 33e3119f-5504-4615-981e-55000c066e80 notes: | This example demonstrates: 1. How to properly fetch sample IDs for each study @@ -222,7 +220,7 @@ resources: resource_type: example 52adfa23-1b07-462e-968b-d09ed12848f0: filepath: cbioportal.json - integration: 26b8655b-4f68-4f70-bb30-5f1397dd201a + integration: 33e3119f-5504-4615-981e-55000c066e80 name: cbioportal resource_id: 52adfa23-1b07-462e-968b-d09ed12848f0 resource_type: file @@ -280,7 +278,7 @@ resources: else: print(f"Error fetching mutations: {mutations_response.status_code}") print("Response:", mutations_response.text) - integration: 26b8655b-4f68-4f70-bb30-5f1397dd201a + integration: 33e3119f-5504-4615-981e-55000c066e80 notes: | It shows: 1. How to properly construct the API endpoint URL with sample list ID @@ -380,7 +378,7 @@ resources: ] print(f"\nRNA expression profiles for {first_study}:") print(rna_profiles[['molecularProfileId', 'name', 'datatype']].to_string()) - integration: 26b8655b-4f68-4f70-bb30-5f1397dd201a + integration: 33e3119f-5504-4615-981e-55000c066e80 notes: null query: |- Example showing how to retrieve and analyze the molecular profiles available for multiple studies from cBioPortal. The code demonstrates how to fetch metadata on the molecular profiles, organize them into DataFrames, and analyze specific information about each profile like the types of profiles available such as RNA expression. @@ -481,7 +479,7 @@ resources: # Save to CSV combined_df.to_csv('all_aml_mutations.csv', index=False) - integration: 26b8655b-4f68-4f70-bb30-5f1397dd201a + integration: 33e3119f-5504-4615-981e-55000c066e80 notes: null query: |- Retrieve and analyze mutation data from multiple AML studies, including data processing and summary statistics. This example demonstrates how to fetch mutations across multiple studies, handle the nested JSON response, and create a comprehensive mutation dataset with key information. @@ -505,7 +503,7 @@ resources: print(f"Description: {study['description']}") print(f"Cancer Type ID: {study['cancerTypeId']}") print("-" * 80) - integration: 26b8655b-4f68-4f70-bb30-5f1397dd201a + integration: 33e3119f-5504-4615-981e-55000c066e80 notes: null query: |- Query and display all colorectal cancer studies from cBioPortal. This example searches for studies with cancer type IDs 'coadread', 'coad', and 'read' (representing colorectal adenocarcinoma, colon adenocarcinoma, and rectal adenocarcinoma respectively). For each study, it displays the name, study ID, description, and cancer type ID. @@ -539,7 +537,7 @@ resources: # Print the DataFrame print(df) - integration: 26b8655b-4f68-4f70-bb30-5f1397dd201a + integration: 33e3119f-5504-4615-981e-55000c066e80 notes: null query: Query for studies related to colorectal cancer using the cancerTypeId 'coadread'. resource_id: c68b74b2-c97c-4cf5-b8b4-7e514db2db65 @@ -622,7 +620,7 @@ resources: print(f" - {change}") if len(details['protein_changes']) > 5: print(f" ... and {len(details['protein_changes'])-5} more changes") - integration: 26b8655b-4f68-4f70-bb30-5f1397dd201a + integration: 33e3119f-5504-4615-981e-55000c066e80 notes: | The example demonstrates how to: 1. Get sample IDs for a study @@ -634,4 +632,11 @@ resources: Query mutation data from cBioPortal for a specific study (Genentech colorectal cancer study) and analyze mutations in key cancer genes (APC, TP53, KRAS, PIK3CA) resource_id: d54d8188-81f5-4fff-a641-230480369fe4 resource_type: example -slug: 26b8655b-4f68-4f70-bb30-5f1397dd201a +slug: adhocspecificationintegration_cbioportal +source: | + This API is through Open API Spec (Swagger). You will be provided the schema below. + All requests go to the following URL: https://www.cbioportal.org/ + + ${cbioportal} +url: null +uuid: 33e3119f-5504-4615-981e-55000c066e80 diff --git a/src/biome/adhoc_data/datasources/cda/api.yaml b/src/biome/adhoc_data/datasources/cda/api.yaml index 0ae4e95..c8d5920 100644 --- a/src/biome/adhoc_data/datasources/cda/api.yaml +++ b/src/biome/adhoc_data/datasources/cda/api.yaml @@ -1,3 +1,4 @@ +datatype: api description: "The Cancer Data Aggregator (CDA) is a service of the National Cancer\ \ Institutes' (NCI) Cancer Research Data Commons. We pull metadata for thousands\ \ of studies hosted at multiple data repositories across NCI, and make it available\ @@ -92,9 +93,19 @@ description: "The Cancer Data Aggregator (CDA) is a service of the National Canc \ DSS has provided values for: ethnicity, file_format, morphology, primary_diagnosis,\ \ race, species, therapeutic_agent, source_material_type (cancer/normal), treatment_type,\ \ and vital_status.\n" -integration_type: api +img_url: null +last_updated: null name: Cancer Data Aggregator -prompt: | +provider: adhoc:specialist_agents +resources: + 5361ce45-01fb-4c16-aa22-31d5397a0bd1: + filepath: cda.yaml + integration: 3a97a492-4947-4cb1-9b40-4dcfbdeee3f3 + name: raw_documentation + resource_id: 5361ce45-01fb-4c16-aa22-31d5397a0bd1 + resource_type: file +slug: adhocspecificationintegration_cancer_data_aggregator +source: | ${raw_documentation} # Additional Instructions: @@ -108,11 +119,5 @@ prompt: | You should make use of the python requests library to interact with the API. Note that the base URL of the API is 'https://cda.datacommons.cancer.gov/' -resources: - 5361ce45-01fb-4c16-aa22-31d5397a0bd1: - filepath: cda.yaml - integration: 16fba0e0-bf97-4d4d-8989-aad9ef934319 - name: raw_documentation - resource_id: 5361ce45-01fb-4c16-aa22-31d5397a0bd1 - resource_type: file -slug: 16fba0e0-bf97-4d4d-8989-aad9ef934319 +url: null +uuid: 3a97a492-4947-4cb1-9b40-4dcfbdeee3f3 diff --git a/src/biome/adhoc_data/datasources/cdc_tracking_network/api.yaml b/src/biome/adhoc_data/datasources/cdc_tracking_network/api.yaml index 6315d41..ed2197d 100644 --- a/src/biome/adhoc_data/datasources/cdc_tracking_network/api.yaml +++ b/src/biome/adhoc_data/datasources/cdc_tracking_network/api.yaml @@ -1,3 +1,4 @@ +datatype: api description: | The National Environmental Public Health Tracking Network (Tracking Network) brings together health data and environmental data from national, state, county, @@ -9,9 +10,25 @@ description: | - Age-stratified data including children under 18 - County-level resolution - Information on asthma, and other conditions -integration_type: api +img_url: null +last_updated: null name: CDC Tracking Network -prompt: | +provider: adhoc:specialist_agents +resources: + 5b62b4c6-536c-4a8d-8e4b-bd295ac8ed72: + filepath: api_examples.md + integration: 62306be3-9b4b-4988-aac9-19d50b64b336 + name: api_examples + resource_id: 5b62b4c6-536c-4a8d-8e4b-bd295ac8ed72 + resource_type: file + 7c5e6353-7e1f-4639-91ee-9dc568887a69: + filepath: user_guide.md + integration: 62306be3-9b4b-4988-aac9-19d50b64b336 + name: user_guide + resource_id: 7c5e6353-7e1f-4639-91ee-9dc568887a69 + resource_type: file +slug: adhocspecificationintegration_cdc_tracking_network +source: | The Tracking Program collects, integrates, analyzes, and disseminates non-infectious disease, environmental, and socio-economic data from various sources. These include a collective of national, state, and local partners. @@ -30,17 +47,5 @@ prompt: | The list of content areas is available at: ${api_examples} -resources: - 5b62b4c6-536c-4a8d-8e4b-bd295ac8ed72: - filepath: api_examples.md - integration: ba3db193-3d39-4793-a525-c569d31bd38b - name: api_examples - resource_id: 5b62b4c6-536c-4a8d-8e4b-bd295ac8ed72 - resource_type: file - 7c5e6353-7e1f-4639-91ee-9dc568887a69: - filepath: user_guide.md - integration: ba3db193-3d39-4793-a525-c569d31bd38b - name: user_guide - resource_id: 7c5e6353-7e1f-4639-91ee-9dc568887a69 - resource_type: file -slug: ba3db193-3d39-4793-a525-c569d31bd38b +url: null +uuid: 62306be3-9b4b-4988-aac9-19d50b64b336 diff --git a/src/biome/adhoc_data/datasources/census_acs/api.yaml b/src/biome/adhoc_data/datasources/census_acs/api.yaml index 89bdba3..d9bc469 100644 --- a/src/biome/adhoc_data/datasources/census_acs/api.yaml +++ b/src/biome/adhoc_data/datasources/census_acs/api.yaml @@ -1,31 +1,16 @@ +datatype: api description: | The Census ACS (American Community Survey) and SF1 (Decennial Census) API returns data that has been collected from the Census Bureau, a database that contains information on the population of the United States. Use this to control for socioeconomic factors (or for other purposes). -integration_type: api +img_url: null +last_updated: null name: Census ACS (American Community Survey) and SF1 -prompt: "The American Community Survey (ACS) is an ongoing survey that provides data\ - \ every\nyear\u2014giving communities the current information they need to make\ - \ important decisions.\nThe ACS covers a broad range of topics about social, economic,\ - \ housing, and demographic\ncharacteristics of the U.S. population.\n\n${web_documentation}\n\ - \nYou will need an API Key to use the Census ACS API.\nIt is available in the environment\ - \ variable:\n- api_key: os.environ.get(\"API_CENSUS\")\n\nYou have an api key, so\ - \ you have access to the census pypi package, and also direct access to the census\ - \ web api.\nPrefer the pypi package if you can, it's easier to use.\n\nData ranges\ - \ from 2005 to 2023.\n\n${sdk_documentation}\n\n## For 1-year datasets when loading\ - \ as file (may or may not apply when using python libraries...)\n### Variable Changes\n\ - Variables, and the values they represent, may change over time. Use the file in\ - \ `${DATASET_FILES_BASE_PATH}/census-acs/2023-1yr-api-changes.csv` as a guide for\ - \ which variables have changed from the prior year for 2023 ACS 1-Year Detailed\ - \ Tables, Data Profiles and Subject Tables. See below for a description of each\ - \ change type.\nNo Change - The variable has not changed from the prior year (most\ - \ variables).\nUpdated - That variable has changed from the prior year and a matching\ - \ variable for the current year has been found.\nNo Match - The variable has changed\ - \ from the prior year and no matching or comparable variable has been found.\n" +provider: adhoc:specialist_agents resources: 76813161-e3dd-400e-b1db-c6534c3fb484: filepath: census_web.md - integration: 50e33776-2cc4-4b05-8135-4ff03d5d42b1 + integration: d6d4d49c-b157-43cb-995a-d66fa3398a72 name: web_documentation resource_id: 76813161-e3dd-400e-b1db-c6534c3fb484 resource_type: file @@ -39,7 +24,7 @@ resources: \ = ['Name', 'Median_Income', 'Poverty_Count', \n 'Median_Rent',\ \ 'Population', 'state', 'county', 'tract']\n\n# Calculate poverty rate\nharris_ses_df['Poverty_Rate']\ \ = harris_ses_df['Poverty_Count'] / harris_ses_df['Population'] * 100\n" - integration: 50e33776-2cc4-4b05-8135-4ff03d5d42b1 + integration: d6d4d49c-b157-43cb-995a-d66fa3398a72 notes: null query: Calculate the poverty rate in Harris County, Texas- Descriptive analysis resource_id: 93d4da90-fe43-4121-8e1b-d9d677637143 @@ -58,15 +43,35 @@ resources: exposure_by_ses = pd.merge(environmental_data, harris_ses_df, on='tract')\n\n\ # Calculate average exposures by income quartile\nquartile_summary = exposure_by_ses.groupby('Income_Quartile')['PM25_Annual',\ \ 'Ozone_Days_Exceeded', 'TRI_Releases'].mean()" - integration: 50e33776-2cc4-4b05-8135-4ff03d5d42b1 + integration: d6d4d49c-b157-43cb-995a-d66fa3398a72 notes: null query: Calculate the poverty rate in Harris County, Texas- Stratified analysis resource_id: 9a1dd209-4e65-49af-a5cc-2f9d73cd8d4b resource_type: example cb8c091d-ff82-46d7-bdea-893aa5b0c6c5: filepath: census_sdk.md - integration: 50e33776-2cc4-4b05-8135-4ff03d5d42b1 + integration: d6d4d49c-b157-43cb-995a-d66fa3398a72 name: sdk_documentation resource_id: cb8c091d-ff82-46d7-bdea-893aa5b0c6c5 resource_type: file -slug: 50e33776-2cc4-4b05-8135-4ff03d5d42b1 +slug: adhocspecificationintegration_census_acs__american_community_survey__and_sf1 +source: "The American Community Survey (ACS) is an ongoing survey that provides data\ + \ every\nyear\u2014giving communities the current information they need to make\ + \ important decisions.\nThe ACS covers a broad range of topics about social, economic,\ + \ housing, and demographic\ncharacteristics of the U.S. population.\n\n${web_documentation}\n\ + \nYou will need an API Key to use the Census ACS API.\nIt is available in the environment\ + \ variable:\n- api_key: os.environ.get(\"API_CENSUS\")\n\nYou have an api key, so\ + \ you have access to the census pypi package, and also direct access to the census\ + \ web api.\nPrefer the pypi package if you can, it's easier to use.\n\nData ranges\ + \ from 2005 to 2023.\n\n${sdk_documentation}\n\n## For 1-year datasets when loading\ + \ as file (may or may not apply when using python libraries...)\n### Variable Changes\n\ + Variables, and the values they represent, may change over time. Use the file in\ + \ `${DATASET_FILES_BASE_PATH}/census-acs/2023-1yr-api-changes.csv` as a guide for\ + \ which variables have changed from the prior year for 2023 ACS 1-Year Detailed\ + \ Tables, Data Profiles and Subject Tables. See below for a description of each\ + \ change type.\nNo Change - The variable has not changed from the prior year (most\ + \ variables).\nUpdated - That variable has changed from the prior year and a matching\ + \ variable for the current year has been found.\nNo Match - The variable has changed\ + \ from the prior year and no matching or comparable variable has been found.\n" +url: null +uuid: d6d4d49c-b157-43cb-995a-d66fa3398a72 diff --git a/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml b/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml index f66ff83..37d4d19 100644 --- a/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml +++ b/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml @@ -1,12 +1,32 @@ +datatype: dataset description: | This dataset contains current asthma prevalence, the estimated percentage of Californians who have ever been diagnosed with asthma by a health care provider AND report they still have asthma and/or had an asthma episode or attack within the past 12 months, statewide and by county. The data are stratified by age group (all ages, 0-17, 18+, 0-4, 5-17, 18-64, 65+) and reported for 2-year periods. -integration_type: dataset +img_url: null +last_updated: null name: CHIS California Asthma -prompt: | +provider: adhoc:specialist_agents +resources: + ca7ff323-7bef-489d-9d2c-760628f40825: + code: "import pandas as pd\nimport os\n\n# Define the file path using the correct\ + \ filename\nfile_path = os.path.join('{{DATASET_FILES_BASE_PATH}}', 'chis_california_asthma',\ + \ 'current-asthma-prevalence-by-county-2015_2022.csv')\n\ndf = pd.read_csv(file_path,\ + \ encoding='latin1') # latin1 encoding has worked before, not specifying it\ + \ hasn't\n\n# Check the column names to make sure we're using the right ones\n\ + print(\"Column names:\", df.columns.tolist())\n\n# Display a sample of the data\ + \ to understand its structure\nprint(\"\\nSample data:\")\nprint(df.head(2))\n\ + \ " + integration: 5aefdb54-5aa3-4c53-a7a4-d0483e4577fd + notes: null + query: Open the california chis asthma dataset and display columns/first 2 rows + to understand the structure + resource_id: ca7ff323-7bef-489d-9d2c-760628f40825 + resource_type: example +slug: adhocspecificationintegration_chis_california_asthma +source: | You have access to a file-based California CHIS Data-Current Asthma Prevalence by County 2015-2022 file, located at `${DATASET_FILES_BASE_PATH}/chis_california_asthma/current-asthma-prevalence-by-county-2015_2022.csv`. @@ -27,20 +47,5 @@ prompt: | "Agre Group" possible values are: "All Ages", "0-17 years", "18+ years", "0-4 years", "5-17 years", "18-64 years", "65+ years" You can use this API to answer questions on the current asthma prevalence in California by county and age group. -resources: - ca7ff323-7bef-489d-9d2c-760628f40825: - code: "import pandas as pd\nimport os\n\n# Define the file path using the correct\ - \ filename\nfile_path = os.path.join('{{DATASET_FILES_BASE_PATH}}', 'chis_california_asthma',\ - \ 'current-asthma-prevalence-by-county-2015_2022.csv')\n\ndf = pd.read_csv(file_path,\ - \ encoding='latin1') # latin1 encoding has worked before, not specifying it\ - \ hasn't\n\n# Check the column names to make sure we're using the right ones\n\ - print(\"Column names:\", df.columns.tolist())\n\n# Display a sample of the data\ - \ to understand its structure\nprint(\"\\nSample data:\")\nprint(df.head(2))\n\ - \ " - integration: fd660ac8-f7cc-477a-bbcb-38df4d330711 - notes: null - query: Open the california chis asthma dataset and display columns/first 2 rows - to understand the structure - resource_id: ca7ff323-7bef-489d-9d2c-760628f40825 - resource_type: example -slug: fd660ac8-f7cc-477a-bbcb-38df4d330711 +url: null +uuid: 5aefdb54-5aa3-4c53-a7a4-d0483e4577fd diff --git a/src/biome/adhoc_data/datasources/epa_tri/api.yaml b/src/biome/adhoc_data/datasources/epa_tri/api.yaml index dacfdc3..1e76445 100644 --- a/src/biome/adhoc_data/datasources/epa_tri/api.yaml +++ b/src/biome/adhoc_data/datasources/epa_tri/api.yaml @@ -1,100 +1,13 @@ +datatype: dataset description: | The Toxics Release Inventory (TRI) is a resource for learning about toxic chemical releases and pollution prevention activities reported by industrial and federal facilities. TRI data support informed decision-making by communities, government agencies, companies, and others. -integration_type: dataset +img_url: null +last_updated: null name: EPA Toxic Release Inventory (TRI) -prompt: | - You have access to a file-based EPA Toxic Release Inventory (TRI) data, from 2014 to 2023, at the `${DATASET_FILES_BASE_PATH}/epa-tri/EPA_TRI_Toxics_2014_2023.csv` file. - The report is just one file- there's a year column, and then a list of chemicals released, and the amount released. - It also contains geospatial information such as latitude, longitude, state/zip, and even EPA region. - - # Overview of the EPA TRI Dataset - TRI tracks the waste management of certain toxic chemicals that may pose a threat to human health and the environment. U.S. facilities in different industry sectors must report annually how much of each chemical they release into the environment and/or managed through recycling, energy recovery and treatment, as well as any practices implemented to prevent or reduce the generation of chemical waste. - - Contains information about toxic chemical releases reported by industrial and federal facilities; key features of this dataset include: - - Facility information: Names, IDs, geographic coordinates, and locations - - Chemical data: Chemical names and release amounts (in pounds) - - Geographic information: Latitude, longitude, state, county, EPA region, ZIP codes, and Census Block Groups - - Risk metrics: RSEI (Risk-Screening Environmental Indicators) Hazard scores - - Temporal coverage: 10 years of data (2014-2023) - - In general, chemicals covered by the TRI Program are those that cause: - - Cancer or other chronic human health effects - - Significant adverse acute human health effects - - Significant adverse environmental effects - - There are currently 799 individually listed chemicals and 33 chemical categories covered by the TRI Program. Facilities that manufacture, process or otherwise use these chemicals in amounts above established levels must submit annual reporting forms for each chemical. Note that the TRI chemical list doesn't include all toxic chemicals used in the U.S. - - The Toxics Release Inventory (TRI) is a resource for learning about toxic chemical releases and pollution prevention activities reported by industrial and federal facilities. TRI data support informed decision-making by communities, government agencies, companies, and others. - - ## Data Dictionary: - - "TRI Facility Name": Name of the facility reporting chemical releases - - "TRI Facility ID": Unique identifier for the facility - - "Year": Reporting year (2014-2023) - - "Census Block Group": Geographic identifier for census block group - - "ZIP Code": Postal code of the facility - - "City": City where the facility is located - - "County": County where the facility is located- format: "COUNTY-NAME-ONLY, STATE 2-DIGIT CODE" (value in all caps) - - "State": State where the facility is located - - "EPA Region": EPA administrative region (numbered 1-10) - - "Latitude"/"Longitude": Geographic coordinates of the facility - - "Chemical": Name of the toxic chemical being reported - - "Releases (lb)": Total amount of chemical released to the environment in pounds - - "Waste Managed (lb)": Total amount of chemical waste managed in pounds - - "RSEI Hazard": Risk-Screening Environmental Indicators score, which measures the relative hazard of chemicals - - ## Working with the Data: - 1. When loading the data, ensure you specify the index_col=False parameter, since pandas is inferring the wrong index column and shifting the data without it: - ```python - import pandas as pd - tri_data = pd.read_csv('${DATASET_FILES_BASE_PATH}/epa-tri/EPA_TRI_Toxics_2014_2023.csv', index_col=False) - ``` - - 2. Common data analysis tasks: - - Filter by year: `tri_data[tri_data['Year'] == 2023]` - - Filter by chemical: `tri_data[tri_data['Chemical'] == 'Lead']` - - Filter by state: `tri_data[tri_data['State'] == 'California']` - - Find top polluters: `tri_data.sort_values('Releases (lb)', ascending=False).head(10)` - - Analyze trends over time: `tri_data.groupby('Year')['Releases (lb)'].sum().plot()` - - 3. Geospatial analysis: - - Create a map of facilities: Use latitude/longitude with libraries like folium or geopandas - - Regional analysis: Group by EPA Region or State to compare pollution levels - - 4. Data quality considerations: - - Check for missing values: `tri_data.isnull().sum()` - - Some chemicals may have different reporting thresholds - - RSEI Hazard scores provide context on relative toxicity beyond just release amounts - - Since this dataset contains a column for EPA Region, we can use this when relating to other more anonymous data sources - that only use these properties- but do store latitude/longitude to correlate with the other data sources that have a higher resolution. - - When checking for respiratory chemicals, always include the CAS number, not just the name- either that or check that the - non-CAS-included name you use is a substring of the value in the Chemical column when trying to make a match. - Example Chemical column values: Manganese compounds (N450), Ammonia (7664-41-7), Nitrate compounds (water dissociable) (N511) - - # Web Documentation - - ## What industries are included in the TRI? - Facilities that report to TRI are typically larger facilities involved in manufacturing, metal mining, electric power generation, chemical manufacturing and hazardous waste treatment. Not all industry sectors are covered by the TRI Program, and not all facilities in covered sectors are required to report to TRI. - - The TRI Program is also different because the data it collects are: - - - annual, collected each July and made publicly available online; - - multimedia, reflecting chemical emissions to air, water and land; and - - broad, encompassing source reduction and other pollution prevention practices. - - Numerous EPA programs use TRI data and information to: - - - provide a more complete picture of environmental performance at facilities and corporations; - - identify facilities that are potentially out of compliance with regulations or operating permits; - - support technical analysis for regulation; - - improve data quality across EPA; and - - develop program priorities and projects. - - Under various environmental laws (identified in the figure below), other EPA programs also collect information about regulated chemicals. Users looking for information not available in the TRI can check the databases associated with these other programs. The Clean Air Act's National Emissions Inventory (NEI), for example, can be used to find estimates of air releases for facilities that do not report to the TRI or for mobile sources such as cars, which are not covered by the TRI. +provider: adhoc:specialist_agents resources: 125d85a4-0a2a-4306-b0d3-cc3c9451bb8e: code: "import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\n\ @@ -123,7 +36,7 @@ resources: \ plt.text(bar.get_x() + bar.get_width()/2., height + 0.5,\n f'{height:.1f}M',\ \ ha='center', va='bottom')\n\nplt.tight_layout()\nplt.savefig('top_zips_by_releases.png')\n\ plt.show()\n" - integration: f9653810-18ac-4108-a1ea-7f7c2e31274a + integration: b57b39a7-882d-4f69-93c9-c3e3dee96063 notes: null query: |- Analyzing and visualizing toxic releases by ZIP code. This example shows how to aggregate TRI data by geographic units (ZIP codes), calculate total releases and hazard scores, and create a bar chart visualization of the top areas with highest toxic releases. @@ -145,7 +58,7 @@ resources: print(f"Found {len(orange_county)} toxic release records containing 'ORANGE, FL' in the County name.") print("\nSample of the data:") print(orange_county.head()) - integration: f9653810-18ac-4108-a1ea-7f7c2e31274a + integration: b57b39a7-882d-4f69-93c9-c3e3dee96063 notes: null query: Open the TRI data file with index_col=False, since otherwise columns are shifted, and find data for Orange County, FL. @@ -200,7 +113,7 @@ resources: ]] result - integration: f9653810-18ac-4108-a1ea-7f7c2e31274a + integration: b57b39a7-882d-4f69-93c9-c3e3dee96063 notes: null query: |- Capture any asthma-respiratory related hazard incident from the TRI- EPA's Toxic Release Inventory data- you have access to this api. FOr now focus on problems in 2022, in the county of Harris, TX. @@ -242,7 +155,7 @@ resources: \ in Toxic Releases')\nplt.xlabel('Year')\nplt.ylabel('Releases (Million lb)')\n\ plt.legend()\nplt.grid(True, linestyle='--', alpha=0.7)\nplt.tight_layout()\n\ plt.savefig('yearly_condition_trends.png')\nplt.show()\n" - integration: f9653810-18ac-4108-a1ea-7f7c2e31274a + integration: b57b39a7-882d-4f69-93c9-c3e3dee96063 notes: null query: |- Identifying and analyzing chemicals related to specific health conditions. This example demonstrates how to filter TRI data for chemicals associated with a particular health condition (like asthma), calculate their proportion of total releases, and analyze trends over time. This approach can be adapted for various health conditions by modifying the list of relevant chemicals. @@ -282,7 +195,7 @@ resources: \ fill_opacity=0.6,\n opacity=0.8\n ).add_to(marker_cluster)\n\ \n# Save the map\nmap_file = 'facilities_map.html'\nm.save(map_file)\nprint(f\"\ Interactive map saved to: {map_file}\")\n" - integration: f9653810-18ac-4108-a1ea-7f7c2e31274a + integration: b57b39a7-882d-4f69-93c9-c3e3dee96063 notes: null query: |- Creating an interactive map of toxic release facilities. This example shows how to use the folium library to generate an interactive web map displaying the locations of facilities, with circle sizes proportional to release amounts and popup information showing detailed facility data. This visualization helps identify spatial patterns and hotspots of toxic releases. @@ -328,7 +241,7 @@ resources: ]] print(result) - integration: f9653810-18ac-4108-a1ea-7f7c2e31274a + integration: b57b39a7-882d-4f69-93c9-c3e3dee96063 notes: null query: |- Query to retrieve asthma-respiratory related hazard incidents from the EPA's TRI for the year 2022 in Harris County, TX. @@ -365,7 +278,7 @@ resources: \ fraction', \n fontsize=12, bbox=dict(boxstyle=\"round,pad=0.3\"\ , fc=\"white\", ec=\"gray\", alpha=0.8))\n\nplt.tight_layout()\nplt.savefig('releases_vs_hazard.png')\n\ plt.show()" - integration: f9653810-18ac-4108-a1ea-7f7c2e31274a + integration: b57b39a7-882d-4f69-93c9-c3e3dee96063 notes: null query: |- Analyzing the relationship between release amounts and hazard scores. This example demonstrates how to investigate the correlation between the quantity of toxic releases and their associated RSEI Hazard scores using scatter plots, trend lines, and correlation statistics. This analysis helps understand whether larger releases necessarily correspond to higher health risks. @@ -393,7 +306,7 @@ resources: \ Total Releases:\")\nfor i, (name, lat, lon, releases, hazard, chemicals, years)\ \ in enumerate(facility_summary.head(10).values, 1):\n print(f\"{i}. {name}:\ \ {releases:,.2f} lb, RSEI Hazard: {hazard:,.2f}, Chemicals: {chemicals}\")\n" - integration: f9653810-18ac-4108-a1ea-7f7c2e31274a + integration: b57b39a7-882d-4f69-93c9-c3e3dee96063 notes: null query: |- Basic loading, cleaning, and summarizing of EPA TRI data for a specific county. This example demonstrates how to handle the comma-separated numeric values in the dataset, filter for a specific geographic area, and create a summary of facilities with their total releases, hazard scores, and chemical counts. @@ -446,10 +359,102 @@ resources: result = result.sort_values('Releases (lb)', ascending=False) print("\nResults:") print(result.head()) - integration: f9653810-18ac-4108-a1ea-7f7c2e31274a + integration: b57b39a7-882d-4f69-93c9-c3e3dee96063 notes: null query: |- When not finding results for a combined query, try a more systematic approach by finding items and counting length by one by one, in order to discover if there is matching data, but the query is not returning any results. This is to discover amonia references for the county of 'HARRIS, TX'. resource_id: e2bd90f9-24a6-41e5-a3ba-1aca807d8990 resource_type: example -slug: f9653810-18ac-4108-a1ea-7f7c2e31274a +slug: adhocspecificationintegration_epa_toxic_release_inventory__tri_ +source: | + You have access to a file-based EPA Toxic Release Inventory (TRI) data, from 2014 to 2023, at the `${DATASET_FILES_BASE_PATH}/epa-tri/EPA_TRI_Toxics_2014_2023.csv` file. + The report is just one file- there's a year column, and then a list of chemicals released, and the amount released. + It also contains geospatial information such as latitude, longitude, state/zip, and even EPA region. + + # Overview of the EPA TRI Dataset + TRI tracks the waste management of certain toxic chemicals that may pose a threat to human health and the environment. U.S. facilities in different industry sectors must report annually how much of each chemical they release into the environment and/or managed through recycling, energy recovery and treatment, as well as any practices implemented to prevent or reduce the generation of chemical waste. + + Contains information about toxic chemical releases reported by industrial and federal facilities; key features of this dataset include: + - Facility information: Names, IDs, geographic coordinates, and locations + - Chemical data: Chemical names and release amounts (in pounds) + - Geographic information: Latitude, longitude, state, county, EPA region, ZIP codes, and Census Block Groups + - Risk metrics: RSEI (Risk-Screening Environmental Indicators) Hazard scores + - Temporal coverage: 10 years of data (2014-2023) + + In general, chemicals covered by the TRI Program are those that cause: + - Cancer or other chronic human health effects + - Significant adverse acute human health effects + - Significant adverse environmental effects + + There are currently 799 individually listed chemicals and 33 chemical categories covered by the TRI Program. Facilities that manufacture, process or otherwise use these chemicals in amounts above established levels must submit annual reporting forms for each chemical. Note that the TRI chemical list doesn't include all toxic chemicals used in the U.S. + + The Toxics Release Inventory (TRI) is a resource for learning about toxic chemical releases and pollution prevention activities reported by industrial and federal facilities. TRI data support informed decision-making by communities, government agencies, companies, and others. + + ## Data Dictionary: + - "TRI Facility Name": Name of the facility reporting chemical releases + - "TRI Facility ID": Unique identifier for the facility + - "Year": Reporting year (2014-2023) + - "Census Block Group": Geographic identifier for census block group + - "ZIP Code": Postal code of the facility + - "City": City where the facility is located + - "County": County where the facility is located- format: "COUNTY-NAME-ONLY, STATE 2-DIGIT CODE" (value in all caps) + - "State": State where the facility is located + - "EPA Region": EPA administrative region (numbered 1-10) + - "Latitude"/"Longitude": Geographic coordinates of the facility + - "Chemical": Name of the toxic chemical being reported + - "Releases (lb)": Total amount of chemical released to the environment in pounds + - "Waste Managed (lb)": Total amount of chemical waste managed in pounds + - "RSEI Hazard": Risk-Screening Environmental Indicators score, which measures the relative hazard of chemicals + + ## Working with the Data: + 1. When loading the data, ensure you specify the index_col=False parameter, since pandas is inferring the wrong index column and shifting the data without it: + ```python + import pandas as pd + tri_data = pd.read_csv('${DATASET_FILES_BASE_PATH}/epa-tri/EPA_TRI_Toxics_2014_2023.csv', index_col=False) + ``` + + 2. Common data analysis tasks: + - Filter by year: `tri_data[tri_data['Year'] == 2023]` + - Filter by chemical: `tri_data[tri_data['Chemical'] == 'Lead']` + - Filter by state: `tri_data[tri_data['State'] == 'California']` + - Find top polluters: `tri_data.sort_values('Releases (lb)', ascending=False).head(10)` + - Analyze trends over time: `tri_data.groupby('Year')['Releases (lb)'].sum().plot()` + + 3. Geospatial analysis: + - Create a map of facilities: Use latitude/longitude with libraries like folium or geopandas + - Regional analysis: Group by EPA Region or State to compare pollution levels + + 4. Data quality considerations: + - Check for missing values: `tri_data.isnull().sum()` + - Some chemicals may have different reporting thresholds + - RSEI Hazard scores provide context on relative toxicity beyond just release amounts + + Since this dataset contains a column for EPA Region, we can use this when relating to other more anonymous data sources + that only use these properties- but do store latitude/longitude to correlate with the other data sources that have a higher resolution. + + When checking for respiratory chemicals, always include the CAS number, not just the name- either that or check that the + non-CAS-included name you use is a substring of the value in the Chemical column when trying to make a match. + Example Chemical column values: Manganese compounds (N450), Ammonia (7664-41-7), Nitrate compounds (water dissociable) (N511) + + # Web Documentation + + ## What industries are included in the TRI? + Facilities that report to TRI are typically larger facilities involved in manufacturing, metal mining, electric power generation, chemical manufacturing and hazardous waste treatment. Not all industry sectors are covered by the TRI Program, and not all facilities in covered sectors are required to report to TRI. + + The TRI Program is also different because the data it collects are: + + - annual, collected each July and made publicly available online; + - multimedia, reflecting chemical emissions to air, water and land; and + - broad, encompassing source reduction and other pollution prevention practices. + + Numerous EPA programs use TRI data and information to: + + - provide a more complete picture of environmental performance at facilities and corporations; + - identify facilities that are potentially out of compliance with regulations or operating permits; + - support technical analysis for regulation; + - improve data quality across EPA; and + - develop program priorities and projects. + + Under various environmental laws (identified in the figure below), other EPA programs also collect information about regulated chemicals. Users looking for information not available in the TRI can check the databases associated with these other programs. The Clean Air Act's National Emissions Inventory (NEI), for example, can be used to find estimates of air releases for facilities that do not report to the TRI or for mobile sources such as cars, which are not covered by the TRI. +url: null +uuid: b57b39a7-882d-4f69-93c9-c3e3dee96063 diff --git a/src/biome/adhoc_data/datasources/faers/api.yaml b/src/biome/adhoc_data/datasources/faers/api.yaml index 5ec76d7..a62bb39 100644 --- a/src/biome/adhoc_data/datasources/faers/api.yaml +++ b/src/biome/adhoc_data/datasources/faers/api.yaml @@ -1,25 +1,17 @@ +datatype: api description: | Drug Adverse Event Overview The openFDA drug adverse event API returns data that has been collected from the FDA Adverse Event Reporting System (FAERS), a database that contains information on adverse event and medication error reports submitted to FDA. -integration_type: api +img_url: null +last_updated: null name: FDA drug adverse event FAERS API -prompt: | - ${web_documentation} - - # Additional Instructions: - - You will need an API Key to use any openFDA APIs. It is available in the environment variable: - - API Key: os.environ.get("API_OPENFDA") - Do not use placeholder auth values from the OpenAPI spec nor prompt the user for these values, just use the environment variable. - - # Searchable Fields Reference: - ${fields_reference} +provider: adhoc:specialist_agents resources: 66215565-2562-455a-aedc-80cdb06f6c1a: filepath: faers_fields_reference.csv - integration: da3ecb53-dfc6-467d-a70d-43ab287e8254 + integration: baf45232-38d9-401b-b1e5-8c5621917442 name: fields_reference resource_id: 66215565-2562-455a-aedc-80cdb06f6c1a resource_type: file @@ -56,7 +48,7 @@ resources: top_med = df.iloc[0] print(f"\nMedication with highest number of adverse events:") print(f"{top_med['term']}: {top_med['count']} reports") - integration: da3ecb53-dfc6-467d-a70d-43ab287e8254 + integration: baf45232-38d9-401b-b1e5-8c5621917442 notes: null query: Can you tell me which medication have the highest adverse effects reported from all of the available? @@ -64,8 +56,21 @@ resources: resource_type: example f9afc1aa-371f-4318-9fb1-cd5030db39fe: filepath: faers_web.md - integration: da3ecb53-dfc6-467d-a70d-43ab287e8254 + integration: baf45232-38d9-401b-b1e5-8c5621917442 name: web_documentation resource_id: f9afc1aa-371f-4318-9fb1-cd5030db39fe resource_type: file -slug: da3ecb53-dfc6-467d-a70d-43ab287e8254 +slug: adhocspecificationintegration_fda_drug_adverse_event_faers_api +source: | + ${web_documentation} + + # Additional Instructions: + + You will need an API Key to use any openFDA APIs. It is available in the environment variable: + - API Key: os.environ.get("API_OPENFDA") + Do not use placeholder auth values from the OpenAPI spec nor prompt the user for these values, just use the environment variable. + + # Searchable Fields Reference: + ${fields_reference} +url: null +uuid: baf45232-38d9-401b-b1e5-8c5621917442 diff --git a/src/biome/adhoc_data/datasources/gdc/api.yaml b/src/biome/adhoc_data/datasources/gdc/api.yaml index 085d7c9..a4554c4 100644 --- a/src/biome/adhoc_data/datasources/gdc/api.yaml +++ b/src/biome/adhoc_data/datasources/gdc/api.yaml @@ -1,58 +1,17 @@ +datatype: api description: | The NCI's Genomic Data Commons (GDC) provides the cancer research community with a repository and computational platform for cancer researchers who need to understand cancer, its clinical progression, and response to therapy. The GDC supports several cancer genome programs at the NCI Center for Cancer Genomics (CCG), including The Cancer Genome Atlas (TCGA) and Therapeutically Applicable Research to Generate Effective Treatments (TARGET). -integration_type: api +img_url: null +last_updated: null name: Genomics Data Commons -prompt: | - # GDC API Documentation - - Below you will find the GDC API documentation. Note that - there are several endpoints available--some are discussed in the `Search and Retrieval` - section, and others are discussed in the `Data Analysis` section. - - When a user is searching for cases related to gene mutations you should use the `/ssm_occurrences` endpoint. - - ${raw_documentation} - - # Additional Instructions: - - You will be given python code to query the GDC API. - - When querying GDC, change the `format` from "TSV" to "JSON" - - If you download a file by file-id, the output will be TSV. - Save the file with the proper extension. - - When filtering you should rely on the fields provided for a given endpoint in the following mapping. - This mapping is a dictionary where the keys are the endpoints and the values are the fields that are available for that endpoint. - - ``` - ${mappings} - ``` - - These are the fields that are available for filtering and also the fields that can be returned. You can use - the facets to filter the fields that are available for a given endpoint. - - A list of fields and their respective choices/facets are as follows: - - ``` - ${facets} - ``` - - Note this list of facets may be incomplete as it changes over time. Remember to consult it - when you are filtering. If you are unsure about options available for a given field, you should consider - just using the wildcard operator. For example, if you are looking for cases for "paragangliomas and glomus tumors" - and getting no results, you should consider just using `*paragangliomas*`. Be sure to let the user know that you are using - the wildcard operator so it might not be as specific as they would like. - - If a user asks you to filter based on the type of cancer, it is often productive to filter on - disease type, primary diagnosis, or primary site. +provider: adhoc:specialist_agents resources: 4b9845c7-d61b-440d-880e-b3d2b315a415: filepath: facets.txt - integration: 7f4da82e-9df8-4235-82cc-00dcbceea9e8 + integration: 39771f58-c791-43e8-a19d-c2095c338567 name: facets resource_id: 4b9845c7-d61b-440d-880e-b3d2b315a415 resource_type: file @@ -102,7 +61,7 @@ resources: print(f"There are {len(all_cases_df)} cases in GDC in total") # Display the DataFrame all_cases_df.head() - integration: 7f4da82e-9df8-4235-82cc-00dcbceea9e8 + integration: 39771f58-c791-43e8-a19d-c2095c338567 notes: null query: |- Get all cases from GDC with no filters. Fetch only the case ids and paginate through the results to fetch all available cases. Store the results as a dataframe @@ -167,7 +126,7 @@ resources: ssms = pd.DataFrame(json_normalize(all_ssms)) ssms.head() - integration: 7f4da82e-9df8-4235-82cc-00dcbceea9e8 + integration: 39771f58-c791-43e8-a19d-c2095c338567 notes: null query: Query the GDC for cases of myeloid leukemia with a JAK2 somatic mutation and return the results as a pandas dataframe. @@ -175,7 +134,7 @@ resources: resource_type: example 74735957-03a3-4706-aa40-f8bf200553cc: filepath: gdc.md - integration: 7f4da82e-9df8-4235-82cc-00dcbceea9e8 + integration: 39771f58-c791-43e8-a19d-c2095c338567 name: raw_documentation resource_id: 74735957-03a3-4706-aa40-f8bf200553cc resource_type: file @@ -209,7 +168,7 @@ resources: \ in lung cancer cases for males under 45 who never smoked:\")\n lung_mutation_df\ \ = pd.DataFrame(json_normalize(mutations))\nelse:\n print(f\"Error: {response.status_code}\ \ - {response.text}\")\n \nlung_mutation_df.head()" - integration: 7f4da82e-9df8-4235-82cc-00dcbceea9e8 + integration: 39771f58-c791-43e8-a19d-c2095c338567 notes: null query: |- Get simple somatic mutation occurrences (cases) from GDC where the primary site is bronchus and lung, the gender is male, the age at diagnosis is less than 45 years old, and the patient is a lifelong non-smoker. Return the results as a pandas dataframe. @@ -284,7 +243,7 @@ resources: print(f"There are {len(cases_df)} cases in GDC for AML") # Display the DataFrame cases_df.head() - integration: 7f4da82e-9df8-4235-82cc-00dcbceea9e8 + integration: 39771f58-c791-43e8-a19d-c2095c338567 notes: null query: |- Get all cases where disease type is myeloid leukemia from GDC and return the results as a pandas dataframe. Paginate through the results to fetch all available cases @@ -292,8 +251,54 @@ resources: resource_type: example a96216e1-8a2b-4fab-90e8-565d1db62fbe: filepath: gdc_mappings.json - integration: 7f4da82e-9df8-4235-82cc-00dcbceea9e8 + integration: 39771f58-c791-43e8-a19d-c2095c338567 name: mappings resource_id: a96216e1-8a2b-4fab-90e8-565d1db62fbe resource_type: file -slug: 7f4da82e-9df8-4235-82cc-00dcbceea9e8 +slug: adhocspecificationintegration_genomics_data_commons +source: | + # GDC API Documentation + + Below you will find the GDC API documentation. Note that + there are several endpoints available--some are discussed in the `Search and Retrieval` + section, and others are discussed in the `Data Analysis` section. + + When a user is searching for cases related to gene mutations you should use the `/ssm_occurrences` endpoint. + + ${raw_documentation} + + # Additional Instructions: + + You will be given python code to query the GDC API. + + When querying GDC, change the `format` from "TSV" to "JSON" + + If you download a file by file-id, the output will be TSV. + Save the file with the proper extension. + + When filtering you should rely on the fields provided for a given endpoint in the following mapping. + This mapping is a dictionary where the keys are the endpoints and the values are the fields that are available for that endpoint. + + ``` + ${mappings} + ``` + + These are the fields that are available for filtering and also the fields that can be returned. You can use + the facets to filter the fields that are available for a given endpoint. + + A list of fields and their respective choices/facets are as follows: + + ``` + ${facets} + ``` + + Note this list of facets may be incomplete as it changes over time. Remember to consult it + when you are filtering. If you are unsure about options available for a given field, you should consider + just using the wildcard operator. For example, if you are looking for cases for "paragangliomas and glomus tumors" + and getting no results, you should consider just using `*paragangliomas*`. Be sure to let the user know that you are using + the wildcard operator so it might not be as specific as they would like. + + If a user asks you to filter based on the type of cancer, it is often productive to filter on + disease type, primary diagnosis, or primary site. +url: null +uuid: 39771f58-c791-43e8-a19d-c2095c338567 diff --git a/src/biome/adhoc_data/datasources/gras/api.yaml b/src/biome/adhoc_data/datasources/gras/api.yaml index 2548c11..993e8c4 100644 --- a/src/biome/adhoc_data/datasources/gras/api.yaml +++ b/src/biome/adhoc_data/datasources/gras/api.yaml @@ -1,38 +1,13 @@ +datatype: dataset description: | The FDA Generally Recognized as Safe (GRAS) is a csv file (export) data with notices filed from the FDA GRAS database from years 1998 to 2024. You may use this tool to check for food additives introductions or changes. Use this tool to answer questions such as: In which year was Pea fiber food additive/preservative introduced into the US food supply? -integration_type: dataset +img_url: null +last_updated: null name: FDA Generally Recognized as Safe (GRAS) -prompt: "You have access to the FDA Generally Recognized as Safe (GRAS), which is\ - \ a csv file data with notices filed from the FDA GRAS database from years 1998\ - \ to 2024.\nYou may use this to check for food additives introductions, and correlate\ - \ with other data sources for analysis.\n\n\"GRAS\" is an acronym for the phrase\ - \ Generally Recognized As Safe.\nUnder sections 201(s) and 409 of the Federal Food,\ - \ Drug, and Cosmetic Act,\nany substance that is intentionally added to food is\ - \ a food additive.\n\n# Additional Instructions:\nUse the raw csv notices file to\ - \ answer questions about GRAS substances. Load the file from:\n\n${DATASET_FILES_BASE_PATH}/fda-gras/GRASNotices.csv\n\ - \nThese are the columns available (separated by semicolons, some newlines/spaces):\n\ - GRAS Notice (GRN) No.\t; Substance; Intended Use; Basis; Notifier;\tNotifier Address;\n\ - Date of filing;\tGRN Part 1;\tGRN Part 2;\tGRN Part 3;\tGRN Part 4;\tGRN Part 5;\t\ - GRN Part 6;\tGRN Part 7;\nDate of closure; Date of correction letter;\tFDA's Letter;\t\ - Date additional correspondence;\tAdditional correspondence;\nDate additinoal correspondence\ - \ 2;\tAdditional correspondence 2; Date additional correspondence 3;\nAdditional\ - \ correspondence 3; Date additional correspondence 4; Additional correspondence\ - \ 4;\nResubmission; Resubmitted; Notes;\tRelated submission\n\nThe Gras Notice GRN\ - \ No is wrapped by spredsheet-software-like function..\nExample:\n=T(\"1\") for\ - \ \"GRAS Notice (GRN) No.\" = 1\n\nThe file which was downloaded from FDA GRAS Notices;\ - \ http://www.hfpappexternal.fda.gov/scripts/fdcc/?set=GRASNotices;\nFile last updated\ - \ by FDA on 3/3/2025.\n\nIf the \"FDA's Letter\" column contains \" FDA has no questions\"\ - \ it means the FDA did not challenge the notice, closest thing to \"approved\".\n\ - Id the \"FDA's Letter\" column contains \"FDA has questions\" or \"Pending\", or\ - \ similar, it means the FDA has questions about the notice, and the notice is not\ - \ approved yet.\nIf the \"FDA's Letter\" column contains \"Notice does not provide\ - \ a basis for a GRAS determination\"- it means it challenged the notice, and the\ - \ notice was not approved.\nUse the stats of the \"FDA's Letter\" column to know\ - \ if a food additive is actually approved to be introduced into the US food supply\ - \ or not.\n" +provider: adhoc:specialist_agents resources: fd30b48e-e83b-41a2-a969-49ebc58da112: code: |- @@ -57,9 +32,39 @@ resources: print(matches[['GRAS Notice (GRN) No.', 'Substance', 'Date of filing', "FDA's Letter"]]) else: print(f"No GRAS notices found for {search_term}") - integration: 2a6f1149-8ff2-4c21-b528-cbf8b18bf30e + integration: d83c8b7e-e51d-499a-8927-79f109fe3f66 notes: null query: Is there any FDA GRAS notice for Dioctyl sodium sulfosuccinate as an additive? resource_id: fd30b48e-e83b-41a2-a969-49ebc58da112 resource_type: example -slug: 2a6f1149-8ff2-4c21-b528-cbf8b18bf30e +slug: adhocspecificationintegration_fda_generally_recognized_as_safe__gras_ +source: "You have access to the FDA Generally Recognized as Safe (GRAS), which is\ + \ a csv file data with notices filed from the FDA GRAS database from years 1998\ + \ to 2024.\nYou may use this to check for food additives introductions, and correlate\ + \ with other data sources for analysis.\n\n\"GRAS\" is an acronym for the phrase\ + \ Generally Recognized As Safe.\nUnder sections 201(s) and 409 of the Federal Food,\ + \ Drug, and Cosmetic Act,\nany substance that is intentionally added to food is\ + \ a food additive.\n\n# Additional Instructions:\nUse the raw csv notices file to\ + \ answer questions about GRAS substances. Load the file from:\n\n${DATASET_FILES_BASE_PATH}/fda-gras/GRASNotices.csv\n\ + \nThese are the columns available (separated by semicolons, some newlines/spaces):\n\ + GRAS Notice (GRN) No.\t; Substance; Intended Use; Basis; Notifier;\tNotifier Address;\n\ + Date of filing;\tGRN Part 1;\tGRN Part 2;\tGRN Part 3;\tGRN Part 4;\tGRN Part 5;\t\ + GRN Part 6;\tGRN Part 7;\nDate of closure; Date of correction letter;\tFDA's Letter;\t\ + Date additional correspondence;\tAdditional correspondence;\nDate additinoal correspondence\ + \ 2;\tAdditional correspondence 2; Date additional correspondence 3;\nAdditional\ + \ correspondence 3; Date additional correspondence 4; Additional correspondence\ + \ 4;\nResubmission; Resubmitted; Notes;\tRelated submission\n\nThe Gras Notice GRN\ + \ No is wrapped by spredsheet-software-like function..\nExample:\n=T(\"1\") for\ + \ \"GRAS Notice (GRN) No.\" = 1\n\nThe file which was downloaded from FDA GRAS Notices;\ + \ http://www.hfpappexternal.fda.gov/scripts/fdcc/?set=GRASNotices;\nFile last updated\ + \ by FDA on 3/3/2025.\n\nIf the \"FDA's Letter\" column contains \" FDA has no questions\"\ + \ it means the FDA did not challenge the notice, closest thing to \"approved\".\n\ + Id the \"FDA's Letter\" column contains \"FDA has questions\" or \"Pending\", or\ + \ similar, it means the FDA has questions about the notice, and the notice is not\ + \ approved yet.\nIf the \"FDA's Letter\" column contains \"Notice does not provide\ + \ a basis for a GRAS determination\"- it means it challenged the notice, and the\ + \ notice was not approved.\nUse the stats of the \"FDA's Letter\" column to know\ + \ if a food additive is actually approved to be introduced into the US food supply\ + \ or not.\n" +url: null +uuid: d83c8b7e-e51d-499a-8927-79f109fe3f66 diff --git a/src/biome/adhoc_data/datasources/hpa/api.yaml b/src/biome/adhoc_data/datasources/hpa/api.yaml index 4a3a5af..2279b7b 100644 --- a/src/biome/adhoc_data/datasources/hpa/api.yaml +++ b/src/biome/adhoc_data/datasources/hpa/api.yaml @@ -1,3 +1,4 @@ +datatype: api description: | The Human Protein Atlas is a Swedish-based program initiated in 2003 with the aim to map all the human proteins in cells, tissues, and organs using an integration of various omics @@ -31,15 +32,14 @@ description: | In addition the Human Protein Atlas has been appointed Global Core Biodata Resource (GCBR) by the Global Biodata Coalition. The Human Protein Atlas consortium is mainly funded by the Knut and Alice Wallenberg Foundation. -integration_type: api +img_url: null +last_updated: null name: Human Protein Atlas -prompt: '${raw_documentation} - - ' +provider: adhoc:specialist_agents resources: 04c53147-cc0a-42f5-8e1d-6427a595948b: filepath: hpa_docs.md - integration: 6f995eeb-b836-4d32-8e4a-827e49d5ccf7 + integration: 9653bf97-8668-4ce2-8c88-653f2d2b3749 name: raw_documentation resource_id: 04c53147-cc0a-42f5-8e1d-6427a595948b resource_type: file @@ -79,10 +79,15 @@ resources: print(f"Subcellular main location: {data.get('Subcellular main location', 'N/A')}") if 'Subcellular additional location' in data: print(f"Subcellular additional location: {data.get('Subcellular additional location', 'N/A')}") - integration: 6f995eeb-b836-4d32-8e4a-827e49d5ccf7 + integration: 9653bf97-8668-4ce2-8c88-653f2d2b3749 notes: null query: |- Query the Human Protein Atlas API for RNA and protein expression summary of a gene (JAK2) using its Ensembl ID. The example demonstrates how to retrieve and display tissue specificity, distribution, and subcellular localization information. resource_id: 92183e68-5e56-4487-a92f-8a1163b4e178 resource_type: example -slug: 6f995eeb-b836-4d32-8e4a-827e49d5ccf7 +slug: adhocspecificationintegration_human_protein_atlas +source: '${raw_documentation} + + ' +url: null +uuid: 9653bf97-8668-4ce2-8c88-653f2d2b3749 diff --git a/src/biome/adhoc_data/datasources/idc/api.yaml b/src/biome/adhoc_data/datasources/idc/api.yaml index d64be59..849f120 100644 --- a/src/biome/adhoc_data/datasources/idc/api.yaml +++ b/src/biome/adhoc_data/datasources/idc/api.yaml @@ -1,3 +1,4 @@ +datatype: api description: | NCI Imaging Data Commons (IDC) is a cloud-based environment containing publicly available cancer imaging data co-located with the analysis and exploration tools and resources. @@ -7,18 +8,10 @@ description: | - Commercial-friendly: >95% of the data in IDC is covered by the permissive CC-BY license, which allows commercial reuse (small subset of data is covered by the CC-NC license); each file in IDC is tagged with the license to make it easier for you to understand and follow the rules - Cloud-based: All of the data in IDC is available from both Google and AWS public buckets: fast and free to download, no out-of-cloud egress fees - Harmonized: All of the images and image-derived data in IDC is harmonized into standard DICOM representation -integration_type: api +img_url: null +last_updated: null name: Imaging Data Commons -prompt: | - ${raw_documentation} - - # Additional Instructions: - - Be sure to import and instantiate the client for the Imaging Data Commons API: - ```python - from idc_index import index - client = index.IDCClient() - ``` +provider: adhoc:specialist_agents resources: 090cdd92-54e0-4a30-82ab-a2e9c67b50c7: code: "# Query to list all studies in the 'cmb_aml' collection\nquery_all_studies\ @@ -26,7 +19,7 @@ resources: \ collection_id = 'cmb_aml'\n\"\"\"\n\ntry:\n df_all_studies = client.sql_query(query_all_studies)\n\ \ print(\"All studies in the 'cmb_aml' collection:\")\n print(df_all_studies)\n\ except Exception as e:\n print(f\"An error occurred: {str(e)}\")\n" - integration: 43858948-4b58-4312-b281-7a65e9c0d38d + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 notes: null query: List all studies for a given collection resource_id: 090cdd92-54e0-4a30-82ab-a2e9c67b50c7 @@ -63,7 +56,7 @@ resources: print("\nTotal number of records:", len(df)) except Exception as e: print(f"An error occurred: {str(e)}") - integration: 43858948-4b58-4312-b281-7a65e9c0d38d + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 notes: null query: Get slide microscopy data resource_id: 1be1fe03-4bb2-4240-a95e-96efb9c1d4d8 @@ -92,7 +85,7 @@ resources: \ 'cmb_aml')\n\"\"\"\n\ntry:\n df_study = client.sql_query(query_study)\n\ \ print(\"\\nStudy-level information:\")\n print(df_study)\nexcept Exception\ \ as e:\n print(f\"An error occurred querying study data: {str(e)}\")\n" - integration: 43858948-4b58-4312-b281-7a65e9c0d38d + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 notes: null query: Get additional information for microscopy data for a specific collection resource_id: 25d73a5f-adb2-495c-9588-8237bb39883d @@ -104,7 +97,7 @@ resources: \ LIKE '%aml%'\n\"\"\"\n\ntry:\n df_collections = client.sql_query(query)\n\ \ print(\"Collections with 'aml' in their name:\")\n print(df_collections)\n\ except Exception as e:\n print(f\"An error occurred: {str(e)}\")\n" - integration: 43858948-4b58-4312-b281-7a65e9c0d38d + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 notes: null query: Find collections in the Imaging Data Commons with 'aml' in their name, likely related to Acute Myeloid Leukemia. @@ -128,7 +121,7 @@ resources: print(df.columns.tolist()) except Exception as e: print(f"An error occurred: {str(e)}") - integration: 43858948-4b58-4312-b281-7a65e9c0d38d + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 notes: null query: Get columns available in IDC index resource_id: 49f63b7d-8e99-4e88-ac9d-86db0d831a8f @@ -157,7 +150,7 @@ resources: # Execute the download s5cmd_binary = client.s5cmdPath !{s5cmd_binary} --no-sign-request --endpoint-url https://s3.amazonaws.com run download_manifest.txt - integration: 43858948-4b58-4312-b281-7a65e9c0d38d + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 notes: null query: Download bone marrow biopsy images from the 'cmb_aml' collection for a specific study using IDC's public S3 bucket. @@ -171,7 +164,7 @@ resources: \ df_studies = client.sql_query(query)\n print(\"Studies in 'cptac_aml':\"\ )\n print(df_studies)\nexcept Exception as e:\n print(f\"An error occurred:\ \ {str(e)}\")\n" - integration: 43858948-4b58-4312-b281-7a65e9c0d38d + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 notes: null query: Fetch studies for a given collection in the Imaging Data Commons (IDC), specifically for the 'cptac_aml' collection. @@ -192,7 +185,7 @@ resources: """ df = client.sql_query(query) print(df.head()) - integration: 43858948-4b58-4312-b281-7a65e9c0d38d + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 notes: null query: Retrieve imagery URLs for the 'XR_Biopsy_BoneMarrow' study in the 'cmb_aml' collection from IDC. @@ -200,7 +193,7 @@ resources: resource_type: example a155ea7d-f42f-4546-8d4c-b1135994ea47: filepath: idc.md - integration: 43858948-4b58-4312-b281-7a65e9c0d38d + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 name: raw_documentation resource_id: a155ea7d-f42f-4546-8d4c-b1135994ea47 resource_type: file @@ -218,7 +211,7 @@ resources: plt.title('Bone Marrow Biopsy Image') plt.axis('off') plt.show() - integration: 43858948-4b58-4312-b281-7a65e9c0d38d + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 notes: null query: Basic plot of dicom image from file resource_id: bbfbce09-abfe-4565-a50b-6afc299df2d3 @@ -249,7 +242,7 @@ resources: No pixel data found in the DICOM file\")\n\nexcept Exception as e:\n print(f\"\ An error occurred: {str(e)}\")\nfinally:\n # Clean up\n if os.path.exists(local_file):\n\ \ os.remove(local_file)\n os.rmdir(temp_dir)\n" - integration: 43858948-4b58-4312-b281-7a65e9c0d38d + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 notes: null query: Download and visualize a slide microscopy image with pydicom and matplotlib resource_id: c19b78ab-29cb-4be3-8069-1ef22302dfb2 @@ -265,7 +258,7 @@ resources: \ size_mb = df['series_size_MB'].iloc[0]\n print(f\"\\nFile size:\ \ {size_mb:.2f} MB\")\n print(f\"Download URL: {url}\")\nexcept Exception\ \ as e:\n print(f\"An error occurred: {str(e)}\")\n" - integration: 43858948-4b58-4312-b281-7a65e9c0d38d + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 notes: null query: Query to get download URLs for bone marrow slides resource_id: dc0ce97c-46a2-406b-a582-a51be49367c9 @@ -285,7 +278,7 @@ resources: \ to a CSV file for further analysis\n df.to_csv('aml_pathology_data.csv',\ \ index=False)\n print(\"\\nData has been saved to 'aml_pathology_data.csv'\"\ )\n\nexcept Exception as e:\n print(f\"An error occurred: {str(e)}\") \n" - integration: 43858948-4b58-4312-b281-7a65e9c0d38d + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 notes: null query: Get slide microscopy data for a specific collection resource_id: e0279959-d257-438e-8513-fb40391082c1 @@ -339,9 +332,21 @@ resources: except Exception as e: print(f"\nError getting bucket information: {str(e)}") - integration: 43858948-4b58-4312-b281-7a65e9c0d38d + integration: b9f7983b-97b3-43c6-9b12-13b810e91ac0 notes: null query: Get list of files in an S3 bucket for a specific collection on IDC resource_id: f5ffd878-c1c9-46e9-b83c-6fd54fe975a5 resource_type: example -slug: 43858948-4b58-4312-b281-7a65e9c0d38d +slug: adhocspecificationintegration_imaging_data_commons +source: | + ${raw_documentation} + + # Additional Instructions: + + Be sure to import and instantiate the client for the Imaging Data Commons API: + ```python + from idc_index import index + client = index.IDCClient() + ``` +url: null +uuid: b9f7983b-97b3-43c6-9b12-13b810e91ac0 diff --git a/src/biome/adhoc_data/datasources/indra/api.yaml b/src/biome/adhoc_data/datasources/indra/api.yaml index 0c14457..1f4c39e 100644 --- a/src/biome/adhoc_data/datasources/indra/api.yaml +++ b/src/biome/adhoc_data/datasources/indra/api.yaml @@ -1,3 +1,4 @@ +datatype: api description: | INDRA CoGEx (Context Graph Extension) is an automatically assembled biomedical knowledge graph which integrates causal mechanisms from INDRA with non-causal contextual relations including properties, ontology, and data. @@ -10,30 +11,10 @@ description: | data scientists, and healthcare professionals to explore and analyze rich biomedical data efficiently, supporting advancements in drug discovery, clinical decision-making, and research. With its robust architecture, the INDRA CoGEx Query API facilitates the discovery of meaningful patterns and supports informed decision-making in complex biomedical scenarios. -integration_type: api +img_url: null +last_updated: null name: INDRA Context Graph Extension (CoGEx) -prompt: | - ${raw_documentation} - - # Additional Instructions: - - You should make use of the python requests library to interact with the API. - Note that the base URL of the API is 'https://discovery.indra.bio/api/' - - For example, here is an example request: - - curl -X 'POST' \ - 'https://discovery.indra.bio/api/get_go_terms_for_gene' \ - -H 'accept: application/json' \ - -H 'Content-Type: application/json' \ - -d '{ - "gene": [ - "HGNC", - "9896" - ], - "include_indirect": true - }' - ``` +provider: adhoc:specialist_agents resources: b7c4d45f-287b-47e2-aa9b-5ee5bae07353: code: | @@ -75,7 +56,7 @@ resources: print("\nShared Pathways Response:") print(shared_pathways_response.json()) - integration: c3befe4c-07ef-445b-8bfc-32debea298bf + integration: 417f20f1-4f56-4f6c-b11a-c40d84258557 notes: null query: |- Perform a gene pathway analysis for JAK2 to find associated pathways and shared pathways with other genes like STAT3, JAK3, and EGFR. @@ -136,7 +117,7 @@ resources: for data in evidence_data.values(): all_pmids.update(data['pmids']) print(f"Total unique PMIDs: {len(all_pmids)}") - integration: c3befe4c-07ef-445b-8bfc-32debea298bf + integration: 417f20f1-4f56-4f6c-b11a-c40d84258557 notes: null query: Retrieve literature evidence related to AML using MeSH term queries, including child terms, and summarize the findings. @@ -218,7 +199,7 @@ resources: print(f"Is kinase: {results['is_kinase']}") print(f"Is phosphatase: {results['is_phosphatase']}") print(f"Is transcription factor: {results['is_transcription_factor']}") - integration: c3befe4c-07ef-445b-8bfc-32debea298bf + integration: 417f20f1-4f56-4f6c-b11a-c40d84258557 notes: null query: |- This example demonstrates how to analyze the JAK2 gene using the INDRA CoGEx API to retrieve GO terms, pathway associations, and check if it is a kinase, phosphatase, or transcription factor. @@ -226,8 +207,32 @@ resources: resource_type: example f49c8aa7-727e-444e-a3a2-314a5f0d6631: filepath: indra.json - integration: c3befe4c-07ef-445b-8bfc-32debea298bf + integration: 417f20f1-4f56-4f6c-b11a-c40d84258557 name: raw_documentation resource_id: f49c8aa7-727e-444e-a3a2-314a5f0d6631 resource_type: file -slug: c3befe4c-07ef-445b-8bfc-32debea298bf +slug: adhocspecificationintegration_indra_context_graph_extension__cogex_ +source: | + ${raw_documentation} + + # Additional Instructions: + + You should make use of the python requests library to interact with the API. + Note that the base URL of the API is 'https://discovery.indra.bio/api/' + + For example, here is an example request: + + curl -X 'POST' \ + 'https://discovery.indra.bio/api/get_go_terms_for_gene' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "gene": [ + "HGNC", + "9896" + ], + "include_indirect": true + }' + ``` +url: null +uuid: 417f20f1-4f56-4f6c-b11a-c40d84258557 diff --git a/src/biome/adhoc_data/datasources/netrias/api.yaml b/src/biome/adhoc_data/datasources/netrias/api.yaml index f93ab8f..c74e752 100644 --- a/src/biome/adhoc_data/datasources/netrias/api.yaml +++ b/src/biome/adhoc_data/datasources/netrias/api.yaml @@ -1,3 +1,4 @@ +datatype: api description: | The Netrias Harmonization REST API leverages AI-driven harmonization approaches to standardize cancer-related terminologies from the Cancer Data Service (CDS), managed by the Center for @@ -13,30 +14,10 @@ description: | for more reliable data integration and analysis. The current intended application is for harmonization of terms in study metadata from cancer researchers delivering research data to the Cancer Research Data Commons. -integration_type: api +img_url: null +last_updated: null name: Netrias Harmonization API -prompt: "${raw_documentation}\n\n# Additional Instructions:\n\nThe REST API is accessible\ - \ at https://apieval.netriaslabs.cloud/.\nIt uses API Keys to track access. It is\ - \ just applied as an additional HTTP header `x-api-key: `\n\n**IMPORTANT**:\ - \ the API Key is available in the environment variable `NETRIAS_KEY`.\n\nFor CDS\ - \ standard terms, commonly referred to as Common Data Elements or CDEs, with large\ - \ numbers of permissible values,\nNetrias builds fine tuned LLMs from synthetically\ - \ generated training data. If a CDE has a small number of permissible values,\n\ - we apply in-context learning approaches where we craft a prompt to feed an LLM API\ - \ chat endpoint (e.g. OpenAI) requesting it\ngenerate harmonizations with no or\ - \ few examples. When used as a tool, this may lead to significant variations in\ - \ response\nlatency as the fine tuned models may take some time to load into a GPU,\ - \ tokenize the input, and infer the intended response.\n\nThe REST API also supports\ - \ the harmonization of terms from ontologies supporting Sage Bionetworks' data repositories\ - \ focused\non rare diseases and cancers. In a demonstration integration of the API\ - \ with Sage's Synapse chatbot application the teams\nfocused on four types of variations.\ - \ Here are some examples:\n\n1. Typos\n a. neurofibromatisis type ! \u2192 Neurofibromatosis\ - \ type 1\n b. whole exme squencing \u2192 whole exome sequencing\n\n2. Exact\ - \ synonyms\n a. NF-1 \u2192 Neurofibromatosis type 1\n b. WEX \u2192 whole\ - \ exome sequencing\n\n3. Narrow synonyms\n a. Watson Syndrome \u2192 Neurofibromatosis\ - \ type 1\n b. clinical whole-exome sequencing \u2192 whole exome sequencing\n\ - \n4. Broad synonyms\n a. NF \u2192 Neurofibromatosis type 1\n b. DNA sequencing\ - \ \u2192 whole exome sequencing\n" +provider: adhoc:specialist_agents resources: 2030d84d-840e-44a1-89d4-f027e2158a31: code: "import requests\nimport os\nimport json\nimport difflib\n\n# Get API key\ @@ -59,7 +40,7 @@ resources: \ for '{drug_term}'\")\n else:\n print(\"Could not find therapeutic_agents\ \ in DataHub standards\")\nelse:\n print(f\"Error getting standards: {standards_response.status_code}\"\ )\n print(standards_response.text)\n" - integration: 495ac407-7042-4eda-95ca-e10850e00081 + integration: 9fc4893e-ad7d-4ade-b7ab-2a70653602a6 notes: |- This example demonstrates how to harmonize a drug term that includes a non-standard suffix (e.g., "zorubicin_9012") by: 1. Retrieving the therapeutic_agents standard from the DataHub @@ -72,8 +53,32 @@ resources: resource_type: example 8242a126-8ef1-48de-aa94-e0ccc7e58a28: filepath: netrias.json - integration: 495ac407-7042-4eda-95ca-e10850e00081 + integration: 9fc4893e-ad7d-4ade-b7ab-2a70653602a6 name: raw_documentation resource_id: 8242a126-8ef1-48de-aa94-e0ccc7e58a28 resource_type: file -slug: 495ac407-7042-4eda-95ca-e10850e00081 +slug: adhocspecificationintegration_netrias_harmonization_api +source: "${raw_documentation}\n\n# Additional Instructions:\n\nThe REST API is accessible\ + \ at https://apieval.netriaslabs.cloud/.\nIt uses API Keys to track access. It is\ + \ just applied as an additional HTTP header `x-api-key: `\n\n**IMPORTANT**:\ + \ the API Key is available in the environment variable `NETRIAS_KEY`.\n\nFor CDS\ + \ standard terms, commonly referred to as Common Data Elements or CDEs, with large\ + \ numbers of permissible values,\nNetrias builds fine tuned LLMs from synthetically\ + \ generated training data. If a CDE has a small number of permissible values,\n\ + we apply in-context learning approaches where we craft a prompt to feed an LLM API\ + \ chat endpoint (e.g. OpenAI) requesting it\ngenerate harmonizations with no or\ + \ few examples. When used as a tool, this may lead to significant variations in\ + \ response\nlatency as the fine tuned models may take some time to load into a GPU,\ + \ tokenize the input, and infer the intended response.\n\nThe REST API also supports\ + \ the harmonization of terms from ontologies supporting Sage Bionetworks' data repositories\ + \ focused\non rare diseases and cancers. In a demonstration integration of the API\ + \ with Sage's Synapse chatbot application the teams\nfocused on four types of variations.\ + \ Here are some examples:\n\n1. Typos\n a. neurofibromatisis type ! \u2192 Neurofibromatosis\ + \ type 1\n b. whole exme squencing \u2192 whole exome sequencing\n\n2. Exact\ + \ synonyms\n a. NF-1 \u2192 Neurofibromatosis type 1\n b. WEX \u2192 whole\ + \ exome sequencing\n\n3. Narrow synonyms\n a. Watson Syndrome \u2192 Neurofibromatosis\ + \ type 1\n b. clinical whole-exome sequencing \u2192 whole exome sequencing\n\ + \n4. Broad synonyms\n a. NF \u2192 Neurofibromatosis type 1\n b. DNA sequencing\ + \ \u2192 whole exome sequencing\n" +url: null +uuid: 9fc4893e-ad7d-4ade-b7ab-2a70653602a6 diff --git a/src/biome/adhoc_data/datasources/nhanes_dietary/api.yaml b/src/biome/adhoc_data/datasources/nhanes_dietary/api.yaml index 5e6d2b7..412aa1f 100644 --- a/src/biome/adhoc_data/datasources/nhanes_dietary/api.yaml +++ b/src/biome/adhoc_data/datasources/nhanes_dietary/api.yaml @@ -1,9 +1,20 @@ +datatype: dataset description: | API for accessing and analyzing NHANES dietary data, including food consumption patterns and processed food analysis across different survey cycles from 1999 to 2023, with demographic information. -integration_type: dataset +img_url: null +last_updated: null name: NHANES Dietary Data -prompt: | +provider: adhoc:specialist_agents +resources: + 19e60e0b-82e2-4af2-b6f2-212287b43277: + filepath: docs.md + integration: 3b474df1-115a-4194-8395-509669f57be5 + name: web_docs + resource_id: 19e60e0b-82e2-4af2-b6f2-212287b43277 + resource_type: file +slug: adhocspecificationintegration_nhanes_dietary_data +source: | You have access to NHANES Dietary Data files from multiple survey cycles, including dietary intake and demographic information: # 1999-2000 NHANES Cycle @@ -60,11 +71,5 @@ prompt: | dietary patterns by demographic characteristics such as age, gender, race/ethnicity, and socioeconomic status. ${web_docs} -resources: - 19e60e0b-82e2-4af2-b6f2-212287b43277: - filepath: docs.md - integration: a37bcc79-b195-4ade-bd08-2d44d558facd - name: web_docs - resource_id: 19e60e0b-82e2-4af2-b6f2-212287b43277 - resource_type: file -slug: a37bcc79-b195-4ade-bd08-2d44d558facd +url: null +uuid: 3b474df1-115a-4194-8395-509669f57be5 diff --git a/src/biome/adhoc_data/datasources/nsch/api.yaml b/src/biome/adhoc_data/datasources/nsch/api.yaml index 21a51e2..bb4a444 100644 --- a/src/biome/adhoc_data/datasources/nsch/api.yaml +++ b/src/biome/adhoc_data/datasources/nsch/api.yaml @@ -1,114 +1,12 @@ +datatype: dataset description: | You have access to knowledge from the National Survey of Children's Health (NSCH) data, available every year from 2016 to 2023. This dataset API can help answer questions about children's health, healthcare access, family functioning, neighborhood characteristics, and social determinants of health for children aged 0-17 years in the United States. -integration_type: dataset +img_url: null +last_updated: null name: National Survey of Children's Health (NSCH) -prompt: | - You have access to file-based National Survey of Children's Health (NSCH) data - from 2016 to 2023, located under the `${DATASET_FILES_BASE_PATH}/census-nsch` directory. - - The NSCH files are located and named per year as follows: - - ${DATASET_FILES_BASE_PATH}/census-nsch/2023e_topical.sas7bdat - - ${DATASET_FILES_BASE_PATH}/census-nsch/2022e_topical.sas7bdat - - ${DATASET_FILES_BASE_PATH}/census-nsch/2021e_topical.sas7bdat - - ${DATASET_FILES_BASE_PATH}/census-nsch/2020e_topical.sas7bdat - - ${DATASET_FILES_BASE_PATH}/census-nsch/2019e_topical.sas7bdat - - ${DATASET_FILES_BASE_PATH}/census-nsch/2018e_topical.sas7bdat - - ${DATASET_FILES_BASE_PATH}/census-nsch/2017e_topical.sas7bdat - - ${DATASET_FILES_BASE_PATH}/census-nsch/2016e_topical.sas7bdat - - Codebook: ${DATASET_FILES_BASE_PATH}/census-nsch/nsch_dictionary_codebook.csv - - The NSCH is a national survey that collects information on the health and well-being of children ages 0-17 years - in the United States and the factors that may relate to their well-being. The survey includes data on physical and - mental health, access to quality health care, and the child's family, neighborhood, school, and social context. - - The NSCH is also designed to assess the prevalence and impact of special health care needs among children in the US and - explores the extent to which children with special health care needs (CSHCN) have medical homes, - adequate health insurance, access to needed services, and adequate care coordination. - Other topics may include functional difficulties, transition services, - shared decision-making, and satisfaction with care. - Information is collected from parents or caregivers who know about the child's health. - It also examines the physical and emotional health of children ages 0-17 years of age. - These factors include access to - and quality of - health care, family interactions, parental health, - neighborhood characteristics, as well as school and after-school experiences (example: child screen time). - - To read these SAS7BDAT files with pandas, use: - ```python - import pandas as pd - filename = "${DATASET_FILES_BASE_PATH}/census-nsch/2023e_topical.sas7bdat" - df = pd.read_sas(filename, format="sas7bdat") - ``` - - The NSCH datasets include state identifiers (FIPSST) and other geographic information that can be used - for regional analysis. Check which geographic variables are available in each year's dataset before - attempting to use them. - - When working with NSCH data, be aware that: - 1. Variable names may change slightly between survey years - 2. Many variables use coded responses (e.g. subset for examples: 1=Yes||2=No, 1 = Never served in the military||2 = Only on active duty for training in the Reserves - You'll need to use the nsch codebook to decode these variable response codes- - The codebook is available at ${DATASET_FILES_BASE_PATH}/census-nsch/nsch_dictionary_codebook.csv - You may open it with pandas and find the "Variable" name, the "Question", "Description", and its "Response Code"s, etc. - The Response Code options/alternatives per column are separated by a double pipe (||),such as "1=Yes||2=No" - In the codebook, the Question itself is under the "Question" column - - You'll see mentions of "screener" and "topical" in the code book- some screener properties are present in the topical files, - which makes it confusing, unfortunately. - - Codebook columns: - ['Variable', # => variable code/name - 'Source', # => source, a question category from "Topical", "Screener", "Operational" - 'Topic', - 'Survey Years', => available in which file years (eg if 2016, 2017), only check for that variable in the corresponding files, may not be available in files for other years - 'Response Code', => 1=Yes||2=No, etc - 'Description', => description of the question/variable - 'Question', => Question text as a human would ask it - 'Type', - 'Universe', - 'Availability', - 'Imputation Strategy', - 'User Notes'] - - Open it with pandas like so: - ``` - codebook = pd.read_csv("${DATASET_FILES_BASE_PATH}/census-nsch/nsch_dictionary_codebook.csv", index_col=False) # ensure index_col=False, otherwise the dataframe will be shifted! - ``` - - The NSCH is not an API but a collection of dataset files from 2016 to 2023 in SAS7BDAT format. These datasets provide rich information on: - - Physical and mental health conditions (asthma, allergies, ADHD, etc.) - - Access to healthcare - - Family, neighborhood, and social context - - Housing and environmental factors - - Insurance coverage and care coordination - - The geographic resolution for this dataset is at the state level- it only contains state FIPS codes, not county FIPS codes. - The FIPS Code column is called "FIPSST" and is available in all years. - - Instead of assuming a specific column or variable exists, or if you wish to match with a specific column pattern, - try to compare substrings instead if an exact match is not found. For example, if you need data related to asthma, try - to check for the 'asth' substring in the column names. Another example: children age may be under SC_AGE_YEARS (not CHILDREN_AGE), as another example. - Use the codebook, filter out variables or meanings, don't just assume a random color code column of, say, K6Q65A, exists - since it may have existed in previous or new years- the surveys change and the codes aren't obvious. - - # Key Variables for Health Conditions Analysis - ## Asthma and Allergies - - K2Q40A: Asthma diagnosis (1=Yes, 2=No) - - ALLERGIES: Allergies diagnosis (1=Yes, 2=No) - - ALLERGIES_CURR: Current allergies (1=Yes, 2=No) - ## Other Health Conditions - - K2Q31A: ADHD diagnosis - - K2Q32A: Depression diagnosis - - K2Q33A: Anxiety diagnosis - - DIABETES: Diabetes diagnosis - - AUTOIMMUNE: Autoimmune disease diagnosis - ## Housing and Environmental Factors - - K9Q41: Smoke inside home - - ACE1: Difficulty covering basics like food or housing - - FOODSIT: Food situation in household - - TENURE: Housing tenure (owned/rented) - - Environmental factors are primarily self-reported by parents/caregivers. +provider: adhoc:specialist_agents resources: 118cc8be-f400-40e0-bac7-b10745d888fb: code: "import pandas as pd\nimport numpy as np\n\n# Read the codebook\ncodebook\ @@ -128,7 +26,7 @@ resources: \ # Assuming 1 = Yes\n 'Percentage': percentages.get(1, 0)\n })\n\n\ # Create results dataframe\nresults_df = pd.DataFrame(results)\nresults_df =\ \ results_df.sort_values('Percentage', ascending=False)\n\nprint(results_df)\n" - integration: f7052ea8-d8f1-4203-b368-60fc23f02846 + integration: 92d74583-521c-45e1-b65f-535b1501996d notes: null query: Retrieve health conditions reported in the National Survey of Children's Health, including ADHD, autism, and asthma. @@ -140,7 +38,7 @@ resources: \ is shifted!\ncodebook = pd.read_csv(\"{{DATASET_FILES_BASE_PATH}}/census-nsch/nsch_dictionary_codebook.csv\"\ , index_col=False) \n# Display the first few rows\nprint(codebook.head())\n\ \n# and columns if needed\nprint(codebook.columns.tolist())\n" - integration: f7052ea8-d8f1-4203-b368-60fc23f02846 + integration: 92d74583-521c-45e1-b65f-535b1501996d notes: null query: Check the codebook columns head and available columns resource_id: 4cdaa736-1ca8-4158-8d3f-60de680da7f3 @@ -153,7 +51,7 @@ resources: existing_cols = df_2023.columns.tolist() print(existing_cols) - integration: f7052ea8-d8f1-4203-b368-60fc23f02846 + integration: 92d74583-521c-45e1-b65f-535b1501996d notes: null query: Check the existing columns in the dataset to see if the screentime variable and which geo columns are included. @@ -278,10 +176,117 @@ resources: \ 'decreased'\nprint(f\"4. Texas Trends: The childhood asthma rate in Texas\ \ {texas_trend} from {texas_data.iloc[0]['has_asthma']:.1%} in 2016 to {texas_data.iloc[-1]['has_asthma']:.1%}\ \ in 2023.\")\n" - integration: f7052ea8-d8f1-4203-b368-60fc23f02846 + integration: 92d74583-521c-45e1-b65f-535b1501996d notes: null query: |- Analyze trends in childhood asthma and environmental factors over time (2016-2023) using NSCH data. Load multiple years of data, harmonize variables across years, and analyze the relationship between asthma prevalence and environmental factors like smoke exposure, housing insecurity, and food insecurity. Include state-level comparisons and create a composite environmental risk score. Do so specifically for Texas. resource_id: e2681d9f-b1b5-441b-9deb-bd203d9de6f4 resource_type: example -slug: f7052ea8-d8f1-4203-b368-60fc23f02846 +slug: adhocspecificationintegration_national_survey_of_children_s_health__nsch_ +source: | + You have access to file-based National Survey of Children's Health (NSCH) data + from 2016 to 2023, located under the `${DATASET_FILES_BASE_PATH}/census-nsch` directory. + + The NSCH files are located and named per year as follows: + - ${DATASET_FILES_BASE_PATH}/census-nsch/2023e_topical.sas7bdat + - ${DATASET_FILES_BASE_PATH}/census-nsch/2022e_topical.sas7bdat + - ${DATASET_FILES_BASE_PATH}/census-nsch/2021e_topical.sas7bdat + - ${DATASET_FILES_BASE_PATH}/census-nsch/2020e_topical.sas7bdat + - ${DATASET_FILES_BASE_PATH}/census-nsch/2019e_topical.sas7bdat + - ${DATASET_FILES_BASE_PATH}/census-nsch/2018e_topical.sas7bdat + - ${DATASET_FILES_BASE_PATH}/census-nsch/2017e_topical.sas7bdat + - ${DATASET_FILES_BASE_PATH}/census-nsch/2016e_topical.sas7bdat + - Codebook: ${DATASET_FILES_BASE_PATH}/census-nsch/nsch_dictionary_codebook.csv + + The NSCH is a national survey that collects information on the health and well-being of children ages 0-17 years + in the United States and the factors that may relate to their well-being. The survey includes data on physical and + mental health, access to quality health care, and the child's family, neighborhood, school, and social context. + + The NSCH is also designed to assess the prevalence and impact of special health care needs among children in the US and + explores the extent to which children with special health care needs (CSHCN) have medical homes, + adequate health insurance, access to needed services, and adequate care coordination. + Other topics may include functional difficulties, transition services, + shared decision-making, and satisfaction with care. + Information is collected from parents or caregivers who know about the child's health. + It also examines the physical and emotional health of children ages 0-17 years of age. + These factors include access to - and quality of - health care, family interactions, parental health, + neighborhood characteristics, as well as school and after-school experiences (example: child screen time). + + To read these SAS7BDAT files with pandas, use: + ```python + import pandas as pd + filename = "${DATASET_FILES_BASE_PATH}/census-nsch/2023e_topical.sas7bdat" + df = pd.read_sas(filename, format="sas7bdat") + ``` + + The NSCH datasets include state identifiers (FIPSST) and other geographic information that can be used + for regional analysis. Check which geographic variables are available in each year's dataset before + attempting to use them. + + When working with NSCH data, be aware that: + 1. Variable names may change slightly between survey years + 2. Many variables use coded responses (e.g. subset for examples: 1=Yes||2=No, 1 = Never served in the military||2 = Only on active duty for training in the Reserves + You'll need to use the nsch codebook to decode these variable response codes- + The codebook is available at ${DATASET_FILES_BASE_PATH}/census-nsch/nsch_dictionary_codebook.csv + You may open it with pandas and find the "Variable" name, the "Question", "Description", and its "Response Code"s, etc. + The Response Code options/alternatives per column are separated by a double pipe (||),such as "1=Yes||2=No" + In the codebook, the Question itself is under the "Question" column + + You'll see mentions of "screener" and "topical" in the code book- some screener properties are present in the topical files, + which makes it confusing, unfortunately. + + Codebook columns: + ['Variable', # => variable code/name + 'Source', # => source, a question category from "Topical", "Screener", "Operational" + 'Topic', + 'Survey Years', => available in which file years (eg if 2016, 2017), only check for that variable in the corresponding files, may not be available in files for other years + 'Response Code', => 1=Yes||2=No, etc + 'Description', => description of the question/variable + 'Question', => Question text as a human would ask it + 'Type', + 'Universe', + 'Availability', + 'Imputation Strategy', + 'User Notes'] + + Open it with pandas like so: + ``` + codebook = pd.read_csv("${DATASET_FILES_BASE_PATH}/census-nsch/nsch_dictionary_codebook.csv", index_col=False) # ensure index_col=False, otherwise the dataframe will be shifted! + ``` + + The NSCH is not an API but a collection of dataset files from 2016 to 2023 in SAS7BDAT format. These datasets provide rich information on: + - Physical and mental health conditions (asthma, allergies, ADHD, etc.) + - Access to healthcare + - Family, neighborhood, and social context + - Housing and environmental factors + - Insurance coverage and care coordination + + The geographic resolution for this dataset is at the state level- it only contains state FIPS codes, not county FIPS codes. + The FIPS Code column is called "FIPSST" and is available in all years. + + Instead of assuming a specific column or variable exists, or if you wish to match with a specific column pattern, + try to compare substrings instead if an exact match is not found. For example, if you need data related to asthma, try + to check for the 'asth' substring in the column names. Another example: children age may be under SC_AGE_YEARS (not CHILDREN_AGE), as another example. + Use the codebook, filter out variables or meanings, don't just assume a random color code column of, say, K6Q65A, exists + since it may have existed in previous or new years- the surveys change and the codes aren't obvious. + + # Key Variables for Health Conditions Analysis + ## Asthma and Allergies + - K2Q40A: Asthma diagnosis (1=Yes, 2=No) + - ALLERGIES: Allergies diagnosis (1=Yes, 2=No) + - ALLERGIES_CURR: Current allergies (1=Yes, 2=No) + ## Other Health Conditions + - K2Q31A: ADHD diagnosis + - K2Q32A: Depression diagnosis + - K2Q33A: Anxiety diagnosis + - DIABETES: Diabetes diagnosis + - AUTOIMMUNE: Autoimmune disease diagnosis + ## Housing and Environmental Factors + - K9Q41: Smoke inside home + - ACE1: Difficulty covering basics like food or housing + - FOODSIT: Food situation in household + - TENURE: Housing tenure (owned/rented) + + Environmental factors are primarily self-reported by parents/caregivers. +url: null +uuid: 92d74583-521c-45e1-b65f-535b1501996d diff --git a/src/biome/adhoc_data/datasources/pdc/api.yaml b/src/biome/adhoc_data/datasources/pdc/api.yaml index 65b0977..3b8685d 100644 --- a/src/biome/adhoc_data/datasources/pdc/api.yaml +++ b/src/biome/adhoc_data/datasources/pdc/api.yaml @@ -1,3 +1,4 @@ +datatype: api description: "The objectives of the National Cancer Institute\u2019s Proteomic Data\ \ Commons (PDC) are (1) to make cancer-related proteomic datasets easily accessible\ \ to the public, and (2) facilitate direct multiomics integration in support of\ @@ -24,16 +25,10 @@ description: "The objectives of the National Cancer Institute\u2019s Proteomic D \ serves as a private user data store and also data submission portal.\n* Distributes\ \ controlled access data, such as the patient-specific protein fasta sequence databases,\ \ with dbGaP authorization and eRA Commons authentication.\n" -integration_type: api +img_url: null +last_updated: null name: Proteomics Data Commons -prompt: | - ${raw_documentation} - - # Additional Instructions: - - This API is through GraphQL. You will be provided the schema. - All requests go to the following URL: https://pdc.cancer.gov/graphql - Make all GraphQL requests to that URL. +provider: adhoc:specialist_agents resources: 07112af0-449d-4d88-b914-acba9f2ae57b: code: | @@ -74,7 +69,7 @@ resources: print(json.dumps(file_info, indent=2)) else: print(f"Error: {response.status_code} - {response.text}") - integration: bcaf77af-17e1-4811-9520-64d48e198988 + integration: 021899d9-96c0-4b2e-9d18-9c3fa3c0251c notes: null query: |- Fetch raw mass spec data for a specific study using study name, data category, and file type in the Proteomics Data Commons (PDC). @@ -128,7 +123,7 @@ resources: print(json.dumps(case_info, indent=2)) else: print(f"Error: {response.status_code} - {response.text}") - integration: bcaf77af-17e1-4811-9520-64d48e198988 + integration: 021899d9-96c0-4b2e-9d18-9c3fa3c0251c notes: null query: Fetch detailed information about a case using its case ID in the Proteomics Data Commons (PDC). @@ -172,7 +167,7 @@ resources: print(json.dumps(files, indent=2)) else: print(f"Error: {response.status_code} - {response.text}") - integration: bcaf77af-17e1-4811-9520-64d48e198988 + integration: 021899d9-96c0-4b2e-9d18-9c3fa3c0251c notes: null query: |- This example demonstrates how to query the Proteomics Data Commons for studies with open standard data for processed mass spec using the getPaginatedUIFile query. @@ -236,7 +231,7 @@ resources: print(f"Error decoding JSON response: {e}") print("Response content:", response_stat5a.text) print("Response content:", response_stat5b.text) - integration: bcaf77af-17e1-4811-9520-64d48e198988 + integration: 021899d9-96c0-4b2e-9d18-9c3fa3c0251c notes: null query: |- Retrieve quantitative mass spectrometry data for STAT5A and STAT5B proteins using the getPaginatedUIGeneAliquotSpectralCount query. @@ -244,7 +239,7 @@ resources: resource_type: example 75e500f5-31c0-4683-995b-26fc49d51426: filepath: pdc_schema.graphql - integration: bcaf77af-17e1-4811-9520-64d48e198988 + integration: 021899d9-96c0-4b2e-9d18-9c3fa3c0251c name: raw_documentation resource_id: 75e500f5-31c0-4683-995b-26fc49d51426 resource_type: file @@ -325,7 +320,7 @@ resources: except json.JSONDecodeError as e: print(f"Error decoding JSON response: {e}") print("Response content:", response.text) - integration: bcaf77af-17e1-4811-9520-64d48e198988 + integration: 021899d9-96c0-4b2e-9d18-9c3fa3c0251c notes: null query: Load samples into a pandas dataframe resource_id: 8708cae9-d62e-468d-9d78-3f61ac1913d6 @@ -402,7 +397,7 @@ resources: except json.JSONDecodeError as e: print(f"Error decoding JSON response: {e}") print("Response content:", response.text) - integration: bcaf77af-17e1-4811-9520-64d48e198988 + integration: 021899d9-96c0-4b2e-9d18-9c3fa3c0251c notes: null query: Fetch all available metadata for a study using its PDC Study ID. resource_id: 893e05e7-c0db-4482-aebe-1b09c9abfd4d @@ -472,7 +467,7 @@ resources: # Print the number of studies found and the first 5 studies print(f"Found {len(lung_cancer_studies)} studies with primary site 'Lung' or 'Bronchus and lung'.") print(list(lung_cancer_studies)[:5]) - integration: bcaf77af-17e1-4811-9520-64d48e198988 + integration: 021899d9-96c0-4b2e-9d18-9c3fa3c0251c notes: null query: |- This example demonstrates how to find lung cancer studies in the Proteomics Data Commons by querying for studies with primary sites 'Lung' and 'Bronchus and lung', and combining the results. @@ -514,7 +509,7 @@ resources: except requests.exceptions.RequestException as e: print(f"An error occurred: {e}") - integration: bcaf77af-17e1-4811-9520-64d48e198988 + integration: 021899d9-96c0-4b2e-9d18-9c3fa3c0251c notes: null query: Query PDC to find studies with a specific disease type (in this case 'Acute Myeloid Leukemia') @@ -586,7 +581,7 @@ resources: # Display the first few studies for study in list(lung_cancer_studies)[:5]: print(study) - integration: bcaf77af-17e1-4811-9520-64d48e198988 + integration: 021899d9-96c0-4b2e-9d18-9c3fa3c0251c notes: null query: |- Query the Proteomics Data Commons (PDC) to find studies with primary site 'Lung' or 'Bronchus and lung' using separate queries and combining results. @@ -630,7 +625,7 @@ resources: print(json.dumps(file_info, indent=2)) else: print(f"Error: {response.status_code} - {response.text}") - integration: bcaf77af-17e1-4811-9520-64d48e198988 + integration: 021899d9-96c0-4b2e-9d18-9c3fa3c0251c notes: null query: Retrieve open standard data files for a specific study using the Proteomics Data Commons (PDC) API. @@ -706,7 +701,7 @@ resources: except json.JSONDecodeError as e: print(f"Error decoding JSON response: {e}") print("Response content:", response.text) - integration: bcaf77af-17e1-4811-9520-64d48e198988 + integration: 021899d9-96c0-4b2e-9d18-9c3fa3c0251c notes: null query: obtain quantitative data resource_id: c68f970e-4d39-4979-bcf2-43a823490773 @@ -744,7 +739,7 @@ resources: print(json.dumps(cases_info, indent=2)) else: print(f"Error: {response.status_code} - {response.text}") - integration: bcaf77af-17e1-4811-9520-64d48e198988 + integration: 021899d9-96c0-4b2e-9d18-9c3fa3c0251c notes: null query: Fetch all cases based on a study name resource_id: d119d4cc-347e-40bc-b3d3-f17f6e1ee16e @@ -790,7 +785,7 @@ resources: except json.JSONDecodeError as e: print(f"Error decoding JSON response: {e}") print("Response content:", response.text) - integration: bcaf77af-17e1-4811-9520-64d48e198988 + integration: 021899d9-96c0-4b2e-9d18-9c3fa3c0251c notes: null query: obtain quantitative mass spec data on a specific protein (STAT5) resource_id: d2fde53a-0ba6-4c00-acaf-e27d4218480b @@ -832,7 +827,7 @@ resources: print(f"Signed URL for the file: {signed_url}") else: print(f"Error: {response.status_code} - {response.text}") - integration: bcaf77af-17e1-4811-9520-64d48e198988 + integration: 021899d9-96c0-4b2e-9d18-9c3fa3c0251c notes: null query: |- Retrieve a signed URL for downloading a specific file (e.g. raw mass spec file) in a study using the Proteomics Data Commons (PDC) API. @@ -868,10 +863,20 @@ resources: r')\n pdc_cases_in_gdc.append(case)\n\nprint(f\"Found {len(pdc_cases_in_gdc)}\ \ cases that have external references to GDC.\")\n\npdc_in_gdc_df = pd.DataFrame(json_normalize(pdc_cases_in_gdc))\n\ pdc_in_gdc_df.head()" - integration: bcaf77af-17e1-4811-9520-64d48e198988 + integration: 021899d9-96c0-4b2e-9d18-9c3fa3c0251c notes: null query: |- Get all clinical data (cases) from PDC and filter for those that have external references to GDC. Return the results as a pandas dataframe. resource_id: ed1fa5df-2d51-48ef-871b-b09abc5defc9 resource_type: example -slug: bcaf77af-17e1-4811-9520-64d48e198988 +slug: adhocspecificationintegration_proteomics_data_commons +source: | + ${raw_documentation} + + # Additional Instructions: + + This API is through GraphQL. You will be provided the schema. + All requests go to the following URL: https://pdc.cancer.gov/graphql + Make all GraphQL requests to that URL. +url: null +uuid: 021899d9-96c0-4b2e-9d18-9c3fa3c0251c diff --git a/src/biome/adhoc_data/datasources/synapse/api.yaml b/src/biome/adhoc_data/datasources/synapse/api.yaml index 2d55848..ce6e0a5 100644 --- a/src/biome/adhoc_data/datasources/synapse/api.yaml +++ b/src/biome/adhoc_data/datasources/synapse/api.yaml @@ -1,10 +1,43 @@ +datatype: api description: | The NF Synapse API is a RESTful API that allows you to interact with the Neurofibromatosis Data Portal. It additionally contains a Python client that can be used to interact with the API (on the package `synapseclient`, already installed in the environment). The code drafter has access both to the openapi spec, the web docs, and the python client documentation. -integration_type: api +img_url: null +last_updated: null name: NF Synapse -prompt: | +provider: adhoc:specialist_agents +resources: + 45324841-c013-4942-8b5f-db85856d7e48: + filepath: web_docs.md + integration: 68a50df6-fa9c-4ef4-af32-0c2502c28b7b + name: web_docs + resource_id: 45324841-c013-4942-8b5f-db85856d7e48 + resource_type: file + 8d29393b-bc38-441f-aee1-88b60d1322b6: + filepath: SynapseOpenApiSpec.json + integration: 68a50df6-fa9c-4ef4-af32-0c2502c28b7b + name: raw_documentation + resource_id: 8d29393b-bc38-441f-aee1-88b60d1322b6 + resource_type: file + 920140d9-ce8a-4cef-ad8c-418146706556: + code: "```\nimport synapseclient, os\nfrom pathlib import Path\n\ndownloads_dir\ + \ = Path(__file__).parent.parent / \"downloads\" / \"synapse\"\n\nsyn = synapseclient.Synapse(cache_root_dir=downloads_dir)\ + \ \nsyn.login(authToken=os.environ.get(\"API_SYNAPSE\")) # pass in the authToken\ + \ on python client login!\n```" + integration: 68a50df6-fa9c-4ef4-af32-0c2502c28b7b + notes: null + query: Login to the Synapse API using the python client + resource_id: 920140d9-ce8a-4cef-ad8c-418146706556 + resource_type: example + b9801ff2-37bb-44d0-9f47-9ce3d18804b5: + filepath: nf_programmatic_webdocs.md + integration: 68a50df6-fa9c-4ef4-af32-0c2502c28b7b + name: python_client + resource_id: b9801ff2-37bb-44d0-9f47-9ce3d18804b5 + resource_type: file +slug: adhocspecificationintegration_nf_synapse +source: | # OpenAPI Spec JSON: ${raw_documentation} @@ -77,33 +110,5 @@ prompt: | syn = synapseclient.Synapse(cache_root_dir=downloads_dir) syn.login(authToken=os.environ.get("API_SYNAPSE")) # pass in the authToken on python client login! ``` -resources: - 45324841-c013-4942-8b5f-db85856d7e48: - filepath: web_docs.md - integration: c6b31ee9-05d7-48fe-a9b3-6840df604e9c - name: web_docs - resource_id: 45324841-c013-4942-8b5f-db85856d7e48 - resource_type: file - 8d29393b-bc38-441f-aee1-88b60d1322b6: - filepath: SynapseOpenApiSpec.json - integration: c6b31ee9-05d7-48fe-a9b3-6840df604e9c - name: raw_documentation - resource_id: 8d29393b-bc38-441f-aee1-88b60d1322b6 - resource_type: file - 920140d9-ce8a-4cef-ad8c-418146706556: - code: "```\nimport synapseclient, os\nfrom pathlib import Path\n\ndownloads_dir\ - \ = Path(__file__).parent.parent / \"downloads\" / \"synapse\"\n\nsyn = synapseclient.Synapse(cache_root_dir=downloads_dir)\ - \ \nsyn.login(authToken=os.environ.get(\"API_SYNAPSE\")) # pass in the authToken\ - \ on python client login!\n```" - integration: c6b31ee9-05d7-48fe-a9b3-6840df604e9c - notes: null - query: Login to the Synapse API using the python client - resource_id: 920140d9-ce8a-4cef-ad8c-418146706556 - resource_type: example - b9801ff2-37bb-44d0-9f47-9ce3d18804b5: - filepath: nf_programmatic_webdocs.md - integration: c6b31ee9-05d7-48fe-a9b3-6840df604e9c - name: python_client - resource_id: b9801ff2-37bb-44d0-9f47-9ce3d18804b5 - resource_type: file -slug: c6b31ee9-05d7-48fe-a9b3-6840df604e9c +url: null +uuid: 68a50df6-fa9c-4ef4-af32-0c2502c28b7b diff --git a/src/biome/adhoc_data/datasources/usda_foodcentral/api.yaml b/src/biome/adhoc_data/datasources/usda_foodcentral/api.yaml index 3b0a732..da03f8e 100644 --- a/src/biome/adhoc_data/datasources/usda_foodcentral/api.yaml +++ b/src/biome/adhoc_data/datasources/usda_foodcentral/api.yaml @@ -1,9 +1,46 @@ +datatype: api description: | This openapi is the primary place to obtain data from the USDA's FoodData Central (nutrition) database. It is intended to assist in incorporating nutrient data into any analysis we need. -integration_type: api +img_url: null +last_updated: null name: USDA FoodData Central API -prompt: | +provider: adhoc:specialist_agents +resources: + 0e410001-bac7-4cb7-9f32-b9064c64a4c1: + filepath: foodcentral_openapi.json + integration: d5e56bdc-4ac0-4ad6-aebe-77a9096081ff + name: raw_documentation + resource_id: 0e410001-bac7-4cb7-9f32-b9064c64a4c1 + resource_type: file + 4164944f-a6d4-452c-a290-3ab7bd2e9d9e: + code: |- + import requests + import os + + # Define the API endpoint and parameters + url = 'https://api.nal.usda.gov/fdc/v1/foods/search' + + api_key = os.environ.get("API_USDA_FDC") + + params = {'query': 'raw acerola juice', 'api_key': api_key} + + # Make the request to the USDA FoodData Central API + response = requests.get(url, params=params) + + # Check if the request was successful + if response.status_code == 200: + data = response.json() + # Extract the first food item from the results + food_item = data['foods'][0] + food_item + integration: d5e56bdc-4ac0-4ad6-aebe-77a9096081ff + notes: null + query: Search for nutrients for raw acerola juice + resource_id: 4164944f-a6d4-452c-a290-3ab7bd2e9d9e + resource_type: example +slug: adhocspecificationintegration_usda_fooddata_central_api +source: | This API is the primary place to obtain data from the USDA's FoodData Central database. # OpenAPI Spec JSON: @@ -71,37 +108,5 @@ prompt: | ### SR Legacy Historic data on food components including nutrients derived from analyses, calculations, and published literature -resources: - 0e410001-bac7-4cb7-9f32-b9064c64a4c1: - filepath: foodcentral_openapi.json - integration: a77cc48a-579a-4145-b1bd-80ad50d482ec - name: raw_documentation - resource_id: 0e410001-bac7-4cb7-9f32-b9064c64a4c1 - resource_type: file - 4164944f-a6d4-452c-a290-3ab7bd2e9d9e: - code: |- - import requests - import os - - # Define the API endpoint and parameters - url = 'https://api.nal.usda.gov/fdc/v1/foods/search' - - api_key = os.environ.get("API_USDA_FDC") - - params = {'query': 'raw acerola juice', 'api_key': api_key} - - # Make the request to the USDA FoodData Central API - response = requests.get(url, params=params) - - # Check if the request was successful - if response.status_code == 200: - data = response.json() - # Extract the first food item from the results - food_item = data['foods'][0] - food_item - integration: a77cc48a-579a-4145-b1bd-80ad50d482ec - notes: null - query: Search for nutrients for raw acerola juice - resource_id: 4164944f-a6d4-452c-a290-3ab7bd2e9d9e - resource_type: example -slug: a77cc48a-579a-4145-b1bd-80ad50d482ec +url: null +uuid: d5e56bdc-4ac0-4ad6-aebe-77a9096081ff diff --git a/src/biome/adhoc_data/datasources/usgs_pesticide/api.yaml b/src/biome/adhoc_data/datasources/usgs_pesticide/api.yaml index 1a09213..3437b45 100644 --- a/src/biome/adhoc_data/datasources/usgs_pesticide/api.yaml +++ b/src/biome/adhoc_data/datasources/usgs_pesticide/api.yaml @@ -1,9 +1,14 @@ +datatype: dataset description: | The USGS Pesticide National Synthesis Project contains annual agricultural pesticide use data by county, on a 2-5 year interval, from 2000 to 2019. -integration_type: dataset +img_url: null +last_updated: null name: USGS Pesticide National Synthesis Project -prompt: | +provider: adhoc:specialist_agents +resources: {} +slug: adhocspecificationintegration_usgs_pesticide_national_synthesis_project +source: | You have access to a file-based USGS Pesticide National Synthesis Project data, for annual agricultural pesticide use by county and pesticide compound. We've added files on a 2-5 year interval, from 2000 to 2019, under the `${DATASET_FILES_BASE_PATH}/usgs_pesticide` directory. The exact files are located in: @@ -41,5 +46,5 @@ prompt: | You may use the python `us` library to convert FIPS codes to state and county names, and use pandas to read the tab-delimited text files. -resources: {} -slug: 19c8907f-cd98-4a3e-8d72-fe466b20087e +url: null +uuid: bc539ebb-0ce9-4f66-9522-e7f516e2c022 diff --git a/src/biome/adhoc_data/datasources/waterservices_usgs/api.yaml b/src/biome/adhoc_data/datasources/waterservices_usgs/api.yaml index 14eb51d..eb04bb4 100644 --- a/src/biome/adhoc_data/datasources/waterservices_usgs/api.yaml +++ b/src/biome/adhoc_data/datasources/waterservices_usgs/api.yaml @@ -1,21 +1,16 @@ +datatype: api description: | You can use this service to retrieve daily statistical data about the hundreds of thousands of hydrologic sites served by the USGS. Especially crucial to analyze water quality to correlate with other data sources. -integration_type: api +img_url: null +last_updated: null name: USGS Water Services API -prompt: | - You can use this service to retrieve daily statistical data about the hundreds of - thousands of hydrologic sites served by the USGS. Various output formats available, prefer JSON. - - ${web_documentation} - - ### Site Types (siteType) minor filter codes - ${site_types} +provider: adhoc:specialist_agents resources: 4c84149a-8c62-40c1-aae9-9bae79c4c2fc: filepath: usgs_waterservices_web.md - integration: c0deb2c9-90bf-476c-9ed0-c5f6070ebee8 + integration: b0b57684-6f63-4436-9489-853c493da6ec name: web_documentation resource_id: 4c84149a-8c62-40c1-aae9-9bae79c4c2fc resource_type: file @@ -34,15 +29,25 @@ resources: \ 'endDt': end_date.strftime('%Y-%m-%d'),\n 'parameterCd': ','.join(params),\n\ \ 'siteStatus': 'active'\n }\n\n # Make request\n response =\ \ requests.get(base_url, params=params)" - integration: c0deb2c9-90bf-476c-9ed0-c5f6070ebee8 + integration: b0b57684-6f63-4436-9489-853c493da6ec notes: null query: Get water quality data for Seminole County, Florida resource_id: 57592feb-0f49-4aea-8d78-570b3f674123 resource_type: example ebee2534-6b90-4b45-8f05-03801d2e7a85: filepath: site_types.md - integration: c0deb2c9-90bf-476c-9ed0-c5f6070ebee8 + integration: b0b57684-6f63-4436-9489-853c493da6ec name: site_types resource_id: ebee2534-6b90-4b45-8f05-03801d2e7a85 resource_type: file -slug: c0deb2c9-90bf-476c-9ed0-c5f6070ebee8 +slug: adhocspecificationintegration_usgs_water_services_api +source: | + You can use this service to retrieve daily statistical data about the hundreds of + thousands of hydrologic sites served by the USGS. Various output formats available, prefer JSON. + + ${web_documentation} + + ### Site Types (siteType) minor filter codes + ${site_types} +url: null +uuid: b0b57684-6f63-4436-9489-853c493da6ec From 38137ae55023f444e4e21db2c874b11b57bf1b0a Mon Sep 17 00:00:00 2001 From: Satchel Baldwin Date: Tue, 15 Jul 2025 14:01:02 -0500 Subject: [PATCH 19/21] add: changes for beaker-kernel integrations typing changes --- src/biome/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/biome/context.py b/src/biome/context.py index 8716c3d..ba9fc9b 100644 --- a/src/biome/context.py +++ b/src/biome/context.py @@ -41,7 +41,7 @@ def __init__(self): ] super().__init__(display_name="Biome Second Test Integration") def list_integrations(self): - return [asdict(i) for i in self.integrations] + return self.integrations def get_integration(self, integration_id): pass def list_resources(self, integration_id, resource_type=None): From a226ca8f088a56d919af0c9132d3e737a2ecb79c Mon Sep 17 00:00:00 2001 From: Satchel Baldwin Date: Tue, 15 Jul 2025 16:03:23 -0500 Subject: [PATCH 20/21] fix interpolation migration bugs causing invalid APIs --- .../datasources/chis_california_asthma/api.yaml | 2 +- src/biome/adhoc_data/datasources/pdc/api.yaml | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml b/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml index 37d4d19..53ce898 100644 --- a/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml +++ b/src/biome/adhoc_data/datasources/chis_california_asthma/api.yaml @@ -12,7 +12,7 @@ provider: adhoc:specialist_agents resources: ca7ff323-7bef-489d-9d2c-760628f40825: code: "import pandas as pd\nimport os\n\n# Define the file path using the correct\ - \ filename\nfile_path = os.path.join('{{DATASET_FILES_BASE_PATH}}', 'chis_california_asthma',\ + \ filename\nfile_path = os.path.join('${DATASET_FILES_BASE_PATH}', 'chis_california_asthma',\ \ 'current-asthma-prevalence-by-county-2015_2022.csv')\n\ndf = pd.read_csv(file_path,\ \ encoding='latin1') # latin1 encoding has worked before, not specifying it\ \ hasn't\n\n# Check the column names to make sure we're using the right ones\n\ diff --git a/src/biome/adhoc_data/datasources/pdc/api.yaml b/src/biome/adhoc_data/datasources/pdc/api.yaml index 3b8685d..a75cbed 100644 --- a/src/biome/adhoc_data/datasources/pdc/api.yaml +++ b/src/biome/adhoc_data/datasources/pdc/api.yaml @@ -181,8 +181,8 @@ resources: url = 'https://pdc.cancer.gov/graphql' query = """ - query ($gene_name: String!, $offset: Int, $limit: Int) { - getPaginatedUIGeneAliquotSpectralCount(gene_name: $gene_name, offset: $offset, limit: $limit) { + query ($$gene_name: String!, $offset: Int, $limit: Int) { + getPaginatedUIGeneAliquotSpectralCount(gene_name: $$gene_name, offset: $offset, limit: $limit) { total uiGeneAliquotSpectralCounts { aliquot_id @@ -253,9 +253,9 @@ resources: # Updated query with correct schema query = """ - query ($gene_name: String!) { + query ($$gene_name: String!) { getPaginatedUIGeneAliquotSpectralCount( - gene_name: $gene_name, + gene_name: $$gene_name, offset: 0, limit: 100 # Increased limit to get more samples ) { @@ -640,9 +640,9 @@ resources: # Corrected query structure query = """ - query ($gene_name: String!) { + query ($$gene_name: String!) { getPaginatedUIGeneAliquotSpectralCount( - gene_name: $gene_name, + gene_name: $$gene_name, offset: 0, limit: 10 ) { @@ -753,8 +753,8 @@ resources: # Query for STAT5B protein expression data query = """ - query ($gene_name: String!) { - getPaginatedUIGeneAliquotSpectralCount(gene_name: $gene_name, offset: 0, limit: 10) { + query ($$gene_name: String!) { + getPaginatedUIGeneAliquotSpectralCount(gene_name: $$gene_name, offset: 0, limit: 10) { total uiGeneAliquotSpectralCounts { aliquot_id From 325eefdebe5e93c9ba6aaf8a831826220d6602ff Mon Sep 17 00:00:00 2001 From: Satchel Baldwin Date: Tue, 15 Jul 2025 16:07:21 -0500 Subject: [PATCH 21/21] update pyproject toml and docker builds --- Dockerfile | 11 +++-------- pyproject.toml | 7 +++---- rest_api/Dockerfile | 1 - 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index 85e14b2..d03c776 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,9 @@ COPY --chown=1000:1000 . /jupyter RUN rm -f /jupyter/.beaker.conf RUN rm -f /jupyter/.env -RUN uv pip install --system -v -e /jupyter +RUN uv pip install --system /jupyter + +RUN chown jupyter:jupyter -R /usr/local/lib/python3.11/site-packages/biome/adhoc_data/ RUN mkdir -m 777 /var/run/beaker @@ -28,12 +30,5 @@ ENV BEAKER_SUBKERNEL_USER=user ENV BEAKER_RUN_PATH=/var/run/beaker ENV BEAKER_APP=biome.app.BiomeApp -# subkernel access / file pane -RUN ln -s /jupyter/src/biome/datasources /home/jupyter/datasources -RUN ln -s /jupyter/data /home/jupyter/data - -RUN ln -s /jupyter/src/biome/datasources /home/user/datasources -RUN ln -s /jupyter/data /home/user/data - # Service CMD ["python", "-m", "beaker_kernel.service.server", "--ip", "0.0.0.0"] diff --git a/pyproject.toml b/pyproject.toml index 0f78da3..ca5d80e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "biome" dynamic = ["version"] description = '' readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.10" license = "MIT" keywords = [] authors = [ @@ -25,9 +25,9 @@ classifiers = [ "Programming Language :: Python :: Implementation :: PyPy", ] dependencies = [ - "beaker-kernel~=1.11.0", + "beaker-kernel @ file:///jupyter/beaker_kernel-1.11.2-py3-none-any.whl", "adhoc-api~=2.4.0", - "archytas~=1.4.0", + "archytas~=1.5.0", "requests", "PyYAML", "idc-index", @@ -43,7 +43,6 @@ dependencies = [ "xlrd~=2.0.1", "geopandas~=1.0.1", "scikit-learn~=1.6.1", - "synapseclient" ] [project.urls] diff --git a/rest_api/Dockerfile b/rest_api/Dockerfile index 329de1b..16f9b5d 100644 --- a/rest_api/Dockerfile +++ b/rest_api/Dockerfile @@ -5,7 +5,6 @@ FROM python:3.11.5 EXPOSE 8001 COPY ./rest_api /rest_api -COPY ./data /data WORKDIR /rest_api RUN pip install --upgrade --no-cache-dir pip