diff --git a/chat-ui/Dockerfile b/chat-ui/Dockerfile index 60219b9..f09ed7a 100644 --- a/chat-ui/Dockerfile +++ b/chat-ui/Dockerfile @@ -8,9 +8,11 @@ RUN pip install --upgrade --no-cache-dir hatch pip beaker-kernel USER jupyter WORKDIR /jupyter -COPY ./beaker-biome /opt/beaker-biome +COPY ./beaker-biome /jupyter/beaker-biome USER root -RUN chown -R jupyter:users /opt/beaker-biome +RUN chown -R jupyter:users /jupyter/beaker-biome USER jupyter -RUN cd /opt/beaker-biome && hatch build -RUN pip install /opt/beaker-biome/dist/beaker_biome*.whl \ No newline at end of file + +# RUN cd /opt/beaker-biome && hatch build +# RUN pip install /opt/beaker-biome/dist/beaker_biome*.whl +RUN pip install -e /jupyter/beaker-biome diff --git a/chat-ui/beaker-biome/pyproject.toml b/chat-ui/beaker-biome/pyproject.toml index b7ae43d..b9129ce 100644 --- a/chat-ui/beaker-biome/pyproject.toml +++ b/chat-ui/beaker-biome/pyproject.toml @@ -27,6 +27,7 @@ classifiers = [ dependencies = [ "beaker-kernel>=1.6.5", "requests", + "google-generativeai" ] [project.urls] diff --git a/chat-ui/beaker-biome/src/beaker_biome/biome/agent.py b/chat-ui/beaker-biome/src/beaker_biome/biome/agent.py index c799eb1..43de36d 100644 --- a/chat-ui/beaker-biome/src/beaker_biome/biome/agent.py +++ b/chat-ui/beaker-biome/src/beaker_biome/biome/agent.py @@ -5,18 +5,67 @@ from time import sleep import asyncio -from archytas.tool_utils import AgentRef, LoopControllerRef, tool +from archytas.tool_utils import AgentRef, LoopControllerRef, ReactContextRef, tool from typing import Any from beaker_kernel.lib.agent import BaseAgent from beaker_kernel.lib.context import BaseContext +import google.generativeai as genai +from google.generativeai import caching logger = logging.getLogger(__name__) BIOME_URL = "http://biome_api:8082" - +disease_types = """ +- adenomas and adenocarcinomas +- ductal and lobular neoplasms +- myeloid leukemias +- epithelial neoplasms, nos +- squamous cell neoplasms +- gliomas +- lymphoid leukemias +- cystic, mucinous and serous neoplasms +- nevi and melanomas +- neuroepitheliomatous neoplasms +- acute lymphoblastic leukemia +- plasma cell tumors +- complex mixed and stromal neoplasms +- mature b-cell lymphomas +- transitional cell papillomas and carcinomas +- not applicable +- osseous and chondromatous neoplasms +- germ cell neoplasms +- mesothelial neoplasms +- not reported +- acinar cell neoplasms +- paragangliomas and glomus tumors +- chronic myeloproliferative disorders +- neoplasms, nos +- thymic epithelial neoplasms +- myomatous neoplasms +- complex epithelial neoplasms +- soft tissue tumors and sarcomas, nos +- lipomatous neoplasms +- meningiomas +- fibromatous neoplasms +- specialized gonadal neoplasms +- unknown +- miscellaneous tumors +- adnexal and skin appendage neoplasms +- basal cell neoplasms +- mucoepidermoid neoplasms +- myelodysplastic syndromes +- nerve sheath tumors +- leukemias, nos +- synovial-like neoplasms +- fibroepithelial neoplasms +- miscellaneous bone tumors +- blood vessel tumors +- mature t- and nk-cell lymphomas +- _missing +""" class BiomeAgent(BaseAgent): """ @@ -33,201 +82,358 @@ class BiomeAgent(BaseAgent): of a flow could be looking through all the data sources, picking one, finding a dataset using the URL, and then finally loading that dataset into a pandas dataframe. """ + GDC_MODEL_DISPLAY_NAME='GDC_CACHE_API_DOCS' + GDC_MODEL='models/gemini-1.5-flash-001' def __init__(self, context: BaseContext = None, tools: list = None, **kwargs): - libraries = { - } + import os + genai.configure(api_key=os.environ["GEMINI_API_KEY"]) + self.gemini = {} super().__init__(context, tools, **kwargs) - - def update_job_status(self, job_id, status): - self.context.send_response("iopub", - "job_status", { - "job_id": job_id, - "status": status - }, - ) - - # TODO: Formatting of these messages should be left to the Analyst-UI in the future. - async def poll_query(self, job_id: str): - # Poll result - status = "queued" - result = None - while status == "queued": - response = requests.get(f"{BIOME_URL}/jobs/{job_id}").json() - status = response["status"] - sleep(1) - - self.update_job_status(job_id, status) - - #asyncio.create_task(self.poll_query_logs(job_id)) - while status == "started": - response = requests.get(f"{BIOME_URL}/jobs/{job_id}/logs").json() - self.context.send_response("iopub", - "job_logs", { - "job_id": job_id, - "logs": response, - }, - ) - response = requests.get(f"{BIOME_URL}/jobs/{job_id}").json() - status = response["status"] - sleep(5) - - self.update_job_status(job_id, status) - - # Handle result - if status != "finished": - self.update_job_status(job_id, status) - self.context.send_response("iopub", - "job_failure", { - "job_id": job_id, - "response": response - }, - ) - - result = response["result"] # TODO: Bubble up better cell type - self.context.send_response("iopub", - "job_response", { - "job_id": job_id, - "response": result['answer'], - "raw": result - }, - ) + self.try_load_cache() - @tool(autosummarize=True) - async def search(self, query: str) -> list[dict[str, Any]]: - """ - Search for data sources in the Biome app. Results will be matched semantically - and string distance. Use this to find a data source. You don't need live - web searches. If the user asks about data sources, use this tool. + def try_load_cache(self): + content = caching.CachedContent.list() + is_cached = False + for cache_object in content: + if cache_object.display_name == self.GDC_MODEL_DISPLAY_NAME: + is_cached = True + break + if not is_cached: + self.gemini_info({"cache": "Cache was not found."}) + cache_object = self.build_cache() + else: + self.gemini_info({"cache": "Cache was found."}) + self.gemini['model'] = genai.GenerativeModel.from_cached_content(cached_content=cache_object) + self.gemini['chat'] = self.gemini['model'].start_chat() - Be sure to use the `display_search` tool for the output. Ensure you always use `display_search` after. + def build_cache(self): + import pathlib + import datetime + agent_dir = pathlib.Path(__file__).resolve().parent + with open(f'{agent_dir}/docs.md', 'r') as f: + docs = f.read() - Args: - query (str): The query used to find the datasource. - Returns: - list: A JSON-formatted string containing a list of strings. - The list should contain only the `name` field and no other field - of the data sources found, ordered from most relevant to least relevant. - Ensure that only the name field is present. - An example is provided surrounded in backticks. - ``` - ["Proteomics Data Commons", ""Office of Cancer Clinical Proteomics Research", "UniProt"] - ``` + prompt = """ + You are an assistant who will help me query the genomics data commons API. + You should write clean python code to solve specific queries I pose. You should + write it as though it will be directly used in a Jupyter notebook. You should not + include backticks or "python" at the top of the code blocks, that is unnecessary. + You should not provide explanation unless I ask a follow up question. + Assume pandas is installed and is imported with `import pandas as pd`. Also assume `requests`, `json`, + and `os` are imported properly. """ - endpoint = f"{BIOME_URL}/sources" - response = requests.get(endpoint, params={"query": query}) - raw_sources = response.json()['sources'] - sources = [ - # Include only necessary fields to ensure LLM context length is not exceeded. - { - "id": source["id"], - "name": source["content"]["Web Page Descriptions"]["name"], - "initials": source["content"]["Web Page Descriptions"]["initials"], - "purpose": source["content"]["Web Page Descriptions"]["purpose"], - "links": source["content"]["Information on Links on Web Page"], - "base_url": source.get("base_url", None) - } for source in raw_sources - ] - return sources - - @tool(autosummarize=True) - async def display_search(self, results: list[str], agent:AgentRef, loop: LoopControllerRef): - """ - Once search has been performed, this tool will display it to the user. - Args: - results (list[str]): The query used to find the datasource. - """ - # sometimes it wraps the output - if isinstance(results, dict): - results = results.get("results", results) - endpoint = f"{BIOME_URL}/sources" - response = requests.get(endpoint, params={ - "simple_query_string": { - "fields": ["content.Web Page Descriptions.name"], - "query": "|".join(results) - } - }) - raw_sources = response.json()['sources'] - sources = [ - { - "id": source["id"], - "name": source["content"]["Web Page Descriptions"]["name"], - "initials": source["content"]["Web Page Descriptions"]["initials"], - "purpose": source["content"]["Web Page Descriptions"]["purpose"], - "links": source["content"]["Information on Links on Web Page"], - "base_url": source.get("base_url", None), - "logo": source.get("logo", None) - } for source in raw_sources - ] - # match sources to ordering from previous llm step by dict to avoid n^2 - - sources_map = { source.get("name", ""): source for source in sources } - ordered_sources = [sources_map[name] for name in results] - self.context.send_response("iopub", - "data_sources", { - "sources": ordered_sources - }, - ) - loop.set_state(loop.STOP_SUCCESS) - - # TODO(DESIGN): Deal with long running jobs in tools - # - # Option 1: We can return the job id and the agent can poll for the result. - # This will require a job status tool. Once the status is done, we can either - # check the result if it's a query or check the data source if it's a scan. - # This feels a bit messy though that the job creation has a similar return - # output on queue but getting the result is very different for each job. - # - # Option 2: We can wait for the job and return it to the agent when it's done - # - # Option 3: We can maybe leverage new widgets in the Analyst UI?? - # - - # CHOOSING OPTION 1 FOR THE TIME BEING - @tool() - async def query_page(self, task: str, base_url: str, agent: AgentRef, loop: LoopControllerRef): - """ - Run an action over a *specific* source in the Biome app and return the results. - Find the url from a data source by using `search` tool first and - picking the most relevant one. + cached_content = f""" + You will be given the entire API documentation, but first I want to give you a list of available + disease types since this is a common search parameter: - This kicks off a long-running job so you'll have to just return the ID to the user - instead of the result. + {disease_types} - This can be used to ask questions about a data source or download some kind - of artifact from it. This tool just kicks off a job where an AI crawls the website - and performs the task. + If you are specifying a disease type, it MUST be from this enumerated list. - Args: - task (str): Task given in natural language to perform over URL. - base_url (str): URL to run query over. + Now, I will provide you the extensive API documentation. When you write code against this API + you should avail yourself of the appropriate query parameters, your understanding of the response model + and be cognizant that not all data is public and thus may require a token, etc. When you are downloading + things you should log progress. When you are doing complex things try to break them down and + implement appropriate exception handling. Ok here we go, here are the docs: + + Unless you receive a 403 forbidden, assume the endpoints are unauthenticated. + If the user says the API does not require authentication, OMIT code about tokens and token handling and token headers. + + {docs} """ - response = requests.post( f"{BIOME_URL}/jobs/query", json={"user_task": task, "url": base_url}) - job_id = response.json()["job_id"] + cache = caching.CachedContent.create( + model=self.GDC_MODEL, + display_name=self.GDC_MODEL_DISPLAY_NAME, + contents=[cached_content], + ttl=datetime.timedelta(minutes=30), + system_instruction=prompt + ) + return cache + + def gemini_info(self, info: dict): self.context.send_response("iopub", - "job_create", { - "job_id": job_id, - "task": task, - "url": base_url + "gemini_info", { + "body": info }, - ) - asyncio.create_task(self.poll_query(job_id)) - loop.set_state(loop.STOP_SUCCESS) + ) + + def gemini_error(self, error: dict): + self.context.send_response("iopub", + "gemini_error", { + "body": error + }, + ) @tool() - async def scan(self, base_url: str, agent:AgentRef, loop: LoopControllerRef) -> str: + async def interact_with_api(self, goal: str, agent: AgentRef, loop: LoopControllerRef, react_context: ReactContextRef) -> str: """ - Profiles the given web page and adds it to the data sources in the Biome app. + This tool provides interaction with external APIs with a second agent. + You will query external APIs through this tool. + Based on what that code returns and the user's goal, continue to interact with the API to get to that goal. - This kicks off a long-running job so you'll have to just return the ID to the user - instead of the result. + The output will either be a summary of the code output or an error. + If it is an error, instruct the second agent to fix the code and retry. Args: - base_url (str): The url to scan and add as a data source. + goal (str): The task given to the second agent. If the user states the API is unauthenticated, relay that information here. Returns: - str: Job ID to poll for the result. + str: A summary of the current step being run, along with the collected stdout, stderr, returned result, display_data items, and any + errors that may have occurred, or just an error. + """ - response = requests.post( f"{BIOME_URL}/jobs/scan", json={"uris": [base_url]}) - job_id = response.json()["job_id"] - asyncio.create_task(self.poll_query(job_id)) - return job_id \ No newline at end of file + self.gemini_info({'goal': goal}) + try: + agent_response = self.gemini['chat'].send_message(goal).text + prefixes = ['```python', '```'] + suffixes = ['```', '```\n'] + for prefix in prefixes: + if agent_response.startswith(prefix): + agent_response = agent_response[len(prefix):] + for suffix in suffixes: + if agent_response.endswith(suffix): + agent_response = agent_response[:-len(suffix)] + agent_response = '\n'.join([ + 'import pandas as pd', + 'import os', + 'import json', + 'import requests', + agent_response + ]) + except Exception as e: + self.gemini_error({'error': str(e)}) + return f"The agent failed to produce valid code: {str(e)}" + + fixed_code = await agent.query(f"""The code you received is will be listed below the line of dashes. +Please fix the python code for syntax errors and only return the python code with fixed syntax errors. +Ensure the output has no formatting and return just the code, please. + +If the output has formatting like backticks or a language specifier, be sure to remove all formatting +and return nothing but the code itself with no additional text. + +Example: + Input: + ```python + print(a) + ``` + Output: + print(a) + Input: + ```python + de f fn_b(b): + print(this is an unescaped string) + def fn_a(a): + print(a) + ``` + This code has been fixed to correctly solve the task. + Output: + def fn_b(b): + print("this is an unescaped string") + def fn(a): + print(a) + +---------- + +{agent_response} +""") + try: + evaluation = await agent.tools['run_code'](fixed_code, agent, loop, react_context) + except Exception as e: + self.gemini_error({'error': str(e)}) + return f""" + The second agent failed to create valid code. Instruct it to rerun. The error was {str(e)}. The code will be provided for fixes or retry. + """ + return evaluation + # def update_job_status(self, job_id, status): + # self.context.send_response("iopub", + # "job_status", { + # "job_id": job_id, + # "status": status + # }, + # ) + + # # TODO: Formatting of these messages should be left to the Analyst-UI in the future. + # async def poll_query(self, job_id: str): + # # Poll result + # status = "queued" + # result = None + # while status == "queued": + # response = requests.get(f"{BIOME_URL}/jobs/{job_id}").json() + # status = response["status"] + # sleep(1) + + # self.update_job_status(job_id, status) + + # #asyncio.create_task(self.poll_query_logs(job_id)) + # while status == "started": + # response = requests.get(f"{BIOME_URL}/jobs/{job_id}/logs").json() + # self.context.send_response("iopub", + # "job_logs", { + # "job_id": job_id, + # "logs": response, + # }, + # ) + # response = requests.get(f"{BIOME_URL}/jobs/{job_id}").json() + # status = response["status"] + # sleep(5) + + # self.update_job_status(job_id, status) + + # # Handle result + # if status != "finished": + # self.update_job_status(job_id, status) + # self.context.send_response("iopub", + # "job_failure", { + # "job_id": job_id, + # "response": response + # }, + # ) + + # result = response["result"] # TODO: Bubble up better cell type + # self.context.send_response("iopub", + # "job_response", { + # "job_id": job_id, + # "response": result['answer'], + # "raw": result + # }, + # ) + + # @tool(autosummarize=True) + # async def search(self, query: str) -> list[dict[str, Any]]: + # """ + # Search for data sources in the Biome app. Results will be matched semantically + # and string distance. Use this to find a data source. You don't need live + # web searches. If the user asks about data sources, use this tool. + + # Be sure to use the `display_search` tool for the output. Ensure you always use `display_search` after. + + # Args: + # query (str): The query used to find the datasource. + # Returns: + # list: A JSON-formatted string containing a list of strings. + # The list should contain only the `name` field and no other field + # of the data sources found, ordered from most relevant to least relevant. + # Ensure that only the name field is present. + # An example is provided surrounded in backticks. + # ``` + # ["Proteomics Data Commons", ""Office of Cancer Clinical Proteomics Research", "UniProt"] + # ``` + # """ + + # endpoint = f"{BIOME_URL}/sources" + # response = requests.get(endpoint, params={"query": query}) + # raw_sources = response.json()['sources'] + # sources = [ + # # Include only necessary fields to ensure LLM context length is not exceeded. + # { + # "id": source["id"], + # "name": source["content"]["Web Page Descriptions"]["name"], + # "initials": source["content"]["Web Page Descriptions"]["initials"], + # "purpose": source["content"]["Web Page Descriptions"]["purpose"], + # "links": source["content"]["Information on Links on Web Page"], + # "base_url": source.get("base_url", None) + # } for source in raw_sources + # ] + # return sources + + # @tool(autosummarize=True) + # async def display_search(self, results: list[str], agent:AgentRef, loop: LoopControllerRef): + # """ + # Once search has been performed, this tool will display it to the user. + # Args: + # results (list[str]): The query used to find the datasource. + # """ + # # sometimes it wraps the output + # if isinstance(results, dict): + # results = results.get("results", results) + # endpoint = f"{BIOME_URL}/sources" + # response = requests.get(endpoint, params={ + # "simple_query_string": { + # "fields": ["content.Web Page Descriptions.name"], + # "query": "|".join(results) + # } + # }) + # raw_sources = response.json()['sources'] + # sources = [ + # { + # "id": source["id"], + # "name": source["content"]["Web Page Descriptions"]["name"], + # "initials": source["content"]["Web Page Descriptions"]["initials"], + # "purpose": source["content"]["Web Page Descriptions"]["purpose"], + # "links": source["content"]["Information on Links on Web Page"], + # "base_url": source.get("base_url", None), + # "logo": source.get("logo", None) + # } for source in raw_sources + # ] + # # match sources to ordering from previous llm step by dict to avoid n^2 + + # sources_map = { source.get("name", ""): source for source in sources } + # ordered_sources = [sources_map[name] for name in results] + # self.context.send_response("iopub", + # "data_sources", { + # "sources": ordered_sources + # }, + # ) + # loop.set_state(loop.STOP_SUCCESS) + + # # TODO(DESIGN): Deal with long running jobs in tools + # # + # # Option 1: We can return the job id and the agent can poll for the result. + # # This will require a job status tool. Once the status is done, we can either + # # check the result if it's a query or check the data source if it's a scan. + # # This feels a bit messy though that the job creation has a similar return + # # output on queue but getting the result is very different for each job. + # # + # # Option 2: We can wait for the job and return it to the agent when it's done + # # + # # Option 3: We can maybe leverage new widgets in the Analyst UI?? + # # + + # # CHOOSING OPTION 1 FOR THE TIME BEING + # @tool() + # async def query_page(self, task: str, base_url: str, agent: AgentRef, loop: LoopControllerRef): + # """ + # Run an action over a *specific* source in the Biome app and return the results. + # Find the url from a data source by using `search` tool first and + # picking the most relevant one. + + # This kicks off a long-running job so you'll have to just return the ID to the user + # instead of the result. + + # This can be used to ask questions about a data source or download some kind + # of artifact from it. This tool just kicks off a job where an AI crawls the website + # and performs the task. + + # Args: + # task (str): Task given in natural language to perform over URL. + # base_url (str): URL to run query over. + # """ + # response = requests.post( f"{BIOME_URL}/jobs/query", json={"user_task": task, "url": base_url}) + # job_id = response.json()["job_id"] + # self.context.send_response("iopub", + # "job_create", { + # "job_id": job_id, + # "task": task, + # "url": base_url + # }, + # ) + # asyncio.create_task(self.poll_query(job_id)) + # loop.set_state(loop.STOP_SUCCESS) + + # @tool() + # async def scan(self, base_url: str, agent:AgentRef, loop: LoopControllerRef) -> str: + # """ + # Profiles the given web page and adds it to the data sources in the Biome app. + + # This kicks off a long-running job so you'll have to just return the ID to the user + # instead of the result. + + # Args: + # base_url (str): The url to scan and add as a data source. + # Returns: + # str: Job ID to poll for the result. + # """ + # response = requests.post( f"{BIOME_URL}/jobs/scan", json={"uris": [base_url]}) + # job_id = response.json()["job_id"] + # asyncio.create_task(self.poll_query(job_id)) + # return job_id diff --git a/chat-ui/beaker-biome/src/beaker_biome/biome/docs.md b/chat-ui/beaker-biome/src/beaker_biome/biome/docs.md new file mode 100644 index 0000000..a9bec19 --- /dev/null +++ b/chat-ui/beaker-biome/src/beaker_biome/biome/docs.md @@ -0,0 +1,47966 @@ +#Preparing for Data Downloads and Uploads + +## Overview + +The GDC Data Transfer Tool is intended to be used in conjunction with the [GDC Data Portal](https://portal.gdc.cancer.gov) and the [GDC Data Submission Portal](https://portal.gdc.cancer.gov/submission/) to transfer data to or from the GDC. First, the GDC Data Portal's interface is used to generate a manifest file or obtain UUID(s) and (for Controlled-Access Data) an authentication token. The GDC Data Transfer Tool is then used to transfer the data files listed in the manifest file or identified by UUID(s). + +##Downloads +### Obtaining a Manifest File for Data Download + +The GDC Data Transfer Tool supports downloading multiple files listed in a GDC manifest file. Manifest files can be generated and downloaded directly from the GDC Data Portal: + +First, select the data files of interest. Click the *Cart* button in the row corresponding to the file desired. The button will turn blue to indicate that the file has been selected. + +![GDC Data Portal: Selecting Files of Interest](images/06-20_Data-Portal-File-Selection.png "Selecting Files of Interest") + + +Once all files of interest have been selected, click on the *Cart* button in the upper right-hand corner. This will bring up the cart page, which provides an overview of all currently selected files. This list of files can be downloaded as a manifest file by clicking on the *Download Cart* button and selecting *Manifest* from the drop down. + +![GDC Data Portal: Cart Page](images/06-20-v2_Data-Portal-Cart-Page.png) + +### Obtaining UUIDs for Data Download + +A manifest file is not required to download files from GDC. The GDC Data Transfer Tool will accept file UUID(s) instead of a manifest file for downloading individual data files. To obtain a data file's UUID from the GDC Data Portal, click the file name to find its detail page including its GDC UUID. + +![GDC Data Portal: Detailed File Page](images/06-20_Data-portal-file-detail-pagev2.png) + + +### Obtaining an Authentication Token for Data Downloads + +The GDC Data Transfer Tool requires an authentication token to download from GDC data portal to download Controlled-Access Data. Tokens can be generated and downloaded directly from the GDC Data Portal. + +To generate a token, first log in to the GDC Data Portal by clicking the *Login* button in the top right corner of the page. This will redirect to the eRA Commons login page. After successful authentication, the GDC Data Portal will display the username in place of the *Login* button. Here, the user Ian Miller is logged in to the GDC Data Portal, indicated by the username IANMILLER. + +Clicking the username will open a drop-down menu. Select *Download Token* from the menu to generate an authentication token. + +![GDC Data Portal User Dropdown Menu](images/06-20_auth_example_download_token-2.png) + + +**NOTE:** The authentication token should be kept in a secure location, as it allows access to all data accessible by the associated user. + +##Uploads +### Obtaining a Manifest File for Data Uploads +Multiple data file uploads are supported by the GDC Data Transfer Tool via a manifest file. Manifest files can be generated and downloaded directly from the GDC Submission Portal. A project's manifest file can be downloaded from the projects's dashboard. + +![GDC Submission Portal Manifest Download](images/10-10-16_manifest_upload.png) + + +**NOTE:** To download a project's manifest file click on the _Download Manifest_ button located on the home page of the project, just below the four status charts. A manifest will be generated for the entire project or if previous files have already been upload only the files that remain to be uploaded. + +A manifest for individual files can also be downloaded from the transaction tab and browse tab pages of the submission portal's project. More information on the process can be found under the Submission Portal's documentation section entitled [Uploading the Submittable Data File to the GDC](https://docs.gdc.cancer.gov/Data_Submission_Portal/Users_Guide/Data_Submission_Walkthrough/#uploading-the-submittable-data-file-to-the-gdc). + +### Obtaining UUIDs for Data Uploads +A UUID can be used for data submission with the Data Transfer Tool. The UUID for submittable data uploads can be obtained from the Submission Portal or from the API GraphQL endpoint. In the Submission Portal the UUID for a data file can be found in the Manifest YAML file located in the _id:_ row located under the file size entry. + +![Submission Manifest yaml file](images/10-18_yaml_submission_UUID_example.png) + + + A second location to obtain a UUID in the Submission Portal is on the Browse Tab page. Under the Submittable Data Files section a UUID can be found by opening up the file's detail page. By clicking on the Submitter ID of the upload file a new window will display a Summary of the file's details, which contains the UUID. + +![Submission Portal Browse Page Details](images/submission_portal_browse_page_UUID.png) + +###GraphQL + +A UUID can be obtained from the API GraphQL endpoint. An overview of what GraphQL and its uses is located on the API documentation page section [Querying Submitted Data Using GraphQL](https://docs.gdc.cancer.gov/API/Users_Guide/Submission/#querying-submitted-data-using-graphql) + +The following example will query the endpoint to produce a UUID along with submitter_id, file_name, and project_id. + +=== "GraphQl_Bare" + + ```GraphQL + { + submitted_unaligned_reads (project_id: "GDC-INTERNAL", submitter_id: "Blood-00001-aliquot_lane1_barcode23.fastq") { + id + submitter_id + file_name + project_id + } + } + ``` + +=== "Escaped_Json" + + ```json + {\n submitted_unaligned_reads (project_id: \"GDC-INTERNAL\", submitter_id: \"Blood-00001-aliquot_lane1_barcode23.fastq\") {\n id\n submitter_id\n file_name\n project_id\n}\n}\n + + ``` + +=== "Query_json" + + ```json + { + "query": "{\n \n submitted_unaligned_reads (project_id: \"GDC-INTERNAL\", submitter_id: \"Blood-00001-aliquot_lane1_barcode23.fastq\") {\n id\n submitter_id\n file_name\n project_id\n}\n}", + "variables": null + } + ``` + +=== "Shell_command" + + ```Shell + export token=ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTO + $ curl --request POST --header "X-Auth-Token: $token" 'https://api.gdc.cancer.gov/v0/submission/graphql' -d@data.json + ``` + +=== "API_Response" + + ```json + { + "data": { + "submitted_unaligned_reads": [ + { + "file_name": "dummy.fastq", + "id": "616eab2f-791a-4641-8cd6-ee195a10a201", + "project_id": "GDC-INTERNAL", + "submitter_id": "Blood-00001-aliquot_lane1_barcode23.fastq" + } + ] + } + ``` + +### Obtaining an Authentication Token for Data Uploads +While biospecimen and clinical metadata may be uploaded via the GDC Data Submission Portal, file upload must be done using the Data Transfer Tool or API. An authentication token is required for data upload and can be generated on the GDC Data Submission Portal. + +To generate a token, first log in to the GDC Data Submission Portal by clicking the *Login* button in the top right corner of the page. This will create a popup window that will redirect to the eRA Commons login page. After successful authentication, the GDC Submission Portal will display the username in place of the *Login* button. Here, the user Ian Miller is logged in to the GDC Submission Portal, indicated by the username IANMILLER. + +Clicking the username will open a drop-down menu. Select *Download Token* from the menu to generate an authentication token. + +![GDC Submission Portal User Dropdown Menu](images/10-27_Submission_Portal_Auth_Download_Tab.png) + + + +#Data Downloads with the Data Transfer Tool UI + +##Data Transfer Tool UI: Overview +The UI version of the Data Transfer Tool was created for users who prefer a graphical interface over the command line or have limited command line experience. The command line version is recommended for those users with more command line experience, require large data transfers of GDC data, or need to download a large numbers of data files. + +###System Recommendations + +The system recommendations for using the GDC Data Transfer Tool are as follows: + +* OS: Linux (Ubuntu 14.x or later), OS X (10.9 Mavericks or later), or Windows (7 or later) +* CPU: At least four 64-bit cores, Intel or AMD +* RAM: At least 2 GiB +* Storage: Enterprise-class storage system capable of at least 1 Gb/s (gigabit per second) write throughput and sufficient free space for BAM files. + +###Binary Distributions + +Binary distributions are available on the GDC Transfer Tool page. To install the GDC Data Transfer UI download the respective binary distribution and unzip the distribution's archive to a location on the target system that is easily accessible. + +###Binary Installation +Once the binary has been positioned in an appropriate location on the client's file system the application will need to run though a one-time installation process. On first execution the binary install splash screen will appear showing the progress of the installation. A hidden directory is created within the user's home directory labeled dtt that holds configuration and executable files. + +![GDC DTT UI Installation](images/GDC_DTT_UI_INSTALLv7.png "GDC Data Transfer Tool UI Install") + + +###Preparing for Data Download + +The GDC Data Transfer Tool UI is a stand-alone client application intended to work with data file information stored on the GDC Data Portals. Data download information must first be gathered from the GDC Data Portal. From there a manifest file can be [generated](https://docs.gdc.cancer.gov/Data_Transfer_Tool/Users_Guide/Preparing_for_Data_Download_and_Upload/#obtaining-a-manifest-file-for-data-download) to supply the client. Alternatively, individual file UUIDs can be provided to the UUID entry window located on the Download tab in the client. + +![GDC DTT UI Start Page](images/DTT_UI_Start_Page.png) +##Downloads with UUIDs +The Data Transfer Tool UI can download files by individual UUID. UUIDs can be entered into the client while on the download tab. The single entry field labeled "Enter UUID(s)" allows the user to enter UUIDs individually. + +To obtain a data file's UUID from the GDC Data Portal, click on the file name to display the file's summary page which includes vital information such as its GDC UUID. + +##Downloads with Manifest +A portal-generated manifest file can be used with the Data Transfer Tool UI. From the Download tab home page click on the Select Manifest File button. A file system search window will popup allowing navigation to the manifest file. + +![GDC DTT UI Manifest Button Example](images/Manifest_button_DTT_UI_Start_Window.png "GDC Data Transfer Tool UI Manifest Button") + +##Download Progress Page +The Download Progress Page is the command console for the Data Transfer Tool UI and allows users to monitor downloads. Progress of all downloads including the ability to start, stop, and restart a download are performed on the Download Progress Page. Once file UUIDs or a manifest has been added to the queue the download can be started by clicking on the download button located at the lower right hand side of the page. + +![GDC DTT UI Download Progress Page_Download](images/Download_Progress_Page_download.png) + +Once a download has completed, information about the downloads can be viewed from the Completed tab located at the bottom of the page. Any Stopped or Failed downloads can also be viewed from their respective labels located at the bottom of the Status page. + +![DTT_UI_Download_Completed_Tab](images/DTT_UI_download_completed.png) + +##Controlled Access File Downloads + +Some files in the GDC are controlled access. If you require access to these files please review the process outlined in the documentation [Obtaining Access to Controlled Data](https://gdc.cancer.gov/access-data/obtaining-access-controlled-data). After appropriate authorization has been granted an access token can be generated to allow the Data Transfer Tool UI application access to the requested data files. Documentation explaining the process of generating a token is located in the [Obtaining an Authentication Token for Data Downloads](https://docs.gdc.cancer.gov/Data_Transfer_Tool/Users_Guide/Preparing_for_Data_Download_and_Upload/#obtaining-an-authentication-token-for-data-downloads). Once a token has been downloaded to a secure location on the client's local filesystem the Data Transfer Tool UI can now access it. + +![No_Token_icon](images/No_Token_file_dtt_ui.png) + +The current status of client authorization is viewable in the upper right corner of the application. If the image and wording on the token manager access button is in red then no valid client token file has been uploaded. To upload a valid token file click on the token status button. The token manager window should appear allowing either a drag and drop token file upload or a file navigation window can be opened to navigate to the file location. + +![Token Manager Window](images/Token_Manager_Window.png) + +The token manager will verify access and display the projects for which the user has access. To complete the token upload process click on the save button within the Token Manager window. + +![Valid Token](images/validated_token.png) + +##Settings and Advanced Settings + +While the default download options will work for the majority of use cases, there are a vareity of ways to customize or modify the download process within the DTT UI. Details of each of the settings are listed below. + +![DTT Settings and Advanced Settings Page](images/DTT_Settings_Page.png) + +| Settings | Details +|----------|---------| +| Server URL | Default: https://api.gdc.cancer.gov | +| Number of Client Connections: Default (3) | Number of concurrent client threads | +| Destination Folder: Default (User's Home Directory) | User selectable download file location | +| Calculate Inbound Segment and check Md5sum on Restart: Default (On) | Verify previous partial downloaded files via segment check sum | +| Calculates check sums on previous downloaded files Default (On) | Verify downloaded files via file level check sum | +| Save Logs: - Download Navigation windows for client downloads | Export download or token log files via drop down and export log button | +| Debug Logging: Default (Off) | Enable debug level logging for file downloads | +| Block Size (Bytes): Default (1048576) | HTTP chunk size transfers | +| Save Interval (Bytes): Default (1000000) | save interval in bytes | +| Auto Retry: Default (On) | Enables auto retries of failed downloads | +| Retry(s): Default (5) | Number of retry attempts to download a file after failure | +| Seconds between Retrys: Default (5) | Number of seconds between retires | + + +# Troubleshooting Guide + +If you encounter issues when using the Data Transfer Tool for downloading files please reference the section below for helpful hints and recommendations. + +## Speed Performance During Download +The Data Transfer Tool has two performance tuning options that are presented during download operations. The two options are: + +* -n OR --n-processes - The "-n" OR "--n-processes" option assists with assigning the number of threads to the download process. The default is 4 and can not be lowered below three threads. + +* --http-chunk-size - The "--http-chunk-size" setting can improve performance but we do not provide any hard settings due to the eclectic nature of client networks and their connections to the internet but instead encourage clients to experiment with changing the default setting of 1048576 bytes to larger size ranges. + +## Very Large Manifests +Some clients have needed to create very large manifest files to satisfy the scope of their work. Using very large manifest files can from time to time lead to the end user experiencing network time outs or dropped connections due to network topologies, internet connections, or load on client side systems. Network time out or dropped network connect can manifest as a hung or unresponsive download session. To help mitigate these issues we recommended that clients breakup their manifest files into smaller chunks. + +## General Tips +* To avoid running into older software bugs/conflicts we recommend you always use the latest version of the client whenever possible. +* When experiencing download problems while using an access tokens try downloading a new token first before reporting it to the GDC Help Desk. + +## Logging +For troubleshooting purposes the GDC User Services Team may request that you run the command line application with the following flags { --debug --log-file }. These flags will run the application in debug mode and create a logfile file with the debug logs in it. +Example Usage: + +=== "Debug-Logfile" + + ```shell + gdc-client download -m lung.manifest.txt -t token.file --debug --log-file logfile.txt + ``` + +## OS Compatibility with the Data Transfer Tool +The Data transfer Tool is offered in three OS compatible versions; Mac OS, Windows, and Ubuntu Linux. We have successfully tested the Ubuntu binary on CentOS 7.x and 8.x, RHEL 7/8 and Scientific Linux 7/8 with the client but have had problems with CentOS 6.x and RHEL 6 and SL6. To work around this problem we have asked users to build their own client from our [github](https://github.com/NCI-GDC/gdc-client) repository with the assistance of an instruction document that we provide on request via the GDC Helpdesk. + +## Network Troubleshooting + +Network problems can appear as dropped network connections or even a stalled application. The GDC Helpdesk might request more network information to assist in diagnosing the problem. The two tests they will request the end user to run are ping and traceroute (tracert on the windows platform) against our api servers. Please capture the output from these tests into a text file and attach it to the reply email. + +Examples: + +=== "Ping" + + ```shell + >ping api.gdc.cancer.gov + PING api.gdc.cancer.gov (192.170.230.246): 56 data bytes + 64 bytes from 192.170.230.246: icmp_seq=0 ttl=249 time=4.235 ms + 64 bytes from 192.170.230.246: icmp_seq=1 ttl=249 time=4.783 ms + ``` + +=== "Traceroute" + + ```shell + >traceroute api.gdc.cancer.gov + traceroute to api.gdc.cancer.gov (192.170.230.246), 64 hops max, 52 byte packets + 1 h01-391-250-v1011.gw.uchicago.net (10.151.0.2) 4.595 ms 3.602 ms 3.322 ms + 2 h01-391-250-to-b65-ll129-300.p2p.uchicago.net (10.5.1.32) 13.285 ms 9.241 ms 5.156 ms + 3 b65-ll129-300-to-borderfw.p2p.uchicago.net (192.170.192.32) 3.218 ms 3.364 ms 3.396 ms + 4 borderfw-to-b65-ll129-500.p2p.uchicago.net (192.170.192.36) 3.605 ms 3.741 ms 3.833 ms + 5 b65-scidmz-01-to-b65-ll129-500.uchicago.net (128.135.247.182) 4.223 ms 5.428 ms 3.999 ms + 6 192.170.224.97 (192.170.224.97) 4.003 ms 3.970 ms 6.260 ms + 7 lnk-g30-scidist-01.scidmz.uchicago.net (192.170.224.66) 4.530 ms 4.649 ms 6.021 ms + 8 192.170.230.246 (192.170.230.246) 4.158 ms 4.273 ms 5.134 ms + ``` + +## Common Error Codes +This is a list of the most common error codes the Data Transfer Tool generates and their meaning +