From b1e27961a181dfefa1dd090f50d705f6ee84033f Mon Sep 17 00:00:00 2001 From: Satchel Baldwin Date: Wed, 18 Sep 2024 14:39:30 -0500 Subject: [PATCH 1/6] add initial gemini integration --- chat-ui/beaker-biome/pyproject.toml | 1 + .../src/beaker_biome/biome/agent.py | 514 +- .../src/beaker_biome/biome/docs.md | 47966 ++++++++++++++++ chat-ui/src/ChatInterface.vue | 4 +- 4 files changed, 48301 insertions(+), 184 deletions(-) create mode 100644 chat-ui/beaker-biome/src/beaker_biome/biome/docs.md diff --git a/chat-ui/beaker-biome/pyproject.toml b/chat-ui/beaker-biome/pyproject.toml index b7ae43d..b9129ce 100644 --- a/chat-ui/beaker-biome/pyproject.toml +++ b/chat-ui/beaker-biome/pyproject.toml @@ -27,6 +27,7 @@ classifiers = [ dependencies = [ "beaker-kernel>=1.6.5", "requests", + "google-generativeai" ] [project.urls] diff --git a/chat-ui/beaker-biome/src/beaker_biome/biome/agent.py b/chat-ui/beaker-biome/src/beaker_biome/biome/agent.py index c799eb1..ab6ee7c 100644 --- a/chat-ui/beaker-biome/src/beaker_biome/biome/agent.py +++ b/chat-ui/beaker-biome/src/beaker_biome/biome/agent.py @@ -11,12 +11,60 @@ from beaker_kernel.lib.agent import BaseAgent from beaker_kernel.lib.context import BaseContext +import google.generativeai as genai logger = logging.getLogger(__name__) BIOME_URL = "http://biome_api:8082" - +disease_types = """ +- adenomas and adenocarcinomas +- ductal and lobular neoplasms +- myeloid leukemias +- epithelial neoplasms, nos +- squamous cell neoplasms +- gliomas +- lymphoid leukemias +- cystic, mucinous and serous neoplasms +- nevi and melanomas +- neuroepitheliomatous neoplasms +- acute lymphoblastic leukemia +- plasma cell tumors +- complex mixed and stromal neoplasms +- mature b-cell lymphomas +- transitional cell papillomas and carcinomas +- not applicable +- osseous and chondromatous neoplasms +- germ cell neoplasms +- mesothelial neoplasms +- not reported +- acinar cell neoplasms +- paragangliomas and glomus tumors +- chronic myeloproliferative disorders +- neoplasms, nos +- thymic epithelial neoplasms +- myomatous neoplasms +- complex epithelial neoplasms +- soft tissue tumors and sarcomas, nos +- lipomatous neoplasms +- meningiomas +- fibromatous neoplasms +- specialized gonadal neoplasms +- unknown +- miscellaneous tumors +- adnexal and skin appendage neoplasms +- basal cell neoplasms +- mucoepidermoid neoplasms +- myelodysplastic syndromes +- nerve sheath tumors +- leukemias, nos +- synovial-like neoplasms +- fibroepithelial neoplasms +- miscellaneous bone tumors +- blood vessel tumors +- mature t- and nk-cell lymphomas +- _missing +""" class BiomeAgent(BaseAgent): """ @@ -34,200 +82,304 @@ class BiomeAgent(BaseAgent): dataframe. """ def __init__(self, context: BaseContext = None, tools: list = None, **kwargs): - libraries = { - } + import os + import pathlib + agent_dir = pathlib.Path(__file__).resolve().parent + + with open(f'{agent_dir}/docs.md', 'r') as f: + docs = f.read() + self.gemini = {} + genai.configure(api_key=os.environ["GEMINI_API_KEY"]) + self.gemini['prompt'] = f""" + You are an assistant who will help me query the genomics data commons API. + You should write clean python code to solve specific queries I pose. You should + write it as though it will be directly used in a Jupyter notebook. You should not + include backticks or "python" at the top of the code blocks, that is unnecessary. + You should not provide explanation unless I ask a follow up question. + Assume pandas is installed and is imported with `import pandas as pd`. Also assume `requests`, `json`, + and `os` are imported properly. + + You will be given the entire API documentation, but first I want to give you a list of available + disease types since this is a common search parameter: + + {disease_types} + + If you are specifying a disease type, it MUST be from this enumerated list. + + Now, I will provide you the extensive API documentation. When you write code against this API + you should avail yourself of the appropriate query parameters, your understanding of the response model + and be cognizant that not all data is public and thus may require a token, etc. When you are downloading + things you should log progress. When you are doing complex things try to break them down and + implement appropriate exception handling. Ok here we go, here are the docs: + + {docs} + """ + self.gemini['model'] = genai.GenerativeModel("gemini-1.5-pro") + self.gemini['chat'] = self.gemini['model'].start_chat( + history=[ + {"role": "user", "parts": self.gemini['prompt']}, + {"role": "model", "parts": "Great to meet you. Let me help you query that API!"}, + ] + ) super().__init__(context, tools, **kwargs) - def update_job_status(self, job_id, status): - self.context.send_response("iopub", - "job_status", { - "job_id": job_id, - "status": status - }, - ) - - # TODO: Formatting of these messages should be left to the Analyst-UI in the future. - async def poll_query(self, job_id: str): - # Poll result - status = "queued" - result = None - while status == "queued": - response = requests.get(f"{BIOME_URL}/jobs/{job_id}").json() - status = response["status"] - sleep(1) - - self.update_job_status(job_id, status) - - #asyncio.create_task(self.poll_query_logs(job_id)) - while status == "started": - response = requests.get(f"{BIOME_URL}/jobs/{job_id}/logs").json() - self.context.send_response("iopub", - "job_logs", { - "job_id": job_id, - "logs": response, - }, - ) - response = requests.get(f"{BIOME_URL}/jobs/{job_id}").json() - status = response["status"] - sleep(5) - - self.update_job_status(job_id, status) - - # Handle result - if status != "finished": - self.update_job_status(job_id, status) - self.context.send_response("iopub", - "job_failure", { - "job_id": job_id, - "response": response - }, - ) - - result = response["result"] # TODO: Bubble up better cell type + def gemini_info(self, info: dict): self.context.send_response("iopub", - "job_response", { - "job_id": job_id, - "response": result['answer'], - "raw": result + "gemini_info", { + "body": info + }, + ) + + def gemini_error(self, error: dict): + self.context.send_response("iopub", + "gemini_error", { + "body": error }, ) - @tool(autosummarize=True) - async def search(self, query: str) -> list[dict[str, Any]]: - """ - Search for data sources in the Biome app. Results will be matched semantically - and string distance. Use this to find a data source. You don't need live - web searches. If the user asks about data sources, use this tool. - - Be sure to use the `display_search` tool for the output. Ensure you always use `display_search` after. - Args: - query (str): The query used to find the datasource. - Returns: - list: A JSON-formatted string containing a list of strings. - The list should contain only the `name` field and no other field - of the data sources found, ordered from most relevant to least relevant. - Ensure that only the name field is present. - An example is provided surrounded in backticks. - ``` - ["Proteomics Data Commons", ""Office of Cancer Clinical Proteomics Research", "UniProt"] - ``` + @tool() + async def interact_with_api(self, goal: str, agent: AgentRef): """ + This tool provides interaction with external APIs with a second agent. + You will query external APIs through this tool and have code returned, which will be executed. + Based on what that code returns and the user's goal, continue to interact with the API to get to that goal. + + If there is an error running the code given back, instruct the second agent + to fix it and run this tool again with a goal of getting that code to successfully run. - endpoint = f"{BIOME_URL}/sources" - response = requests.get(endpoint, params={"query": query}) - raw_sources = response.json()['sources'] - sources = [ - # Include only necessary fields to ensure LLM context length is not exceeded. - { - "id": source["id"], - "name": source["content"]["Web Page Descriptions"]["name"], - "initials": source["content"]["Web Page Descriptions"]["initials"], - "purpose": source["content"]["Web Page Descriptions"]["purpose"], - "links": source["content"]["Information on Links on Web Page"], - "base_url": source.get("base_url", None) - } for source in raw_sources - ] - return sources - - @tool(autosummarize=True) - async def display_search(self, results: list[str], agent:AgentRef, loop: LoopControllerRef): - """ - Once search has been performed, this tool will display it to the user. Args: - results (list[str]): The query used to find the datasource. - """ - # sometimes it wraps the output - if isinstance(results, dict): - results = results.get("results", results) - endpoint = f"{BIOME_URL}/sources" - response = requests.get(endpoint, params={ - "simple_query_string": { - "fields": ["content.Web Page Descriptions.name"], - "query": "|".join(results) - } - }) - raw_sources = response.json()['sources'] - sources = [ - { - "id": source["id"], - "name": source["content"]["Web Page Descriptions"]["name"], - "initials": source["content"]["Web Page Descriptions"]["initials"], - "purpose": source["content"]["Web Page Descriptions"]["purpose"], - "links": source["content"]["Information on Links on Web Page"], - "base_url": source.get("base_url", None), - "logo": source.get("logo", None) - } for source in raw_sources - ] - # match sources to ordering from previous llm step by dict to avoid n^2 - - sources_map = { source.get("name", ""): source for source in sources } - ordered_sources = [sources_map[name] for name in results] - self.context.send_response("iopub", - "data_sources", { - "sources": ordered_sources - }, - ) - loop.set_state(loop.STOP_SUCCESS) - - # TODO(DESIGN): Deal with long running jobs in tools - # - # Option 1: We can return the job id and the agent can poll for the result. - # This will require a job status tool. Once the status is done, we can either - # check the result if it's a query or check the data source if it's a scan. - # This feels a bit messy though that the job creation has a similar return - # output on queue but getting the result is very different for each job. - # - # Option 2: We can wait for the job and return it to the agent when it's done - # - # Option 3: We can maybe leverage new widgets in the Analyst UI?? - # - - # CHOOSING OPTION 1 FOR THE TIME BEING - @tool() - async def query_page(self, task: str, base_url: str, agent: AgentRef, loop: LoopControllerRef): + goal (str): The task given to the second agent """ - Run an action over a *specific* source in the Biome app and return the results. - Find the url from a data source by using `search` tool first and - picking the most relevant one. + self.gemini_info({'goal': goal}) + try: + agent_response = self.gemini['chat'].send_message(goal).text + prefixes = ['```python', '```'] + suffixes = ['```', '```\n'] + for prefix in prefixes: + if agent_response.startswith(prefix): + agent_response = agent_response[len(prefix):] + for suffix in suffixes: + if agent_response.endswith(suffix): + agent_response = agent_response[:-len(suffix)] + agent_response = '\n'.join([ + 'import pandas as pd', + 'import os', + 'import json', + 'import requests', + agent_response + ]) + + except Exception as e: + self.gemini_error({'error': str(e)}) + return + self.gemini_info({'response': agent_response}) + try: + evaluation = await agent.context.evaluate(agent_response) + except Exception as e: + self.gemini_error({'error': str(e)}) + agent.add_context(f"The second agent failed to create valid code. Instruct it to rerun. The error was {str(e)}.") + return + self.gemini_info({'eval': str(evaluation)}) + if evaluation.get('result', {}).get('status', None) != 'ok': + agent.add_context("The second agent failed to create valid code. Instruct it to rerun.") + return + agent.add_context(""" + The code ran successfully. If this completes the user's original task, inform them and stop. + If there is still more to do, inform the user and proceed with the next API interaction + using the results from what you just ran. + """) - This kicks off a long-running job so you'll have to just return the ID to the user - instead of the result. + # def update_job_status(self, job_id, status): + # self.context.send_response("iopub", + # "job_status", { + # "job_id": job_id, + # "status": status + # }, + # ) - This can be used to ask questions about a data source or download some kind - of artifact from it. This tool just kicks off a job where an AI crawls the website - and performs the task. + # # TODO: Formatting of these messages should be left to the Analyst-UI in the future. + # async def poll_query(self, job_id: str): + # # Poll result + # status = "queued" + # result = None + # while status == "queued": + # response = requests.get(f"{BIOME_URL}/jobs/{job_id}").json() + # status = response["status"] + # sleep(1) + + # self.update_job_status(job_id, status) - Args: - task (str): Task given in natural language to perform over URL. - base_url (str): URL to run query over. - """ - response = requests.post( f"{BIOME_URL}/jobs/query", json={"user_task": task, "url": base_url}) - job_id = response.json()["job_id"] - self.context.send_response("iopub", - "job_create", { - "job_id": job_id, - "task": task, - "url": base_url - }, - ) - asyncio.create_task(self.poll_query(job_id)) - loop.set_state(loop.STOP_SUCCESS) + # #asyncio.create_task(self.poll_query_logs(job_id)) + # while status == "started": + # response = requests.get(f"{BIOME_URL}/jobs/{job_id}/logs").json() + # self.context.send_response("iopub", + # "job_logs", { + # "job_id": job_id, + # "logs": response, + # }, + # ) + # response = requests.get(f"{BIOME_URL}/jobs/{job_id}").json() + # status = response["status"] + # sleep(5) - @tool() - async def scan(self, base_url: str, agent:AgentRef, loop: LoopControllerRef) -> str: - """ - Profiles the given web page and adds it to the data sources in the Biome app. + # self.update_job_status(job_id, status) - This kicks off a long-running job so you'll have to just return the ID to the user - instead of the result. + # # Handle result + # if status != "finished": + # self.update_job_status(job_id, status) + # self.context.send_response("iopub", + # "job_failure", { + # "job_id": job_id, + # "response": response + # }, + # ) - Args: - base_url (str): The url to scan and add as a data source. - Returns: - str: Job ID to poll for the result. - """ - response = requests.post( f"{BIOME_URL}/jobs/scan", json={"uris": [base_url]}) - job_id = response.json()["job_id"] - asyncio.create_task(self.poll_query(job_id)) - return job_id \ No newline at end of file + # result = response["result"] # TODO: Bubble up better cell type + # self.context.send_response("iopub", + # "job_response", { + # "job_id": job_id, + # "response": result['answer'], + # "raw": result + # }, + # ) + + # @tool(autosummarize=True) + # async def search(self, query: str) -> list[dict[str, Any]]: + # """ + # Search for data sources in the Biome app. Results will be matched semantically + # and string distance. Use this to find a data source. You don't need live + # web searches. If the user asks about data sources, use this tool. + + # Be sure to use the `display_search` tool for the output. Ensure you always use `display_search` after. + + # Args: + # query (str): The query used to find the datasource. + # Returns: + # list: A JSON-formatted string containing a list of strings. + # The list should contain only the `name` field and no other field + # of the data sources found, ordered from most relevant to least relevant. + # Ensure that only the name field is present. + # An example is provided surrounded in backticks. + # ``` + # ["Proteomics Data Commons", ""Office of Cancer Clinical Proteomics Research", "UniProt"] + # ``` + # """ + + # endpoint = f"{BIOME_URL}/sources" + # response = requests.get(endpoint, params={"query": query}) + # raw_sources = response.json()['sources'] + # sources = [ + # # Include only necessary fields to ensure LLM context length is not exceeded. + # { + # "id": source["id"], + # "name": source["content"]["Web Page Descriptions"]["name"], + # "initials": source["content"]["Web Page Descriptions"]["initials"], + # "purpose": source["content"]["Web Page Descriptions"]["purpose"], + # "links": source["content"]["Information on Links on Web Page"], + # "base_url": source.get("base_url", None) + # } for source in raw_sources + # ] + # return sources + + # @tool(autosummarize=True) + # async def display_search(self, results: list[str], agent:AgentRef, loop: LoopControllerRef): + # """ + # Once search has been performed, this tool will display it to the user. + # Args: + # results (list[str]): The query used to find the datasource. + # """ + # # sometimes it wraps the output + # if isinstance(results, dict): + # results = results.get("results", results) + # endpoint = f"{BIOME_URL}/sources" + # response = requests.get(endpoint, params={ + # "simple_query_string": { + # "fields": ["content.Web Page Descriptions.name"], + # "query": "|".join(results) + # } + # }) + # raw_sources = response.json()['sources'] + # sources = [ + # { + # "id": source["id"], + # "name": source["content"]["Web Page Descriptions"]["name"], + # "initials": source["content"]["Web Page Descriptions"]["initials"], + # "purpose": source["content"]["Web Page Descriptions"]["purpose"], + # "links": source["content"]["Information on Links on Web Page"], + # "base_url": source.get("base_url", None), + # "logo": source.get("logo", None) + # } for source in raw_sources + # ] + # # match sources to ordering from previous llm step by dict to avoid n^2 + + # sources_map = { source.get("name", ""): source for source in sources } + # ordered_sources = [sources_map[name] for name in results] + # self.context.send_response("iopub", + # "data_sources", { + # "sources": ordered_sources + # }, + # ) + # loop.set_state(loop.STOP_SUCCESS) + + # # TODO(DESIGN): Deal with long running jobs in tools + # # + # # Option 1: We can return the job id and the agent can poll for the result. + # # This will require a job status tool. Once the status is done, we can either + # # check the result if it's a query or check the data source if it's a scan. + # # This feels a bit messy though that the job creation has a similar return + # # output on queue but getting the result is very different for each job. + # # + # # Option 2: We can wait for the job and return it to the agent when it's done + # # + # # Option 3: We can maybe leverage new widgets in the Analyst UI?? + # # + + # # CHOOSING OPTION 1 FOR THE TIME BEING + # @tool() + # async def query_page(self, task: str, base_url: str, agent: AgentRef, loop: LoopControllerRef): + # """ + # Run an action over a *specific* source in the Biome app and return the results. + # Find the url from a data source by using `search` tool first and + # picking the most relevant one. + + # This kicks off a long-running job so you'll have to just return the ID to the user + # instead of the result. + + # This can be used to ask questions about a data source or download some kind + # of artifact from it. This tool just kicks off a job where an AI crawls the website + # and performs the task. + + # Args: + # task (str): Task given in natural language to perform over URL. + # base_url (str): URL to run query over. + # """ + # response = requests.post( f"{BIOME_URL}/jobs/query", json={"user_task": task, "url": base_url}) + # job_id = response.json()["job_id"] + # self.context.send_response("iopub", + # "job_create", { + # "job_id": job_id, + # "task": task, + # "url": base_url + # }, + # ) + # asyncio.create_task(self.poll_query(job_id)) + # loop.set_state(loop.STOP_SUCCESS) + + # @tool() + # async def scan(self, base_url: str, agent:AgentRef, loop: LoopControllerRef) -> str: + # """ + # Profiles the given web page and adds it to the data sources in the Biome app. + + # This kicks off a long-running job so you'll have to just return the ID to the user + # instead of the result. + + # Args: + # base_url (str): The url to scan and add as a data source. + # Returns: + # str: Job ID to poll for the result. + # """ + # response = requests.post( f"{BIOME_URL}/jobs/scan", json={"uris": [base_url]}) + # job_id = response.json()["job_id"] + # asyncio.create_task(self.poll_query(job_id)) + # return job_id diff --git a/chat-ui/beaker-biome/src/beaker_biome/biome/docs.md b/chat-ui/beaker-biome/src/beaker_biome/biome/docs.md new file mode 100644 index 0000000..a9bec19 --- /dev/null +++ b/chat-ui/beaker-biome/src/beaker_biome/biome/docs.md @@ -0,0 +1,47966 @@ +#Preparing for Data Downloads and Uploads + +## Overview + +The GDC Data Transfer Tool is intended to be used in conjunction with the [GDC Data Portal](https://portal.gdc.cancer.gov) and the [GDC Data Submission Portal](https://portal.gdc.cancer.gov/submission/) to transfer data to or from the GDC. First, the GDC Data Portal's interface is used to generate a manifest file or obtain UUID(s) and (for Controlled-Access Data) an authentication token. The GDC Data Transfer Tool is then used to transfer the data files listed in the manifest file or identified by UUID(s). + +##Downloads +### Obtaining a Manifest File for Data Download + +The GDC Data Transfer Tool supports downloading multiple files listed in a GDC manifest file. Manifest files can be generated and downloaded directly from the GDC Data Portal: + +First, select the data files of interest. Click the *Cart* button in the row corresponding to the file desired. The button will turn blue to indicate that the file has been selected. + +![GDC Data Portal: Selecting Files of Interest](images/06-20_Data-Portal-File-Selection.png "Selecting Files of Interest") + + +Once all files of interest have been selected, click on the *Cart* button in the upper right-hand corner. This will bring up the cart page, which provides an overview of all currently selected files. This list of files can be downloaded as a manifest file by clicking on the *Download Cart* button and selecting *Manifest* from the drop down. + +![GDC Data Portal: Cart Page](images/06-20-v2_Data-Portal-Cart-Page.png) + +### Obtaining UUIDs for Data Download + +A manifest file is not required to download files from GDC. The GDC Data Transfer Tool will accept file UUID(s) instead of a manifest file for downloading individual data files. To obtain a data file's UUID from the GDC Data Portal, click the file name to find its detail page including its GDC UUID. + +![GDC Data Portal: Detailed File Page](images/06-20_Data-portal-file-detail-pagev2.png) + + +### Obtaining an Authentication Token for Data Downloads + +The GDC Data Transfer Tool requires an authentication token to download from GDC data portal to download Controlled-Access Data. Tokens can be generated and downloaded directly from the GDC Data Portal. + +To generate a token, first log in to the GDC Data Portal by clicking the *Login* button in the top right corner of the page. This will redirect to the eRA Commons login page. After successful authentication, the GDC Data Portal will display the username in place of the *Login* button. Here, the user Ian Miller is logged in to the GDC Data Portal, indicated by the username IANMILLER. + +Clicking the username will open a drop-down menu. Select *Download Token* from the menu to generate an authentication token. + +![GDC Data Portal User Dropdown Menu](images/06-20_auth_example_download_token-2.png) + + +**NOTE:** The authentication token should be kept in a secure location, as it allows access to all data accessible by the associated user. + +##Uploads +### Obtaining a Manifest File for Data Uploads +Multiple data file uploads are supported by the GDC Data Transfer Tool via a manifest file. Manifest files can be generated and downloaded directly from the GDC Submission Portal. A project's manifest file can be downloaded from the projects's dashboard. + +![GDC Submission Portal Manifest Download](images/10-10-16_manifest_upload.png) + + +**NOTE:** To download a project's manifest file click on the _Download Manifest_ button located on the home page of the project, just below the four status charts. A manifest will be generated for the entire project or if previous files have already been upload only the files that remain to be uploaded. + +A manifest for individual files can also be downloaded from the transaction tab and browse tab pages of the submission portal's project. More information on the process can be found under the Submission Portal's documentation section entitled [Uploading the Submittable Data File to the GDC](https://docs.gdc.cancer.gov/Data_Submission_Portal/Users_Guide/Data_Submission_Walkthrough/#uploading-the-submittable-data-file-to-the-gdc). + +### Obtaining UUIDs for Data Uploads +A UUID can be used for data submission with the Data Transfer Tool. The UUID for submittable data uploads can be obtained from the Submission Portal or from the API GraphQL endpoint. In the Submission Portal the UUID for a data file can be found in the Manifest YAML file located in the _id:_ row located under the file size entry. + +![Submission Manifest yaml file](images/10-18_yaml_submission_UUID_example.png) + + + A second location to obtain a UUID in the Submission Portal is on the Browse Tab page. Under the Submittable Data Files section a UUID can be found by opening up the file's detail page. By clicking on the Submitter ID of the upload file a new window will display a Summary of the file's details, which contains the UUID. + +![Submission Portal Browse Page Details](images/submission_portal_browse_page_UUID.png) + +###GraphQL + +A UUID can be obtained from the API GraphQL endpoint. An overview of what GraphQL and its uses is located on the API documentation page section [Querying Submitted Data Using GraphQL](https://docs.gdc.cancer.gov/API/Users_Guide/Submission/#querying-submitted-data-using-graphql) + +The following example will query the endpoint to produce a UUID along with submitter_id, file_name, and project_id. + +=== "GraphQl_Bare" + + ```GraphQL + { + submitted_unaligned_reads (project_id: "GDC-INTERNAL", submitter_id: "Blood-00001-aliquot_lane1_barcode23.fastq") { + id + submitter_id + file_name + project_id + } + } + ``` + +=== "Escaped_Json" + + ```json + {\n submitted_unaligned_reads (project_id: \"GDC-INTERNAL\", submitter_id: \"Blood-00001-aliquot_lane1_barcode23.fastq\") {\n id\n submitter_id\n file_name\n project_id\n}\n}\n + + ``` + +=== "Query_json" + + ```json + { + "query": "{\n \n submitted_unaligned_reads (project_id: \"GDC-INTERNAL\", submitter_id: \"Blood-00001-aliquot_lane1_barcode23.fastq\") {\n id\n submitter_id\n file_name\n project_id\n}\n}", + "variables": null + } + ``` + +=== "Shell_command" + + ```Shell + export token=ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTOKEN-01234567890+AlPhAnUmErIcToKeN=0123456789-ALPHANUMERICTO + $ curl --request POST --header "X-Auth-Token: $token" 'https://api.gdc.cancer.gov/v0/submission/graphql' -d@data.json + ``` + +=== "API_Response" + + ```json + { + "data": { + "submitted_unaligned_reads": [ + { + "file_name": "dummy.fastq", + "id": "616eab2f-791a-4641-8cd6-ee195a10a201", + "project_id": "GDC-INTERNAL", + "submitter_id": "Blood-00001-aliquot_lane1_barcode23.fastq" + } + ] + } + ``` + +### Obtaining an Authentication Token for Data Uploads +While biospecimen and clinical metadata may be uploaded via the GDC Data Submission Portal, file upload must be done using the Data Transfer Tool or API. An authentication token is required for data upload and can be generated on the GDC Data Submission Portal. + +To generate a token, first log in to the GDC Data Submission Portal by clicking the *Login* button in the top right corner of the page. This will create a popup window that will redirect to the eRA Commons login page. After successful authentication, the GDC Submission Portal will display the username in place of the *Login* button. Here, the user Ian Miller is logged in to the GDC Submission Portal, indicated by the username IANMILLER. + +Clicking the username will open a drop-down menu. Select *Download Token* from the menu to generate an authentication token. + +![GDC Submission Portal User Dropdown Menu](images/10-27_Submission_Portal_Auth_Download_Tab.png) + + + +#Data Downloads with the Data Transfer Tool UI + +##Data Transfer Tool UI: Overview +The UI version of the Data Transfer Tool was created for users who prefer a graphical interface over the command line or have limited command line experience. The command line version is recommended for those users with more command line experience, require large data transfers of GDC data, or need to download a large numbers of data files. + +###System Recommendations + +The system recommendations for using the GDC Data Transfer Tool are as follows: + +* OS: Linux (Ubuntu 14.x or later), OS X (10.9 Mavericks or later), or Windows (7 or later) +* CPU: At least four 64-bit cores, Intel or AMD +* RAM: At least 2 GiB +* Storage: Enterprise-class storage system capable of at least 1 Gb/s (gigabit per second) write throughput and sufficient free space for BAM files. + +###Binary Distributions + +Binary distributions are available on the GDC Transfer Tool page. To install the GDC Data Transfer UI download the respective binary distribution and unzip the distribution's archive to a location on the target system that is easily accessible. + +###Binary Installation +Once the binary has been positioned in an appropriate location on the client's file system the application will need to run though a one-time installation process. On first execution the binary install splash screen will appear showing the progress of the installation. A hidden directory is created within the user's home directory labeled dtt that holds configuration and executable files. + +![GDC DTT UI Installation](images/GDC_DTT_UI_INSTALLv7.png "GDC Data Transfer Tool UI Install") + + +###Preparing for Data Download + +The GDC Data Transfer Tool UI is a stand-alone client application intended to work with data file information stored on the GDC Data Portals. Data download information must first be gathered from the GDC Data Portal. From there a manifest file can be [generated](https://docs.gdc.cancer.gov/Data_Transfer_Tool/Users_Guide/Preparing_for_Data_Download_and_Upload/#obtaining-a-manifest-file-for-data-download) to supply the client. Alternatively, individual file UUIDs can be provided to the UUID entry window located on the Download tab in the client. + +![GDC DTT UI Start Page](images/DTT_UI_Start_Page.png) +##Downloads with UUIDs +The Data Transfer Tool UI can download files by individual UUID. UUIDs can be entered into the client while on the download tab. The single entry field labeled "Enter UUID(s)" allows the user to enter UUIDs individually. + +To obtain a data file's UUID from the GDC Data Portal, click on the file name to display the file's summary page which includes vital information such as its GDC UUID. + +##Downloads with Manifest +A portal-generated manifest file can be used with the Data Transfer Tool UI. From the Download tab home page click on the Select Manifest File button. A file system search window will popup allowing navigation to the manifest file. + +![GDC DTT UI Manifest Button Example](images/Manifest_button_DTT_UI_Start_Window.png "GDC Data Transfer Tool UI Manifest Button") + +##Download Progress Page +The Download Progress Page is the command console for the Data Transfer Tool UI and allows users to monitor downloads. Progress of all downloads including the ability to start, stop, and restart a download are performed on the Download Progress Page. Once file UUIDs or a manifest has been added to the queue the download can be started by clicking on the download button located at the lower right hand side of the page. + +![GDC DTT UI Download Progress Page_Download](images/Download_Progress_Page_download.png) + +Once a download has completed, information about the downloads can be viewed from the Completed tab located at the bottom of the page. Any Stopped or Failed downloads can also be viewed from their respective labels located at the bottom of the Status page. + +![DTT_UI_Download_Completed_Tab](images/DTT_UI_download_completed.png) + +##Controlled Access File Downloads + +Some files in the GDC are controlled access. If you require access to these files please review the process outlined in the documentation [Obtaining Access to Controlled Data](https://gdc.cancer.gov/access-data/obtaining-access-controlled-data). After appropriate authorization has been granted an access token can be generated to allow the Data Transfer Tool UI application access to the requested data files. Documentation explaining the process of generating a token is located in the [Obtaining an Authentication Token for Data Downloads](https://docs.gdc.cancer.gov/Data_Transfer_Tool/Users_Guide/Preparing_for_Data_Download_and_Upload/#obtaining-an-authentication-token-for-data-downloads). Once a token has been downloaded to a secure location on the client's local filesystem the Data Transfer Tool UI can now access it. + +![No_Token_icon](images/No_Token_file_dtt_ui.png) + +The current status of client authorization is viewable in the upper right corner of the application. If the image and wording on the token manager access button is in red then no valid client token file has been uploaded. To upload a valid token file click on the token status button. The token manager window should appear allowing either a drag and drop token file upload or a file navigation window can be opened to navigate to the file location. + +![Token Manager Window](images/Token_Manager_Window.png) + +The token manager will verify access and display the projects for which the user has access. To complete the token upload process click on the save button within the Token Manager window. + +![Valid Token](images/validated_token.png) + +##Settings and Advanced Settings + +While the default download options will work for the majority of use cases, there are a vareity of ways to customize or modify the download process within the DTT UI. Details of each of the settings are listed below. + +![DTT Settings and Advanced Settings Page](images/DTT_Settings_Page.png) + +| Settings | Details +|----------|---------| +| Server URL | Default: https://api.gdc.cancer.gov | +| Number of Client Connections: Default (3) | Number of concurrent client threads | +| Destination Folder: Default (User's Home Directory) | User selectable download file location | +| Calculate Inbound Segment and check Md5sum on Restart: Default (On) | Verify previous partial downloaded files via segment check sum | +| Calculates check sums on previous downloaded files Default (On) | Verify downloaded files via file level check sum | +| Save Logs: - Download Navigation windows for client downloads | Export download or token log files via drop down and export log button | +| Debug Logging: Default (Off) | Enable debug level logging for file downloads | +| Block Size (Bytes): Default (1048576) | HTTP chunk size transfers | +| Save Interval (Bytes): Default (1000000) | save interval in bytes | +| Auto Retry: Default (On) | Enables auto retries of failed downloads | +| Retry(s): Default (5) | Number of retry attempts to download a file after failure | +| Seconds between Retrys: Default (5) | Number of seconds between retires | + + +# Troubleshooting Guide + +If you encounter issues when using the Data Transfer Tool for downloading files please reference the section below for helpful hints and recommendations. + +## Speed Performance During Download +The Data Transfer Tool has two performance tuning options that are presented during download operations. The two options are: + +* -n OR --n-processes - The "-n" OR "--n-processes" option assists with assigning the number of threads to the download process. The default is 4 and can not be lowered below three threads. + +* --http-chunk-size - The "--http-chunk-size" setting can improve performance but we do not provide any hard settings due to the eclectic nature of client networks and their connections to the internet but instead encourage clients to experiment with changing the default setting of 1048576 bytes to larger size ranges. + +## Very Large Manifests +Some clients have needed to create very large manifest files to satisfy the scope of their work. Using very large manifest files can from time to time lead to the end user experiencing network time outs or dropped connections due to network topologies, internet connections, or load on client side systems. Network time out or dropped network connect can manifest as a hung or unresponsive download session. To help mitigate these issues we recommended that clients breakup their manifest files into smaller chunks. + +## General Tips +* To avoid running into older software bugs/conflicts we recommend you always use the latest version of the client whenever possible. +* When experiencing download problems while using an access tokens try downloading a new token first before reporting it to the GDC Help Desk. + +## Logging +For troubleshooting purposes the GDC User Services Team may request that you run the command line application with the following flags { --debug --log-file }. These flags will run the application in debug mode and create a logfile file with the debug logs in it. +Example Usage: + +=== "Debug-Logfile" + + ```shell + gdc-client download -m lung.manifest.txt -t token.file --debug --log-file logfile.txt + ``` + +## OS Compatibility with the Data Transfer Tool +The Data transfer Tool is offered in three OS compatible versions; Mac OS, Windows, and Ubuntu Linux. We have successfully tested the Ubuntu binary on CentOS 7.x and 8.x, RHEL 7/8 and Scientific Linux 7/8 with the client but have had problems with CentOS 6.x and RHEL 6 and SL6. To work around this problem we have asked users to build their own client from our [github](https://github.com/NCI-GDC/gdc-client) repository with the assistance of an instruction document that we provide on request via the GDC Helpdesk. + +## Network Troubleshooting + +Network problems can appear as dropped network connections or even a stalled application. The GDC Helpdesk might request more network information to assist in diagnosing the problem. The two tests they will request the end user to run are ping and traceroute (tracert on the windows platform) against our api servers. Please capture the output from these tests into a text file and attach it to the reply email. + +Examples: + +=== "Ping" + + ```shell + >ping api.gdc.cancer.gov + PING api.gdc.cancer.gov (192.170.230.246): 56 data bytes + 64 bytes from 192.170.230.246: icmp_seq=0 ttl=249 time=4.235 ms + 64 bytes from 192.170.230.246: icmp_seq=1 ttl=249 time=4.783 ms + ``` + +=== "Traceroute" + + ```shell + >traceroute api.gdc.cancer.gov + traceroute to api.gdc.cancer.gov (192.170.230.246), 64 hops max, 52 byte packets + 1 h01-391-250-v1011.gw.uchicago.net (10.151.0.2) 4.595 ms 3.602 ms 3.322 ms + 2 h01-391-250-to-b65-ll129-300.p2p.uchicago.net (10.5.1.32) 13.285 ms 9.241 ms 5.156 ms + 3 b65-ll129-300-to-borderfw.p2p.uchicago.net (192.170.192.32) 3.218 ms 3.364 ms 3.396 ms + 4 borderfw-to-b65-ll129-500.p2p.uchicago.net (192.170.192.36) 3.605 ms 3.741 ms 3.833 ms + 5 b65-scidmz-01-to-b65-ll129-500.uchicago.net (128.135.247.182) 4.223 ms 5.428 ms 3.999 ms + 6 192.170.224.97 (192.170.224.97) 4.003 ms 3.970 ms 6.260 ms + 7 lnk-g30-scidist-01.scidmz.uchicago.net (192.170.224.66) 4.530 ms 4.649 ms 6.021 ms + 8 192.170.230.246 (192.170.230.246) 4.158 ms 4.273 ms 5.134 ms + ``` + +## Common Error Codes +This is a list of the most common error codes the Data Transfer Tool generates and their meaning +