From 7c4c50cb88c1dc1cf20740ae73eed6b324c30688 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Sun, 17 Mar 2019 06:39:36 +0000 Subject: [PATCH 01/32] Latest version copied to anamailyan fork --- const.py | 162 +++++++++---------- database.py | 350 ++++++++++++++++++++++++++++++++++++----- load_local_database.py | 2 +- query_bigquery.py | 2 +- schema.graphql | 34 +++- schema.py | 84 +++++++--- utilities.py | 106 ++++++++++--- 7 files changed, 577 insertions(+), 163 deletions(-) diff --git a/const.py b/const.py index 00131c7..e524d1d 100644 --- a/const.py +++ b/const.py @@ -15,7 +15,7 @@ class DB(object): def __init__(self): import schema - self.CURRENT_CASEANNOTATION = schema.CaseAnnotation() # not currently used + self.CURRENT_CASE_ANNOTATION = schema.CaseAnnotation() # not currently used self.CURRENT_USER = schema.User(username="null") # no users are currently being used @@ -37,49 +37,103 @@ def __init__(self): self.CURRENT_PROJECTS = { "1":schema.Project(id="1", project_id="NHSII-DemoA", name="NHSII-DemoA", program=self.CURRENT_PROGRAMS[0], summary=schema.Summary(case_count=2, file_count=6, - data_categories=self.DATA_CATEGORIES, experimental_strategies=self.EXPERIMENTAL_STRATEGIES, file_size=15), + data_categories=self.DATA_CATEGORIES, experimental_strategies=[self.EXPERIMENTAL_STRATEGIES[0]], file_size=15), primary_site=["Stool"]), "2":schema.Project(id="2", project_id="NHSII-DemoB", name="NHSII-DemoB", program=self.CURRENT_PROGRAMS[0], summary=schema.Summary(case_count=2, file_count=6, - data_categories=self.DATA_CATEGORIES, experimental_strategies=self.EXPERIMENTAL_STRATEGIES, file_size=15), + data_categories=self.DATA_CATEGORIES, experimental_strategies=[self.EXPERIMENTAL_STRATEGIES[1]], file_size=15), primary_site=["Stool"]), } self.CURRENT_FILE_CASES = { - "1":schema.FileCase(1,"Case1",self.CURRENT_PROJECTS["1"]), - "2":schema.FileCase(2,"Case2",self.CURRENT_PROJECTS["1"]), - "3":schema.FileCase(3,"Case3",self.CURRENT_PROJECTS["2"]), - "4":schema.FileCase(4,"Case4",self.CURRENT_PROJECTS["2"]), + "1":schema.FileCase(1,"Case1",self.CURRENT_PROJECTS["1"], + schema.Demographic("not hispanic or latino","male","white"), "Stool"), + "2":schema.FileCase(2,"Case2",self.CURRENT_PROJECTS["1"], + schema.Demographic("not hispanic or latino","male","white"), "Stool"), + "3":schema.FileCase(3,"Case3",self.CURRENT_PROJECTS["2"], + schema.Demographic("not hispanic or latino","female","white"), "Stool"), + "4":schema.FileCase(4,"Case4",self.CURRENT_PROJECTS["2"], + schema.Demographic("hispanic or latino","female","white"), "Stool") } + self.CASE_FILES = { + "1": schema.CaseFile(1,experimental_strategy="WMGX", + data_category="Raw Reads",data_format="Fastq", + platform="Illumina MiSeq",access="controlled"), + "2": schema.CaseFile(2,experimental_strategy="WMGX", + data_category="Taxonomic Profile",data_format="TSV", + platform="Illumina MiSeq",access="open"), + "3": schema.CaseFile(3,experimental_strategy="WMGX", + data_category="Gene Families",data_format="TSV", + platform="Illumina MiSeq",access="open"), + "4": schema.CaseFile(4,experimental_strategy="WMGX", + data_category="Raw Reads",data_format="Fastq", + platform="Illumina MiSeq",access="controlled"), + "5": schema.CaseFile(5,experimental_strategy="WMGX", + data_category="Taxonomic Profile",data_format="TSV", + platform="Illumina MiSeq",access="open"), + "6": schema.CaseFile(6,experimental_strategy="WMGX", + data_category="Gene Families",data_format="TSV", + platform="Illumina MiSeq",access="open"), + "7": schema.CaseFile(7,experimental_strategy="16S", + data_category="Raw Reads",data_format="Fastq", + platform="Illumina HiSeq",access="controlled"), + "8": schema.CaseFile(8,experimental_strategy="16S", + data_category="Taxonomic Profile",data_format="TSV", + platform="Illumina HiSeq",access="open"), + "9": schema.CaseFile(9,experimental_strategy="16S", + data_category="Gene Families",data_format="TSV", + platform="Illumina HiSeq",access="open"), + "10": schema.CaseFile(10,experimental_strategy="16S", + data_category="Raw Reads",data_format="Fastq", + platform="Illumina HiSeq",access="controlled"), + "11": schema.CaseFile(11,experimental_strategy="16S", + data_category="Taxonomic Profile",data_format="TSV", + platform="Illumina HiSeq",access="open"), + "12": schema.CaseFile(12,experimental_strategy="16S", + data_category="Gene Families",data_format="TSV", + platform="Illumina HiSeq",access="open"), + } self.FILE_SIZES = { "gene": 300000000, "raw": 5000000000, "taxa": 200000000 } self.TEST_FILES = { "1": schema.File(1, "demoA_sample1_raw_reads.fastq","case1","sample1", "controlled", - self.FILE_SIZES["raw"], "Raw Reads", "Illumina", "Fastq", "WMGX", schema.FileCases(hits=["1"])), + self.FILE_SIZES["raw"], "Raw Reads", "Fastq", "Illumina MiSeq", "WMGX", + "demoA_sample1_raw_reads.fastq", schema.FileCases(hits=[self.CURRENT_FILE_CASES["1"]])), "2": schema.File(2, "demoA_sample1_taxonomic_profile.tsv","case1","sample1", "open", - self.FILE_SIZES["taxa"], "Taxonomic Profile", "Illumina", "TSV", "WMGX", schema.FileCases(hits=["1"])), + self.FILE_SIZES["taxa"], "Taxonomic Profile", "TSV", "Illumina MiSeq", "WMGX", + "demoA_sample1_taxonomic_profile.tsv", schema.FileCases(hits=[self.CURRENT_FILE_CASES["1"]])), "3": schema.File(3, "demoA_sample1_gene_families.tsv","case1","sample1", "open", - self.FILE_SIZES["gene"], "Gene Families", "Illumina", "TSV", "WMGX", schema.FileCases(hits=["1"])), + self.FILE_SIZES["gene"], "Gene Families", "TSV", "Illumina MiSeq", "WMGX", + "demoA_sample1_gene_families.tsv", schema.FileCases(hits=[self.CURRENT_FILE_CASES["1"]])), "4": schema.File(4, "demoA_sample2_raw_reads.fastq","case2","sample2", "controlled", - self.FILE_SIZES["raw"], "Raw Reads", "Illumina", "Fastq", "WMGX", schema.FileCases(hits=["2"])), + self.FILE_SIZES["raw"], "Raw Reads", "Fastq", "Illumina MiSeq", "WMGX", + "demoA_sample2_raw_reads.fastq", schema.FileCases(hits=[self.CURRENT_FILE_CASES["2"]])), "5": schema.File(5, "demoA_sample2_taxonomic_profile.tsv","case2","sample2", "open", - self.FILE_SIZES["taxa"], "Taxonomic Profile", "Illumina", "TSV", "WMGX", schema.FileCases(hits=["2"])), + self.FILE_SIZES["taxa"], "Taxonomic Profile", "TSV", "Illumina MiSeq", "WMGX", + "demoA_sample2_taxonomic_profile.tsv", schema.FileCases(hits=[self.CURRENT_FILE_CASES["2"]])), "6": schema.File(6, "demoA_sample2_gene_families.tsv","case2","sample2", "open", - self.FILE_SIZES["gene"], "Gene Families", "Illumina", "TSV", "WMGX", schema.FileCases(hits=["2"])), + self.FILE_SIZES["gene"], "Gene Families", "TSV", "Illumina MiSeq", "WMGX", + "demoA_sample2_gene_families.tsv", schema.FileCases(hits=[self.CURRENT_FILE_CASES["2"]])), "7": schema.File(7, "demoB_sample3_raw_reads.fastq","case3","sample3", "controlled", - self.FILE_SIZES["raw"], "Raw Reads", "Illumina", "Fastq", "WMGX", schema.FileCases(hits=["3"])), + self.FILE_SIZES["raw"], "Raw Reads", "Fastq", "Illumina HiSeq", "16S", + "demoB_sample3_raw_reads.fastq", schema.FileCases(hits=[self.CURRENT_FILE_CASES["3"]])), "8": schema.File(8, "demoB_sample3_taxonomic_profile.tsv","case3","sample3", "open", - self.FILE_SIZES["taxa"], "Taxonomic Profile", "Illumina", "TSV", "WMGX", schema.FileCases(hits=["3"])), + self.FILE_SIZES["taxa"], "Taxonomic Profile", "TSV", "Illumina HiSeq","16S", + "demoB_sample3_taxonomic_profile.tsv", schema.FileCases(hits=[self.CURRENT_FILE_CASES["3"]])), "9": schema.File(9, "demoB_sample3_gene_families.tsv","case3","sample3", "open", - self.FILE_SIZES["gene"], "Gene Families", "Illumina", "TSV", "WMGX", schema.FileCases(hits=["3"])), + self.FILE_SIZES["gene"], "Gene Families", "TSV", "Illumina HiSeq", "16S", + "demoB_sample3_gene_families.tsv", schema.FileCases(hits=[self.CURRENT_FILE_CASES["3"]])), "10": schema.File(10, "demoB_sample4_raw_reads.fastq","case4","sample4", "controlled", - self.FILE_SIZES["raw"], "Raw Reads", "Illumina", "Fastq", "WMGX", schema.FileCases(hits=["4"])), + self.FILE_SIZES["raw"], "Raw Reads", "Fastq", "Illumina HiSeq", "16S", + "demoB_sample4_raw_reads.fastq", schema.FileCases(hits=[self.CURRENT_FILE_CASES["4"]])), "11": schema.File(11, "demoB_sample4_taxonomic_profile.tsv","case4","sample4", "open", - self.FILE_SIZES["taxa"], "Taxonomic Profile", "Illumina", "TSV", "WMGX", schema.FileCases(hits=["4"])), + self.FILE_SIZES["taxa"], "Taxonomic Profile", "TSV", "Illumina HiSeq", "16S", + "demoB_sample4_taxonomic_profile.tsv", schema.FileCases(hits=[self.CURRENT_FILE_CASES["4"]])), "12": schema.File(12, "demoB_sample4_gene_families.tsv","case4","sample4", "open", - self.FILE_SIZES["gene"], "Gene Families", "Illumina", "TSV", "WMGX", schema.FileCases(hits=["4"])), + self.FILE_SIZES["gene"], "Gene Families", "TSV", "Illumina HiSeq", "16S", + "demoB_sample4_gene_families.tsv", schema.FileCases(hits=[self.CURRENT_FILE_CASES["4"]])), } self.CURRENT_COUNTS = schema.Count( @@ -95,80 +149,28 @@ def __init__(self): demographic=schema.Demographic("not hispanic or latino","male","white"), project=self.CURRENT_PROJECTS["1"], summary=schema.Summary(case_count=1,file_count=1,file_size=1, - data_categories=self.DATA_CATEGORIES_SINGLE_CASE)), + data_categories=self.DATA_CATEGORIES_SINGLE_CASE), + files=schema.CaseFiles(hits=[self.CASE_FILES["1"],self.CASE_FILES["2"],self.CASE_FILES["3"]])), "2":schema.Case(2,case_id="Case2",primary_site="Stool", demographic=schema.Demographic("not hispanic or latino","male","white"), project=self.CURRENT_PROJECTS["1"], summary=schema.Summary(case_count=1,file_count=1,file_size=1, - data_categories=self.DATA_CATEGORIES_SINGLE_CASE)), + data_categories=self.DATA_CATEGORIES_SINGLE_CASE), + files=schema.CaseFiles(hits=[self.CASE_FILES["4"],self.CASE_FILES["5"],self.CASE_FILES["6"]])), "3":schema.Case(3,case_id="Case3",primary_site="Stool", demographic=schema.Demographic("not hispanic or latino","female","white"), project=self.CURRENT_PROJECTS["2"], summary=schema.Summary(case_count=1,file_count=1,file_size=1, - data_categories=self.DATA_CATEGORIES_SINGLE_CASE)), + data_categories=self.DATA_CATEGORIES_SINGLE_CASE), + files=schema.CaseFiles(hits=[self.CASE_FILES["7"],self.CASE_FILES["8"],self.CASE_FILES["9"]])), "4":schema.Case(4,case_id="Case4",primary_site="Stool", - demographic=schema.Demographic("not hispanic or latino","female","white"), + demographic=schema.Demographic("hispanic or latino","female","white"), project=self.CURRENT_PROJECTS["2"], summary=schema.Summary(case_count=1,file_count=1,file_size=1, - data_categories=self.DATA_CATEGORIES_SINGLE_CASE)) + data_categories=self.DATA_CATEGORIES_SINGLE_CASE), + files=schema.CaseFiles(hits=[self.CASE_FILES["10"],self.CASE_FILES["11"],self.CASE_FILES["12"]])) } self.CURRENT_FILES = schema.Files(hits=self.TEST_FILES.keys()) self.CURRENT_CASES = schema.RepositoryCases(hits=self.TEST_CASES.keys()) - self.FILE_AGGREGATIONS=schema.FileAggregations( - data_category=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=4, key="Raw Reads"), - schema.Bucket(doc_count=4, key="Taxonomic Profile"), - schema.Bucket(doc_count=4, key="Gene Families")]), - experimental_strategy=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=6, key="WMGX"), - schema.Bucket(doc_count=6, key="16S")]), - data_format=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=4, key="Fastq"), - schema.Bucket(doc_count=8, key="TSV")]), - platform=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=6, key="Illumina MiSeq"), - schema.Bucket(doc_count=6, key="Illumina HiSeq")]), - cases__primary_site=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=12, key="Stool")]), - cases__project__project_id=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=6, key="NHSII-DemoA"), - schema.Bucket(doc_count=6, key="NHSII-DemoB")]), - access=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=4, key="open"), - schema.Bucket(doc_count=8, key="controlled")])) - - self.CASE_AGGREGATIONS=schema.CaseAggregations( - demographic__ethnicity=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=10, key="not hispanic or latino"), - schema.Bucket(doc_count=2, key="hispanic or latino")]), - demographic__gender=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=8, key="male"), - schema.Bucket(doc_count=4, key="female")]), - demographic__race=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=8, key="white"), - schema.Bucket(doc_count=4, key="asian")]), - primary_site=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=12, key="Stool")]), - project__project_id=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=6, key="NHSII-DemoA"), - schema.Bucket(doc_count=6, key="NHSII-DemoB")]), - project__program__name=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=12, key="NHSII")]), - ) - - self.PROJECT_AGGREGATIONS=schema.ProjectAggregations( - primary_site=schema.Aggregations(buckets=[schema.Bucket(doc_count=45, key="Stool")]), - program__name=schema.Aggregations(buckets=[schema.Bucket(doc_count=45, key="NHSII")]), - project_id=schema.Aggregations(buckets=[schema.Bucket(doc_count=15, key="NHSII-DemoA"), - schema.Bucket(doc_count=15, key="NHSII-DemoB"), - schema.Bucket(doc_count=15, key="NHSII-DemoC")]), - summary__data_categories__data_category=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=15, key="Raw Reads"), - schema.Bucket(doc_count=15, key="Gene Families"), - schema.Bucket(doc_count=15, key="Taxonomic Profiles")]), - summary__experimental_strategies__experimental_strategy=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=30, key="WMGX"), - schema.Bucket(doc_count=15, key="16S")])) - diff --git a/database.py b/database.py index c248c32..a45fd1a 100644 --- a/database.py +++ b/database.py @@ -1,76 +1,346 @@ +# Calls to obtain values from database -# Calls to obtain values from the data structures (to be populated by the local database next) +import mysql.connector as mariadb +import json -import const +def add_key_increment(dictionary, key): + if not key in dictionary: + dictionary[key]=0 + dictionary[key]+=1 class Data(object): - def __getattr__(self, name): - # load data if not already loaded - if name == "data": - self.load_data() + db_conn = mariadb.connect(user='root', password='DanDa1osh', db='portal_ui') + cursor = db_conn.cursor(buffered=True) - def load_data(self): - self.data = const.DB() - - def get_total_projects_count(self): - return len(self.data.CURRENT_PROJECTS.keys()) + def fetch_results(self,query): + self.cursor.execute(query) + row_headers=[x[0] for x in self.cursor.description] + rows = self.cursor.fetchall() + json_data=[] + for row in rows: + json_data.append(dict(zip(row_headers,row))) + return json_data def get_current_projects(self): - return [self.get_project(project_id) for project_id in self.data.CURRENT_PROJECTS.keys()] + projects_json=self.fetch_results("select `id` from `project`") + print("json_data inside get_current_projects") + print(projects_json) + return [self.get_project(project['id']) for project in projects_json] def get_user(self): - return self.data.CURRENT_USER - - def get_filecase(self,id): - return self.data.CURRENT_FILE_CASES[id] + import schema + return schema.User(username="null") def get_project(self,id): - return self.data.CURRENT_PROJECTS[id] + import schema - def get_file(self,id): - return self.data.TEST_FILES[id] + print("id inside get_project") + print(id) + project_json=self.fetch_results( + "select id, project_id, name as pr_name, program from `project` where `id` =" + str(id)) + project_id=project_json[0]['project_id'] + proj_id=project_json[0]['id'] + proj_name=project_json[0]['pr_name'] + proj_program=["HPFS"] + #proj_program=project_json[0]['program'] + #proj_primary_site=project_json[0]['primary_site'] + proj_primary_site=["Stool"] + + file_count_json=self.fetch_results("select count(id) as file_count from file_sample where project='"+project_id+"'") + proj_file_count=file_count_json[0]['file_count'] + + part_count_json=self.fetch_results( + "select count(distinct participant) as part_count from file_sample where project='"+project_id+"'") + proj_part_count=part_count_json[0]['part_count'] + + data_cat_json=self.fetch_results("select distinct data_category from file_sample where project='"+project_id+"'") + proj_data_cat=[data_cat['data_category'] for data_cat in data_cat_json] - def get_total_files(self): - return len(self.data.CURRENT_FILES.hits) + exp_str_json=self.fetch_results("select distinct experimental_strategy from file_sample where project='"+project_id+"'") + proj_exp_str=[exp_str['experimental_strategy'] for exp_str in exp_str_json] + + sum_file_size_json=self.fetch_results( + "select sum(file_size) as sum_file_size from file_sample where project='"+project_id+"'") + proj_file_size=sum_file_size_json[0]['sum_file_size'] + print("values inside get_project") + print( + proj_id,project_id,proj_name,proj_program,proj_part_count,proj_file_count,proj_data_cat,proj_exp_str,proj_file_size,proj_primary_site) + + return schema.Project( + id=proj_id, + project_id=project_id, + name=proj_name, + program=proj_program, + summary=schema.Summary( + case_count=proj_part_count, + file_count=proj_file_count, + data_categories=proj_data_cat, + experimental_strategies=proj_exp_str, + file_size=proj_file_size), + primary_site=proj_primary_site) + + + def get_file(self,id): + import schema - def get_total_case_annotations(self): - return 1 # not currently being used + file_json=self.fetch_results("select * from `file_sample` where `id` ="+ str(id)) + project_json=self.fetch_results( + "select id , project_id, primary_site from project where project_id='"+file_json[0]['project']+"'") + part_json=self.fetch_results( + "select id, entity_participant_id from participant where entity_participant_id='"+file_json[0]['participant']+"'") + print("file and project json inside get_file") + print(file_json[0]['sample']) + print(project_json[0]['project_id']) + return schema.File( + id=file_json[0]['id'], + name=file_json[0]['file_name'], + file_id=file_json[0]['file_id'], + participant=file_json[0]['participant'], + sample=file_json[0]['sample'], + access=file_json[0]['access'], + file_size=file_json[0]['file_size'], + data_category=file_json[0]['data_category'], + data_format=file_json[0]['data_format'], + platform=file_json[0]['platform'], + experimental_strategy=file_json[0]['experimental_strategy'], + file_name=file_json[0]['file_name'], + cases=schema.FileCases( + hits=[schema.FileCase( + part_json[0]['id'], + case_id=file_json[0]['participant'], + project=self.get_project(project_json[0]['id']), + demographic=schema.Demographic("not hispanic or latino","male","white"), + #primary_site=project_json[0]['primary_site'] + primary_site="Stool")])) def get_case_annotation(self): - return self.data.CURRENT_CASE_ANNOTATION + import schema + return schema.CaseAnnotation() def get_current_files(self): - return self.data.CURRENT_FILES + import schema + + files_json=self.fetch_results("select `id` from `file_sample`") + file_hits=[] + for item in files_json: + file_hits.append(item['id']) + print("file_hits inside current files") + print(file_hits) + return schema.Files(hits=file_hits) def get_current_cases(self): - return self.data.CURRENT_CASES + import schema + + cases_json=self.fetch_results("select id from participant") + return schema.RepositoryCases(hits=[case['id'] for case in cases_json]) def get_case(self,id): - return self.data.TEST_CASES[id] + import schema + + case_json=self.fetch_results("select * from participant where id="+str(id)) + part_id=case_json[0]['entity_participant_id'] + + case_count_json=self.fetch_results( + "select count(entity_participant_id) as case_count from participant where entity_participant_id="+str(part_id)) + case_count=case_count_json[0]['case_count'] + + file_count_json=self.fetch_results( + "select count(id) as file_count from file_sample where participant="+str(part_id)) + file_count=file_count_json[0]['file_count'] + + file_size_json=self.fetch_results( + "select sum(file_size) as filesize from file_sample where participant="+str(part_id)) + file_size=file_size_json[0]['filesize'] + + print("inside get case") + print(part_id) + proj_json=self.fetch_results('''select distinct file_sample.project, project.id + from file_sample, project where file_sample.participant='''+str(part_id)+''' + and project.project_id=file_sample.project''') + proj_id=proj_json[0]['id'] + print(proj_id) + + cat_raw_json=self.fetch_results( + "select count(id) as raw_count from file_sample where participant="+str(part_id)+" and data_category='Raw Reads'") + cat_raw_count=cat_raw_json[0]['raw_count'] + + cat_taxa_json=self.fetch_results( + "select count(id) as taxa_count from file_sample where participant="+str(part_id)+" and data_category='Taxonomic Profiles'") + cat_taxa_count=cat_taxa_json[0]['taxa_count'] + + cat_gene_json=self.fetch_results( + "select count(id) as gene_count from file_sample where participant="+str(part_id)+" and data_category='Gene Families'") + cat_gene_count=cat_gene_json[0]['gene_count'] + + files_json=self.fetch_results("select * from file_sample where participant="+str(part_id)) + + return schema.Case(id, + case_id="entity_participant_id", + primary_site="Stool", + demographic=schema.Demographic("not hispanic or latino","male","white"), + project=self.get_project(proj_id), + summary=schema.Summary( + case_count=case_count, + file_count=file_count, + file_size=file_size, + data_categories=[schema.DataCategories(file_count=cat_raw_count, data_category="Raw Reads"), + schema.DataCategories(file_count=cat_gene_count, data_category="Gene Families"), + schema.DataCategories(file_count=cat_taxa_count, data_category="Taxonomic Profiles")]), + files=schema.CaseFiles(hits=[schema.CaseFile( + case_file['id'], + experimental_strategy=case_file['experimental_strategy'], + data_category=case_file['data_category'], + data_format=case_file['data_format'], + platform=case_file['platform'], + access=case_file['access']) for case_file in files_json])) + + + def get_project_aggregations(self, projects): + import schema + + # compile aggregations from project + aggregates = {"primary_site": {}, "program__name": {}, + "project_id": {}, + "summary__data_categories__data_category": {}, + "summary__experimental_strategies__experimental_strategy": {}} + print("projects inside aggregations") + + for project in projects: + print( + project.id, project.project_id,project.summary.data_categories,project.primary_site, + project.program,project.summary.experimental_strategies) + add_key_increment(aggregates["primary_site"],project.primary_site[0]) + add_key_increment(aggregates["project_id"], project.project_id) + add_key_increment(aggregates["program__name"], project.program[0]) + for item in project.summary.data_categories: + add_key_increment(aggregates["summary__data_categories__data_category"], item) + for item in project.summary.experimental_strategies: + add_key_increment(aggregates["summary__experimental_strategies__experimental_strategy"], item) + + project_aggregates=schema.ProjectAggregations( + primary_site=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["primary_site"].items()]), + project_id=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["project_id"].items()]), + program__name=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["program__name"].items()]), + summary__data_categories__data_category=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["summary__data_categories__data_category"].items()]), + summary__experimental_strategies__experimental_strategy=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["summary__experimental_strategies__experimental_strategy"].items()])) + + return project_aggregates - def get_project_aggregations(self): - return self.data.PROJECT_AGGREGATIONS def get_current_counts(self): - return self.data.CURRENT_COUNTS + import schema + + projects_json = self.fetch_results("select count(`project_id`)as projects_count from project") + projects = projects_json[0]['projects_count'] + + participants_json = self.fetch_results("select count(`entity_participant_id`) as participants_count from `participant`") + participants = participants_json[0]['participants_count'] + + samples_json = self.fetch_results("select count(`sample`) as samples_count from `sample`") + samples = samples_json[0]['samples_count'] + + dataFormats_json = self.fetch_results("select count(distinct `data_format`) as data_format_count from `file_sample`") + dataFormats = dataFormats_json[0]['data_format_count'] + + rawFiles_json = self.fetch_results( + "select count(`id`) as rawfiles_count from `file_sample` where type='rawFiles'") + rawFiles = rawFiles_json[0]['rawfiles_count'] - def get_file_aggregations(self): - return self.data.FILE_AGGREGATIONS + prFiles_json = self.fetch_results( + "select count(`id`) as prfiles_count from `file_sample` where type='processedFiles'") + prFiles = prFiles_json[0]['prfiles_count'] - def get_case_aggregations(self): - return self.data.CASE_AGGREGATIONS + return schema.Count( + projects=projects, + participants=participants, + samples=samples, + dataFormats=dataFormats, + rawFiles=rawFiles, + processedFiles=prFiles) - def get_total_cases_per_file(self): - return 1 # this is currently not being used - def get_total_cases(self): - return len(self.data.TEST_CASES.keys()) + def get_file_aggregations(self, files): + import schema + + # aggregate file data + aggregates = {"data_category": {}, "experimental_strategy": {}, + "data_format": {}, "platform": {}, "cases__primary_site": {}, + "cases__project__project_id": {}, "access": {}} + + for file in files: + print(" values inside file aggregation") + print(file.data_category,file.experimental_strategy,file.data_format,file.platform,file.access) + add_key_increment(aggregates["data_category"], file.data_category) + add_key_increment(aggregates["experimental_strategy"], file.experimental_strategy) + add_key_increment(aggregates["data_format"], file.data_format) + add_key_increment(aggregates["platform"], file.platform) + add_key_increment(aggregates["access"], file.access) + project = file.cases.hits[0].project + add_key_increment(aggregates["cases__primary_site"], project.primary_site[0]) + add_key_increment(aggregates["cases__project__project_id"], project.project_id) + + file_aggregates = schema.FileAggregations( + data_category=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["data_category"].items()]), + experimental_strategy=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["experimental_strategy"].items()]), + data_format=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["data_format"].items()]), + platform=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["platform"].items()]), + cases__primary_site=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["cases__primary_site"].items()]), + access=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["access"].items()]), + cases__project__project_id=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["cases__project__project_id"].items()])) + + return file_aggregates + + def get_case_aggregations(self, cases): + import schema + + # aggregate case data + aggregates = {"demographic__ethnicity": {}, "demographic__gender": {}, + "demographic__race": {}, "primary_site": {}, "project__project_id": {}, + "project__program__name": {}} + + for case in cases: + add_key_increment(aggregates["demographic__ethnicity"], case.demographic.ethnicity) + add_key_increment(aggregates["demographic__gender"], case.demographic.gender) + add_key_increment(aggregates["demographic__race"], case.demographic.race) + add_key_increment(aggregates["primary_site"], case.primary_site) + add_key_increment(aggregates["project__project_id"], case.project.project_id) + add_key_increment(aggregates["project__program__name"], case.project.program.name) + + case_aggregates=schema.CaseAggregations( + demographic__ethnicity=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["demographic__ethnicity"].items()]), + demographic__gender=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["demographic__gender"].items()]), + demographic__race=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["demographic__race"].items()]), + primary_site=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["primary_site"].items()]), + project__project_id=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["project__project_id"].items()]), + project__program__name=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["project__program__name"].items()])) + + return case_aggregates def get_facets(self): return "null" # this is not currently being used def get_cart_file_size(self): - return self.data.CURRENT_FILE_SIZE + import schema -data = Data() + file_json=self.fetch_results("select sum(file_size) as sum_size from file_sample") + return schema.FileSize(file_json[0]['sum_size']) +data = Data() diff --git a/load_local_database.py b/load_local_database.py index f83f0f0..944566d 100644 --- a/load_local_database.py +++ b/load_local_database.py @@ -23,7 +23,7 @@ def parse_arguments(args): required=False) parser.add_argument( "--key-file", - default="biom-mass-fdcadb440fdf.json", + default="/home/hutlab_public/compute_engine_service_account_key/biom-mass-8dc9ab934396.json", help="google project service account private key file path\n]", required=False) parser.add_argument( diff --git a/query_bigquery.py b/query_bigquery.py index 2d179ab..770c7ad 100644 --- a/query_bigquery.py +++ b/query_bigquery.py @@ -22,7 +22,7 @@ def parse_arguments(args): required=False) parser.add_argument( "--key-file", - default="biom-mass-fdcadb440fdf.json", + default="/home/hutlab_public/compute_engine_service_account_key/biom-mass-8dc9ab934396.json", help="google project service account private key file path\n]", required=False) parser.add_argument( diff --git a/schema.graphql b/schema.graphql index 3d41bba..fe55875 100644 --- a/schema.graphql +++ b/schema.graphql @@ -26,6 +26,7 @@ type Case implements Node { project: Project summary: Summary annotations: CaseAnnotations + files: CaseFiles } type CaseAggregations { @@ -75,6 +76,35 @@ type CaseEdge { cursor: String! } +type CaseFile implements Node { + # The ID of the object. + id: ID! + case_id: String + experimental_strategy: String + data_category: String + data_format: String + platform: String + access: String +} + +type CaseFileConnection { + pageInfo: PageInfo! + edges: [CaseFileEdge]! + total: Int +} + +type CaseFileEdge { + # The item at the end of the edge + node: CaseFile + + # A cursor for use in pagination + cursor: String! +} + +type CaseFiles { + hits(first: Int, offset: Int, sort: [Sort], filters: FiltersArgument, before: String, after: String, last: Int): CaseFileConnection +} + type Count { projects: String participants: String @@ -128,9 +158,9 @@ type File implements Node { data_format: String platform: String experimental_strategy: String + file_name: String cases: FileCases file_id: String - file_name: String type: String } @@ -149,6 +179,8 @@ type FileCase implements Node { id: ID! case_id: String project: Project + demographic: Demographic + primary_site: String } type FileCaseConnection { diff --git a/schema.py b/schema.py index a7e1403..13b23d4 100644 --- a/schema.py +++ b/schema.py @@ -86,12 +86,19 @@ class Meta: def get_node(cls, info, id): return data.get_project(id) +class Demographic(graphene.ObjectType): + ethnicity = graphene.String() + gender = graphene.String() + race = graphene.String() + class FileCase(graphene.ObjectType): class Meta: interfaces = (graphene.relay.Node,) case_id = graphene.String() project = graphene.Field(Project) + demographic = graphene.Field(Demographic) + primary_site = graphene.String() class FileCaseConnection(graphene.relay.Connection): class Meta: @@ -100,7 +107,7 @@ class Meta: total = graphene.Int() def resolve_total(self, info): - return data.get_total_cases_per_file() + return len(self.edges) class FileCases(graphene.ObjectType): hits = graphene.relay.ConnectionField(FileCaseConnection, @@ -109,9 +116,6 @@ class FileCases(graphene.ObjectType): sort=graphene.List(Sort), filters=FiltersArgument()) - def resolve_hits(self, info, first=None, offset=None, sort=None, filters=None): - return [data.get_filecase(file_id) for file_id in self.hits] - class File(graphene.ObjectType): class Meta: interfaces = (graphene.relay.Node,) @@ -125,19 +129,16 @@ class Meta: data_format = graphene.String() platform = graphene.String() experimental_strategy = graphene.String() + file_name = graphene.String() cases = graphene.Field(FileCases) file_id = graphene.String() - file_name = graphene.String() type = graphene.String() def resolve_file_id(self, info): return self.name - def resolve_file_name(self, info): - return self.name - def resolve_type(self, info): return self.data_format @@ -151,7 +152,7 @@ class Meta: total = graphene.Int() def resolve_total(self, info): - return data.get_total_projects_count() + return len(self.edges) class FileConnection(graphene.relay.Connection): class Meta: @@ -159,7 +160,7 @@ class Meta: total = graphene.Int() def resolve_total(self, info): - return data.get_total_files() + return len(self.edges) class Bucket(graphene.ObjectType): doc_count = graphene.Int() @@ -196,10 +197,15 @@ class Files(graphene.ObjectType): aggregations_filter_themselves=graphene.Boolean()) def resolve_hits(self, info, first=None, score=None, offset=None, sort=None, filters=None): - return [data.get_file(file_id) for file_id in self.hits] + all_files = [data.get_file(file_id) for file_id in self.hits] + filtered_files = utilities.filter_hits(all_files, filters, "files") + sorted_files = utilities.sort_hits(filtered_files, sort) + return sorted_files def resolve_aggregations(self, info, filters=None, aggregations_filter_themselves=None): - return data.get_file_aggregations() + all_files = [data.get_file(file_id) for file_id in self.hits] + filtered_files = utilities.filter_hits(all_files, filters, "files") + return data.get_file_aggregations(filtered_files) def resolve_facets(self, info, filters=None, facets=None): return data.get_facets() @@ -226,7 +232,7 @@ class Meta: total = graphene.Int() def resolve_total(self, info): - return data.get_total_case_annotations() + return len(self.edges) class CaseAnnotations(graphene.ObjectType): hits = graphene.relay.ConnectionField(CaseAnnotationConnection, @@ -237,12 +243,35 @@ class CaseAnnotations(graphene.ObjectType): filters=FiltersArgument()) def resolve_hits(self, info, first=None, score=None, offset=None, sort=None, filters=None): + # filters are not currently in use return data.get_case_annotation() -class Demographic(graphene.ObjectType): - ethnicity = graphene.String() - gender = graphene.String() - race = graphene.String() +class CaseFile(graphene.ObjectType): + class Meta: + interfaces = (graphene.relay.Node,) + + case_id = graphene.String() + experimental_strategy = graphene.String() + data_category = graphene.String() + data_format = graphene.String() + platform = graphene.String() + access = graphene.String() + +class CaseFileConnection(graphene.relay.Connection): + class Meta: + node = CaseFile + + total = graphene.Int() + + def resolve_total(self, info): + return len(self.edges) + +class CaseFiles(graphene.ObjectType): + hits = graphene.relay.ConnectionField(CaseFileConnection, + first=graphene.Int(), + offset=graphene.Int(), + sort=graphene.List(Sort), + filters=FiltersArgument()) class Case(graphene.ObjectType): class Meta: @@ -257,6 +286,8 @@ class Meta: summary = graphene.Field(Summary) annotations = graphene.Field(CaseAnnotations) + files = graphene.Field(CaseFiles) + def resolve_submitter_id(self, info): return self.case_id @@ -267,7 +298,7 @@ class Meta: total = graphene.Int() def resolve_total(self, info): - return data.get_total_cases() + return len(self.edges) class RepositoryCases(graphene.ObjectType): hits = graphene.relay.ConnectionField(CaseConnection, @@ -284,10 +315,16 @@ class RepositoryCases(graphene.ObjectType): facets=graphene.List(graphene.String)) def resolve_hits(self, info, first=None, score=None, offset=None, sort=None, filters=None): - return [data.get_case(case_id) for case_id in self.hits] + all_cases = [data.get_case(case_id) for case_id in self.hits] + filtered_cases = utilities.filter_hits(all_cases, filters, "cases") + sorted_cases = utilities.sort_hits(filtered_cases, sort) + return sorted_cases def resolve_aggregations(self, info, filters=None, aggregations_filter_themselves=None): - return data.get_case_aggregations() + all_cases = [data.get_case(case_id) for case_id in self.hits] + filtered_cases = utilities.filter_hits(all_cases, filters, "cases") + case_aggregations = data.get_case_aggregations(filtered_cases) + return case_aggregations def resolve_facets(self, info, filters=None, facets=None): return data.get_facets() @@ -321,11 +358,14 @@ class Projects(graphene.ObjectType): def resolve_hits(self, info, first=None, offset=None, sort=None, filters=None): projects = data.get_current_projects() - filtered_projects = utilities.filter_hits(projects, filters) + filtered_projects = utilities.filter_hits(projects, filters, "projects") return filtered_projects def resolve_aggregations(self, info, filters=None, aggregations_filter_themselves=None): - return data.get_project_aggregations() + projects = data.get_current_projects() + filtered_projects = utilities.filter_hits(projects, filters, "projects") + project_aggregations = data.get_project_aggregations(filtered_projects) + return project_aggregations class FileSize(graphene.ObjectType): value = graphene.Float() diff --git a/utilities.py b/utilities.py index 651257e..72d8f3d 100644 --- a/utilities.py +++ b/utilities.py @@ -1,40 +1,110 @@ import json +import ast import collections -def get_class_member_value(obj,members,offset=1): - levels = members.split(".")[offset:] - final_value = getattr(obj, levels.pop(0)) +def get_class_member_value(obj, levels, subset=False): + final_value = obj for item in levels: final_value = getattr(final_value, item) - if isinstance(final_value, list): - sub_levels = ".".join(levels[levels.index(item)+1:]) - final_value = [get_class_member_value(sub_value,sub_levels,offset=0) for sub_value in final_value] + sub_levels = levels[levels.index(item)+1:] + if not isinstance(sub_levels, list): + sub_levels = [sub_levels] + final_value = [get_class_member_value(sub_value,sub_levels,subset=True) for sub_value in final_value] break - if not isinstance(final_value, list) and offset !=0: + if not isinstance(final_value, list) and not subset: final_value = [final_value] return final_value -def filter_hits(hits, filters): +def check_for_match(value_selected, hit_values): + # check if the values for the hit match one or more of the values selected + + match = False + if isinstance(value_selected, list): + for item in value_selected: + if item in hit_values: + match = True + break + elif value_selected in hit_values: + match = True + + return match + +def filter_hits(hits, filters, object_name): # Filter the hits based on the json string provided if not filters: return hits - - filtered_set = set() + + all_filtered_sets = [] for content in filters["content"]: field=content["content"]["field"] - value_selected=content["content"]["value"][0] - for item in hits: - hit_values=get_class_member_value(item,field) - if value_selected in hit_values: - filtered_set.add(item) - return list(filtered_set) - -# The functions were based on code from + value_selected=content["content"]["value"] + + # check if this filter is for this object + levels = field.split(".") + top_level = levels.pop(0) + if top_level != object_name: + if not top_level in dir(hits[0]): + all_filtered_sets.append(set(hits)) + else: + filtered_set=set() + for item in hits: + # find object that contains the search values + hit_values = [] + for search_item in getattr(item, top_level).hits: + hit_values+=get_class_member_value(search_item, levels) + if check_for_match(value_selected, hit_values): + filtered_set.add(item) + all_filtered_sets.append(list(filtered_set)) + else: + filtered_set=set() + for item in hits: + hit_values=get_class_member_value(item,levels) + if check_for_match(value_selected, hit_values): + filtered_set.add(item) + all_filtered_sets.append(list(filtered_set)) + + # reduce sets + final_set = set(all_filtered_sets.pop(0)) + for next_set in all_filtered_sets: + final_set = final_set.intersection(next_set) + + return list(final_set) + +def sort_hits(hits, sort): + # Sort the hits based on the string provided + + if not sort: + return hits + + sorted_hits = hits + for content in sort: + eval_content = ast.literal_eval(content) + field=eval_content["field"] + levels=field.split(".") + + # reverse sort order if set + order=eval_content["order"] + reverse_sort_order = False + if order != "asc": + reverse_sort_order = True + + # get the values for all of the hits + hit_value_pairs = [] + for item in sorted_hits: + hit_value_pairs.append((item, get_class_member_value(item,levels)[0])) + + hit_values = collections.OrderedDict(hit_value_pairs) + + sorted_hits=sorted(hit_values, key=hit_values.get, reverse=reverse_sort_order) + + return sorted_hits + +# The functions below were based on code from # https://github.com/nderkach/python-grahql-api def _json_object_hook(d): From bbc1397bccd2d33af6231af5c7a57a8cee08d2cd Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Sun, 17 Mar 2019 06:41:44 +0000 Subject: [PATCH 02/32] Change db user --- database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database.py b/database.py index a45fd1a..e44230f 100644 --- a/database.py +++ b/database.py @@ -9,7 +9,7 @@ def add_key_increment(dictionary, key): dictionary[key]+=1 class Data(object): - db_conn = mariadb.connect(user='root', password='DanDa1osh', db='portal_ui') + db_conn = mariadb.connect(user='biom-mass', password='', db='portal_ui') cursor = db_conn.cursor(buffered=True) def fetch_results(self,query): From c39a5d8e16e08be9074af943ba7030647638593f Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Sun, 17 Mar 2019 19:42:29 +0000 Subject: [PATCH 03/32] More details to project --- database.py | 87 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 63 insertions(+), 24 deletions(-) diff --git a/database.py b/database.py index e44230f..6411d80 100644 --- a/database.py +++ b/database.py @@ -9,7 +9,7 @@ def add_key_increment(dictionary, key): dictionary[key]+=1 class Data(object): - db_conn = mariadb.connect(user='biom-mass', password='', db='portal_ui') + db_conn = mariadb.connect(user='biom_mass', password='DanDa1osh', db='portal_ui') cursor = db_conn.cursor(buffered=True) def fetch_results(self,query): @@ -37,12 +37,12 @@ def get_project(self,id): print("id inside get_project") print(id) project_json=self.fetch_results( - "select id, project_id, name as pr_name, program from `project` where `id` =" + str(id)) + "select * from `project` where `id` =" + str(id)) project_id=project_json[0]['project_id'] proj_id=project_json[0]['id'] - proj_name=project_json[0]['pr_name'] + proj_name=project_json[0]['name'] proj_program=["HPFS"] - #proj_program=project_json[0]['program'] + proj_program=project_json[0]['program'] #proj_primary_site=project_json[0]['primary_site'] proj_primary_site=["Stool"] @@ -53,33 +53,72 @@ def get_project(self,id): "select count(distinct participant) as part_count from file_sample where project='"+project_id+"'") proj_part_count=part_count_json[0]['part_count'] - data_cat_json=self.fetch_results("select distinct data_category from file_sample where project='"+project_id+"'") - proj_data_cat=[data_cat['data_category'] for data_cat in data_cat_json] - - exp_str_json=self.fetch_results("select distinct experimental_strategy from file_sample where project='"+project_id+"'") - proj_exp_str=[exp_str['experimental_strategy'] for exp_str in exp_str_json] - sum_file_size_json=self.fetch_results( "select sum(file_size) as sum_file_size from file_sample where project='"+project_id+"'") proj_file_size=sum_file_size_json[0]['sum_file_size'] print("values inside get_project") print( - proj_id,project_id,proj_name,proj_program,proj_part_count,proj_file_count,proj_data_cat,proj_exp_str,proj_file_size,proj_primary_site) + proj_id,project_id,proj_name,proj_program,proj_part_count,proj_file_count,proj_file_size,proj_primary_site) return schema.Project( id=proj_id, project_id=project_id, name=proj_name, - program=proj_program, + program=schema.Program(name=proj_program), summary=schema.Summary( case_count=proj_part_count, file_count=proj_file_count, - data_categories=proj_data_cat, - experimental_strategies=proj_exp_str, + data_categories=self.get_data_categories("project",project_id), + experimental_strategies=self.get_experimental_strategies("project",project_id), file_size=proj_file_size), primary_site=proj_primary_site) + def get_data_categories(self,table,id): + import schema + + print("inside get_data_cat") + print(id) + + data_cat_json=self.fetch_results( + "select distinct data_category from file_sample where "+ table+"='" + str(id)+"'") + data_cat=[] + for item in data_cat_json: + item_files_json=self.fetch_results( + "select count(id) as file_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") + item_part_json=self.fetch_results( + "select count(distinct participant) as case_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") + data_cat.append( + schema.DataCategories( + case_count=item_part_json[0]['case_count'], + file_count=item_files_json[0]['file_count'], + data_category=item['data_category'])) + + return data_cat + + + + def get_experimental_strategies(self,table,id): + import schema + + print("inside get exp_str") + print(table,id) + exp_str_json=self.fetch_results( + "select distinct experimental_strategy from file_sample where "+ table+"='" + str(id) +"'") + exp_str=[] + for item in exp_str_json: + item_files_json=self.fetch_results( + "select count(id) as file_count from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") + item_part_json=self.fetch_results( + "select count(distinct participant) as case_count from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") + exp_str.append( + schema.ExperimentalStrategies( + case_count=item_part_json[0]['case_count'], + file_count=item_files_json[0]['file_count'], + experimental_strategy=item['experimental_strategy'])) + + return exp_str + def get_file(self,id): import schema @@ -154,10 +193,10 @@ def get_case(self,id): print("inside get case") print(part_id) - proj_json=self.fetch_results('''select distinct file_sample.project, project.id + proj_json=self.fetch_results('''select distinct file_sample.project, project.id as proj_id from file_sample, project where file_sample.participant='''+str(part_id)+''' and project.project_id=file_sample.project''') - proj_id=proj_json[0]['id'] + proj_id=proj_json[0]['proj_id'] print(proj_id) cat_raw_json=self.fetch_results( @@ -175,7 +214,7 @@ def get_case(self,id): files_json=self.fetch_results("select * from file_sample where participant="+str(part_id)) return schema.Case(id, - case_id="entity_participant_id", + case_id=part_id, primary_site="Stool", demographic=schema.Demographic("not hispanic or latino","male","white"), project=self.get_project(proj_id), @@ -207,15 +246,15 @@ def get_project_aggregations(self, projects): for project in projects: print( - project.id, project.project_id,project.summary.data_categories,project.primary_site, - project.program,project.summary.experimental_strategies) + project.id, project.project_id,project.primary_site, + project.program) add_key_increment(aggregates["primary_site"],project.primary_site[0]) add_key_increment(aggregates["project_id"], project.project_id) - add_key_increment(aggregates["program__name"], project.program[0]) + add_key_increment(aggregates["program__name"], project.program.name) for item in project.summary.data_categories: - add_key_increment(aggregates["summary__data_categories__data_category"], item) + add_key_increment(aggregates["summary__data_categories__data_category"], item.data_category) for item in project.summary.experimental_strategies: - add_key_increment(aggregates["summary__experimental_strategies__experimental_strategy"], item) + add_key_increment(aggregates["summary__experimental_strategies__experimental_strategy"], item.experimental_strategy) project_aggregates=schema.ProjectAggregations( primary_site=schema.Aggregations( @@ -314,9 +353,9 @@ def get_case_aggregations(self, cases): add_key_increment(aggregates["demographic__ethnicity"], case.demographic.ethnicity) add_key_increment(aggregates["demographic__gender"], case.demographic.gender) add_key_increment(aggregates["demographic__race"], case.demographic.race) - add_key_increment(aggregates["primary_site"], case.primary_site) + add_key_increment(aggregates["primary_site"], case.primary_site[0]) add_key_increment(aggregates["project__project_id"], case.project.project_id) - add_key_increment(aggregates["project__program__name"], case.project.program.name) + add_key_increment(aggregates["project__program__name"], case.project.program[0]) case_aggregates=schema.CaseAggregations( demographic__ethnicity=schema.Aggregations( From fe154ecc079e7a498c8c0c53539d53eba77417a3 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Mon, 18 Mar 2019 06:43:10 +0000 Subject: [PATCH 04/32] Adding fields from HPFS data and some ifs to test nulls --- database.py | 91 +++++++++++++++++++++++------------------------------ schema.py | 32 +++++++++++++++++++ 2 files changed, 71 insertions(+), 52 deletions(-) diff --git a/database.py b/database.py index 6411d80..ac780c6 100644 --- a/database.py +++ b/database.py @@ -9,7 +9,7 @@ def add_key_increment(dictionary, key): dictionary[key]+=1 class Data(object): - db_conn = mariadb.connect(user='biom_mass', password='DanDa1osh', db='portal_ui') + db_conn = mariadb.connect(user='biom_mass', password='', db='portal_ui') cursor = db_conn.cursor(buffered=True) def fetch_results(self,query): @@ -23,8 +23,6 @@ def fetch_results(self,query): def get_current_projects(self): projects_json=self.fetch_results("select `id` from `project`") - print("json_data inside get_current_projects") - print(projects_json) return [self.get_project(project['id']) for project in projects_json] def get_user(self): @@ -34,14 +32,13 @@ def get_user(self): def get_project(self,id): import schema - print("id inside get_project") + print("inside get_project") print(id) project_json=self.fetch_results( "select * from `project` where `id` =" + str(id)) project_id=project_json[0]['project_id'] proj_id=project_json[0]['id'] proj_name=project_json[0]['name'] - proj_program=["HPFS"] proj_program=project_json[0]['program'] #proj_primary_site=project_json[0]['primary_site'] proj_primary_site=["Stool"] @@ -56,9 +53,6 @@ def get_project(self,id): sum_file_size_json=self.fetch_results( "select sum(file_size) as sum_file_size from file_sample where project='"+project_id+"'") proj_file_size=sum_file_size_json[0]['sum_file_size'] - print("values inside get_project") - print( - proj_id,project_id,proj_name,proj_program,proj_part_count,proj_file_count,proj_file_size,proj_primary_site) return schema.Project( id=proj_id, @@ -78,7 +72,7 @@ def get_data_categories(self,table,id): import schema print("inside get_data_cat") - print(id) + print(table,id) data_cat_json=self.fetch_results( "select distinct data_category from file_sample where "+ table+"='" + str(id)+"'") @@ -86,12 +80,22 @@ def get_data_categories(self,table,id): for item in data_cat_json: item_files_json=self.fetch_results( "select count(id) as file_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") + item_part_json=self.fetch_results( "select count(distinct participant) as case_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") + + if len(item_part_json) > 0: + case_count=item_part_json[0]['case_count'] + else: + case_count=0 + if len(item_files_json) > 0: + file_count=item_files_json[0]['file_count'] + else: + file_count=0 data_cat.append( schema.DataCategories( - case_count=item_part_json[0]['case_count'], - file_count=item_files_json[0]['file_count'], + case_count=case_count, + file_count=file_count, data_category=item['data_category'])) return data_cat @@ -127,13 +131,11 @@ def get_file(self,id): "select id , project_id, primary_site from project where project_id='"+file_json[0]['project']+"'") part_json=self.fetch_results( "select id, entity_participant_id from participant where entity_participant_id='"+file_json[0]['participant']+"'") - print("file and project json inside get_file") - print(file_json[0]['sample']) + print("inside get_file") print(project_json[0]['project_id']) return schema.File( id=file_json[0]['id'], name=file_json[0]['file_name'], - file_id=file_json[0]['file_id'], participant=file_json[0]['participant'], sample=file_json[0]['sample'], access=file_json[0]['access'], @@ -150,7 +152,8 @@ def get_file(self,id): project=self.get_project(project_json[0]['id']), demographic=schema.Demographic("not hispanic or latino","male","white"), #primary_site=project_json[0]['primary_site'] - primary_site="Stool")])) + primary_site="Stool")]), + file_id=file_json[0]['file_id']) def get_case_annotation(self): import schema @@ -163,8 +166,6 @@ def get_current_files(self): file_hits=[] for item in files_json: file_hits.append(item['id']) - print("file_hits inside current files") - print(file_hits) return schema.Files(hits=file_hits) def get_current_cases(self): @@ -193,24 +194,14 @@ def get_case(self,id): print("inside get case") print(part_id) - proj_json=self.fetch_results('''select distinct file_sample.project, project.id as proj_id - from file_sample, project where file_sample.participant='''+str(part_id)+''' - and project.project_id=file_sample.project''') + proj_query="select distinct file_sample.project, project.id as proj_id " + proj_query=proj_query+"from file_sample, project where file_sample.participant='" + proj_query=proj_query+str(part_id)+"' and project.project_id=file_sample.project limit 1" + print(proj_query) + proj_json=self.fetch_results(proj_query) proj_id=proj_json[0]['proj_id'] print(proj_id) - cat_raw_json=self.fetch_results( - "select count(id) as raw_count from file_sample where participant="+str(part_id)+" and data_category='Raw Reads'") - cat_raw_count=cat_raw_json[0]['raw_count'] - - cat_taxa_json=self.fetch_results( - "select count(id) as taxa_count from file_sample where participant="+str(part_id)+" and data_category='Taxonomic Profiles'") - cat_taxa_count=cat_taxa_json[0]['taxa_count'] - - cat_gene_json=self.fetch_results( - "select count(id) as gene_count from file_sample where participant="+str(part_id)+" and data_category='Gene Families'") - cat_gene_count=cat_gene_json[0]['gene_count'] - files_json=self.fetch_results("select * from file_sample where participant="+str(part_id)) return schema.Case(id, @@ -222,10 +213,9 @@ def get_case(self,id): case_count=case_count, file_count=file_count, file_size=file_size, - data_categories=[schema.DataCategories(file_count=cat_raw_count, data_category="Raw Reads"), - schema.DataCategories(file_count=cat_gene_count, data_category="Gene Families"), - schema.DataCategories(file_count=cat_taxa_count, data_category="Taxonomic Profiles")]), - files=schema.CaseFiles(hits=[schema.CaseFile( + data_categories=self.get_data_categories("participant",part_id)), + # experimental_strategies=self.get_experimental_strategies("participant",part_id)), + files=schema.CaseFiles(hits=[schema.CaseFile( case_file['id'], experimental_strategy=case_file['experimental_strategy'], data_category=case_file['data_category'], @@ -245,9 +235,6 @@ def get_project_aggregations(self, projects): print("projects inside aggregations") for project in projects: - print( - project.id, project.project_id,project.primary_site, - project.program) add_key_increment(aggregates["primary_site"],project.primary_site[0]) add_key_increment(aggregates["project_id"], project.project_id) add_key_increment(aggregates["program__name"], project.program.name) @@ -310,18 +297,18 @@ def get_file_aggregations(self, files): aggregates = {"data_category": {}, "experimental_strategy": {}, "data_format": {}, "platform": {}, "cases__primary_site": {}, "cases__project__project_id": {}, "access": {}} + if len(files) > 0: + for file in files: + + add_key_increment(aggregates["data_category"], file.data_category) + add_key_increment(aggregates["experimental_strategy"], file.experimental_strategy) + add_key_increment(aggregates["data_format"], file.data_format) + add_key_increment(aggregates["platform"], file.platform) + add_key_increment(aggregates["access"], file.access) - for file in files: - print(" values inside file aggregation") - print(file.data_category,file.experimental_strategy,file.data_format,file.platform,file.access) - add_key_increment(aggregates["data_category"], file.data_category) - add_key_increment(aggregates["experimental_strategy"], file.experimental_strategy) - add_key_increment(aggregates["data_format"], file.data_format) - add_key_increment(aggregates["platform"], file.platform) - add_key_increment(aggregates["access"], file.access) - project = file.cases.hits[0].project - add_key_increment(aggregates["cases__primary_site"], project.primary_site[0]) - add_key_increment(aggregates["cases__project__project_id"], project.project_id) + project = file.cases.hits[0].project + add_key_increment(aggregates["cases__primary_site"], project.primary_site[0]) + add_key_increment(aggregates["cases__project__project_id"], project.project_id) file_aggregates = schema.FileAggregations( data_category=schema.Aggregations( @@ -339,7 +326,7 @@ def get_file_aggregations(self, files): cases__project__project_id=schema.Aggregations( buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["cases__project__project_id"].items()])) - return file_aggregates + return file_aggregates def get_case_aggregations(self, cases): import schema @@ -355,7 +342,7 @@ def get_case_aggregations(self, cases): add_key_increment(aggregates["demographic__race"], case.demographic.race) add_key_increment(aggregates["primary_site"], case.primary_site[0]) add_key_increment(aggregates["project__project_id"], case.project.project_id) - add_key_increment(aggregates["project__program__name"], case.project.program[0]) + add_key_increment(aggregates["project__program__name"], case.project.program.name) case_aggregates=schema.CaseAggregations( demographic__ethnicity=schema.Aggregations( diff --git a/schema.py b/schema.py index 13b23d4..ed76e93 100644 --- a/schema.py +++ b/schema.py @@ -91,6 +91,38 @@ class Demographic(graphene.ObjectType): gender = graphene.String() race = graphene.String() +class Metadata(graphene.ObjectType): + age2012 = graphene.Int() + totMETs1 = graphene.String() + weightLbs = graphene.String() + DaysSince1Jan12 = graphene.Int() + drAlcohol = graphene.Float() + drB12 = graphene.Float() + drCalories = graphene.Float() + drCarbs = graphene.Float() + drCholine = graphene.Float() + drFat = graphene.Float() + drFiber = graphene.Float() + drFolate = graphene.Float() + drIron = graphene.Float() + drProtein = graphene.Float() + participant = graphene.Int() + q2Alcohol = graphene.String() + q2B12 = graphene.String() + q2Calories = graphene.String() + q2Carbs = graphene.String() + q2Choline = graphene.String() + q2Fat = graphene.String() + q2Fiber = graphene.String() + q2Folate = graphene.String() + q2Iron = graphene.String() + q2Protein = graphene.String() + Time = graphene.String() + week = graphene.Int() + non_ribosomal_proteins = graphene.Float() + ribosomal_Proteins = graphene.Float() + + class FileCase(graphene.ObjectType): class Meta: interfaces = (graphene.relay.Node,) From 9a528bb6b1107186af9e7a83e8c831e573131bf4 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Tue, 19 Mar 2019 16:46:28 +0000 Subject: [PATCH 05/32] Adding schema changes --- schema.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/schema.py b/schema.py index 5aaa2fd..03ea5a5 100644 --- a/schema.py +++ b/schema.py @@ -91,7 +91,6 @@ class Demographic(graphene.ObjectType): gender = graphene.String() race = graphene.String() -<<<<<<< HEAD class Metadata(graphene.ObjectType): age2012 = graphene.Int() totMETs1 = graphene.String() @@ -123,9 +122,6 @@ class Metadata(graphene.ObjectType): non_ribosomal_proteins = graphene.Float() ribosomal_Proteins = graphene.Float() - -======= ->>>>>>> upstream/master class FileCase(graphene.ObjectType): class Meta: interfaces = (graphene.relay.Node,) From e232d1f84b7f1d53b0fc9bca8014a2bf3e0a88bb Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Tue, 19 Mar 2019 18:00:13 +0000 Subject: [PATCH 06/32] Reverting to clean version without ifs --- database.py | 80 ++++++++++++++++++++--------------------------------- schema.py | 0 2 files changed, 30 insertions(+), 50 deletions(-) mode change 100644 => 100755 database.py mode change 100644 => 100755 schema.py diff --git a/database.py b/database.py old mode 100644 new mode 100755 index 908641f..157151c --- a/database.py +++ b/database.py @@ -9,7 +9,7 @@ def add_key_increment(dictionary, key): dictionary[key]+=1 class Data(object): - db_conn = mariadb.connect(user='biom_mass', password='', db='portal_ui') + db_conn = mariadb.connect(user='biom_mass', password='DanDa1osh', db='portal_ui') cursor = db_conn.cursor(buffered=True) def fetch_results(self,query): @@ -78,31 +78,25 @@ def get_data_categories(self,table,id): "select distinct data_category from file_sample where "+ table+"='" + str(id)+"'") data_cat=[] for item in data_cat_json: - - if table is "participant": - item_files_json=self.fetch_results( - "select count(id) as file_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") + item_files_json=self.fetch_results( + "select count(id) as file_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") + + item_part_json=self.fetch_results( + "select count(distinct participant) as case_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") + + if len(item_part_json) > 0: + case_count=item_part_json[0]['case_count'] + else: + case_count=0 + if len(item_files_json) > 0: + file_count=item_files_json[0]['file_count'] + else: file_count=0 - if len(item_files_json) > 0: - file_count=item_files_json[0]['file_count'] - - data_cat.append( - schema.DataCategories( - file_count=file_count, - data_category=item['data_category'])) - - elif table is "project": - item_part_json=self.fetch_results( - "select count(distinct participant) as case_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") - - case_count=0 - if len(item_part_json) > 0: - case_count=item_part_json[0]['case_count'] - - data_cat.append( - schema.DataCategories( - case_count=case_count, - data_category=item['data_category'])) + data_cat.append( + schema.DataCategories( + case_count=case_count, + file_count=file_count, + data_category=item['data_category'])) return data_cat @@ -115,31 +109,17 @@ def get_experimental_strategies(self,table,id): print(table,id) exp_str_json=self.fetch_results( "select distinct experimental_strategy from file_sample where "+ table+"='" + str(id) +"'") - exp_str=[] for item in exp_str_json: item_files_json=self.fetch_results( - "select count(id) as filecount from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") - print("select count(id) as filecount from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") - -# item_part_json=self.fetch_results( -# "select count(distinct participant) as case_count from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") -# -# case_count=0 -# if len(item_part_json) > 0: -# case_count=item_part_json[0]['case_count'] - print(len(item_files_json)) - file_count=0 - if len(item_files_json)> 0: - file_count=item_files_json[0]['filecount'] - print( item) - print(file_count) + "select count(id) as file_count from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") + item_part_json=self.fetch_results( + "select count(distinct participant) as case_count from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") exp_str.append( schema.ExperimentalStrategies( -# case_count=case_count, - file_count=file_count, - experimental_strategy="wmgx")) -# experimental_strategy=item['experimental_strategy'])) + case_count=item_part_json[0]['case_count'], + file_count=item_files_json[0]['file_count'], + experimental_strategy=item['experimental_strategy'])) return exp_str @@ -325,11 +305,11 @@ def get_file_aggregations(self, files): add_key_increment(aggregates["data_format"], file.data_format) add_key_increment(aggregates["platform"], file.platform) add_key_increment(aggregates["access"], file.access) - if len(file.cases.hits) > 0: - project = file.cases.hits[0].project - add_key_increment(aggregates["cases__primary_site"], project.primary_site[0]) - add_key_increment(aggregates["cases__project__project_id"], project.project_id) - + + project = file.cases.hits[0].project + add_key_increment(aggregates["cases__primary_site"], project.primary_site[0]) + add_key_increment(aggregates["cases__project__project_id"], project.project_id) + file_aggregates = schema.FileAggregations( data_category=schema.Aggregations( buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["data_category"].items()]), diff --git a/schema.py b/schema.py old mode 100644 new mode 100755 From b63f67decd83b7898a536f0935ab31c36ebe9062 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Tue, 19 Mar 2019 18:03:05 +0000 Subject: [PATCH 07/32] Typo fix --- database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database.py b/database.py index 157151c..ac780c6 100755 --- a/database.py +++ b/database.py @@ -9,7 +9,7 @@ def add_key_increment(dictionary, key): dictionary[key]+=1 class Data(object): - db_conn = mariadb.connect(user='biom_mass', password='DanDa1osh', db='portal_ui') + db_conn = mariadb.connect(user='biom_mass', password='', db='portal_ui') cursor = db_conn.cursor(buffered=True) def fetch_results(self,query): From d7dc041401008bb7b8fb093fdaf08d4cf1851a2b Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Wed, 20 Mar 2019 23:16:48 +0000 Subject: [PATCH 08/32] open and close db connection/cursor --- database.py | 135 ++++++++++++++++++++++++---------------------------- 1 file changed, 63 insertions(+), 72 deletions(-) diff --git a/database.py b/database.py index ac780c6..cd1b12e 100755 --- a/database.py +++ b/database.py @@ -2,6 +2,7 @@ import mysql.connector as mariadb import json +import schema def add_key_increment(dictionary, key): if not key in dictionary: @@ -9,16 +10,21 @@ def add_key_increment(dictionary, key): dictionary[key]+=1 class Data(object): - db_conn = mariadb.connect(user='biom_mass', password='', db='portal_ui') - cursor = db_conn.cursor(buffered=True) def fetch_results(self,query): - self.cursor.execute(query) - row_headers=[x[0] for x in self.cursor.description] - rows = self.cursor.fetchall() + + db_conn = mariadb.connect(user='biom_mass', password='', db='portal_ui') + cursor = db_conn.cursor(buffered=True) + cursor.execute(query) + + row_headers=[x[0] for x in cursor.description] + rows = cursor.fetchall() json_data=[] for row in rows: json_data.append(dict(zip(row_headers,row))) + cursor.close() + db_conn.close() + return json_data def get_current_projects(self): @@ -26,14 +32,10 @@ def get_current_projects(self): return [self.get_project(project['id']) for project in projects_json] def get_user(self): - import schema return schema.User(username="null") def get_project(self,id): - import schema - print("inside get_project") - print(id) project_json=self.fetch_results( "select * from `project` where `id` =" + str(id)) project_id=project_json[0]['project_id'] @@ -43,7 +45,8 @@ def get_project(self,id): #proj_primary_site=project_json[0]['primary_site'] proj_primary_site=["Stool"] - file_count_json=self.fetch_results("select count(id) as file_count from file_sample where project='"+project_id+"'") + file_count_json=self.fetch_results( + "select count(id) as file_count from file_sample where project='"+project_id+"'") proj_file_count=file_count_json[0]['file_count'] part_count_json=self.fetch_results( @@ -69,70 +72,75 @@ def get_project(self,id): def get_data_categories(self,table,id): - import schema - - print("inside get_data_cat") - print(table,id) data_cat_json=self.fetch_results( "select distinct data_category from file_sample where "+ table+"='" + str(id)+"'") data_cat=[] for item in data_cat_json: - item_files_json=self.fetch_results( - "select count(id) as file_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") - item_part_json=self.fetch_results( - "select count(distinct participant) as case_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") + if table is "participant": + item_files_json=self.fetch_results( + "select count(id) as file_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") + file_count=0 + if len(item_files_json) > 0: + file_count=item_files_json[0]['file_count'] + + data_cat.append( + schema.DataCategories( + file_count=file_count, + data_category=item['data_category'])) + elif table is "project": + item_part_json=self.fetch_results( + "select count(distinct participant) as case_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") - if len(item_part_json) > 0: - case_count=item_part_json[0]['case_count'] - else: case_count=0 - if len(item_files_json) > 0: - file_count=item_files_json[0]['file_count'] - else: - file_count=0 - data_cat.append( - schema.DataCategories( - case_count=case_count, - file_count=file_count, - data_category=item['data_category'])) + if len(item_part_json) > 0: + case_count=item_part_json[0]['case_count'] + data_cat.append( + schema.DataCategories( + case_count=case_count, + data_category=item['data_category'])) return data_cat def get_experimental_strategies(self,table,id): - import schema - print("inside get exp_str") - print(table,id) exp_str_json=self.fetch_results( "select distinct experimental_strategy from file_sample where "+ table+"='" + str(id) +"'") + exp_str=[] for item in exp_str_json: item_files_json=self.fetch_results( - "select count(id) as file_count from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") - item_part_json=self.fetch_results( - "select count(distinct participant) as case_count from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") + "select count(id) as filecount from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") +# item_part_json=self.fetch_results( +# "select count(distinct participant) as case_count from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") +# +# case_count=0 +# if len(item_part_json) > 0: +# case_count=item_part_json[0]['case_count'] + + file_count=0 + if len(item_files_json) > 0: + file_count=item_files_json[0]['filecount'] exp_str.append( schema.ExperimentalStrategies( - case_count=item_part_json[0]['case_count'], - file_count=item_files_json[0]['file_count'], - experimental_strategy=item['experimental_strategy'])) +# case_count=case_count, + file_count=file_count, + experimental_strategy="wmgx")) +# experimental_strategy=item['experimental_strategy'])) return exp_str def get_file(self,id): - import schema file_json=self.fetch_results("select * from `file_sample` where `id` ="+ str(id)) project_json=self.fetch_results( "select id , project_id, primary_site from project where project_id='"+file_json[0]['project']+"'") part_json=self.fetch_results( "select id, entity_participant_id from participant where entity_participant_id='"+file_json[0]['participant']+"'") - print("inside get_file") - print(project_json[0]['project_id']) + return schema.File( id=file_json[0]['id'], name=file_json[0]['file_name'], @@ -156,26 +164,19 @@ def get_file(self,id): file_id=file_json[0]['file_id']) def get_case_annotation(self): - import schema return schema.CaseAnnotation() def get_current_files(self): - import schema files_json=self.fetch_results("select `id` from `file_sample`") - file_hits=[] - for item in files_json: - file_hits.append(item['id']) - return schema.Files(hits=file_hits) + return schema.Files(hits=[item['id'] for item in files_json]) def get_current_cases(self): - import schema cases_json=self.fetch_results("select id from participant") return schema.RepositoryCases(hits=[case['id'] for case in cases_json]) def get_case(self,id): - import schema case_json=self.fetch_results("select * from participant where id="+str(id)) part_id=case_json[0]['entity_participant_id'] @@ -192,15 +193,11 @@ def get_case(self,id): "select sum(file_size) as filesize from file_sample where participant="+str(part_id)) file_size=file_size_json[0]['filesize'] - print("inside get case") - print(part_id) proj_query="select distinct file_sample.project, project.id as proj_id " proj_query=proj_query+"from file_sample, project where file_sample.participant='" proj_query=proj_query+str(part_id)+"' and project.project_id=file_sample.project limit 1" - print(proj_query) proj_json=self.fetch_results(proj_query) proj_id=proj_json[0]['proj_id'] - print(proj_id) files_json=self.fetch_results("select * from file_sample where participant="+str(part_id)) @@ -225,14 +222,12 @@ def get_case(self,id): def get_project_aggregations(self, projects): - import schema # compile aggregations from project aggregates = {"primary_site": {}, "program__name": {}, "project_id": {}, "summary__data_categories__data_category": {}, "summary__experimental_strategies__experimental_strategy": {}} - print("projects inside aggregations") for project in projects: add_key_increment(aggregates["primary_site"],project.primary_site[0]) @@ -259,11 +254,10 @@ def get_project_aggregations(self, projects): def get_current_counts(self): - import schema projects_json = self.fetch_results("select count(`project_id`)as projects_count from project") projects = projects_json[0]['projects_count'] - + participants_json = self.fetch_results("select count(`entity_participant_id`) as participants_count from `participant`") participants = participants_json[0]['participants_count'] @@ -291,24 +285,23 @@ def get_current_counts(self): def get_file_aggregations(self, files): - import schema # aggregate file data aggregates = {"data_category": {}, "experimental_strategy": {}, "data_format": {}, "platform": {}, "cases__primary_site": {}, "cases__project__project_id": {}, "access": {}} - if len(files) > 0: - for file in files: - add_key_increment(aggregates["data_category"], file.data_category) - add_key_increment(aggregates["experimental_strategy"], file.experimental_strategy) - add_key_increment(aggregates["data_format"], file.data_format) - add_key_increment(aggregates["platform"], file.platform) - add_key_increment(aggregates["access"], file.access) + for file in files: + + add_key_increment(aggregates["data_category"], file.data_category) + add_key_increment(aggregates["experimental_strategy"], file.experimental_strategy) + add_key_increment(aggregates["data_format"], file.data_format) + add_key_increment(aggregates["platform"], file.platform) + add_key_increment(aggregates["access"], file.access) - project = file.cases.hits[0].project - add_key_increment(aggregates["cases__primary_site"], project.primary_site[0]) - add_key_increment(aggregates["cases__project__project_id"], project.project_id) + project = file.cases.hits[0].project + add_key_increment(aggregates["cases__primary_site"], project.primary_site[0]) + add_key_increment(aggregates["cases__project__project_id"], project.project_id) file_aggregates = schema.FileAggregations( data_category=schema.Aggregations( @@ -326,10 +319,9 @@ def get_file_aggregations(self, files): cases__project__project_id=schema.Aggregations( buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["cases__project__project_id"].items()])) - return file_aggregates + return file_aggregates def get_case_aggregations(self, cases): - import schema # aggregate case data aggregates = {"demographic__ethnicity": {}, "demographic__gender": {}, @@ -364,7 +356,6 @@ def get_facets(self): return "null" # this is not currently being used def get_cart_file_size(self): - import schema file_json=self.fetch_results("select sum(file_size) as sum_size from file_sample") return schema.FileSize(file_json[0]['sum_size']) From 9341bd20c56ab079112d29d6e6d609d009454b80 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Thu, 21 Mar 2019 02:41:33 +0000 Subject: [PATCH 09/32] Adding HPFS fields to schema --- database.py | 64 +++++++++++++++++++++++++++++++++++++++++++---------- schema.py | 7 ++++-- 2 files changed, 57 insertions(+), 14 deletions(-) diff --git a/database.py b/database.py index cd1b12e..b7a37fd 100755 --- a/database.py +++ b/database.py @@ -4,6 +4,8 @@ import json import schema + +# increments key for aggregations def add_key_increment(dictionary, key): if not key in dictionary: dictionary[key]=0 @@ -11,12 +13,13 @@ def add_key_increment(dictionary, key): class Data(object): + # connects to db and runs query def fetch_results(self,query): - db_conn = mariadb.connect(user='biom_mass', password='', db='portal_ui') + db_conn = mariadb.connect(user='biom_mass', password='DanDa1osh', db='portal_ui') cursor = db_conn.cursor(buffered=True) cursor.execute(query) - + # response json row_headers=[x[0] for x in cursor.description] rows = cursor.fetchall() json_data=[] @@ -27,13 +30,16 @@ def fetch_results(self,query): return json_data + # get all projects from db def get_current_projects(self): projects_json=self.fetch_results("select `id` from `project`") return [self.get_project(project['id']) for project in projects_json] + # get user (currently none) def get_user(self): return schema.User(username="null") + # get project object details from db def get_project(self,id): project_json=self.fetch_results( @@ -42,8 +48,8 @@ def get_project(self,id): proj_id=project_json[0]['id'] proj_name=project_json[0]['name'] proj_program=project_json[0]['program'] - #proj_primary_site=project_json[0]['primary_site'] - proj_primary_site=["Stool"] + proj_primary_site=[project_json[0]['primary_site']] + #proj_primary_site=["Stool"] file_count_json=self.fetch_results( "select count(id) as file_count from file_sample where project='"+project_id+"'") @@ -71,6 +77,7 @@ def get_project(self,id): primary_site=proj_primary_site) + # get data categories file and case counts from db for a project or a participant def get_data_categories(self,table,id): data_cat_json=self.fetch_results( @@ -104,7 +111,7 @@ def get_data_categories(self,table,id): return data_cat - + # get experimental strategies file count from db for a project def get_experimental_strategies(self,table,id): exp_str_json=self.fetch_results( @@ -128,11 +135,13 @@ def get_experimental_strategies(self,table,id): schema.ExperimentalStrategies( # case_count=case_count, file_count=file_count, - experimental_strategy="wmgx")) -# experimental_strategy=item['experimental_strategy'])) + # experimental_strategy="wmgx")) + experimental_strategy=item['experimental_strategy'])) return exp_str + + # get details of file object from db def get_file(self,id): file_json=self.fetch_results("select * from `file_sample` where `id` ="+ str(id)) @@ -159,23 +168,27 @@ def get_file(self,id): case_id=file_json[0]['participant'], project=self.get_project(project_json[0]['id']), demographic=schema.Demographic("not hispanic or latino","male","white"), - #primary_site=project_json[0]['primary_site'] - primary_site="Stool")]), + primary_site=project_json[0]['primary_site'])]), + #primary_site="Stool")]), file_id=file_json[0]['file_id']) + # get annotations (currently not in use) def get_case_annotation(self): return schema.CaseAnnotation() + # get ids of all files from db def get_current_files(self): files_json=self.fetch_results("select `id` from `file_sample`") return schema.Files(hits=[item['id'] for item in files_json]) + # get ids of all cases from db def get_current_cases(self): cases_json=self.fetch_results("select id from participant") return schema.RepositoryCases(hits=[case['id'] for case in cases_json]) + # get details of case object from db def get_case(self,id): case_json=self.fetch_results("select * from participant where id="+str(id)) @@ -194,24 +207,44 @@ def get_case(self,id): file_size=file_size_json[0]['filesize'] proj_query="select distinct file_sample.project, project.id as proj_id " - proj_query=proj_query+"from file_sample, project where file_sample.participant='" + proj_query=proj_query+"from file_sample, project where file_sample.participant='" proj_query=proj_query+str(part_id)+"' and project.project_id=file_sample.project limit 1" proj_json=self.fetch_results(proj_query) proj_id=proj_json[0]['proj_id'] + # query to get files from file_sample table files_json=self.fetch_results("select * from file_sample where participant="+str(part_id)) + # query to get metadata from prticipant table + metadata_part_json=self.fetch_results( + "select * from participant where entity_participant_id="+str(part_id)) + metadata_part_values=[] + for key, value in metadata_part_json[0].items(): + metadata_part_values.append(value) + metadata_participant=schema.MetadataParticipant(metadata_part_values) + + # query to get metadata from sample table + metadata_json=self.fetch_results("select * from sample where participant="+str(part_id)) + metadata_list=[] + for metadata in metadata_json: + metadata_values=[] + for key, value in metadata.items(): + metadata_values.append(value) + metadata_list.append(schema.MetadataSample(metadata_values)) + return schema.Case(id, case_id=part_id, primary_site="Stool", demographic=schema.Demographic("not hispanic or latino","male","white"), + metadata_participant=metadata_participant, + metadata_sample=metadata_list, project=self.get_project(proj_id), summary=schema.Summary( case_count=case_count, file_count=file_count, file_size=file_size, data_categories=self.get_data_categories("participant",part_id)), - # experimental_strategies=self.get_experimental_strategies("participant",part_id)), + #experimental_strategies=self.get_experimental_strategies("participant",part_id)), files=schema.CaseFiles(hits=[schema.CaseFile( case_file['id'], experimental_strategy=case_file['experimental_strategy'], @@ -221,6 +254,7 @@ def get_case(self,id): access=case_file['access']) for case_file in files_json])) + def get_project_aggregations(self, projects): # compile aggregations from project @@ -252,7 +286,7 @@ def get_project_aggregations(self, projects): return project_aggregates - + # get all counts for front page summary from db def get_current_counts(self): projects_json = self.fetch_results("select count(`project_id`)as projects_count from project") @@ -284,6 +318,7 @@ def get_current_counts(self): processedFiles=prFiles) + def get_file_aggregations(self, files): # aggregate file data @@ -321,6 +356,8 @@ def get_file_aggregations(self, files): return file_aggregates + + def get_case_aggregations(self, cases): # aggregate case data @@ -352,9 +389,12 @@ def get_case_aggregations(self, cases): return case_aggregates + # get facet (currently not in use) def get_facets(self): return "null" # this is not currently being used + + # get total size of all files def get_cart_file_size(self): file_json=self.fetch_results("select sum(file_size) as sum_size from file_sample") diff --git a/schema.py b/schema.py index 03ea5a5..b8c6ee9 100755 --- a/schema.py +++ b/schema.py @@ -91,10 +91,12 @@ class Demographic(graphene.ObjectType): gender = graphene.String() race = graphene.String() -class Metadata(graphene.ObjectType): +class MetadataParticipant(graphene.ObjectType): age2012 = graphene.Int() totMETs1 = graphene.String() weightLbs = graphene.String() + +class MetadataSample(graphene.ObjectType): DaysSince1Jan12 = graphene.Int() drAlcohol = graphene.Float() drB12 = graphene.Float() @@ -311,8 +313,9 @@ class Meta: case_id = graphene.String() primary_site = graphene.String() submitter_id = graphene.String() - demographic = graphene.Field(Demographic) + metadata_participant = graphene.Field(MetadataParticipant) + metadata_sample = graphene.List(MetadataSample) project = graphene.Field(Project) summary = graphene.Field(Summary) annotations = graphene.Field(CaseAnnotations) From 1cc4318805c00e24b22d600c46bccba7344f0503 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Thu, 21 Mar 2019 03:35:35 +0000 Subject: [PATCH 10/32] Typo fix --- database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database.py b/database.py index b7a37fd..1440383 100755 --- a/database.py +++ b/database.py @@ -16,7 +16,7 @@ class Data(object): # connects to db and runs query def fetch_results(self,query): - db_conn = mariadb.connect(user='biom_mass', password='DanDa1osh', db='portal_ui') + db_conn = mariadb.connect(user='biom_mass', password='', db='portal_ui') cursor = db_conn.cursor(buffered=True) cursor.execute(query) # response json From 00de0471da20cfded2a6f4bbc66eb8a22c159779 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Thu, 21 Mar 2019 18:10:22 +0000 Subject: [PATCH 11/32] Adding couple missing fields to Metadata --- database.py | 19 ++++++++++--------- schema.py | 12 ++++++++---- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/database.py b/database.py index 1440383..f9f483c 100755 --- a/database.py +++ b/database.py @@ -169,7 +169,6 @@ def get_file(self,id): project=self.get_project(project_json[0]['id']), demographic=schema.Demographic("not hispanic or latino","male","white"), primary_site=project_json[0]['primary_site'])]), - #primary_site="Stool")]), file_id=file_json[0]['file_id']) # get annotations (currently not in use) @@ -218,19 +217,20 @@ def get_case(self,id): # query to get metadata from prticipant table metadata_part_json=self.fetch_results( "select * from participant where entity_participant_id="+str(part_id)) - metadata_part_values=[] - for key, value in metadata_part_json[0].items(): - metadata_part_values.append(value) - metadata_participant=schema.MetadataParticipant(metadata_part_values) + + del metadata_part_json[0]['updated'] + metadata_part_json[0]['participant'] = metadata_part_json[0]['entity_participant_id'] + del metadata_part_json[0]['entity_participant_id'] + metadata_participant=schema.MetadataParticipant(**metadata_part_json[0]) + # query to get metadata from sample table metadata_json=self.fetch_results("select * from sample where participant="+str(part_id)) metadata_list=[] for metadata in metadata_json: - metadata_values=[] - for key, value in metadata.items(): - metadata_values.append(value) - metadata_list.append(schema.MetadataSample(metadata_values)) + del metadata['updated'] + metadata_list.append(schema.MetadataSample(**metadata)) + return schema.Case(id, case_id=part_id, @@ -373,6 +373,7 @@ def get_case_aggregations(self, cases): add_key_increment(aggregates["project__project_id"], case.project.project_id) add_key_increment(aggregates["project__program__name"], case.project.program.name) + case_aggregates=schema.CaseAggregations( demographic__ethnicity=schema.Aggregations( buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["demographic__ethnicity"].items()]), diff --git a/schema.py b/schema.py index b8c6ee9..6ad1c68 100755 --- a/schema.py +++ b/schema.py @@ -92,11 +92,17 @@ class Demographic(graphene.ObjectType): race = graphene.String() class MetadataParticipant(graphene.ObjectType): - age2012 = graphene.Int() + id = graphene.Int() + participant = graphene.Int() + age_2012 = graphene.Int() totMETs1 = graphene.String() - weightLbs = graphene.String() + weight_lbs = graphene.String() class MetadataSample(graphene.ObjectType): + id = graphene.Int() + project = graphene.String() + sample = graphene.String() + participant = graphene.Int() DaysSince1Jan12 = graphene.Int() drAlcohol = graphene.Float() drB12 = graphene.Float() @@ -121,8 +127,6 @@ class MetadataSample(graphene.ObjectType): q2Protein = graphene.String() Time = graphene.String() week = graphene.Int() - non_ribosomal_proteins = graphene.Float() - ribosomal_Proteins = graphene.Float() class FileCase(graphene.ObjectType): class Meta: From fdfffe673e06500c2f16589a927979a178ccf4a6 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Thu, 21 Mar 2019 23:40:37 +0000 Subject: [PATCH 12/32] reverting to front page only on dev branch --- database.py | 268 ++++++++-------------------------------------------- 1 file changed, 38 insertions(+), 230 deletions(-) diff --git a/database.py b/database.py index f9f483c..7bd6403 100755 --- a/database.py +++ b/database.py @@ -1,11 +1,8 @@ -# Calls to obtain values from database - +# Calls to obtain values from the data structures (to be populated by the local database next) import mysql.connector as mariadb import json -import schema - +import const -# increments key for aggregations def add_key_increment(dictionary, key): if not key in dictionary: dictionary[key]=0 @@ -16,7 +13,7 @@ class Data(object): # connects to db and runs query def fetch_results(self,query): - db_conn = mariadb.connect(user='biom_mass', password='', db='portal_ui') + db_conn = mariadb.connect(user='biom_mass', password='DanDa1osh', db='portal_ui') cursor = db_conn.cursor(buffered=True) cursor.execute(query) # response json @@ -30,232 +27,46 @@ def fetch_results(self,query): return json_data - # get all projects from db + + def load_data(self): + self.data = const.DB() + def get_current_projects(self): - projects_json=self.fetch_results("select `id` from `project`") - return [self.get_project(project['id']) for project in projects_json] + self.load_data() + return [self.get_project(project_id) for project_id in self.data.CURRENT_PROJECTS.keys()] - # get user (currently none) def get_user(self): - return schema.User(username="null") + self.load_data() + return self.data.CURRENT_USER - # get project object details from db def get_project(self,id): + self.load_data() + return self.data.CURRENT_PROJECTS[id] - project_json=self.fetch_results( - "select * from `project` where `id` =" + str(id)) - project_id=project_json[0]['project_id'] - proj_id=project_json[0]['id'] - proj_name=project_json[0]['name'] - proj_program=project_json[0]['program'] - proj_primary_site=[project_json[0]['primary_site']] - #proj_primary_site=["Stool"] - - file_count_json=self.fetch_results( - "select count(id) as file_count from file_sample where project='"+project_id+"'") - proj_file_count=file_count_json[0]['file_count'] - - part_count_json=self.fetch_results( - "select count(distinct participant) as part_count from file_sample where project='"+project_id+"'") - proj_part_count=part_count_json[0]['part_count'] - - sum_file_size_json=self.fetch_results( - "select sum(file_size) as sum_file_size from file_sample where project='"+project_id+"'") - proj_file_size=sum_file_size_json[0]['sum_file_size'] - - return schema.Project( - id=proj_id, - project_id=project_id, - name=proj_name, - program=schema.Program(name=proj_program), - summary=schema.Summary( - case_count=proj_part_count, - file_count=proj_file_count, - data_categories=self.get_data_categories("project",project_id), - experimental_strategies=self.get_experimental_strategies("project",project_id), - file_size=proj_file_size), - primary_site=proj_primary_site) - - - # get data categories file and case counts from db for a project or a participant - def get_data_categories(self,table,id): - - data_cat_json=self.fetch_results( - "select distinct data_category from file_sample where "+ table+"='" + str(id)+"'") - data_cat=[] - for item in data_cat_json: - - if table is "participant": - item_files_json=self.fetch_results( - "select count(id) as file_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") - file_count=0 - if len(item_files_json) > 0: - file_count=item_files_json[0]['file_count'] - - data_cat.append( - schema.DataCategories( - file_count=file_count, - data_category=item['data_category'])) - elif table is "project": - item_part_json=self.fetch_results( - "select count(distinct participant) as case_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") - - case_count=0 - if len(item_part_json) > 0: - case_count=item_part_json[0]['case_count'] - data_cat.append( - schema.DataCategories( - case_count=case_count, - data_category=item['data_category'])) - - return data_cat - - - # get experimental strategies file count from db for a project - def get_experimental_strategies(self,table,id): - - exp_str_json=self.fetch_results( - "select distinct experimental_strategy from file_sample where "+ table+"='" + str(id) +"'") - - exp_str=[] - for item in exp_str_json: - item_files_json=self.fetch_results( - "select count(id) as filecount from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") -# item_part_json=self.fetch_results( -# "select count(distinct participant) as case_count from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") -# -# case_count=0 -# if len(item_part_json) > 0: -# case_count=item_part_json[0]['case_count'] - - file_count=0 - if len(item_files_json) > 0: - file_count=item_files_json[0]['filecount'] - exp_str.append( - schema.ExperimentalStrategies( -# case_count=case_count, - file_count=file_count, - # experimental_strategy="wmgx")) - experimental_strategy=item['experimental_strategy'])) - - return exp_str - - - # get details of file object from db def get_file(self,id): + self.load_data() + return self.data.TEST_FILES[id] - file_json=self.fetch_results("select * from `file_sample` where `id` ="+ str(id)) - project_json=self.fetch_results( - "select id , project_id, primary_site from project where project_id='"+file_json[0]['project']+"'") - part_json=self.fetch_results( - "select id, entity_participant_id from participant where entity_participant_id='"+file_json[0]['participant']+"'") - - return schema.File( - id=file_json[0]['id'], - name=file_json[0]['file_name'], - participant=file_json[0]['participant'], - sample=file_json[0]['sample'], - access=file_json[0]['access'], - file_size=file_json[0]['file_size'], - data_category=file_json[0]['data_category'], - data_format=file_json[0]['data_format'], - platform=file_json[0]['platform'], - experimental_strategy=file_json[0]['experimental_strategy'], - file_name=file_json[0]['file_name'], - cases=schema.FileCases( - hits=[schema.FileCase( - part_json[0]['id'], - case_id=file_json[0]['participant'], - project=self.get_project(project_json[0]['id']), - demographic=schema.Demographic("not hispanic or latino","male","white"), - primary_site=project_json[0]['primary_site'])]), - file_id=file_json[0]['file_id']) - - # get annotations (currently not in use) def get_case_annotation(self): - return schema.CaseAnnotation() + self.load_data() + return self.data.CURRENT_CASE_ANNOTATION - # get ids of all files from db def get_current_files(self): + self.load_data() + return self.data.CURRENT_FILES - files_json=self.fetch_results("select `id` from `file_sample`") - return schema.Files(hits=[item['id'] for item in files_json]) - - # get ids of all cases from db def get_current_cases(self): + self.load_data() + return self.data.CURRENT_CASES - cases_json=self.fetch_results("select id from participant") - return schema.RepositoryCases(hits=[case['id'] for case in cases_json]) - - # get details of case object from db def get_case(self,id): - - case_json=self.fetch_results("select * from participant where id="+str(id)) - part_id=case_json[0]['entity_participant_id'] - - case_count_json=self.fetch_results( - "select count(entity_participant_id) as case_count from participant where entity_participant_id="+str(part_id)) - case_count=case_count_json[0]['case_count'] - - file_count_json=self.fetch_results( - "select count(id) as file_count from file_sample where participant="+str(part_id)) - file_count=file_count_json[0]['file_count'] - - file_size_json=self.fetch_results( - "select sum(file_size) as filesize from file_sample where participant="+str(part_id)) - file_size=file_size_json[0]['filesize'] - - proj_query="select distinct file_sample.project, project.id as proj_id " - proj_query=proj_query+"from file_sample, project where file_sample.participant='" - proj_query=proj_query+str(part_id)+"' and project.project_id=file_sample.project limit 1" - proj_json=self.fetch_results(proj_query) - proj_id=proj_json[0]['proj_id'] - - # query to get files from file_sample table - files_json=self.fetch_results("select * from file_sample where participant="+str(part_id)) - - # query to get metadata from prticipant table - metadata_part_json=self.fetch_results( - "select * from participant where entity_participant_id="+str(part_id)) - - del metadata_part_json[0]['updated'] - metadata_part_json[0]['participant'] = metadata_part_json[0]['entity_participant_id'] - del metadata_part_json[0]['entity_participant_id'] - metadata_participant=schema.MetadataParticipant(**metadata_part_json[0]) - - - # query to get metadata from sample table - metadata_json=self.fetch_results("select * from sample where participant="+str(part_id)) - metadata_list=[] - for metadata in metadata_json: - del metadata['updated'] - metadata_list.append(schema.MetadataSample(**metadata)) - - - return schema.Case(id, - case_id=part_id, - primary_site="Stool", - demographic=schema.Demographic("not hispanic or latino","male","white"), - metadata_participant=metadata_participant, - metadata_sample=metadata_list, - project=self.get_project(proj_id), - summary=schema.Summary( - case_count=case_count, - file_count=file_count, - file_size=file_size, - data_categories=self.get_data_categories("participant",part_id)), - #experimental_strategies=self.get_experimental_strategies("participant",part_id)), - files=schema.CaseFiles(hits=[schema.CaseFile( - case_file['id'], - experimental_strategy=case_file['experimental_strategy'], - data_category=case_file['data_category'], - data_format=case_file['data_format'], - platform=case_file['platform'], - access=case_file['access']) for case_file in files_json])) - - + self.load_data() + return self.data.TEST_CASES[id] def get_project_aggregations(self, projects): + import schema + + self.load_data() # compile aggregations from project aggregates = {"primary_site": {}, "program__name": {}, @@ -264,7 +75,7 @@ def get_project_aggregations(self, projects): "summary__experimental_strategies__experimental_strategy": {}} for project in projects: - add_key_increment(aggregates["primary_site"],project.primary_site[0]) + add_key_increment(aggregates["primary_site"], project.primary_site[0]) add_key_increment(aggregates["project_id"], project.project_id) add_key_increment(aggregates["program__name"], project.program.name) for item in project.summary.data_categories: @@ -286,8 +97,10 @@ def get_project_aggregations(self, projects): return project_aggregates - # get all counts for front page summary from db + + # get all counts for front page summary from db def get_current_counts(self): + import schema projects_json = self.fetch_results("select count(`project_id`)as projects_count from project") projects = projects_json[0]['projects_count'] @@ -318,22 +131,21 @@ def get_current_counts(self): processedFiles=prFiles) - def get_file_aggregations(self, files): + import schema + self.load_data() # aggregate file data aggregates = {"data_category": {}, "experimental_strategy": {}, "data_format": {}, "platform": {}, "cases__primary_site": {}, "cases__project__project_id": {}, "access": {}} for file in files: - add_key_increment(aggregates["data_category"], file.data_category) add_key_increment(aggregates["experimental_strategy"], file.experimental_strategy) add_key_increment(aggregates["data_format"], file.data_format) add_key_increment(aggregates["platform"], file.platform) add_key_increment(aggregates["access"], file.access) - project = file.cases.hits[0].project add_key_increment(aggregates["cases__primary_site"], project.primary_site[0]) add_key_increment(aggregates["cases__project__project_id"], project.project_id) @@ -356,9 +168,10 @@ def get_file_aggregations(self, files): return file_aggregates - - def get_case_aggregations(self, cases): + import schema + + self.load_data() # aggregate case data aggregates = {"demographic__ethnicity": {}, "demographic__gender": {}, @@ -369,11 +182,10 @@ def get_case_aggregations(self, cases): add_key_increment(aggregates["demographic__ethnicity"], case.demographic.ethnicity) add_key_increment(aggregates["demographic__gender"], case.demographic.gender) add_key_increment(aggregates["demographic__race"], case.demographic.race) - add_key_increment(aggregates["primary_site"], case.primary_site[0]) + add_key_increment(aggregates["primary_site"], case.primary_site) add_key_increment(aggregates["project__project_id"], case.project.project_id) add_key_increment(aggregates["project__program__name"], case.project.program.name) - case_aggregates=schema.CaseAggregations( demographic__ethnicity=schema.Aggregations( buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["demographic__ethnicity"].items()]), @@ -390,15 +202,11 @@ def get_case_aggregations(self, cases): return case_aggregates - # get facet (currently not in use) def get_facets(self): return "null" # this is not currently being used - - # get total size of all files def get_cart_file_size(self): - - file_json=self.fetch_results("select sum(file_size) as sum_size from file_sample") - return schema.FileSize(file_json[0]['sum_size']) + self.load_data() + return self.data.CURRENT_FILE_SIZE data = Data() From 253198b7f8fe9264cf6c975f3e1934f01cec5070 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Thu, 21 Mar 2019 23:46:23 +0000 Subject: [PATCH 13/32] reverting to front page changes only on dev --- load_local_database.py | 2 +- query_bigquery.py | 2 +- schema.py | 43 ++---------------------------------------- 3 files changed, 4 insertions(+), 43 deletions(-) diff --git a/load_local_database.py b/load_local_database.py index 944566d..f83f0f0 100644 --- a/load_local_database.py +++ b/load_local_database.py @@ -23,7 +23,7 @@ def parse_arguments(args): required=False) parser.add_argument( "--key-file", - default="/home/hutlab_public/compute_engine_service_account_key/biom-mass-8dc9ab934396.json", + default="biom-mass-fdcadb440fdf.json", help="google project service account private key file path\n]", required=False) parser.add_argument( diff --git a/query_bigquery.py b/query_bigquery.py index 770c7ad..2d179ab 100644 --- a/query_bigquery.py +++ b/query_bigquery.py @@ -22,7 +22,7 @@ def parse_arguments(args): required=False) parser.add_argument( "--key-file", - default="/home/hutlab_public/compute_engine_service_account_key/biom-mass-8dc9ab934396.json", + default="biom-mass-fdcadb440fdf.json", help="google project service account private key file path\n]", required=False) parser.add_argument( diff --git a/schema.py b/schema.py index 6ad1c68..0cbc3de 100755 --- a/schema.py +++ b/schema.py @@ -1,4 +1,5 @@ + # Schema for firecloud api to graphql import json @@ -91,43 +92,6 @@ class Demographic(graphene.ObjectType): gender = graphene.String() race = graphene.String() -class MetadataParticipant(graphene.ObjectType): - id = graphene.Int() - participant = graphene.Int() - age_2012 = graphene.Int() - totMETs1 = graphene.String() - weight_lbs = graphene.String() - -class MetadataSample(graphene.ObjectType): - id = graphene.Int() - project = graphene.String() - sample = graphene.String() - participant = graphene.Int() - DaysSince1Jan12 = graphene.Int() - drAlcohol = graphene.Float() - drB12 = graphene.Float() - drCalories = graphene.Float() - drCarbs = graphene.Float() - drCholine = graphene.Float() - drFat = graphene.Float() - drFiber = graphene.Float() - drFolate = graphene.Float() - drIron = graphene.Float() - drProtein = graphene.Float() - participant = graphene.Int() - q2Alcohol = graphene.String() - q2B12 = graphene.String() - q2Calories = graphene.String() - q2Carbs = graphene.String() - q2Choline = graphene.String() - q2Fat = graphene.String() - q2Fiber = graphene.String() - q2Folate = graphene.String() - q2Iron = graphene.String() - q2Protein = graphene.String() - Time = graphene.String() - week = graphene.Int() - class FileCase(graphene.ObjectType): class Meta: interfaces = (graphene.relay.Node,) @@ -317,9 +281,8 @@ class Meta: case_id = graphene.String() primary_site = graphene.String() submitter_id = graphene.String() + demographic = graphene.Field(Demographic) - metadata_participant = graphene.Field(MetadataParticipant) - metadata_sample = graphene.List(MetadataSample) project = graphene.Field(Project) summary = graphene.Field(Summary) annotations = graphene.Field(CaseAnnotations) @@ -465,5 +428,3 @@ def resolve_samples(self, info, namespace, workspace): json_result = query_firecloud.call_api(url) obj_result = utilities.json2obj(json.dumps(json_result)) return obj_result - - From 13095a8147a5700c7fbb87191ef9fdcfee5f5543 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Thu, 21 Mar 2019 23:49:32 +0000 Subject: [PATCH 14/32] reverting to front page changes only on dev --- schema.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schema.py b/schema.py index 0cbc3de..490ece2 100755 --- a/schema.py +++ b/schema.py @@ -1,5 +1,4 @@ - # Schema for firecloud api to graphql import json @@ -428,3 +427,5 @@ def resolve_samples(self, info, namespace, workspace): json_result = query_firecloud.call_api(url) obj_result = utilities.json2obj(json.dumps(json_result)) return obj_result + + From 0c7941021569818bec12796efca0081825efc9b6 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Thu, 21 Mar 2019 23:51:20 +0000 Subject: [PATCH 15/32] reverting to front page changes only on dev --- schema.py | 1 - 1 file changed, 1 deletion(-) diff --git a/schema.py b/schema.py index 490ece2..f22950e 100755 --- a/schema.py +++ b/schema.py @@ -428,4 +428,3 @@ def resolve_samples(self, info, namespace, workspace): obj_result = utilities.json2obj(json.dumps(json_result)) return obj_result - From 9952cbfd023be6b7c3b15e57e9cfc2ad688d3ff4 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Thu, 21 Mar 2019 23:53:42 +0000 Subject: [PATCH 16/32] reverting to front page changes only on dev --- schema.py | 1 + 1 file changed, 1 insertion(+) diff --git a/schema.py b/schema.py index f22950e..490ece2 100755 --- a/schema.py +++ b/schema.py @@ -428,3 +428,4 @@ def resolve_samples(self, info, namespace, workspace): obj_result = utilities.json2obj(json.dumps(json_result)) return obj_result + From 95db21d9a6e51c457e60aaa47bf10f586adf4ab1 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Fri, 22 Mar 2019 19:46:19 +0000 Subject: [PATCH 17/32] Fix typo --- database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database.py b/database.py index 7bd6403..8e523cc 100755 --- a/database.py +++ b/database.py @@ -13,7 +13,7 @@ class Data(object): # connects to db and runs query def fetch_results(self,query): - db_conn = mariadb.connect(user='biom_mass', password='DanDa1osh', db='portal_ui') + db_conn = mariadb.connect(user='biom_mass', password='', db='portal_ui') cursor = db_conn.cursor(buffered=True) cursor.execute(query) # response json From bf1532252880ed14f02c758d7d7ee1acd05e595e Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Sat, 23 Mar 2019 00:35:11 +0000 Subject: [PATCH 18/32] Making count queries union --- database.py | 55 +++++++++++++++++++++-------------------------------- 1 file changed, 22 insertions(+), 33 deletions(-) mode change 100755 => 100644 database.py diff --git a/database.py b/database.py old mode 100755 new mode 100644 index 8e523cc..6bf6519 --- a/database.py +++ b/database.py @@ -1,6 +1,6 @@ # Calls to obtain values from the data structures (to be populated by the local database next) import mysql.connector as mariadb -import json +import os import const def add_key_increment(dictionary, key): @@ -10,22 +10,24 @@ def add_key_increment(dictionary, key): class Data(object): + db_conn = mariadb.connect(user='biom_mass', password=os.environ['BIOM_MASS'], db='portal_ui') + # connects to db and runs query def fetch_results(self,query): - db_conn = mariadb.connect(user='biom_mass', password='', db='portal_ui') - cursor = db_conn.cursor(buffered=True) + cursor = self.db_conn.cursor(buffered=True) cursor.execute(query) - # response json + + # response row_headers=[x[0] for x in cursor.description] rows = cursor.fetchall() - json_data=[] + data=[] for row in rows: - json_data.append(dict(zip(row_headers,row))) + data.append(dict(zip(row_headers,row))) + cursor.close() - db_conn.close() - return json_data + return data def load_data(self): @@ -102,33 +104,20 @@ def get_project_aggregations(self, projects): def get_current_counts(self): import schema - projects_json = self.fetch_results("select count(`project_id`)as projects_count from project") - projects = projects_json[0]['projects_count'] - - participants_json = self.fetch_results("select count(`entity_participant_id`) as participants_count from `participant`") - participants = participants_json[0]['participants_count'] - - samples_json = self.fetch_results("select count(`sample`) as samples_count from `sample`") - samples = samples_json[0]['samples_count'] - - dataFormats_json = self.fetch_results("select count(distinct `data_format`) as data_format_count from `file_sample`") - dataFormats = dataFormats_json[0]['data_format_count'] - - rawFiles_json = self.fetch_results( - "select count(`id`) as rawfiles_count from `file_sample` where type='rawFiles'") - rawFiles = rawFiles_json[0]['rawfiles_count'] - - prFiles_json = self.fetch_results( - "select count(`id`) as prfiles_count from `file_sample` where type='processedFiles'") - prFiles = prFiles_json[0]['prfiles_count'] + counts_data = self.fetch_results('''select count(id) as countid from project + union all select count(id) as countid from participant + union all select count(id) as countid from sample + union all select count(distinct data_format) as countid from file_sample + union all select count(id) as countid from file_sample where type="rawFiles" + union all select count(id) as countid from file_sample where type="processedFiles"''') return schema.Count( - projects=projects, - participants=participants, - samples=samples, - dataFormats=dataFormats, - rawFiles=rawFiles, - processedFiles=prFiles) + projects=counts_data[0]['countid'], + participants=counts_data[1]['countid'], + samples=counts_data[2]['countid'], + dataFormats=counts_data[3]['countid'], + rawFiles=counts_data[4]['countid'], + processedFiles=counts_data[5]['countid']) def get_file_aggregations(self, files): From f9b47c6b961e794fba7ed0cb81e809369dbcebdd Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Sat, 23 Mar 2019 04:18:56 +0000 Subject: [PATCH 19/32] adding version to schema --- database.py | 7 +++++++ schema.graphql | 9 +++++++++ schema.py | 11 +++++++++++ 3 files changed, 27 insertions(+) mode change 100755 => 100644 schema.py diff --git a/database.py b/database.py index 6bf6519..3ee0782 100644 --- a/database.py +++ b/database.py @@ -41,6 +41,13 @@ def get_user(self): self.load_data() return self.data.CURRENT_USER + # get current version from db + def get_version(self): + version_data=self.fetch_results("select * from version order by updated desc limit 1 ") + del version_data[0]['updated'] + del version_data[0]['id'] + return schema.Version(**version_data[0]) + def get_project(self,id): self.load_data() return self.data.CURRENT_PROJECTS[id] diff --git a/schema.graphql b/schema.graphql index fe55875..dee8e9b 100644 --- a/schema.graphql +++ b/schema.graphql @@ -1,3 +1,4 @@ + type Aggregations { buckets: [Bucket] } @@ -114,6 +115,14 @@ type Count { processedFiles: String } +type Version{ + data_release: String + commit: String + version: Float + tag: String + status: String +} + type DataCategories { case_count: Int file_count: Int diff --git a/schema.py b/schema.py old mode 100755 new mode 100644 index 490ece2..1b448d1 --- a/schema.py +++ b/schema.py @@ -43,6 +43,13 @@ class Count(graphene.ObjectType): class User(graphene.ObjectType): username = graphene.String() +class Version(graphene.ObjectType): + data_release = graphene.String() + commit = graphene.String() + version = graphene.Float() + tag = graphene.String() + status = graphene.String() + class Program(graphene.ObjectType): name = graphene.String() program_id = graphene.String() @@ -386,6 +393,7 @@ class Root(graphene.ObjectType): repository = graphene.Field(Repository) projects = graphene.Field(Projects) cart_summary = graphene.Field(CartSummary) + version = graphene.Field(Version) def resolve_user(self, info): return data.get_user() @@ -402,6 +410,9 @@ def resolve_projects(self, info): def resolve_cart_summary(self, info): return CartSummary(self) + def resolve_version(self, info): + return data.get_current_version() + class Query(graphene.ObjectType): ## Portal API ## From 4704a87f1545cd99c80cfb7477fe23a0ac63907f Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Sat, 23 Mar 2019 04:43:10 +0000 Subject: [PATCH 20/32] function name change --- database.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/database.py b/database.py index 3ee0782..3ab8687 100644 --- a/database.py +++ b/database.py @@ -42,7 +42,9 @@ def get_user(self): return self.data.CURRENT_USER # get current version from db - def get_version(self): + def get_current_version(self): + import schema + version_data=self.fetch_results("select * from version order by updated desc limit 1 ") del version_data[0]['updated'] del version_data[0]['id'] From 94ed77cfd738e248f57fae038d94ef7e56a50a16 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Sat, 23 Mar 2019 18:34:25 +0000 Subject: [PATCH 21/32] Adding command to set env var for password --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a67d7e1..3cc710b 100644 --- a/README.md +++ b/README.md @@ -137,8 +137,9 @@ then run 1. ``CREATE USER 'username'@'localhost' IDENTIFIED BY 'password';`` 2. ``GRANT ALL PRIVILEGES ON *.* TO 'username'@'localhost' WITH GRANT OPTION;`` 3. ``flush privileges;`` - More information https://dev.mysql.com/doc/refman/5.5/en/adding-users.html +4. Set environment variable BIOM_MASS that holds user's password + ``export BIOM_MASS="password"`` load_local_database.py script needs mysql.connect module. From a81be3fcbd0de44688f6e4f16cab683dfda346da Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Mon, 25 Mar 2019 15:53:25 +0000 Subject: [PATCH 22/32] adding init and exit for db conn open and close --- database.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/database.py b/database.py index 3ab8687..7da0e23 100644 --- a/database.py +++ b/database.py @@ -10,7 +10,11 @@ def add_key_increment(dictionary, key): class Data(object): - db_conn = mariadb.connect(user='biom_mass', password=os.environ['BIOM_MASS'], db='portal_ui') + def __init__(self): + self.db_conn = mariadb.connect(user='biom_mass', password=os.environ['BIOM_MASS'], db='portal_ui') + + def __exit__(self): + self.db.conn.close() # connects to db and runs query def fetch_results(self,query): From 0eef992f8b9d161c61741b92338cddf5d517c9de Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Wed, 27 Mar 2019 01:48:23 +0000 Subject: [PATCH 23/32] Using connection pool, c extension, pulling version from db and using cursor dictionary option --- database.py | 43 +++++++++++++++++++++++++------------------ schema.graphql | 8 -------- schema.py | 11 ----------- server.py | 6 +++++- 4 files changed, 30 insertions(+), 38 deletions(-) diff --git a/database.py b/database.py index 7da0e23..29022de 100644 --- a/database.py +++ b/database.py @@ -1,8 +1,10 @@ # Calls to obtain values from the data structures (to be populated by the local database next) -import mysql.connector as mariadb +import mysql.connector +from mysql.connector import pooling import os import const +# add increment for aggregations def add_key_increment(dictionary, key): if not key in dictionary: dictionary[key]=0 @@ -11,28 +13,39 @@ def add_key_increment(dictionary, key): class Data(object): def __init__(self): - self.db_conn = mariadb.connect(user='biom_mass', password=os.environ['BIOM_MASS'], db='portal_ui') + self.conn_pool = mysql.connector.connect(use_pure=False, pool_name="portal", + pool_size=32, + pool_reset_session=True, + database='portal_ui', + user='biom_mass', + password=os.environ['BIOM_MASS']) def __exit__(self): - self.db.conn.close() + self.conn.close() # connects to db and runs query def fetch_results(self,query): - cursor = self.db_conn.cursor(buffered=True) + conn = mysql.connector.connect(pool_name="portal") + cursor = conn.cursor(buffered=True,dictionary=True) cursor.execute(query) # response - row_headers=[x[0] for x in cursor.description] - rows = cursor.fetchall() data=[] - for row in rows: - data.append(dict(zip(row_headers,row))) + for row in cursor: + data.append(row) cursor.close() - + conn.close() return data + # get current version from db + def get_current_version(self): + version_data=self.fetch_results("select * from version order by updated desc limit 1 ") + del version_data[0]['updated'] + del version_data[0]['id'] + return version_data[0] + def load_data(self): self.data = const.DB() @@ -45,15 +58,6 @@ def get_user(self): self.load_data() return self.data.CURRENT_USER - # get current version from db - def get_current_version(self): - import schema - - version_data=self.fetch_results("select * from version order by updated desc limit 1 ") - del version_data[0]['updated'] - del version_data[0]['id'] - return schema.Version(**version_data[0]) - def get_project(self,id): self.load_data() return self.data.CURRENT_PROJECTS[id] @@ -212,3 +216,6 @@ def get_cart_file_size(self): return self.data.CURRENT_FILE_SIZE data = Data() + +VERSION = data.get_current_version() + diff --git a/schema.graphql b/schema.graphql index dee8e9b..b44d2c6 100644 --- a/schema.graphql +++ b/schema.graphql @@ -115,14 +115,6 @@ type Count { processedFiles: String } -type Version{ - data_release: String - commit: String - version: Float - tag: String - status: String -} - type DataCategories { case_count: Int file_count: Int diff --git a/schema.py b/schema.py index 1b448d1..490ece2 100644 --- a/schema.py +++ b/schema.py @@ -43,13 +43,6 @@ class Count(graphene.ObjectType): class User(graphene.ObjectType): username = graphene.String() -class Version(graphene.ObjectType): - data_release = graphene.String() - commit = graphene.String() - version = graphene.Float() - tag = graphene.String() - status = graphene.String() - class Program(graphene.ObjectType): name = graphene.String() program_id = graphene.String() @@ -393,7 +386,6 @@ class Root(graphene.ObjectType): repository = graphene.Field(Repository) projects = graphene.Field(Projects) cart_summary = graphene.Field(CartSummary) - version = graphene.Field(Version) def resolve_user(self, info): return data.get_user() @@ -410,9 +402,6 @@ def resolve_projects(self, info): def resolve_cart_summary(self, info): return CartSummary(self) - def resolve_version(self, info): - return data.get_current_version() - class Query(graphene.ObjectType): ## Portal API ## diff --git a/server.py b/server.py index 4154751..fa6cd40 100644 --- a/server.py +++ b/server.py @@ -10,11 +10,13 @@ import os import flask +import flask_cors import flask_graphql import graphene import schema import const +import database NAME = "firecloud_graphql" HOST = "0.0.0.0" @@ -41,6 +43,8 @@ def process_query(request, schema): def main(): # create the graphql flask app app = flask.Flask(NAME) + # allow initial OPTIONS requests + flask_cors.CORS(app) # add the root graphql queries @app.route('/graphql', methods=["POST"]) @@ -55,7 +59,7 @@ def get_schema(name): # add static endpoint for version/status @app.route('/status', methods=["GET"]) def get_version(): - return flask.jsonify(const.VERSION) + return flask.jsonify(database.VERSION) # add end point for graphql gui app.add_url_rule('/test', view_func=flask_graphql.GraphQLView.as_view( From a2bd028363751eb799583161a440b3b1fe3b27da Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Wed, 27 Mar 2019 01:53:12 +0000 Subject: [PATCH 24/32] Fixing typo --- database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database.py b/database.py index 29022de..07ea229 100644 --- a/database.py +++ b/database.py @@ -21,7 +21,7 @@ def __init__(self): password=os.environ['BIOM_MASS']) def __exit__(self): - self.conn.close() + self.conn_pool.close() # connects to db and runs query def fetch_results(self,query): From ffa8ce1bb227bb44bcd008b0fdee61ca8cc6278a Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Wed, 27 Mar 2019 01:58:04 +0000 Subject: [PATCH 25/32] fixing white space --- schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema.py b/schema.py index 490ece2..d106c4a 100644 --- a/schema.py +++ b/schema.py @@ -428,4 +428,4 @@ def resolve_samples(self, info, namespace, workspace): obj_result = utilities.json2obj(json.dumps(json_result)) return obj_result - + From 9c96015fc3b37b77e8eeea59d79c36e263b2b254 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Wed, 27 Mar 2019 02:04:22 +0000 Subject: [PATCH 26/32] Removing spaces --- schema.graphql | 1 - 1 file changed, 1 deletion(-) diff --git a/schema.graphql b/schema.graphql index b44d2c6..fe55875 100644 --- a/schema.graphql +++ b/schema.graphql @@ -1,4 +1,3 @@ - type Aggregations { buckets: [Bucket] } From 829e51d3c4fb03b44748c2c903de7a08a6f9a8ea Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Wed, 27 Mar 2019 16:40:03 +0000 Subject: [PATCH 27/32] moving pool creation and closing out of fetch --- database.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/database.py b/database.py index 07ea229..1c71602 100644 --- a/database.py +++ b/database.py @@ -23,25 +23,17 @@ def __init__(self): def __exit__(self): self.conn_pool.close() - # connects to db and runs query - def fetch_results(self,query): - - conn = mysql.connector.connect(pool_name="portal") - cursor = conn.cursor(buffered=True,dictionary=True) + # runs query and returns results + def fetch_results(self,cursor,query): cursor.execute(query) - - # response - data=[] - for row in cursor: - data.append(row) - - cursor.close() - conn.close() - return data + return [row for row in cursor] # get current version from db def get_current_version(self): - version_data=self.fetch_results("select * from version order by updated desc limit 1 ") + + conn = mysql.connector.connect(pool_name="portal") + cursor = conn.cursor(buffered=True,dictionary=True) + version_data=self.fetch_results(cursor,"select * from version order by updated desc limit 1 ") del version_data[0]['updated'] del version_data[0]['id'] return version_data[0] @@ -121,7 +113,10 @@ def get_project_aggregations(self, projects): def get_current_counts(self): import schema - counts_data = self.fetch_results('''select count(id) as countid from project + conn = mysql.connector.connect(pool_name="portal") + cursor = conn.cursor(buffered=True,dictionary=True) + counts_data = self.fetch_results(cursor, + '''select count(id) as countid from project union all select count(id) as countid from participant union all select count(id) as countid from sample union all select count(distinct data_format) as countid from file_sample From f884da3b1ffd18d589f96d92ecea7dac57464b80 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Wed, 27 Mar 2019 18:45:49 +0000 Subject: [PATCH 28/32] 32 pools is too much, setting 10 --- database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database.py b/database.py index 1c71602..f71be55 100644 --- a/database.py +++ b/database.py @@ -14,7 +14,7 @@ class Data(object): def __init__(self): self.conn_pool = mysql.connector.connect(use_pure=False, pool_name="portal", - pool_size=32, + pool_size=10, pool_reset_session=True, database='portal_ui', user='biom_mass', From e8f18cd6b6919e8733c0e4f853b63a9b9ff65ab2 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Thu, 28 Mar 2019 20:39:47 +0000 Subject: [PATCH 29/32] placing pool creation in fetch_results and fixing version --- database.py | 27 ++++++++++++++++----------- load_local_database.py | 4 ++-- server.py | 4 ++-- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/database.py b/database.py index f71be55..1378054 100644 --- a/database.py +++ b/database.py @@ -23,17 +23,26 @@ def __init__(self): def __exit__(self): self.conn_pool.close() + def get_pool(self): + conn = mysql.connector.connect(pool_name="portal") + cursor = conn.cursor(buffered=True,dictionary=True) + return conn, cursor + + def release_pool(self,conn,cursor): + cursor.close() + conn.close() + # runs query and returns results - def fetch_results(self,cursor,query): + def fetch_results(self,query): + conn,cursor=self.get_pool() cursor.execute(query) - return [row for row in cursor] + data=[row for row in cursor] + self.release_pool(conn,cursor) + return data # get current version from db def get_current_version(self): - - conn = mysql.connector.connect(pool_name="portal") - cursor = conn.cursor(buffered=True,dictionary=True) - version_data=self.fetch_results(cursor,"select * from version order by updated desc limit 1 ") + version_data=self.fetch_results("select * from version order by updated desc limit 1 ") del version_data[0]['updated'] del version_data[0]['id'] return version_data[0] @@ -113,10 +122,7 @@ def get_project_aggregations(self, projects): def get_current_counts(self): import schema - conn = mysql.connector.connect(pool_name="portal") - cursor = conn.cursor(buffered=True,dictionary=True) - counts_data = self.fetch_results(cursor, - '''select count(id) as countid from project + counts_data = self.fetch_results('''select count(id) as countid from project union all select count(id) as countid from participant union all select count(id) as countid from sample union all select count(distinct data_format) as countid from file_sample @@ -212,5 +218,4 @@ def get_cart_file_size(self): data = Data() -VERSION = data.get_current_version() diff --git a/load_local_database.py b/load_local_database.py index f83f0f0..1a58ba8 100644 --- a/load_local_database.py +++ b/load_local_database.py @@ -23,7 +23,7 @@ def parse_arguments(args): required=False) parser.add_argument( "--key-file", - default="biom-mass-fdcadb440fdf.json", + default="/home/hutlab_public/compute_engine_service_account_key/biom-mass-8dc9ab934396.json", help="google project service account private key file path\n]", required=False) parser.add_argument( @@ -256,7 +256,7 @@ def main(): new_version = str(0.1) now = datetime.datetime.now() - release_date ="Date Release "+new_version+" "+ now.strftime("%b %d, %Y") + release_date ="Data Release "+new_version+" - "+ now.strftime("%b %d, %Y") commit ="commit_"+now.strftime("%m%d%Y") query_insert_version ='''INSERT INTO `version` ( commit, diff --git a/server.py b/server.py index fa6cd40..4676bc0 100644 --- a/server.py +++ b/server.py @@ -16,7 +16,7 @@ import schema import const -import database +from database import data NAME = "firecloud_graphql" HOST = "0.0.0.0" @@ -59,7 +59,7 @@ def get_schema(name): # add static endpoint for version/status @app.route('/status', methods=["GET"]) def get_version(): - return flask.jsonify(database.VERSION) + return flask.jsonify(data.get_current_version()) # add end point for graphql gui app.add_url_rule('/test', view_func=flask_graphql.GraphQLView.as_view( From e3e2826208f9e2cb843fb7ec428ba7e0dbf5d8a6 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Fri, 29 Mar 2019 02:27:55 +0000 Subject: [PATCH 30/32] comments added --- database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database.py b/database.py index 1378054..4fc1d74 100644 --- a/database.py +++ b/database.py @@ -23,11 +23,13 @@ def __init__(self): def __exit__(self): self.conn_pool.close() + # get connection from pool def get_pool(self): conn = mysql.connector.connect(pool_name="portal") cursor = conn.cursor(buffered=True,dictionary=True) return conn, cursor + # release connection back to pool def release_pool(self,conn,cursor): cursor.close() conn.close() @@ -217,5 +219,3 @@ def get_cart_file_size(self): return self.data.CURRENT_FILE_SIZE data = Data() - - From d50b49c8345a849c2c7df375e99202143b43ea3e Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Tue, 2 Apr 2019 20:53:55 +0000 Subject: [PATCH 31/32] Moving queries to dbqueries.py --- database.py | 22 ++++++-------- dbqueries.py | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 13 deletions(-) create mode 100644 dbqueries.py diff --git a/database.py b/database.py index 4fc1d74..49db8d8 100644 --- a/database.py +++ b/database.py @@ -2,6 +2,7 @@ import mysql.connector from mysql.connector import pooling import os +import dbqueries import const # add increment for aggregations @@ -44,7 +45,7 @@ def fetch_results(self,query): # get current version from db def get_current_version(self): - version_data=self.fetch_results("select * from version order by updated desc limit 1 ") + version_data=self.fetch_results(dbqueries.version_query) del version_data[0]['updated'] del version_data[0]['id'] return version_data[0] @@ -124,20 +125,15 @@ def get_project_aggregations(self, projects): def get_current_counts(self): import schema - counts_data = self.fetch_results('''select count(id) as countid from project - union all select count(id) as countid from participant - union all select count(id) as countid from sample - union all select count(distinct data_format) as countid from file_sample - union all select count(id) as countid from file_sample where type="rawFiles" - union all select count(id) as countid from file_sample where type="processedFiles"''') + counts_data = self.fetch_results(dbqueries.all_counts_query) return schema.Count( - projects=counts_data[0]['countid'], - participants=counts_data[1]['countid'], - samples=counts_data[2]['countid'], - dataFormats=counts_data[3]['countid'], - rawFiles=counts_data[4]['countid'], - processedFiles=counts_data[5]['countid']) + projects=counts_data[0]['total'], + participants=counts_data[1]['total'], + samples=counts_data[2]['total'], + dataFormats=counts_data[3]['total'], + rawFiles=counts_data[4]['total'], + processedFiles=counts_data[5]['total']) def get_file_aggregations(self, files): diff --git a/dbqueries.py b/dbqueries.py new file mode 100644 index 0000000..17e89b6 --- /dev/null +++ b/dbqueries.py @@ -0,0 +1,83 @@ +# This file holds all queries run in database.py + + + +all_counts_query='''select count(id) as total from project + union all select count(id) as total from participant + union all select count(id) as total from sample + union all select count(distinct data_format) as total from file_sample + union all select count(id) as total from file_sample where type="rawFiles" + union all select count(id) as total from file_sample where type="processedFiles"''' +projects_query="select id from project" +version_query="select * from version order by updated desc limit 1" +files_query="select id from file_sample" +cases_query="select id from participant" +files_size_query="select sum(file_size) as sum_size from file_sample" +files_total_query="select count(id) as total from file_sample" +cases_total_query="select count(distinct id) as total from participant" + +def project_query(p_id): + return "select * from project where id =" + str(p_id) + +def project_counts_query(p_id): + query="select count(id) as total from file_sample where project='"+p_id+"'" + query=query+" union all " + query=query+"select count(distinct participant) as total from file_sample where project='"+p_id+"'" + query=query+" union all " + query=query+"select sum(file_size) as total from file_sample where project='"+p_id+"'" + return query + +def data_cat_query(table, p_id): + return "select distinct data_category from file_sample where "+ table+"='" + str(p_id)+"'" + +def data_cat_counts_query(table,p_id,data_cat): + query="select count(id) as total from file_sample where data_category='"+data_cat+"' and "+table+"='"+str(p_id)+"'" + query=query+" union all " + query=query+"select count(distinct participant) as total from file_sample where data_category='"+data_cat+"' and "+table+"='"+str(p_id)+"'" + return query + +def exp_str_query(table,p_id): + return "select distinct experimental_strategy from file_sample where "+ table+"='" + str(p_id) +"'" + +def exp_str_counts_query(table,p_id,exp_str): + query="select count(id) as total from file_sample where experimental_strategy='"+exp_str+"' and "+table+"='"+str(p_id)+"'" + query=query+" union all " + query=query+ "select count(distinct participant) as total from file_sample where experimental_strategy='"+exp_str+"' and "+table+"='"+str(p_id)+"'" + return query + +def file_query(f_id): + query='''select file_sample.*, + project.id as p_id, project.project_id as proj_id,project.primary_site, + participant.id as part_id + from file_sample,project,participant where file_sample.id='''+str(f_id) + query=query+''' and file_sample.project=project.project_id and + file_sample.participant=participant.entity_participant_id''' + return query + +def case_query(p_id): + return '''select participant.entity_participant_id, file_sample.id, file_sample.experimental_strategy, + file_sample.data_category, file_sample.platform, file_sample.access,file_sample.data_format + from participant, file_sample + where participant.id='''+str(p_id)+" and file_sample.participant=participant.entity_participant_id" + +def case_counts_query(p_id): + query="select count(id) as total from participant where entity_participant_id="+str(p_id) + query=query+" union all " + query=query+ "select count(id) as total from file_sample where participant="+str(p_id) + query=query+" union all " + query=query+"select sum(file_size) as total from file_sample where participant="+str(p_id) + return query + +def case_project_query(p_id): + query="select distinct file_sample.project, project.id as p_id, project.primary_site " + query=query+"from file_sample, project where file_sample.participant='" + query=query+str(p_id)+"' and project.project_id=file_sample.project limit 1" + return query + + +def metadata_part_query(p_id): + return "select * from participant where entity_participant_id="+str(p_id) + +def metadata_sample_query(p_id): + return "select * from sample where participant="+str(p_id) + From ece3f26f5d3a494221ef0b3459e2b3ee392f8492 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Tue, 2 Apr 2019 21:00:55 +0000 Subject: [PATCH 32/32] Remove all but front page queries --- dbqueries.py | 76 +--------------------------------------------------- 1 file changed, 1 insertion(+), 75 deletions(-) diff --git a/dbqueries.py b/dbqueries.py index 17e89b6..0f96b0d 100644 --- a/dbqueries.py +++ b/dbqueries.py @@ -1,83 +1,9 @@ # This file holds all queries run in database.py - - - all_counts_query='''select count(id) as total from project union all select count(id) as total from participant union all select count(id) as total from sample union all select count(distinct data_format) as total from file_sample union all select count(id) as total from file_sample where type="rawFiles" union all select count(id) as total from file_sample where type="processedFiles"''' -projects_query="select id from project" -version_query="select * from version order by updated desc limit 1" -files_query="select id from file_sample" -cases_query="select id from participant" -files_size_query="select sum(file_size) as sum_size from file_sample" -files_total_query="select count(id) as total from file_sample" -cases_total_query="select count(distinct id) as total from participant" - -def project_query(p_id): - return "select * from project where id =" + str(p_id) - -def project_counts_query(p_id): - query="select count(id) as total from file_sample where project='"+p_id+"'" - query=query+" union all " - query=query+"select count(distinct participant) as total from file_sample where project='"+p_id+"'" - query=query+" union all " - query=query+"select sum(file_size) as total from file_sample where project='"+p_id+"'" - return query - -def data_cat_query(table, p_id): - return "select distinct data_category from file_sample where "+ table+"='" + str(p_id)+"'" - -def data_cat_counts_query(table,p_id,data_cat): - query="select count(id) as total from file_sample where data_category='"+data_cat+"' and "+table+"='"+str(p_id)+"'" - query=query+" union all " - query=query+"select count(distinct participant) as total from file_sample where data_category='"+data_cat+"' and "+table+"='"+str(p_id)+"'" - return query - -def exp_str_query(table,p_id): - return "select distinct experimental_strategy from file_sample where "+ table+"='" + str(p_id) +"'" - -def exp_str_counts_query(table,p_id,exp_str): - query="select count(id) as total from file_sample where experimental_strategy='"+exp_str+"' and "+table+"='"+str(p_id)+"'" - query=query+" union all " - query=query+ "select count(distinct participant) as total from file_sample where experimental_strategy='"+exp_str+"' and "+table+"='"+str(p_id)+"'" - return query - -def file_query(f_id): - query='''select file_sample.*, - project.id as p_id, project.project_id as proj_id,project.primary_site, - participant.id as part_id - from file_sample,project,participant where file_sample.id='''+str(f_id) - query=query+''' and file_sample.project=project.project_id and - file_sample.participant=participant.entity_participant_id''' - return query - -def case_query(p_id): - return '''select participant.entity_participant_id, file_sample.id, file_sample.experimental_strategy, - file_sample.data_category, file_sample.platform, file_sample.access,file_sample.data_format - from participant, file_sample - where participant.id='''+str(p_id)+" and file_sample.participant=participant.entity_participant_id" - -def case_counts_query(p_id): - query="select count(id) as total from participant where entity_participant_id="+str(p_id) - query=query+" union all " - query=query+ "select count(id) as total from file_sample where participant="+str(p_id) - query=query+" union all " - query=query+"select sum(file_size) as total from file_sample where participant="+str(p_id) - return query - -def case_project_query(p_id): - query="select distinct file_sample.project, project.id as p_id, project.primary_site " - query=query+"from file_sample, project where file_sample.participant='" - query=query+str(p_id)+"' and project.project_id=file_sample.project limit 1" - return query - - -def metadata_part_query(p_id): - return "select * from participant where entity_participant_id="+str(p_id) - -def metadata_sample_query(p_id): - return "select * from sample where participant="+str(p_id) +version_query="select * from version order by updated desc limit 1"