From 7c4c50cb88c1dc1cf20740ae73eed6b324c30688 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Sun, 17 Mar 2019 06:39:36 +0000 Subject: [PATCH 01/34] Latest version copied to anamailyan fork --- const.py | 162 +++++++++---------- database.py | 350 ++++++++++++++++++++++++++++++++++++----- load_local_database.py | 2 +- query_bigquery.py | 2 +- schema.graphql | 34 +++- schema.py | 84 +++++++--- utilities.py | 106 ++++++++++--- 7 files changed, 577 insertions(+), 163 deletions(-) diff --git a/const.py b/const.py index 00131c7..e524d1d 100644 --- a/const.py +++ b/const.py @@ -15,7 +15,7 @@ class DB(object): def __init__(self): import schema - self.CURRENT_CASEANNOTATION = schema.CaseAnnotation() # not currently used + self.CURRENT_CASE_ANNOTATION = schema.CaseAnnotation() # not currently used self.CURRENT_USER = schema.User(username="null") # no users are currently being used @@ -37,49 +37,103 @@ def __init__(self): self.CURRENT_PROJECTS = { "1":schema.Project(id="1", project_id="NHSII-DemoA", name="NHSII-DemoA", program=self.CURRENT_PROGRAMS[0], summary=schema.Summary(case_count=2, file_count=6, - data_categories=self.DATA_CATEGORIES, experimental_strategies=self.EXPERIMENTAL_STRATEGIES, file_size=15), + data_categories=self.DATA_CATEGORIES, experimental_strategies=[self.EXPERIMENTAL_STRATEGIES[0]], file_size=15), primary_site=["Stool"]), "2":schema.Project(id="2", project_id="NHSII-DemoB", name="NHSII-DemoB", program=self.CURRENT_PROGRAMS[0], summary=schema.Summary(case_count=2, file_count=6, - data_categories=self.DATA_CATEGORIES, experimental_strategies=self.EXPERIMENTAL_STRATEGIES, file_size=15), + data_categories=self.DATA_CATEGORIES, experimental_strategies=[self.EXPERIMENTAL_STRATEGIES[1]], file_size=15), primary_site=["Stool"]), } self.CURRENT_FILE_CASES = { - "1":schema.FileCase(1,"Case1",self.CURRENT_PROJECTS["1"]), - "2":schema.FileCase(2,"Case2",self.CURRENT_PROJECTS["1"]), - "3":schema.FileCase(3,"Case3",self.CURRENT_PROJECTS["2"]), - "4":schema.FileCase(4,"Case4",self.CURRENT_PROJECTS["2"]), + "1":schema.FileCase(1,"Case1",self.CURRENT_PROJECTS["1"], + schema.Demographic("not hispanic or latino","male","white"), "Stool"), + "2":schema.FileCase(2,"Case2",self.CURRENT_PROJECTS["1"], + schema.Demographic("not hispanic or latino","male","white"), "Stool"), + "3":schema.FileCase(3,"Case3",self.CURRENT_PROJECTS["2"], + schema.Demographic("not hispanic or latino","female","white"), "Stool"), + "4":schema.FileCase(4,"Case4",self.CURRENT_PROJECTS["2"], + schema.Demographic("hispanic or latino","female","white"), "Stool") } + self.CASE_FILES = { + "1": schema.CaseFile(1,experimental_strategy="WMGX", + data_category="Raw Reads",data_format="Fastq", + platform="Illumina MiSeq",access="controlled"), + "2": schema.CaseFile(2,experimental_strategy="WMGX", + data_category="Taxonomic Profile",data_format="TSV", + platform="Illumina MiSeq",access="open"), + "3": schema.CaseFile(3,experimental_strategy="WMGX", + data_category="Gene Families",data_format="TSV", + platform="Illumina MiSeq",access="open"), + "4": schema.CaseFile(4,experimental_strategy="WMGX", + data_category="Raw Reads",data_format="Fastq", + platform="Illumina MiSeq",access="controlled"), + "5": schema.CaseFile(5,experimental_strategy="WMGX", + data_category="Taxonomic Profile",data_format="TSV", + platform="Illumina MiSeq",access="open"), + "6": schema.CaseFile(6,experimental_strategy="WMGX", + data_category="Gene Families",data_format="TSV", + platform="Illumina MiSeq",access="open"), + "7": schema.CaseFile(7,experimental_strategy="16S", + data_category="Raw Reads",data_format="Fastq", + platform="Illumina HiSeq",access="controlled"), + "8": schema.CaseFile(8,experimental_strategy="16S", + data_category="Taxonomic Profile",data_format="TSV", + platform="Illumina HiSeq",access="open"), + "9": schema.CaseFile(9,experimental_strategy="16S", + data_category="Gene Families",data_format="TSV", + platform="Illumina HiSeq",access="open"), + "10": schema.CaseFile(10,experimental_strategy="16S", + data_category="Raw Reads",data_format="Fastq", + platform="Illumina HiSeq",access="controlled"), + "11": schema.CaseFile(11,experimental_strategy="16S", + data_category="Taxonomic Profile",data_format="TSV", + platform="Illumina HiSeq",access="open"), + "12": schema.CaseFile(12,experimental_strategy="16S", + data_category="Gene Families",data_format="TSV", + platform="Illumina HiSeq",access="open"), + } self.FILE_SIZES = { "gene": 300000000, "raw": 5000000000, "taxa": 200000000 } self.TEST_FILES = { "1": schema.File(1, "demoA_sample1_raw_reads.fastq","case1","sample1", "controlled", - self.FILE_SIZES["raw"], "Raw Reads", "Illumina", "Fastq", "WMGX", schema.FileCases(hits=["1"])), + self.FILE_SIZES["raw"], "Raw Reads", "Fastq", "Illumina MiSeq", "WMGX", + "demoA_sample1_raw_reads.fastq", schema.FileCases(hits=[self.CURRENT_FILE_CASES["1"]])), "2": schema.File(2, "demoA_sample1_taxonomic_profile.tsv","case1","sample1", "open", - self.FILE_SIZES["taxa"], "Taxonomic Profile", "Illumina", "TSV", "WMGX", schema.FileCases(hits=["1"])), + self.FILE_SIZES["taxa"], "Taxonomic Profile", "TSV", "Illumina MiSeq", "WMGX", + "demoA_sample1_taxonomic_profile.tsv", schema.FileCases(hits=[self.CURRENT_FILE_CASES["1"]])), "3": schema.File(3, "demoA_sample1_gene_families.tsv","case1","sample1", "open", - self.FILE_SIZES["gene"], "Gene Families", "Illumina", "TSV", "WMGX", schema.FileCases(hits=["1"])), + self.FILE_SIZES["gene"], "Gene Families", "TSV", "Illumina MiSeq", "WMGX", + "demoA_sample1_gene_families.tsv", schema.FileCases(hits=[self.CURRENT_FILE_CASES["1"]])), "4": schema.File(4, "demoA_sample2_raw_reads.fastq","case2","sample2", "controlled", - self.FILE_SIZES["raw"], "Raw Reads", "Illumina", "Fastq", "WMGX", schema.FileCases(hits=["2"])), + self.FILE_SIZES["raw"], "Raw Reads", "Fastq", "Illumina MiSeq", "WMGX", + "demoA_sample2_raw_reads.fastq", schema.FileCases(hits=[self.CURRENT_FILE_CASES["2"]])), "5": schema.File(5, "demoA_sample2_taxonomic_profile.tsv","case2","sample2", "open", - self.FILE_SIZES["taxa"], "Taxonomic Profile", "Illumina", "TSV", "WMGX", schema.FileCases(hits=["2"])), + self.FILE_SIZES["taxa"], "Taxonomic Profile", "TSV", "Illumina MiSeq", "WMGX", + "demoA_sample2_taxonomic_profile.tsv", schema.FileCases(hits=[self.CURRENT_FILE_CASES["2"]])), "6": schema.File(6, "demoA_sample2_gene_families.tsv","case2","sample2", "open", - self.FILE_SIZES["gene"], "Gene Families", "Illumina", "TSV", "WMGX", schema.FileCases(hits=["2"])), + self.FILE_SIZES["gene"], "Gene Families", "TSV", "Illumina MiSeq", "WMGX", + "demoA_sample2_gene_families.tsv", schema.FileCases(hits=[self.CURRENT_FILE_CASES["2"]])), "7": schema.File(7, "demoB_sample3_raw_reads.fastq","case3","sample3", "controlled", - self.FILE_SIZES["raw"], "Raw Reads", "Illumina", "Fastq", "WMGX", schema.FileCases(hits=["3"])), + self.FILE_SIZES["raw"], "Raw Reads", "Fastq", "Illumina HiSeq", "16S", + "demoB_sample3_raw_reads.fastq", schema.FileCases(hits=[self.CURRENT_FILE_CASES["3"]])), "8": schema.File(8, "demoB_sample3_taxonomic_profile.tsv","case3","sample3", "open", - self.FILE_SIZES["taxa"], "Taxonomic Profile", "Illumina", "TSV", "WMGX", schema.FileCases(hits=["3"])), + self.FILE_SIZES["taxa"], "Taxonomic Profile", "TSV", "Illumina HiSeq","16S", + "demoB_sample3_taxonomic_profile.tsv", schema.FileCases(hits=[self.CURRENT_FILE_CASES["3"]])), "9": schema.File(9, "demoB_sample3_gene_families.tsv","case3","sample3", "open", - self.FILE_SIZES["gene"], "Gene Families", "Illumina", "TSV", "WMGX", schema.FileCases(hits=["3"])), + self.FILE_SIZES["gene"], "Gene Families", "TSV", "Illumina HiSeq", "16S", + "demoB_sample3_gene_families.tsv", schema.FileCases(hits=[self.CURRENT_FILE_CASES["3"]])), "10": schema.File(10, "demoB_sample4_raw_reads.fastq","case4","sample4", "controlled", - self.FILE_SIZES["raw"], "Raw Reads", "Illumina", "Fastq", "WMGX", schema.FileCases(hits=["4"])), + self.FILE_SIZES["raw"], "Raw Reads", "Fastq", "Illumina HiSeq", "16S", + "demoB_sample4_raw_reads.fastq", schema.FileCases(hits=[self.CURRENT_FILE_CASES["4"]])), "11": schema.File(11, "demoB_sample4_taxonomic_profile.tsv","case4","sample4", "open", - self.FILE_SIZES["taxa"], "Taxonomic Profile", "Illumina", "TSV", "WMGX", schema.FileCases(hits=["4"])), + self.FILE_SIZES["taxa"], "Taxonomic Profile", "TSV", "Illumina HiSeq", "16S", + "demoB_sample4_taxonomic_profile.tsv", schema.FileCases(hits=[self.CURRENT_FILE_CASES["4"]])), "12": schema.File(12, "demoB_sample4_gene_families.tsv","case4","sample4", "open", - self.FILE_SIZES["gene"], "Gene Families", "Illumina", "TSV", "WMGX", schema.FileCases(hits=["4"])), + self.FILE_SIZES["gene"], "Gene Families", "TSV", "Illumina HiSeq", "16S", + "demoB_sample4_gene_families.tsv", schema.FileCases(hits=[self.CURRENT_FILE_CASES["4"]])), } self.CURRENT_COUNTS = schema.Count( @@ -95,80 +149,28 @@ def __init__(self): demographic=schema.Demographic("not hispanic or latino","male","white"), project=self.CURRENT_PROJECTS["1"], summary=schema.Summary(case_count=1,file_count=1,file_size=1, - data_categories=self.DATA_CATEGORIES_SINGLE_CASE)), + data_categories=self.DATA_CATEGORIES_SINGLE_CASE), + files=schema.CaseFiles(hits=[self.CASE_FILES["1"],self.CASE_FILES["2"],self.CASE_FILES["3"]])), "2":schema.Case(2,case_id="Case2",primary_site="Stool", demographic=schema.Demographic("not hispanic or latino","male","white"), project=self.CURRENT_PROJECTS["1"], summary=schema.Summary(case_count=1,file_count=1,file_size=1, - data_categories=self.DATA_CATEGORIES_SINGLE_CASE)), + data_categories=self.DATA_CATEGORIES_SINGLE_CASE), + files=schema.CaseFiles(hits=[self.CASE_FILES["4"],self.CASE_FILES["5"],self.CASE_FILES["6"]])), "3":schema.Case(3,case_id="Case3",primary_site="Stool", demographic=schema.Demographic("not hispanic or latino","female","white"), project=self.CURRENT_PROJECTS["2"], summary=schema.Summary(case_count=1,file_count=1,file_size=1, - data_categories=self.DATA_CATEGORIES_SINGLE_CASE)), + data_categories=self.DATA_CATEGORIES_SINGLE_CASE), + files=schema.CaseFiles(hits=[self.CASE_FILES["7"],self.CASE_FILES["8"],self.CASE_FILES["9"]])), "4":schema.Case(4,case_id="Case4",primary_site="Stool", - demographic=schema.Demographic("not hispanic or latino","female","white"), + demographic=schema.Demographic("hispanic or latino","female","white"), project=self.CURRENT_PROJECTS["2"], summary=schema.Summary(case_count=1,file_count=1,file_size=1, - data_categories=self.DATA_CATEGORIES_SINGLE_CASE)) + data_categories=self.DATA_CATEGORIES_SINGLE_CASE), + files=schema.CaseFiles(hits=[self.CASE_FILES["10"],self.CASE_FILES["11"],self.CASE_FILES["12"]])) } self.CURRENT_FILES = schema.Files(hits=self.TEST_FILES.keys()) self.CURRENT_CASES = schema.RepositoryCases(hits=self.TEST_CASES.keys()) - self.FILE_AGGREGATIONS=schema.FileAggregations( - data_category=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=4, key="Raw Reads"), - schema.Bucket(doc_count=4, key="Taxonomic Profile"), - schema.Bucket(doc_count=4, key="Gene Families")]), - experimental_strategy=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=6, key="WMGX"), - schema.Bucket(doc_count=6, key="16S")]), - data_format=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=4, key="Fastq"), - schema.Bucket(doc_count=8, key="TSV")]), - platform=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=6, key="Illumina MiSeq"), - schema.Bucket(doc_count=6, key="Illumina HiSeq")]), - cases__primary_site=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=12, key="Stool")]), - cases__project__project_id=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=6, key="NHSII-DemoA"), - schema.Bucket(doc_count=6, key="NHSII-DemoB")]), - access=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=4, key="open"), - schema.Bucket(doc_count=8, key="controlled")])) - - self.CASE_AGGREGATIONS=schema.CaseAggregations( - demographic__ethnicity=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=10, key="not hispanic or latino"), - schema.Bucket(doc_count=2, key="hispanic or latino")]), - demographic__gender=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=8, key="male"), - schema.Bucket(doc_count=4, key="female")]), - demographic__race=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=8, key="white"), - schema.Bucket(doc_count=4, key="asian")]), - primary_site=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=12, key="Stool")]), - project__project_id=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=6, key="NHSII-DemoA"), - schema.Bucket(doc_count=6, key="NHSII-DemoB")]), - project__program__name=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=12, key="NHSII")]), - ) - - self.PROJECT_AGGREGATIONS=schema.ProjectAggregations( - primary_site=schema.Aggregations(buckets=[schema.Bucket(doc_count=45, key="Stool")]), - program__name=schema.Aggregations(buckets=[schema.Bucket(doc_count=45, key="NHSII")]), - project_id=schema.Aggregations(buckets=[schema.Bucket(doc_count=15, key="NHSII-DemoA"), - schema.Bucket(doc_count=15, key="NHSII-DemoB"), - schema.Bucket(doc_count=15, key="NHSII-DemoC")]), - summary__data_categories__data_category=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=15, key="Raw Reads"), - schema.Bucket(doc_count=15, key="Gene Families"), - schema.Bucket(doc_count=15, key="Taxonomic Profiles")]), - summary__experimental_strategies__experimental_strategy=schema.Aggregations(buckets=[ - schema.Bucket(doc_count=30, key="WMGX"), - schema.Bucket(doc_count=15, key="16S")])) - diff --git a/database.py b/database.py index c248c32..a45fd1a 100644 --- a/database.py +++ b/database.py @@ -1,76 +1,346 @@ +# Calls to obtain values from database -# Calls to obtain values from the data structures (to be populated by the local database next) +import mysql.connector as mariadb +import json -import const +def add_key_increment(dictionary, key): + if not key in dictionary: + dictionary[key]=0 + dictionary[key]+=1 class Data(object): - def __getattr__(self, name): - # load data if not already loaded - if name == "data": - self.load_data() + db_conn = mariadb.connect(user='root', password='DanDa1osh', db='portal_ui') + cursor = db_conn.cursor(buffered=True) - def load_data(self): - self.data = const.DB() - - def get_total_projects_count(self): - return len(self.data.CURRENT_PROJECTS.keys()) + def fetch_results(self,query): + self.cursor.execute(query) + row_headers=[x[0] for x in self.cursor.description] + rows = self.cursor.fetchall() + json_data=[] + for row in rows: + json_data.append(dict(zip(row_headers,row))) + return json_data def get_current_projects(self): - return [self.get_project(project_id) for project_id in self.data.CURRENT_PROJECTS.keys()] + projects_json=self.fetch_results("select `id` from `project`") + print("json_data inside get_current_projects") + print(projects_json) + return [self.get_project(project['id']) for project in projects_json] def get_user(self): - return self.data.CURRENT_USER - - def get_filecase(self,id): - return self.data.CURRENT_FILE_CASES[id] + import schema + return schema.User(username="null") def get_project(self,id): - return self.data.CURRENT_PROJECTS[id] + import schema - def get_file(self,id): - return self.data.TEST_FILES[id] + print("id inside get_project") + print(id) + project_json=self.fetch_results( + "select id, project_id, name as pr_name, program from `project` where `id` =" + str(id)) + project_id=project_json[0]['project_id'] + proj_id=project_json[0]['id'] + proj_name=project_json[0]['pr_name'] + proj_program=["HPFS"] + #proj_program=project_json[0]['program'] + #proj_primary_site=project_json[0]['primary_site'] + proj_primary_site=["Stool"] + + file_count_json=self.fetch_results("select count(id) as file_count from file_sample where project='"+project_id+"'") + proj_file_count=file_count_json[0]['file_count'] + + part_count_json=self.fetch_results( + "select count(distinct participant) as part_count from file_sample where project='"+project_id+"'") + proj_part_count=part_count_json[0]['part_count'] + + data_cat_json=self.fetch_results("select distinct data_category from file_sample where project='"+project_id+"'") + proj_data_cat=[data_cat['data_category'] for data_cat in data_cat_json] - def get_total_files(self): - return len(self.data.CURRENT_FILES.hits) + exp_str_json=self.fetch_results("select distinct experimental_strategy from file_sample where project='"+project_id+"'") + proj_exp_str=[exp_str['experimental_strategy'] for exp_str in exp_str_json] + + sum_file_size_json=self.fetch_results( + "select sum(file_size) as sum_file_size from file_sample where project='"+project_id+"'") + proj_file_size=sum_file_size_json[0]['sum_file_size'] + print("values inside get_project") + print( + proj_id,project_id,proj_name,proj_program,proj_part_count,proj_file_count,proj_data_cat,proj_exp_str,proj_file_size,proj_primary_site) + + return schema.Project( + id=proj_id, + project_id=project_id, + name=proj_name, + program=proj_program, + summary=schema.Summary( + case_count=proj_part_count, + file_count=proj_file_count, + data_categories=proj_data_cat, + experimental_strategies=proj_exp_str, + file_size=proj_file_size), + primary_site=proj_primary_site) + + + def get_file(self,id): + import schema - def get_total_case_annotations(self): - return 1 # not currently being used + file_json=self.fetch_results("select * from `file_sample` where `id` ="+ str(id)) + project_json=self.fetch_results( + "select id , project_id, primary_site from project where project_id='"+file_json[0]['project']+"'") + part_json=self.fetch_results( + "select id, entity_participant_id from participant where entity_participant_id='"+file_json[0]['participant']+"'") + print("file and project json inside get_file") + print(file_json[0]['sample']) + print(project_json[0]['project_id']) + return schema.File( + id=file_json[0]['id'], + name=file_json[0]['file_name'], + file_id=file_json[0]['file_id'], + participant=file_json[0]['participant'], + sample=file_json[0]['sample'], + access=file_json[0]['access'], + file_size=file_json[0]['file_size'], + data_category=file_json[0]['data_category'], + data_format=file_json[0]['data_format'], + platform=file_json[0]['platform'], + experimental_strategy=file_json[0]['experimental_strategy'], + file_name=file_json[0]['file_name'], + cases=schema.FileCases( + hits=[schema.FileCase( + part_json[0]['id'], + case_id=file_json[0]['participant'], + project=self.get_project(project_json[0]['id']), + demographic=schema.Demographic("not hispanic or latino","male","white"), + #primary_site=project_json[0]['primary_site'] + primary_site="Stool")])) def get_case_annotation(self): - return self.data.CURRENT_CASE_ANNOTATION + import schema + return schema.CaseAnnotation() def get_current_files(self): - return self.data.CURRENT_FILES + import schema + + files_json=self.fetch_results("select `id` from `file_sample`") + file_hits=[] + for item in files_json: + file_hits.append(item['id']) + print("file_hits inside current files") + print(file_hits) + return schema.Files(hits=file_hits) def get_current_cases(self): - return self.data.CURRENT_CASES + import schema + + cases_json=self.fetch_results("select id from participant") + return schema.RepositoryCases(hits=[case['id'] for case in cases_json]) def get_case(self,id): - return self.data.TEST_CASES[id] + import schema + + case_json=self.fetch_results("select * from participant where id="+str(id)) + part_id=case_json[0]['entity_participant_id'] + + case_count_json=self.fetch_results( + "select count(entity_participant_id) as case_count from participant where entity_participant_id="+str(part_id)) + case_count=case_count_json[0]['case_count'] + + file_count_json=self.fetch_results( + "select count(id) as file_count from file_sample where participant="+str(part_id)) + file_count=file_count_json[0]['file_count'] + + file_size_json=self.fetch_results( + "select sum(file_size) as filesize from file_sample where participant="+str(part_id)) + file_size=file_size_json[0]['filesize'] + + print("inside get case") + print(part_id) + proj_json=self.fetch_results('''select distinct file_sample.project, project.id + from file_sample, project where file_sample.participant='''+str(part_id)+''' + and project.project_id=file_sample.project''') + proj_id=proj_json[0]['id'] + print(proj_id) + + cat_raw_json=self.fetch_results( + "select count(id) as raw_count from file_sample where participant="+str(part_id)+" and data_category='Raw Reads'") + cat_raw_count=cat_raw_json[0]['raw_count'] + + cat_taxa_json=self.fetch_results( + "select count(id) as taxa_count from file_sample where participant="+str(part_id)+" and data_category='Taxonomic Profiles'") + cat_taxa_count=cat_taxa_json[0]['taxa_count'] + + cat_gene_json=self.fetch_results( + "select count(id) as gene_count from file_sample where participant="+str(part_id)+" and data_category='Gene Families'") + cat_gene_count=cat_gene_json[0]['gene_count'] + + files_json=self.fetch_results("select * from file_sample where participant="+str(part_id)) + + return schema.Case(id, + case_id="entity_participant_id", + primary_site="Stool", + demographic=schema.Demographic("not hispanic or latino","male","white"), + project=self.get_project(proj_id), + summary=schema.Summary( + case_count=case_count, + file_count=file_count, + file_size=file_size, + data_categories=[schema.DataCategories(file_count=cat_raw_count, data_category="Raw Reads"), + schema.DataCategories(file_count=cat_gene_count, data_category="Gene Families"), + schema.DataCategories(file_count=cat_taxa_count, data_category="Taxonomic Profiles")]), + files=schema.CaseFiles(hits=[schema.CaseFile( + case_file['id'], + experimental_strategy=case_file['experimental_strategy'], + data_category=case_file['data_category'], + data_format=case_file['data_format'], + platform=case_file['platform'], + access=case_file['access']) for case_file in files_json])) + + + def get_project_aggregations(self, projects): + import schema + + # compile aggregations from project + aggregates = {"primary_site": {}, "program__name": {}, + "project_id": {}, + "summary__data_categories__data_category": {}, + "summary__experimental_strategies__experimental_strategy": {}} + print("projects inside aggregations") + + for project in projects: + print( + project.id, project.project_id,project.summary.data_categories,project.primary_site, + project.program,project.summary.experimental_strategies) + add_key_increment(aggregates["primary_site"],project.primary_site[0]) + add_key_increment(aggregates["project_id"], project.project_id) + add_key_increment(aggregates["program__name"], project.program[0]) + for item in project.summary.data_categories: + add_key_increment(aggregates["summary__data_categories__data_category"], item) + for item in project.summary.experimental_strategies: + add_key_increment(aggregates["summary__experimental_strategies__experimental_strategy"], item) + + project_aggregates=schema.ProjectAggregations( + primary_site=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["primary_site"].items()]), + project_id=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["project_id"].items()]), + program__name=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["program__name"].items()]), + summary__data_categories__data_category=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["summary__data_categories__data_category"].items()]), + summary__experimental_strategies__experimental_strategy=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["summary__experimental_strategies__experimental_strategy"].items()])) + + return project_aggregates - def get_project_aggregations(self): - return self.data.PROJECT_AGGREGATIONS def get_current_counts(self): - return self.data.CURRENT_COUNTS + import schema + + projects_json = self.fetch_results("select count(`project_id`)as projects_count from project") + projects = projects_json[0]['projects_count'] + + participants_json = self.fetch_results("select count(`entity_participant_id`) as participants_count from `participant`") + participants = participants_json[0]['participants_count'] + + samples_json = self.fetch_results("select count(`sample`) as samples_count from `sample`") + samples = samples_json[0]['samples_count'] + + dataFormats_json = self.fetch_results("select count(distinct `data_format`) as data_format_count from `file_sample`") + dataFormats = dataFormats_json[0]['data_format_count'] + + rawFiles_json = self.fetch_results( + "select count(`id`) as rawfiles_count from `file_sample` where type='rawFiles'") + rawFiles = rawFiles_json[0]['rawfiles_count'] - def get_file_aggregations(self): - return self.data.FILE_AGGREGATIONS + prFiles_json = self.fetch_results( + "select count(`id`) as prfiles_count from `file_sample` where type='processedFiles'") + prFiles = prFiles_json[0]['prfiles_count'] - def get_case_aggregations(self): - return self.data.CASE_AGGREGATIONS + return schema.Count( + projects=projects, + participants=participants, + samples=samples, + dataFormats=dataFormats, + rawFiles=rawFiles, + processedFiles=prFiles) - def get_total_cases_per_file(self): - return 1 # this is currently not being used - def get_total_cases(self): - return len(self.data.TEST_CASES.keys()) + def get_file_aggregations(self, files): + import schema + + # aggregate file data + aggregates = {"data_category": {}, "experimental_strategy": {}, + "data_format": {}, "platform": {}, "cases__primary_site": {}, + "cases__project__project_id": {}, "access": {}} + + for file in files: + print(" values inside file aggregation") + print(file.data_category,file.experimental_strategy,file.data_format,file.platform,file.access) + add_key_increment(aggregates["data_category"], file.data_category) + add_key_increment(aggregates["experimental_strategy"], file.experimental_strategy) + add_key_increment(aggregates["data_format"], file.data_format) + add_key_increment(aggregates["platform"], file.platform) + add_key_increment(aggregates["access"], file.access) + project = file.cases.hits[0].project + add_key_increment(aggregates["cases__primary_site"], project.primary_site[0]) + add_key_increment(aggregates["cases__project__project_id"], project.project_id) + + file_aggregates = schema.FileAggregations( + data_category=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["data_category"].items()]), + experimental_strategy=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["experimental_strategy"].items()]), + data_format=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["data_format"].items()]), + platform=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["platform"].items()]), + cases__primary_site=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["cases__primary_site"].items()]), + access=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["access"].items()]), + cases__project__project_id=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["cases__project__project_id"].items()])) + + return file_aggregates + + def get_case_aggregations(self, cases): + import schema + + # aggregate case data + aggregates = {"demographic__ethnicity": {}, "demographic__gender": {}, + "demographic__race": {}, "primary_site": {}, "project__project_id": {}, + "project__program__name": {}} + + for case in cases: + add_key_increment(aggregates["demographic__ethnicity"], case.demographic.ethnicity) + add_key_increment(aggregates["demographic__gender"], case.demographic.gender) + add_key_increment(aggregates["demographic__race"], case.demographic.race) + add_key_increment(aggregates["primary_site"], case.primary_site) + add_key_increment(aggregates["project__project_id"], case.project.project_id) + add_key_increment(aggregates["project__program__name"], case.project.program.name) + + case_aggregates=schema.CaseAggregations( + demographic__ethnicity=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["demographic__ethnicity"].items()]), + demographic__gender=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["demographic__gender"].items()]), + demographic__race=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["demographic__race"].items()]), + primary_site=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["primary_site"].items()]), + project__project_id=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["project__project_id"].items()]), + project__program__name=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["project__program__name"].items()])) + + return case_aggregates def get_facets(self): return "null" # this is not currently being used def get_cart_file_size(self): - return self.data.CURRENT_FILE_SIZE + import schema -data = Data() + file_json=self.fetch_results("select sum(file_size) as sum_size from file_sample") + return schema.FileSize(file_json[0]['sum_size']) +data = Data() diff --git a/load_local_database.py b/load_local_database.py index f83f0f0..944566d 100644 --- a/load_local_database.py +++ b/load_local_database.py @@ -23,7 +23,7 @@ def parse_arguments(args): required=False) parser.add_argument( "--key-file", - default="biom-mass-fdcadb440fdf.json", + default="/home/hutlab_public/compute_engine_service_account_key/biom-mass-8dc9ab934396.json", help="google project service account private key file path\n]", required=False) parser.add_argument( diff --git a/query_bigquery.py b/query_bigquery.py index 2d179ab..770c7ad 100644 --- a/query_bigquery.py +++ b/query_bigquery.py @@ -22,7 +22,7 @@ def parse_arguments(args): required=False) parser.add_argument( "--key-file", - default="biom-mass-fdcadb440fdf.json", + default="/home/hutlab_public/compute_engine_service_account_key/biom-mass-8dc9ab934396.json", help="google project service account private key file path\n]", required=False) parser.add_argument( diff --git a/schema.graphql b/schema.graphql index 3d41bba..fe55875 100644 --- a/schema.graphql +++ b/schema.graphql @@ -26,6 +26,7 @@ type Case implements Node { project: Project summary: Summary annotations: CaseAnnotations + files: CaseFiles } type CaseAggregations { @@ -75,6 +76,35 @@ type CaseEdge { cursor: String! } +type CaseFile implements Node { + # The ID of the object. + id: ID! + case_id: String + experimental_strategy: String + data_category: String + data_format: String + platform: String + access: String +} + +type CaseFileConnection { + pageInfo: PageInfo! + edges: [CaseFileEdge]! + total: Int +} + +type CaseFileEdge { + # The item at the end of the edge + node: CaseFile + + # A cursor for use in pagination + cursor: String! +} + +type CaseFiles { + hits(first: Int, offset: Int, sort: [Sort], filters: FiltersArgument, before: String, after: String, last: Int): CaseFileConnection +} + type Count { projects: String participants: String @@ -128,9 +158,9 @@ type File implements Node { data_format: String platform: String experimental_strategy: String + file_name: String cases: FileCases file_id: String - file_name: String type: String } @@ -149,6 +179,8 @@ type FileCase implements Node { id: ID! case_id: String project: Project + demographic: Demographic + primary_site: String } type FileCaseConnection { diff --git a/schema.py b/schema.py index a7e1403..13b23d4 100644 --- a/schema.py +++ b/schema.py @@ -86,12 +86,19 @@ class Meta: def get_node(cls, info, id): return data.get_project(id) +class Demographic(graphene.ObjectType): + ethnicity = graphene.String() + gender = graphene.String() + race = graphene.String() + class FileCase(graphene.ObjectType): class Meta: interfaces = (graphene.relay.Node,) case_id = graphene.String() project = graphene.Field(Project) + demographic = graphene.Field(Demographic) + primary_site = graphene.String() class FileCaseConnection(graphene.relay.Connection): class Meta: @@ -100,7 +107,7 @@ class Meta: total = graphene.Int() def resolve_total(self, info): - return data.get_total_cases_per_file() + return len(self.edges) class FileCases(graphene.ObjectType): hits = graphene.relay.ConnectionField(FileCaseConnection, @@ -109,9 +116,6 @@ class FileCases(graphene.ObjectType): sort=graphene.List(Sort), filters=FiltersArgument()) - def resolve_hits(self, info, first=None, offset=None, sort=None, filters=None): - return [data.get_filecase(file_id) for file_id in self.hits] - class File(graphene.ObjectType): class Meta: interfaces = (graphene.relay.Node,) @@ -125,19 +129,16 @@ class Meta: data_format = graphene.String() platform = graphene.String() experimental_strategy = graphene.String() + file_name = graphene.String() cases = graphene.Field(FileCases) file_id = graphene.String() - file_name = graphene.String() type = graphene.String() def resolve_file_id(self, info): return self.name - def resolve_file_name(self, info): - return self.name - def resolve_type(self, info): return self.data_format @@ -151,7 +152,7 @@ class Meta: total = graphene.Int() def resolve_total(self, info): - return data.get_total_projects_count() + return len(self.edges) class FileConnection(graphene.relay.Connection): class Meta: @@ -159,7 +160,7 @@ class Meta: total = graphene.Int() def resolve_total(self, info): - return data.get_total_files() + return len(self.edges) class Bucket(graphene.ObjectType): doc_count = graphene.Int() @@ -196,10 +197,15 @@ class Files(graphene.ObjectType): aggregations_filter_themselves=graphene.Boolean()) def resolve_hits(self, info, first=None, score=None, offset=None, sort=None, filters=None): - return [data.get_file(file_id) for file_id in self.hits] + all_files = [data.get_file(file_id) for file_id in self.hits] + filtered_files = utilities.filter_hits(all_files, filters, "files") + sorted_files = utilities.sort_hits(filtered_files, sort) + return sorted_files def resolve_aggregations(self, info, filters=None, aggregations_filter_themselves=None): - return data.get_file_aggregations() + all_files = [data.get_file(file_id) for file_id in self.hits] + filtered_files = utilities.filter_hits(all_files, filters, "files") + return data.get_file_aggregations(filtered_files) def resolve_facets(self, info, filters=None, facets=None): return data.get_facets() @@ -226,7 +232,7 @@ class Meta: total = graphene.Int() def resolve_total(self, info): - return data.get_total_case_annotations() + return len(self.edges) class CaseAnnotations(graphene.ObjectType): hits = graphene.relay.ConnectionField(CaseAnnotationConnection, @@ -237,12 +243,35 @@ class CaseAnnotations(graphene.ObjectType): filters=FiltersArgument()) def resolve_hits(self, info, first=None, score=None, offset=None, sort=None, filters=None): + # filters are not currently in use return data.get_case_annotation() -class Demographic(graphene.ObjectType): - ethnicity = graphene.String() - gender = graphene.String() - race = graphene.String() +class CaseFile(graphene.ObjectType): + class Meta: + interfaces = (graphene.relay.Node,) + + case_id = graphene.String() + experimental_strategy = graphene.String() + data_category = graphene.String() + data_format = graphene.String() + platform = graphene.String() + access = graphene.String() + +class CaseFileConnection(graphene.relay.Connection): + class Meta: + node = CaseFile + + total = graphene.Int() + + def resolve_total(self, info): + return len(self.edges) + +class CaseFiles(graphene.ObjectType): + hits = graphene.relay.ConnectionField(CaseFileConnection, + first=graphene.Int(), + offset=graphene.Int(), + sort=graphene.List(Sort), + filters=FiltersArgument()) class Case(graphene.ObjectType): class Meta: @@ -257,6 +286,8 @@ class Meta: summary = graphene.Field(Summary) annotations = graphene.Field(CaseAnnotations) + files = graphene.Field(CaseFiles) + def resolve_submitter_id(self, info): return self.case_id @@ -267,7 +298,7 @@ class Meta: total = graphene.Int() def resolve_total(self, info): - return data.get_total_cases() + return len(self.edges) class RepositoryCases(graphene.ObjectType): hits = graphene.relay.ConnectionField(CaseConnection, @@ -284,10 +315,16 @@ class RepositoryCases(graphene.ObjectType): facets=graphene.List(graphene.String)) def resolve_hits(self, info, first=None, score=None, offset=None, sort=None, filters=None): - return [data.get_case(case_id) for case_id in self.hits] + all_cases = [data.get_case(case_id) for case_id in self.hits] + filtered_cases = utilities.filter_hits(all_cases, filters, "cases") + sorted_cases = utilities.sort_hits(filtered_cases, sort) + return sorted_cases def resolve_aggregations(self, info, filters=None, aggregations_filter_themselves=None): - return data.get_case_aggregations() + all_cases = [data.get_case(case_id) for case_id in self.hits] + filtered_cases = utilities.filter_hits(all_cases, filters, "cases") + case_aggregations = data.get_case_aggregations(filtered_cases) + return case_aggregations def resolve_facets(self, info, filters=None, facets=None): return data.get_facets() @@ -321,11 +358,14 @@ class Projects(graphene.ObjectType): def resolve_hits(self, info, first=None, offset=None, sort=None, filters=None): projects = data.get_current_projects() - filtered_projects = utilities.filter_hits(projects, filters) + filtered_projects = utilities.filter_hits(projects, filters, "projects") return filtered_projects def resolve_aggregations(self, info, filters=None, aggregations_filter_themselves=None): - return data.get_project_aggregations() + projects = data.get_current_projects() + filtered_projects = utilities.filter_hits(projects, filters, "projects") + project_aggregations = data.get_project_aggregations(filtered_projects) + return project_aggregations class FileSize(graphene.ObjectType): value = graphene.Float() diff --git a/utilities.py b/utilities.py index 651257e..72d8f3d 100644 --- a/utilities.py +++ b/utilities.py @@ -1,40 +1,110 @@ import json +import ast import collections -def get_class_member_value(obj,members,offset=1): - levels = members.split(".")[offset:] - final_value = getattr(obj, levels.pop(0)) +def get_class_member_value(obj, levels, subset=False): + final_value = obj for item in levels: final_value = getattr(final_value, item) - if isinstance(final_value, list): - sub_levels = ".".join(levels[levels.index(item)+1:]) - final_value = [get_class_member_value(sub_value,sub_levels,offset=0) for sub_value in final_value] + sub_levels = levels[levels.index(item)+1:] + if not isinstance(sub_levels, list): + sub_levels = [sub_levels] + final_value = [get_class_member_value(sub_value,sub_levels,subset=True) for sub_value in final_value] break - if not isinstance(final_value, list) and offset !=0: + if not isinstance(final_value, list) and not subset: final_value = [final_value] return final_value -def filter_hits(hits, filters): +def check_for_match(value_selected, hit_values): + # check if the values for the hit match one or more of the values selected + + match = False + if isinstance(value_selected, list): + for item in value_selected: + if item in hit_values: + match = True + break + elif value_selected in hit_values: + match = True + + return match + +def filter_hits(hits, filters, object_name): # Filter the hits based on the json string provided if not filters: return hits - - filtered_set = set() + + all_filtered_sets = [] for content in filters["content"]: field=content["content"]["field"] - value_selected=content["content"]["value"][0] - for item in hits: - hit_values=get_class_member_value(item,field) - if value_selected in hit_values: - filtered_set.add(item) - return list(filtered_set) - -# The functions were based on code from + value_selected=content["content"]["value"] + + # check if this filter is for this object + levels = field.split(".") + top_level = levels.pop(0) + if top_level != object_name: + if not top_level in dir(hits[0]): + all_filtered_sets.append(set(hits)) + else: + filtered_set=set() + for item in hits: + # find object that contains the search values + hit_values = [] + for search_item in getattr(item, top_level).hits: + hit_values+=get_class_member_value(search_item, levels) + if check_for_match(value_selected, hit_values): + filtered_set.add(item) + all_filtered_sets.append(list(filtered_set)) + else: + filtered_set=set() + for item in hits: + hit_values=get_class_member_value(item,levels) + if check_for_match(value_selected, hit_values): + filtered_set.add(item) + all_filtered_sets.append(list(filtered_set)) + + # reduce sets + final_set = set(all_filtered_sets.pop(0)) + for next_set in all_filtered_sets: + final_set = final_set.intersection(next_set) + + return list(final_set) + +def sort_hits(hits, sort): + # Sort the hits based on the string provided + + if not sort: + return hits + + sorted_hits = hits + for content in sort: + eval_content = ast.literal_eval(content) + field=eval_content["field"] + levels=field.split(".") + + # reverse sort order if set + order=eval_content["order"] + reverse_sort_order = False + if order != "asc": + reverse_sort_order = True + + # get the values for all of the hits + hit_value_pairs = [] + for item in sorted_hits: + hit_value_pairs.append((item, get_class_member_value(item,levels)[0])) + + hit_values = collections.OrderedDict(hit_value_pairs) + + sorted_hits=sorted(hit_values, key=hit_values.get, reverse=reverse_sort_order) + + return sorted_hits + +# The functions below were based on code from # https://github.com/nderkach/python-grahql-api def _json_object_hook(d): From bbc1397bccd2d33af6231af5c7a57a8cee08d2cd Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Sun, 17 Mar 2019 06:41:44 +0000 Subject: [PATCH 02/34] Change db user --- database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database.py b/database.py index a45fd1a..e44230f 100644 --- a/database.py +++ b/database.py @@ -9,7 +9,7 @@ def add_key_increment(dictionary, key): dictionary[key]+=1 class Data(object): - db_conn = mariadb.connect(user='root', password='DanDa1osh', db='portal_ui') + db_conn = mariadb.connect(user='biom-mass', password='', db='portal_ui') cursor = db_conn.cursor(buffered=True) def fetch_results(self,query): From c39a5d8e16e08be9074af943ba7030647638593f Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Sun, 17 Mar 2019 19:42:29 +0000 Subject: [PATCH 03/34] More details to project --- database.py | 87 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 63 insertions(+), 24 deletions(-) diff --git a/database.py b/database.py index e44230f..6411d80 100644 --- a/database.py +++ b/database.py @@ -9,7 +9,7 @@ def add_key_increment(dictionary, key): dictionary[key]+=1 class Data(object): - db_conn = mariadb.connect(user='biom-mass', password='', db='portal_ui') + db_conn = mariadb.connect(user='biom_mass', password='DanDa1osh', db='portal_ui') cursor = db_conn.cursor(buffered=True) def fetch_results(self,query): @@ -37,12 +37,12 @@ def get_project(self,id): print("id inside get_project") print(id) project_json=self.fetch_results( - "select id, project_id, name as pr_name, program from `project` where `id` =" + str(id)) + "select * from `project` where `id` =" + str(id)) project_id=project_json[0]['project_id'] proj_id=project_json[0]['id'] - proj_name=project_json[0]['pr_name'] + proj_name=project_json[0]['name'] proj_program=["HPFS"] - #proj_program=project_json[0]['program'] + proj_program=project_json[0]['program'] #proj_primary_site=project_json[0]['primary_site'] proj_primary_site=["Stool"] @@ -53,33 +53,72 @@ def get_project(self,id): "select count(distinct participant) as part_count from file_sample where project='"+project_id+"'") proj_part_count=part_count_json[0]['part_count'] - data_cat_json=self.fetch_results("select distinct data_category from file_sample where project='"+project_id+"'") - proj_data_cat=[data_cat['data_category'] for data_cat in data_cat_json] - - exp_str_json=self.fetch_results("select distinct experimental_strategy from file_sample where project='"+project_id+"'") - proj_exp_str=[exp_str['experimental_strategy'] for exp_str in exp_str_json] - sum_file_size_json=self.fetch_results( "select sum(file_size) as sum_file_size from file_sample where project='"+project_id+"'") proj_file_size=sum_file_size_json[0]['sum_file_size'] print("values inside get_project") print( - proj_id,project_id,proj_name,proj_program,proj_part_count,proj_file_count,proj_data_cat,proj_exp_str,proj_file_size,proj_primary_site) + proj_id,project_id,proj_name,proj_program,proj_part_count,proj_file_count,proj_file_size,proj_primary_site) return schema.Project( id=proj_id, project_id=project_id, name=proj_name, - program=proj_program, + program=schema.Program(name=proj_program), summary=schema.Summary( case_count=proj_part_count, file_count=proj_file_count, - data_categories=proj_data_cat, - experimental_strategies=proj_exp_str, + data_categories=self.get_data_categories("project",project_id), + experimental_strategies=self.get_experimental_strategies("project",project_id), file_size=proj_file_size), primary_site=proj_primary_site) + def get_data_categories(self,table,id): + import schema + + print("inside get_data_cat") + print(id) + + data_cat_json=self.fetch_results( + "select distinct data_category from file_sample where "+ table+"='" + str(id)+"'") + data_cat=[] + for item in data_cat_json: + item_files_json=self.fetch_results( + "select count(id) as file_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") + item_part_json=self.fetch_results( + "select count(distinct participant) as case_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") + data_cat.append( + schema.DataCategories( + case_count=item_part_json[0]['case_count'], + file_count=item_files_json[0]['file_count'], + data_category=item['data_category'])) + + return data_cat + + + + def get_experimental_strategies(self,table,id): + import schema + + print("inside get exp_str") + print(table,id) + exp_str_json=self.fetch_results( + "select distinct experimental_strategy from file_sample where "+ table+"='" + str(id) +"'") + exp_str=[] + for item in exp_str_json: + item_files_json=self.fetch_results( + "select count(id) as file_count from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") + item_part_json=self.fetch_results( + "select count(distinct participant) as case_count from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") + exp_str.append( + schema.ExperimentalStrategies( + case_count=item_part_json[0]['case_count'], + file_count=item_files_json[0]['file_count'], + experimental_strategy=item['experimental_strategy'])) + + return exp_str + def get_file(self,id): import schema @@ -154,10 +193,10 @@ def get_case(self,id): print("inside get case") print(part_id) - proj_json=self.fetch_results('''select distinct file_sample.project, project.id + proj_json=self.fetch_results('''select distinct file_sample.project, project.id as proj_id from file_sample, project where file_sample.participant='''+str(part_id)+''' and project.project_id=file_sample.project''') - proj_id=proj_json[0]['id'] + proj_id=proj_json[0]['proj_id'] print(proj_id) cat_raw_json=self.fetch_results( @@ -175,7 +214,7 @@ def get_case(self,id): files_json=self.fetch_results("select * from file_sample where participant="+str(part_id)) return schema.Case(id, - case_id="entity_participant_id", + case_id=part_id, primary_site="Stool", demographic=schema.Demographic("not hispanic or latino","male","white"), project=self.get_project(proj_id), @@ -207,15 +246,15 @@ def get_project_aggregations(self, projects): for project in projects: print( - project.id, project.project_id,project.summary.data_categories,project.primary_site, - project.program,project.summary.experimental_strategies) + project.id, project.project_id,project.primary_site, + project.program) add_key_increment(aggregates["primary_site"],project.primary_site[0]) add_key_increment(aggregates["project_id"], project.project_id) - add_key_increment(aggregates["program__name"], project.program[0]) + add_key_increment(aggregates["program__name"], project.program.name) for item in project.summary.data_categories: - add_key_increment(aggregates["summary__data_categories__data_category"], item) + add_key_increment(aggregates["summary__data_categories__data_category"], item.data_category) for item in project.summary.experimental_strategies: - add_key_increment(aggregates["summary__experimental_strategies__experimental_strategy"], item) + add_key_increment(aggregates["summary__experimental_strategies__experimental_strategy"], item.experimental_strategy) project_aggregates=schema.ProjectAggregations( primary_site=schema.Aggregations( @@ -314,9 +353,9 @@ def get_case_aggregations(self, cases): add_key_increment(aggregates["demographic__ethnicity"], case.demographic.ethnicity) add_key_increment(aggregates["demographic__gender"], case.demographic.gender) add_key_increment(aggregates["demographic__race"], case.demographic.race) - add_key_increment(aggregates["primary_site"], case.primary_site) + add_key_increment(aggregates["primary_site"], case.primary_site[0]) add_key_increment(aggregates["project__project_id"], case.project.project_id) - add_key_increment(aggregates["project__program__name"], case.project.program.name) + add_key_increment(aggregates["project__program__name"], case.project.program[0]) case_aggregates=schema.CaseAggregations( demographic__ethnicity=schema.Aggregations( From fe154ecc079e7a498c8c0c53539d53eba77417a3 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Mon, 18 Mar 2019 06:43:10 +0000 Subject: [PATCH 04/34] Adding fields from HPFS data and some ifs to test nulls --- database.py | 91 +++++++++++++++++++++++------------------------------ schema.py | 32 +++++++++++++++++++ 2 files changed, 71 insertions(+), 52 deletions(-) diff --git a/database.py b/database.py index 6411d80..ac780c6 100644 --- a/database.py +++ b/database.py @@ -9,7 +9,7 @@ def add_key_increment(dictionary, key): dictionary[key]+=1 class Data(object): - db_conn = mariadb.connect(user='biom_mass', password='DanDa1osh', db='portal_ui') + db_conn = mariadb.connect(user='biom_mass', password='', db='portal_ui') cursor = db_conn.cursor(buffered=True) def fetch_results(self,query): @@ -23,8 +23,6 @@ def fetch_results(self,query): def get_current_projects(self): projects_json=self.fetch_results("select `id` from `project`") - print("json_data inside get_current_projects") - print(projects_json) return [self.get_project(project['id']) for project in projects_json] def get_user(self): @@ -34,14 +32,13 @@ def get_user(self): def get_project(self,id): import schema - print("id inside get_project") + print("inside get_project") print(id) project_json=self.fetch_results( "select * from `project` where `id` =" + str(id)) project_id=project_json[0]['project_id'] proj_id=project_json[0]['id'] proj_name=project_json[0]['name'] - proj_program=["HPFS"] proj_program=project_json[0]['program'] #proj_primary_site=project_json[0]['primary_site'] proj_primary_site=["Stool"] @@ -56,9 +53,6 @@ def get_project(self,id): sum_file_size_json=self.fetch_results( "select sum(file_size) as sum_file_size from file_sample where project='"+project_id+"'") proj_file_size=sum_file_size_json[0]['sum_file_size'] - print("values inside get_project") - print( - proj_id,project_id,proj_name,proj_program,proj_part_count,proj_file_count,proj_file_size,proj_primary_site) return schema.Project( id=proj_id, @@ -78,7 +72,7 @@ def get_data_categories(self,table,id): import schema print("inside get_data_cat") - print(id) + print(table,id) data_cat_json=self.fetch_results( "select distinct data_category from file_sample where "+ table+"='" + str(id)+"'") @@ -86,12 +80,22 @@ def get_data_categories(self,table,id): for item in data_cat_json: item_files_json=self.fetch_results( "select count(id) as file_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") + item_part_json=self.fetch_results( "select count(distinct participant) as case_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") + + if len(item_part_json) > 0: + case_count=item_part_json[0]['case_count'] + else: + case_count=0 + if len(item_files_json) > 0: + file_count=item_files_json[0]['file_count'] + else: + file_count=0 data_cat.append( schema.DataCategories( - case_count=item_part_json[0]['case_count'], - file_count=item_files_json[0]['file_count'], + case_count=case_count, + file_count=file_count, data_category=item['data_category'])) return data_cat @@ -127,13 +131,11 @@ def get_file(self,id): "select id , project_id, primary_site from project where project_id='"+file_json[0]['project']+"'") part_json=self.fetch_results( "select id, entity_participant_id from participant where entity_participant_id='"+file_json[0]['participant']+"'") - print("file and project json inside get_file") - print(file_json[0]['sample']) + print("inside get_file") print(project_json[0]['project_id']) return schema.File( id=file_json[0]['id'], name=file_json[0]['file_name'], - file_id=file_json[0]['file_id'], participant=file_json[0]['participant'], sample=file_json[0]['sample'], access=file_json[0]['access'], @@ -150,7 +152,8 @@ def get_file(self,id): project=self.get_project(project_json[0]['id']), demographic=schema.Demographic("not hispanic or latino","male","white"), #primary_site=project_json[0]['primary_site'] - primary_site="Stool")])) + primary_site="Stool")]), + file_id=file_json[0]['file_id']) def get_case_annotation(self): import schema @@ -163,8 +166,6 @@ def get_current_files(self): file_hits=[] for item in files_json: file_hits.append(item['id']) - print("file_hits inside current files") - print(file_hits) return schema.Files(hits=file_hits) def get_current_cases(self): @@ -193,24 +194,14 @@ def get_case(self,id): print("inside get case") print(part_id) - proj_json=self.fetch_results('''select distinct file_sample.project, project.id as proj_id - from file_sample, project where file_sample.participant='''+str(part_id)+''' - and project.project_id=file_sample.project''') + proj_query="select distinct file_sample.project, project.id as proj_id " + proj_query=proj_query+"from file_sample, project where file_sample.participant='" + proj_query=proj_query+str(part_id)+"' and project.project_id=file_sample.project limit 1" + print(proj_query) + proj_json=self.fetch_results(proj_query) proj_id=proj_json[0]['proj_id'] print(proj_id) - cat_raw_json=self.fetch_results( - "select count(id) as raw_count from file_sample where participant="+str(part_id)+" and data_category='Raw Reads'") - cat_raw_count=cat_raw_json[0]['raw_count'] - - cat_taxa_json=self.fetch_results( - "select count(id) as taxa_count from file_sample where participant="+str(part_id)+" and data_category='Taxonomic Profiles'") - cat_taxa_count=cat_taxa_json[0]['taxa_count'] - - cat_gene_json=self.fetch_results( - "select count(id) as gene_count from file_sample where participant="+str(part_id)+" and data_category='Gene Families'") - cat_gene_count=cat_gene_json[0]['gene_count'] - files_json=self.fetch_results("select * from file_sample where participant="+str(part_id)) return schema.Case(id, @@ -222,10 +213,9 @@ def get_case(self,id): case_count=case_count, file_count=file_count, file_size=file_size, - data_categories=[schema.DataCategories(file_count=cat_raw_count, data_category="Raw Reads"), - schema.DataCategories(file_count=cat_gene_count, data_category="Gene Families"), - schema.DataCategories(file_count=cat_taxa_count, data_category="Taxonomic Profiles")]), - files=schema.CaseFiles(hits=[schema.CaseFile( + data_categories=self.get_data_categories("participant",part_id)), + # experimental_strategies=self.get_experimental_strategies("participant",part_id)), + files=schema.CaseFiles(hits=[schema.CaseFile( case_file['id'], experimental_strategy=case_file['experimental_strategy'], data_category=case_file['data_category'], @@ -245,9 +235,6 @@ def get_project_aggregations(self, projects): print("projects inside aggregations") for project in projects: - print( - project.id, project.project_id,project.primary_site, - project.program) add_key_increment(aggregates["primary_site"],project.primary_site[0]) add_key_increment(aggregates["project_id"], project.project_id) add_key_increment(aggregates["program__name"], project.program.name) @@ -310,18 +297,18 @@ def get_file_aggregations(self, files): aggregates = {"data_category": {}, "experimental_strategy": {}, "data_format": {}, "platform": {}, "cases__primary_site": {}, "cases__project__project_id": {}, "access": {}} + if len(files) > 0: + for file in files: + + add_key_increment(aggregates["data_category"], file.data_category) + add_key_increment(aggregates["experimental_strategy"], file.experimental_strategy) + add_key_increment(aggregates["data_format"], file.data_format) + add_key_increment(aggregates["platform"], file.platform) + add_key_increment(aggregates["access"], file.access) - for file in files: - print(" values inside file aggregation") - print(file.data_category,file.experimental_strategy,file.data_format,file.platform,file.access) - add_key_increment(aggregates["data_category"], file.data_category) - add_key_increment(aggregates["experimental_strategy"], file.experimental_strategy) - add_key_increment(aggregates["data_format"], file.data_format) - add_key_increment(aggregates["platform"], file.platform) - add_key_increment(aggregates["access"], file.access) - project = file.cases.hits[0].project - add_key_increment(aggregates["cases__primary_site"], project.primary_site[0]) - add_key_increment(aggregates["cases__project__project_id"], project.project_id) + project = file.cases.hits[0].project + add_key_increment(aggregates["cases__primary_site"], project.primary_site[0]) + add_key_increment(aggregates["cases__project__project_id"], project.project_id) file_aggregates = schema.FileAggregations( data_category=schema.Aggregations( @@ -339,7 +326,7 @@ def get_file_aggregations(self, files): cases__project__project_id=schema.Aggregations( buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["cases__project__project_id"].items()])) - return file_aggregates + return file_aggregates def get_case_aggregations(self, cases): import schema @@ -355,7 +342,7 @@ def get_case_aggregations(self, cases): add_key_increment(aggregates["demographic__race"], case.demographic.race) add_key_increment(aggregates["primary_site"], case.primary_site[0]) add_key_increment(aggregates["project__project_id"], case.project.project_id) - add_key_increment(aggregates["project__program__name"], case.project.program[0]) + add_key_increment(aggregates["project__program__name"], case.project.program.name) case_aggregates=schema.CaseAggregations( demographic__ethnicity=schema.Aggregations( diff --git a/schema.py b/schema.py index 13b23d4..ed76e93 100644 --- a/schema.py +++ b/schema.py @@ -91,6 +91,38 @@ class Demographic(graphene.ObjectType): gender = graphene.String() race = graphene.String() +class Metadata(graphene.ObjectType): + age2012 = graphene.Int() + totMETs1 = graphene.String() + weightLbs = graphene.String() + DaysSince1Jan12 = graphene.Int() + drAlcohol = graphene.Float() + drB12 = graphene.Float() + drCalories = graphene.Float() + drCarbs = graphene.Float() + drCholine = graphene.Float() + drFat = graphene.Float() + drFiber = graphene.Float() + drFolate = graphene.Float() + drIron = graphene.Float() + drProtein = graphene.Float() + participant = graphene.Int() + q2Alcohol = graphene.String() + q2B12 = graphene.String() + q2Calories = graphene.String() + q2Carbs = graphene.String() + q2Choline = graphene.String() + q2Fat = graphene.String() + q2Fiber = graphene.String() + q2Folate = graphene.String() + q2Iron = graphene.String() + q2Protein = graphene.String() + Time = graphene.String() + week = graphene.Int() + non_ribosomal_proteins = graphene.Float() + ribosomal_Proteins = graphene.Float() + + class FileCase(graphene.ObjectType): class Meta: interfaces = (graphene.relay.Node,) From 9a528bb6b1107186af9e7a83e8c831e573131bf4 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Tue, 19 Mar 2019 16:46:28 +0000 Subject: [PATCH 05/34] Adding schema changes --- schema.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/schema.py b/schema.py index 5aaa2fd..03ea5a5 100644 --- a/schema.py +++ b/schema.py @@ -91,7 +91,6 @@ class Demographic(graphene.ObjectType): gender = graphene.String() race = graphene.String() -<<<<<<< HEAD class Metadata(graphene.ObjectType): age2012 = graphene.Int() totMETs1 = graphene.String() @@ -123,9 +122,6 @@ class Metadata(graphene.ObjectType): non_ribosomal_proteins = graphene.Float() ribosomal_Proteins = graphene.Float() - -======= ->>>>>>> upstream/master class FileCase(graphene.ObjectType): class Meta: interfaces = (graphene.relay.Node,) From e232d1f84b7f1d53b0fc9bca8014a2bf3e0a88bb Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Tue, 19 Mar 2019 18:00:13 +0000 Subject: [PATCH 06/34] Reverting to clean version without ifs --- database.py | 80 ++++++++++++++++++++--------------------------------- schema.py | 0 2 files changed, 30 insertions(+), 50 deletions(-) mode change 100644 => 100755 database.py mode change 100644 => 100755 schema.py diff --git a/database.py b/database.py old mode 100644 new mode 100755 index 908641f..157151c --- a/database.py +++ b/database.py @@ -9,7 +9,7 @@ def add_key_increment(dictionary, key): dictionary[key]+=1 class Data(object): - db_conn = mariadb.connect(user='biom_mass', password='', db='portal_ui') + db_conn = mariadb.connect(user='biom_mass', password='DanDa1osh', db='portal_ui') cursor = db_conn.cursor(buffered=True) def fetch_results(self,query): @@ -78,31 +78,25 @@ def get_data_categories(self,table,id): "select distinct data_category from file_sample where "+ table+"='" + str(id)+"'") data_cat=[] for item in data_cat_json: - - if table is "participant": - item_files_json=self.fetch_results( - "select count(id) as file_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") + item_files_json=self.fetch_results( + "select count(id) as file_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") + + item_part_json=self.fetch_results( + "select count(distinct participant) as case_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") + + if len(item_part_json) > 0: + case_count=item_part_json[0]['case_count'] + else: + case_count=0 + if len(item_files_json) > 0: + file_count=item_files_json[0]['file_count'] + else: file_count=0 - if len(item_files_json) > 0: - file_count=item_files_json[0]['file_count'] - - data_cat.append( - schema.DataCategories( - file_count=file_count, - data_category=item['data_category'])) - - elif table is "project": - item_part_json=self.fetch_results( - "select count(distinct participant) as case_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") - - case_count=0 - if len(item_part_json) > 0: - case_count=item_part_json[0]['case_count'] - - data_cat.append( - schema.DataCategories( - case_count=case_count, - data_category=item['data_category'])) + data_cat.append( + schema.DataCategories( + case_count=case_count, + file_count=file_count, + data_category=item['data_category'])) return data_cat @@ -115,31 +109,17 @@ def get_experimental_strategies(self,table,id): print(table,id) exp_str_json=self.fetch_results( "select distinct experimental_strategy from file_sample where "+ table+"='" + str(id) +"'") - exp_str=[] for item in exp_str_json: item_files_json=self.fetch_results( - "select count(id) as filecount from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") - print("select count(id) as filecount from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") - -# item_part_json=self.fetch_results( -# "select count(distinct participant) as case_count from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") -# -# case_count=0 -# if len(item_part_json) > 0: -# case_count=item_part_json[0]['case_count'] - print(len(item_files_json)) - file_count=0 - if len(item_files_json)> 0: - file_count=item_files_json[0]['filecount'] - print( item) - print(file_count) + "select count(id) as file_count from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") + item_part_json=self.fetch_results( + "select count(distinct participant) as case_count from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") exp_str.append( schema.ExperimentalStrategies( -# case_count=case_count, - file_count=file_count, - experimental_strategy="wmgx")) -# experimental_strategy=item['experimental_strategy'])) + case_count=item_part_json[0]['case_count'], + file_count=item_files_json[0]['file_count'], + experimental_strategy=item['experimental_strategy'])) return exp_str @@ -325,11 +305,11 @@ def get_file_aggregations(self, files): add_key_increment(aggregates["data_format"], file.data_format) add_key_increment(aggregates["platform"], file.platform) add_key_increment(aggregates["access"], file.access) - if len(file.cases.hits) > 0: - project = file.cases.hits[0].project - add_key_increment(aggregates["cases__primary_site"], project.primary_site[0]) - add_key_increment(aggregates["cases__project__project_id"], project.project_id) - + + project = file.cases.hits[0].project + add_key_increment(aggregates["cases__primary_site"], project.primary_site[0]) + add_key_increment(aggregates["cases__project__project_id"], project.project_id) + file_aggregates = schema.FileAggregations( data_category=schema.Aggregations( buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["data_category"].items()]), diff --git a/schema.py b/schema.py old mode 100644 new mode 100755 From b63f67decd83b7898a536f0935ab31c36ebe9062 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Tue, 19 Mar 2019 18:03:05 +0000 Subject: [PATCH 07/34] Typo fix --- database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database.py b/database.py index 157151c..ac780c6 100755 --- a/database.py +++ b/database.py @@ -9,7 +9,7 @@ def add_key_increment(dictionary, key): dictionary[key]+=1 class Data(object): - db_conn = mariadb.connect(user='biom_mass', password='DanDa1osh', db='portal_ui') + db_conn = mariadb.connect(user='biom_mass', password='', db='portal_ui') cursor = db_conn.cursor(buffered=True) def fetch_results(self,query): From d7dc041401008bb7b8fb093fdaf08d4cf1851a2b Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Wed, 20 Mar 2019 23:16:48 +0000 Subject: [PATCH 08/34] open and close db connection/cursor --- database.py | 135 ++++++++++++++++++++++++---------------------------- 1 file changed, 63 insertions(+), 72 deletions(-) diff --git a/database.py b/database.py index ac780c6..cd1b12e 100755 --- a/database.py +++ b/database.py @@ -2,6 +2,7 @@ import mysql.connector as mariadb import json +import schema def add_key_increment(dictionary, key): if not key in dictionary: @@ -9,16 +10,21 @@ def add_key_increment(dictionary, key): dictionary[key]+=1 class Data(object): - db_conn = mariadb.connect(user='biom_mass', password='', db='portal_ui') - cursor = db_conn.cursor(buffered=True) def fetch_results(self,query): - self.cursor.execute(query) - row_headers=[x[0] for x in self.cursor.description] - rows = self.cursor.fetchall() + + db_conn = mariadb.connect(user='biom_mass', password='', db='portal_ui') + cursor = db_conn.cursor(buffered=True) + cursor.execute(query) + + row_headers=[x[0] for x in cursor.description] + rows = cursor.fetchall() json_data=[] for row in rows: json_data.append(dict(zip(row_headers,row))) + cursor.close() + db_conn.close() + return json_data def get_current_projects(self): @@ -26,14 +32,10 @@ def get_current_projects(self): return [self.get_project(project['id']) for project in projects_json] def get_user(self): - import schema return schema.User(username="null") def get_project(self,id): - import schema - print("inside get_project") - print(id) project_json=self.fetch_results( "select * from `project` where `id` =" + str(id)) project_id=project_json[0]['project_id'] @@ -43,7 +45,8 @@ def get_project(self,id): #proj_primary_site=project_json[0]['primary_site'] proj_primary_site=["Stool"] - file_count_json=self.fetch_results("select count(id) as file_count from file_sample where project='"+project_id+"'") + file_count_json=self.fetch_results( + "select count(id) as file_count from file_sample where project='"+project_id+"'") proj_file_count=file_count_json[0]['file_count'] part_count_json=self.fetch_results( @@ -69,70 +72,75 @@ def get_project(self,id): def get_data_categories(self,table,id): - import schema - - print("inside get_data_cat") - print(table,id) data_cat_json=self.fetch_results( "select distinct data_category from file_sample where "+ table+"='" + str(id)+"'") data_cat=[] for item in data_cat_json: - item_files_json=self.fetch_results( - "select count(id) as file_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") - item_part_json=self.fetch_results( - "select count(distinct participant) as case_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") + if table is "participant": + item_files_json=self.fetch_results( + "select count(id) as file_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") + file_count=0 + if len(item_files_json) > 0: + file_count=item_files_json[0]['file_count'] + + data_cat.append( + schema.DataCategories( + file_count=file_count, + data_category=item['data_category'])) + elif table is "project": + item_part_json=self.fetch_results( + "select count(distinct participant) as case_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") - if len(item_part_json) > 0: - case_count=item_part_json[0]['case_count'] - else: case_count=0 - if len(item_files_json) > 0: - file_count=item_files_json[0]['file_count'] - else: - file_count=0 - data_cat.append( - schema.DataCategories( - case_count=case_count, - file_count=file_count, - data_category=item['data_category'])) + if len(item_part_json) > 0: + case_count=item_part_json[0]['case_count'] + data_cat.append( + schema.DataCategories( + case_count=case_count, + data_category=item['data_category'])) return data_cat def get_experimental_strategies(self,table,id): - import schema - print("inside get exp_str") - print(table,id) exp_str_json=self.fetch_results( "select distinct experimental_strategy from file_sample where "+ table+"='" + str(id) +"'") + exp_str=[] for item in exp_str_json: item_files_json=self.fetch_results( - "select count(id) as file_count from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") - item_part_json=self.fetch_results( - "select count(distinct participant) as case_count from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") + "select count(id) as filecount from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") +# item_part_json=self.fetch_results( +# "select count(distinct participant) as case_count from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") +# +# case_count=0 +# if len(item_part_json) > 0: +# case_count=item_part_json[0]['case_count'] + + file_count=0 + if len(item_files_json) > 0: + file_count=item_files_json[0]['filecount'] exp_str.append( schema.ExperimentalStrategies( - case_count=item_part_json[0]['case_count'], - file_count=item_files_json[0]['file_count'], - experimental_strategy=item['experimental_strategy'])) +# case_count=case_count, + file_count=file_count, + experimental_strategy="wmgx")) +# experimental_strategy=item['experimental_strategy'])) return exp_str def get_file(self,id): - import schema file_json=self.fetch_results("select * from `file_sample` where `id` ="+ str(id)) project_json=self.fetch_results( "select id , project_id, primary_site from project where project_id='"+file_json[0]['project']+"'") part_json=self.fetch_results( "select id, entity_participant_id from participant where entity_participant_id='"+file_json[0]['participant']+"'") - print("inside get_file") - print(project_json[0]['project_id']) + return schema.File( id=file_json[0]['id'], name=file_json[0]['file_name'], @@ -156,26 +164,19 @@ def get_file(self,id): file_id=file_json[0]['file_id']) def get_case_annotation(self): - import schema return schema.CaseAnnotation() def get_current_files(self): - import schema files_json=self.fetch_results("select `id` from `file_sample`") - file_hits=[] - for item in files_json: - file_hits.append(item['id']) - return schema.Files(hits=file_hits) + return schema.Files(hits=[item['id'] for item in files_json]) def get_current_cases(self): - import schema cases_json=self.fetch_results("select id from participant") return schema.RepositoryCases(hits=[case['id'] for case in cases_json]) def get_case(self,id): - import schema case_json=self.fetch_results("select * from participant where id="+str(id)) part_id=case_json[0]['entity_participant_id'] @@ -192,15 +193,11 @@ def get_case(self,id): "select sum(file_size) as filesize from file_sample where participant="+str(part_id)) file_size=file_size_json[0]['filesize'] - print("inside get case") - print(part_id) proj_query="select distinct file_sample.project, project.id as proj_id " proj_query=proj_query+"from file_sample, project where file_sample.participant='" proj_query=proj_query+str(part_id)+"' and project.project_id=file_sample.project limit 1" - print(proj_query) proj_json=self.fetch_results(proj_query) proj_id=proj_json[0]['proj_id'] - print(proj_id) files_json=self.fetch_results("select * from file_sample where participant="+str(part_id)) @@ -225,14 +222,12 @@ def get_case(self,id): def get_project_aggregations(self, projects): - import schema # compile aggregations from project aggregates = {"primary_site": {}, "program__name": {}, "project_id": {}, "summary__data_categories__data_category": {}, "summary__experimental_strategies__experimental_strategy": {}} - print("projects inside aggregations") for project in projects: add_key_increment(aggregates["primary_site"],project.primary_site[0]) @@ -259,11 +254,10 @@ def get_project_aggregations(self, projects): def get_current_counts(self): - import schema projects_json = self.fetch_results("select count(`project_id`)as projects_count from project") projects = projects_json[0]['projects_count'] - + participants_json = self.fetch_results("select count(`entity_participant_id`) as participants_count from `participant`") participants = participants_json[0]['participants_count'] @@ -291,24 +285,23 @@ def get_current_counts(self): def get_file_aggregations(self, files): - import schema # aggregate file data aggregates = {"data_category": {}, "experimental_strategy": {}, "data_format": {}, "platform": {}, "cases__primary_site": {}, "cases__project__project_id": {}, "access": {}} - if len(files) > 0: - for file in files: - add_key_increment(aggregates["data_category"], file.data_category) - add_key_increment(aggregates["experimental_strategy"], file.experimental_strategy) - add_key_increment(aggregates["data_format"], file.data_format) - add_key_increment(aggregates["platform"], file.platform) - add_key_increment(aggregates["access"], file.access) + for file in files: + + add_key_increment(aggregates["data_category"], file.data_category) + add_key_increment(aggregates["experimental_strategy"], file.experimental_strategy) + add_key_increment(aggregates["data_format"], file.data_format) + add_key_increment(aggregates["platform"], file.platform) + add_key_increment(aggregates["access"], file.access) - project = file.cases.hits[0].project - add_key_increment(aggregates["cases__primary_site"], project.primary_site[0]) - add_key_increment(aggregates["cases__project__project_id"], project.project_id) + project = file.cases.hits[0].project + add_key_increment(aggregates["cases__primary_site"], project.primary_site[0]) + add_key_increment(aggregates["cases__project__project_id"], project.project_id) file_aggregates = schema.FileAggregations( data_category=schema.Aggregations( @@ -326,10 +319,9 @@ def get_file_aggregations(self, files): cases__project__project_id=schema.Aggregations( buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["cases__project__project_id"].items()])) - return file_aggregates + return file_aggregates def get_case_aggregations(self, cases): - import schema # aggregate case data aggregates = {"demographic__ethnicity": {}, "demographic__gender": {}, @@ -364,7 +356,6 @@ def get_facets(self): return "null" # this is not currently being used def get_cart_file_size(self): - import schema file_json=self.fetch_results("select sum(file_size) as sum_size from file_sample") return schema.FileSize(file_json[0]['sum_size']) From 9341bd20c56ab079112d29d6e6d609d009454b80 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Thu, 21 Mar 2019 02:41:33 +0000 Subject: [PATCH 09/34] Adding HPFS fields to schema --- database.py | 64 +++++++++++++++++++++++++++++++++++++++++++---------- schema.py | 7 ++++-- 2 files changed, 57 insertions(+), 14 deletions(-) diff --git a/database.py b/database.py index cd1b12e..b7a37fd 100755 --- a/database.py +++ b/database.py @@ -4,6 +4,8 @@ import json import schema + +# increments key for aggregations def add_key_increment(dictionary, key): if not key in dictionary: dictionary[key]=0 @@ -11,12 +13,13 @@ def add_key_increment(dictionary, key): class Data(object): + # connects to db and runs query def fetch_results(self,query): - db_conn = mariadb.connect(user='biom_mass', password='', db='portal_ui') + db_conn = mariadb.connect(user='biom_mass', password='DanDa1osh', db='portal_ui') cursor = db_conn.cursor(buffered=True) cursor.execute(query) - + # response json row_headers=[x[0] for x in cursor.description] rows = cursor.fetchall() json_data=[] @@ -27,13 +30,16 @@ def fetch_results(self,query): return json_data + # get all projects from db def get_current_projects(self): projects_json=self.fetch_results("select `id` from `project`") return [self.get_project(project['id']) for project in projects_json] + # get user (currently none) def get_user(self): return schema.User(username="null") + # get project object details from db def get_project(self,id): project_json=self.fetch_results( @@ -42,8 +48,8 @@ def get_project(self,id): proj_id=project_json[0]['id'] proj_name=project_json[0]['name'] proj_program=project_json[0]['program'] - #proj_primary_site=project_json[0]['primary_site'] - proj_primary_site=["Stool"] + proj_primary_site=[project_json[0]['primary_site']] + #proj_primary_site=["Stool"] file_count_json=self.fetch_results( "select count(id) as file_count from file_sample where project='"+project_id+"'") @@ -71,6 +77,7 @@ def get_project(self,id): primary_site=proj_primary_site) + # get data categories file and case counts from db for a project or a participant def get_data_categories(self,table,id): data_cat_json=self.fetch_results( @@ -104,7 +111,7 @@ def get_data_categories(self,table,id): return data_cat - + # get experimental strategies file count from db for a project def get_experimental_strategies(self,table,id): exp_str_json=self.fetch_results( @@ -128,11 +135,13 @@ def get_experimental_strategies(self,table,id): schema.ExperimentalStrategies( # case_count=case_count, file_count=file_count, - experimental_strategy="wmgx")) -# experimental_strategy=item['experimental_strategy'])) + # experimental_strategy="wmgx")) + experimental_strategy=item['experimental_strategy'])) return exp_str + + # get details of file object from db def get_file(self,id): file_json=self.fetch_results("select * from `file_sample` where `id` ="+ str(id)) @@ -159,23 +168,27 @@ def get_file(self,id): case_id=file_json[0]['participant'], project=self.get_project(project_json[0]['id']), demographic=schema.Demographic("not hispanic or latino","male","white"), - #primary_site=project_json[0]['primary_site'] - primary_site="Stool")]), + primary_site=project_json[0]['primary_site'])]), + #primary_site="Stool")]), file_id=file_json[0]['file_id']) + # get annotations (currently not in use) def get_case_annotation(self): return schema.CaseAnnotation() + # get ids of all files from db def get_current_files(self): files_json=self.fetch_results("select `id` from `file_sample`") return schema.Files(hits=[item['id'] for item in files_json]) + # get ids of all cases from db def get_current_cases(self): cases_json=self.fetch_results("select id from participant") return schema.RepositoryCases(hits=[case['id'] for case in cases_json]) + # get details of case object from db def get_case(self,id): case_json=self.fetch_results("select * from participant where id="+str(id)) @@ -194,24 +207,44 @@ def get_case(self,id): file_size=file_size_json[0]['filesize'] proj_query="select distinct file_sample.project, project.id as proj_id " - proj_query=proj_query+"from file_sample, project where file_sample.participant='" + proj_query=proj_query+"from file_sample, project where file_sample.participant='" proj_query=proj_query+str(part_id)+"' and project.project_id=file_sample.project limit 1" proj_json=self.fetch_results(proj_query) proj_id=proj_json[0]['proj_id'] + # query to get files from file_sample table files_json=self.fetch_results("select * from file_sample where participant="+str(part_id)) + # query to get metadata from prticipant table + metadata_part_json=self.fetch_results( + "select * from participant where entity_participant_id="+str(part_id)) + metadata_part_values=[] + for key, value in metadata_part_json[0].items(): + metadata_part_values.append(value) + metadata_participant=schema.MetadataParticipant(metadata_part_values) + + # query to get metadata from sample table + metadata_json=self.fetch_results("select * from sample where participant="+str(part_id)) + metadata_list=[] + for metadata in metadata_json: + metadata_values=[] + for key, value in metadata.items(): + metadata_values.append(value) + metadata_list.append(schema.MetadataSample(metadata_values)) + return schema.Case(id, case_id=part_id, primary_site="Stool", demographic=schema.Demographic("not hispanic or latino","male","white"), + metadata_participant=metadata_participant, + metadata_sample=metadata_list, project=self.get_project(proj_id), summary=schema.Summary( case_count=case_count, file_count=file_count, file_size=file_size, data_categories=self.get_data_categories("participant",part_id)), - # experimental_strategies=self.get_experimental_strategies("participant",part_id)), + #experimental_strategies=self.get_experimental_strategies("participant",part_id)), files=schema.CaseFiles(hits=[schema.CaseFile( case_file['id'], experimental_strategy=case_file['experimental_strategy'], @@ -221,6 +254,7 @@ def get_case(self,id): access=case_file['access']) for case_file in files_json])) + def get_project_aggregations(self, projects): # compile aggregations from project @@ -252,7 +286,7 @@ def get_project_aggregations(self, projects): return project_aggregates - + # get all counts for front page summary from db def get_current_counts(self): projects_json = self.fetch_results("select count(`project_id`)as projects_count from project") @@ -284,6 +318,7 @@ def get_current_counts(self): processedFiles=prFiles) + def get_file_aggregations(self, files): # aggregate file data @@ -321,6 +356,8 @@ def get_file_aggregations(self, files): return file_aggregates + + def get_case_aggregations(self, cases): # aggregate case data @@ -352,9 +389,12 @@ def get_case_aggregations(self, cases): return case_aggregates + # get facet (currently not in use) def get_facets(self): return "null" # this is not currently being used + + # get total size of all files def get_cart_file_size(self): file_json=self.fetch_results("select sum(file_size) as sum_size from file_sample") diff --git a/schema.py b/schema.py index 03ea5a5..b8c6ee9 100755 --- a/schema.py +++ b/schema.py @@ -91,10 +91,12 @@ class Demographic(graphene.ObjectType): gender = graphene.String() race = graphene.String() -class Metadata(graphene.ObjectType): +class MetadataParticipant(graphene.ObjectType): age2012 = graphene.Int() totMETs1 = graphene.String() weightLbs = graphene.String() + +class MetadataSample(graphene.ObjectType): DaysSince1Jan12 = graphene.Int() drAlcohol = graphene.Float() drB12 = graphene.Float() @@ -311,8 +313,9 @@ class Meta: case_id = graphene.String() primary_site = graphene.String() submitter_id = graphene.String() - demographic = graphene.Field(Demographic) + metadata_participant = graphene.Field(MetadataParticipant) + metadata_sample = graphene.List(MetadataSample) project = graphene.Field(Project) summary = graphene.Field(Summary) annotations = graphene.Field(CaseAnnotations) From 1cc4318805c00e24b22d600c46bccba7344f0503 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Thu, 21 Mar 2019 03:35:35 +0000 Subject: [PATCH 10/34] Typo fix --- database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database.py b/database.py index b7a37fd..1440383 100755 --- a/database.py +++ b/database.py @@ -16,7 +16,7 @@ class Data(object): # connects to db and runs query def fetch_results(self,query): - db_conn = mariadb.connect(user='biom_mass', password='DanDa1osh', db='portal_ui') + db_conn = mariadb.connect(user='biom_mass', password='', db='portal_ui') cursor = db_conn.cursor(buffered=True) cursor.execute(query) # response json From 00de0471da20cfded2a6f4bbc66eb8a22c159779 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Thu, 21 Mar 2019 18:10:22 +0000 Subject: [PATCH 11/34] Adding couple missing fields to Metadata --- database.py | 19 ++++++++++--------- schema.py | 12 ++++++++---- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/database.py b/database.py index 1440383..f9f483c 100755 --- a/database.py +++ b/database.py @@ -169,7 +169,6 @@ def get_file(self,id): project=self.get_project(project_json[0]['id']), demographic=schema.Demographic("not hispanic or latino","male","white"), primary_site=project_json[0]['primary_site'])]), - #primary_site="Stool")]), file_id=file_json[0]['file_id']) # get annotations (currently not in use) @@ -218,19 +217,20 @@ def get_case(self,id): # query to get metadata from prticipant table metadata_part_json=self.fetch_results( "select * from participant where entity_participant_id="+str(part_id)) - metadata_part_values=[] - for key, value in metadata_part_json[0].items(): - metadata_part_values.append(value) - metadata_participant=schema.MetadataParticipant(metadata_part_values) + + del metadata_part_json[0]['updated'] + metadata_part_json[0]['participant'] = metadata_part_json[0]['entity_participant_id'] + del metadata_part_json[0]['entity_participant_id'] + metadata_participant=schema.MetadataParticipant(**metadata_part_json[0]) + # query to get metadata from sample table metadata_json=self.fetch_results("select * from sample where participant="+str(part_id)) metadata_list=[] for metadata in metadata_json: - metadata_values=[] - for key, value in metadata.items(): - metadata_values.append(value) - metadata_list.append(schema.MetadataSample(metadata_values)) + del metadata['updated'] + metadata_list.append(schema.MetadataSample(**metadata)) + return schema.Case(id, case_id=part_id, @@ -373,6 +373,7 @@ def get_case_aggregations(self, cases): add_key_increment(aggregates["project__project_id"], case.project.project_id) add_key_increment(aggregates["project__program__name"], case.project.program.name) + case_aggregates=schema.CaseAggregations( demographic__ethnicity=schema.Aggregations( buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["demographic__ethnicity"].items()]), diff --git a/schema.py b/schema.py index b8c6ee9..6ad1c68 100755 --- a/schema.py +++ b/schema.py @@ -92,11 +92,17 @@ class Demographic(graphene.ObjectType): race = graphene.String() class MetadataParticipant(graphene.ObjectType): - age2012 = graphene.Int() + id = graphene.Int() + participant = graphene.Int() + age_2012 = graphene.Int() totMETs1 = graphene.String() - weightLbs = graphene.String() + weight_lbs = graphene.String() class MetadataSample(graphene.ObjectType): + id = graphene.Int() + project = graphene.String() + sample = graphene.String() + participant = graphene.Int() DaysSince1Jan12 = graphene.Int() drAlcohol = graphene.Float() drB12 = graphene.Float() @@ -121,8 +127,6 @@ class MetadataSample(graphene.ObjectType): q2Protein = graphene.String() Time = graphene.String() week = graphene.Int() - non_ribosomal_proteins = graphene.Float() - ribosomal_Proteins = graphene.Float() class FileCase(graphene.ObjectType): class Meta: From 72fc05475672c16f64cb832b0565adf31bac9561 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Fri, 22 Mar 2019 16:27:10 +0000 Subject: [PATCH 12/34] making sure schema changes are reflected in all files --- database.py | 13 ++++++++++++- schema.graphql | 43 +++++++++++++++++++++++++++++++++++++++++++ schema.py | 3 +++ 3 files changed, 58 insertions(+), 1 deletion(-) mode change 100644 => 100755 schema.graphql diff --git a/database.py b/database.py index f9f483c..505fce5 100755 --- a/database.py +++ b/database.py @@ -16,7 +16,7 @@ class Data(object): # connects to db and runs query def fetch_results(self,query): - db_conn = mariadb.connect(user='biom_mass', password='', db='portal_ui') + db_conn = mariadb.connect(user='biom_mass', password='DanDa1osh', db='portal_ui') cursor = db_conn.cursor(buffered=True) cursor.execute(query) # response json @@ -362,6 +362,8 @@ def get_case_aggregations(self, cases): # aggregate case data aggregates = {"demographic__ethnicity": {}, "demographic__gender": {}, + "metadata_participant__age_2012": {}, "metadata_participant__totMETs1": {}, + "metadata_participant__weight_lbs": {}, "demographic__race": {}, "primary_site": {}, "project__project_id": {}, "project__program__name": {}} @@ -369,6 +371,9 @@ def get_case_aggregations(self, cases): add_key_increment(aggregates["demographic__ethnicity"], case.demographic.ethnicity) add_key_increment(aggregates["demographic__gender"], case.demographic.gender) add_key_increment(aggregates["demographic__race"], case.demographic.race) + add_key_increment(aggregates["metadata_participant__age_2012"],case.metadata_participant.age_2012) + add_key_increment(aggregates["metadata_participant__totMETs1"],case.metadata_participant.totMETs1) + add_key_increment(aggregates["metadata_participant__weight_lbs"], case.metadata_participant.weight_lbs) add_key_increment(aggregates["primary_site"], case.primary_site[0]) add_key_increment(aggregates["project__project_id"], case.project.project_id) add_key_increment(aggregates["project__program__name"], case.project.program.name) @@ -381,6 +386,12 @@ def get_case_aggregations(self, cases): buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["demographic__gender"].items()]), demographic__race=schema.Aggregations( buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["demographic__race"].items()]), + metadata_participant__age_2012=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["metadata_participant__age_2012"].items()]), + metadata_participant__totMETs1=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["metadata_participant__totMETs1"].items()]), + metadata_participant__weight_lbs=schema.Aggregations( + buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["metadata_participant__weight_lbs"].items()]), primary_site=schema.Aggregations( buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["primary_site"].items()]), project__project_id=schema.Aggregations( diff --git a/schema.graphql b/schema.graphql old mode 100644 new mode 100755 index fe55875..1d39a28 --- a/schema.graphql +++ b/schema.graphql @@ -23,16 +23,59 @@ type Case implements Node { primary_site: String submitter_id: String demographic: Demographic + metadata_participant: MetadataParticipant + metadata_sample: [MetadataSample] project: Project summary: Summary annotations: CaseAnnotations files: CaseFiles } +type MetadataParticipant { + id: Int + participant: Int + age_2012: Int + totMETs1: String + weight_lbs: String +} + +type MetadataSample + id: Int + project: String + sample: String + participant: Int + DaysSince1Jan12: Int + drAlcohol: Float + drB12: Float + drCalories: Float + drCarbs: Float + drCholine: Float + drFat: Float + drFiber: Float + drFolate: Float + drIron: Float + drProtein: Float + participant: Int + q2Alcohol: String + q2B12: String + q2Calories: String + q2Carbs: String + q2Choline: String + q2Fat: String + q2Fiber: String + q2Folate: String + q2Iron: String + q2Protein: String + Time: String + week: Int + type CaseAggregations { demographic__ethnicity: Aggregations demographic__gender: Aggregations demographic__race: Aggregations + metadata_participant__age_2012: Aggregations + metadata_participant__totMETs1: Aggregations + metadata_participant__weight_lbs: Aggregations primary_site: Aggregations project__project_id: Aggregations project__program__name: Aggregations diff --git a/schema.py b/schema.py index 6ad1c68..b330cb7 100755 --- a/schema.py +++ b/schema.py @@ -251,6 +251,9 @@ class CaseAggregations(graphene.ObjectType): demographic__ethnicity = graphene.Field(Aggregations) demographic__gender = graphene.Field(Aggregations) demographic__race = graphene.Field(Aggregations) + metadata_participant__age_2012 = graphene.Field(Aggregations) + metadata_participant__totMETs1 = graphene.Field(Aggregations) + metadata_participant__weight_lbs = graphene.Field(Aggregations) primary_site = graphene.Field(Aggregations) project__project_id = graphene.Field(Aggregations) project__program__name = graphene.Field(Aggregations) From e13865e0327fe55ba3e3533871fbb658601f1eac Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Fri, 22 Mar 2019 19:19:37 +0000 Subject: [PATCH 13/34] Adding Version to schema for future use --- database.py | 8 ++++++++ schema.graphql | 8 ++++++++ schema.py | 7 +++++++ 3 files changed, 23 insertions(+) diff --git a/database.py b/database.py index 505fce5..4128d80 100755 --- a/database.py +++ b/database.py @@ -39,6 +39,14 @@ def get_current_projects(self): def get_user(self): return schema.User(username="null") + # get current version from db + def get_version(self): + version_json=self.fetch_results("select * from `version` order by updated desc limit 1 ") + del version_json[0]['updated'] + del version_json[0]['id'] + return schema.Version(**version_json[0]) + + # get project object details from db def get_project(self,id): diff --git a/schema.graphql b/schema.graphql index 1d39a28..c803ef5 100755 --- a/schema.graphql +++ b/schema.graphql @@ -31,6 +31,14 @@ type Case implements Node { files: CaseFiles } +type Version{ + data_release: String + commit: String + version: Float + tag: String + status: String +} + type MetadataParticipant { id: Int participant: Int diff --git a/schema.py b/schema.py index b330cb7..0900329 100755 --- a/schema.py +++ b/schema.py @@ -43,6 +43,13 @@ class Count(graphene.ObjectType): class User(graphene.ObjectType): username = graphene.String() +class Version(graphene.ObjectType): + data_release = graphene.String() + commit = graphene.String() + version = graphene.Float() + tag = graphene.String() + status =graphene.String() + class Program(graphene.ObjectType): name = graphene.String() program_id = graphene.String() From 9c96af7907ecc7560fc4f13ad0dd2334fa43443c Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Sat, 23 Mar 2019 21:58:39 +0000 Subject: [PATCH 14/34] Reducing number of queries --- database.py | 249 ++++++++++++++++++++++++---------------------------- schema.py | 4 + 2 files changed, 118 insertions(+), 135 deletions(-) diff --git a/database.py b/database.py index 4128d80..daf0674 100755 --- a/database.py +++ b/database.py @@ -1,7 +1,7 @@ # Calls to obtain values from database import mysql.connector as mariadb -import json +import os import schema @@ -13,104 +13,95 @@ def add_key_increment(dictionary, key): class Data(object): + + # connects to db and runs query def fetch_results(self,query): - db_conn = mariadb.connect(user='biom_mass', password='DanDa1osh', db='portal_ui') + db_conn = mariadb.connect(user='biom_mass', password=os.environ['BIOM_MASS'], db='portal_ui') cursor = db_conn.cursor(buffered=True) cursor.execute(query) # response json row_headers=[x[0] for x in cursor.description] rows = cursor.fetchall() - json_data=[] + data=[] for row in rows: - json_data.append(dict(zip(row_headers,row))) + data.append(dict(zip(row_headers,row))) cursor.close() db_conn.close() - return json_data + return data # get all projects from db def get_current_projects(self): - projects_json=self.fetch_results("select `id` from `project`") - return [self.get_project(project['id']) for project in projects_json] + projects_data=self.fetch_results("select `id` from `project`") + return [self.get_project(project['id']) for project in projects_data] # get user (currently none) def get_user(self): return schema.User(username="null") # get current version from db - def get_version(self): - version_json=self.fetch_results("select * from `version` order by updated desc limit 1 ") - del version_json[0]['updated'] - del version_json[0]['id'] - return schema.Version(**version_json[0]) + def get_current_version(self): + version_data=self.fetch_results("select * from `version` order by updated desc limit 1 ") + del version_data[0]['updated'] + del version_data[0]['id'] + return schema.Version(**version_data[0]) # get project object details from db def get_project(self,id): - project_json=self.fetch_results( - "select * from `project` where `id` =" + str(id)) - project_id=project_json[0]['project_id'] - proj_id=project_json[0]['id'] - proj_name=project_json[0]['name'] - proj_program=project_json[0]['program'] - proj_primary_site=[project_json[0]['primary_site']] - #proj_primary_site=["Stool"] - - file_count_json=self.fetch_results( - "select count(id) as file_count from file_sample where project='"+project_id+"'") - proj_file_count=file_count_json[0]['file_count'] - - part_count_json=self.fetch_results( - "select count(distinct participant) as part_count from file_sample where project='"+project_id+"'") - proj_part_count=part_count_json[0]['part_count'] + project_data=self.fetch_results("select * from `project` where `id` =" + str(id)) + project_id=project_data[0]['project_id'] - sum_file_size_json=self.fetch_results( - "select sum(file_size) as sum_file_size from file_sample where project='"+project_id+"'") - proj_file_size=sum_file_size_json[0]['sum_file_size'] + counts_query="select count(id) as total from file_sample where project='"+project_id+"'" + counts_query=counts_query+" union all " + counts_query=counts_query+"select count(distinct participant) as total from file_sample where project='"+project_id+"'" + counts_query=counts_query+" union all " + counts_query=counts_query+"select sum(file_size) as total from file_sample where project='"+project_id+"'" + proj_counts_data=self.fetch_results(counts_query) return schema.Project( - id=proj_id, + id=id, project_id=project_id, - name=proj_name, - program=schema.Program(name=proj_program), + name=project_data[0]['name'], + program=schema.Program(name=project_data[0]['program']), summary=schema.Summary( - case_count=proj_part_count, - file_count=proj_file_count, + case_count=proj_counts_data[1]['total'], + file_count=proj_counts_data[0]['total'], data_categories=self.get_data_categories("project",project_id), experimental_strategies=self.get_experimental_strategies("project",project_id), - file_size=proj_file_size), - primary_site=proj_primary_site) + file_size=proj_counts_data[2]['total']), + primary_site=[project_data[0]['primary_site']]) # get data categories file and case counts from db for a project or a participant def get_data_categories(self,table,id): - data_cat_json=self.fetch_results( + data_cat_data=self.fetch_results( "select distinct data_category from file_sample where "+ table+"='" + str(id)+"'") data_cat=[] - for item in data_cat_json: + for item in data_cat_data: if table is "participant": - item_files_json=self.fetch_results( + item_files_data=self.fetch_results( "select count(id) as file_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") file_count=0 - if len(item_files_json) > 0: - file_count=item_files_json[0]['file_count'] + if len(item_files_data) > 0: + file_count=item_files_data[0]['file_count'] data_cat.append( schema.DataCategories( file_count=file_count, data_category=item['data_category'])) elif table is "project": - item_part_json=self.fetch_results( + item_part_data=self.fetch_results( "select count(distinct participant) as case_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") case_count=0 - if len(item_part_json) > 0: - case_count=item_part_json[0]['case_count'] + if len(item_part_data) > 0: + case_count=item_part_data[0]['case_count'] data_cat.append( schema.DataCategories( case_count=case_count, @@ -122,28 +113,27 @@ def get_data_categories(self,table,id): # get experimental strategies file count from db for a project def get_experimental_strategies(self,table,id): - exp_str_json=self.fetch_results( + exp_str_data=self.fetch_results( "select distinct experimental_strategy from file_sample where "+ table+"='" + str(id) +"'") exp_str=[] - for item in exp_str_json: - item_files_json=self.fetch_results( + for item in exp_str_data: + item_files_data=self.fetch_results( "select count(id) as filecount from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") -# item_part_json=self.fetch_results( +# item_part_data=self.fetch_results( # "select count(distinct participant) as case_count from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") # # case_count=0 -# if len(item_part_json) > 0: -# case_count=item_part_json[0]['case_count'] +# if len(item_part_data) > 0: +# case_count=item_part_data[0]['case_count'] file_count=0 - if len(item_files_json) > 0: - file_count=item_files_json[0]['filecount'] + if len(item_files_data) > 0: + file_count=item_files_data[0]['filecount'] exp_str.append( schema.ExperimentalStrategies( # case_count=case_count, file_count=file_count, - # experimental_strategy="wmgx")) experimental_strategy=item['experimental_strategy'])) return exp_str @@ -152,32 +142,35 @@ def get_experimental_strategies(self,table,id): # get details of file object from db def get_file(self,id): - file_json=self.fetch_results("select * from `file_sample` where `id` ="+ str(id)) - project_json=self.fetch_results( - "select id , project_id, primary_site from project where project_id='"+file_json[0]['project']+"'") - part_json=self.fetch_results( - "select id, entity_participant_id from participant where entity_participant_id='"+file_json[0]['participant']+"'") + file_query='''select file_sample.*, + project.id as p_id, project.project_id as proj_id,project.primary_site, + participant.id as part_id + from file_sample,project,participant where file_sample.id='''+str(id) + file_query=file_query+''' and file_sample.project=project.project_id and + file_sample.participant=participant.entity_participant_id''' + file_data=self.fetch_results(file_query) return schema.File( - id=file_json[0]['id'], - name=file_json[0]['file_name'], - participant=file_json[0]['participant'], - sample=file_json[0]['sample'], - access=file_json[0]['access'], - file_size=file_json[0]['file_size'], - data_category=file_json[0]['data_category'], - data_format=file_json[0]['data_format'], - platform=file_json[0]['platform'], - experimental_strategy=file_json[0]['experimental_strategy'], - file_name=file_json[0]['file_name'], + id=id, + name=file_data[0]['file_name'], + participant=file_data[0]['participant'], + sample=file_data[0]['sample'], + access=file_data[0]['access'], + file_size=file_data[0]['file_size'], + data_category=file_data[0]['data_category'], + data_format=file_data[0]['data_format'], + platform=file_data[0]['platform'], + experimental_strategy=file_data[0]['experimental_strategy'], + file_name=file_data[0]['file_name'], cases=schema.FileCases( hits=[schema.FileCase( - part_json[0]['id'], - case_id=file_json[0]['participant'], - project=self.get_project(project_json[0]['id']), + file_data[0]['part_id'], + case_id=file_data[0]['participant'], + project=self.get_project(file_data[0]['p_id']), demographic=schema.Demographic("not hispanic or latino","male","white"), - primary_site=project_json[0]['primary_site'])]), - file_id=file_json[0]['file_id']) + primary_site=file_data[0]['primary_site'])]), + file_id=file_data[0]['file_id']) + # get annotations (currently not in use) def get_case_annotation(self): @@ -186,71 +179,68 @@ def get_case_annotation(self): # get ids of all files from db def get_current_files(self): - files_json=self.fetch_results("select `id` from `file_sample`") - return schema.Files(hits=[item['id'] for item in files_json]) + files_data=self.fetch_results("select `id` from `file_sample`") + return schema.Files(hits=[item['id'] for item in files_data]) # get ids of all cases from db def get_current_cases(self): - cases_json=self.fetch_results("select id from participant") - return schema.RepositoryCases(hits=[case['id'] for case in cases_json]) + cases_data=self.fetch_results("select id from participant") + return schema.RepositoryCases(hits=[case['id'] for case in cases_data]) # get details of case object from db def get_case(self,id): - case_json=self.fetch_results("select * from participant where id="+str(id)) - part_id=case_json[0]['entity_participant_id'] - - case_count_json=self.fetch_results( - "select count(entity_participant_id) as case_count from participant where entity_participant_id="+str(part_id)) - case_count=case_count_json[0]['case_count'] + case_data=self.fetch_results("select * from participant where id="+str(id)) + part_id=case_data[0]['entity_participant_id'] - file_count_json=self.fetch_results( - "select count(id) as file_count from file_sample where participant="+str(part_id)) - file_count=file_count_json[0]['file_count'] + counts_query="select count(id) as total from participant where entity_participant_id="+str(part_id) + counts_query=counts_query+" union all " + counts_query=counts_query+ "select count(id) as total from file_sample where participant="+str(part_id) + counts_query=counts_query+" union all " + counts_query= counts_query+"select sum(file_size) as total from file_sample where participant="+str(part_id) - file_size_json=self.fetch_results( - "select sum(file_size) as filesize from file_sample where participant="+str(part_id)) - file_size=file_size_json[0]['filesize'] + counts_data=self.fetch_results(counts_query) - proj_query="select distinct file_sample.project, project.id as proj_id " + proj_query="select distinct file_sample.project, project.id as p_id, project.primary_site " proj_query=proj_query+"from file_sample, project where file_sample.participant='" proj_query=proj_query+str(part_id)+"' and project.project_id=file_sample.project limit 1" - proj_json=self.fetch_results(proj_query) - proj_id=proj_json[0]['proj_id'] + proj_data=self.fetch_results(proj_query) + + proj_id=proj_data[0]['p_id'] # query to get files from file_sample table - files_json=self.fetch_results("select * from file_sample where participant="+str(part_id)) + files_data=self.fetch_results("select * from file_sample where participant="+str(part_id)) # query to get metadata from prticipant table - metadata_part_json=self.fetch_results( + metadata_part_data=self.fetch_results( "select * from participant where entity_participant_id="+str(part_id)) - del metadata_part_json[0]['updated'] - metadata_part_json[0]['participant'] = metadata_part_json[0]['entity_participant_id'] - del metadata_part_json[0]['entity_participant_id'] - metadata_participant=schema.MetadataParticipant(**metadata_part_json[0]) + del metadata_part_data[0]['updated'] + metadata_part_data[0]['participant'] = metadata_part_data[0]['entity_participant_id'] + del metadata_part_data[0]['entity_participant_id'] + metadata_participant=schema.MetadataParticipant(**metadata_part_data[0]) # query to get metadata from sample table - metadata_json=self.fetch_results("select * from sample where participant="+str(part_id)) + metadata_data=self.fetch_results("select * from sample where participant="+str(part_id)) metadata_list=[] - for metadata in metadata_json: + for metadata in metadata_data: del metadata['updated'] metadata_list.append(schema.MetadataSample(**metadata)) return schema.Case(id, case_id=part_id, - primary_site="Stool", + primary_site=proj_data[0]['primary_site'], demographic=schema.Demographic("not hispanic or latino","male","white"), metadata_participant=metadata_participant, metadata_sample=metadata_list, project=self.get_project(proj_id), summary=schema.Summary( - case_count=case_count, - file_count=file_count, - file_size=file_size, + case_count=counts_data[0]['total'], + file_count=counts_data[1]['total'], + file_size=counts_data[2]['total'], data_categories=self.get_data_categories("participant",part_id)), #experimental_strategies=self.get_experimental_strategies("participant",part_id)), files=schema.CaseFiles(hits=[schema.CaseFile( @@ -259,7 +249,7 @@ def get_case(self,id): data_category=case_file['data_category'], data_format=case_file['data_format'], platform=case_file['platform'], - access=case_file['access']) for case_file in files_json])) + access=case_file['access']) for case_file in files_data])) @@ -294,36 +284,25 @@ def get_project_aggregations(self, projects): return project_aggregates - # get all counts for front page summary from db - def get_current_counts(self): - - projects_json = self.fetch_results("select count(`project_id`)as projects_count from project") - projects = projects_json[0]['projects_count'] - - participants_json = self.fetch_results("select count(`entity_participant_id`) as participants_count from `participant`") - participants = participants_json[0]['participants_count'] - samples_json = self.fetch_results("select count(`sample`) as samples_count from `sample`") - samples = samples_json[0]['samples_count'] - dataFormats_json = self.fetch_results("select count(distinct `data_format`) as data_format_count from `file_sample`") - dataFormats = dataFormats_json[0]['data_format_count'] - - rawFiles_json = self.fetch_results( - "select count(`id`) as rawfiles_count from `file_sample` where type='rawFiles'") - rawFiles = rawFiles_json[0]['rawfiles_count'] + # get all counts for front page summary from db + def get_current_counts(self): - prFiles_json = self.fetch_results( - "select count(`id`) as prfiles_count from `file_sample` where type='processedFiles'") - prFiles = prFiles_json[0]['prfiles_count'] + counts_data = self.fetch_results('''select count(id) as countid from project + union all select count(id) as countid from participant + union all select count(id) as countid from sample + union all select count(distinct data_format) as countid from file_sample + union all select count(id) as countid from file_sample where type="rawFiles" + union all select count(id) as countid from file_sample where type="processedFiles"''') return schema.Count( - projects=projects, - participants=participants, - samples=samples, - dataFormats=dataFormats, - rawFiles=rawFiles, - processedFiles=prFiles) + projects=counts_data[0]['countid'], + participants=counts_data[1]['countid'], + samples=counts_data[2]['countid'], + dataFormats=counts_data[3]['countid'], + rawFiles=counts_data[4]['countid'], + processedFiles=counts_data[5]['countid']) @@ -417,7 +396,7 @@ def get_facets(self): # get total size of all files def get_cart_file_size(self): - file_json=self.fetch_results("select sum(file_size) as sum_size from file_sample") - return schema.FileSize(file_json[0]['sum_size']) + file_data=self.fetch_results("select sum(file_size) as sum_size from file_sample") + return schema.FileSize(file_data[0]['sum_size']) data = Data() diff --git a/schema.py b/schema.py index 0900329..70de3ea 100755 --- a/schema.py +++ b/schema.py @@ -434,6 +434,7 @@ class Root(graphene.ObjectType): repository = graphene.Field(Repository) projects = graphene.Field(Projects) cart_summary = graphene.Field(CartSummary) + version = graphene.Field(Version) def resolve_user(self, info): return data.get_user() @@ -450,6 +451,9 @@ def resolve_projects(self, info): def resolve_cart_summary(self, info): return CartSummary(self) + def resolve_version(self, info): + return data.get_current_version() + class Query(graphene.ObjectType): ## Portal API ## From 734422d59b1a02404da02984b05ffa4f837e7984 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Sat, 23 Mar 2019 22:04:39 +0000 Subject: [PATCH 15/34] Adding command to set environment var for password --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) mode change 100644 => 100755 README.md diff --git a/README.md b/README.md old mode 100644 new mode 100755 index a67d7e1..3cc710b --- a/README.md +++ b/README.md @@ -137,8 +137,9 @@ then run 1. ``CREATE USER 'username'@'localhost' IDENTIFIED BY 'password';`` 2. ``GRANT ALL PRIVILEGES ON *.* TO 'username'@'localhost' WITH GRANT OPTION;`` 3. ``flush privileges;`` - More information https://dev.mysql.com/doc/refman/5.5/en/adding-users.html +4. Set environment variable BIOM_MASS that holds user's password + ``export BIOM_MASS="password"`` load_local_database.py script needs mysql.connect module. From 2e4ecc5f52e62f136270c5782aea71c2a3c2cc29 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Mon, 25 Mar 2019 18:01:09 +0000 Subject: [PATCH 16/34] Removing some test ifs and test cases --- database.py | 58 ++++++++++++++++++----------------------------------- 1 file changed, 19 insertions(+), 39 deletions(-) diff --git a/database.py b/database.py index daf0674..ac7df62 100755 --- a/database.py +++ b/database.py @@ -76,41 +76,29 @@ def get_project(self,id): primary_site=[project_data[0]['primary_site']]) - # get data categories file and case counts from db for a project or a participant + # get data categories file and case counts from db for a project or a participant def get_data_categories(self,table,id): data_cat_data=self.fetch_results( "select distinct data_category from file_sample where "+ table+"='" + str(id)+"'") + data_cat=[] for item in data_cat_data: + count_query="select count(id) as total from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'" + count_query=count_query+" union all " + count_query=count_query+"select count(distinct participant) as total from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'" + count_data=self.fetch_results(count_query) - if table is "participant": - item_files_data=self.fetch_results( - "select count(id) as file_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") - file_count=0 - if len(item_files_data) > 0: - file_count=item_files_data[0]['file_count'] - - data_cat.append( - schema.DataCategories( - file_count=file_count, - data_category=item['data_category'])) - elif table is "project": - item_part_data=self.fetch_results( - "select count(distinct participant) as case_count from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'") - - case_count=0 - if len(item_part_data) > 0: - case_count=item_part_data[0]['case_count'] - data_cat.append( + data_cat.append( schema.DataCategories( - case_count=case_count, + case_count=count_data[1]['total'], + file_count=count_data[0]['total'], data_category=item['data_category'])) return data_cat - # get experimental strategies file count from db for a project + # get experimental strategies file count from db for a project def get_experimental_strategies(self,table,id): exp_str_data=self.fetch_results( @@ -118,22 +106,14 @@ def get_experimental_strategies(self,table,id): exp_str=[] for item in exp_str_data: - item_files_data=self.fetch_results( - "select count(id) as filecount from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") -# item_part_data=self.fetch_results( -# "select count(distinct participant) as case_count from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'") -# -# case_count=0 -# if len(item_part_data) > 0: -# case_count=item_part_data[0]['case_count'] - - file_count=0 - if len(item_files_data) > 0: - file_count=item_files_data[0]['filecount'] + count_query="select count(id) as total from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'" + count_query=count_query+" union all " + count_query=count_query+ "select count(distinct participant) as total from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'" + count_data=self.fetch_results(count_query) exp_str.append( schema.ExperimentalStrategies( -# case_count=case_count, - file_count=file_count, + case_count=count_data[1]['total'], + file_count=count_data[0]['total'], experimental_strategy=item['experimental_strategy'])) return exp_str @@ -241,8 +221,8 @@ def get_case(self,id): case_count=counts_data[0]['total'], file_count=counts_data[1]['total'], file_size=counts_data[2]['total'], - data_categories=self.get_data_categories("participant",part_id)), - #experimental_strategies=self.get_experimental_strategies("participant",part_id)), + data_categories=self.get_data_categories("participant",part_id), + experimental_strategies=self.get_experimental_strategies("participant",part_id)), files=schema.CaseFiles(hits=[schema.CaseFile( case_file['id'], experimental_strategy=case_file['experimental_strategy'], @@ -257,7 +237,7 @@ def get_project_aggregations(self, projects): # compile aggregations from project aggregates = {"primary_site": {}, "program__name": {}, - "project_id": {}, + "project_id": {}, "summary__data_categories__data_category": {}, "summary__experimental_strategies__experimental_strategy": {}} From 3d604c0fac769b79c8805e0508589f7a199d5363 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Mon, 25 Mar 2019 18:47:25 +0000 Subject: [PATCH 17/34] adding placeholders for init and exit for future use --- database.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/database.py b/database.py index ac7df62..aacdd91 100755 --- a/database.py +++ b/database.py @@ -13,7 +13,11 @@ def add_key_increment(dictionary, key): class Data(object): +# def __init__(self): + # self.db_conn = mariadb.connect(user='biom_mass', password=os.environ['BIOM_MASS'], db='portal_ui') +# def __exit__(self): + # self.db.conn.close() # connects to db and runs query def fetch_results(self,query): @@ -21,7 +25,7 @@ def fetch_results(self,query): db_conn = mariadb.connect(user='biom_mass', password=os.environ['BIOM_MASS'], db='portal_ui') cursor = db_conn.cursor(buffered=True) cursor.execute(query) - # response json + # response list of json data row_headers=[x[0] for x in cursor.description] rows = cursor.fetchall() data=[] @@ -171,7 +175,9 @@ def get_current_cases(self): # get details of case object from db def get_case(self,id): - case_data=self.fetch_results("select * from participant where id="+str(id)) + # get entity_participant_id and file info from db + case_data=self.fetch_results('''select participant.entity_participant_id, file_sample.* from participant, file_sample + where participant.id='''+str(id)+" and file_sample.participant=participant.entity_participant_id") part_id=case_data[0]['entity_participant_id'] counts_query="select count(id) as total from participant where entity_participant_id="+str(part_id) @@ -189,8 +195,6 @@ def get_case(self,id): proj_id=proj_data[0]['p_id'] - # query to get files from file_sample table - files_data=self.fetch_results("select * from file_sample where participant="+str(part_id)) # query to get metadata from prticipant table metadata_part_data=self.fetch_results( @@ -229,7 +233,7 @@ def get_case(self,id): data_category=case_file['data_category'], data_format=case_file['data_format'], platform=case_file['platform'], - access=case_file['access']) for case_file in files_data])) + access=case_file['access']) for case_file in case_data])) From b49cfea45edb7fdeb2c1b122bc7a281fe294bdd8 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Wed, 27 Mar 2019 02:51:52 +0000 Subject: [PATCH 18/34] Using pooling, c extension, cursor dictionary, and pulling version --- database.py | 40 +++++++++++++++++++++++++--------------- schema.graphql | 10 +--------- schema.py | 11 ----------- server.py | 9 ++++++--- 4 files changed, 32 insertions(+), 38 deletions(-) mode change 100755 => 100644 database.py mode change 100755 => 100644 schema.py diff --git a/database.py b/database.py old mode 100755 new mode 100644 index aacdd91..f99d6a6 --- a/database.py +++ b/database.py @@ -1,6 +1,8 @@ # Calls to obtain values from database -import mysql.connector as mariadb +import mysql.connector +#from mysql.connector.connection import MySQLConnection +from mysql.connector import pooling import os import schema @@ -13,26 +15,32 @@ def add_key_increment(dictionary, key): class Data(object): -# def __init__(self): - # self.db_conn = mariadb.connect(user='biom_mass', password=os.environ['BIOM_MASS'], db='portal_ui') + def __init__(self): + # self.db_conn = mysql.connector.connect(user='biom_mass', password=os.environ['BIOM_MASS'], db='portal_ui', use_pure = False) -# def __exit__(self): - # self.db.conn.close() + self.conn_pool = mysql.connector.connect(use_pure=False, pool_name="portal", + pool_size=32, + pool_reset_session=True, + database='portal_ui', + user='biom_mass', + password=os.environ['BIOM_MASS']) + # print("Use Pure is--------", use_pure) + + def __exit__(self): + self.conn_pool.close() # connects to db and runs query def fetch_results(self,query): - db_conn = mariadb.connect(user='biom_mass', password=os.environ['BIOM_MASS'], db='portal_ui') - cursor = db_conn.cursor(buffered=True) + # db_conn = mysql.connector.connect(user='biom_mass', password=os.environ['BIOM_MASS'], db='portal_ui', use_pure = False + conn = mysql.connector.connect(pool_name="portal") + cursor = conn.cursor(buffered=True,dictionary=True) cursor.execute(query) - # response list of json data - row_headers=[x[0] for x in cursor.description] - rows = cursor.fetchall() - data=[] - for row in rows: - data.append(dict(zip(row_headers,row))) + + # response list of dict + data=[row for row in cursor] cursor.close() - db_conn.close() + conn.close() return data @@ -50,7 +58,7 @@ def get_current_version(self): version_data=self.fetch_results("select * from `version` order by updated desc limit 1 ") del version_data[0]['updated'] del version_data[0]['id'] - return schema.Version(**version_data[0]) + return version_data[0] # get project object details from db @@ -384,3 +392,5 @@ def get_cart_file_size(self): return schema.FileSize(file_data[0]['sum_size']) data = Data() + +VERSION = data.get_current_version() diff --git a/schema.graphql b/schema.graphql index c803ef5..8a40712 100755 --- a/schema.graphql +++ b/schema.graphql @@ -28,15 +28,7 @@ type Case implements Node { project: Project summary: Summary annotations: CaseAnnotations - files: CaseFiles -} - -type Version{ - data_release: String - commit: String - version: Float - tag: String - status: String + files: CaseFile } type MetadataParticipant { diff --git a/schema.py b/schema.py old mode 100755 new mode 100644 index 70de3ea..b330cb7 --- a/schema.py +++ b/schema.py @@ -43,13 +43,6 @@ class Count(graphene.ObjectType): class User(graphene.ObjectType): username = graphene.String() -class Version(graphene.ObjectType): - data_release = graphene.String() - commit = graphene.String() - version = graphene.Float() - tag = graphene.String() - status =graphene.String() - class Program(graphene.ObjectType): name = graphene.String() program_id = graphene.String() @@ -434,7 +427,6 @@ class Root(graphene.ObjectType): repository = graphene.Field(Repository) projects = graphene.Field(Projects) cart_summary = graphene.Field(CartSummary) - version = graphene.Field(Version) def resolve_user(self, info): return data.get_user() @@ -451,9 +443,6 @@ def resolve_projects(self, info): def resolve_cart_summary(self, info): return CartSummary(self) - def resolve_version(self, info): - return data.get_current_version() - class Query(graphene.ObjectType): ## Portal API ## diff --git a/server.py b/server.py index 4154751..c735249 100644 --- a/server.py +++ b/server.py @@ -10,11 +10,13 @@ import os import flask +import flask_cors import flask_graphql import graphene import schema import const +import database NAME = "firecloud_graphql" HOST = "0.0.0.0" @@ -41,7 +43,8 @@ def process_query(request, schema): def main(): # create the graphql flask app app = flask.Flask(NAME) - + # allow initial OPTIONS requests + flask_cors.CORS(app) # add the root graphql queries @app.route('/graphql', methods=["POST"]) def get_root_schema(): @@ -55,8 +58,8 @@ def get_schema(name): # add static endpoint for version/status @app.route('/status', methods=["GET"]) def get_version(): - return flask.jsonify(const.VERSION) - + return flask.jsonify(database.VERSION) + # app.debug = True # add end point for graphql gui app.add_url_rule('/test', view_func=flask_graphql.GraphQLView.as_view( 'test', schema=graphene.Schema(query=schema.Query, auto_camelcase=False), graphiql=True)) From b7ba27dc01ad8551d2ede2bfc6ee0094f4bd0fb1 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Wed, 27 Mar 2019 16:55:27 +0000 Subject: [PATCH 19/34] moving connection out of fetch results --- database.py | 126 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 76 insertions(+), 50 deletions(-) diff --git a/database.py b/database.py index f99d6a6..94fdfe5 100644 --- a/database.py +++ b/database.py @@ -1,7 +1,6 @@ # Calls to obtain values from database import mysql.connector -#from mysql.connector.connection import MySQLConnection from mysql.connector import pooling import os import schema @@ -16,37 +15,31 @@ def add_key_increment(dictionary, key): class Data(object): def __init__(self): - # self.db_conn = mysql.connector.connect(user='biom_mass', password=os.environ['BIOM_MASS'], db='portal_ui', use_pure = False) - - self.conn_pool = mysql.connector.connect(use_pure=False, pool_name="portal", + self.conn_pool=mysql.connector.connect(use_pure=False, pool_name="portal", pool_size=32, pool_reset_session=True, database='portal_ui', user='biom_mass', password=os.environ['BIOM_MASS']) + # print("Use Pure is--------", use_pure) def __exit__(self): self.conn_pool.close() - # connects to db and runs query - def fetch_results(self,query): - - # db_conn = mysql.connector.connect(user='biom_mass', password=os.environ['BIOM_MASS'], db='portal_ui', use_pure = False - conn = mysql.connector.connect(pool_name="portal") - cursor = conn.cursor(buffered=True,dictionary=True) + # runs query and returns result + def fetch_results(self,cursor,query): cursor.execute(query) + return [row for row in cursor] - # response list of dict - data=[row for row in cursor] - cursor.close() - conn.close() - - return data # get all projects from db def get_current_projects(self): - projects_data=self.fetch_results("select `id` from `project`") + conn = mysql.connector.connect(pool_name="portal") + cursor = conn.cursor(buffered=True,dictionary=True) + projects_data=self.fetch_results(cursor,"select id from project") + cursor.close() + conn.close() return [self.get_project(project['id']) for project in projects_data] # get user (currently none) @@ -55,7 +48,12 @@ def get_user(self): # get current version from db def get_current_version(self): - version_data=self.fetch_results("select * from `version` order by updated desc limit 1 ") + conn = mysql.connector.connect(pool_name="portal") + cursor = conn.cursor(buffered=True,dictionary=True) + version_data=self.fetch_results(cursor, + "select * from version order by updated desc limit 1 ") + cursor.close() + conn.close() del version_data[0]['updated'] del version_data[0]['id'] return version_data[0] @@ -64,7 +62,10 @@ def get_current_version(self): # get project object details from db def get_project(self,id): - project_data=self.fetch_results("select * from `project` where `id` =" + str(id)) + conn = mysql.connector.connect(pool_name="portal") + cursor = conn.cursor(buffered=True,dictionary=True) + project_data=self.fetch_results(cursor, + "select * from project where id =" + str(id)) project_id=project_data[0]['project_id'] counts_query="select count(id) as total from file_sample where project='"+project_id+"'" @@ -72,7 +73,9 @@ def get_project(self,id): counts_query=counts_query+"select count(distinct participant) as total from file_sample where project='"+project_id+"'" counts_query=counts_query+" union all " counts_query=counts_query+"select sum(file_size) as total from file_sample where project='"+project_id+"'" - proj_counts_data=self.fetch_results(counts_query) + proj_counts_data=self.fetch_results(cursor,counts_query) + cursor.close() + conn.close() return schema.Project( id=id, @@ -90,16 +93,16 @@ def get_project(self,id): # get data categories file and case counts from db for a project or a participant def get_data_categories(self,table,id): - - data_cat_data=self.fetch_results( + conn = mysql.connector.connect(pool_name="portal") + cursor = conn.cursor(buffered=True,dictionary=True) + data_cat_data=self.fetch_results(cursor, "select distinct data_category from file_sample where "+ table+"='" + str(id)+"'") - data_cat=[] for item in data_cat_data: count_query="select count(id) as total from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'" count_query=count_query+" union all " count_query=count_query+"select count(distinct participant) as total from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'" - count_data=self.fetch_results(count_query) + count_data=self.fetch_results(cursor,count_query) data_cat.append( schema.DataCategories( @@ -107,40 +110,48 @@ def get_data_categories(self,table,id): file_count=count_data[0]['total'], data_category=item['data_category'])) + cursor.close() + conn.close() return data_cat # get experimental strategies file count from db for a project def get_experimental_strategies(self,table,id): - - exp_str_data=self.fetch_results( + conn = mysql.connector.connect(pool_name="portal") + cursor = conn.cursor(buffered=True,dictionary=True) + exp_str_data=self.fetch_results(cursor, "select distinct experimental_strategy from file_sample where "+ table+"='" + str(id) +"'") - exp_str=[] for item in exp_str_data: count_query="select count(id) as total from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'" count_query=count_query+" union all " count_query=count_query+ "select count(distinct participant) as total from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'" - count_data=self.fetch_results(count_query) + count_data=self.fetch_results(cursor,count_query) exp_str.append( schema.ExperimentalStrategies( case_count=count_data[1]['total'], file_count=count_data[0]['total'], experimental_strategy=item['experimental_strategy'])) + cursor.close() + conn.close() return exp_str # get details of file object from db def get_file(self,id): + conn = mysql.connector.connect(pool_name="portal") + cursor = conn.cursor(buffered=True,dictionary=True) file_query='''select file_sample.*, project.id as p_id, project.project_id as proj_id,project.primary_site, participant.id as part_id from file_sample,project,participant where file_sample.id='''+str(id) file_query=file_query+''' and file_sample.project=project.project_id and file_sample.participant=participant.entity_participant_id''' - file_data=self.fetch_results(file_query) + file_data=self.fetch_results(cursor,file_query) + cursor.close() + conn.close() return schema.File( id=id, @@ -170,21 +181,29 @@ def get_case_annotation(self): # get ids of all files from db def get_current_files(self): - - files_data=self.fetch_results("select `id` from `file_sample`") - return schema.Files(hits=[item['id'] for item in files_data]) + conn = mysql.connector.connect(pool_name="portal") + cursor = conn.cursor(buffered=True,dictionary=True) + files_data=self.fetch_results(cursor,"select `id` from `file_sample`") + cursor.close() + conn.close() + return schema.Files(hits=[file['id'] for file in files_data]) # get ids of all cases from db def get_current_cases(self): - - cases_data=self.fetch_results("select id from participant") + conn = mysql.connector.connect(pool_name="portal") + cursor = conn.cursor(buffered=True,dictionary=True) + cases_data=self.fetch_results(cursor,"select id from participant") + cursor.close() + conn.close() return schema.RepositoryCases(hits=[case['id'] for case in cases_data]) # get details of case object from db def get_case(self,id): # get entity_participant_id and file info from db - case_data=self.fetch_results('''select participant.entity_participant_id, file_sample.* from participant, file_sample + conn = mysql.connector.connect(pool_name="portal") + cursor = conn.cursor(buffered=True,dictionary=True) + case_data=self.fetch_results(cursor,'''select participant.entity_participant_id, file_sample.* from participant, file_sample where participant.id='''+str(id)+" and file_sample.participant=participant.entity_participant_id") part_id=case_data[0]['entity_participant_id'] @@ -193,21 +212,19 @@ def get_case(self,id): counts_query=counts_query+ "select count(id) as total from file_sample where participant="+str(part_id) counts_query=counts_query+" union all " counts_query= counts_query+"select sum(file_size) as total from file_sample where participant="+str(part_id) - - counts_data=self.fetch_results(counts_query) + counts_data=self.fetch_results(cursor,counts_query) proj_query="select distinct file_sample.project, project.id as p_id, project.primary_site " proj_query=proj_query+"from file_sample, project where file_sample.participant='" proj_query=proj_query+str(part_id)+"' and project.project_id=file_sample.project limit 1" - proj_data=self.fetch_results(proj_query) + proj_data=self.fetch_results(cursor,proj_query) proj_id=proj_data[0]['p_id'] # query to get metadata from prticipant table - metadata_part_data=self.fetch_results( + metadata_part_data=self.fetch_results(cursor, "select * from participant where entity_participant_id="+str(part_id)) - del metadata_part_data[0]['updated'] metadata_part_data[0]['participant'] = metadata_part_data[0]['entity_participant_id'] del metadata_part_data[0]['entity_participant_id'] @@ -215,12 +232,15 @@ def get_case(self,id): # query to get metadata from sample table - metadata_data=self.fetch_results("select * from sample where participant="+str(part_id)) + metadata_data=self.fetch_results(cursor, + "select * from sample where participant="+str(part_id)) metadata_list=[] for metadata in metadata_data: del metadata['updated'] metadata_list.append(schema.MetadataSample(**metadata)) + cursor.close() + conn.close() return schema.Case(id, case_id=part_id, @@ -281,13 +301,16 @@ def get_project_aggregations(self, projects): # get all counts for front page summary from db def get_current_counts(self): - counts_data = self.fetch_results('''select count(id) as countid from project - union all select count(id) as countid from participant - union all select count(id) as countid from sample - union all select count(distinct data_format) as countid from file_sample - union all select count(id) as countid from file_sample where type="rawFiles" - union all select count(id) as countid from file_sample where type="processedFiles"''') - + conn = mysql.connector.connect(pool_name="portal") + cursor = conn.cursor(buffered=True,dictionary=True) + counts_data=self.fetch_results(cursor,'''select count(id) as countid from project + union all select count(id) as countid from participant + union all select count(id) as countid from sample + union all select count(distinct data_format) as countid from file_sample + union all select count(id) as countid from file_sample where type="rawFiles" + union all select count(id) as countid from file_sample where type="processedFiles"''') + cursor.close() + conn.close() return schema.Count( projects=counts_data[0]['countid'], participants=counts_data[1]['countid'], @@ -387,8 +410,11 @@ def get_facets(self): # get total size of all files def get_cart_file_size(self): - - file_data=self.fetch_results("select sum(file_size) as sum_size from file_sample") + conn = mysql.connector.connect(pool_name="portal") + cursor = conn.cursor(buffered=True,dictionary=True) + file_data=self.fetch_results(cursor,"select sum(file_size) as sum_size from file_sample") + cursor.close() + conn.close() return schema.FileSize(file_data[0]['sum_size']) data = Data() From ead4a8b9de551966bdcf9d601f35ac3351ce5ebf Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Wed, 27 Mar 2019 18:29:56 +0000 Subject: [PATCH 20/34] experimenting with connection pools --- database.py | 70 ++++++++++++++++++++++++++++------------------------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/database.py b/database.py index 94fdfe5..149fa09 100644 --- a/database.py +++ b/database.py @@ -16,14 +16,12 @@ class Data(object): def __init__(self): self.conn_pool=mysql.connector.connect(use_pure=False, pool_name="portal", - pool_size=32, + pool_size=10, pool_reset_session=True, database='portal_ui', user='biom_mass', password=os.environ['BIOM_MASS']) - # print("Use Pure is--------", use_pure) - def __exit__(self): self.conn_pool.close() @@ -38,9 +36,10 @@ def get_current_projects(self): conn = mysql.connector.connect(pool_name="portal") cursor = conn.cursor(buffered=True,dictionary=True) projects_data=self.fetch_results(cursor,"select id from project") + project_object=[self.get_project(project['id'],cursor) for project in projects_data] cursor.close() conn.close() - return [self.get_project(project['id']) for project in projects_data] + return project_object # get user (currently none) def get_user(self): @@ -60,10 +59,10 @@ def get_current_version(self): # get project object details from db - def get_project(self,id): + def get_project(self,id,cursor): - conn = mysql.connector.connect(pool_name="portal") - cursor = conn.cursor(buffered=True,dictionary=True) + #conn = mysql.connector.connect(pool_name="portal") + #cursor = conn.cursor(buffered=True,dictionary=True) project_data=self.fetch_results(cursor, "select * from project where id =" + str(id)) project_id=project_data[0]['project_id'] @@ -74,10 +73,8 @@ def get_project(self,id): counts_query=counts_query+" union all " counts_query=counts_query+"select sum(file_size) as total from file_sample where project='"+project_id+"'" proj_counts_data=self.fetch_results(cursor,counts_query) - cursor.close() - conn.close() - return schema.Project( + project_object=schema.Project( id=id, project_id=project_id, name=project_data[0]['name'], @@ -85,16 +82,19 @@ def get_project(self,id): summary=schema.Summary( case_count=proj_counts_data[1]['total'], file_count=proj_counts_data[0]['total'], - data_categories=self.get_data_categories("project",project_id), - experimental_strategies=self.get_experimental_strategies("project",project_id), + data_categories=self.get_data_categories("project",project_id,cursor), + experimental_strategies=self.get_experimental_strategies("project",project_id,cursor), file_size=proj_counts_data[2]['total']), primary_site=[project_data[0]['primary_site']]) + #cursor.close() + #conn.close() + return project_object # get data categories file and case counts from db for a project or a participant - def get_data_categories(self,table,id): - conn = mysql.connector.connect(pool_name="portal") - cursor = conn.cursor(buffered=True,dictionary=True) + def get_data_categories(self,table,id,cursor): + #conn = mysql.connector.connect(pool_name="portal") + #cursor = conn.cursor(buffered=True,dictionary=True) data_cat_data=self.fetch_results(cursor, "select distinct data_category from file_sample where "+ table+"='" + str(id)+"'") data_cat=[] @@ -110,15 +110,15 @@ def get_data_categories(self,table,id): file_count=count_data[0]['total'], data_category=item['data_category'])) - cursor.close() - conn.close() + #cursor.close() + #conn.close() return data_cat # get experimental strategies file count from db for a project - def get_experimental_strategies(self,table,id): - conn = mysql.connector.connect(pool_name="portal") - cursor = conn.cursor(buffered=True,dictionary=True) + def get_experimental_strategies(self,table,id,cursor): + #conn = mysql.connector.connect(pool_name="portal") + #cursor = conn.cursor(buffered=True,dictionary=True) exp_str_data=self.fetch_results(cursor, "select distinct experimental_strategy from file_sample where "+ table+"='" + str(id) +"'") exp_str=[] @@ -133,8 +133,8 @@ def get_experimental_strategies(self,table,id): file_count=count_data[0]['total'], experimental_strategy=item['experimental_strategy'])) - cursor.close() - conn.close() + #cursor.close() + #conn.close() return exp_str @@ -150,10 +150,8 @@ def get_file(self,id): file_query=file_query+''' and file_sample.project=project.project_id and file_sample.participant=participant.entity_participant_id''' file_data=self.fetch_results(cursor,file_query) - cursor.close() - conn.close() - return schema.File( + file_object=schema.File( id=id, name=file_data[0]['file_name'], participant=file_data[0]['participant'], @@ -169,11 +167,15 @@ def get_file(self,id): hits=[schema.FileCase( file_data[0]['part_id'], case_id=file_data[0]['participant'], - project=self.get_project(file_data[0]['p_id']), + project=self.get_project(file_data[0]['p_id'],cursor), demographic=schema.Demographic("not hispanic or latino","male","white"), primary_site=file_data[0]['primary_site'])]), file_id=file_data[0]['file_id']) + cursor.close() + conn.close() + return file_object + # get annotations (currently not in use) def get_case_annotation(self): @@ -239,22 +241,19 @@ def get_case(self,id): del metadata['updated'] metadata_list.append(schema.MetadataSample(**metadata)) - cursor.close() - conn.close() - - return schema.Case(id, + case_object=schema.Case(id, case_id=part_id, primary_site=proj_data[0]['primary_site'], demographic=schema.Demographic("not hispanic or latino","male","white"), metadata_participant=metadata_participant, metadata_sample=metadata_list, - project=self.get_project(proj_id), + project=self.get_project(proj_id,cursor), summary=schema.Summary( case_count=counts_data[0]['total'], file_count=counts_data[1]['total'], file_size=counts_data[2]['total'], - data_categories=self.get_data_categories("participant",part_id), - experimental_strategies=self.get_experimental_strategies("participant",part_id)), + data_categories=self.get_data_categories("participant",part_id,cursor), + experimental_strategies=self.get_experimental_strategies("participant",part_id,cursor)), files=schema.CaseFiles(hits=[schema.CaseFile( case_file['id'], experimental_strategy=case_file['experimental_strategy'], @@ -264,6 +263,11 @@ def get_case(self,id): access=case_file['access']) for case_file in case_data])) + cursor.close() + conn.close() + + return case_object + def get_project_aggregations(self, projects): From 4147de154251364b42e1f2afe2d1ee8a2087a269 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Thu, 28 Mar 2019 17:14:40 +0000 Subject: [PATCH 21/34] testing saving projects info --- database.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/database.py b/database.py index 149fa09..f7707c1 100644 --- a/database.py +++ b/database.py @@ -21,9 +21,18 @@ def __init__(self): database='portal_ui', user='biom_mass', password=os.environ['BIOM_MASS']) - - def __exit__(self): - self.conn_pool.close() +# conn = mysql.connector.connect(pool_name="portal") +# cursor = conn.cursor(buffered=True,dictionary=True) +# projects_data=self.fetch_results(cursor,"select id from project") +# self.projects={} +# for project in projects_data: +# self.projects[project['id']]=self.get_project(project['id'],cursor) +# print(self.projects) +# cursor.close() +# conn.close() + +# def __exit__(self): +# self.conn_pool.close() # runs query and returns result def fetch_results(self,cursor,query): @@ -168,6 +177,7 @@ def get_file(self,id): file_data[0]['part_id'], case_id=file_data[0]['participant'], project=self.get_project(file_data[0]['p_id'],cursor), + # project=self.projects[file_data[0]['p_id']].getValue(), demographic=schema.Demographic("not hispanic or latino","male","white"), primary_site=file_data[0]['primary_site'])]), file_id=file_data[0]['file_id']) @@ -248,6 +258,7 @@ def get_case(self,id): metadata_participant=metadata_participant, metadata_sample=metadata_list, project=self.get_project(proj_id,cursor), + #project=self.projects[proj_id].getValue(), summary=schema.Summary( case_count=counts_data[0]['total'], file_count=counts_data[1]['total'], From cd9a8bccf1ca133c59de2ddfc65577d6f5f60e9a Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Thu, 28 Mar 2019 23:40:10 +0000 Subject: [PATCH 22/34] working fast with projects saved --- database.py | 46 ++++++++++++++++------------------------------ 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/database.py b/database.py index f7707c1..7cd584a 100644 --- a/database.py +++ b/database.py @@ -15,24 +15,18 @@ def add_key_increment(dictionary, key): class Data(object): def __init__(self): + self.conn_pool=mysql.connector.connect(use_pure=False, pool_name="portal", pool_size=10, pool_reset_session=True, database='portal_ui', user='biom_mass', password=os.environ['BIOM_MASS']) -# conn = mysql.connector.connect(pool_name="portal") -# cursor = conn.cursor(buffered=True,dictionary=True) -# projects_data=self.fetch_results(cursor,"select id from project") -# self.projects={} -# for project in projects_data: -# self.projects[project['id']]=self.get_project(project['id'],cursor) -# print(self.projects) -# cursor.close() -# conn.close() - -# def __exit__(self): -# self.conn_pool.close() + + self.projects={} + + def __exit__(self): + self.conn_pool.close() # runs query and returns result def fetch_results(self,cursor,query): @@ -46,6 +40,8 @@ def get_current_projects(self): cursor = conn.cursor(buffered=True,dictionary=True) projects_data=self.fetch_results(cursor,"select id from project") project_object=[self.get_project(project['id'],cursor) for project in projects_data] + for project in projects_data: + self.projects[str(project['id'])]=self.get_project(project['id'],cursor) cursor.close() conn.close() return project_object @@ -70,8 +66,6 @@ def get_current_version(self): # get project object details from db def get_project(self,id,cursor): - #conn = mysql.connector.connect(pool_name="portal") - #cursor = conn.cursor(buffered=True,dictionary=True) project_data=self.fetch_results(cursor, "select * from project where id =" + str(id)) project_id=project_data[0]['project_id'] @@ -83,7 +77,7 @@ def get_project(self,id,cursor): counts_query=counts_query+"select sum(file_size) as total from file_sample where project='"+project_id+"'" proj_counts_data=self.fetch_results(cursor,counts_query) - project_object=schema.Project( + return schema.Project( id=id, project_id=project_id, name=project_data[0]['name'], @@ -96,14 +90,10 @@ def get_project(self,id,cursor): file_size=proj_counts_data[2]['total']), primary_site=[project_data[0]['primary_site']]) - #cursor.close() - #conn.close() - return project_object # get data categories file and case counts from db for a project or a participant def get_data_categories(self,table,id,cursor): - #conn = mysql.connector.connect(pool_name="portal") - #cursor = conn.cursor(buffered=True,dictionary=True) + data_cat_data=self.fetch_results(cursor, "select distinct data_category from file_sample where "+ table+"='" + str(id)+"'") data_cat=[] @@ -119,15 +109,12 @@ def get_data_categories(self,table,id,cursor): file_count=count_data[0]['total'], data_category=item['data_category'])) - #cursor.close() - #conn.close() return data_cat # get experimental strategies file count from db for a project def get_experimental_strategies(self,table,id,cursor): - #conn = mysql.connector.connect(pool_name="portal") - #cursor = conn.cursor(buffered=True,dictionary=True) + exp_str_data=self.fetch_results(cursor, "select distinct experimental_strategy from file_sample where "+ table+"='" + str(id) +"'") exp_str=[] @@ -142,8 +129,7 @@ def get_experimental_strategies(self,table,id,cursor): file_count=count_data[0]['total'], experimental_strategy=item['experimental_strategy'])) - #cursor.close() - #conn.close() + return exp_str @@ -176,8 +162,8 @@ def get_file(self,id): hits=[schema.FileCase( file_data[0]['part_id'], case_id=file_data[0]['participant'], - project=self.get_project(file_data[0]['p_id'],cursor), - # project=self.projects[file_data[0]['p_id']].getValue(), + #project=self.get_project(file_data[0]['p_id'],cursor), + project=self.projects[str(file_data[0]['p_id'])], demographic=schema.Demographic("not hispanic or latino","male","white"), primary_site=file_data[0]['primary_site'])]), file_id=file_data[0]['file_id']) @@ -257,8 +243,8 @@ def get_case(self,id): demographic=schema.Demographic("not hispanic or latino","male","white"), metadata_participant=metadata_participant, metadata_sample=metadata_list, - project=self.get_project(proj_id,cursor), - #project=self.projects[proj_id].getValue(), + # project=self.get_project(proj_id,cursor), + project=self.projects[str(proj_id)], summary=schema.Summary( case_count=counts_data[0]['total'], file_count=counts_data[1]['total'], From bfe86e2034f66fac79fe4cf05b93855e79c18578 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Fri, 29 Mar 2019 01:06:47 +0000 Subject: [PATCH 23/34] get pool and release pool added --- database.py | 59 +++++++++++++++++++++++++---------------------------- 1 file changed, 28 insertions(+), 31 deletions(-) diff --git a/database.py b/database.py index 7cd584a..923ed27 100644 --- a/database.py +++ b/database.py @@ -22,12 +22,25 @@ def __init__(self): database='portal_ui', user='biom_mass', password=os.environ['BIOM_MASS']) - + # holds projects info self.projects={} def __exit__(self): self.conn_pool.close() + # get connection from pool + def get_pool(self): + conn = mysql.connector.connect(pool_name="portal") + cursor = conn.cursor(buffered=True,dictionary=True) + return conn, cursor + + # release connection back to pool + def release_pool(self,conn,cursor): + cursor.close() + conn.close() + + + # runs query and returns result def fetch_results(self,cursor,query): cursor.execute(query) @@ -52,12 +65,10 @@ def get_user(self): # get current version from db def get_current_version(self): - conn = mysql.connector.connect(pool_name="portal") - cursor = conn.cursor(buffered=True,dictionary=True) + conn,cursor=get_pull version_data=self.fetch_results(cursor, "select * from version order by updated desc limit 1 ") - cursor.close() - conn.close() + release_pool(conn,cursor) del version_data[0]['updated'] del version_data[0]['id'] return version_data[0] @@ -136,8 +147,7 @@ def get_experimental_strategies(self,table,id,cursor): # get details of file object from db def get_file(self,id): - conn = mysql.connector.connect(pool_name="portal") - cursor = conn.cursor(buffered=True,dictionary=True) + conn,cursor=get_pool() file_query='''select file_sample.*, project.id as p_id, project.project_id as proj_id,project.primary_site, participant.id as part_id @@ -168,8 +178,7 @@ def get_file(self,id): primary_site=file_data[0]['primary_site'])]), file_id=file_data[0]['file_id']) - cursor.close() - conn.close() + release_pool(conn,cursor) return file_object @@ -179,28 +188,23 @@ def get_case_annotation(self): # get ids of all files from db def get_current_files(self): - conn = mysql.connector.connect(pool_name="portal") - cursor = conn.cursor(buffered=True,dictionary=True) + conn,cursor=get_pool() files_data=self.fetch_results(cursor,"select `id` from `file_sample`") - cursor.close() - conn.close() + release_pool(conn,cursor) return schema.Files(hits=[file['id'] for file in files_data]) # get ids of all cases from db def get_current_cases(self): - conn = mysql.connector.connect(pool_name="portal") - cursor = conn.cursor(buffered=True,dictionary=True) + conn,cursor=get_pool() cases_data=self.fetch_results(cursor,"select id from participant") - cursor.close() - conn.close() + release_pool(conn,cursor) return schema.RepositoryCases(hits=[case['id'] for case in cases_data]) # get details of case object from db def get_case(self,id): # get entity_participant_id and file info from db - conn = mysql.connector.connect(pool_name="portal") - cursor = conn.cursor(buffered=True,dictionary=True) + conn,cursor=get_pool() case_data=self.fetch_results(cursor,'''select participant.entity_participant_id, file_sample.* from participant, file_sample where participant.id='''+str(id)+" and file_sample.participant=participant.entity_participant_id") part_id=case_data[0]['entity_participant_id'] @@ -260,9 +264,7 @@ def get_case(self,id): access=case_file['access']) for case_file in case_data])) - cursor.close() - conn.close() - + release_pool(conn,cursor) return case_object @@ -302,16 +304,14 @@ def get_project_aggregations(self, projects): # get all counts for front page summary from db def get_current_counts(self): - conn = mysql.connector.connect(pool_name="portal") - cursor = conn.cursor(buffered=True,dictionary=True) + conn,cursor=get_pool() counts_data=self.fetch_results(cursor,'''select count(id) as countid from project union all select count(id) as countid from participant union all select count(id) as countid from sample union all select count(distinct data_format) as countid from file_sample union all select count(id) as countid from file_sample where type="rawFiles" union all select count(id) as countid from file_sample where type="processedFiles"''') - cursor.close() - conn.close() + release_pool(conn,cursor) return schema.Count( projects=counts_data[0]['countid'], participants=counts_data[1]['countid'], @@ -411,13 +411,10 @@ def get_facets(self): # get total size of all files def get_cart_file_size(self): - conn = mysql.connector.connect(pool_name="portal") - cursor = conn.cursor(buffered=True,dictionary=True) + conn,cursor=get_pull() file_data=self.fetch_results(cursor,"select sum(file_size) as sum_size from file_sample") - cursor.close() - conn.close() + release_pool(conn,cursor) return schema.FileSize(file_data[0]['sum_size']) data = Data() -VERSION = data.get_current_version() From 1a4367726fa89432bda58b5fe89a6f0eb6debf9a Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Fri, 29 Mar 2019 02:32:18 +0000 Subject: [PATCH 24/34] fixing version on front page --- database.py | 34 ++++++++++++++++------------------ server.py | 6 +++--- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/database.py b/database.py index 923ed27..808eae4 100644 --- a/database.py +++ b/database.py @@ -49,14 +49,12 @@ def fetch_results(self,cursor,query): # get all projects from db def get_current_projects(self): - conn = mysql.connector.connect(pool_name="portal") - cursor = conn.cursor(buffered=True,dictionary=True) + conn,cursor=self.get_pool() projects_data=self.fetch_results(cursor,"select id from project") project_object=[self.get_project(project['id'],cursor) for project in projects_data] for project in projects_data: self.projects[str(project['id'])]=self.get_project(project['id'],cursor) - cursor.close() - conn.close() + self.release_pool(conn,cursor) return project_object # get user (currently none) @@ -65,10 +63,10 @@ def get_user(self): # get current version from db def get_current_version(self): - conn,cursor=get_pull + conn,cursor=self.get_pool() version_data=self.fetch_results(cursor, "select * from version order by updated desc limit 1 ") - release_pool(conn,cursor) + self.release_pool(conn,cursor) del version_data[0]['updated'] del version_data[0]['id'] return version_data[0] @@ -147,7 +145,7 @@ def get_experimental_strategies(self,table,id,cursor): # get details of file object from db def get_file(self,id): - conn,cursor=get_pool() + conn,cursor=self.get_pool() file_query='''select file_sample.*, project.id as p_id, project.project_id as proj_id,project.primary_site, participant.id as part_id @@ -178,7 +176,7 @@ def get_file(self,id): primary_site=file_data[0]['primary_site'])]), file_id=file_data[0]['file_id']) - release_pool(conn,cursor) + self.release_pool(conn,cursor) return file_object @@ -188,23 +186,23 @@ def get_case_annotation(self): # get ids of all files from db def get_current_files(self): - conn,cursor=get_pool() + conn,cursor=self.get_pool() files_data=self.fetch_results(cursor,"select `id` from `file_sample`") - release_pool(conn,cursor) + self.release_pool(conn,cursor) return schema.Files(hits=[file['id'] for file in files_data]) # get ids of all cases from db def get_current_cases(self): - conn,cursor=get_pool() + conn,cursor=self.get_pool() cases_data=self.fetch_results(cursor,"select id from participant") - release_pool(conn,cursor) + self.release_pool(conn,cursor) return schema.RepositoryCases(hits=[case['id'] for case in cases_data]) # get details of case object from db def get_case(self,id): # get entity_participant_id and file info from db - conn,cursor=get_pool() + conn,cursor=self.get_pool() case_data=self.fetch_results(cursor,'''select participant.entity_participant_id, file_sample.* from participant, file_sample where participant.id='''+str(id)+" and file_sample.participant=participant.entity_participant_id") part_id=case_data[0]['entity_participant_id'] @@ -264,7 +262,7 @@ def get_case(self,id): access=case_file['access']) for case_file in case_data])) - release_pool(conn,cursor) + self.release_pool(conn,cursor) return case_object @@ -304,14 +302,14 @@ def get_project_aggregations(self, projects): # get all counts for front page summary from db def get_current_counts(self): - conn,cursor=get_pool() + conn,cursor=self.get_pool() counts_data=self.fetch_results(cursor,'''select count(id) as countid from project union all select count(id) as countid from participant union all select count(id) as countid from sample union all select count(distinct data_format) as countid from file_sample union all select count(id) as countid from file_sample where type="rawFiles" union all select count(id) as countid from file_sample where type="processedFiles"''') - release_pool(conn,cursor) + self.release_pool(conn,cursor) return schema.Count( projects=counts_data[0]['countid'], participants=counts_data[1]['countid'], @@ -411,9 +409,9 @@ def get_facets(self): # get total size of all files def get_cart_file_size(self): - conn,cursor=get_pull() + conn,cursor=self.get_pool() file_data=self.fetch_results(cursor,"select sum(file_size) as sum_size from file_sample") - release_pool(conn,cursor) + self.release_pool(conn,cursor) return schema.FileSize(file_data[0]['sum_size']) data = Data() diff --git a/server.py b/server.py index c735249..6d626e0 100644 --- a/server.py +++ b/server.py @@ -15,8 +15,8 @@ import graphene import schema -import const -import database +#import const +from database import data NAME = "firecloud_graphql" HOST = "0.0.0.0" @@ -58,7 +58,7 @@ def get_schema(name): # add static endpoint for version/status @app.route('/status', methods=["GET"]) def get_version(): - return flask.jsonify(database.VERSION) + return flask.jsonify(data.get_current_version()) # app.debug = True # add end point for graphql gui app.add_url_rule('/test', view_func=flask_graphql.GraphQLView.as_view( From 672689e4fc7b7c72710d4da6f90beac35a82b116 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Fri, 29 Mar 2019 02:40:23 +0000 Subject: [PATCH 25/34] fixing version text in db --- load_local_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/load_local_database.py b/load_local_database.py index 944566d..1a58ba8 100644 --- a/load_local_database.py +++ b/load_local_database.py @@ -256,7 +256,7 @@ def main(): new_version = str(0.1) now = datetime.datetime.now() - release_date ="Date Release "+new_version+" "+ now.strftime("%b %d, %Y") + release_date ="Data Release "+new_version+" - "+ now.strftime("%b %d, %Y") commit ="commit_"+now.strftime("%m%d%Y") query_insert_version ='''INSERT INTO `version` ( commit, From b006e654b09e308f2b0adadebbed064d5dcc206f Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Fri, 29 Mar 2019 15:41:52 +0000 Subject: [PATCH 26/34] fixing typo --- schema.graphql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema.graphql b/schema.graphql index 8a40712..1d39a28 100755 --- a/schema.graphql +++ b/schema.graphql @@ -28,7 +28,7 @@ type Case implements Node { project: Project summary: Summary annotations: CaseAnnotations - files: CaseFile + files: CaseFiles } type MetadataParticipant { From d19b20ba104778f80c5c71f250e03f9e2e1b3eef Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Sat, 30 Mar 2019 23:15:18 +0000 Subject: [PATCH 27/34] file returns file_id as id, not name --- schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema.py b/schema.py index b330cb7..9333c93 100644 --- a/schema.py +++ b/schema.py @@ -174,7 +174,7 @@ class Meta: type = graphene.String() def resolve_file_id(self, info): - return self.name + return self.file_id def resolve_type(self, info): return self.data_format From 202b64c16b36107d56523ec5953d431adac31cd5 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Sat, 30 Mar 2019 23:40:14 +0000 Subject: [PATCH 28/34] combine 2 loops --- database.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/database.py b/database.py index 808eae4..db1b906 100644 --- a/database.py +++ b/database.py @@ -51,9 +51,12 @@ def fetch_results(self,cursor,query): def get_current_projects(self): conn,cursor=self.get_pool() projects_data=self.fetch_results(cursor,"select id from project") - project_object=[self.get_project(project['id'],cursor) for project in projects_data] + # project_object=[self.get_project(project['id'],cursor) for project in projects_data] + project_object=[] for project in projects_data: - self.projects[str(project['id'])]=self.get_project(project['id'],cursor) + getproject=self.get_project(project['id'],cursor) + self.projects[str(project['id'])]=getproject + project_object.append(getproject) self.release_pool(conn,cursor) return project_object From eb0672a7afff6388667a371e646e1db3e22abe0e Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Sun, 31 Mar 2019 00:00:58 +0000 Subject: [PATCH 29/34] making metadata weight lbs and totMETs1 float --- schema.graphql | 4 ++-- schema.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/schema.graphql b/schema.graphql index 1d39a28..6623bc5 100755 --- a/schema.graphql +++ b/schema.graphql @@ -35,8 +35,8 @@ type MetadataParticipant { id: Int participant: Int age_2012: Int - totMETs1: String - weight_lbs: String + totMETs1: Float + weight_lbs: Float } type MetadataSample diff --git a/schema.py b/schema.py index 9333c93..5f467f0 100644 --- a/schema.py +++ b/schema.py @@ -95,8 +95,8 @@ class MetadataParticipant(graphene.ObjectType): id = graphene.Int() participant = graphene.Int() age_2012 = graphene.Int() - totMETs1 = graphene.String() - weight_lbs = graphene.String() + totMETs1 = graphene.Float() + weight_lbs = graphene.Float() class MetadataSample(graphene.ObjectType): id = graphene.Int() From e52ec5bd1e69bc3b403fd0091e9f1320da677ca8 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Tue, 2 Apr 2019 18:44:37 +0000 Subject: [PATCH 30/34] Moving query strings to separate file --- database.py | 124 +++++++++++++++++++-------------------------------- dbqueries.py | 79 ++++++++++++++++++++++++++++++++ schema.py | 4 +- 3 files changed, 127 insertions(+), 80 deletions(-) create mode 100644 dbqueries.py diff --git a/database.py b/database.py index db1b906..5837180 100644 --- a/database.py +++ b/database.py @@ -4,6 +4,7 @@ from mysql.connector import pooling import os import schema +import dbqueries # increments key for aggregations @@ -17,7 +18,7 @@ class Data(object): def __init__(self): self.conn_pool=mysql.connector.connect(use_pure=False, pool_name="portal", - pool_size=10, + pool_size=15, pool_reset_session=True, database='portal_ui', user='biom_mass', @@ -50,8 +51,7 @@ def fetch_results(self,cursor,query): # get all projects from db def get_current_projects(self): conn,cursor=self.get_pool() - projects_data=self.fetch_results(cursor,"select id from project") - # project_object=[self.get_project(project['id'],cursor) for project in projects_data] + projects_data=self.fetch_results(cursor,dbqueries.projects_query) project_object=[] for project in projects_data: getproject=self.get_project(project['id'],cursor) @@ -67,8 +67,7 @@ def get_user(self): # get current version from db def get_current_version(self): conn,cursor=self.get_pool() - version_data=self.fetch_results(cursor, - "select * from version order by updated desc limit 1 ") + version_data=self.fetch_results(cursor,dbqueries.version_query) self.release_pool(conn,cursor) del version_data[0]['updated'] del version_data[0]['id'] @@ -78,16 +77,9 @@ def get_current_version(self): # get project object details from db def get_project(self,id,cursor): - project_data=self.fetch_results(cursor, - "select * from project where id =" + str(id)) + project_data=self.fetch_results(cursor,dbqueries.project_query(id)) project_id=project_data[0]['project_id'] - - counts_query="select count(id) as total from file_sample where project='"+project_id+"'" - counts_query=counts_query+" union all " - counts_query=counts_query+"select count(distinct participant) as total from file_sample where project='"+project_id+"'" - counts_query=counts_query+" union all " - counts_query=counts_query+"select sum(file_size) as total from file_sample where project='"+project_id+"'" - proj_counts_data=self.fetch_results(cursor,counts_query) + proj_counts_data=self.fetch_results(cursor,dbqueries.project_counts_query(project_id)) return schema.Project( id=id, @@ -106,16 +98,12 @@ def get_project(self,id,cursor): # get data categories file and case counts from db for a project or a participant def get_data_categories(self,table,id,cursor): - data_cat_data=self.fetch_results(cursor, - "select distinct data_category from file_sample where "+ table+"='" + str(id)+"'") + data_cat_data=self.fetch_results(cursor,dbqueries.data_cat_query(table,id)) data_cat=[] for item in data_cat_data: - count_query="select count(id) as total from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'" - count_query=count_query+" union all " - count_query=count_query+"select count(distinct participant) as total from file_sample where data_category='"+item['data_category']+"' and "+table+"='"+str(id)+"'" - count_data=self.fetch_results(cursor,count_query) + count_data=self.fetch_results(cursor,dbqueries.data_cat_counts_query(table,id,item['data_category'])) - data_cat.append( + data_cat.append( schema.DataCategories( case_count=count_data[1]['total'], file_count=count_data[0]['total'], @@ -127,14 +115,10 @@ def get_data_categories(self,table,id,cursor): # get experimental strategies file count from db for a project def get_experimental_strategies(self,table,id,cursor): - exp_str_data=self.fetch_results(cursor, - "select distinct experimental_strategy from file_sample where "+ table+"='" + str(id) +"'") + exp_str_data=self.fetch_results(cursor,dbqueries.exp_str_query(table,id)) exp_str=[] for item in exp_str_data: - count_query="select count(id) as total from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'" - count_query=count_query+" union all " - count_query=count_query+ "select count(distinct participant) as total from file_sample where experimental_strategy='"+item['experimental_strategy']+"' and "+table+"='"+str(id)+"'" - count_data=self.fetch_results(cursor,count_query) + count_data=self.fetch_results(cursor,dbqueries.exp_str_counts_query(table,id,item['experimental_strategy'])) exp_str.append( schema.ExperimentalStrategies( case_count=count_data[1]['total'], @@ -149,13 +133,7 @@ def get_experimental_strategies(self,table,id,cursor): def get_file(self,id): conn,cursor=self.get_pool() - file_query='''select file_sample.*, - project.id as p_id, project.project_id as proj_id,project.primary_site, - participant.id as part_id - from file_sample,project,participant where file_sample.id='''+str(id) - file_query=file_query+''' and file_sample.project=project.project_id and - file_sample.participant=participant.entity_participant_id''' - file_data=self.fetch_results(cursor,file_query) + file_data=self.fetch_results(cursor,dbqueries.file_query(id)) file_object=schema.File( id=id, @@ -173,7 +151,6 @@ def get_file(self,id): hits=[schema.FileCase( file_data[0]['part_id'], case_id=file_data[0]['participant'], - #project=self.get_project(file_data[0]['p_id'],cursor), project=self.projects[str(file_data[0]['p_id'])], demographic=schema.Demographic("not hispanic or latino","male","white"), primary_site=file_data[0]['primary_site'])]), @@ -190,14 +167,14 @@ def get_case_annotation(self): # get ids of all files from db def get_current_files(self): conn,cursor=self.get_pool() - files_data=self.fetch_results(cursor,"select `id` from `file_sample`") + files_data=self.fetch_results(cursor,dbqueries.files_query) self.release_pool(conn,cursor) return schema.Files(hits=[file['id'] for file in files_data]) # get ids of all cases from db def get_current_cases(self): conn,cursor=self.get_pool() - cases_data=self.fetch_results(cursor,"select id from participant") + cases_data=self.fetch_results(cursor, dbqueries.cases_query) self.release_pool(conn,cursor) return schema.RepositoryCases(hits=[case['id'] for case in cases_data]) @@ -206,28 +183,14 @@ def get_case(self,id): # get entity_participant_id and file info from db conn,cursor=self.get_pool() - case_data=self.fetch_results(cursor,'''select participant.entity_participant_id, file_sample.* from participant, file_sample - where participant.id='''+str(id)+" and file_sample.participant=participant.entity_participant_id") + case_data=self.fetch_results(cursor,dbqueries.case_query(id)) part_id=case_data[0]['entity_participant_id'] - - counts_query="select count(id) as total from participant where entity_participant_id="+str(part_id) - counts_query=counts_query+" union all " - counts_query=counts_query+ "select count(id) as total from file_sample where participant="+str(part_id) - counts_query=counts_query+" union all " - counts_query= counts_query+"select sum(file_size) as total from file_sample where participant="+str(part_id) - counts_data=self.fetch_results(cursor,counts_query) - - proj_query="select distinct file_sample.project, project.id as p_id, project.primary_site " - proj_query=proj_query+"from file_sample, project where file_sample.participant='" - proj_query=proj_query+str(part_id)+"' and project.project_id=file_sample.project limit 1" - proj_data=self.fetch_results(cursor,proj_query) - + counts_data=self.fetch_results(cursor,dbqueries.case_counts_query(part_id)) + proj_data=self.fetch_results(cursor,dbqueries.case_project_query(part_id)) proj_id=proj_data[0]['p_id'] - # query to get metadata from prticipant table - metadata_part_data=self.fetch_results(cursor, - "select * from participant where entity_participant_id="+str(part_id)) + metadata_part_data=self.fetch_results(cursor,dbqueries.metadata_part_query(part_id)) del metadata_part_data[0]['updated'] metadata_part_data[0]['participant'] = metadata_part_data[0]['entity_participant_id'] del metadata_part_data[0]['entity_participant_id'] @@ -235,20 +198,24 @@ def get_case(self,id): # query to get metadata from sample table - metadata_data=self.fetch_results(cursor, - "select * from sample where participant="+str(part_id)) + metadata_data=self.fetch_results(cursor,dbqueries.metadata_sample_query(part_id)) metadata_list=[] for metadata in metadata_data: del metadata['updated'] metadata_list.append(schema.MetadataSample(**metadata)) + # create file hits list + case_files_list=[] + for case_file in case_data: + del case_file['entity_participant_id'] + case_files_list.append(schema.CaseFile(**case_file)) + case_object=schema.Case(id, case_id=part_id, primary_site=proj_data[0]['primary_site'], demographic=schema.Demographic("not hispanic or latino","male","white"), metadata_participant=metadata_participant, metadata_sample=metadata_list, - # project=self.get_project(proj_id,cursor), project=self.projects[str(proj_id)], summary=schema.Summary( case_count=counts_data[0]['total'], @@ -256,13 +223,7 @@ def get_case(self,id): file_size=counts_data[2]['total'], data_categories=self.get_data_categories("participant",part_id,cursor), experimental_strategies=self.get_experimental_strategies("participant",part_id,cursor)), - files=schema.CaseFiles(hits=[schema.CaseFile( - case_file['id'], - experimental_strategy=case_file['experimental_strategy'], - data_category=case_file['data_category'], - data_format=case_file['data_format'], - platform=case_file['platform'], - access=case_file['access']) for case_file in case_data])) + files=schema.CaseFiles(hits=case_files_list)) self.release_pool(conn,cursor) @@ -306,20 +267,15 @@ def get_project_aggregations(self, projects): def get_current_counts(self): conn,cursor=self.get_pool() - counts_data=self.fetch_results(cursor,'''select count(id) as countid from project - union all select count(id) as countid from participant - union all select count(id) as countid from sample - union all select count(distinct data_format) as countid from file_sample - union all select count(id) as countid from file_sample where type="rawFiles" - union all select count(id) as countid from file_sample where type="processedFiles"''') + counts_data=self.fetch_results(cursor,dbqueries.all_counts_query) self.release_pool(conn,cursor) return schema.Count( - projects=counts_data[0]['countid'], - participants=counts_data[1]['countid'], - samples=counts_data[2]['countid'], - dataFormats=counts_data[3]['countid'], - rawFiles=counts_data[4]['countid'], - processedFiles=counts_data[5]['countid']) + projects=counts_data[0]['total'], + participants=counts_data[1]['total'], + samples=counts_data[2]['total'], + dataFormats=counts_data[3]['total'], + rawFiles=counts_data[4]['total'], + processedFiles=counts_data[5]['total']) @@ -413,9 +369,21 @@ def get_facets(self): # get total size of all files def get_cart_file_size(self): conn,cursor=self.get_pool() - file_data=self.fetch_results(cursor,"select sum(file_size) as sum_size from file_sample") + file_data=self.fetch_results(cursor,dbqueries.files_size_query) self.release_pool(conn,cursor) return schema.FileSize(file_data[0]['sum_size']) + def get_files_total(self): + conn,cursor=self.get_pool() + files_count_data=self.fetch_results(cursor,dbqueries.files_total_query) + self.release_pool(conn,cursor) + return files_count_data[0]['total'] + + def get_cases_total(self): + conn,cursor=self.get_pool() + cases_count_data=self.fetch_results(cursor,dbqueries.cases_total_query) + self.release_pool(conn,cursor) + return cases_count_data[0]['total'] + data = Data() diff --git a/dbqueries.py b/dbqueries.py new file mode 100644 index 0000000..799fc62 --- /dev/null +++ b/dbqueries.py @@ -0,0 +1,79 @@ +all_counts_query='''select count(id) as total from project + union all select count(id) as total from participant + union all select count(id) as total from sample + union all select count(distinct data_format) as total from file_sample + union all select count(id) as total from file_sample where type="rawFiles" + union all select count(id) as total from file_sample where type="processedFiles"''' +projects_query="select id from project" +version_query="select * from version order by updated desc limit 1" +files_query="select id from file_sample" +cases_query="select id from participant" +files_size_query="select sum(file_size) as sum_size from file_sample" +files_total_query="select count(id) as total from file_sample" +cases_total_query="select count(distinct id) as total from participant" + +def project_query(p_id): + return "select * from project where id =" + str(p_id) + +def project_counts_query(p_id): + query="select count(id) as total from file_sample where project='"+p_id+"'" + query=query+" union all " + query=query+"select count(distinct participant) as total from file_sample where project='"+p_id+"'" + query=query+" union all " + query=query+"select sum(file_size) as total from file_sample where project='"+p_id+"'" + return query + +def data_cat_query(table, p_id): + return "select distinct data_category from file_sample where "+ table+"='" + str(p_id)+"'" + +def data_cat_counts_query(table,p_id,data_cat): + query="select count(id) as total from file_sample where data_category='"+data_cat+"' and "+table+"='"+str(p_id)+"'" + query=query+" union all " + query=query+"select count(distinct participant) as total from file_sample where data_category='"+data_cat+"' and "+table+"='"+str(p_id)+"'" + return query + +def exp_str_query(table,p_id): + return "select distinct experimental_strategy from file_sample where "+ table+"='" + str(p_id) +"'" + +def exp_str_counts_query(table,p_id,exp_str): + query="select count(id) as total from file_sample where experimental_strategy='"+exp_str+"' and "+table+"='"+str(p_id)+"'" + query=query+" union all " + query=query+ "select count(distinct participant) as total from file_sample where experimental_strategy='"+exp_str+"' and "+table+"='"+str(p_id)+"'" + return query + +def file_query(f_id): + query='''select file_sample.*, + project.id as p_id, project.project_id as proj_id,project.primary_site, + participant.id as part_id + from file_sample,project,participant where file_sample.id='''+str(f_id) + query=query+''' and file_sample.project=project.project_id and + file_sample.participant=participant.entity_participant_id''' + return query + +def case_query(p_id): + return '''select participant.entity_participant_id, file_sample.id, file_sample.experimental_strategy, + file_sample.data_category, file_sample.platform, file_sample.access,file_sample.data_format + from participant, file_sample + where participant.id='''+str(p_id)+" and file_sample.participant=participant.entity_participant_id" + +def case_counts_query(p_id): + query="select count(id) as total from participant where entity_participant_id="+str(p_id) + query=query+" union all " + query=query+ "select count(id) as total from file_sample where participant="+str(p_id) + query=query+" union all " + query=query+"select sum(file_size) as total from file_sample where participant="+str(p_id) + return query + +def case_project_query(p_id): + query="select distinct file_sample.project, project.id as p_id, project.primary_site " + query=query+"from file_sample, project where file_sample.participant='" + query=query+str(p_id)+"' and project.project_id=file_sample.project limit 1" + return query + + +def metadata_part_query(p_id): + return "select * from participant where entity_participant_id="+str(p_id) + +def metadata_sample_query(p_id): + return "select * from sample where participant="+str(p_id) + diff --git a/schema.py b/schema.py index 5f467f0..245d5cb 100644 --- a/schema.py +++ b/schema.py @@ -197,7 +197,7 @@ class Meta: total = graphene.Int() def resolve_total(self, info): - return len(self.edges) + return data.get_files_total() class Bucket(graphene.ObjectType): doc_count = graphene.Int() @@ -339,7 +339,7 @@ class Meta: total = graphene.Int() def resolve_total(self, info): - return len(self.edges) + return data.get_cases_total() class RepositoryCases(graphene.ObjectType): hits = graphene.relay.ConnectionField(CaseConnection, From 7c9f44637d031b7a19aafa74807bce8c1029dfa9 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Tue, 2 Apr 2019 18:47:38 +0000 Subject: [PATCH 31/34] Comments added --- dbqueries.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dbqueries.py b/dbqueries.py index 799fc62..17e89b6 100644 --- a/dbqueries.py +++ b/dbqueries.py @@ -1,3 +1,7 @@ +# This file holds all queries run in database.py + + + all_counts_query='''select count(id) as total from project union all select count(id) as total from participant union all select count(id) as total from sample From 39bad359e7399763e4020831e7364a07735d6e28 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Wed, 3 Apr 2019 03:32:12 +0000 Subject: [PATCH 32/34] Adding metadata_participant to FileCase, portal ui needs it --- database.py | 20 ++++++++++++++++++-- dbqueries.py | 2 +- schema.graphql | 1 + schema.py | 7 +++++-- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/database.py b/database.py index 5837180..8eb9015 100644 --- a/database.py +++ b/database.py @@ -27,7 +27,10 @@ def __init__(self): self.projects={} def __exit__(self): - self.conn_pool.close() + try: + self.conn_pool.close() + except: + print("No connection to close") # get connection from pool def get_pool(self): @@ -153,6 +156,12 @@ def get_file(self,id): case_id=file_data[0]['participant'], project=self.projects[str(file_data[0]['p_id'])], demographic=schema.Demographic("not hispanic or latino","male","white"), + metadata_participant=schema.MetadataParticipant( + file_data[0]['part_id'], + file_data[0]['participant'], + file_data[0]['age_2012'], + file_data[0]['totMETs1'], + file_data[0]['weight_lbs']), primary_site=file_data[0]['primary_site'])]), file_id=file_data[0]['file_id']) @@ -284,7 +293,9 @@ def get_file_aggregations(self, files): # aggregate file data aggregates = {"data_category": {}, "experimental_strategy": {}, "data_format": {}, "platform": {}, "cases__primary_site": {}, - "cases__project__project_id": {}, "access": {}} + "cases__project__project_id": {}, "cases__metadata_participant__age_2012": {}, + "cases__metadata_participant__totMETs1": {},"cases__metadata_participant__weight_lbs": {}, + "access": {}} for file in files: @@ -298,6 +309,11 @@ def get_file_aggregations(self, files): add_key_increment(aggregates["cases__primary_site"], project.primary_site[0]) add_key_increment(aggregates["cases__project__project_id"], project.project_id) + metadata_participant = file.cases.hits[0].metadata_participant + add_key_increment(aggregates["cases__metadata_participant__age_2012"], metadata_participant.age_2012) + add_key_increment(aggregates["cases__metadata_participant__totMETs1"], metadata_participant.totMETs1) + add_key_increment(aggregates["cases__metadata_participant__weight_lbs"], metadata_participant.weight_lbs) + file_aggregates = schema.FileAggregations( data_category=schema.Aggregations( buckets=[schema.Bucket(doc_count=count, key=key) for key,count in aggregates["data_category"].items()]), diff --git a/dbqueries.py b/dbqueries.py index 17e89b6..fdfb02c 100644 --- a/dbqueries.py +++ b/dbqueries.py @@ -48,7 +48,7 @@ def exp_str_counts_query(table,p_id,exp_str): def file_query(f_id): query='''select file_sample.*, project.id as p_id, project.project_id as proj_id,project.primary_site, - participant.id as part_id + participant.id as part_id, participant.age_2012, participant.totMETs1, participant.weight_lbs from file_sample,project,participant where file_sample.id='''+str(f_id) query=query+''' and file_sample.project=project.project_id and file_sample.participant=participant.entity_participant_id''' diff --git a/schema.graphql b/schema.graphql index 6623bc5..5c73dc9 100755 --- a/schema.graphql +++ b/schema.graphql @@ -223,6 +223,7 @@ type FileCase implements Node { case_id: String project: Project demographic: Demographic + metadata_participant: MetadataParticipant primary_site: String } diff --git a/schema.py b/schema.py index 245d5cb..ab40d3b 100644 --- a/schema.py +++ b/schema.py @@ -135,6 +135,7 @@ class Meta: case_id = graphene.String() project = graphene.Field(Project) demographic = graphene.Field(Demographic) + metadata_participant = graphene.Field(MetadataParticipant) primary_site = graphene.String() class FileCaseConnection(graphene.relay.Connection): @@ -197,7 +198,8 @@ class Meta: total = graphene.Int() def resolve_total(self, info): - return data.get_files_total() + # return data.get_files_total() + return len(self.edges) class Bucket(graphene.ObjectType): doc_count = graphene.Int() @@ -339,7 +341,8 @@ class Meta: total = graphene.Int() def resolve_total(self, info): - return data.get_cases_total() + #return data.get_cases_total() + return len(self.edges) class RepositoryCases(graphene.ObjectType): hits = graphene.relay.ConnectionField(CaseConnection, From 9f737c7787343b42dbe9dffde348f0129ccc3e3f Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Wed, 3 Apr 2019 03:47:20 +0000 Subject: [PATCH 33/34] Fixing primary site issue --- database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database.py b/database.py index 8eb9015..b2130c3 100644 --- a/database.py +++ b/database.py @@ -162,7 +162,7 @@ def get_file(self,id): file_data[0]['age_2012'], file_data[0]['totMETs1'], file_data[0]['weight_lbs']), - primary_site=file_data[0]['primary_site'])]), + primary_site=[file_data[0]['primary_site']])]), file_id=file_data[0]['file_id']) self.release_pool(conn,cursor) @@ -221,7 +221,7 @@ def get_case(self,id): case_object=schema.Case(id, case_id=part_id, - primary_site=proj_data[0]['primary_site'], + primary_site=[proj_data[0]['primary_site']], demographic=schema.Demographic("not hispanic or latino","male","white"), metadata_participant=metadata_participant, metadata_sample=metadata_list, From 9805477aff648e012e0033d7ad063bd42b3f0903 Mon Sep 17 00:00:00 2001 From: Ana Mailyan Date: Wed, 3 Apr 2019 04:09:49 +0000 Subject: [PATCH 34/34] primary site fix again --- database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database.py b/database.py index b2130c3..327516a 100644 --- a/database.py +++ b/database.py @@ -221,7 +221,7 @@ def get_case(self,id): case_object=schema.Case(id, case_id=part_id, - primary_site=[proj_data[0]['primary_site']], + primary_site=proj_data[0]['primary_site'], demographic=schema.Demographic("not hispanic or latino","male","white"), metadata_participant=metadata_participant, metadata_sample=metadata_list, @@ -350,7 +350,7 @@ def get_case_aggregations(self, cases): add_key_increment(aggregates["metadata_participant__age_2012"],case.metadata_participant.age_2012) add_key_increment(aggregates["metadata_participant__totMETs1"],case.metadata_participant.totMETs1) add_key_increment(aggregates["metadata_participant__weight_lbs"], case.metadata_participant.weight_lbs) - add_key_increment(aggregates["primary_site"], case.primary_site[0]) + add_key_increment(aggregates["primary_site"], case.primary_site) add_key_increment(aggregates["project__project_id"], case.project.project_id) add_key_increment(aggregates["project__program__name"], case.project.program.name)