From 5d6a9aebbbb5170525dae12c564ffe5ae75bc199 Mon Sep 17 00:00:00 2001 From: Jover Date: Tue, 18 Feb 2020 14:26:15 -0800 Subject: [PATCH 1/7] Add warehouse views for PUMA and neighborhood_district --- schema/deploy/warehouse/views.sql | 12 ++++++++++ schema/deploy/warehouse/views@2020-02-06.sql | 25 ++++++++++++++++++++ schema/revert/warehouse/views.sql | 25 +++++++++++++++++--- schema/revert/warehouse/views@2020-02-06.sql | 8 +++++++ schema/sqitch.plan | 2 ++ schema/verify/warehouse/views@2020-02-06.sql | 7 ++++++ 6 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 schema/deploy/warehouse/views@2020-02-06.sql create mode 100644 schema/revert/warehouse/views@2020-02-06.sql create mode 100644 schema/verify/warehouse/views@2020-02-06.sql diff --git a/schema/deploy/warehouse/views.sql b/schema/deploy/warehouse/views.sql index 6accc6384..295902a55 100644 --- a/schema/deploy/warehouse/views.sql +++ b/schema/deploy/warehouse/views.sql @@ -21,5 +21,17 @@ create or replace view warehouse.tract as comment on view warehouse.tract is 'View of all tract-scale locations; for convenience when joining'; +create or replace view warehouse.puma as + select * from warehouse.location where scale = 'puma'; + +comment on view warehouse.puma is + 'View of all PUMA-scale locations; for convenience when joining'; + +create or replace view warehouse.neighborhood_district as + select * from warehouse.location where scale = 'neighborhood_district'; + +comment on view warehouse.neighborhood_district is + 'View of all neighborhood-district-scale locations; for convenience when joining'; + commit; diff --git a/schema/deploy/warehouse/views@2020-02-06.sql b/schema/deploy/warehouse/views@2020-02-06.sql new file mode 100644 index 000000000..6accc6384 --- /dev/null +++ b/schema/deploy/warehouse/views@2020-02-06.sql @@ -0,0 +1,25 @@ +-- Deploy seattleflu/id3c-customizations:views to pg +-- requires: seattleflu/schema:warehouse/location + +-- Hello! All custom views are defined here. Rework this change with Sqitch +-- to change a view definition or add new views. This workflow helps keep +-- inter-view dependencies manageable. + +begin; + + +create or replace view warehouse.address as + select * from warehouse.location where scale = 'address'; + +comment on view warehouse.address is + 'View of all address-scale locations; for convenience when joining'; + + +create or replace view warehouse.tract as + select * from warehouse.location where scale = 'tract'; + +comment on view warehouse.tract is + 'View of all tract-scale locations; for convenience when joining'; + + +commit; diff --git a/schema/revert/warehouse/views.sql b/schema/revert/warehouse/views.sql index 7702e0bae..d976fc7ad 100644 --- a/schema/revert/warehouse/views.sql +++ b/schema/revert/warehouse/views.sql @@ -1,8 +1,27 @@ --- Revert seattleflu/id3c-customizations:views from pg +-- Deploy seattleflu/id3c-customizations:views to pg +-- requires: seattleflu/schema:warehouse/location + +-- Hello! All custom views are defined here. Rework this change with Sqitch +-- to change a view definition or add new views. This workflow helps keep +-- inter-view dependencies manageable. begin; -drop view warehouse.address; -drop view warehouse.tract; +drop view warehouse.puma; +drop view warehouse.neighborhood_district; + +create or replace view warehouse.address as + select * from warehouse.location where scale = 'address'; + +comment on view warehouse.address is + 'View of all address-scale locations; for convenience when joining'; + + +create or replace view warehouse.tract as + select * from warehouse.location where scale = 'tract'; + +comment on view warehouse.tract is + 'View of all tract-scale locations; for convenience when joining'; + commit; diff --git a/schema/revert/warehouse/views@2020-02-06.sql b/schema/revert/warehouse/views@2020-02-06.sql new file mode 100644 index 000000000..7702e0bae --- /dev/null +++ b/schema/revert/warehouse/views@2020-02-06.sql @@ -0,0 +1,8 @@ +-- Revert seattleflu/id3c-customizations:views from pg + +begin; + +drop view warehouse.address; +drop view warehouse.tract; + +commit; diff --git a/schema/sqitch.plan b/schema/sqitch.plan index cd29636c4..8d39bd68d 100644 --- a/schema/sqitch.plan +++ b/schema/sqitch.plan @@ -78,3 +78,5 @@ warehouse/target/data [warehouse/target/data@2020-02-06] 2020-02-20T17:41:10Z Jo warehouse/site/data [warehouse/site/data@2020-02-20] 2020-02-21T19:49:28Z Kairsten Fay # Update site details for self-test and swabNSend @2020-02-21 2020-02-21T19:55:54Z Kairsten Fay # Schema as of 21 Feb 2020 + +warehouse/views [warehouse/views@2020-02-06] 2020-02-18T22:20:31Z Jover Lee # Add PUMA and neighborhood scale location views diff --git a/schema/verify/warehouse/views@2020-02-06.sql b/schema/verify/warehouse/views@2020-02-06.sql new file mode 100644 index 000000000..91393ae2f --- /dev/null +++ b/schema/verify/warehouse/views@2020-02-06.sql @@ -0,0 +1,7 @@ +-- Verify seattleflu/id3c-customizations:views on pg + +begin; + + + +rollback; From 1fa327b1d3a060182700ddba6b6660eb7106a602 Mon Sep 17 00:00:00 2001 From: Jover Date: Mon, 24 Feb 2020 11:07:30 -0800 Subject: [PATCH 2/7] add shipping.incidence_model_observation_v4 Exports residenct PUMA and neighborhood district instead of census tract --- schema/deploy/shipping/views.sql | 81 +++ schema/deploy/shipping/views@2020-02-21.sql | 672 ++++++++++++++++++++ schema/revert/shipping/views.sql | 80 ++- schema/revert/shipping/views@2020-02-21.sql | 600 +++++++++++++++++ schema/sqitch.plan | 1 + schema/verify/shipping/views.sql | 5 + schema/verify/shipping/views@2020-02-21.sql | 77 +++ 7 files changed, 1515 insertions(+), 1 deletion(-) create mode 100644 schema/deploy/shipping/views@2020-02-21.sql create mode 100644 schema/revert/shipping/views@2020-02-21.sql create mode 100644 schema/verify/shipping/views@2020-02-21.sql diff --git a/schema/deploy/shipping/views.sql b/schema/deploy/shipping/views.sql index e6ed1eee6..c58629eec 100644 --- a/schema/deploy/shipping/views.sql +++ b/schema/deploy/shipping/views.sql @@ -669,4 +669,85 @@ comment on view shipping.sample_with_best_available_encounter_data_v1 is 'Version 1 of view of warehoused samples and their best available encounter date and site details important for metrics calculations'; +create or replace view shipping.incidence_model_observation_v4 as + + select encounter.identifier as encounter, + + to_char((encountered at time zone 'US/Pacific')::date, 'IYYY-"W"IW') as encountered_week, + + site.details->>'type' as site_type, + site.details->>'category' as site_category, + + individual.identifier as individual, + individual.sex, + + age_bin_fine_v2.range as age_range_fine, + age_in_years(lower(age_bin_fine_v2.range)) as age_range_fine_lower, + age_in_years(upper(age_bin_fine_v2.range)) as age_range_fine_upper, + + age_bin_coarse_v2.range as age_range_coarse, + age_in_years(lower(age_bin_coarse_v2.range)) as age_range_coarse_lower, + age_in_years(upper(age_bin_coarse_v2.range)) as age_range_coarse_upper, + + residence_puma, + residence_neighborhood_district, + + coalesce(encounter_responses.flu_shot,fhir.vaccine) as flu_shot, + coalesce(encounter_responses.symptoms,fhir.symptoms) as symptoms, + + sample.identifier as sample + + from warehouse.encounter + join warehouse.individual using (individual_id) + join warehouse.site using (site_id) + left join warehouse.sample using (encounter_id) + left join shipping.age_bin_fine_v2 on age_bin_fine_v2.range @> age + left join shipping.age_bin_coarse_v2 on age_bin_coarse_v2.range @> age + left join shipping.fhir_encounter_details_v1 as fhir using (encounter_id) + left join ( + select + encounter_id, + tract.hierarchy->'puma' as residence_puma, + tract.hierarchy->'neighborhood_district' as residence_neighborhood_district + from warehouse.encounter_location + left join warehouse.location as address using (location_id) + left join warehouse.tract as tract on (address.hierarchy -> 'tract') = tract.identifier + where relation = 'residence' + or relation = 'lodging' + ) as residence using (encounter_id), + + lateral ( + -- XXX TODO: The data in this subquery will be modeled better in the + -- future and the verbosity of extracting data from the JSON details + -- document will go away. + -- -trs, 22 March 2019 + + select -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(nullif(responses."FluShot"[1], 'doNotKnow'), 'dontKnow')::bool as flu_shot, + + -- XXX FIXME: Remove duplicate value collapsing when we're no + -- longer affected by this known Audere data quality issue. + array_distinct(responses."Symptoms") as symptoms + + from jsonb_to_record(encounter.details->'responses') + as responses ( + "FluShot" text[], + "Symptoms" text[])) + as encounter_responses + + order by encountered; + +comment on view shipping.incidence_model_observation_v4 is + 'Version 4 of view of warehoused encounters and important questionnaire responses for modeling and viz teams'; + +revoke all + on shipping.incidence_model_observation_v4 + from "incidence-modeler"; + +grant select + on shipping.incidence_model_observation_v4 + to "incidence-modeler"; + + commit; diff --git a/schema/deploy/shipping/views@2020-02-21.sql b/schema/deploy/shipping/views@2020-02-21.sql new file mode 100644 index 000000000..e6ed1eee6 --- /dev/null +++ b/schema/deploy/shipping/views@2020-02-21.sql @@ -0,0 +1,672 @@ +-- Deploy seattleflu/id3c-customizations:shipping/views to pg +-- requires: seattleflu/schema:shipping/schema + +-- Hello! All shipping views are defined here. Rework this change with Sqitch +-- to change a view definition or add new views. This workflow helps keep +-- inter-view dependencies manageable. + +begin; + +-- This view is versioned as a hedge against future changes. Changing this +-- view in place is fine as long as changes are backwards compatible. Think of +-- the version number as the major part of a semantic versioning scheme. If +-- there needs to be a lag between view development and consumers being +-- updated, copy the view definition into v2 and make changes there. + +create or replace view shipping.reportable_condition_v1 as + + with reportable as ( + select array_agg(lineage) as lineages + from warehouse.organism + where details @> '{"report_to_public_health":true}' + ) + + select + presence_absence.presence_absence_id, + organism.lineage::text, + sample.identifier as sample, + barcode, + site.identifier as site, + presence_absence.details->>'reporting_log' as reporting_log, + sample.details->'_provenance'->>'workbook' as workbook, + sample.details->'_provenance'->>'sheet' as sheet, + sample.details->>'sample_origin' as sample_origin, + sample.details->>'swab_site' as swab_site + + from warehouse.presence_absence + join warehouse.target using (target_id) + join warehouse.organism using (organism_id) + join warehouse.sample using (sample_id) + join warehouse.identifier on ( + sample.identifier = cast(identifier.uuid as text)) + left join warehouse.encounter using (encounter_id) + left join warehouse.site using (site_id) + + where organism.lineage <@ (table reportable) + and present + + order by encountered desc; + + +drop view shipping.metadata_for_augur_build_v2; +create or replace view shipping.metadata_for_augur_build_v2 as + + select sample as strain, + cast(encountered as date) as date, + 'seattle' as region, + -- XXX TODO: Change to PUMA and neighborhoods + residence_census_tract as location, + 'Seattle Flu Study' as authors, + age_range_coarse, + case age_range_coarse <@ '[0 mon, 18 years)'::intervalrange + when 't' then 'child' + when 'f' then 'adult' + end as age_category, + case + when site_type in ('childrensHospital', 'childrensClinic', 'childrensHospital', 'clinic', 'hospital', 'retrospective') then 'clinical' + when site_type in ('childcare' , 'collegeCampus' , 'homelessShelter' , 'port', 'publicSpace', 'workplace') then 'community' + end as site_category, + residence_census_tract, + flu_shot, + sex + + from shipping.incidence_model_observation_v3 + join warehouse.encounter on encounter.identifier = incidence_model_observation_v3.encounter; + +comment on view shipping.metadata_for_augur_build_v2 is + 'View of metadata necessary for SFS augur build'; + + +create or replace view shipping.genomic_sequences_for_augur_build_v1 as + + select distinct on (sample.identifier, organism.lineage, segment) + sample.identifier as sample, + organism.lineage as organism, + genomic_sequence.segment, + round(((length(replace(seq, 'N', '')) * 1.0 / length(seq))), 4) as coverage, + genomic_sequence.seq + from warehouse.sample + join warehouse.consensus_genome using (sample_id) + join warehouse.genomic_sequence using (consensus_genome_id) + join warehouse.organism using (organism_id) + + order by sample.identifier, organism.lineage, segment, coverage desc; + +comment on view shipping.genomic_sequences_for_augur_build_v1 is + 'View of genomic sequences for SFS augur build'; + + +create or replace view shipping.flu_assembly_jobs_v1 as + + select sample.identifier as sfs_uuid, + sample.details ->> 'nwgc_id' as nwgc_id, + sequence_read_set.urls, + target.identifier as target, + o1.lineage as target_linked_organism, + coalesce(o2.lineage, o1.lineage) as assembly_job_organism, + sequence_read_set.details + + from warehouse.sequence_read_set + join warehouse.sample using (sample_id) + join warehouse.presence_absence using (sample_id) + join warehouse.target using (target_id) + join warehouse.organism o1 using (organism_id) + -- Reason for o2 join: For pan-subtype targets like Flu_A_pan, + -- we want to spread out to the more specific lineages for assembly jobs. + -- Since this view is Flu specific, we are hard-coding it to spread just one level down. + left join warehouse.organism o2 on (o2.lineage ~ concat(o1.lineage::text, '.*{1}')::lquery) + + where o1.lineage ~ 'Influenza.*' + and present + -- The sequence read set details currently holds the state of assembly jobs for each lineage. + -- So if null, then definitely no jobs have been completed. + -- If not null, then check if the assembly job organism matches any key in the sequence read set details. + and (sequence_read_set.details is null + or not sequence_read_set.details ? coalesce(o2.lineage, o1.lineage)::text) + + order by sample.details ->> 'nwgc_id'; + +comment on view shipping.flu_assembly_jobs_v1 is + 'View of flu jobs that still need to be run through the assembly pipeline'; + + +create or replace view shipping.return_results_v1 as + + select barcode, + case + when sample_id is null then 'notReceived' + when sample_id is not null and count(present) = 0 then 'processing' + when count(present) > 0 then 'complete' + end as status, + -- We only return the top level organisms for results so we want to omit subtypes + array_agg(distinct subpath(organism, 0, 1)::text) + filter (where present and organism is not null) as organisms_present, + array_agg(distinct subpath(organism, 0, 1)::text) + filter (where not present and organism is not null) as organisms_absent + + from warehouse.identifier + join warehouse.identifier_set using (identifier_set_id) + left join warehouse.sample on uuid::text = sample.collection_identifier + left join shipping.presence_absence_result_v1 on sample.identifier = presence_absence_result_v1.sample + + --These are all past and current collection identifier sets not including self-test + where identifier_set.name in ('collections-seattleflu.org', + 'collections-kiosks', + 'collections-environmental', + 'collections-swab&send', + 'collections-household-observation', + 'collections-household-intervention') and + (organism is null or + -- We only return results for these organisms, so omit all other presence/absence results + organism <@ '{"Adenovirus", "Human_coronavirus", "Enterovirus", "Influenza", "Human_metapneumovirus", "Human_parainfluenza", "Rhinovirus", "RSV"}'::ltree[]) + group by barcode, sample_id + order by barcode; + +comment on view shipping.return_results_v1 is + 'View of barcodes and presence/absence results for return of results on website'; + + +create or replace view shipping.fhir_encounter_details_v1 as + + with + fhir as ( + select encounter_id, + case + when details -> 'Condition' is not null then array_agg(distinct condition."id" order by condition."id") + else null + end as symptoms, + jsonb_object_agg(response."linkId", response.answer order by response.answer) filter (where response."linkId" is not null) as responses + from warehouse.encounter + left join jsonb_to_recordset(details -> 'Condition') as condition("id" text) on true + left join jsonb_to_recordset(details -> 'QuestionnaireResponse') as q("item" jsonb) on true + left join jsonb_to_recordset(q."item") as response("linkId" text, "answer" jsonb) on true + group by encounter_id + ), + + vaccine as ( + --Builds vaccine jsonb object without null values that spans multiple rows. + --Expand the rows into key/value pairs with jsonb_each and + --aggregate that back into a single jsonb value. + -- -Jover 10 Jan 2020 + select encounter_id, + jsonb_object_agg(t.k, t.v order by t.v) filter (where t.k is not null) as vaccine + from ( + select encounter_id, + jsonb_strip_nulls( + jsonb_build_object( + 'vaccine', vaccine."valueBoolean", + 'vaccine_date', vaccine."valueDate")) as vaccine_obj + from fhir, + jsonb_to_record(responses) as response("vaccine" jsonb), + jsonb_to_recordset(response."vaccine") as vaccine("valueBoolean" bool, "valueDate" text)) as vaccine, + jsonb_each(vaccine.vaccine_obj) as t(k,v) + group by encounter_id + ), + + array_responses as ( + select + encounter_id, + case + when responses -> 'race' is not null then array_agg(coalesce(race."valueCoding" ->> 'code', race."valueString") order by race."valueString") + else null + end as race, + case + when responses -> 'insurance' is not null then array_agg(insurance."valueString" order by insurance."valueString") + else null + end as insurance + from fhir + left join jsonb_to_record(responses) as response("race" jsonb, "insurance" jsonb) on true + left join jsonb_to_recordset(response."race") as race ("valueCoding" jsonb, "valueString" text) on true + left join jsonb_to_recordset(response."insurance") as insurance ("valueString" text) on true + group by encounter_id, responses + ), + + boolean_responses as ( + select + encounter_id, + ethnicity."valueBoolean" as hispanic_or_latino, + travel_countries."valueBoolean" as travel_countries, + travel_states."valueBoolean" as travel_states + from fhir + left join jsonb_to_record(responses) as response("ethnicity" jsonb, "travel_countries" jsonb, "travel_states" jsonb) on true + left join jsonb_to_recordset(response."ethnicity") as ethnicity("valueBoolean" bool) on true + left join jsonb_to_recordset(response."travel_countries") as travel_countries("valueBoolean" bool) on true + left join jsonb_to_recordset(response."travel_states") as travel_states("valueBoolean" bool) on true + ) + + + select + encounter_id, + symptoms, + (vaccine ->> 'vaccine')::bool as vaccine, + vaccine ->> 'vaccine_date' as vaccine_date, + race, + insurance, + hispanic_or_latino, + travel_countries, + travel_states + from fhir + left join vaccine using (encounter_id) + left join array_responses using (encounter_id) + left join boolean_responses using (encounter_id); + +comment on view shipping.fhir_encounter_details_v1 is + 'A view of encounter details that are in FHIR format'; + +revoke all + on shipping.fhir_encounter_details_v1 + from "incidence-modeler"; + +grant select + on shipping.fhir_encounter_details_v1 + to "incidence-modeler"; + + +create or replace view shipping.incidence_model_observation_v1 as + + select encounter.identifier as encounter, + + (encountered at time zone 'US/Pacific')::date as encountered_date, + to_char((encountered at time zone 'US/Pacific')::date, 'IYYY-"W"IW') as encountered_week, + + site.details->>'type' as site_type, + + individual.identifier as individual, + individual.sex, + + -- Reporting 90 is more accurate than reporting nothing, and it will + -- never be a real value in our dataset. + -- + -- XXX TODO: This will be pre-processed out of the JSON in the future. + ceiling(age_in_years(age))::int as age, + + age_bin_fine.range as age_range_fine, + lower(age_bin_fine.range) as age_range_fine_lower, + upper(age_bin_fine.range) as age_range_fine_upper, + + age_bin_coarse.range as age_range_coarse, + lower(age_bin_coarse.range) as age_range_coarse_lower, + upper(age_bin_coarse.range) as age_range_coarse_upper, + + residence_census_tract, + work_census_tract, + + coalesce(encounter_responses.flu_shot,fhir.vaccine) as flu_shot, + coalesce(encounter_responses.symptoms,fhir.symptoms) as symptoms, + coalesce(encounter_responses.race,fhir.race) as race, + coalesce(encounter_responses.hispanic_or_latino,fhir.hispanic_or_latino) as hispanic_or_latino, + + sample.identifier as sample + + from warehouse.encounter + join warehouse.individual using (individual_id) + join warehouse.site using (site_id) + left join warehouse.sample using (encounter_id) + left join shipping.age_bin_fine on age_bin_fine.range @> ceiling(age_in_years(age))::int + left join shipping.age_bin_coarse on age_bin_coarse.range @> ceiling(age_in_years(age))::int + left join shipping.fhir_encounter_details_v1 as fhir using (encounter_id) + left join ( + select encounter_id, hierarchy->'tract' as residence_census_tract + from warehouse.encounter_location + left join warehouse.location using (location_id) + where relation = 'residence' + or relation = 'lodging' + ) as residence using (encounter_id) + left join ( + select encounter_id, hierarchy->'tract' as work_census_tract + from warehouse.encounter_location + left join warehouse.location using (location_id) + where relation = 'workplace' + ) as workplace using (encounter_id), + + lateral ( + -- XXX TODO: The data in this subquery will be modeled better in the + -- future and the verbosity of extracting data from the JSON details + -- document will go away. + -- -trs, 22 March 2019 + + select -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(nullif(responses."FluShot"[1], 'doNotKnow'), 'dontKnow')::bool as flu_shot, + + -- XXX FIXME: Remove duplicate value collapsing when we're no + -- longer affected by this known Audere data quality issue. + array_distinct(responses."Symptoms") as symptoms, + array_distinct(responses."Race") as race, + + -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(responses."HispanicLatino"[1], 'preferNotToSay')::bool as hispanic_or_latino + + from jsonb_to_record(encounter.details->'responses') + as responses ( + "FluShot" text[], + "Symptoms" text[], + "Race" text[], + "HispanicLatino" text[])) + as encounter_responses + + order by encountered; + +comment on view shipping.incidence_model_observation_v1 is + 'View of warehoused encounters and important questionnaire responses for modeling and viz teams'; + +revoke all + on shipping.incidence_model_observation_v1 + from "incidence-modeler"; + +grant select + on shipping.incidence_model_observation_v1 + to "incidence-modeler"; + + +create or replace view shipping.incidence_model_observation_v2 as + + select encounter.identifier as encounter, + + (encountered at time zone 'US/Pacific')::date as encountered_date, + to_char((encountered at time zone 'US/Pacific')::date, 'IYYY-"W"IW') as encountered_week, + + site.identifier as site, + site.details->>'type' as site_type, + + individual.identifier as individual, + individual.sex, + + age_in_years(age) as age, + + age_bin_fine_v2.range as age_range_fine, + age_in_years(lower(age_bin_fine_v2.range)) as age_range_fine_lower, + age_in_years(upper(age_bin_fine_v2.range)) as age_range_fine_upper, + + age_bin_coarse_v2.range as age_range_coarse, + age_in_years(lower(age_bin_coarse_v2.range)) as age_range_coarse_lower, + age_in_years(upper(age_bin_coarse_v2.range)) as age_range_coarse_upper, + + residence_census_tract, + work_census_tract, + + coalesce(encounter_responses.flu_shot,fhir.vaccine) as flu_shot, + coalesce(encounter_responses.symptoms,fhir.symptoms) as symptoms, + coalesce(encounter_responses.race,fhir.race) as race, + coalesce(encounter_responses.hispanic_or_latino,fhir.hispanic_or_latino) as hispanic_or_latino, + + sample.identifier as sample + + from warehouse.encounter + join warehouse.individual using (individual_id) + join warehouse.site using (site_id) + left join warehouse.sample using (encounter_id) + left join shipping.age_bin_fine_v2 on age_bin_fine_v2.range @> age + left join shipping.age_bin_coarse_v2 on age_bin_coarse_v2.range @> age + left join shipping.fhir_encounter_details_v1 as fhir using (encounter_id) + left join ( + select encounter_id, hierarchy->'tract' as residence_census_tract + from warehouse.encounter_location + left join warehouse.location using (location_id) + where relation = 'residence' + or relation = 'lodging' + ) as residence using (encounter_id) + left join ( + select encounter_id, hierarchy->'tract' as work_census_tract + from warehouse.encounter_location + left join warehouse.location using (location_id) + where relation = 'workplace' + ) as workplace using (encounter_id), + + lateral ( + -- XXX TODO: The data in this subquery will be modeled better in the + -- future and the verbosity of extracting data from the JSON details + -- document will go away. + -- -trs, 22 March 2019 + + select -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(nullif(responses."FluShot"[1], 'doNotKnow'), 'dontKnow')::bool as flu_shot, + + -- XXX FIXME: Remove duplicate value collapsing when we're no + -- longer affected by this known Audere data quality issue. + array_distinct(responses."Symptoms") as symptoms, + array_distinct(responses."Race") as race, + + -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(responses."HispanicLatino"[1], 'preferNotToSay')::bool as hispanic_or_latino + + from jsonb_to_record(encounter.details->'responses') + as responses ( + "FluShot" text[], + "Symptoms" text[], + "Race" text[], + "HispanicLatino" text[])) + as encounter_responses + + order by encountered; + +comment on view shipping.incidence_model_observation_v2 is + 'Version 2 of view of warehoused encounters and important questionnaire responses for modeling and viz teams'; + +revoke all + on shipping.incidence_model_observation_v2 + from "incidence-modeler"; + +grant select + on shipping.incidence_model_observation_v2 + to "incidence-modeler"; + + +create or replace view shipping.observation_with_presence_absence_result_v1 as + + select target, + present, + present::int as presence, + observation.*, + organism + from shipping.incidence_model_observation_v2 as observation + join shipping.presence_absence_result_v1 using (sample) + order by site, encounter, sample, target; + +comment on view shipping.observation_with_presence_absence_result_v1 is + 'Joined view of shipping.incidence_model_observation_v2 and shipping.presence_absence_result_v1'; + + +create or replace view shipping.incidence_model_observation_v3 as + + select encounter.identifier as encounter, + + to_char((encountered at time zone 'US/Pacific')::date, 'IYYY-"W"IW') as encountered_week, + + site.details->>'type' as site_type, + + individual.identifier as individual, + individual.sex, + + age_bin_fine_v2.range as age_range_fine, + age_in_years(lower(age_bin_fine_v2.range)) as age_range_fine_lower, + age_in_years(upper(age_bin_fine_v2.range)) as age_range_fine_upper, + + age_bin_coarse_v2.range as age_range_coarse, + age_in_years(lower(age_bin_coarse_v2.range)) as age_range_coarse_lower, + age_in_years(upper(age_bin_coarse_v2.range)) as age_range_coarse_upper, + + residence_census_tract, + + coalesce(encounter_responses.flu_shot,fhir.vaccine) as flu_shot, + coalesce(encounter_responses.symptoms,fhir.symptoms) as symptoms, + + sample.identifier as sample + + from warehouse.encounter + join warehouse.individual using (individual_id) + join warehouse.site using (site_id) + left join warehouse.sample using (encounter_id) + left join shipping.age_bin_fine_v2 on age_bin_fine_v2.range @> age + left join shipping.age_bin_coarse_v2 on age_bin_coarse_v2.range @> age + left join shipping.fhir_encounter_details_v1 as fhir using (encounter_id) + left join ( + select encounter_id, hierarchy->'tract' as residence_census_tract + from warehouse.encounter_location + left join warehouse.location using (location_id) + where relation = 'residence' + or relation = 'lodging' + ) as residence using (encounter_id), + + lateral ( + -- XXX TODO: The data in this subquery will be modeled better in the + -- future and the verbosity of extracting data from the JSON details + -- document will go away. + -- -trs, 22 March 2019 + + select -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(nullif(responses."FluShot"[1], 'doNotKnow'), 'dontKnow')::bool as flu_shot, + + -- XXX FIXME: Remove duplicate value collapsing when we're no + -- longer affected by this known Audere data quality issue. + array_distinct(responses."Symptoms") as symptoms + + from jsonb_to_record(encounter.details->'responses') + as responses ( + "FluShot" text[], + "Symptoms" text[])) + as encounter_responses + + order by encountered; + +comment on view shipping.incidence_model_observation_v3 is + 'Version 3 of view of warehoused encounters and important questionnaire responses for modeling and viz teams'; + +revoke all + on shipping.incidence_model_observation_v3 + from "incidence-modeler"; + +grant select + on shipping.incidence_model_observation_v3 + to "incidence-modeler"; + + +create or replace view shipping.observation_with_presence_absence_result_v2 as + + select target, + present, + present::int as presence, + observation.*, + organism + from shipping.incidence_model_observation_v3 as observation + join shipping.presence_absence_result_v1 using (sample) + order by site_type, encounter, sample, target; + +comment on view shipping.observation_with_presence_absence_result_v2 is + 'Joined view of shipping.incidence_model_observation_v3 and shipping.presence_absence_result_v1'; + + +create or replace view shipping.metadata_for_augur_build_v3 as + + select sample.identifier as strain, + coalesce( + encountered_date, + case + when date_or_null(sample.details->>'date') <= current_date + then date_or_null(sample.details->>'date') + end + ) as date, + 'seattle' as region, + -- XXX TODO: Change to PUMA and neighborhoods + residence_census_tract as location, + 'Seattle Flu Study' as authors, + age_range_coarse, + case age_range_coarse <@ '[0 mon, 18 years)'::intervalrange + when 't' then 'child' + when 'f' then 'adult' + end as age_category, + warehouse.site.details->>'category' as site_category, + residence_census_tract, + flu_shot, + sex + + from warehouse.sample + left join shipping.incidence_model_observation_v2 on sample.identifier = incidence_model_observation_v2.sample + left join warehouse.site on site.identifier = incidence_model_observation_v2.site + + where sample.identifier is not null; + +comment on view shipping.metadata_for_augur_build_v3 is + 'View of metadata necessary for SFS augur build'; + + +create or replace view shipping.sample_with_best_available_encounter_data_v1 as + + with specimen_manifest_data as ( + select + sample_id, + date_or_null(details->>'date') as collection_date, + trim(both ' ' from details->>'swab_site') as swab_site, + trim(both ' ' from details->>'sample_origin') as sample_origin + from + warehouse.sample + ), + + site_details as ( + select + site_id, + site.identifier as site, + site.details->>'type' as site_type, + site.details->>'category' as site_category, + coalesce(site.details->>'swab_site', site.details->>'sample_origin') as manifest_regex + from warehouse.site + ), + + samples_with_manifest_data as ( + select + sample_id, + site_id, + coalesce(encountered::date, collection_date) as best_available_encounter_date, + + coalesce( + case + -- Environmental samples must be processed first, because they're + -- often taken at existing human swab sites + when manifest.sample_origin = 'es' + then 'environmental' + else manifest.swab_site + end, + + manifest.sample_origin + ) as site_manifest_details, + + site_id is not null as has_encounter_data + + from warehouse.sample + left join warehouse.encounter using (encounter_id) + left join specimen_manifest_data as manifest using (sample_id) + ) + + select + sample_id, + sample.identifier as sample, + has_encounter_data, + best_available_encounter_date, + + case + when best_available_encounter_date < '2019-10-01'::date then 'Y1' + when best_available_encounter_date < '2020-10-01'::date then 'Y2' + else null + end as season, + + coalesce(site.site_id, site_details.site_id) as best_available_site_id, + coalesce(site.identifier, site_details.site) as best_available_site, + coalesce(site.details->>'type', site_type) as best_available_site_type, + coalesce(site.details->>'category', site_category) as best_available_site_category + + from warehouse.sample + left join samples_with_manifest_data using (sample_id) + left join site_details on (site_manifest_details similar to manifest_regex) + left join warehouse.site on (samples_with_manifest_data.site_id = site.site_id) + where sample.identifier is not null + ; + +comment on view shipping.sample_with_best_available_encounter_data_v1 is + 'Version 1 of view of warehoused samples and their best available encounter date and site details important for metrics calculations'; + + +commit; diff --git a/schema/revert/shipping/views.sql b/schema/revert/shipping/views.sql index c65fd95f3..b14822a4e 100644 --- a/schema/revert/shipping/views.sql +++ b/schema/revert/shipping/views.sql @@ -594,7 +594,85 @@ comment on view shipping.metadata_for_augur_build_v3 is 'View of metadata necessary for SFS augur build'; -drop view shipping.sample_with_best_available_encounter_data_v1; +create or replace view shipping.sample_with_best_available_encounter_data_v1 as + + with specimen_manifest_data as ( + select + sample_id, + date_or_null(details->>'date') as collection_date, + trim(both ' ' from details->>'swab_site') as swab_site, + trim(both ' ' from details->>'sample_origin') as sample_origin + from + warehouse.sample + ), + + site_details as ( + select + site_id, + site.identifier as site, + site.details->>'type' as site_type, + site.details->>'category' as site_category, + coalesce(site.details->>'swab_site', site.details->>'sample_origin') as manifest_regex + from warehouse.site + ), + + samples_with_manifest_data as ( + select + sample_id, + site_id, + coalesce(encountered::date, collection_date) as best_available_encounter_date, + + coalesce( + case + -- Environmental samples must be processed first, because they're + -- often taken at existing human swab sites + when manifest.sample_origin = 'es' + then 'environmental' + else manifest.swab_site + end, + + manifest.sample_origin + ) as site_manifest_details, + + site_id is not null as has_encounter_data + + from warehouse.sample + left join warehouse.encounter using (encounter_id) + left join specimen_manifest_data as manifest using (sample_id) + ) + + select + sample_id, + sample.identifier as sample, + has_encounter_data, + best_available_encounter_date, + + case + when best_available_encounter_date < '2019-10-01'::date then 'Y1' + when best_available_encounter_date < '2020-10-01'::date then 'Y2' + else null + end as season, + + coalesce(site.site_id, site_details.site_id) as best_available_site_id, + coalesce(site.identifier, site_details.site) as best_available_site, + coalesce(site.details->>'type', site_type) as best_available_site_type, + coalesce(site.details->>'category', site_category) as best_available_site_category + + from warehouse.sample + left join samples_with_manifest_data using (sample_id) + left join site_details on (site_manifest_details similar to manifest_regex) + left join warehouse.site on (samples_with_manifest_data.site_id = site.site_id) + where sample.identifier is not null + ; + +comment on view shipping.sample_with_best_available_encounter_data_v1 is + 'Version 1 of view of warehoused samples and their best available encounter date and site details important for metrics calculations'; + + +revoke all + on shipping.incidence_model_observation_v4 + from "incidence-modeler"; +drop view shipping.incidence_model_observation_v4; commit; diff --git a/schema/revert/shipping/views@2020-02-21.sql b/schema/revert/shipping/views@2020-02-21.sql new file mode 100644 index 000000000..c65fd95f3 --- /dev/null +++ b/schema/revert/shipping/views@2020-02-21.sql @@ -0,0 +1,600 @@ +-- Deploy seattleflu/id3c-customizations:shipping/views to pg +-- requires: seattleflu/schema:shipping/schema + +-- Hello! All shipping views are defined here. Rework this change with Sqitch +-- to change a view definition or add new views. This workflow helps keep +-- inter-view dependencies manageable. + +begin; + +-- This view is versioned as a hedge against future changes. Changing this +-- view in place is fine as long as changes are backwards compatible. Think of +-- the version number as the major part of a semantic versioning scheme. If +-- there needs to be a lag between view development and consumers being +-- updated, copy the view definition into v2 and make changes there. + +create or replace view shipping.reportable_condition_v1 as + + with reportable as ( + select array_agg(lineage) as lineages + from warehouse.organism + where details @> '{"report_to_public_health":true}' + ) + + select + presence_absence.presence_absence_id, + organism.lineage::text, + sample.identifier as sample, + barcode, + site.identifier as site, + presence_absence.details->>'reporting_log' as reporting_log, + sample.details->'_provenance'->>'workbook' as workbook, + sample.details->'_provenance'->>'sheet' as sheet, + sample.details->>'sample_origin' as sample_origin, + sample.details->>'swab_site' as swab_site + + from warehouse.presence_absence + join warehouse.target using (target_id) + join warehouse.organism using (organism_id) + join warehouse.sample using (sample_id) + join warehouse.identifier on ( + sample.identifier = cast(identifier.uuid as text)) + left join warehouse.encounter using (encounter_id) + left join warehouse.site using (site_id) + + where organism.lineage <@ (table reportable) + and present + + order by encountered desc; + + +drop view shipping.metadata_for_augur_build_v2; +create or replace view shipping.metadata_for_augur_build_v2 as + + select sample as strain, + cast(encountered as date) as date, + 'seattle' as region, + -- XXX TODO: Change to PUMA and neighborhoods + residence_census_tract as location, + 'Seattle Flu Study' as authors, + age_range_coarse, + case age_range_coarse <@ '[0 mon, 18 years)'::intervalrange + when 't' then 'child' + when 'f' then 'adult' + end as age_category, + case + when site_type in ('childrensHospital', 'childrensClinic', 'childrensHospital', 'clinic', 'hospital', 'retrospective') then 'clinical' + when site_type in ('childcare' , 'collegeCampus' , 'homelessShelter' , 'port', 'publicSpace', 'workplace') then 'community' + end as site_category, + residence_census_tract, + flu_shot, + sex + + from shipping.incidence_model_observation_v3 + join warehouse.encounter on encounter.identifier = incidence_model_observation_v3.encounter; + +comment on view shipping.metadata_for_augur_build_v2 is + 'View of metadata necessary for SFS augur build'; + + +create or replace view shipping.genomic_sequences_for_augur_build_v1 as + + select distinct on (sample.identifier, organism.lineage, segment) + sample.identifier as sample, + organism.lineage as organism, + genomic_sequence.segment, + round(((length(replace(seq, 'N', '')) * 1.0 / length(seq))), 4) as coverage, + genomic_sequence.seq + from warehouse.sample + join warehouse.consensus_genome using (sample_id) + join warehouse.genomic_sequence using (consensus_genome_id) + join warehouse.organism using (organism_id) + + order by sample.identifier, organism.lineage, segment, coverage desc; + +comment on view shipping.genomic_sequences_for_augur_build_v1 is + 'View of genomic sequences for SFS augur build'; + + +create or replace view shipping.flu_assembly_jobs_v1 as + + select sample.identifier as sfs_uuid, + sample.details ->> 'nwgc_id' as nwgc_id, + sequence_read_set.urls, + target.identifier as target, + o1.lineage as target_linked_organism, + coalesce(o2.lineage, o1.lineage) as assembly_job_organism, + sequence_read_set.details + + from warehouse.sequence_read_set + join warehouse.sample using (sample_id) + join warehouse.presence_absence using (sample_id) + join warehouse.target using (target_id) + join warehouse.organism o1 using (organism_id) + -- Reason for o2 join: For pan-subtype targets like Flu_A_pan, + -- we want to spread out to the more specific lineages for assembly jobs. + -- Since this view is Flu specific, we are hard-coding it to spread just one level down. + left join warehouse.organism o2 on (o2.lineage ~ concat(o1.lineage::text, '.*{1}')::lquery) + + where o1.lineage ~ 'Influenza.*' + and present + -- The sequence read set details currently holds the state of assembly jobs for each lineage. + -- So if null, then definitely no jobs have been completed. + -- If not null, then check if the assembly job organism matches any key in the sequence read set details. + and (sequence_read_set.details is null + or not sequence_read_set.details ? coalesce(o2.lineage, o1.lineage)::text) + + order by sample.details ->> 'nwgc_id'; + +comment on view shipping.flu_assembly_jobs_v1 is + 'View of flu jobs that still need to be run through the assembly pipeline'; + + +create or replace view shipping.return_results_v1 as + + select barcode, + case + when sample_id is null then 'notReceived' + when sample_id is not null and count(present) = 0 then 'processing' + when count(present) > 0 then 'complete' + end as status, + -- We only return the top level organisms for results so we want to omit subtypes + array_agg(distinct subpath(organism, 0, 1)::text) + filter (where present and organism is not null) as organisms_present, + array_agg(distinct subpath(organism, 0, 1)::text) + filter (where not present and organism is not null) as organisms_absent + + from warehouse.identifier + join warehouse.identifier_set using (identifier_set_id) + left join warehouse.sample on uuid::text = sample.collection_identifier + left join shipping.presence_absence_result_v1 on sample.identifier = presence_absence_result_v1.sample + + --These are all past and current collection identifier sets not including self-test + where identifier_set.name in ('collections-seattleflu.org', + 'collections-kiosks', + 'collections-environmental', + 'collections-swab&send', + 'collections-household-observation', + 'collections-household-intervention') and + (organism is null or + -- We only return results for these organisms, so omit all other presence/absence results + organism <@ '{"Adenovirus", "Human_coronavirus", "Enterovirus", "Influenza", "Human_metapneumovirus", "Human_parainfluenza", "Rhinovirus", "RSV"}'::ltree[]) + group by barcode, sample_id + order by barcode; + +comment on view shipping.return_results_v1 is + 'View of barcodes and presence/absence results for return of results on website'; + + +create or replace view shipping.fhir_encounter_details_v1 as + + with + fhir as ( + select encounter_id, + case + when details -> 'Condition' is not null then array_agg(distinct condition."id" order by condition."id") + else null + end as symptoms, + jsonb_object_agg(response."linkId", response.answer order by response.answer) filter (where response."linkId" is not null) as responses + from warehouse.encounter + left join jsonb_to_recordset(details -> 'Condition') as condition("id" text) on true + left join jsonb_to_recordset(details -> 'QuestionnaireResponse') as q("item" jsonb) on true + left join jsonb_to_recordset(q."item") as response("linkId" text, "answer" jsonb) on true + group by encounter_id + ), + + vaccine as ( + --Builds vaccine jsonb object without null values that spans multiple rows. + --Expand the rows into key/value pairs with jsonb_each and + --aggregate that back into a single jsonb value. + -- -Jover 10 Jan 2020 + select encounter_id, + jsonb_object_agg(t.k, t.v order by t.v) filter (where t.k is not null) as vaccine + from ( + select encounter_id, + jsonb_strip_nulls( + jsonb_build_object( + 'vaccine', vaccine."valueBoolean", + 'vaccine_date', vaccine."valueDate")) as vaccine_obj + from fhir, + jsonb_to_record(responses) as response("vaccine" jsonb), + jsonb_to_recordset(response."vaccine") as vaccine("valueBoolean" bool, "valueDate" text)) as vaccine, + jsonb_each(vaccine.vaccine_obj) as t(k,v) + group by encounter_id + ), + + array_responses as ( + select + encounter_id, + case + when responses -> 'race' is not null then array_agg(coalesce(race."valueCoding" ->> 'code', race."valueString") order by race."valueString") + else null + end as race, + case + when responses -> 'insurance' is not null then array_agg(insurance."valueString" order by insurance."valueString") + else null + end as insurance + from fhir + left join jsonb_to_record(responses) as response("race" jsonb, "insurance" jsonb) on true + left join jsonb_to_recordset(response."race") as race ("valueCoding" jsonb, "valueString" text) on true + left join jsonb_to_recordset(response."insurance") as insurance ("valueString" text) on true + group by encounter_id, responses + ), + + boolean_responses as ( + select + encounter_id, + ethnicity."valueBoolean" as hispanic_or_latino, + travel_countries."valueBoolean" as travel_countries, + travel_states."valueBoolean" as travel_states + from fhir + left join jsonb_to_record(responses) as response("ethnicity" jsonb, "travel_countries" jsonb, "travel_states" jsonb) on true + left join jsonb_to_recordset(response."ethnicity") as ethnicity("valueBoolean" bool) on true + left join jsonb_to_recordset(response."travel_countries") as travel_countries("valueBoolean" bool) on true + left join jsonb_to_recordset(response."travel_states") as travel_states("valueBoolean" bool) on true + ) + + + select + encounter_id, + symptoms, + (vaccine ->> 'vaccine')::bool as vaccine, + vaccine ->> 'vaccine_date' as vaccine_date, + race, + insurance, + hispanic_or_latino, + travel_countries, + travel_states + from fhir + left join vaccine using (encounter_id) + left join array_responses using (encounter_id) + left join boolean_responses using (encounter_id); + +comment on view shipping.fhir_encounter_details_v1 is + 'A view of encounter details that are in FHIR format'; + +revoke all + on shipping.fhir_encounter_details_v1 + from "incidence-modeler"; + +grant select + on shipping.fhir_encounter_details_v1 + to "incidence-modeler"; + + +create or replace view shipping.incidence_model_observation_v1 as + + select encounter.identifier as encounter, + + (encountered at time zone 'US/Pacific')::date as encountered_date, + to_char((encountered at time zone 'US/Pacific')::date, 'IYYY-"W"IW') as encountered_week, + + site.details->>'type' as site_type, + + individual.identifier as individual, + individual.sex, + + -- Reporting 90 is more accurate than reporting nothing, and it will + -- never be a real value in our dataset. + -- + -- XXX TODO: This will be pre-processed out of the JSON in the future. + ceiling(age_in_years(age))::int as age, + + age_bin_fine.range as age_range_fine, + lower(age_bin_fine.range) as age_range_fine_lower, + upper(age_bin_fine.range) as age_range_fine_upper, + + age_bin_coarse.range as age_range_coarse, + lower(age_bin_coarse.range) as age_range_coarse_lower, + upper(age_bin_coarse.range) as age_range_coarse_upper, + + residence_census_tract, + work_census_tract, + + coalesce(encounter_responses.flu_shot,fhir.vaccine) as flu_shot, + coalesce(encounter_responses.symptoms,fhir.symptoms) as symptoms, + coalesce(encounter_responses.race,fhir.race) as race, + coalesce(encounter_responses.hispanic_or_latino,fhir.hispanic_or_latino) as hispanic_or_latino, + + sample.identifier as sample + + from warehouse.encounter + join warehouse.individual using (individual_id) + join warehouse.site using (site_id) + left join warehouse.sample using (encounter_id) + left join shipping.age_bin_fine on age_bin_fine.range @> ceiling(age_in_years(age))::int + left join shipping.age_bin_coarse on age_bin_coarse.range @> ceiling(age_in_years(age))::int + left join shipping.fhir_encounter_details_v1 as fhir using (encounter_id) + left join ( + select encounter_id, hierarchy->'tract' as residence_census_tract + from warehouse.encounter_location + left join warehouse.location using (location_id) + where relation = 'residence' + or relation = 'lodging' + ) as residence using (encounter_id) + left join ( + select encounter_id, hierarchy->'tract' as work_census_tract + from warehouse.encounter_location + left join warehouse.location using (location_id) + where relation = 'workplace' + ) as workplace using (encounter_id), + + lateral ( + -- XXX TODO: The data in this subquery will be modeled better in the + -- future and the verbosity of extracting data from the JSON details + -- document will go away. + -- -trs, 22 March 2019 + + select -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(nullif(responses."FluShot"[1], 'doNotKnow'), 'dontKnow')::bool as flu_shot, + + -- XXX FIXME: Remove duplicate value collapsing when we're no + -- longer affected by this known Audere data quality issue. + array_distinct(responses."Symptoms") as symptoms, + array_distinct(responses."Race") as race, + + -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(responses."HispanicLatino"[1], 'preferNotToSay')::bool as hispanic_or_latino + + from jsonb_to_record(encounter.details->'responses') + as responses ( + "FluShot" text[], + "Symptoms" text[], + "Race" text[], + "HispanicLatino" text[])) + as encounter_responses + + order by encountered; + +comment on view shipping.incidence_model_observation_v1 is + 'View of warehoused encounters and important questionnaire responses for modeling and viz teams'; + +revoke all + on shipping.incidence_model_observation_v1 + from "incidence-modeler"; + +grant select + on shipping.incidence_model_observation_v1 + to "incidence-modeler"; + + +create or replace view shipping.incidence_model_observation_v2 as + + select encounter.identifier as encounter, + + (encountered at time zone 'US/Pacific')::date as encountered_date, + to_char((encountered at time zone 'US/Pacific')::date, 'IYYY-"W"IW') as encountered_week, + + site.identifier as site, + site.details->>'type' as site_type, + + individual.identifier as individual, + individual.sex, + + age_in_years(age) as age, + + age_bin_fine_v2.range as age_range_fine, + age_in_years(lower(age_bin_fine_v2.range)) as age_range_fine_lower, + age_in_years(upper(age_bin_fine_v2.range)) as age_range_fine_upper, + + age_bin_coarse_v2.range as age_range_coarse, + age_in_years(lower(age_bin_coarse_v2.range)) as age_range_coarse_lower, + age_in_years(upper(age_bin_coarse_v2.range)) as age_range_coarse_upper, + + residence_census_tract, + work_census_tract, + + coalesce(encounter_responses.flu_shot,fhir.vaccine) as flu_shot, + coalesce(encounter_responses.symptoms,fhir.symptoms) as symptoms, + coalesce(encounter_responses.race,fhir.race) as race, + coalesce(encounter_responses.hispanic_or_latino,fhir.hispanic_or_latino) as hispanic_or_latino, + + sample.identifier as sample + + from warehouse.encounter + join warehouse.individual using (individual_id) + join warehouse.site using (site_id) + left join warehouse.sample using (encounter_id) + left join shipping.age_bin_fine_v2 on age_bin_fine_v2.range @> age + left join shipping.age_bin_coarse_v2 on age_bin_coarse_v2.range @> age + left join shipping.fhir_encounter_details_v1 as fhir using (encounter_id) + left join ( + select encounter_id, hierarchy->'tract' as residence_census_tract + from warehouse.encounter_location + left join warehouse.location using (location_id) + where relation = 'residence' + or relation = 'lodging' + ) as residence using (encounter_id) + left join ( + select encounter_id, hierarchy->'tract' as work_census_tract + from warehouse.encounter_location + left join warehouse.location using (location_id) + where relation = 'workplace' + ) as workplace using (encounter_id), + + lateral ( + -- XXX TODO: The data in this subquery will be modeled better in the + -- future and the verbosity of extracting data from the JSON details + -- document will go away. + -- -trs, 22 March 2019 + + select -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(nullif(responses."FluShot"[1], 'doNotKnow'), 'dontKnow')::bool as flu_shot, + + -- XXX FIXME: Remove duplicate value collapsing when we're no + -- longer affected by this known Audere data quality issue. + array_distinct(responses."Symptoms") as symptoms, + array_distinct(responses."Race") as race, + + -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(responses."HispanicLatino"[1], 'preferNotToSay')::bool as hispanic_or_latino + + from jsonb_to_record(encounter.details->'responses') + as responses ( + "FluShot" text[], + "Symptoms" text[], + "Race" text[], + "HispanicLatino" text[])) + as encounter_responses + + order by encountered; + +comment on view shipping.incidence_model_observation_v2 is + 'Version 2 of view of warehoused encounters and important questionnaire responses for modeling and viz teams'; + +revoke all + on shipping.incidence_model_observation_v2 + from "incidence-modeler"; + +grant select + on shipping.incidence_model_observation_v2 + to "incidence-modeler"; + + +create or replace view shipping.observation_with_presence_absence_result_v1 as + + select target, + present, + present::int as presence, + observation.*, + organism + from shipping.incidence_model_observation_v2 as observation + join shipping.presence_absence_result_v1 using (sample) + order by site, encounter, sample, target; + +comment on view shipping.observation_with_presence_absence_result_v1 is + 'Joined view of shipping.incidence_model_observation_v2 and shipping.presence_absence_result_v1'; + + +create or replace view shipping.incidence_model_observation_v3 as + + select encounter.identifier as encounter, + + to_char((encountered at time zone 'US/Pacific')::date, 'IYYY-"W"IW') as encountered_week, + + site.details->>'type' as site_type, + + individual.identifier as individual, + individual.sex, + + age_bin_fine_v2.range as age_range_fine, + age_in_years(lower(age_bin_fine_v2.range)) as age_range_fine_lower, + age_in_years(upper(age_bin_fine_v2.range)) as age_range_fine_upper, + + age_bin_coarse_v2.range as age_range_coarse, + age_in_years(lower(age_bin_coarse_v2.range)) as age_range_coarse_lower, + age_in_years(upper(age_bin_coarse_v2.range)) as age_range_coarse_upper, + + residence_census_tract, + + coalesce(encounter_responses.flu_shot,fhir.vaccine) as flu_shot, + coalesce(encounter_responses.symptoms,fhir.symptoms) as symptoms, + + sample.identifier as sample + + from warehouse.encounter + join warehouse.individual using (individual_id) + join warehouse.site using (site_id) + left join warehouse.sample using (encounter_id) + left join shipping.age_bin_fine_v2 on age_bin_fine_v2.range @> age + left join shipping.age_bin_coarse_v2 on age_bin_coarse_v2.range @> age + left join shipping.fhir_encounter_details_v1 as fhir using (encounter_id) + left join ( + select encounter_id, hierarchy->'tract' as residence_census_tract + from warehouse.encounter_location + left join warehouse.location using (location_id) + where relation = 'residence' + or relation = 'lodging' + ) as residence using (encounter_id), + + lateral ( + -- XXX TODO: The data in this subquery will be modeled better in the + -- future and the verbosity of extracting data from the JSON details + -- document will go away. + -- -trs, 22 March 2019 + + select -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(nullif(responses."FluShot"[1], 'doNotKnow'), 'dontKnow')::bool as flu_shot, + + -- XXX FIXME: Remove duplicate value collapsing when we're no + -- longer affected by this known Audere data quality issue. + array_distinct(responses."Symptoms") as symptoms + + from jsonb_to_record(encounter.details->'responses') + as responses ( + "FluShot" text[], + "Symptoms" text[])) + as encounter_responses + + order by encountered; + +comment on view shipping.incidence_model_observation_v3 is + 'Version 3 of view of warehoused encounters and important questionnaire responses for modeling and viz teams'; + +revoke all + on shipping.incidence_model_observation_v3 + from "incidence-modeler"; + +grant select + on shipping.incidence_model_observation_v3 + to "incidence-modeler"; + + +create or replace view shipping.observation_with_presence_absence_result_v2 as + + select target, + present, + present::int as presence, + observation.*, + organism + from shipping.incidence_model_observation_v3 as observation + join shipping.presence_absence_result_v1 using (sample) + order by site_type, encounter, sample, target; + +comment on view shipping.observation_with_presence_absence_result_v2 is + 'Joined view of shipping.incidence_model_observation_v3 and shipping.presence_absence_result_v1'; + + +create or replace view shipping.metadata_for_augur_build_v3 as + + select sample.identifier as strain, + coalesce( + encountered_date, + case + when date_or_null(sample.details->>'date') <= current_date + then date_or_null(sample.details->>'date') + end + ) as date, + 'seattle' as region, + -- XXX TODO: Change to PUMA and neighborhoods + residence_census_tract as location, + 'Seattle Flu Study' as authors, + age_range_coarse, + case age_range_coarse <@ '[0 mon, 18 years)'::intervalrange + when 't' then 'child' + when 'f' then 'adult' + end as age_category, + warehouse.site.details->>'category' as site_category, + residence_census_tract, + flu_shot, + sex + + from warehouse.sample + left join shipping.incidence_model_observation_v2 on sample.identifier = incidence_model_observation_v2.sample + left join warehouse.site on site.identifier = incidence_model_observation_v2.site + + where sample.identifier is not null; + +comment on view shipping.metadata_for_augur_build_v3 is + 'View of metadata necessary for SFS augur build'; + + +drop view shipping.sample_with_best_available_encounter_data_v1; + + +commit; diff --git a/schema/sqitch.plan b/schema/sqitch.plan index 8d39bd68d..508a80afd 100644 --- a/schema/sqitch.plan +++ b/schema/sqitch.plan @@ -80,3 +80,4 @@ warehouse/site/data [warehouse/site/data@2020-02-20] 2020-02-21T19:49:28Z Kairst @2020-02-21 2020-02-21T19:55:54Z Kairsten Fay # Schema as of 21 Feb 2020 warehouse/views [warehouse/views@2020-02-06] 2020-02-18T22:20:31Z Jover Lee # Add PUMA and neighborhood scale location views +shipping/views [shipping/views@2020-02-21] 2020-02-24T19:03:29Z Jover Lee # Add incidence model view v4 with PUMA and neighborhood district diff --git a/schema/verify/shipping/views.sql b/schema/verify/shipping/views.sql index cef8eb02d..b4e5b1b20 100644 --- a/schema/verify/shipping/views.sql +++ b/schema/verify/shipping/views.sql @@ -74,4 +74,9 @@ select 1/(count(*) = 1)::int where array[table_schema, table_name]::text[] = pg_catalog.parse_ident('shipping.sample_with_best_available_encounter_data_v1'); +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.incidence_model_observation_v4'); + rollback; diff --git a/schema/verify/shipping/views@2020-02-21.sql b/schema/verify/shipping/views@2020-02-21.sql new file mode 100644 index 000000000..cef8eb02d --- /dev/null +++ b/schema/verify/shipping/views@2020-02-21.sql @@ -0,0 +1,77 @@ +-- Verify seattleflu/id3c-customizations:shipping/views on pg +-- requires: seattleflu/schema:shipping/schema + +begin; + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.reportable_condition_v1'); + +-- Verify that the view has been dropped +select 1/(count(*) = 0)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.metadata_for_augur_build_v1'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.metadata_for_augur_build_v2'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.genomic_sequences_for_augur_build_v1'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.flu_assembly_jobs_v1'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.return_results_v1'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.fhir_encounter_details_v1'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.incidence_model_observation_v1'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.incidence_model_observation_v2'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.observation_with_presence_absence_result_v1'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.incidence_model_observation_v3'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.observation_with_presence_absence_result_v2'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.metadata_for_augur_build_v3'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.sample_with_best_available_encounter_data_v1'); + +rollback; From 56efa0e0d2f7575acbec159d60ad2ea76c34517d Mon Sep 17 00:00:00 2001 From: Jover Date: Mon, 24 Feb 2020 11:45:47 -0800 Subject: [PATCH 3/7] Tag schema --- schema/sqitch.plan | 1 + 1 file changed, 1 insertion(+) diff --git a/schema/sqitch.plan b/schema/sqitch.plan index 508a80afd..b2c8e1af8 100644 --- a/schema/sqitch.plan +++ b/schema/sqitch.plan @@ -81,3 +81,4 @@ warehouse/site/data [warehouse/site/data@2020-02-20] 2020-02-21T19:49:28Z Kairst warehouse/views [warehouse/views@2020-02-06] 2020-02-18T22:20:31Z Jover Lee # Add PUMA and neighborhood scale location views shipping/views [shipping/views@2020-02-21] 2020-02-24T19:03:29Z Jover Lee # Add incidence model view v4 with PUMA and neighborhood district +@2020-02-24 2020-02-24T19:27:10Z Jover Lee # Schema as of 24 Feb 2020 From a32307672758b6a55179835ccb1b312a9b63a25a Mon Sep 17 00:00:00 2001 From: Jover Date: Mon, 24 Feb 2020 11:45:56 -0800 Subject: [PATCH 4/7] add shipping.metadata_for_augur_build_v4 Exports neighborhood or PUMA for location and removes census tract. --- schema/deploy/shipping/views.sql | 33 + schema/deploy/shipping/views@2020-02-24.sql | 753 ++++++++++++++++++++ schema/revert/shipping/views.sql | 78 +- schema/revert/shipping/views@2020-02-24.sql | 678 ++++++++++++++++++ schema/sqitch.plan | 1 + schema/verify/shipping/views.sql | 5 + schema/verify/shipping/views@2020-02-24.sql | 82 +++ 7 files changed, 1629 insertions(+), 1 deletion(-) create mode 100644 schema/deploy/shipping/views@2020-02-24.sql create mode 100644 schema/revert/shipping/views@2020-02-24.sql create mode 100644 schema/verify/shipping/views@2020-02-24.sql diff --git a/schema/deploy/shipping/views.sql b/schema/deploy/shipping/views.sql index c58629eec..ca1beaa14 100644 --- a/schema/deploy/shipping/views.sql +++ b/schema/deploy/shipping/views.sql @@ -750,4 +750,37 @@ grant select to "incidence-modeler"; +create or replace view shipping.metadata_for_augur_build_v4 as + + select sample.identifier as strain, + coalesce( + encountered, + case + when date_or_null(sample.details->>'date') <= current_date + then date_or_null(sample.details->>'date') + end + ) as date, + 'Seattle' as region, + coalesce(residence_neighborhood_district, residence_puma) as location, + 'Seattle Flu Study' as authors, + age_range_coarse, + case age_range_coarse <@ '[0 mon, 18 years)'::intervalrange + when 't' then 'child' + when 'f' then 'adult' + end as age_category, + site_category, + flu_shot, + sex + + from warehouse.sample + join warehouse.consensus_genome using (sample_id) + left join warehouse.encounter using (encounter_id) + left join shipping.incidence_model_observation_v4 on sample.identifier = incidence_model_observation_v4.sample + + where sample.identifier is not null and consensus_genome_id is not null; + +comment on view shipping.metadata_for_augur_build_v4 is + 'View of metadata necessary for SFS augur build'; + + commit; diff --git a/schema/deploy/shipping/views@2020-02-24.sql b/schema/deploy/shipping/views@2020-02-24.sql new file mode 100644 index 000000000..c58629eec --- /dev/null +++ b/schema/deploy/shipping/views@2020-02-24.sql @@ -0,0 +1,753 @@ +-- Deploy seattleflu/id3c-customizations:shipping/views to pg +-- requires: seattleflu/schema:shipping/schema + +-- Hello! All shipping views are defined here. Rework this change with Sqitch +-- to change a view definition or add new views. This workflow helps keep +-- inter-view dependencies manageable. + +begin; + +-- This view is versioned as a hedge against future changes. Changing this +-- view in place is fine as long as changes are backwards compatible. Think of +-- the version number as the major part of a semantic versioning scheme. If +-- there needs to be a lag between view development and consumers being +-- updated, copy the view definition into v2 and make changes there. + +create or replace view shipping.reportable_condition_v1 as + + with reportable as ( + select array_agg(lineage) as lineages + from warehouse.organism + where details @> '{"report_to_public_health":true}' + ) + + select + presence_absence.presence_absence_id, + organism.lineage::text, + sample.identifier as sample, + barcode, + site.identifier as site, + presence_absence.details->>'reporting_log' as reporting_log, + sample.details->'_provenance'->>'workbook' as workbook, + sample.details->'_provenance'->>'sheet' as sheet, + sample.details->>'sample_origin' as sample_origin, + sample.details->>'swab_site' as swab_site + + from warehouse.presence_absence + join warehouse.target using (target_id) + join warehouse.organism using (organism_id) + join warehouse.sample using (sample_id) + join warehouse.identifier on ( + sample.identifier = cast(identifier.uuid as text)) + left join warehouse.encounter using (encounter_id) + left join warehouse.site using (site_id) + + where organism.lineage <@ (table reportable) + and present + + order by encountered desc; + + +drop view shipping.metadata_for_augur_build_v2; +create or replace view shipping.metadata_for_augur_build_v2 as + + select sample as strain, + cast(encountered as date) as date, + 'seattle' as region, + -- XXX TODO: Change to PUMA and neighborhoods + residence_census_tract as location, + 'Seattle Flu Study' as authors, + age_range_coarse, + case age_range_coarse <@ '[0 mon, 18 years)'::intervalrange + when 't' then 'child' + when 'f' then 'adult' + end as age_category, + case + when site_type in ('childrensHospital', 'childrensClinic', 'childrensHospital', 'clinic', 'hospital', 'retrospective') then 'clinical' + when site_type in ('childcare' , 'collegeCampus' , 'homelessShelter' , 'port', 'publicSpace', 'workplace') then 'community' + end as site_category, + residence_census_tract, + flu_shot, + sex + + from shipping.incidence_model_observation_v3 + join warehouse.encounter on encounter.identifier = incidence_model_observation_v3.encounter; + +comment on view shipping.metadata_for_augur_build_v2 is + 'View of metadata necessary for SFS augur build'; + + +create or replace view shipping.genomic_sequences_for_augur_build_v1 as + + select distinct on (sample.identifier, organism.lineage, segment) + sample.identifier as sample, + organism.lineage as organism, + genomic_sequence.segment, + round(((length(replace(seq, 'N', '')) * 1.0 / length(seq))), 4) as coverage, + genomic_sequence.seq + from warehouse.sample + join warehouse.consensus_genome using (sample_id) + join warehouse.genomic_sequence using (consensus_genome_id) + join warehouse.organism using (organism_id) + + order by sample.identifier, organism.lineage, segment, coverage desc; + +comment on view shipping.genomic_sequences_for_augur_build_v1 is + 'View of genomic sequences for SFS augur build'; + + +create or replace view shipping.flu_assembly_jobs_v1 as + + select sample.identifier as sfs_uuid, + sample.details ->> 'nwgc_id' as nwgc_id, + sequence_read_set.urls, + target.identifier as target, + o1.lineage as target_linked_organism, + coalesce(o2.lineage, o1.lineage) as assembly_job_organism, + sequence_read_set.details + + from warehouse.sequence_read_set + join warehouse.sample using (sample_id) + join warehouse.presence_absence using (sample_id) + join warehouse.target using (target_id) + join warehouse.organism o1 using (organism_id) + -- Reason for o2 join: For pan-subtype targets like Flu_A_pan, + -- we want to spread out to the more specific lineages for assembly jobs. + -- Since this view is Flu specific, we are hard-coding it to spread just one level down. + left join warehouse.organism o2 on (o2.lineage ~ concat(o1.lineage::text, '.*{1}')::lquery) + + where o1.lineage ~ 'Influenza.*' + and present + -- The sequence read set details currently holds the state of assembly jobs for each lineage. + -- So if null, then definitely no jobs have been completed. + -- If not null, then check if the assembly job organism matches any key in the sequence read set details. + and (sequence_read_set.details is null + or not sequence_read_set.details ? coalesce(o2.lineage, o1.lineage)::text) + + order by sample.details ->> 'nwgc_id'; + +comment on view shipping.flu_assembly_jobs_v1 is + 'View of flu jobs that still need to be run through the assembly pipeline'; + + +create or replace view shipping.return_results_v1 as + + select barcode, + case + when sample_id is null then 'notReceived' + when sample_id is not null and count(present) = 0 then 'processing' + when count(present) > 0 then 'complete' + end as status, + -- We only return the top level organisms for results so we want to omit subtypes + array_agg(distinct subpath(organism, 0, 1)::text) + filter (where present and organism is not null) as organisms_present, + array_agg(distinct subpath(organism, 0, 1)::text) + filter (where not present and organism is not null) as organisms_absent + + from warehouse.identifier + join warehouse.identifier_set using (identifier_set_id) + left join warehouse.sample on uuid::text = sample.collection_identifier + left join shipping.presence_absence_result_v1 on sample.identifier = presence_absence_result_v1.sample + + --These are all past and current collection identifier sets not including self-test + where identifier_set.name in ('collections-seattleflu.org', + 'collections-kiosks', + 'collections-environmental', + 'collections-swab&send', + 'collections-household-observation', + 'collections-household-intervention') and + (organism is null or + -- We only return results for these organisms, so omit all other presence/absence results + organism <@ '{"Adenovirus", "Human_coronavirus", "Enterovirus", "Influenza", "Human_metapneumovirus", "Human_parainfluenza", "Rhinovirus", "RSV"}'::ltree[]) + group by barcode, sample_id + order by barcode; + +comment on view shipping.return_results_v1 is + 'View of barcodes and presence/absence results for return of results on website'; + + +create or replace view shipping.fhir_encounter_details_v1 as + + with + fhir as ( + select encounter_id, + case + when details -> 'Condition' is not null then array_agg(distinct condition."id" order by condition."id") + else null + end as symptoms, + jsonb_object_agg(response."linkId", response.answer order by response.answer) filter (where response."linkId" is not null) as responses + from warehouse.encounter + left join jsonb_to_recordset(details -> 'Condition') as condition("id" text) on true + left join jsonb_to_recordset(details -> 'QuestionnaireResponse') as q("item" jsonb) on true + left join jsonb_to_recordset(q."item") as response("linkId" text, "answer" jsonb) on true + group by encounter_id + ), + + vaccine as ( + --Builds vaccine jsonb object without null values that spans multiple rows. + --Expand the rows into key/value pairs with jsonb_each and + --aggregate that back into a single jsonb value. + -- -Jover 10 Jan 2020 + select encounter_id, + jsonb_object_agg(t.k, t.v order by t.v) filter (where t.k is not null) as vaccine + from ( + select encounter_id, + jsonb_strip_nulls( + jsonb_build_object( + 'vaccine', vaccine."valueBoolean", + 'vaccine_date', vaccine."valueDate")) as vaccine_obj + from fhir, + jsonb_to_record(responses) as response("vaccine" jsonb), + jsonb_to_recordset(response."vaccine") as vaccine("valueBoolean" bool, "valueDate" text)) as vaccine, + jsonb_each(vaccine.vaccine_obj) as t(k,v) + group by encounter_id + ), + + array_responses as ( + select + encounter_id, + case + when responses -> 'race' is not null then array_agg(coalesce(race."valueCoding" ->> 'code', race."valueString") order by race."valueString") + else null + end as race, + case + when responses -> 'insurance' is not null then array_agg(insurance."valueString" order by insurance."valueString") + else null + end as insurance + from fhir + left join jsonb_to_record(responses) as response("race" jsonb, "insurance" jsonb) on true + left join jsonb_to_recordset(response."race") as race ("valueCoding" jsonb, "valueString" text) on true + left join jsonb_to_recordset(response."insurance") as insurance ("valueString" text) on true + group by encounter_id, responses + ), + + boolean_responses as ( + select + encounter_id, + ethnicity."valueBoolean" as hispanic_or_latino, + travel_countries."valueBoolean" as travel_countries, + travel_states."valueBoolean" as travel_states + from fhir + left join jsonb_to_record(responses) as response("ethnicity" jsonb, "travel_countries" jsonb, "travel_states" jsonb) on true + left join jsonb_to_recordset(response."ethnicity") as ethnicity("valueBoolean" bool) on true + left join jsonb_to_recordset(response."travel_countries") as travel_countries("valueBoolean" bool) on true + left join jsonb_to_recordset(response."travel_states") as travel_states("valueBoolean" bool) on true + ) + + + select + encounter_id, + symptoms, + (vaccine ->> 'vaccine')::bool as vaccine, + vaccine ->> 'vaccine_date' as vaccine_date, + race, + insurance, + hispanic_or_latino, + travel_countries, + travel_states + from fhir + left join vaccine using (encounter_id) + left join array_responses using (encounter_id) + left join boolean_responses using (encounter_id); + +comment on view shipping.fhir_encounter_details_v1 is + 'A view of encounter details that are in FHIR format'; + +revoke all + on shipping.fhir_encounter_details_v1 + from "incidence-modeler"; + +grant select + on shipping.fhir_encounter_details_v1 + to "incidence-modeler"; + + +create or replace view shipping.incidence_model_observation_v1 as + + select encounter.identifier as encounter, + + (encountered at time zone 'US/Pacific')::date as encountered_date, + to_char((encountered at time zone 'US/Pacific')::date, 'IYYY-"W"IW') as encountered_week, + + site.details->>'type' as site_type, + + individual.identifier as individual, + individual.sex, + + -- Reporting 90 is more accurate than reporting nothing, and it will + -- never be a real value in our dataset. + -- + -- XXX TODO: This will be pre-processed out of the JSON in the future. + ceiling(age_in_years(age))::int as age, + + age_bin_fine.range as age_range_fine, + lower(age_bin_fine.range) as age_range_fine_lower, + upper(age_bin_fine.range) as age_range_fine_upper, + + age_bin_coarse.range as age_range_coarse, + lower(age_bin_coarse.range) as age_range_coarse_lower, + upper(age_bin_coarse.range) as age_range_coarse_upper, + + residence_census_tract, + work_census_tract, + + coalesce(encounter_responses.flu_shot,fhir.vaccine) as flu_shot, + coalesce(encounter_responses.symptoms,fhir.symptoms) as symptoms, + coalesce(encounter_responses.race,fhir.race) as race, + coalesce(encounter_responses.hispanic_or_latino,fhir.hispanic_or_latino) as hispanic_or_latino, + + sample.identifier as sample + + from warehouse.encounter + join warehouse.individual using (individual_id) + join warehouse.site using (site_id) + left join warehouse.sample using (encounter_id) + left join shipping.age_bin_fine on age_bin_fine.range @> ceiling(age_in_years(age))::int + left join shipping.age_bin_coarse on age_bin_coarse.range @> ceiling(age_in_years(age))::int + left join shipping.fhir_encounter_details_v1 as fhir using (encounter_id) + left join ( + select encounter_id, hierarchy->'tract' as residence_census_tract + from warehouse.encounter_location + left join warehouse.location using (location_id) + where relation = 'residence' + or relation = 'lodging' + ) as residence using (encounter_id) + left join ( + select encounter_id, hierarchy->'tract' as work_census_tract + from warehouse.encounter_location + left join warehouse.location using (location_id) + where relation = 'workplace' + ) as workplace using (encounter_id), + + lateral ( + -- XXX TODO: The data in this subquery will be modeled better in the + -- future and the verbosity of extracting data from the JSON details + -- document will go away. + -- -trs, 22 March 2019 + + select -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(nullif(responses."FluShot"[1], 'doNotKnow'), 'dontKnow')::bool as flu_shot, + + -- XXX FIXME: Remove duplicate value collapsing when we're no + -- longer affected by this known Audere data quality issue. + array_distinct(responses."Symptoms") as symptoms, + array_distinct(responses."Race") as race, + + -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(responses."HispanicLatino"[1], 'preferNotToSay')::bool as hispanic_or_latino + + from jsonb_to_record(encounter.details->'responses') + as responses ( + "FluShot" text[], + "Symptoms" text[], + "Race" text[], + "HispanicLatino" text[])) + as encounter_responses + + order by encountered; + +comment on view shipping.incidence_model_observation_v1 is + 'View of warehoused encounters and important questionnaire responses for modeling and viz teams'; + +revoke all + on shipping.incidence_model_observation_v1 + from "incidence-modeler"; + +grant select + on shipping.incidence_model_observation_v1 + to "incidence-modeler"; + + +create or replace view shipping.incidence_model_observation_v2 as + + select encounter.identifier as encounter, + + (encountered at time zone 'US/Pacific')::date as encountered_date, + to_char((encountered at time zone 'US/Pacific')::date, 'IYYY-"W"IW') as encountered_week, + + site.identifier as site, + site.details->>'type' as site_type, + + individual.identifier as individual, + individual.sex, + + age_in_years(age) as age, + + age_bin_fine_v2.range as age_range_fine, + age_in_years(lower(age_bin_fine_v2.range)) as age_range_fine_lower, + age_in_years(upper(age_bin_fine_v2.range)) as age_range_fine_upper, + + age_bin_coarse_v2.range as age_range_coarse, + age_in_years(lower(age_bin_coarse_v2.range)) as age_range_coarse_lower, + age_in_years(upper(age_bin_coarse_v2.range)) as age_range_coarse_upper, + + residence_census_tract, + work_census_tract, + + coalesce(encounter_responses.flu_shot,fhir.vaccine) as flu_shot, + coalesce(encounter_responses.symptoms,fhir.symptoms) as symptoms, + coalesce(encounter_responses.race,fhir.race) as race, + coalesce(encounter_responses.hispanic_or_latino,fhir.hispanic_or_latino) as hispanic_or_latino, + + sample.identifier as sample + + from warehouse.encounter + join warehouse.individual using (individual_id) + join warehouse.site using (site_id) + left join warehouse.sample using (encounter_id) + left join shipping.age_bin_fine_v2 on age_bin_fine_v2.range @> age + left join shipping.age_bin_coarse_v2 on age_bin_coarse_v2.range @> age + left join shipping.fhir_encounter_details_v1 as fhir using (encounter_id) + left join ( + select encounter_id, hierarchy->'tract' as residence_census_tract + from warehouse.encounter_location + left join warehouse.location using (location_id) + where relation = 'residence' + or relation = 'lodging' + ) as residence using (encounter_id) + left join ( + select encounter_id, hierarchy->'tract' as work_census_tract + from warehouse.encounter_location + left join warehouse.location using (location_id) + where relation = 'workplace' + ) as workplace using (encounter_id), + + lateral ( + -- XXX TODO: The data in this subquery will be modeled better in the + -- future and the verbosity of extracting data from the JSON details + -- document will go away. + -- -trs, 22 March 2019 + + select -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(nullif(responses."FluShot"[1], 'doNotKnow'), 'dontKnow')::bool as flu_shot, + + -- XXX FIXME: Remove duplicate value collapsing when we're no + -- longer affected by this known Audere data quality issue. + array_distinct(responses."Symptoms") as symptoms, + array_distinct(responses."Race") as race, + + -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(responses."HispanicLatino"[1], 'preferNotToSay')::bool as hispanic_or_latino + + from jsonb_to_record(encounter.details->'responses') + as responses ( + "FluShot" text[], + "Symptoms" text[], + "Race" text[], + "HispanicLatino" text[])) + as encounter_responses + + order by encountered; + +comment on view shipping.incidence_model_observation_v2 is + 'Version 2 of view of warehoused encounters and important questionnaire responses for modeling and viz teams'; + +revoke all + on shipping.incidence_model_observation_v2 + from "incidence-modeler"; + +grant select + on shipping.incidence_model_observation_v2 + to "incidence-modeler"; + + +create or replace view shipping.observation_with_presence_absence_result_v1 as + + select target, + present, + present::int as presence, + observation.*, + organism + from shipping.incidence_model_observation_v2 as observation + join shipping.presence_absence_result_v1 using (sample) + order by site, encounter, sample, target; + +comment on view shipping.observation_with_presence_absence_result_v1 is + 'Joined view of shipping.incidence_model_observation_v2 and shipping.presence_absence_result_v1'; + + +create or replace view shipping.incidence_model_observation_v3 as + + select encounter.identifier as encounter, + + to_char((encountered at time zone 'US/Pacific')::date, 'IYYY-"W"IW') as encountered_week, + + site.details->>'type' as site_type, + + individual.identifier as individual, + individual.sex, + + age_bin_fine_v2.range as age_range_fine, + age_in_years(lower(age_bin_fine_v2.range)) as age_range_fine_lower, + age_in_years(upper(age_bin_fine_v2.range)) as age_range_fine_upper, + + age_bin_coarse_v2.range as age_range_coarse, + age_in_years(lower(age_bin_coarse_v2.range)) as age_range_coarse_lower, + age_in_years(upper(age_bin_coarse_v2.range)) as age_range_coarse_upper, + + residence_census_tract, + + coalesce(encounter_responses.flu_shot,fhir.vaccine) as flu_shot, + coalesce(encounter_responses.symptoms,fhir.symptoms) as symptoms, + + sample.identifier as sample + + from warehouse.encounter + join warehouse.individual using (individual_id) + join warehouse.site using (site_id) + left join warehouse.sample using (encounter_id) + left join shipping.age_bin_fine_v2 on age_bin_fine_v2.range @> age + left join shipping.age_bin_coarse_v2 on age_bin_coarse_v2.range @> age + left join shipping.fhir_encounter_details_v1 as fhir using (encounter_id) + left join ( + select encounter_id, hierarchy->'tract' as residence_census_tract + from warehouse.encounter_location + left join warehouse.location using (location_id) + where relation = 'residence' + or relation = 'lodging' + ) as residence using (encounter_id), + + lateral ( + -- XXX TODO: The data in this subquery will be modeled better in the + -- future and the verbosity of extracting data from the JSON details + -- document will go away. + -- -trs, 22 March 2019 + + select -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(nullif(responses."FluShot"[1], 'doNotKnow'), 'dontKnow')::bool as flu_shot, + + -- XXX FIXME: Remove duplicate value collapsing when we're no + -- longer affected by this known Audere data quality issue. + array_distinct(responses."Symptoms") as symptoms + + from jsonb_to_record(encounter.details->'responses') + as responses ( + "FluShot" text[], + "Symptoms" text[])) + as encounter_responses + + order by encountered; + +comment on view shipping.incidence_model_observation_v3 is + 'Version 3 of view of warehoused encounters and important questionnaire responses for modeling and viz teams'; + +revoke all + on shipping.incidence_model_observation_v3 + from "incidence-modeler"; + +grant select + on shipping.incidence_model_observation_v3 + to "incidence-modeler"; + + +create or replace view shipping.observation_with_presence_absence_result_v2 as + + select target, + present, + present::int as presence, + observation.*, + organism + from shipping.incidence_model_observation_v3 as observation + join shipping.presence_absence_result_v1 using (sample) + order by site_type, encounter, sample, target; + +comment on view shipping.observation_with_presence_absence_result_v2 is + 'Joined view of shipping.incidence_model_observation_v3 and shipping.presence_absence_result_v1'; + + +create or replace view shipping.metadata_for_augur_build_v3 as + + select sample.identifier as strain, + coalesce( + encountered_date, + case + when date_or_null(sample.details->>'date') <= current_date + then date_or_null(sample.details->>'date') + end + ) as date, + 'seattle' as region, + -- XXX TODO: Change to PUMA and neighborhoods + residence_census_tract as location, + 'Seattle Flu Study' as authors, + age_range_coarse, + case age_range_coarse <@ '[0 mon, 18 years)'::intervalrange + when 't' then 'child' + when 'f' then 'adult' + end as age_category, + warehouse.site.details->>'category' as site_category, + residence_census_tract, + flu_shot, + sex + + from warehouse.sample + left join shipping.incidence_model_observation_v2 on sample.identifier = incidence_model_observation_v2.sample + left join warehouse.site on site.identifier = incidence_model_observation_v2.site + + where sample.identifier is not null; + +comment on view shipping.metadata_for_augur_build_v3 is + 'View of metadata necessary for SFS augur build'; + + +create or replace view shipping.sample_with_best_available_encounter_data_v1 as + + with specimen_manifest_data as ( + select + sample_id, + date_or_null(details->>'date') as collection_date, + trim(both ' ' from details->>'swab_site') as swab_site, + trim(both ' ' from details->>'sample_origin') as sample_origin + from + warehouse.sample + ), + + site_details as ( + select + site_id, + site.identifier as site, + site.details->>'type' as site_type, + site.details->>'category' as site_category, + coalesce(site.details->>'swab_site', site.details->>'sample_origin') as manifest_regex + from warehouse.site + ), + + samples_with_manifest_data as ( + select + sample_id, + site_id, + coalesce(encountered::date, collection_date) as best_available_encounter_date, + + coalesce( + case + -- Environmental samples must be processed first, because they're + -- often taken at existing human swab sites + when manifest.sample_origin = 'es' + then 'environmental' + else manifest.swab_site + end, + + manifest.sample_origin + ) as site_manifest_details, + + site_id is not null as has_encounter_data + + from warehouse.sample + left join warehouse.encounter using (encounter_id) + left join specimen_manifest_data as manifest using (sample_id) + ) + + select + sample_id, + sample.identifier as sample, + has_encounter_data, + best_available_encounter_date, + + case + when best_available_encounter_date < '2019-10-01'::date then 'Y1' + when best_available_encounter_date < '2020-10-01'::date then 'Y2' + else null + end as season, + + coalesce(site.site_id, site_details.site_id) as best_available_site_id, + coalesce(site.identifier, site_details.site) as best_available_site, + coalesce(site.details->>'type', site_type) as best_available_site_type, + coalesce(site.details->>'category', site_category) as best_available_site_category + + from warehouse.sample + left join samples_with_manifest_data using (sample_id) + left join site_details on (site_manifest_details similar to manifest_regex) + left join warehouse.site on (samples_with_manifest_data.site_id = site.site_id) + where sample.identifier is not null + ; + +comment on view shipping.sample_with_best_available_encounter_data_v1 is + 'Version 1 of view of warehoused samples and their best available encounter date and site details important for metrics calculations'; + + +create or replace view shipping.incidence_model_observation_v4 as + + select encounter.identifier as encounter, + + to_char((encountered at time zone 'US/Pacific')::date, 'IYYY-"W"IW') as encountered_week, + + site.details->>'type' as site_type, + site.details->>'category' as site_category, + + individual.identifier as individual, + individual.sex, + + age_bin_fine_v2.range as age_range_fine, + age_in_years(lower(age_bin_fine_v2.range)) as age_range_fine_lower, + age_in_years(upper(age_bin_fine_v2.range)) as age_range_fine_upper, + + age_bin_coarse_v2.range as age_range_coarse, + age_in_years(lower(age_bin_coarse_v2.range)) as age_range_coarse_lower, + age_in_years(upper(age_bin_coarse_v2.range)) as age_range_coarse_upper, + + residence_puma, + residence_neighborhood_district, + + coalesce(encounter_responses.flu_shot,fhir.vaccine) as flu_shot, + coalesce(encounter_responses.symptoms,fhir.symptoms) as symptoms, + + sample.identifier as sample + + from warehouse.encounter + join warehouse.individual using (individual_id) + join warehouse.site using (site_id) + left join warehouse.sample using (encounter_id) + left join shipping.age_bin_fine_v2 on age_bin_fine_v2.range @> age + left join shipping.age_bin_coarse_v2 on age_bin_coarse_v2.range @> age + left join shipping.fhir_encounter_details_v1 as fhir using (encounter_id) + left join ( + select + encounter_id, + tract.hierarchy->'puma' as residence_puma, + tract.hierarchy->'neighborhood_district' as residence_neighborhood_district + from warehouse.encounter_location + left join warehouse.location as address using (location_id) + left join warehouse.tract as tract on (address.hierarchy -> 'tract') = tract.identifier + where relation = 'residence' + or relation = 'lodging' + ) as residence using (encounter_id), + + lateral ( + -- XXX TODO: The data in this subquery will be modeled better in the + -- future and the verbosity of extracting data from the JSON details + -- document will go away. + -- -trs, 22 March 2019 + + select -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(nullif(responses."FluShot"[1], 'doNotKnow'), 'dontKnow')::bool as flu_shot, + + -- XXX FIXME: Remove duplicate value collapsing when we're no + -- longer affected by this known Audere data quality issue. + array_distinct(responses."Symptoms") as symptoms + + from jsonb_to_record(encounter.details->'responses') + as responses ( + "FluShot" text[], + "Symptoms" text[])) + as encounter_responses + + order by encountered; + +comment on view shipping.incidence_model_observation_v4 is + 'Version 4 of view of warehoused encounters and important questionnaire responses for modeling and viz teams'; + +revoke all + on shipping.incidence_model_observation_v4 + from "incidence-modeler"; + +grant select + on shipping.incidence_model_observation_v4 + to "incidence-modeler"; + + +commit; diff --git a/schema/revert/shipping/views.sql b/schema/revert/shipping/views.sql index b14822a4e..228b409fe 100644 --- a/schema/revert/shipping/views.sql +++ b/schema/revert/shipping/views.sql @@ -669,10 +669,86 @@ comment on view shipping.sample_with_best_available_encounter_data_v1 is 'Version 1 of view of warehoused samples and their best available encounter date and site details important for metrics calculations'; +create or replace view shipping.incidence_model_observation_v4 as + + select encounter.identifier as encounter, + + to_char((encountered at time zone 'US/Pacific')::date, 'IYYY-"W"IW') as encountered_week, + + site.details->>'type' as site_type, + site.details->>'category' as site_category, + + individual.identifier as individual, + individual.sex, + + age_bin_fine_v2.range as age_range_fine, + age_in_years(lower(age_bin_fine_v2.range)) as age_range_fine_lower, + age_in_years(upper(age_bin_fine_v2.range)) as age_range_fine_upper, + + age_bin_coarse_v2.range as age_range_coarse, + age_in_years(lower(age_bin_coarse_v2.range)) as age_range_coarse_lower, + age_in_years(upper(age_bin_coarse_v2.range)) as age_range_coarse_upper, + + residence_puma, + residence_neighborhood_district, + + coalesce(encounter_responses.flu_shot,fhir.vaccine) as flu_shot, + coalesce(encounter_responses.symptoms,fhir.symptoms) as symptoms, + + sample.identifier as sample + + from warehouse.encounter + join warehouse.individual using (individual_id) + join warehouse.site using (site_id) + left join warehouse.sample using (encounter_id) + left join shipping.age_bin_fine_v2 on age_bin_fine_v2.range @> age + left join shipping.age_bin_coarse_v2 on age_bin_coarse_v2.range @> age + left join shipping.fhir_encounter_details_v1 as fhir using (encounter_id) + left join ( + select + encounter_id, + tract.hierarchy->'puma' as residence_puma, + tract.hierarchy->'neighborhood_district' as residence_neighborhood_district + from warehouse.encounter_location + left join warehouse.location as address using (location_id) + left join warehouse.tract as tract on (address.hierarchy -> 'tract') = tract.identifier + where relation = 'residence' + or relation = 'lodging' + ) as residence using (encounter_id), + + lateral ( + -- XXX TODO: The data in this subquery will be modeled better in the + -- future and the verbosity of extracting data from the JSON details + -- document will go away. + -- -trs, 22 March 2019 + + select -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(nullif(responses."FluShot"[1], 'doNotKnow'), 'dontKnow')::bool as flu_shot, + + -- XXX FIXME: Remove duplicate value collapsing when we're no + -- longer affected by this known Audere data quality issue. + array_distinct(responses."Symptoms") as symptoms + + from jsonb_to_record(encounter.details->'responses') + as responses ( + "FluShot" text[], + "Symptoms" text[])) + as encounter_responses + + order by encountered; + +comment on view shipping.incidence_model_observation_v4 is + 'Version 4 of view of warehoused encounters and important questionnaire responses for modeling and viz teams'; + revoke all on shipping.incidence_model_observation_v4 from "incidence-modeler"; -drop view shipping.incidence_model_observation_v4; +grant select + on shipping.incidence_model_observation_v4 + to "incidence-modeler"; + +drop view shipping.metadata_for_augur_build_v4; commit; diff --git a/schema/revert/shipping/views@2020-02-24.sql b/schema/revert/shipping/views@2020-02-24.sql new file mode 100644 index 000000000..b14822a4e --- /dev/null +++ b/schema/revert/shipping/views@2020-02-24.sql @@ -0,0 +1,678 @@ +-- Deploy seattleflu/id3c-customizations:shipping/views to pg +-- requires: seattleflu/schema:shipping/schema + +-- Hello! All shipping views are defined here. Rework this change with Sqitch +-- to change a view definition or add new views. This workflow helps keep +-- inter-view dependencies manageable. + +begin; + +-- This view is versioned as a hedge against future changes. Changing this +-- view in place is fine as long as changes are backwards compatible. Think of +-- the version number as the major part of a semantic versioning scheme. If +-- there needs to be a lag between view development and consumers being +-- updated, copy the view definition into v2 and make changes there. + +create or replace view shipping.reportable_condition_v1 as + + with reportable as ( + select array_agg(lineage) as lineages + from warehouse.organism + where details @> '{"report_to_public_health":true}' + ) + + select + presence_absence.presence_absence_id, + organism.lineage::text, + sample.identifier as sample, + barcode, + site.identifier as site, + presence_absence.details->>'reporting_log' as reporting_log, + sample.details->'_provenance'->>'workbook' as workbook, + sample.details->'_provenance'->>'sheet' as sheet, + sample.details->>'sample_origin' as sample_origin, + sample.details->>'swab_site' as swab_site + + from warehouse.presence_absence + join warehouse.target using (target_id) + join warehouse.organism using (organism_id) + join warehouse.sample using (sample_id) + join warehouse.identifier on ( + sample.identifier = cast(identifier.uuid as text)) + left join warehouse.encounter using (encounter_id) + left join warehouse.site using (site_id) + + where organism.lineage <@ (table reportable) + and present + + order by encountered desc; + + +drop view shipping.metadata_for_augur_build_v2; +create or replace view shipping.metadata_for_augur_build_v2 as + + select sample as strain, + cast(encountered as date) as date, + 'seattle' as region, + -- XXX TODO: Change to PUMA and neighborhoods + residence_census_tract as location, + 'Seattle Flu Study' as authors, + age_range_coarse, + case age_range_coarse <@ '[0 mon, 18 years)'::intervalrange + when 't' then 'child' + when 'f' then 'adult' + end as age_category, + case + when site_type in ('childrensHospital', 'childrensClinic', 'childrensHospital', 'clinic', 'hospital', 'retrospective') then 'clinical' + when site_type in ('childcare' , 'collegeCampus' , 'homelessShelter' , 'port', 'publicSpace', 'workplace') then 'community' + end as site_category, + residence_census_tract, + flu_shot, + sex + + from shipping.incidence_model_observation_v3 + join warehouse.encounter on encounter.identifier = incidence_model_observation_v3.encounter; + +comment on view shipping.metadata_for_augur_build_v2 is + 'View of metadata necessary for SFS augur build'; + + +create or replace view shipping.genomic_sequences_for_augur_build_v1 as + + select distinct on (sample.identifier, organism.lineage, segment) + sample.identifier as sample, + organism.lineage as organism, + genomic_sequence.segment, + round(((length(replace(seq, 'N', '')) * 1.0 / length(seq))), 4) as coverage, + genomic_sequence.seq + from warehouse.sample + join warehouse.consensus_genome using (sample_id) + join warehouse.genomic_sequence using (consensus_genome_id) + join warehouse.organism using (organism_id) + + order by sample.identifier, organism.lineage, segment, coverage desc; + +comment on view shipping.genomic_sequences_for_augur_build_v1 is + 'View of genomic sequences for SFS augur build'; + + +create or replace view shipping.flu_assembly_jobs_v1 as + + select sample.identifier as sfs_uuid, + sample.details ->> 'nwgc_id' as nwgc_id, + sequence_read_set.urls, + target.identifier as target, + o1.lineage as target_linked_organism, + coalesce(o2.lineage, o1.lineage) as assembly_job_organism, + sequence_read_set.details + + from warehouse.sequence_read_set + join warehouse.sample using (sample_id) + join warehouse.presence_absence using (sample_id) + join warehouse.target using (target_id) + join warehouse.organism o1 using (organism_id) + -- Reason for o2 join: For pan-subtype targets like Flu_A_pan, + -- we want to spread out to the more specific lineages for assembly jobs. + -- Since this view is Flu specific, we are hard-coding it to spread just one level down. + left join warehouse.organism o2 on (o2.lineage ~ concat(o1.lineage::text, '.*{1}')::lquery) + + where o1.lineage ~ 'Influenza.*' + and present + -- The sequence read set details currently holds the state of assembly jobs for each lineage. + -- So if null, then definitely no jobs have been completed. + -- If not null, then check if the assembly job organism matches any key in the sequence read set details. + and (sequence_read_set.details is null + or not sequence_read_set.details ? coalesce(o2.lineage, o1.lineage)::text) + + order by sample.details ->> 'nwgc_id'; + +comment on view shipping.flu_assembly_jobs_v1 is + 'View of flu jobs that still need to be run through the assembly pipeline'; + + +create or replace view shipping.return_results_v1 as + + select barcode, + case + when sample_id is null then 'notReceived' + when sample_id is not null and count(present) = 0 then 'processing' + when count(present) > 0 then 'complete' + end as status, + -- We only return the top level organisms for results so we want to omit subtypes + array_agg(distinct subpath(organism, 0, 1)::text) + filter (where present and organism is not null) as organisms_present, + array_agg(distinct subpath(organism, 0, 1)::text) + filter (where not present and organism is not null) as organisms_absent + + from warehouse.identifier + join warehouse.identifier_set using (identifier_set_id) + left join warehouse.sample on uuid::text = sample.collection_identifier + left join shipping.presence_absence_result_v1 on sample.identifier = presence_absence_result_v1.sample + + --These are all past and current collection identifier sets not including self-test + where identifier_set.name in ('collections-seattleflu.org', + 'collections-kiosks', + 'collections-environmental', + 'collections-swab&send', + 'collections-household-observation', + 'collections-household-intervention') and + (organism is null or + -- We only return results for these organisms, so omit all other presence/absence results + organism <@ '{"Adenovirus", "Human_coronavirus", "Enterovirus", "Influenza", "Human_metapneumovirus", "Human_parainfluenza", "Rhinovirus", "RSV"}'::ltree[]) + group by barcode, sample_id + order by barcode; + +comment on view shipping.return_results_v1 is + 'View of barcodes and presence/absence results for return of results on website'; + + +create or replace view shipping.fhir_encounter_details_v1 as + + with + fhir as ( + select encounter_id, + case + when details -> 'Condition' is not null then array_agg(distinct condition."id" order by condition."id") + else null + end as symptoms, + jsonb_object_agg(response."linkId", response.answer order by response.answer) filter (where response."linkId" is not null) as responses + from warehouse.encounter + left join jsonb_to_recordset(details -> 'Condition') as condition("id" text) on true + left join jsonb_to_recordset(details -> 'QuestionnaireResponse') as q("item" jsonb) on true + left join jsonb_to_recordset(q."item") as response("linkId" text, "answer" jsonb) on true + group by encounter_id + ), + + vaccine as ( + --Builds vaccine jsonb object without null values that spans multiple rows. + --Expand the rows into key/value pairs with jsonb_each and + --aggregate that back into a single jsonb value. + -- -Jover 10 Jan 2020 + select encounter_id, + jsonb_object_agg(t.k, t.v order by t.v) filter (where t.k is not null) as vaccine + from ( + select encounter_id, + jsonb_strip_nulls( + jsonb_build_object( + 'vaccine', vaccine."valueBoolean", + 'vaccine_date', vaccine."valueDate")) as vaccine_obj + from fhir, + jsonb_to_record(responses) as response("vaccine" jsonb), + jsonb_to_recordset(response."vaccine") as vaccine("valueBoolean" bool, "valueDate" text)) as vaccine, + jsonb_each(vaccine.vaccine_obj) as t(k,v) + group by encounter_id + ), + + array_responses as ( + select + encounter_id, + case + when responses -> 'race' is not null then array_agg(coalesce(race."valueCoding" ->> 'code', race."valueString") order by race."valueString") + else null + end as race, + case + when responses -> 'insurance' is not null then array_agg(insurance."valueString" order by insurance."valueString") + else null + end as insurance + from fhir + left join jsonb_to_record(responses) as response("race" jsonb, "insurance" jsonb) on true + left join jsonb_to_recordset(response."race") as race ("valueCoding" jsonb, "valueString" text) on true + left join jsonb_to_recordset(response."insurance") as insurance ("valueString" text) on true + group by encounter_id, responses + ), + + boolean_responses as ( + select + encounter_id, + ethnicity."valueBoolean" as hispanic_or_latino, + travel_countries."valueBoolean" as travel_countries, + travel_states."valueBoolean" as travel_states + from fhir + left join jsonb_to_record(responses) as response("ethnicity" jsonb, "travel_countries" jsonb, "travel_states" jsonb) on true + left join jsonb_to_recordset(response."ethnicity") as ethnicity("valueBoolean" bool) on true + left join jsonb_to_recordset(response."travel_countries") as travel_countries("valueBoolean" bool) on true + left join jsonb_to_recordset(response."travel_states") as travel_states("valueBoolean" bool) on true + ) + + + select + encounter_id, + symptoms, + (vaccine ->> 'vaccine')::bool as vaccine, + vaccine ->> 'vaccine_date' as vaccine_date, + race, + insurance, + hispanic_or_latino, + travel_countries, + travel_states + from fhir + left join vaccine using (encounter_id) + left join array_responses using (encounter_id) + left join boolean_responses using (encounter_id); + +comment on view shipping.fhir_encounter_details_v1 is + 'A view of encounter details that are in FHIR format'; + +revoke all + on shipping.fhir_encounter_details_v1 + from "incidence-modeler"; + +grant select + on shipping.fhir_encounter_details_v1 + to "incidence-modeler"; + + +create or replace view shipping.incidence_model_observation_v1 as + + select encounter.identifier as encounter, + + (encountered at time zone 'US/Pacific')::date as encountered_date, + to_char((encountered at time zone 'US/Pacific')::date, 'IYYY-"W"IW') as encountered_week, + + site.details->>'type' as site_type, + + individual.identifier as individual, + individual.sex, + + -- Reporting 90 is more accurate than reporting nothing, and it will + -- never be a real value in our dataset. + -- + -- XXX TODO: This will be pre-processed out of the JSON in the future. + ceiling(age_in_years(age))::int as age, + + age_bin_fine.range as age_range_fine, + lower(age_bin_fine.range) as age_range_fine_lower, + upper(age_bin_fine.range) as age_range_fine_upper, + + age_bin_coarse.range as age_range_coarse, + lower(age_bin_coarse.range) as age_range_coarse_lower, + upper(age_bin_coarse.range) as age_range_coarse_upper, + + residence_census_tract, + work_census_tract, + + coalesce(encounter_responses.flu_shot,fhir.vaccine) as flu_shot, + coalesce(encounter_responses.symptoms,fhir.symptoms) as symptoms, + coalesce(encounter_responses.race,fhir.race) as race, + coalesce(encounter_responses.hispanic_or_latino,fhir.hispanic_or_latino) as hispanic_or_latino, + + sample.identifier as sample + + from warehouse.encounter + join warehouse.individual using (individual_id) + join warehouse.site using (site_id) + left join warehouse.sample using (encounter_id) + left join shipping.age_bin_fine on age_bin_fine.range @> ceiling(age_in_years(age))::int + left join shipping.age_bin_coarse on age_bin_coarse.range @> ceiling(age_in_years(age))::int + left join shipping.fhir_encounter_details_v1 as fhir using (encounter_id) + left join ( + select encounter_id, hierarchy->'tract' as residence_census_tract + from warehouse.encounter_location + left join warehouse.location using (location_id) + where relation = 'residence' + or relation = 'lodging' + ) as residence using (encounter_id) + left join ( + select encounter_id, hierarchy->'tract' as work_census_tract + from warehouse.encounter_location + left join warehouse.location using (location_id) + where relation = 'workplace' + ) as workplace using (encounter_id), + + lateral ( + -- XXX TODO: The data in this subquery will be modeled better in the + -- future and the verbosity of extracting data from the JSON details + -- document will go away. + -- -trs, 22 March 2019 + + select -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(nullif(responses."FluShot"[1], 'doNotKnow'), 'dontKnow')::bool as flu_shot, + + -- XXX FIXME: Remove duplicate value collapsing when we're no + -- longer affected by this known Audere data quality issue. + array_distinct(responses."Symptoms") as symptoms, + array_distinct(responses."Race") as race, + + -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(responses."HispanicLatino"[1], 'preferNotToSay')::bool as hispanic_or_latino + + from jsonb_to_record(encounter.details->'responses') + as responses ( + "FluShot" text[], + "Symptoms" text[], + "Race" text[], + "HispanicLatino" text[])) + as encounter_responses + + order by encountered; + +comment on view shipping.incidence_model_observation_v1 is + 'View of warehoused encounters and important questionnaire responses for modeling and viz teams'; + +revoke all + on shipping.incidence_model_observation_v1 + from "incidence-modeler"; + +grant select + on shipping.incidence_model_observation_v1 + to "incidence-modeler"; + + +create or replace view shipping.incidence_model_observation_v2 as + + select encounter.identifier as encounter, + + (encountered at time zone 'US/Pacific')::date as encountered_date, + to_char((encountered at time zone 'US/Pacific')::date, 'IYYY-"W"IW') as encountered_week, + + site.identifier as site, + site.details->>'type' as site_type, + + individual.identifier as individual, + individual.sex, + + age_in_years(age) as age, + + age_bin_fine_v2.range as age_range_fine, + age_in_years(lower(age_bin_fine_v2.range)) as age_range_fine_lower, + age_in_years(upper(age_bin_fine_v2.range)) as age_range_fine_upper, + + age_bin_coarse_v2.range as age_range_coarse, + age_in_years(lower(age_bin_coarse_v2.range)) as age_range_coarse_lower, + age_in_years(upper(age_bin_coarse_v2.range)) as age_range_coarse_upper, + + residence_census_tract, + work_census_tract, + + coalesce(encounter_responses.flu_shot,fhir.vaccine) as flu_shot, + coalesce(encounter_responses.symptoms,fhir.symptoms) as symptoms, + coalesce(encounter_responses.race,fhir.race) as race, + coalesce(encounter_responses.hispanic_or_latino,fhir.hispanic_or_latino) as hispanic_or_latino, + + sample.identifier as sample + + from warehouse.encounter + join warehouse.individual using (individual_id) + join warehouse.site using (site_id) + left join warehouse.sample using (encounter_id) + left join shipping.age_bin_fine_v2 on age_bin_fine_v2.range @> age + left join shipping.age_bin_coarse_v2 on age_bin_coarse_v2.range @> age + left join shipping.fhir_encounter_details_v1 as fhir using (encounter_id) + left join ( + select encounter_id, hierarchy->'tract' as residence_census_tract + from warehouse.encounter_location + left join warehouse.location using (location_id) + where relation = 'residence' + or relation = 'lodging' + ) as residence using (encounter_id) + left join ( + select encounter_id, hierarchy->'tract' as work_census_tract + from warehouse.encounter_location + left join warehouse.location using (location_id) + where relation = 'workplace' + ) as workplace using (encounter_id), + + lateral ( + -- XXX TODO: The data in this subquery will be modeled better in the + -- future and the verbosity of extracting data from the JSON details + -- document will go away. + -- -trs, 22 March 2019 + + select -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(nullif(responses."FluShot"[1], 'doNotKnow'), 'dontKnow')::bool as flu_shot, + + -- XXX FIXME: Remove duplicate value collapsing when we're no + -- longer affected by this known Audere data quality issue. + array_distinct(responses."Symptoms") as symptoms, + array_distinct(responses."Race") as race, + + -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(responses."HispanicLatino"[1], 'preferNotToSay')::bool as hispanic_or_latino + + from jsonb_to_record(encounter.details->'responses') + as responses ( + "FluShot" text[], + "Symptoms" text[], + "Race" text[], + "HispanicLatino" text[])) + as encounter_responses + + order by encountered; + +comment on view shipping.incidence_model_observation_v2 is + 'Version 2 of view of warehoused encounters and important questionnaire responses for modeling and viz teams'; + +revoke all + on shipping.incidence_model_observation_v2 + from "incidence-modeler"; + +grant select + on shipping.incidence_model_observation_v2 + to "incidence-modeler"; + + +create or replace view shipping.observation_with_presence_absence_result_v1 as + + select target, + present, + present::int as presence, + observation.*, + organism + from shipping.incidence_model_observation_v2 as observation + join shipping.presence_absence_result_v1 using (sample) + order by site, encounter, sample, target; + +comment on view shipping.observation_with_presence_absence_result_v1 is + 'Joined view of shipping.incidence_model_observation_v2 and shipping.presence_absence_result_v1'; + + +create or replace view shipping.incidence_model_observation_v3 as + + select encounter.identifier as encounter, + + to_char((encountered at time zone 'US/Pacific')::date, 'IYYY-"W"IW') as encountered_week, + + site.details->>'type' as site_type, + + individual.identifier as individual, + individual.sex, + + age_bin_fine_v2.range as age_range_fine, + age_in_years(lower(age_bin_fine_v2.range)) as age_range_fine_lower, + age_in_years(upper(age_bin_fine_v2.range)) as age_range_fine_upper, + + age_bin_coarse_v2.range as age_range_coarse, + age_in_years(lower(age_bin_coarse_v2.range)) as age_range_coarse_lower, + age_in_years(upper(age_bin_coarse_v2.range)) as age_range_coarse_upper, + + residence_census_tract, + + coalesce(encounter_responses.flu_shot,fhir.vaccine) as flu_shot, + coalesce(encounter_responses.symptoms,fhir.symptoms) as symptoms, + + sample.identifier as sample + + from warehouse.encounter + join warehouse.individual using (individual_id) + join warehouse.site using (site_id) + left join warehouse.sample using (encounter_id) + left join shipping.age_bin_fine_v2 on age_bin_fine_v2.range @> age + left join shipping.age_bin_coarse_v2 on age_bin_coarse_v2.range @> age + left join shipping.fhir_encounter_details_v1 as fhir using (encounter_id) + left join ( + select encounter_id, hierarchy->'tract' as residence_census_tract + from warehouse.encounter_location + left join warehouse.location using (location_id) + where relation = 'residence' + or relation = 'lodging' + ) as residence using (encounter_id), + + lateral ( + -- XXX TODO: The data in this subquery will be modeled better in the + -- future and the verbosity of extracting data from the JSON details + -- document will go away. + -- -trs, 22 March 2019 + + select -- XXX FIXME: Remove use of nullif() when we're no longer + -- dealing with raw response values. + nullif(nullif(responses."FluShot"[1], 'doNotKnow'), 'dontKnow')::bool as flu_shot, + + -- XXX FIXME: Remove duplicate value collapsing when we're no + -- longer affected by this known Audere data quality issue. + array_distinct(responses."Symptoms") as symptoms + + from jsonb_to_record(encounter.details->'responses') + as responses ( + "FluShot" text[], + "Symptoms" text[])) + as encounter_responses + + order by encountered; + +comment on view shipping.incidence_model_observation_v3 is + 'Version 3 of view of warehoused encounters and important questionnaire responses for modeling and viz teams'; + +revoke all + on shipping.incidence_model_observation_v3 + from "incidence-modeler"; + +grant select + on shipping.incidence_model_observation_v3 + to "incidence-modeler"; + + +create or replace view shipping.observation_with_presence_absence_result_v2 as + + select target, + present, + present::int as presence, + observation.*, + organism + from shipping.incidence_model_observation_v3 as observation + join shipping.presence_absence_result_v1 using (sample) + order by site_type, encounter, sample, target; + +comment on view shipping.observation_with_presence_absence_result_v2 is + 'Joined view of shipping.incidence_model_observation_v3 and shipping.presence_absence_result_v1'; + + +create or replace view shipping.metadata_for_augur_build_v3 as + + select sample.identifier as strain, + coalesce( + encountered_date, + case + when date_or_null(sample.details->>'date') <= current_date + then date_or_null(sample.details->>'date') + end + ) as date, + 'seattle' as region, + -- XXX TODO: Change to PUMA and neighborhoods + residence_census_tract as location, + 'Seattle Flu Study' as authors, + age_range_coarse, + case age_range_coarse <@ '[0 mon, 18 years)'::intervalrange + when 't' then 'child' + when 'f' then 'adult' + end as age_category, + warehouse.site.details->>'category' as site_category, + residence_census_tract, + flu_shot, + sex + + from warehouse.sample + left join shipping.incidence_model_observation_v2 on sample.identifier = incidence_model_observation_v2.sample + left join warehouse.site on site.identifier = incidence_model_observation_v2.site + + where sample.identifier is not null; + +comment on view shipping.metadata_for_augur_build_v3 is + 'View of metadata necessary for SFS augur build'; + + +create or replace view shipping.sample_with_best_available_encounter_data_v1 as + + with specimen_manifest_data as ( + select + sample_id, + date_or_null(details->>'date') as collection_date, + trim(both ' ' from details->>'swab_site') as swab_site, + trim(both ' ' from details->>'sample_origin') as sample_origin + from + warehouse.sample + ), + + site_details as ( + select + site_id, + site.identifier as site, + site.details->>'type' as site_type, + site.details->>'category' as site_category, + coalesce(site.details->>'swab_site', site.details->>'sample_origin') as manifest_regex + from warehouse.site + ), + + samples_with_manifest_data as ( + select + sample_id, + site_id, + coalesce(encountered::date, collection_date) as best_available_encounter_date, + + coalesce( + case + -- Environmental samples must be processed first, because they're + -- often taken at existing human swab sites + when manifest.sample_origin = 'es' + then 'environmental' + else manifest.swab_site + end, + + manifest.sample_origin + ) as site_manifest_details, + + site_id is not null as has_encounter_data + + from warehouse.sample + left join warehouse.encounter using (encounter_id) + left join specimen_manifest_data as manifest using (sample_id) + ) + + select + sample_id, + sample.identifier as sample, + has_encounter_data, + best_available_encounter_date, + + case + when best_available_encounter_date < '2019-10-01'::date then 'Y1' + when best_available_encounter_date < '2020-10-01'::date then 'Y2' + else null + end as season, + + coalesce(site.site_id, site_details.site_id) as best_available_site_id, + coalesce(site.identifier, site_details.site) as best_available_site, + coalesce(site.details->>'type', site_type) as best_available_site_type, + coalesce(site.details->>'category', site_category) as best_available_site_category + + from warehouse.sample + left join samples_with_manifest_data using (sample_id) + left join site_details on (site_manifest_details similar to manifest_regex) + left join warehouse.site on (samples_with_manifest_data.site_id = site.site_id) + where sample.identifier is not null + ; + +comment on view shipping.sample_with_best_available_encounter_data_v1 is + 'Version 1 of view of warehoused samples and their best available encounter date and site details important for metrics calculations'; + + +revoke all + on shipping.incidence_model_observation_v4 + from "incidence-modeler"; +drop view shipping.incidence_model_observation_v4; + + +commit; diff --git a/schema/sqitch.plan b/schema/sqitch.plan index b2c8e1af8..2d749abe1 100644 --- a/schema/sqitch.plan +++ b/schema/sqitch.plan @@ -82,3 +82,4 @@ warehouse/site/data [warehouse/site/data@2020-02-20] 2020-02-21T19:49:28Z Kairst warehouse/views [warehouse/views@2020-02-06] 2020-02-18T22:20:31Z Jover Lee # Add PUMA and neighborhood scale location views shipping/views [shipping/views@2020-02-21] 2020-02-24T19:03:29Z Jover Lee # Add incidence model view v4 with PUMA and neighborhood district @2020-02-24 2020-02-24T19:27:10Z Jover Lee # Schema as of 24 Feb 2020 +shipping/views [shipping/views@2020-02-24] 2020-02-24T19:31:05Z Jover Lee # Add augur build metadata view v4 to export neighborhood or PUMA diff --git a/schema/verify/shipping/views.sql b/schema/verify/shipping/views.sql index b4e5b1b20..9d5122dc9 100644 --- a/schema/verify/shipping/views.sql +++ b/schema/verify/shipping/views.sql @@ -79,4 +79,9 @@ select 1/(count(*) = 1)::int where array[table_schema, table_name]::text[] = pg_catalog.parse_ident('shipping.incidence_model_observation_v4'); +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.metadata_for_augur_build_v4'); + rollback; diff --git a/schema/verify/shipping/views@2020-02-24.sql b/schema/verify/shipping/views@2020-02-24.sql new file mode 100644 index 000000000..b4e5b1b20 --- /dev/null +++ b/schema/verify/shipping/views@2020-02-24.sql @@ -0,0 +1,82 @@ +-- Verify seattleflu/id3c-customizations:shipping/views on pg +-- requires: seattleflu/schema:shipping/schema + +begin; + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.reportable_condition_v1'); + +-- Verify that the view has been dropped +select 1/(count(*) = 0)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.metadata_for_augur_build_v1'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.metadata_for_augur_build_v2'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.genomic_sequences_for_augur_build_v1'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.flu_assembly_jobs_v1'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.return_results_v1'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.fhir_encounter_details_v1'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.incidence_model_observation_v1'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.incidence_model_observation_v2'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.observation_with_presence_absence_result_v1'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.incidence_model_observation_v3'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.observation_with_presence_absence_result_v2'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.metadata_for_augur_build_v3'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.sample_with_best_available_encounter_data_v1'); + +select 1/(count(*) = 1)::int + from information_schema.views + where array[table_schema, table_name]::text[] + = pg_catalog.parse_ident('shipping.incidence_model_observation_v4'); + +rollback; From ce1d866e2ac46e9789d8d01461f2ee5ef5fe8f37 Mon Sep 17 00:00:00 2001 From: Jover Date: Mon, 24 Feb 2020 11:50:18 -0800 Subject: [PATCH 5/7] augur-build-exporter: Update grants Add `shipping.incidence_model_observation_v4`, `warehouse.puma`, and `warehouse.neighborhood_district`. Remove grants on `warehouse.tract`. --- .../roles/augur-build-exporter/grants.sql | 6 +++-- .../grants@2020-02-24.sql | 22 +++++++++++++++++++ .../roles/augur-build-exporter/grants.sql | 3 ++- .../grants@2020-02-24.sql | 21 ++++++++++++++++++ schema/sqitch.plan | 1 + .../roles/augur-build-exporter/grants.sql | 8 +++++-- .../grants@2020-02-24.sql | 17 ++++++++++++++ 7 files changed, 73 insertions(+), 5 deletions(-) create mode 100644 schema/deploy/roles/augur-build-exporter/grants@2020-02-24.sql create mode 100644 schema/revert/roles/augur-build-exporter/grants@2020-02-24.sql create mode 100644 schema/verify/roles/augur-build-exporter/grants@2020-02-24.sql diff --git a/schema/deploy/roles/augur-build-exporter/grants.sql b/schema/deploy/roles/augur-build-exporter/grants.sql index dfe1b223d..d857a5c84 100644 --- a/schema/deploy/roles/augur-build-exporter/grants.sql +++ b/schema/deploy/roles/augur-build-exporter/grants.sql @@ -13,10 +13,12 @@ grant usage to "augur-build-exporter"; grant select - on shipping.metadata_for_augur_build_v3, + on shipping.metadata_for_augur_build_v4, + shipping.metadata_for_augur_build_v3, shipping.metadata_for_augur_build_v2, shipping.genomic_sequences_for_augur_build_v1, - warehouse.tract + warehouse.puma, + warehouse.neighborhood_district to "augur-build-exporter"; commit; diff --git a/schema/deploy/roles/augur-build-exporter/grants@2020-02-24.sql b/schema/deploy/roles/augur-build-exporter/grants@2020-02-24.sql new file mode 100644 index 000000000..dfe1b223d --- /dev/null +++ b/schema/deploy/roles/augur-build-exporter/grants@2020-02-24.sql @@ -0,0 +1,22 @@ +-- Deploy seattleflu/id3c-customizations:roles/augur-build-exporter/grants to pg + +begin; + +revoke all on database :"DBNAME" from "augur-build-exporter"; +revoke all on schema receiving, warehouse, shipping from "augur-build-exporter"; +revoke all on all tables in schema receiving, warehouse, shipping from "augur-build-exporter"; + +grant connect on database :"DBNAME" to "augur-build-exporter"; + +grant usage + on schema shipping, warehouse + to "augur-build-exporter"; + +grant select + on shipping.metadata_for_augur_build_v3, + shipping.metadata_for_augur_build_v2, + shipping.genomic_sequences_for_augur_build_v1, + warehouse.tract + to "augur-build-exporter"; + +commit; diff --git a/schema/revert/roles/augur-build-exporter/grants.sql b/schema/revert/roles/augur-build-exporter/grants.sql index 2da3e83fb..dfe1b223d 100644 --- a/schema/revert/roles/augur-build-exporter/grants.sql +++ b/schema/revert/roles/augur-build-exporter/grants.sql @@ -13,7 +13,8 @@ grant usage to "augur-build-exporter"; grant select - on shipping.metadata_for_augur_build_v2, + on shipping.metadata_for_augur_build_v3, + shipping.metadata_for_augur_build_v2, shipping.genomic_sequences_for_augur_build_v1, warehouse.tract to "augur-build-exporter"; diff --git a/schema/revert/roles/augur-build-exporter/grants@2020-02-24.sql b/schema/revert/roles/augur-build-exporter/grants@2020-02-24.sql new file mode 100644 index 000000000..2da3e83fb --- /dev/null +++ b/schema/revert/roles/augur-build-exporter/grants@2020-02-24.sql @@ -0,0 +1,21 @@ +-- Deploy seattleflu/id3c-customizations:roles/augur-build-exporter/grants to pg + +begin; + +revoke all on database :"DBNAME" from "augur-build-exporter"; +revoke all on schema receiving, warehouse, shipping from "augur-build-exporter"; +revoke all on all tables in schema receiving, warehouse, shipping from "augur-build-exporter"; + +grant connect on database :"DBNAME" to "augur-build-exporter"; + +grant usage + on schema shipping, warehouse + to "augur-build-exporter"; + +grant select + on shipping.metadata_for_augur_build_v2, + shipping.genomic_sequences_for_augur_build_v1, + warehouse.tract + to "augur-build-exporter"; + +commit; diff --git a/schema/sqitch.plan b/schema/sqitch.plan index 2d749abe1..4e232c4e3 100644 --- a/schema/sqitch.plan +++ b/schema/sqitch.plan @@ -83,3 +83,4 @@ warehouse/views [warehouse/views@2020-02-06] 2020-02-18T22:20:31Z Jover Lee # Add incidence model view v4 with PUMA and neighborhood district @2020-02-24 2020-02-24T19:27:10Z Jover Lee # Schema as of 24 Feb 2020 shipping/views [shipping/views@2020-02-24] 2020-02-24T19:31:05Z Jover Lee # Add augur build metadata view v4 to export neighborhood or PUMA +roles/augur-build-exporter/grants [roles/augur-build-exporter/grants@2020-02-24] 2020-02-24T19:48:06Z Jover Lee # Update grants to augur-build-exporter for v4 diff --git a/schema/verify/roles/augur-build-exporter/grants.sql b/schema/verify/roles/augur-build-exporter/grants.sql index a52b87dd1..89a3e2172 100644 --- a/schema/verify/roles/augur-build-exporter/grants.sql +++ b/schema/verify/roles/augur-build-exporter/grants.sql @@ -4,14 +4,18 @@ begin; select 1/pg_catalog.has_database_privilege('augur-build-exporter', :'DBNAME', 'connect')::int; select 1/pg_catalog.has_schema_privilege('augur-build-exporter', 'shipping', 'usage')::int; +select 1/pg_catalog.has_table_privilege('augur-build-exporter', 'shipping.metadata_for_augur_build_v4', 'select')::int; select 1/pg_catalog.has_table_privilege('augur-build-exporter', 'shipping.metadata_for_augur_build_v3', 'select')::int; select 1/pg_catalog.has_table_privilege('augur-build-exporter', 'shipping.metadata_for_augur_build_v2', 'select')::int; select 1/pg_catalog.has_table_privilege('augur-build-exporter', 'shipping.genomic_sequences_for_augur_build_v1', 'select')::int; -select 1/pg_catalog.has_table_privilege('augur-build-exporter', 'warehouse.tract', 'select')::int; +select 1/pg_catalog.has_table_privilege('augur-build-exporter', 'warehouse.puma', 'select')::int; +select 1/pg_catalog.has_table_privilege('augur-build-exporter', 'warehouse.neighborhood_district', 'select')::int; +select 1/(not pg_catalog.has_table_privilege('augur-build-exporter', 'shipping.metadata_for_augur_build_v4', 'insert,update,delete'))::int; select 1/(not pg_catalog.has_table_privilege('augur-build-exporter', 'shipping.metadata_for_augur_build_v3', 'insert,update,delete'))::int; select 1/(not pg_catalog.has_table_privilege('augur-build-exporter', 'shipping.metadata_for_augur_build_v2', 'insert,update,delete'))::int; select 1/(not pg_catalog.has_table_privilege('augur-build-exporter', 'shipping.genomic_sequences_for_augur_build_v1', 'insert,update,delete'))::int; -select 1/(not pg_catalog.has_table_privilege('augur-build-exporter', 'warehouse.tract', 'insert,update,delete'))::int; +select 1/(not pg_catalog.has_table_privilege('augur-build-exporter', 'warehouse.puma', 'insert,update,delete'))::int; +select 1/(not pg_catalog.has_table_privilege('augur-build-exporter', 'warehouse.neighborhood_district', 'insert,update,delete'))::int; rollback; diff --git a/schema/verify/roles/augur-build-exporter/grants@2020-02-24.sql b/schema/verify/roles/augur-build-exporter/grants@2020-02-24.sql new file mode 100644 index 000000000..a52b87dd1 --- /dev/null +++ b/schema/verify/roles/augur-build-exporter/grants@2020-02-24.sql @@ -0,0 +1,17 @@ +-- Verify seattleflu/id3c-customizations:roles/augur-build-exporter/grants on pg + +begin; + +select 1/pg_catalog.has_database_privilege('augur-build-exporter', :'DBNAME', 'connect')::int; +select 1/pg_catalog.has_schema_privilege('augur-build-exporter', 'shipping', 'usage')::int; +select 1/pg_catalog.has_table_privilege('augur-build-exporter', 'shipping.metadata_for_augur_build_v3', 'select')::int; +select 1/pg_catalog.has_table_privilege('augur-build-exporter', 'shipping.metadata_for_augur_build_v2', 'select')::int; +select 1/pg_catalog.has_table_privilege('augur-build-exporter', 'shipping.genomic_sequences_for_augur_build_v1', 'select')::int; +select 1/pg_catalog.has_table_privilege('augur-build-exporter', 'warehouse.tract', 'select')::int; + +select 1/(not pg_catalog.has_table_privilege('augur-build-exporter', 'shipping.metadata_for_augur_build_v3', 'insert,update,delete'))::int; +select 1/(not pg_catalog.has_table_privilege('augur-build-exporter', 'shipping.metadata_for_augur_build_v2', 'insert,update,delete'))::int; +select 1/(not pg_catalog.has_table_privilege('augur-build-exporter', 'shipping.genomic_sequences_for_augur_build_v1', 'insert,update,delete'))::int; +select 1/(not pg_catalog.has_table_privilege('augur-build-exporter', 'warehouse.tract', 'insert,update,delete'))::int; + +rollback; From a7e2faaa8c9a86dc7c553a0932e1952d430bd36a Mon Sep 17 00:00:00 2001 From: Jover Date: Mon, 24 Feb 2020 11:51:17 -0800 Subject: [PATCH 6/7] Tag schema --- schema/sqitch.plan | 1 + 1 file changed, 1 insertion(+) diff --git a/schema/sqitch.plan b/schema/sqitch.plan index 4e232c4e3..c941e574f 100644 --- a/schema/sqitch.plan +++ b/schema/sqitch.plan @@ -84,3 +84,4 @@ shipping/views [shipping/views@2020-02-21] 2020-02-24T19:03:29Z Jover Lee # Schema as of 24 Feb 2020 shipping/views [shipping/views@2020-02-24] 2020-02-24T19:31:05Z Jover Lee # Add augur build metadata view v4 to export neighborhood or PUMA roles/augur-build-exporter/grants [roles/augur-build-exporter/grants@2020-02-24] 2020-02-24T19:48:06Z Jover Lee # Update grants to augur-build-exporter for v4 +@2020-02-24-b 2020-02-24T19:51:04Z Jover Lee # Schema as of later on 24 Feb 2020 From f5d974462e64dd44ac6615f2403375ff6766f08e Mon Sep 17 00:00:00 2001 From: Jover Date: Wed, 19 Feb 2020 14:09:12 -0800 Subject: [PATCH 7/7] api: Add metadata for augur build v4 endpoint --- lib/seattleflu/id3c/api/routes.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/seattleflu/id3c/api/routes.py b/lib/seattleflu/id3c/api/routes.py index 55471a8bd..eb7ffcc62 100644 --- a/lib/seattleflu/id3c/api/routes.py +++ b/lib/seattleflu/id3c/api/routes.py @@ -13,6 +13,9 @@ api_v3 = Blueprint('api_v3', 'api_v3', url_prefix='/v3') blueprints.append(api_v3) +api_v4 = Blueprint('api_v4', 'api_v4', url_prefix='/v4') +blueprints.append(api_v4) + @api_v1.route("/shipping/return-results/", methods = ['GET']) @cross_origin(origins=[ "https://seattleflu.org", @@ -62,6 +65,19 @@ def get_metadata_v3(session): return Response((row[0] + '\n' for row in metadata), mimetype="application/x-ndjson") +@api_v4.route("/shipping/augur-build-metadata", methods = ['GET']) +@authenticated_datastore_session_required +def get_metadata_v4(session): + """ + Export metadata needed for SFS augur build + """ + LOG.debug("Exporting metadata for SFS augur build") + + metadata = datastore.fetch_rows_from_table(session, ("shipping", "metadata_for_augur_build_v4")) + + return Response((row[0] + '\n' for row in metadata), mimetype="application/x-ndjson") + + @api_v1.route("/shipping/genomic-data//", methods = ['GET']) @authenticated_datastore_session_required def get_genomic_data(lineage, segment, session):