From 730e9241893d4ab77e4c22fd926ff20db0870b97 Mon Sep 17 00:00:00 2001 From: kleinone Date: Fri, 23 May 2014 18:08:10 -0700 Subject: [PATCH 01/62] Display number of selected samples in dropdown row. Added dropdown form stub for creating new sample lists. Removed print button. --- .../templates/seqpeek/mutations_map.hbs | 14 ++++++++- .../seqpeek/sample_list_dropdown_caption.hbs | 1 + app/scripts/views/seqpeek/view.js | 29 ++++++++++++++----- 3 files changed, 35 insertions(+), 9 deletions(-) create mode 100644 app/scripts/templates/seqpeek/sample_list_dropdown_caption.hbs diff --git a/app/scripts/templates/seqpeek/mutations_map.hbs b/app/scripts/templates/seqpeek/mutations_map.hbs index 46ebbe4..a8a27b8 100644 --- a/app/scripts/templates/seqpeek/mutations_map.hbs +++ b/app/scripts/templates/seqpeek/mutations_map.hbs @@ -27,9 +27,21 @@ {{/each}} +
  • + + + + +
    +
    + + + +
    +
    +
    • -
    diff --git a/app/scripts/templates/seqpeek/sample_list_dropdown_caption.hbs b/app/scripts/templates/seqpeek/sample_list_dropdown_caption.hbs new file mode 100644 index 0000000..c1e48b1 --- /dev/null +++ b/app/scripts/templates/seqpeek/sample_list_dropdown_caption.hbs @@ -0,0 +1 @@ + {{caption}} \ No newline at end of file diff --git a/app/scripts/views/seqpeek/view.js b/app/scripts/views/seqpeek/view.js index fd297c5..6019e12 100644 --- a/app/scripts/views/seqpeek/view.js +++ b/app/scripts/views/seqpeek/view.js @@ -4,10 +4,11 @@ define([ "seqpeek/util/data_adapters", "seqpeek/builders/builder_for_existing_elements", "hbs!templates/seqpeek/mutations_map", - "hbs!templates/seqpeek/mutations_map_table" + "hbs!templates/seqpeek/mutations_map_table", + "hbs!templates/seqpeek/sample_list_dropdown_caption" ], function ($, _, Backbone, d3, vq, - ProteinDomainModel, SeqPeekDataAdapters, SeqPeekBuilder, MutationsMapTpl, MutationsMapTableTpl) { + ProteinDomainModel, SeqPeekDataAdapters, SeqPeekBuilder, MutationsMapTpl, MutationsMapTableTpl, SampleListCaptionTpl) { var VARIANT_TRACK_MAX_HEIGHT = 150; var TICK_TRACK_HEIGHT = 25; var REGION_TRACK_HEIGHT = 10; @@ -110,10 +111,6 @@ define([ this.__enable_seqpeek_selection(); }, - "click .btn.seqpeek-print-ids": function(e) { - this.__print_selected_samples(); - }, - "click .btn.seqpeek-toggle-bars": function(e) { if (this.sample_track_type_user_setting == "bar_plot") { this.sample_track_type_user_setting = "sample_plot"; @@ -170,6 +167,8 @@ define([ }) })); + this.__update_sample_list_dropdown(); + return this; }, @@ -654,10 +653,24 @@ define([ __seqpeek_selection_handler: function(id_list) { this.selected_patient_ids = id_list; + + this.__update_sample_list_dropdown(); }, - __print_selected_samples: function() { - console.log(this.selected_patient_ids); + __update_sample_list_dropdown: function() { + var num_selected = this.selected_patient_ids.length; + var caption; + + if (num_selected == 0 || num_selected > 1) { + caption = num_selected + " Samples Selected"; + } + else { + caption = "1 Sample Selected"; + } + + this.$el.find(".sample-list-dropdown").html(SampleListCaptionTpl({ + caption: caption + })); } }); }); From 4eba9c8275ce2ef85802964b98e1a44604cf0d77 Mon Sep 17 00:00:00 2001 From: kleinone Date: Tue, 27 May 2014 13:58:28 -0700 Subject: [PATCH 02/62] Implemented sample list storing --- app/scripts/views/seqpeek/view.js | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/app/scripts/views/seqpeek/view.js b/app/scripts/views/seqpeek/view.js index 6019e12..0493c49 100644 --- a/app/scripts/views/seqpeek/view.js +++ b/app/scripts/views/seqpeek/view.js @@ -119,6 +119,10 @@ define([ this.sample_track_type_user_setting = "bar_plot"; } this.__render(); + }, + + "click .add-new-list": function() { + this.__store_sample_list(); } }, @@ -169,6 +173,11 @@ define([ this.__update_sample_list_dropdown(); + // Stop the dropdown from being hidden when the text field is clicked + this.$(".new-list-name").on("click", function(event) { + event.stopPropagation(); + }); + return this; }, @@ -671,6 +680,28 @@ define([ this.$el.find(".sample-list-dropdown").html(SampleListCaptionTpl({ caption: caption })); + }, + + __store_sample_list: function() { + var list_label = this.$el.find(".new-list-name").val(); + + if (list_label.length == 0 || this.selected_patient_ids.length == 0) { + return; + } + + this.$el.find(".new-list-name").val(""); + + var sample_list_document = { + "label": list_label, + "samples": this.selected_patient_ids + }; + + Backbone.sync("create", new Backbone.Model(sample_list_document), { + "url": "svc/collections/samplelists", "success": function() { + console.log("Succesfully created samplelist."); + console.log(arguments); + } + }); } }); }); From 50f17a0698efcf083ac2db85577bc679fd98d8fd Mon Sep 17 00:00:00 2001 From: kleinone Date: Tue, 27 May 2014 17:12:46 -0700 Subject: [PATCH 03/62] Added Sample Lists control to atlas --- app/scripts/templates/gs/atlas.hbs | 6 ++ .../templates/samplelist/container.hbs | 32 ++++++++ .../templates/seqpeek/mutations_map.hbs | 1 - app/scripts/views/gs/atlas.js | 22 +++++- app/scripts/views/samplelist/control.js | 77 +++++++++++++++++++ app/scripts/views/seqpeek/view.js | 3 +- 6 files changed, 136 insertions(+), 5 deletions(-) create mode 100644 app/scripts/templates/samplelist/container.hbs create mode 100644 app/scripts/views/samplelist/control.js diff --git a/app/scripts/templates/gs/atlas.hbs b/app/scripts/templates/gs/atlas.hbs index 0aeaa19..9f4ac2e 100644 --- a/app/scripts/templates/gs/atlas.hbs +++ b/app/scripts/templates/gs/atlas.hbs @@ -31,6 +31,9 @@
  • Clinical and Sample Variables
    • +
  • + Sample Lists +
  • @@ -56,6 +59,9 @@
    +
    +
    +
    diff --git a/app/scripts/templates/samplelist/container.hbs b/app/scripts/templates/samplelist/container.hbs new file mode 100644 index 0000000..38ca15e --- /dev/null +++ b/app/scripts/templates/samplelist/container.hbs @@ -0,0 +1,32 @@ +{{#if samplelists.length}} +

    Sample Lists

    +
    +
    +
    +
      + {{#each samplelists}} + {{label}} + {{/each}} +
    +
    + +
    + {{#each samplelists}} +
    +
    {{number_samples}} Samples
    + +
    +
    + + +
    +
    + {{/each}} +
    +
    +
    +{{else}} +

    Sample Lists

    +
    No sample lists stored
    +{{/if}} diff --git a/app/scripts/templates/seqpeek/mutations_map.hbs b/app/scripts/templates/seqpeek/mutations_map.hbs index a8a27b8..5351a7f 100644 --- a/app/scripts/templates/seqpeek/mutations_map.hbs +++ b/app/scripts/templates/seqpeek/mutations_map.hbs @@ -36,7 +36,6 @@
    -
    • diff --git a/app/scripts/views/gs/atlas.js b/app/scripts/views/gs/atlas.js index af6f327..ca9c1e3 100644 --- a/app/scripts/views/gs/atlas.js +++ b/app/scripts/views/gs/atlas.js @@ -5,14 +5,16 @@ define([ "views/gs/atlas_map", "views/genes/control", "views/clinvarlist/control", + "views/samplelist/control", "views/gs/tumor_types_control", "views/datamodel_collector/control", "views/collected_maps/control", "views/datasheets/control" + ], function ($, _, Backbone, AtlasTpl, MapsListContainerTpl, AtlasMapView, - GenelistControl, ClinicalListControl, TumorTypesControl, DatamodelCollectorControl, CollectedMapsControl, - DatasheetsControl) { + GenelistControl, ClinicalListControl, SampleListControl, TumorTypesControl, DatamodelCollectorControl, + CollectedMapsControl, DatasheetsControl) { return Backbone.View.extend({ "datasheetsControl": new DatasheetsControl({}), @@ -56,6 +58,7 @@ define([ this.model.set("atlas_map_views", []); this.model.on("load", this.__init_genelist_control, this); this.model.on("load", this.__init_clinicallist_control, this); + this.model.on("load", this.__init_samplelist_control, this); this.model.on("load", this.__init_tumortypes_control, this); this.model.on("load", this.__init_datamodel_collector, this); // this.model.on("load", this.__init_collected_maps_control, this); @@ -100,6 +103,21 @@ define([ this.$el.find(".clinvarlist-container").html(this.clinicalListControl.render().el); }, + __init_samplelist_control: function() { + this.sampleListControl = new SampleListControl({}); + this.sampleListControl.on("updated", function (ev) { + console.debug("atlas.__init_samplelist_control:updated:" + JSON.stringify(ev)); + if (ev["reorder"]) { + console.debug("atlas.__init_samplelist_control:updated:reorder:ignore"); + return; + } + + this.__reload_all_maps(); + }, this); + + this.$el.find(".samplelist-container").html(this.sampleListControl.render().el); + }, + __init_collected_maps_control: function() { this.collectedMapsControl = new CollectedMapsControl({}); this.collectedMapsControl.on("selected", function (ev) { diff --git a/app/scripts/views/samplelist/control.js b/app/scripts/views/samplelist/control.js new file mode 100644 index 0000000..5fe0dc7 --- /dev/null +++ b/app/scripts/views/samplelist/control.js @@ -0,0 +1,77 @@ +define([ + "jquery", + "underscore", + "backbone", + "views/genes/typeahead", + "hbs!templates/samplelist/container" +], +function ($, _, Backbone, + TypeAhead, + Tpl +) { + return Backbone.View.extend({ + samplelists_collection: new Backbone.Collection([], { "url": "svc/collections/samplelists" }), + itemizers: {}, + + events: { + "click .list-remover": function(e) { + var listid = $(e.target).data("id"); + + Backbone.sync("delete", new Backbone.Model({}), { + "url": "svc/collections/samplelists/" + listid, "success": this.__refresh + }); + }, + + "click .list-refresh": function() { + _.defer(this.__refresh); + } + }, + + initialize: function() { + _.bindAll(this, "__load", "__refresh", "__ready"); + }, + + render: function() { + this.samplelists_collection.fetch({ "success": this.__ready }); + this.samplelists_collection.on("change", function(item) { + if (_.isEmpty(item)) return; + Backbone.sync("update", item, { + "url": "svc/collections/samplelists/" + item.get("id"), "success": this.__refresh + }); + }); + + return this; + }, + + __ready: function() { + this.__load(); + this.trigger("ready"); + }, + + __refresh: function() { + this.samplelists_collection.fetch({ "success": this.__load }); + }, + + __load: function() { + var samplelists = _.map(this.samplelists_collection["models"], function(model) { + var samples = model.get("samples"); + var text_content = samples.join("\n"); + + return { + "id": model.get("id"), + "label": model.get("label"), + "samples": samples, + "text": text_content, + "number_samples": samples.length + }; + }); + + this.$el.html(Tpl({ "samplelists": _.sortBy(samplelists, "sort") })); + }, + + get_current: function() { + var currentSampleListId = this.$el.find(".nav-tabs").find("li.active").data("id"); + return this.itemizers[currentSampleListId].model.get("genes"); + } + }); +}); diff --git a/app/scripts/views/seqpeek/view.js b/app/scripts/views/seqpeek/view.js index 0493c49..4efb145 100644 --- a/app/scripts/views/seqpeek/view.js +++ b/app/scripts/views/seqpeek/view.js @@ -698,8 +698,7 @@ define([ Backbone.sync("create", new Backbone.Model(sample_list_document), { "url": "svc/collections/samplelists", "success": function() { - console.log("Succesfully created samplelist."); - console.log(arguments); + console.log("Succesfully stored samplelist"); } }); } From 1c90dded7073541ffb5e53b4fca34ec5e6562cb8 Mon Sep 17 00:00:00 2001 From: kleinone Date: Fri, 30 May 2014 19:13:56 -0700 Subject: [PATCH 04/62] Added item list collection to WebApp. Sample Lists in Atlas and sample list dropdown both use the collection in WebApp. Selected samples can be added to existing sample list in SeqPeek view. --- .../templates/seqpeek/mutations_map.hbs | 3 ++ app/scripts/views/samplelist/control.js | 50 ++++++++----------- .../seqpeek/sample_list_operations_view.js | 50 +++++++++++++++++++ app/scripts/views/seqpeek/view.js | 38 +++++++++++--- app/scripts/webapp.js | 20 ++++++-- 5 files changed, 121 insertions(+), 40 deletions(-) create mode 100644 app/scripts/views/seqpeek/sample_list_operations_view.js diff --git a/app/scripts/templates/seqpeek/mutations_map.hbs b/app/scripts/templates/seqpeek/mutations_map.hbs index 5351a7f..0618901 100644 --- a/app/scripts/templates/seqpeek/mutations_map.hbs +++ b/app/scripts/templates/seqpeek/mutations_map.hbs @@ -37,6 +37,9 @@ +
    + +
    diff --git a/app/scripts/views/samplelist/control.js b/app/scripts/views/samplelist/control.js index 5fe0dc7..ecd6d46 100644 --- a/app/scripts/views/samplelist/control.js +++ b/app/scripts/views/samplelist/control.js @@ -2,24 +2,18 @@ define([ "jquery", "underscore", "backbone", - "views/genes/typeahead", "hbs!templates/samplelist/container" ], function ($, _, Backbone, - TypeAhead, Tpl ) { return Backbone.View.extend({ - samplelists_collection: new Backbone.Collection([], { "url": "svc/collections/samplelists" }), - itemizers: {}, - + collection: null, events: { "click .list-remover": function(e) { var listid = $(e.target).data("id"); - Backbone.sync("delete", new Backbone.Model({}), { - "url": "svc/collections/samplelists/" + listid, "success": this.__refresh - }); + this.collection.remove(listid); }, "click .list-refresh": function() { @@ -28,45 +22,43 @@ function ($, _, Backbone, }, initialize: function() { - _.bindAll(this, "__load", "__refresh", "__ready"); + _.bindAll(this, "__refresh"); }, - render: function() { - this.samplelists_collection.fetch({ "success": this.__ready }); - this.samplelists_collection.on("change", function(item) { - if (_.isEmpty(item)) return; - Backbone.sync("update", item, { - "url": "svc/collections/samplelists/" + item.get("id"), "success": this.__refresh - }); - }); + __set_collection: function() { + if (WebApp !== undefined && this.collection === null) { + this.collection = WebApp.getItemSets(); - return this; + this.collection.on("add", this.__refresh, this); + this.collection.on("remove", this.__refresh, this); + this.collection.on("change", this.__refresh, this); + } }, - __ready: function() { - this.__load(); - this.trigger("ready"); + render: function() { + this.__set_collection(); + this.__refresh(); + + return this; }, __refresh: function() { - this.samplelists_collection.fetch({ "success": this.__load }); - }, + var data = this.collection.toJSON(); - __load: function() { - var samplelists = _.map(this.samplelists_collection["models"], function(model) { - var samples = model.get("samples"); + var template_data = _.map(data, function(model) { + var samples = model["samples"]; var text_content = samples.join("\n"); return { - "id": model.get("id"), - "label": model.get("label"), + "id": model["id"], + "label": model["label"], "samples": samples, "text": text_content, "number_samples": samples.length }; }); - this.$el.html(Tpl({ "samplelists": _.sortBy(samplelists, "sort") })); + this.$el.html(Tpl({ "samplelists": _.sortBy(template_data, "sort") })); }, get_current: function() { diff --git a/app/scripts/views/seqpeek/sample_list_operations_view.js b/app/scripts/views/seqpeek/sample_list_operations_view.js new file mode 100644 index 0000000..a1fe58a --- /dev/null +++ b/app/scripts/views/seqpeek/sample_list_operations_view.js @@ -0,0 +1,50 @@ +define([ + "jquery", + "underscore", + "backbone", + "hbs!templates/seqpeek/sample_list_operations" +], +function ($, _, Backbone, + Tpl + ) { + + return Backbone.View.extend({ + events: { + "click .list-union-op": function(e) { + var listid = $(e.target).data("id"); + this.trigger("list:union", this.collection.get(listid)); + } + }, + + initialize: function() { + _.bindAll(this, "__refresh"); + + this.collection.on("add remove change reset", this.__refresh, this); + }, + + render: function() { + this.__refresh(); + + return this; + }, + + __refresh: function() { + var data = this.collection.toJSON(); + + var template_data = _.map(data, function(model) { + var samples = model["samples"]; + var text_content = samples.join("\n"); + + return { + "id": model["id"], + "label": model["label"], + "samples": samples, + "text": text_content, + "number_samples": samples.length + }; + }); + + this.$el.html(Tpl({ "samplelists": _.sortBy(template_data, "sort") })); + } + }); +}); diff --git a/app/scripts/views/seqpeek/view.js b/app/scripts/views/seqpeek/view.js index 4efb145..bb08ae7 100644 --- a/app/scripts/views/seqpeek/view.js +++ b/app/scripts/views/seqpeek/view.js @@ -3,12 +3,17 @@ define([ "models/gs/protein_domain_model", "seqpeek/util/data_adapters", "seqpeek/builders/builder_for_existing_elements", + "views/seqpeek/sample_list_operations_view", "hbs!templates/seqpeek/mutations_map", "hbs!templates/seqpeek/mutations_map_table", "hbs!templates/seqpeek/sample_list_dropdown_caption" ], function ($, _, Backbone, d3, vq, - ProteinDomainModel, SeqPeekDataAdapters, SeqPeekBuilder, MutationsMapTpl, MutationsMapTableTpl, SampleListCaptionTpl) { + ProteinDomainModel, SeqPeekDataAdapters, SeqPeekBuilder, + SampleListOperationsView, + MutationsMapTpl, MutationsMapTableTpl, + SampleListCaptionTpl + ) { var VARIANT_TRACK_MAX_HEIGHT = 150; var TICK_TRACK_HEIGHT = 25; var REGION_TRACK_HEIGHT = 10; @@ -137,6 +142,17 @@ define([ this.sample_track_type_user_setting = null; this.selected_patient_ids = []; + + this.samplelists = WebApp.getItemSets(); + + this.sample_list_op_view = new SampleListOperationsView({ + collection: this.samplelists + }); + + this.sample_list_op_view.on("list:union", this.__sample_list_union, this); + + this.samplelists.on("add", this.__update_stored_samplelists, this); + this.samplelists.on("remove", this.__update_stored_samplelists, this); }, render: function() { @@ -173,6 +189,8 @@ define([ this.__update_sample_list_dropdown(); + this.$el.find(".sample-list-operations").html(this.sample_list_op_view.render().el); + // Stop the dropdown from being hidden when the text field is clicked this.$(".new-list-name").on("click", function(event) { event.stopPropagation(); @@ -682,6 +700,18 @@ define([ })); }, + __sample_list_union: function(target_list_model) { + if (this.selected_patient_ids.length > 0) { + var sample_id_set = target_list_model.get("samples"); + Array.prototype.push.apply(sample_id_set, this.selected_patient_ids); + target_list_model.set({ + "samples": sample_id_set + }); + + this.samplelists.updateListModel(target_list_model); + } + }, + __store_sample_list: function() { var list_label = this.$el.find(".new-list-name").val(); @@ -696,11 +726,7 @@ define([ "samples": this.selected_patient_ids }; - Backbone.sync("create", new Backbone.Model(sample_list_document), { - "url": "svc/collections/samplelists", "success": function() { - console.log("Succesfully stored samplelist"); - } - }); + this.samplelists.addList(sample_list_document); } }); }); diff --git a/app/scripts/webapp.js b/app/scripts/webapp.js index fbd0573..feadbba 100755 --- a/app/scripts/webapp.js +++ b/app/scripts/webapp.js @@ -1,8 +1,11 @@ define(["jquery", "underscore", "backbone", "router", "models/sessions", "models/datamodel", "models/lookups", - "views/items_grid_view", "views/pivot_data_view", "views/search_control"], - function ($, _, Backbone, AppRouter, SessionsCollection, Datamodel, LookupsModel, ItemGridView, PivotDataView, SearchControl) { + "views/items_grid_view", "views/pivot_data_view", "views/search_control", + "models/gs/item_set"], + function ($, _, Backbone, AppRouter, SessionsCollection, Datamodel, LookupsModel, ItemGridView, PivotDataView, SearchControl, + GeneSpotItemSet + ) { WebApp = { Events: _.extend(Backbone.Events), @@ -23,7 +26,8 @@ define(["jquery", "underscore", "backbone", }, LocalSession: new Backbone.Model(), // TODO : Add Sync UserPreferences: new Backbone.Model(), - Search: new SearchControl() + Search: new SearchControl(), + ItemSets: new GeneSpotItemSet() }; WebApp.initialize = function () { @@ -70,7 +74,9 @@ define(["jquery", "underscore", "backbone", } }); - WebApp.LocalSession.fetch({ "url": "svc/collections/local_session" }) + WebApp.LocalSession.fetch({ "url": "svc/collections/local_session" }); + + WebApp.ItemSets.fetch(); }; WebApp.alert = function(alertEl, timeout) { @@ -80,7 +86,11 @@ define(["jquery", "underscore", "backbone", }, timeout || 2000); }; - _.bindAll(WebApp, "initialize"); + WebApp.getItemSets = function() { + return this.ItemSets; + }; + + _.bindAll(WebApp, "initialize", "getItemSets"); return WebApp; }); \ No newline at end of file From 576dd2cd6e468e91c630fa6fc0ec216012deecfd Mon Sep 17 00:00:00 2001 From: kleinone Date: Mon, 2 Jun 2014 17:17:33 -0700 Subject: [PATCH 05/62] Fixed sample list union operation. Fixed tabs in sample list dropdown in SeqPeek view. Fixed typo in Atlas samplelist template. --- app/scripts/templates/samplelist/container.hbs | 2 +- app/scripts/views/seqpeek/view.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/scripts/templates/samplelist/container.hbs b/app/scripts/templates/samplelist/container.hbs index 38ca15e..68ff729 100644 --- a/app/scripts/templates/samplelist/container.hbs +++ b/app/scripts/templates/samplelist/container.hbs @@ -15,7 +15,7 @@ {{#each samplelists}}
    {{number_samples}} Samples
    - +
    diff --git a/app/scripts/views/seqpeek/view.js b/app/scripts/views/seqpeek/view.js index bb08ae7..f8ff67f 100644 --- a/app/scripts/views/seqpeek/view.js +++ b/app/scripts/views/seqpeek/view.js @@ -705,7 +705,7 @@ define([ var sample_id_set = target_list_model.get("samples"); Array.prototype.push.apply(sample_id_set, this.selected_patient_ids); target_list_model.set({ - "samples": sample_id_set + "samples": _.unique(sample_id_set) }); this.samplelists.updateListModel(target_list_model); From 0009e211f6c776d1b9bc74f7bbfce2a95366320b Mon Sep 17 00:00:00 2001 From: kleinone Date: Mon, 2 Jun 2014 17:18:48 -0700 Subject: [PATCH 06/62] Cleanup --- app/scripts/views/samplelist/control.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/scripts/views/samplelist/control.js b/app/scripts/views/samplelist/control.js index ecd6d46..4c74400 100644 --- a/app/scripts/views/samplelist/control.js +++ b/app/scripts/views/samplelist/control.js @@ -5,6 +5,7 @@ define([ "hbs!templates/samplelist/container" ], function ($, _, Backbone, + SampleListContentsView, Tpl ) { return Backbone.View.extend({ @@ -29,9 +30,7 @@ function ($, _, Backbone, if (WebApp !== undefined && this.collection === null) { this.collection = WebApp.getItemSets(); - this.collection.on("add", this.__refresh, this); - this.collection.on("remove", this.__refresh, this); - this.collection.on("change", this.__refresh, this); + this.collection.on("add remove change", this.__refresh, this); } }, From c58d10d4bd934585c44ee313aa941b2fa6479fb2 Mon Sep 17 00:00:00 2001 From: kleinone Date: Mon, 2 Jun 2014 17:21:12 -0700 Subject: [PATCH 07/62] Cleanup --- app/scripts/views/samplelist/control.js | 1 - 1 file changed, 1 deletion(-) diff --git a/app/scripts/views/samplelist/control.js b/app/scripts/views/samplelist/control.js index 4c74400..0d7513a 100644 --- a/app/scripts/views/samplelist/control.js +++ b/app/scripts/views/samplelist/control.js @@ -5,7 +5,6 @@ define([ "hbs!templates/samplelist/container" ], function ($, _, Backbone, - SampleListContentsView, Tpl ) { return Backbone.View.extend({ From 2fe64e66a8195203c46ee1e65b98819be16d6d01 Mon Sep 17 00:00:00 2001 From: kleinone Date: Wed, 11 Jun 2014 12:59:52 -0700 Subject: [PATCH 08/62] Added sample list operations template. Iterate through collection in sample list views. --- .../seqpeek/sample_list_operations.hbs | 32 +++++++++++++++++++ app/scripts/views/samplelist/control.js | 32 ++++--------------- .../seqpeek/sample_list_operations_view.js | 18 +++++------ 3 files changed, 47 insertions(+), 35 deletions(-) create mode 100644 app/scripts/templates/seqpeek/sample_list_operations.hbs diff --git a/app/scripts/templates/seqpeek/sample_list_operations.hbs b/app/scripts/templates/seqpeek/sample_list_operations.hbs new file mode 100644 index 0000000..4ab1c1a --- /dev/null +++ b/app/scripts/templates/seqpeek/sample_list_operations.hbs @@ -0,0 +1,32 @@ +{{#if samplelists.length}} +

    Sample Lists

    +
    +
    +
    +
      + {{#each samplelists}} + {{label}} + {{/each}} +
    +
    + +
    + {{#each samplelists}} +
    +
    {{number_samples}} Samples
    + +
    +
    + + +
    +
    + {{/each}} +
    +
    +
    +{{else}} +

    Sample Lists

    +
    No sample lists stored
    +{{/if}} diff --git a/app/scripts/views/samplelist/control.js b/app/scripts/views/samplelist/control.js index 0d7513a..0781309 100644 --- a/app/scripts/views/samplelist/control.js +++ b/app/scripts/views/samplelist/control.js @@ -8,7 +8,6 @@ function ($, _, Backbone, Tpl ) { return Backbone.View.extend({ - collection: null, events: { "click .list-remover": function(e) { var listid = $(e.target).data("id"); @@ -22,34 +21,20 @@ function ($, _, Backbone, }, initialize: function() { - _.bindAll(this, "__refresh"); - }, - - __set_collection: function() { - if (WebApp !== undefined && this.collection === null) { - this.collection = WebApp.getItemSets(); + _.bindAll(this, "render"); - this.collection.on("add remove change", this.__refresh, this); - } + this.collection = WebApp.getItemSets(); + this.collection.on("add remove change", this.render, this); }, render: function() { - this.__set_collection(); - this.__refresh(); - - return this; - }, - - __refresh: function() { - var data = this.collection.toJSON(); - - var template_data = _.map(data, function(model) { - var samples = model["samples"]; + var template_data = this.collection.map(function(model) { + var samples = model.get("samples"); var text_content = samples.join("\n"); return { "id": model["id"], - "label": model["label"], + "label": model.get("label"), "samples": samples, "text": text_content, "number_samples": samples.length @@ -57,11 +42,8 @@ function ($, _, Backbone, }); this.$el.html(Tpl({ "samplelists": _.sortBy(template_data, "sort") })); - }, - get_current: function() { - var currentSampleListId = this.$el.find(".nav-tabs").find("li.active").data("id"); - return this.itemizers[currentSampleListId].model.get("genes"); + return this; } }); }); diff --git a/app/scripts/views/seqpeek/sample_list_operations_view.js b/app/scripts/views/seqpeek/sample_list_operations_view.js index a1fe58a..ccb58a1 100644 --- a/app/scripts/views/seqpeek/sample_list_operations_view.js +++ b/app/scripts/views/seqpeek/sample_list_operations_view.js @@ -1,8 +1,8 @@ define([ - "jquery", - "underscore", - "backbone", - "hbs!templates/seqpeek/sample_list_operations" + "jquery", + "underscore", + "backbone", + "hbs!templates/seqpeek/sample_list_operations" ], function ($, _, Backbone, Tpl @@ -19,7 +19,7 @@ function ($, _, Backbone, initialize: function() { _.bindAll(this, "__refresh"); - this.collection.on("add remove change reset", this.__refresh, this); + this.collection.on("add remove change", this.__refresh, this); }, render: function() { @@ -29,15 +29,13 @@ function ($, _, Backbone, }, __refresh: function() { - var data = this.collection.toJSON(); - - var template_data = _.map(data, function(model) { - var samples = model["samples"]; + var template_data = this.collection.map(function(model) { + var samples = model.get("samples"); var text_content = samples.join("\n"); return { "id": model["id"], - "label": model["label"], + "label": model.get("label"), "samples": samples, "text": text_content, "number_samples": samples.length From eedfb01869750a50eab43816def020644ed94d0c Mon Sep 17 00:00:00 2001 From: kleinone Date: Wed, 11 Jun 2014 17:15:01 -0700 Subject: [PATCH 09/62] Sample lists can be created and updated from Atlas --- app/scripts/templates/samplelist/container.hbs | 12 +++++++++--- app/scripts/views/samplelist/control.js | 18 +++++++++++++++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/app/scripts/templates/samplelist/container.hbs b/app/scripts/templates/samplelist/container.hbs index 68ff729..1939161 100644 --- a/app/scripts/templates/samplelist/container.hbs +++ b/app/scripts/templates/samplelist/container.hbs @@ -1,5 +1,9 @@ +

    Sample Lists

    +
    + + +
    {{#if samplelists.length}} -

    Sample Lists

    @@ -15,7 +19,10 @@ {{#each samplelists}}
    {{number_samples}} Samples
    - + +
    + +
    @@ -27,6 +34,5 @@
    {{else}} -

    Sample Lists

    No sample lists stored
    {{/if}} diff --git a/app/scripts/views/samplelist/control.js b/app/scripts/views/samplelist/control.js index 0781309..376ecd5 100644 --- a/app/scripts/views/samplelist/control.js +++ b/app/scripts/views/samplelist/control.js @@ -11,12 +11,28 @@ function ($, _, Backbone, events: { "click .list-remover": function(e) { var listid = $(e.target).data("id"); - this.collection.remove(listid); }, + "click .list-update": function(e) { + var listid = $(e.target).data("id"); + var text_content = this.$el.find("#samplelist-contents-" + listid).val(); + var sample_ids = text_content.match(/\S+/g); + this.collection.updateSampleList(listid, sample_ids); + }, + "click .list-refresh": function() { _.defer(this.__refresh); + }, + + "click .add-new-list": function(e) { + var list_label = this.$el.find(".new-list-name").val(); + + if (list_label.length == 0) { + return; + } + + this.collection.addSampleList(list_label, []); } }, From 5cfc06ab400fef972abe6a7e6694938ff7feefe3 Mon Sep 17 00:00:00 2001 From: kleinone Date: Wed, 11 Jun 2014 17:15:47 -0700 Subject: [PATCH 10/62] Fixed calls to sample list collection --- app/scripts/templates/seqpeek/mutations_map.hbs | 4 ++-- app/scripts/views/seqpeek/view.js | 15 ++------------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/app/scripts/templates/seqpeek/mutations_map.hbs b/app/scripts/templates/seqpeek/mutations_map.hbs index 0618901..85d6898 100644 --- a/app/scripts/templates/seqpeek/mutations_map.hbs +++ b/app/scripts/templates/seqpeek/mutations_map.hbs @@ -27,8 +27,8 @@ {{/each}} -
  • - +
  • + diff --git a/app/scripts/views/seqpeek/view.js b/app/scripts/views/seqpeek/view.js index f8ff67f..754712c 100644 --- a/app/scripts/views/seqpeek/view.js +++ b/app/scripts/views/seqpeek/view.js @@ -702,13 +702,7 @@ define([ __sample_list_union: function(target_list_model) { if (this.selected_patient_ids.length > 0) { - var sample_id_set = target_list_model.get("samples"); - Array.prototype.push.apply(sample_id_set, this.selected_patient_ids); - target_list_model.set({ - "samples": _.unique(sample_id_set) - }); - - this.samplelists.updateListModel(target_list_model); + this.samplelists.updateSampleListByUnion(target_list_model["id"], this.selected_patient_ids); } }, @@ -721,12 +715,7 @@ define([ this.$el.find(".new-list-name").val(""); - var sample_list_document = { - "label": list_label, - "samples": this.selected_patient_ids - }; - - this.samplelists.addList(sample_list_document); + this.samplelists.addSampleList(label, this.selected_patient_ids); } }); }); From 899ef509e30242779cd2b53a6e84220fdf65c839 Mon Sep 17 00:00:00 2001 From: kleinone Date: Wed, 11 Jun 2014 17:21:20 -0700 Subject: [PATCH 11/62] Added item set --- app/scripts/models/gs/item_set.js | 77 +++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 app/scripts/models/gs/item_set.js diff --git a/app/scripts/models/gs/item_set.js b/app/scripts/models/gs/item_set.js new file mode 100644 index 0000000..10c4518 --- /dev/null +++ b/app/scripts/models/gs/item_set.js @@ -0,0 +1,77 @@ +define([ + "jquery", + "underscore", + "backbone" +], +function ($, _, Backbone +) { + var URL = "svc/collections/samplelists"; + + return Backbone.Collection.extend({ + "url": URL, + + "model": Backbone.Model.extend({ + idAttribute: "_id" + }), + + initialize: function() { + this.on("add", this.__add_handler, this); + this.on("remove", this.__remove_handler, this); + this.on("change", this.__change_handler, this); + }, + + __add_handler: function() { + + }, + + __remove_handler: function(model, collection, index) { + Backbone.sync("delete", new Backbone.Model({}), { + "url": URL + "/" + model["id"], "success": this.__refresh + }); + }, + + __change_handler: function() { + + }, + + updateSampleList: function(model_id, sample_list) { + var model = this.get(model_id); + var successFn = _.bind(function() { + model.trigger("change"); + }, this); + + model.set({ + samples: sample_list + }); + + this.sync("update", model, { + url: URL + "/" + model["id"], + success: successFn, + context: this + }); + }, + + updateSampleListByUnion: function(model_id, sample_list) { + var sample_id_set = this.get(model_id).get("samples"); + Array.prototype.push.apply(sample_id_set, sample_list); + this.updateSampleList(model_id, _.unique(sample_id_set)); + }, + + addSampleList: function(label, sample_id_array) { + var sample_list_model = new this.model({ + "label": label, + "samples": sample_id_array + }); + + var successFn = _.bind(function(response, status) { + this.add(_.extend(sample_list_model, {"id": response["id"]})); + }, this); + + this.sync("create", sample_list_model, { + url: URL, + success: successFn, + context: this + }); + } + }); +}); From 399ed54df4d22c20e20b9ccc2ba04ece28cfda85 Mon Sep 17 00:00:00 2001 From: mdmiller53 Date: Thu, 12 Jun 2014 12:06:18 -0700 Subject: [PATCH 12/62] completed implmentation for adding statistics to feature matrix --- db/scripts/generate_stats_var_lookup.js | 114 +++++++++++------------- 1 file changed, 53 insertions(+), 61 deletions(-) diff --git a/db/scripts/generate_stats_var_lookup.js b/db/scripts/generate_stats_var_lookup.js index 59a7459..b81bbf1 100644 --- a/db/scripts/generate_stats_var_lookup.js +++ b/db/scripts/generate_stats_var_lookup.js @@ -1,5 +1,5 @@ /* - usage: mongo :/ fn_2_ffn_clin.js + usage: mongo :/ generate_stats_var_lookup.js */ function getCounts(values) { var retVal = {}; @@ -16,110 +16,102 @@ function getCounts(values) { return retVal; } +function median(values) { + values.sort(function(a, b){return a-b}); + var half = Math.floor(values.length/2); + if(values.length % 2) + retVal = values[half]; + else + retVal = (values[half-1] + values[half]) / 2.0; + return retVal; +} + function getStats(type, values) { var count = 0; - var retVal = [{}, {}, {}]; - retVal[0]["total"] = 0; - retVal[0]["valid"] = 0; + var retVal = new Object(); + retVal["counts"] = new Object(); + retVal["counts"]["total"] = 0; + retVal["counts"]["valid"] = 0; + + retVal["categories"] = new Object(); - retVal[2]["max"] = Number.MIN_VALUE; - retVal[2]["min"] = Number.MAX_VALUE; - retVal[2]["mean"] = 0; - retVal[2]["median"] = new Array(); + retVal["numeric"] = new Object(); + retVal["numeric"]["max"] = -Number.MAX_VALUE; + retVal["numeric"]["min"] = Number.MAX_VALUE; + retVal["numeric"]["mean"] = 0; + retVal["numeric"]["median"] = new Array(); for (var key in values) { - retVal[0]["total"]++; + retVal["counts"]["total"]++; if (values[key] != 'NA') { - retVal[0]["valid"]++; + retVal["counts"]["valid"]++; } - if (values[key] in retVal[1]) { - retVal[1][values[key]]++; + if (values[key] in retVal["categories"]) { + retVal["categories"][values[key]]++; } else { count++; - retVal[1][values[key]] = 1; + retVal["categories"][values[key]] = 1; } if ("N" == type && "NA" != values[key]) { - if (retVal[2]["max"] < values[key]) { - retVal[2]["max"] = values[key]; + if (retVal["numeric"]["max"] < values[key]) { + retVal["numeric"]["max"] = values[key]; } - if (retVal[2]["min"] > values[key]) { - retVal[2]["min"] = values[key]; + if (retVal["numeric"]["min"] > values[key]) { + retVal["numeric"]["min"] = values[key]; } - retVal[2]["mean"] += values[key]; - retval[2]["median"].push(values[key]); + retVal["numeric"]["mean"] += values[key]; + retVal["numeric"]["median"].push(values[key]); } } - + if (!((("C" == type || "B" == type) && count > 0) || (count > 0 && count < 21))) { - retVal[1] = null; + retVal["categories"] = null; } - - if ("N" != type || 0 == retVal[0]["valid"]) { - retVal[2] = null; + + if ("N" != type || 0 == retVal["counts"]["valid"]) { + retVal["numeric"] = null; } else { - retVal[2]["mean"] = retVal[2]["mean"] / retVal[0]["valid"]; + retVal["numeric"]["mean"] = retVal["numeric"]["mean"] / retVal["counts"]["valid"]; - retVal[2]["stddev"] = 0; + retVal["numeric"]["stddev"] = 0; for (var key in values) { if (values[key] != 'NA') { - retVal[2]["stddev"] += Math.pow((values[key] - retVal[2]["mean"]), 2); + retVal["numeric"]["stddev"] += Math.pow((values[key] - retVal["numeric"]["mean"]), 2); } } - retVal[2]["stddev"] = Math.sqrt(retVal[2]["stddev"] / retVal[0]["valid"]); + retVal["numeric"]["stddev"] = Math.sqrt(retVal["numeric"]["stddev"] / retVal["counts"]["valid"]); - retVal[2]["median"].sort(function(a, b){return a-b}); - retVal[2]["mad"] = retVal[2]["median"]; - var half = Math.floor(retVal[2]["median"].length/2); - if(retVal[2]["median"].length % 2) - retVal[2]["median"] = retVal[2]["median"][half]; - else - retVal[2]["median"] = (retVal[2]["median"][half-1] + retVal[2]["median"][half]) / 2.0; + retVal["numeric"]["mad"] = retVal["numeric"]["median"]; + retVal["numeric"]["median"] = median(retVal["numeric"]["median"]); - for (i = 0; i < retVal[2]["mad"].length; i++) { - retVal[2]["mad"][i] = Math.abs(retVal[2]["mad"][i] - retVal[2]["median"]); - retVal[2]["mad"].sort(function(a, b){return a-b}); - var half = Math.floor(retVal[2]["mad"].length/2); - if(retVal[2]["mad"].length % 2) - retVal[2]["mad"] = retVal[2]["mad"][half]; - else - retVal[2]["mad"] = (retVal[2]["mad"][half-1] + retVal[2]["mad"][half]) / 2.0; + for (i = 0; i < retVal["numeric"]["mad"].length; i++) { + retVal["numeric"]["mad"][i] = Math.abs(retVal["numeric"]["mad"][i] - retVal["numeric"]["median"]); } - + retVal["numeric"]["mad"] = median(retVal["numeric"]["mad"]); } - return retVal; }; print('start: ' + new Date()); var count = 0; -db.getSiblingDB("skcm_stats_lookup_test").dropDatabase(); -db.getSiblingDB("skcm_stats_lookup_test").fm_stats.ensureIndex({"id": 1}, {unique: true}); db.feature_matrix.find().forEach( function(doc) { if (0 == (count++ % 4096)) { print("processing record " + count + " " + new Date()); } - sibDB = db.getSiblingDB("skcm_stats_lookup_test"); var stats = getStats(doc["type"], doc["values"]); - var entry = { - "id":doc["id"], - "counts":stats[0] + db.feature_matrix.update({"id":doc["id"]}, {$set: {"statistics.counts": stats["counts"]}}); + if (stats["categories"]) { + db.feature_matrix.update({"id":doc["id"]}, {$set: {"statistics.categories":stats["categories"]}}) } - sibDB.fm_stats.insert(entry); - - var cats = stats[1]; - if (null != cats) { - sibDB.fm_stats.update({"id":doc["id"]}, {$set: {"categories":cats}}) - } - - var numeric = stats[2]; - if (null != numeric) { - sibDB.fm_stats.update({"id":doc["id"]}, {$set: {"numeric":numeric}}) + if (stats["numeric"]) { + db.feature_matrix.update({"id":doc["id"]}, {$set: {"statistics.numeric":stats["numeric"]}}) } } ) +print('total count: ' + count); print('end: ' + new Date()); From 51c03e9bd66b0bfa11d00064bdb4d23e36ddac74 Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Tue, 17 Jun 2014 14:17:35 -0700 Subject: [PATCH 13/62] changed filter for mutated samples to use code_potential_somatic --- db/scripts/mapreduce_mutated_samples.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/scripts/mapreduce_mutated_samples.js b/db/scripts/mapreduce_mutated_samples.js index 055d88a..f530dd3 100644 --- a/db/scripts/mapreduce_mutated_samples.js +++ b/db/scripts/mapreduce_mutated_samples.js @@ -56,7 +56,7 @@ var finalize = function(key, reducedValue) { var query = { "source":"GNAB", - "code":"y_n_somatic", + "code":"code_potential_somatic", "type": "B", "gene": { "$exists": true } }; From 7da98d89ed55578adf54565825b0dfaaf70ca8c7 Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Tue, 17 Jun 2014 14:23:25 -0700 Subject: [PATCH 14/62] generating unique list of feature tags --- db/scripts/featurematrix_tags_grouped.js | 39 +++++++++++++++ db/scripts/featurematrix_tags_lookup.js | 50 +++++++++++++++++++ .../generate_featurematrix_tags_lookup.js | 44 ---------------- 3 files changed, 89 insertions(+), 44 deletions(-) create mode 100644 db/scripts/featurematrix_tags_grouped.js create mode 100644 db/scripts/featurematrix_tags_lookup.js delete mode 100644 db/scripts/generate_featurematrix_tags_lookup.js diff --git a/db/scripts/featurematrix_tags_grouped.js b/db/scripts/featurematrix_tags_grouped.js new file mode 100644 index 0000000..4afb06e --- /dev/null +++ b/db/scripts/featurematrix_tags_grouped.js @@ -0,0 +1,39 @@ +/* +- Reads from "feature_tags" collection in LOOKUPS database +- Writes to map reduce output to "feature_tags_grouped" collection + +Usage: + + mongo --host=$HOST $DB_NAME featurematrix_tags_grouped.js + +Example of the type of features targeted for aggregation by this script. Must be run on each tumor_type feature matrix independently. +{ + "_id" : ObjectId("53a0a595044458b47fe9b0a9"), + "tag" : "10p15.1", + "feature_id" : "C:CNVR:10p15.1:chr10:5610350:5628570::BLCA-TP_Gistic_ROI_d_amp", + "source" : "CNVR", + "db_name" : "BLCA" +} + */ + +var result = db.feature_tags.aggregate([ + { + "$group": { + "_id": "$tag", + "tumor_types": { + "$addToSet": "$db_name" + } + } + } +]); +print("aggregate completed"); + +result["result"].forEach(function(d) { + db.feature_tags_grouped.insert({ + "tag": d["_id"], + "tumor_types": d["tumor_types"] + }); +}); +print("insert completed"); + +print("feature_tags_grouped:" + db.feature_tags_grouped.count()); diff --git a/db/scripts/featurematrix_tags_lookup.js b/db/scripts/featurematrix_tags_lookup.js new file mode 100644 index 0000000..7d3b366 --- /dev/null +++ b/db/scripts/featurematrix_tags_lookup.js @@ -0,0 +1,50 @@ +/* +- Reads from "feature_matrix" collection +- Aggregates tags and writes individual entries per tag to lookups database + +Usage: + + mongo --host=$HOST $DB_NAME featurematrix_tags_lookup.js --eval="var lookupsDbUri='hostname:port/LOOKUPS';" + +Example of the type of features targeted for aggregation by this script. Must be run on each tumor_type feature matrix independently. +{ + "_id" : ObjectId("52d484d43d8a1d038104f407"), + "id" : "B:GNAB:A1BG:chr19:58858172:58874214:-:y_n_somatic", + "strand" : "-", + "tags" : [ "KLHDC9", "PFDN2", "ARHGAP30", "PVRL4", "SRGAP2", "1q23.3" ] + "end" : 58874214, + "start" : 58858172, + "source" : "GNAB", + "chr" : "19", + "code" : "y_n_somatic", + "type" : "B" +} + */ + +var query = { + "$and": [ + { "tags": { "$exists": true } }, + { "tags": { "$nin": ["NO_MATCH"] } } + ] +}; + +var lookupsDb = connect(lookupsDbUri); +var db_name = db["_name"]; + +db.feature_matrix.find(query).forEach(function(d) { + var feature_tags = d["tags"]; + if (feature_tags === undefined || feature_tags.length <= 0) return; + + var feature_id = d["id"]; + var source = d["source"]; + feature_tags.forEach(function(tag) { + if (tag && tag != "") { + lookupsDb["feature_tags"].insert({ + "tag": tag, + "feature_id": feature_id, + "source": source, + "db_name": db_name + }); + } + }); +}); diff --git a/db/scripts/generate_featurematrix_tags_lookup.js b/db/scripts/generate_featurematrix_tags_lookup.js deleted file mode 100644 index 4461a4b..0000000 --- a/db/scripts/generate_featurematrix_tags_lookup.js +++ /dev/null @@ -1,44 +0,0 @@ -/* -- Extracts tags from "feature_matrix" collection into GLOBAL lookup collection - -Usage: - - mongo --host=$HOST $DB_NAME generate_featurematrix_tags_lookup.js --eval="var lookupsDbUri='hostname:port/LOOKUPS';" - */ - -print("script:started"); -var lookupsDb = connect(lookupsDbUri); -var db_name = db["_name"]; - -print("initial check=LOOKUPS:" + lookupsDb.all_tags.count()); - -print("update:started"); -db.feature_matrix.find( - { - "$and": [ - { "tags": { "$exists": true } }, - { "tags": { "$nin": ["NO_MATCH"] } } - ] - }, - { - "tags": true, - "id": true - } -).forEach(function (doc) { - if (!doc) return; - var tags = doc["tags"] || []; - if (tags.length <= 0) return; - - for (var i = 0; i < tags.length; i++) { - var tag = tags[i]; - if (tag) { - var details = { "id": doc["id"], "tumor_type": db_name }; - lookupsDb["all_tags"].update({ "tag": tag }, { "$push": { "features": details } }, true); - } - } -}); - -print("update:completed"); - -print("final check=LOOKUPS:" + lookupsDb.all_tags.count()); -print("script:completed"); From 43e27a1846fbdb539e14c642a24b117f9a1bdf18 Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Tue, 17 Jun 2014 14:23:52 -0700 Subject: [PATCH 15/62] updated database reference fixed sorting in autocomplete --- app/configurations/atlas.json | 2 +- app/scripts/views/genes/typeahead.js | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/configurations/atlas.json b/app/configurations/atlas.json index 6c4ab7e..8863ef6 100644 --- a/app/configurations/atlas.json +++ b/app/configurations/atlas.json @@ -1,6 +1,6 @@ { "default_genelist": ["BRCA1", "RAD51", "TP53", "KRAS", "CC2D1A"], - "all_tags_url": "svc/datastores/FFN/LOOKUPS/all_tags", + "all_tags_url": "svc/datastores/FFN/LOOKUPS/feature_tags_grouped", "maps": [ { "id": "mutations_combo", diff --git a/app/scripts/views/genes/typeahead.js b/app/scripts/views/genes/typeahead.js index 71786c6..02def51 100644 --- a/app/scripts/views/genes/typeahead.js +++ b/app/scripts/views/genes/typeahead.js @@ -9,7 +9,6 @@ define([ "jquery", "underscore", "backbone" ], var all_tags_url = this.options["url"] + "/search/tag"; this.$el.typeahead({ "source": function(q, p) { - if (!q || q.length < 2) return; $.ajax({ "url": all_tags_url, "data": { "term": q }, @@ -18,11 +17,13 @@ define([ "jquery", "underscore", "backbone" ], "success": function (json) { if (json && json["items"]) { var matching_tags = _.uniq(_.pluck(json["items"], "tag")); - if (!_.isEmpty(matching_tags)) p(matching_tags) + if (!_.isEmpty(matching_tags)) p(matching_tags.sort()); } } }); }, + "items": 16, + "minLength": 2, "updater": this.__typed }); return this; From af598b488356be0702c0977922fbd12d58d00133 Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Wed, 18 Jun 2014 11:36:08 -0700 Subject: [PATCH 16/62] added prominent link to TCGA publication guidelines --- app/configurations/display.json | 8 ++++++-- app/scripts/templates/gs/atlas.hbs | 4 ++++ app/scripts/templates/gs/atlas_quick_tutorial.hbs | 6 +++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/app/configurations/display.json b/app/configurations/display.json index 83d514c..6536297 100644 --- a/app/configurations/display.json +++ b/app/configurations/display.json @@ -7,13 +7,17 @@ "url": "http://cancergenome.nih.gov" }, { - "label": "Cancer Regulome Home", - "url": "http://www.cancerregulome.org" + "label": "TCGA Publication Guidelines", + "url": "http://cancergenome.nih.gov/publications/publicationguidelines" }, { "divider": true, "header": "About Us" }, + { + "label": "Cancer Regulome Home", + "url": "http://www.cancerregulome.org" + }, { "label": "Institute for Systems Biology", "url": "http://www.systemsbiology.org" diff --git a/app/scripts/templates/gs/atlas.hbs b/app/scripts/templates/gs/atlas.hbs index 0aeaa19..d3e5a4b 100644 --- a/app/scripts/templates/gs/atlas.hbs +++ b/app/scripts/templates/gs/atlas.hbs @@ -38,6 +38,10 @@
  • Data Sheets
    • +
  • +
  • + TCGA Publication Guidelines +
    • diff --git a/app/scripts/templates/gs/atlas_quick_tutorial.hbs b/app/scripts/templates/gs/atlas_quick_tutorial.hbs index 9106288..621e95c 100644 --- a/app/scripts/templates/gs/atlas_quick_tutorial.hbs +++ b/app/scripts/templates/gs/atlas_quick_tutorial.hbs @@ -1,5 +1,9 @@
    - This guide reviews the controls used on this application. It is available from the Visualizations menu. +

    + Before publishing any work using this data,
    + please consult the TCGA Publication Guidelines +

    +
    This guide reviews the controls used on this application. It is available from the Visualizations menu.
    From fb2d1e82df80a24550064a3c11ac6b35b2e712ec Mon Sep 17 00:00:00 2001 From: kleinone Date: Wed, 18 Jun 2014 13:19:17 -0700 Subject: [PATCH 17/62] Fixed list update --- app/scripts/models/gs/item_set.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/scripts/models/gs/item_set.js b/app/scripts/models/gs/item_set.js index 10c4518..e1d0ffc 100644 --- a/app/scripts/models/gs/item_set.js +++ b/app/scripts/models/gs/item_set.js @@ -34,6 +34,11 @@ function ($, _, Backbone }, + __createModelForSync: function(model) { + var data = _.omit(model.toJSON(), "uri", "id", "_id"); + return new Backbone.Model(data); + }, + updateSampleList: function(model_id, sample_list) { var model = this.get(model_id); var successFn = _.bind(function() { @@ -44,7 +49,7 @@ function ($, _, Backbone samples: sample_list }); - this.sync("update", model, { + this.sync("update", this.__createModelForSync(model), { url: URL + "/" + model["id"], success: successFn, context: this From f69df7ad9ac73426cb777771a23eb1b10d364007 Mon Sep 17 00:00:00 2001 From: kleinone Date: Wed, 18 Jun 2014 15:02:39 -0700 Subject: [PATCH 18/62] Removed feature matrices mutation map datamodel --- app/configurations/atlas.json | 8 -------- 1 file changed, 8 deletions(-) diff --git a/app/configurations/atlas.json b/app/configurations/atlas.json index 8863ef6..07d0019 100644 --- a/app/configurations/atlas.json +++ b/app/configurations/atlas.json @@ -23,14 +23,6 @@ "url_suffix": "/mutation_summary" }, "mutsig": "datamodel/mutations/mutsig_rankings", - "features": { - "uri": "datamodel/tcga_datawarehouse", - "url_suffix": "/feature_matrix", - "base_query": { - "source": "GNAB", - "label": "y_n_somatic" - } - }, "mutated_samples": { "uri": "datamodel/tcga_datawarehouse", "url_suffix": "/mutated_samples" From 8f0948c05576a49427581aa57aa60136c8be243c Mon Sep 17 00:00:00 2001 From: kleinone Date: Wed, 18 Jun 2014 17:08:57 -0700 Subject: [PATCH 19/62] Add "no data" label to genes for which data is not found. Draw track grid even if no data is found for selected gene. --- app/scripts/views/seqpeek/view.js | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/app/scripts/views/seqpeek/view.js b/app/scripts/views/seqpeek/view.js index 754712c..ea80744 100644 --- a/app/scripts/views/seqpeek/view.js +++ b/app/scripts/views/seqpeek/view.js @@ -155,6 +155,19 @@ define([ this.samplelists.on("remove", this.__update_stored_samplelists, this); }, + __update_gene_dropdown_labels: function(gene_to_uniprot_mapping) { + _.each(this.genes, function(gene_label) { + var $el = this.$el.find(".seqpeek-gene-selector a[data-id=" + gene_label + "]"); + + if (_.has(gene_to_uniprot_mapping, gene_label)) { + $el.text(gene_label); + } + else { + $el.text(gene_label + " NO DATA"); + } + }, this); + }, + render: function() { console.debug("seqpeek/view.render"); @@ -268,6 +281,19 @@ define([ var seqpeek_data = []; + this.__update_gene_dropdown_labels(this.gene_to_uniprot_mapping); + + if (! _.has(this.gene_to_uniprot_mapping, this.selected_gene)) { + this.$(".mutations_map_table").html(MutationsMapTableTpl({ + "items": data_items, + "total": { + samples: "No data", + percentOf: "NA" + }})); + + return; + } + var uniprot_id = this.gene_to_uniprot_mapping[this.selected_gene]; var protein_data = this.found_protein_domains[uniprot_id]; @@ -320,6 +346,10 @@ define([ this.__render_tracks(seqpeek_data, region_data, protein_data, seqpeek_tick_track_element, seqpeek_domain_track_element); }, + __render_no_data: function(mutation_data) { + + }, + __render_tracks: function(mutation_data, region_array, protein_data, seqpeek_tick_track_element, seqpeek_domain_track_element) { console.debug("seqpeek/view.__render_tracks"); From fc4b9c02eef917386c7883d120e59430ad2dffc0 Mon Sep 17 00:00:00 2001 From: kleinone Date: Thu, 19 Jun 2014 17:33:18 -0700 Subject: [PATCH 20/62] Fixed buttons in template --- app/scripts/templates/seqpeek/mutations_map.hbs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/app/scripts/templates/seqpeek/mutations_map.hbs b/app/scripts/templates/seqpeek/mutations_map.hbs index 85d6898..62f9c6f 100644 --- a/app/scripts/templates/seqpeek/mutations_map.hbs +++ b/app/scripts/templates/seqpeek/mutations_map.hbs @@ -28,7 +28,7 @@
  • - + @@ -38,12 +38,18 @@
    • - +
    - - - +
  • + +
    • +
  • + +
    • +
  • + +
    • From b3779a3b2527f9d62272c09ad164a2f4a8d23842 Mon Sep 17 00:00:00 2001 From: kleinone Date: Thu, 19 Jun 2014 17:33:47 -0700 Subject: [PATCH 21/62] Sample list dropdown opens and closes only by clicking the nav bar --- app/scripts/views/seqpeek/view.js | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/app/scripts/views/seqpeek/view.js b/app/scripts/views/seqpeek/view.js index ea80744..234dedd 100644 --- a/app/scripts/views/seqpeek/view.js +++ b/app/scripts/views/seqpeek/view.js @@ -204,9 +204,20 @@ define([ this.$el.find(".sample-list-operations").html(this.sample_list_op_view.render().el); - // Stop the dropdown from being hidden when the text field is clicked - this.$(".new-list-name").on("click", function(event) { - event.stopPropagation(); + var $sample_list_dropdown = $(this.$el.find("a.sample-list-dropdown")); + + // Manually open and close the sample list dialog dropdown. The dropdown + // will not close when clicking outside. + $sample_list_dropdown.on("click.dropdown.data-api", function(e) { + var parent = $(this.parentNode); + var is_open = parent.hasClass("open"); + + if (is_open == false) { + parent.addClass("open"); + } + else { + parent.removeClass("open"); + } }); return this; @@ -745,7 +756,7 @@ define([ this.$el.find(".new-list-name").val(""); - this.samplelists.addSampleList(label, this.selected_patient_ids); + this.samplelists.addSampleList(list_label, this.selected_patient_ids); } }); }); From 40c76993a676d66ae29600ca84951ba1f53a18b5 Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Mon, 23 Jun 2014 14:20:02 -0700 Subject: [PATCH 22/62] implemented autocomplete against new all_clinical database --- app/configurations/atlas.json | 1 + app/configurations/lookups.json | 3 -- app/scripts/views/clinvarlist/control.js | 2 +- app/scripts/views/clinvarlist/typeahead.js | 39 ++++++++++++++-------- app/scripts/views/gs/atlas.js | 2 +- 5 files changed, 28 insertions(+), 19 deletions(-) diff --git a/app/configurations/atlas.json b/app/configurations/atlas.json index 07d0019..48e1f57 100644 --- a/app/configurations/atlas.json +++ b/app/configurations/atlas.json @@ -1,6 +1,7 @@ { "default_genelist": ["BRCA1", "RAD51", "TP53", "KRAS", "CC2D1A"], "all_tags_url": "svc/datastores/FFN/LOOKUPS/feature_tags_grouped", + "all_clinical_url": "svc/datastores/FFN/LOOKUPS/all_clinical", "maps": [ { "id": "mutations_combo", diff --git a/app/configurations/lookups.json b/app/configurations/lookups.json index 5d1ff06..096441e 100644 --- a/app/configurations/lookups.json +++ b/app/configurations/lookups.json @@ -8,9 +8,6 @@ "NA": "Other" } }, - "clinical_variables": { - "url": "svc/datastores/FFN/LOOKUPS/clinical_variables" - }, "chromosomes": { "url": "svc/data/lookups/chromosomes", "model": "models/annotations" diff --git a/app/scripts/views/clinvarlist/control.js b/app/scripts/views/clinvarlist/control.js index cf78251..9bf3c55 100644 --- a/app/scripts/views/clinvarlist/control.js +++ b/app/scripts/views/clinvarlist/control.js @@ -78,7 +78,7 @@ define(["jquery", "underscore", "backbone", var itemizer = this.itemizers[gl_model.get("_id")] = new Itemizer({"el": $glList.find(".clinvar-selector"), "model": gl_model }); itemizer.render(); - var typeahead = new TypeAhead({ "el": $glList.find(".clin-typeahead") }); + var typeahead = new TypeAhead({ "el": $glList.find(".clin-typeahead"), "url": this.options["url"] }); typeahead.render(); typeahead.on("typed", function(clin) { var cv_from_model = _.map(gl_model.get("clinical_variables"), function(g) {return g;}); diff --git a/app/scripts/views/clinvarlist/typeahead.js b/app/scripts/views/clinvarlist/typeahead.js index 5f2bcc4..4fb2f3a 100644 --- a/app/scripts/views/clinvarlist/typeahead.js +++ b/app/scripts/views/clinvarlist/typeahead.js @@ -1,34 +1,45 @@ define([ "jquery", "underscore", "backbone" ], function ($, _, Backbone) { return Backbone.View.extend({ + "clinical_variables_by_label": {}, + initialize: function() { _.bindAll(this, "render", "__typed"); }, render: function() { - var clinical_variables = _.extend({}, WebApp.Lookups.get("clinical_variables").get("items")); - if (_.isEmpty(clinical_variables)) return this; - - this.clinical_variables_by_label = _.groupBy(clinical_variables, "label"); + var url = this.options["url"] + "/search/label"; + var label_bucket = this.clinical_variables_by_label; this.$el.typeahead({ - source: function (q, p) { - p(_.compact(_.flatten(_.map(q.toLowerCase().split(" "), function (qi) { - return _.map(clinical_variables, function (item) { - if (item["label"].toLowerCase().indexOf(qi) >= 0) return item["label"]; - return null; - }); - })))); - }, + "source": function (q, p) { + $.ajax({ + "url": url, + "data": { "term": q }, + "traditional": true, + "dataType": "json", + "success": function (json) { + if (json && json["items"]) { + var matching_labels = _.compact(_.map(json["items"], function(item) { + label_bucket[item["label"]] = item; + return item["label"]; + })); - updater: this.__typed + if (!_.isEmpty(matching_labels)) p(matching_labels.sort()); + } + } + }); + }, + "items": 16, + "minLength": 2, + "updater": this.__typed }); return this; }, __typed: function(clin) { - this.trigger("typed", _.first(this.clinical_variables_by_label[clin])); + this.trigger("typed", this.clinical_variables_by_label[clin]); return ""; } }); diff --git a/app/scripts/views/gs/atlas.js b/app/scripts/views/gs/atlas.js index 2e796e3..f17e1b0 100644 --- a/app/scripts/views/gs/atlas.js +++ b/app/scripts/views/gs/atlas.js @@ -92,7 +92,7 @@ define([ }, __init_clinicallist_control: function() { - this.clinicalListControl = new ClinicalListControl({}); + this.clinicalListControl = new ClinicalListControl({ "url": this.model.get("all_clinical_url") }); this.clinicalListControl.on("updated", function (ev) { console.debug("atlas.__init_clinicallist_control:updated:" + JSON.stringify(ev)); if (ev["reorder"]) { From 43f39365e21e91088f468fcd889d532a756b0c98 Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Tue, 24 Jun 2014 10:51:58 -0700 Subject: [PATCH 23/62] added ordering of feature sources #69 --- app/configurations/atlas.json | 3 +++ app/scripts/views/fmx_distributions/view.js | 9 +++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/app/configurations/atlas.json b/app/configurations/atlas.json index 48e1f57..94bb7dd 100644 --- a/app/configurations/atlas.json +++ b/app/configurations/atlas.json @@ -68,6 +68,9 @@ "url_suffix": "/feature_matrix", "query_clinical_variables": true } + }, + "feature_sources_order": { + "GEXP": 1 } } ] diff --git a/app/scripts/views/fmx_distributions/view.js b/app/scripts/views/fmx_distributions/view.js index cc52bee..f9f4af6 100644 --- a/app/scripts/views/fmx_distributions/view.js +++ b/app/scripts/views/fmx_distributions/view.js @@ -266,8 +266,13 @@ define(["jquery", "underscore", "backbone", var feature_sources = _.map(_.keys(fd_by_gene || {}), function (source) { var s_uid = uid++ + "-" + axis; fdefs_uid_by_source[source] = s_uid; - return { "uid": s_uid, "label": source.toUpperCase(), "item_class": "feature_defs" }; - }); + + var order_dict = this.options["feature_sources_order"] || {}; + var order = order_dict[source.toUpperCase()] || 100; + return { "uid": s_uid, "label": source.toUpperCase(), "item_class": "feature_defs", "order": order }; + }, this); + + feature_sources = _.sortBy(feature_sources, "order"); $feature_selector.append(FeatureDefsTpl({"axis": axis, "feature_sources": feature_sources})); _.each(fd_by_gene, function (features, source) { From 840565932e7a56e9262d950f6857a4460c61ab7e Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Tue, 24 Jun 2014 11:26:22 -0700 Subject: [PATCH 24/62] corrected data source --- app/configurations/atlas.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/configurations/atlas.json b/app/configurations/atlas.json index 94bb7dd..0827922 100644 --- a/app/configurations/atlas.json +++ b/app/configurations/atlas.json @@ -1,6 +1,6 @@ { "default_genelist": ["BRCA1", "RAD51", "TP53", "KRAS", "CC2D1A"], - "all_tags_url": "svc/datastores/FFN/LOOKUPS/feature_tags_grouped", + "all_tags_url": "svc/datastores/FFN/LOOKUPS/all_tags", "all_clinical_url": "svc/datastores/FFN/LOOKUPS/all_clinical", "maps": [ { From 027435a71ac47ae733fe64413fcd390e4ee59751 Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Tue, 24 Jun 2014 13:58:09 -0700 Subject: [PATCH 25/62] implemented uniqueness identifier to facilitate grouping features by source #72 --- app/scripts/views/fmx_distributions/view.js | 39 +++++++--- db/scripts/featurematrix_featureunique_id.js | 78 ++++++++++++++++++++ 2 files changed, 107 insertions(+), 10 deletions(-) create mode 100644 db/scripts/featurematrix_featureunique_id.js diff --git a/app/scripts/views/fmx_distributions/view.js b/app/scripts/views/fmx_distributions/view.js index f9f4af6..de68f98 100644 --- a/app/scripts/views/fmx_distributions/view.js +++ b/app/scripts/views/fmx_distributions/view.js @@ -214,7 +214,7 @@ define(["jquery", "underscore", "backbone", __load_fdefs_clinvars: function (tumor_type) { console.debug("fmx-dist.__load_fdefs_clinvars(" + tumor_type + ")"); _.each(this.options["clinical_variables"], function(item) { - this.feature_definitions_by_id[item.id] = _.extend({}, item); + this.feature_definitions_by_id[item["unid"] || item["id"]] = _.extend({}, item); }, this); this.__aggregate(tumor_type, this.model["clinical_features"]["by_tumor_type"][tumor_type]); }, @@ -222,10 +222,23 @@ define(["jquery", "underscore", "backbone", __aggregate: function(tumor_type, model) { console.debug("fmx-dist.__aggregate(" + tumor_type + ")"); _.each(model.get("items"), function(item) { - var a_f_by_id = this.aggregate_features_by_id[item.id]; - if (!a_f_by_id) a_f_by_id = this.aggregate_features_by_id[item.id] = {}; - a_f_by_id[tumor_type] = item; - this.feature_definitions_by_id[item.id] = _.omit(item, "values"); + var a_f_by_id = this.aggregate_features_by_id[item["unid"] || item["id"]]; + if (!a_f_by_id) a_f_by_id = this.aggregate_features_by_id[item["unid"] || item["id"]] = {}; + + if (_.has(a_f_by_id, tumor_type)) { + var existing = a_f_by_id[tumor_type]; + var overlap_values = _.extend({}, existing["values"], item["values"]); + _.each(_.keys(overlap_values), function(key) { + var value = overlap_values[key]; + if (_.isEqual(value, "NA")) value = item["values"][key]; + if (_.isEqual(value, "NA")) value = existing["values"][key]; + overlap_values[key] = value; + }); + a_f_by_id[tumor_type] = _.extend({}, existing, item, { "values": overlap_values }); + } else { + a_f_by_id[tumor_type] = item; + } + this.feature_definitions_by_id[item["unid"] || item["id"]] = _.omit(item, "values"); }, this); }, @@ -277,8 +290,12 @@ define(["jquery", "underscore", "backbone", _.each(fd_by_gene, function (features, source) { var collapserUL = $feature_selector.find("#tab-pane-" + fdefs_uid_by_source[source]); - _.each(_.sortBy(features, "label"), function (feature) { - collapserUL.append(LineItemTpl({ "label": feature["label"], "id": feature["id"], "a_class": "feature-selector-" + axis })); + var grouped_by_unid = _.groupBy(features, "unid"); + if (_.isEmpty(grouped_by_unid)) grouped_by_unid = _.groupBy(features, "id"); + _.each(grouped_by_unid, function(grouped_features) { + var feature = _.first(grouped_features); + if (!feature) return; + collapserUL.append(LineItemTpl({ "label": feature["label"], "id": feature["unid"] || feature["id"], "a_class": "feature-selector-" + axis })); }); }); @@ -336,9 +353,9 @@ define(["jquery", "underscore", "backbone", var data = null; if (this.selected_tumor_type) { - data = this.__visdata([this.selected_tumor_type], X_feature.id, Y_feature.id); + data = this.__visdata([this.selected_tumor_type], X_feature, Y_feature); } else { - data = this.__visdata(this.options["tumor_types"], X_feature.id, Y_feature.id); + data = this.__visdata(this.options["tumor_types"], X_feature, Y_feature); } if (_.isEmpty(data)) { this.latest_data = []; @@ -411,7 +428,9 @@ define(["jquery", "underscore", "backbone", } }, - __visdata: function (tumor_types, X_feature_id, Y_feature_id) { + __visdata: function (tumor_types, X_f, Y_f) { + var X_feature_id = X_f["unid"] || X_f["id"]; + var Y_feature_id = Y_f["unid"] || Y_f["id"]; var data = _.map(tumor_types, function (tumor_type) { var stl = this.sample_types_lookup[tumor_type] || {}; diff --git a/db/scripts/featurematrix_featureunique_id.js b/db/scripts/featurematrix_featureunique_id.js new file mode 100644 index 0000000..79afdca --- /dev/null +++ b/db/scripts/featurematrix_featureunique_id.js @@ -0,0 +1,78 @@ +/* +- Reads from "feature_matrix" collection +- Updates each feature with a "unid" that more coarsely defines uniqueness + +Usage: + + mongo --host=$HOST $DB_NAME featurematrix_featureunique_id.js + +Example of the type of features targeted for aggregation by this script. Must be run on each tumor_type feature matrix independently. +{ + "_id": "537bde73ab47b3606a2b2f5f", + "end": 7590856, + "chromosome": "chr17", + "start": 7565097, + "tags": [ + "TP53" + ], + "label": "TP53 (7157)", + "source": "GEXP", + "platform": "mRNAseq", + "gene": "TP53", + "type": "N", + "id": "N:GEXP:TP53:chr17:7565097:7590856:-:7157", + "strand": "-" +} + +Uniqueness_id -> "unid": "N:GEXP:TP53:7157" +*/ + +var db_name = db["_name"]; + +var UNID_builder = function(doc, key_ids) { + var keys = []; + key_ids.forEach(function(key_id) { + if (doc[key_id]) keys.push(doc[key_id]); + }); + return keys.join(":"); +}; + +var UNID_generators = { + "GEXP": function(doc) { + return UNID_builder(doc, ["type", "source", "gene", "code"]); + }, + "GNAB": function(doc) { + return UNID_builder(doc, ["type", "source", "gene", "code"]); + }, + "MIRN": function(doc) { + return UNID_builder(doc, ["type", "source", "accession_number", "code"]); + }, + "CNVR": function(doc) { + return UNID_builder(doc, ["type", "source", "chromosome", "start", "end", "code"]); + }, + "METH": function(doc) { + return UNID_builder(doc, ["type", "source", "probe"]); + }, + "RPPA": function(doc) { + return UNID_builder(doc, ["type", "source", "antibody"]); + }, + "CLIN": function(doc) { + return UNID_builder(doc, ["type", "source", "code"]); + }, + "SAMP": function(doc) { + return UNID_builder(doc, ["type", "source", "code"]); + } +}; + +for (var src in UNID_generators) { + print("[" + db_name + "]:updating:" + src + ":" + db["feature_matrix"].find({ "source": src }).count()); + + var unid_generator = UNID_generators[src]; + db["feature_matrix"].find({ "source": src }).forEach(function(doc) { + db["feature_matrix"].update({ + "_id": doc["_id"] + }, { + "$set": { "unid": unid_generator(doc) } + }); + }); +} \ No newline at end of file From a4b5570afaf9b5cc1f600f4638c7152ec410bcc7 Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Tue, 24 Jun 2014 15:05:11 -0700 Subject: [PATCH 26/62] color by labels for non-continuous variables #74 --- .../templates/fmx_distributions/container.hbs | 2 +- app/scripts/views/fmx_distributions/view.js | 37 ++++++++++++++++--- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/app/scripts/templates/fmx_distributions/container.hbs b/app/scripts/templates/fmx_distributions/container.hbs index 95662f3..750c43e 100755 --- a/app/scripts/templates/fmx_distributions/container.hbs +++ b/app/scripts/templates/fmx_distributions/container.hbs @@ -68,7 +68,7 @@ Color By -
    {{label}}
    - {{description}} + {{{description}}} {{#if publications}}
    Related Publications
    {{label}}
    - {{description}} + {{{description}}} {{#if publications}}
    Related Publications
      diff --git a/app/scripts/templates/gs/maps_list_container.hbs b/app/scripts/templates/gs/maps_list_container.hbs index 03306d4..5c0e810 100644 --- a/app/scripts/templates/gs/maps_list_container.hbs +++ b/app/scripts/templates/gs/maps_list_container.hbs @@ -3,14 +3,14 @@ {{#unless disabled}}
      {{label}}
      -

      {{description}}

      +

      {{{description}}}

      {{/unless}} {{#if disabled}}
      {{label}}
      -

      {{description}}

      +

      {{{description}}}

      This map is not currently available

      From 799b89b14cb052ef784e05c6d6ce4684cf099a68 Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Fri, 27 Jun 2014 11:17:39 -0700 Subject: [PATCH 34/62] filtering out numerical features from color by menu #70 --- app/scripts/views/fmx_distributions/view.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/scripts/views/fmx_distributions/view.js b/app/scripts/views/fmx_distributions/view.js index 6e02f5c..167671d 100644 --- a/app/scripts/views/fmx_distributions/view.js +++ b/app/scripts/views/fmx_distributions/view.js @@ -121,7 +121,9 @@ define(["jquery", "underscore", "backbone", this.$el.html(Tpl({ "id": this.id, "genes": this.options["genes"], - "clinical_variables": this.options["clinical_variables"], + "clinical_variables": _.filter(this.options["clinical_variables"], function(cv) { + return (_.has(cv, "id") && !_.isEqual(cv["id"].substring(0,2), "N:")); + }), "tumor_types": this.options["tumor_types"], "sample_types": this.sample_types, "selected_genes": this.selected_genes From 6ead996657f333a00c7371379a2c4e9bc003db79 Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Fri, 27 Jun 2014 11:40:00 -0700 Subject: [PATCH 35/62] renames --- db/scripts/{insert_featurematrix.py => featurematrix_insert.py} | 0 ...ated_samples.js => featurematrix_mapreduce_mutated_samples.js} | 0 ...nsert_mutationsummary_mongodb.py => mutationsummary_insert.py} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename db/scripts/{insert_featurematrix.py => featurematrix_insert.py} (100%) rename db/scripts/{mapreduce_mutated_samples.js => featurematrix_mapreduce_mutated_samples.js} (100%) rename db/scripts/{insert_mutationsummary_mongodb.py => mutationsummary_insert.py} (100%) diff --git a/db/scripts/insert_featurematrix.py b/db/scripts/featurematrix_insert.py similarity index 100% rename from db/scripts/insert_featurematrix.py rename to db/scripts/featurematrix_insert.py diff --git a/db/scripts/mapreduce_mutated_samples.js b/db/scripts/featurematrix_mapreduce_mutated_samples.js similarity index 100% rename from db/scripts/mapreduce_mutated_samples.js rename to db/scripts/featurematrix_mapreduce_mutated_samples.js diff --git a/db/scripts/insert_mutationsummary_mongodb.py b/db/scripts/mutationsummary_insert.py similarity index 100% rename from db/scripts/insert_mutationsummary_mongodb.py rename to db/scripts/mutationsummary_insert.py From 4dccef5c7f9d33f58bf4beb8cca1ffa2338fb429 Mon Sep 17 00:00:00 2001 From: kleinone Date: Mon, 30 Jun 2014 10:52:22 -0700 Subject: [PATCH 36/62] Working on non-coding mode --- .../templates/seqpeek/mutations_map_table.hbs | 2 +- app/scripts/views/seqpeek/view.js | 61 +++++++++++++++---- 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/app/scripts/templates/seqpeek/mutations_map_table.hbs b/app/scripts/templates/seqpeek/mutations_map_table.hbs index 3583579..fbb06c4 100644 --- a/app/scripts/templates/seqpeek/mutations_map_table.hbs +++ b/app/scripts/templates/seqpeek/mutations_map_table.hbs @@ -17,7 +17,7 @@ {{/each}} - ALL + COMBINED {{total.samples}} diff --git a/app/scripts/views/seqpeek/view.js b/app/scripts/views/seqpeek/view.js index c0425b6..2714bc7 100644 --- a/app/scripts/views/seqpeek/view.js +++ b/app/scripts/views/seqpeek/view.js @@ -13,19 +13,21 @@ define([ SampleListOperationsView, MutationsMapTpl, MutationsMapTableTpl, SampleListCaptionTpl - ) { + ) { + var VARIANT_TRACK_MAX_HEIGHT = 150; var TICK_TRACK_HEIGHT = 25; var REGION_TRACK_HEIGHT = 10; var PROTEIN_DOMAIN_TRACK_HEIGHT = 40; var VIEWPORT_WIDTH = 1000; - //var POSITION_FIELD_NAME = "amino_acid_position"; - var POSITION_FIELD_NAME = "chromosome_position"; + var AMINO_ACID_POSITION_FIELD_NAME = "amino_acid_position"; + var COORDINATE_FIELD_NAME = "chromosome_position"; var TYPE_FIELD_NAME = "mutation_type"; var AMINO_ACID_MUTATION_FIELD_NAME = "amino_acid_mutation"; var AMINO_ACID_WILDTYPE_FIELD_NAME = "amino_acid_wildtype"; var DNA_CHANGE_FIELD_NAME = "dna_change"; + var UNIPROT_FIELD_NAME = "uniprot_id"; var GROUP_BY_CATEGORIES = { "Mutation Type": TYPE_FIELD_NAME, @@ -35,6 +37,8 @@ define([ } }; + var VISUALIZATION_MODE = "ALL"; + var MUTATION_TYPE_COLOR_MAP = { Nonsense_Mutation: "red", Silent: "green", @@ -176,7 +180,8 @@ define([ this.genes = this.options["genes"] || []; if (!_.isEmpty(this.genes)) this.selected_gene = _.first(this.genes); - var renderFn = _.after(1 + (2 * this.tumor_types.length), this.__load_protein_domains); + //var renderFn = _.after(1 + (2 * this.tumor_types.length), this.__load_protein_domains); + var renderFn = _.after(1 + (2 * this.tumor_types.length), this.__preprocess_data); this.model["mutsig"].on("load", renderFn, this); @@ -408,7 +413,7 @@ define([ variant_layout: { variant_width: 5.0 }, - variant_data_location_field: POSITION_FIELD_NAME, + variant_data_location_field: COORDINATE_FIELD_NAME, variant_data_type_field: this.selected_group_by, variant_data_source_field: "patient_id", selection_handler: _.bind(this.__seqpeek_selection_handler, this) @@ -438,7 +443,12 @@ define([ guid: track_guid, hovercard_content: { "Protein location": function(d) { - return d["start_aa"] + " - " + d["end_aa"]; + if (d["type"] == "exon") { + return d["start_aa"] + " - " + d["end_aa"]; + } + else { + return d["start"] + " - " + d["end"]; + } }, "Protein length": function () { return protein_data["length"]; @@ -535,7 +545,7 @@ define([ __find_maximum_samples_in_location: function(mutation_data) { var track_maximums = []; _.each(mutation_data, function(track_obj) { - var grouped_data = SeqPeekDataAdapters.group_by_location(track_obj.variants, this.selected_group_by, POSITION_FIELD_NAME); + var grouped_data = SeqPeekDataAdapters.group_by_location(track_obj.variants, this.selected_group_by, COORDINATE_FIELD_NAME); SeqPeekDataAdapters.apply_statistics(grouped_data, function() {return 'all';}); var max_number_of_samples_in_position = d3.max(grouped_data, function(data_by_location) { @@ -553,12 +563,17 @@ define([ __add_data_track: function(track_obj, seqpeek_builder, track_guid, track_target_svg) { var track_type = track_obj.track_type || this.sample_track_type_user_setting; + var variants = track_obj.variants; + variants.sort(function(x, y) { + return (parseInt(x["chromosome_position"]) - parseInt(y["chromosome_position"])); + }); + if (track_type == "sample_plot") { - return seqpeek_builder.addSamplePlotTrackWithArrayData(track_obj.variants, track_target_svg, { + return seqpeek_builder.addSamplePlotTrackWithArrayData(variants, track_target_svg, { guid: track_guid, hovercard_content: { "Location": function (d) { - return d[POSITION_FIELD_NAME]; + return d[COORDINATE_FIELD_NAME]; }, "Amino Acid Mutation": function (d) { return d[AMINO_ACID_MUTATION_FIELD_NAME]; @@ -601,20 +616,25 @@ define([ }, __build_regions: function(data, protein_start, protein_end) { + var itercount = 0; + console.log("__br"); data.sort(function(x, y) { return (parseInt(x["chromosome_position"]) - parseInt(y["chromosome_position"])); }); var split = _.reduce(data, function(memo, data_point, index, input_array) { + itercount += 1; var has_uniprot = _.has(data_point, "uniprot_id"); if (memo.last_has_uniprot === null) { + //console.log(itercount, "-"); memo.current_array.push(data_point); memo.last_has_uniprot = has_uniprot; return memo; } if (has_uniprot != memo.last_has_uniprot) { + //console.log(itercount, "|"); memo.split_array.push({ coding: memo.last_has_uniprot, data: _.clone(memo.current_array) @@ -623,6 +643,7 @@ define([ memo.current_array = [data_point]; } else { + //console.log(itercount, "*"); memo.current_array.push(data_point); } @@ -673,6 +694,20 @@ define([ return region_info.data; }, + __preprocess_data: function() { + _.each(this.model["mutations"]["by_tumor_type"], function(data, tumor_type) { + if (_.isArray(data)) { + _.each(data, function(d) { + d[COORDINATE_FIELD_NAME] = parseInt(d[COORDINATE_FIELD_NAME]); + + if (_.has(d, UNIPROT_FIELD_NAME)) { + d[AMINO_ACID_POSITION_FIELD_NAME] = parseInt(d[AMINO_ACID_POSITION_FIELD_NAME]); + } + }); + } + }, this); + }, + __filter_data: function(data_by_tumor_type) { console.debug("seqpeek/view.__filter_data:" + this.selected_gene); @@ -686,7 +721,7 @@ define([ filtered[tumor_type] = _.filter(data, function(item) { return (_.has(item, "gene") && _.isEqual(item["gene"].toLowerCase(), lowercase_gene) && - _.has(item, POSITION_FIELD_NAME)); + _.has(item, UNIPROT_FIELD_NAME)); }, this); } else { if (_.has(data, "gene") && _.isEqual(data["gene"], lowercase_gene)) { @@ -726,7 +761,7 @@ define([ return { variants: all_variants, - tumor_type: "ALL", + tumor_type: "COMBINED", track_type: "bar_plot" }; }, @@ -784,8 +819,8 @@ define([ var gene_to_uniprot_mapping = _.reduce(items, function(memo, item) { var gene_label = item["gene"]; - if (!_.has(memo, gene_label)) { - memo[gene_label] = item["uniprot_id"]; + if (!_.has(memo, gene_label) && _.has(item, UNIPROT_FIELD_NAME)) { + memo[gene_label] = item[UNIPROT_FIELD_NAME]; } return memo; }, {}); From 8ffd2e316bf0f0b2e4b0a20a8b1ab9516bb9e435 Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Mon, 30 Jun 2014 15:35:32 -0700 Subject: [PATCH 37/62] script to insert copy number gistic all_data_by_genes.txt from Firehose (truncates sample ids) --- db/scripts/copynumbergistic_insert.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 db/scripts/copynumbergistic_insert.py diff --git a/db/scripts/copynumbergistic_insert.py b/db/scripts/copynumbergistic_insert.py new file mode 100644 index 0000000..e69de29 From e89f2c438deeb113b68cc6e9c0085c2c1846e021 Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Mon, 30 Jun 2014 15:38:25 -0700 Subject: [PATCH 38/62] script to insert copy number gistic all_data_by_genes.txt from Firehose (truncates sample ids) --- db/scripts/copynumbergistic_insert.py | 75 +++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/db/scripts/copynumbergistic_insert.py b/db/scripts/copynumbergistic_insert.py index e69de29..f62fbb1 100644 --- a/db/scripts/copynumbergistic_insert.py +++ b/db/scripts/copynumbergistic_insert.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python + +import argparse +import csv +import pymongo +import logging + +from utilities import configure_logging + +def extract_records(file_path): + logging.info("extract_records(%s)" % file_path) + + with open(file_path, "rb") as csvfile: + csvreader = csv.reader(csvfile, delimiter="\t") + ids = csvreader.next()[4:] + + for row in csvreader: + gene_symbol = row[0] + locus_id = row[1] + cytoband = row[2] + values = row[4:] + + if len(values) != len(ids): raise Exception("mismatched values and features: %s (%s/%s)" % (gene_symbol, str(len(values)), str(len(ids)))) + + record = {} + record["gene"] = gene_symbol + record["locus"] = locus_id + record["cytoband"] = cytoband + record["values"] = values_dict(ids, values) + + yield record + + logging.info("extract_records(%s):samples=%s" % (file_path, str(len(ids)))) + +def values_dict(ids, values): + result = {} + for i, v in zip(ids, values): + trunc_i = i + float_v = v + + if len(i) > 16: trunc_i = i[0:15] + if v != "NA": float_v = float(v) + + result[trunc_i] = float_v + return result + +def main(): + parser = argparse.ArgumentParser(description="Utility to import Firehose Copy Number Gistic2 data to MongoDB") + parser.add_argument("--host", required=True, help="MongoDB host name") + parser.add_argument("--port", required=True, type=int, help="MongoDB port") + parser.add_argument("--db", required=True, help="Database name") + parser.add_argument("--f", required=True, help="Path to copy number gistic2 file") + parser.add_argument("--loglevel", default="INFO", help="Logging Level") + args = parser.parse_args() + + configure_logging(args.loglevel.upper()) + + logging.info("import file: %s" % args.f) + logging.info("uploading to %s:%s/%s" % (args.host, args.port, args.db)) + + conn = pymongo.Connection(args.host, args.port) + collection = conn[args.db]["copy_number_gistic"] + + count = 0 + for record in extract_records(args.f): + collection.insert(record) + count += 1 + + logging.info("inserted count=%s" % count) + + conn.close() + +if __name__ == "__main__": + main() + From eceedd3f9cf636eaf3eaacd459fc45c2343ee764 Mon Sep 17 00:00:00 2001 From: kleinone Date: Mon, 30 Jun 2014 16:03:55 -0700 Subject: [PATCH 39/62] Added y-axis scale --- app/scripts/views/seqpeek/view.js | 123 +++++++++++++++++++++++++----- 1 file changed, 105 insertions(+), 18 deletions(-) diff --git a/app/scripts/views/seqpeek/view.js b/app/scripts/views/seqpeek/view.js index fae9915..a2a4788 100644 --- a/app/scripts/views/seqpeek/view.js +++ b/app/scripts/views/seqpeek/view.js @@ -13,12 +13,17 @@ define([ SampleListOperationsView, MutationsMapTpl, MutationsMapTableTpl, SampleListCaptionTpl - ) { + ) { + + var Y_AXIS_SCALE_WIDTH = 50; + var VARIANT_TRACK_MAX_HEIGHT = 150; var TICK_TRACK_HEIGHT = 25; var REGION_TRACK_HEIGHT = 10; var PROTEIN_DOMAIN_TRACK_HEIGHT = 40; var VIEWPORT_WIDTH = 1000; + var SAMPLE_PLOT_TRACK_STEM_HEIGHT = 30; + var TRACK_SVG_WIDTH = VIEWPORT_WIDTH + Y_AXIS_SCALE_WIDTH; var POSITION_FIELD_NAME = "amino_acid_position"; var TYPE_FIELD_NAME = "mutation_type"; @@ -368,7 +373,7 @@ define([ bar_plot_tracks: { bar_width: 5.0, height: VARIANT_TRACK_MAX_HEIGHT, - stem_height: 30, + stem_height: SAMPLE_PLOT_TRACK_STEM_HEIGHT, color_scheme: this.selected_bar_plot_color_by }, sample_plot_tracks: { @@ -408,21 +413,26 @@ define([ var track_guid = "C" + vq.utils.VisUtils.guid(); var track_elements_svg = d3.select(track_obj.target_element) .append("svg") - .attr("width", VIEWPORT_WIDTH) + .attr("width", TRACK_SVG_WIDTH) .attr("height", VARIANT_TRACK_MAX_HEIGHT + REGION_TRACK_HEIGHT) .attr("id", track_guid) .style("pointer-events", "none"); var sample_plot_track_g = track_elements_svg .append("g") - .style("pointer-events", "none"); + .style("pointer-events", "none") + .call(this.__set_track_g_position); var region_track_g = track_elements_svg + .append("g") + .style("pointer-events", "none") + .call(this.__set_track_g_position) .append("g") .style("pointer-events", "none"); track_obj.track_info = this.__add_data_track(track_obj, seqpeek, track_guid, sample_plot_track_g); track_obj.variant_track_svg = track_elements_svg; + track_obj.sample_plot_track_g = sample_plot_track_g; seqpeek.addRegionScaleTrackToElement(region_track_g, { guid: track_guid, @@ -449,23 +459,27 @@ define([ track_obj.region_track_svg = region_track_g; }, this); - var tick_track_svg = d3.select(seqpeek_tick_track_element) + var tick_track_g = d3.select(seqpeek_tick_track_element) .append("svg") - .attr("width", VIEWPORT_WIDTH) - .attr("height", TICK_TRACK_HEIGHT) - .style("pointer-events", "none"); + .attr("width", TRACK_SVG_WIDTH) + .attr("height", TICK_TRACK_HEIGHT) + .style("pointer-events", "none") + .append("svg:g") + .call(this.__set_track_g_position); - seqpeek.addTickTrackToElement(tick_track_svg); + seqpeek.addTickTrackToElement(tick_track_g); var protein_domain_track_guid = "C" + vq.utils.VisUtils.guid(); - var protein_domain_track_svg = d3.select(seqpeek_domain_track_element) + var protein_domain_track_g = d3.select(seqpeek_domain_track_element) .append("svg") - .attr("width", VIEWPORT_WIDTH) - .attr("height", PROTEIN_DOMAIN_TRACK_HEIGHT) - .attr("id", protein_domain_track_guid) - .style("pointer-events", "none"); - - seqpeek.addProteinDomainTrackToElement(protein_data["matches"], protein_domain_track_svg, { + .attr("width", TRACK_SVG_WIDTH) + .attr("height", PROTEIN_DOMAIN_TRACK_HEIGHT) + .attr("id", protein_domain_track_guid) + .style("pointer-events", "none") + .append("svg:g") + .call(this.__set_track_g_position); + + seqpeek.addProteinDomainTrackToElement(protein_data["matches"], protein_domain_track_g, { guid: protein_domain_track_guid, hovercard_content: { "DB": function(d) { @@ -511,14 +525,87 @@ define([ track_obj.variant_track_svg.attr("height", total_track_height); track_obj.region_track_svg - .attr("transform", "translate(0," + (variant_track_height) + ")") - }); + .attr("transform", "translate(0," + (variant_track_height) + ")"); + + this.__render_scales(track_obj.variant_track_svg, total_track_height, track_instance.statistics); + }, this); seqpeek.render(); this.seqpeek = seqpeek; }, + __set_track_g_position: function(track_selector) { + track_selector + .attr("transform", "translate(" + Y_AXIS_SCALE_WIDTH + ",0)"); + }, + + __render_scales: function(track_selector, total_track_height, track_statistics) { + var right = Y_AXIS_SCALE_WIDTH - 10; + + var axis = track_selector + .append("svg:g") + .attr("class", "y-axis") + .attr("transform", "translate(0," + total_track_height + ")"); + + axis + .append("svg:line") + .attr("y1", 0) + .attr("x1", right) + .attr("y2", -total_track_height) + .attr("x2", right) + .style("stroke", "black"); + + var domain = [ + track_statistics.min_samples_in_location, + track_statistics.max_samples_in_location + ]; + + var scale_start = -(REGION_TRACK_HEIGHT + SAMPLE_PLOT_TRACK_STEM_HEIGHT); + + var scale = d3.scale.linear().domain(domain).range([scale_start, -total_track_height]); + var ticks = [ + { + text: domain[0], + y: scale(domain[0]), + text_y: -5 + }, + { + text: domain[1], + y: scale(domain[1]), + text_y: +13 + }, + ]; + + var tick_g = axis + .selectAll(".tick") + .data(ticks) + .enter() + .append("svg:g") + .attr("class", "y-axis-tick") + .attr("transform", function(d) { + return "translate(0," + d.y + ")"; + }); + + tick_g + .append("svg:line") + .attr("y1", 0.0) + .attr("y2", 0.0) + .attr("x1", right - 10) + .attr("x2", right) + .style("stroke", "black"); + tick_g + .append("svg:text") + .attr("x", right - 15) + .attr("y", function(d) { + return d.text_y; + }) + .text(function(d) { + return d.text; + }) + .style("text-anchor", "end"); + }, + __find_maximum_samples_in_location: function(mutation_data) { var track_maximums = []; _.each(mutation_data, function(track_obj) { From baddf92db3bf3a41402a002bea87084b41cc9ec1 Mon Sep 17 00:00:00 2001 From: kleinone Date: Mon, 30 Jun 2014 16:12:45 -0700 Subject: [PATCH 40/62] Adjusted top tick --- app/scripts/views/seqpeek/view.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/scripts/views/seqpeek/view.js b/app/scripts/views/seqpeek/view.js index a2a4788..e20ed4b 100644 --- a/app/scripts/views/seqpeek/view.js +++ b/app/scripts/views/seqpeek/view.js @@ -572,7 +572,7 @@ define([ }, { text: domain[1], - y: scale(domain[1]), + y: scale(domain[1]) + 1, text_y: +13 }, ]; From 095c1fb3805d16f443662155c0090f0ad0183868 Mon Sep 17 00:00:00 2001 From: kleinone Date: Mon, 30 Jun 2014 18:28:03 -0700 Subject: [PATCH 41/62] Added SeqPeek mini locator to mutations map. Updated SeqPeek dependency in bower.json. --- .../templates/seqpeek/mutations_map.hbs | 3 +++ app/scripts/views/seqpeek/view.js | 25 ++++++++++++++++--- bower.json | 2 +- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/app/scripts/templates/seqpeek/mutations_map.hbs b/app/scripts/templates/seqpeek/mutations_map.hbs index 62f9c6f..5a8599f 100644 --- a/app/scripts/templates/seqpeek/mutations_map.hbs +++ b/app/scripts/templates/seqpeek/mutations_map.hbs @@ -51,5 +51,8 @@
      • +
    • + +
    diff --git a/app/scripts/views/seqpeek/view.js b/app/scripts/views/seqpeek/view.js index e20ed4b..d809485 100644 --- a/app/scripts/views/seqpeek/view.js +++ b/app/scripts/views/seqpeek/view.js @@ -3,13 +3,14 @@ define([ "models/gs/protein_domain_model", "seqpeek/util/data_adapters", "seqpeek/builders/builder_for_existing_elements", + "seqpeek/util/mini_locator", "views/seqpeek/sample_list_operations_view", "hbs!templates/seqpeek/mutations_map", "hbs!templates/seqpeek/mutations_map_table", "hbs!templates/seqpeek/sample_list_dropdown_caption" ], function ($, _, Backbone, d3, vq, - ProteinDomainModel, SeqPeekDataAdapters, SeqPeekBuilder, + ProteinDomainModel, SeqPeekDataAdapters, SeqPeekBuilder, SeqPeekMiniLocatorFactory, SampleListOperationsView, MutationsMapTpl, MutationsMapTableTpl, SampleListCaptionTpl @@ -530,11 +531,28 @@ define([ this.__render_scales(track_obj.variant_track_svg, total_track_height, track_instance.statistics); }, this); + + var mini_locator_scale = 200 / seqpeek.getRegionMetadata().total_width; + this.__create_mini_locator(seqpeek.getProcessedRegionData(), mini_locator_scale); + + seqpeek.scrollEventCallback(_.bind(function(d) { + this.mini_locator.render(d.visible_min_x, d.visible_max_x) + }, this)); seqpeek.render(); this.seqpeek = seqpeek; }, + __create_mini_locator: function(region_data, scale) { + var mini_locator_canvas = this.$el.find(".seqpeek-mini-locator")[0]; + + this.mini_locator = SeqPeekMiniLocatorFactory.create(mini_locator_canvas) + .data(region_data) + .scale(scale); + + this.mini_locator.render(0, 1000); + }, + __set_track_g_position: function(track_selector) { track_selector .attr("transform", "translate(" + Y_AXIS_SCALE_WIDTH + ",0)"); @@ -542,6 +560,7 @@ define([ __render_scales: function(track_selector, total_track_height, track_statistics) { var right = Y_AXIS_SCALE_WIDTH - 10; + var scale_start = -(REGION_TRACK_HEIGHT + SAMPLE_PLOT_TRACK_STEM_HEIGHT); var axis = track_selector .append("svg:g") @@ -550,7 +569,7 @@ define([ axis .append("svg:line") - .attr("y1", 0) + .attr("y1", scale_start) .attr("x1", right) .attr("y2", -total_track_height) .attr("x2", right) @@ -561,8 +580,6 @@ define([ track_statistics.max_samples_in_location ]; - var scale_start = -(REGION_TRACK_HEIGHT + SAMPLE_PLOT_TRACK_STEM_HEIGHT); - var scale = d3.scale.linear().domain(domain).range([scale_start, -total_track_height]); var ticks = [ { diff --git a/bower.json b/bower.json index e7fff5e..b7507d8 100644 --- a/bower.json +++ b/bower.json @@ -22,7 +22,7 @@ "carve": "~0.0.7", "x2js": "*", "cytoscape": "~2.2.1", - "seqpeek": "git://github.com/IlyaLab/SeqPeek.git#b4d49cd120ede4aaf8fac796ed757e1aeaec1da0" + "seqpeek": "git://github.com/IlyaLab/SeqPeek.git#e45d5d849328a492b3efd255594a1c1aa379d268" }, "resolutions": { "underscore": "~1.5.2", From 0490849fa7aa958b9294261766961b38e9d8f496 Mon Sep 17 00:00:00 2001 From: kleinone Date: Tue, 1 Jul 2014 13:26:44 -0700 Subject: [PATCH 42/62] Moved mini locator to header row of table. Resized mini locator. --- app/scripts/templates/seqpeek/mutations_map.hbs | 3 --- app/scripts/templates/seqpeek/mutations_map_table.hbs | 2 +- app/scripts/views/seqpeek/view.js | 10 +++++++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/app/scripts/templates/seqpeek/mutations_map.hbs b/app/scripts/templates/seqpeek/mutations_map.hbs index 5a8599f..62f9c6f 100644 --- a/app/scripts/templates/seqpeek/mutations_map.hbs +++ b/app/scripts/templates/seqpeek/mutations_map.hbs @@ -51,8 +51,5 @@
    • -
  • - -
    • diff --git a/app/scripts/templates/seqpeek/mutations_map_table.hbs b/app/scripts/templates/seqpeek/mutations_map_table.hbs index fbb06c4..dfeb872 100644 --- a/app/scripts/templates/seqpeek/mutations_map_table.hbs +++ b/app/scripts/templates/seqpeek/mutations_map_table.hbs @@ -4,7 +4,7 @@ Tumor Type MutSig Rank Samples (#) - + diff --git a/app/scripts/views/seqpeek/view.js b/app/scripts/views/seqpeek/view.js index d809485..6b640b0 100644 --- a/app/scripts/views/seqpeek/view.js +++ b/app/scripts/views/seqpeek/view.js @@ -15,6 +15,8 @@ define([ MutationsMapTpl, MutationsMapTableTpl, SampleListCaptionTpl ) { + var MINI_LOCATOR_WIDTH = 400; + var MINI_LOCATOR_HEIGHT = 24; var Y_AXIS_SCALE_WIDTH = 50; @@ -532,7 +534,7 @@ define([ }, this); - var mini_locator_scale = 200 / seqpeek.getRegionMetadata().total_width; + var mini_locator_scale = MINI_LOCATOR_WIDTH / seqpeek.getRegionMetadata().total_width; this.__create_mini_locator(seqpeek.getProcessedRegionData(), mini_locator_scale); seqpeek.scrollEventCallback(_.bind(function(d) { @@ -544,9 +546,11 @@ define([ }, __create_mini_locator: function(region_data, scale) { - var mini_locator_canvas = this.$el.find(".seqpeek-mini-locator")[0]; + var $mini_locator = this.$el.find(".seqpeek-mini-locator") + .attr("width", MINI_LOCATOR_WIDTH) + .attr("height", MINI_LOCATOR_HEIGHT); - this.mini_locator = SeqPeekMiniLocatorFactory.create(mini_locator_canvas) + this.mini_locator = SeqPeekMiniLocatorFactory.create($mini_locator[0]) .data(region_data) .scale(scale); From fc95bfe595b66109a6b34cab70353ca0e5bcce04 Mon Sep 17 00:00:00 2001 From: kleinone Date: Tue, 1 Jul 2014 13:28:42 -0700 Subject: [PATCH 43/62] Updated SeqPeek dependency --- bower.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bower.json b/bower.json index b7507d8..835b7ae 100644 --- a/bower.json +++ b/bower.json @@ -22,7 +22,7 @@ "carve": "~0.0.7", "x2js": "*", "cytoscape": "~2.2.1", - "seqpeek": "git://github.com/IlyaLab/SeqPeek.git#e45d5d849328a492b3efd255594a1c1aa379d268" + "seqpeek": "git://github.com/IlyaLab/SeqPeek.git#2e82f276f03d9e37dcafcefe32ef7c78ed724294" }, "resolutions": { "underscore": "~1.5.2", From 580ff96461a950bb27527b2a6f9f368290c0b19e Mon Sep 17 00:00:00 2001 From: kleinone Date: Tue, 1 Jul 2014 17:50:19 -0700 Subject: [PATCH 44/62] Fixed calls to SeqPeek mini locator API. Send in coordinates to mini locator on scroll events. Updated SeqPeek dependency in bower.json. --- app/scripts/views/seqpeek/view.js | 14 +++++++++----- bower.json | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/app/scripts/views/seqpeek/view.js b/app/scripts/views/seqpeek/view.js index 6b640b0..50baac5 100644 --- a/app/scripts/views/seqpeek/view.js +++ b/app/scripts/views/seqpeek/view.js @@ -533,28 +533,32 @@ define([ this.__render_scales(track_obj.variant_track_svg, total_track_height, track_instance.statistics); }, this); + var regions_start_coordinate = seqpeek.getRegionMetadata().start_coordinate; + var regions_end_coordinate = seqpeek.getRegionMetadata().end_coordinate; var mini_locator_scale = MINI_LOCATOR_WIDTH / seqpeek.getRegionMetadata().total_width; - this.__create_mini_locator(seqpeek.getProcessedRegionData(), mini_locator_scale); + this.__create_mini_locator(seqpeek.getProcessedRegionData(), seqpeek.region_layout, mini_locator_scale, regions_start_coordinate, regions_end_coordinate); seqpeek.scrollEventCallback(_.bind(function(d) { - this.mini_locator.render(d.visible_min_x, d.visible_max_x) + var visible_coordinates = d.visible_coordinates; + this.mini_locator.render(visible_coordinates[0], visible_coordinates[1]); }, this)); seqpeek.render(); this.seqpeek = seqpeek; }, - __create_mini_locator: function(region_data, scale) { + __create_mini_locator: function(region_data, region_layout, scale, start_coordinate, end_coordinate) { var $mini_locator = this.$el.find(".seqpeek-mini-locator") .attr("width", MINI_LOCATOR_WIDTH) .attr("height", MINI_LOCATOR_HEIGHT); this.mini_locator = SeqPeekMiniLocatorFactory.create($mini_locator[0]) .data(region_data) + .region_layout(region_layout) .scale(scale); - this.mini_locator.render(0, 1000); + this.mini_locator.render(start_coordinate, end_coordinate); }, __set_track_g_position: function(track_selector) { @@ -595,7 +599,7 @@ define([ text: domain[1], y: scale(domain[1]) + 1, text_y: +13 - }, + } ]; var tick_g = axis diff --git a/bower.json b/bower.json index 835b7ae..189b341 100644 --- a/bower.json +++ b/bower.json @@ -22,7 +22,7 @@ "carve": "~0.0.7", "x2js": "*", "cytoscape": "~2.2.1", - "seqpeek": "git://github.com/IlyaLab/SeqPeek.git#2e82f276f03d9e37dcafcefe32ef7c78ed724294" + "seqpeek": "git://github.com/IlyaLab/SeqPeek.git#ced34636358c9c88e481e03439db756d32ac6669" }, "resolutions": { "underscore": "~1.5.2", From 14fa07c3160075f1bd580fcb7454f9f80f0bb1eb Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Thu, 3 Jul 2014 18:41:24 -0700 Subject: [PATCH 45/62] fixing how unid is used for feature selection null checks --- app/scripts/views/fmx_distributions/view.js | 29 ++++++++++++--------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/app/scripts/views/fmx_distributions/view.js b/app/scripts/views/fmx_distributions/view.js index 167671d..a69dfbb 100644 --- a/app/scripts/views/fmx_distributions/view.js +++ b/app/scripts/views/fmx_distributions/view.js @@ -216,17 +216,15 @@ define(["jquery", "underscore", "backbone", __load_fdefs_clinvars: function (tumor_type) { console.debug("fmx-dist.__load_fdefs_clinvars(" + tumor_type + ")"); - _.each(this.options["clinical_variables"], function(item) { - this.feature_definitions_by_id[item["unid"] || item["id"]] = _.extend({}, item); - }, this); this.__aggregate(tumor_type, this.model["clinical_features"]["by_tumor_type"][tumor_type]); }, __aggregate: function(tumor_type, model) { console.debug("fmx-dist.__aggregate(" + tumor_type + ")"); _.each(model.get("items"), function(item) { - var a_f_by_id = this.aggregate_features_by_id[item["unid"] || item["id"]]; - if (!a_f_by_id) a_f_by_id = this.aggregate_features_by_id[item["unid"] || item["id"]] = {}; + var unid_or_id = item["unid"] || item["id"]; + var a_f_by_id = this.aggregate_features_by_id[unid_or_id]; + if (!a_f_by_id) a_f_by_id = this.aggregate_features_by_id[unid_or_id] = {}; if (_.has(a_f_by_id, tumor_type)) { var existing = a_f_by_id[tumor_type]; @@ -241,7 +239,10 @@ define(["jquery", "underscore", "backbone", } else { a_f_by_id[tumor_type] = item; } - this.feature_definitions_by_id[item["unid"] || item["id"]] = _.omit(item, "values"); + + var omit_values = _.omit(_.extend({}, item), "values"); + this.feature_definitions_by_id[item["unid"]] = omit_values; + this.feature_definitions_by_id[item["id"]] = omit_values; }, this); }, @@ -378,10 +379,12 @@ define(["jquery", "underscore", "backbone", var Y_feature = this.feature_definitions_by_id[this.selected_features["y"]]; var data = null; - if (this.selected_tumor_type) { - data = this.__visdata([this.selected_tumor_type], X_feature, Y_feature); - } else { - data = this.__visdata(this.options["tumor_types"], X_feature, Y_feature); + if (X_feature && Y_feature) { + if (this.selected_tumor_type) { + data = this.__visdata([this.selected_tumor_type], X_feature, Y_feature); + } else { + data = this.__visdata(this.options["tumor_types"], X_feature, Y_feature); + } } if (_.isEmpty(data)) { this.latest_data = []; @@ -463,10 +466,12 @@ define(["jquery", "underscore", "backbone", var stl = this.sample_types_lookup[tumor_type] || {}; var X_feature_by_tumor_type = this.aggregate_features_by_id[X_feature_id] || {}; - var X_feature = X_feature_by_tumor_type[tumor_type] || {}; + var X_feature = X_feature_by_tumor_type[tumor_type]; + if (!X_feature) return null; var Y_feature_by_tumor_type = this.aggregate_features_by_id[Y_feature_id] || {}; - var Y_feature = Y_feature_by_tumor_type[tumor_type] || {}; + var Y_feature = Y_feature_by_tumor_type[tumor_type]; + if (!Y_feature) return null; var Cby_feature_by_tumor_type = this.aggregate_features_by_id[this.selected_color_by] || {}; var Cby_feature = Cby_feature_by_tumor_type[tumor_type] || {}; From 5180e25d2ed19c67e0b4bd303b71f085ae102506 Mon Sep 17 00:00:00 2001 From: Michael Miller Date: Mon, 7 Jul 2014 09:13:10 -0700 Subject: [PATCH 46/62] updates to complete the FFN/FFV and statistics features --- db/scripts/add_fm_custom_labels.py | 160 ++++++++++++++++++++---- db/scripts/generate_stats_var_lookup.js | 38 +++--- db/scripts/update_fm_with_ffn_lookup.js | 137 +++++++------------- 3 files changed, 195 insertions(+), 140 deletions(-) diff --git a/db/scripts/add_fm_custom_labels.py b/db/scripts/add_fm_custom_labels.py index 3b4cd7d..612e241 100644 --- a/db/scripts/add_fm_custom_labels.py +++ b/db/scripts/add_fm_custom_labels.py @@ -1,46 +1,147 @@ ''' Created on Jun 4, 2014 +usage: add_fm_custom_labels.py --host --port --tumor --root +NOTE: --db will generally be equal to --tumor +NOTE: --root must not end with a '/' + @author: michael ''' import argparse import logging -from os import path +import os import pymongo import re +import traceback from utilities import configure_logging ffnPattern = re.compile('FFN.*\.tsv$') +clusterGroupPattern = re.compile('^[CG]([0-9]+)$') + +def removeClusterOrGroup(name): + match = clusterGroupPattern.match(name) + if match: + return match.group(1) + else: + return name + +def display(value): + value = value.replace('_', ' ') + return value[0].upper() + value[1:] -def addLabels(collection, path): - logging.info(collection) +def makeLabel(label, fv, typeformat): + if not typeformat or typeformat == 'Default': + return label + ': ' + fv + elif typeformat == 'Append': + return fv + ' ' + label + elif typeformat == 'AppendWithParentheses': + return fv + ' (' + label + ')' + elif typeformat == 'DoNotDisplay': + return fv + elif typeformat == 'Prepend': + return label + ' ' + fv + print '''Didn't fall into any case!!!''' + +def updateCategoryFeatures(collection, ffv_infos): + catOnePattern = r'I\(([^,]+)\|%s\)' + catTwoPattern = r'I\((.+),(.+)\|%s\)' + for name, ffv_info in ffv_infos.iteritems(): + regex = re.compile(catOnePattern % (name)) + docsOne = collection.find({"id": regex}) + print 'for name %s found %d docs' % (name, docsOne.count()) + for doc in docsOne: + try: + nlabel = ffv_info[0] + match = regex.match(doc['id'].split(':')[2]) + fv = ffv_info[1].get(match.group(1), removeClusterOrGroup(match.group(1))) + collection.update({'id': doc['id']}, {"$set": {"label": makeLabel(display(nlabel), display(fv), ffv_info[2])}}) + except Exception as e: + traceback.print_exc() + print 'problem with updateCategoryFeatures(%s-%s): %s: %s' % (doc['id'], fv, name, ffv_info) + raise e + docsOne.close() + + regex = re.compile(catTwoPattern % (name)) + docsTwo = collection.find({"id": regex}) + print 'for name %s found %d docs' % (name, docsTwo.count()) + for doc in docsTwo: + try: + nlabel = ffv_info[0] + match = regex.match(doc['id'].split(':')[2]) + fv1 = ffv_info[1].get(match.group(1), removeClusterOrGroup(match.group(1))) + fv2 = ffv_info[1].get(match.group(2), removeClusterOrGroup(match.group(2))) + collection.update({'id': doc['id']}, {"$set": {"label": makeLabel(display(nlabel), display(fv1) + ' vs ' + display(fv2), ffv_info[2])}}) + except Exception as e: + traceback.print_exc() + print 'problem with updateCategoryFeatures(%s-%s vs %s): %s: %s' % (doc['id'], fv1, fv2, name, ffv_info) + raise e + docsTwo.close() + + +def updateLabels(collection, path, ffv_infos): + logging.info('\ncollection: %s file: %s' % (collection, path)) + count = 0 with open(path, 'r') as labels: + labels.readline() for line in labels: - if '#' == line[0]: - continue - fields = line.strip().split('\t') - if 2 > len(fields): - raise ValueError('did not find two fields!!!') - if collection.find_one({'id': fields[0]}): - logging.info('found %s, updating label to %s', fields[0], fields[1]) - collection.update({'id': fields[0]}, {"$set": {"label": fields[1]}}) - else: - logging.info("didn't find %s", (fields[0])) + try: + count += 1 + if '#' == line[0]: + continue + fields = line.strip().split('\t') + if 2 > len(fields): + raise ValueError('did not find two fields!!!') + + fields += [None, None] + doc = collection.find_one({'id': fields[0]}) + ffv_info = [None, {}, None] + if doc: + if 'Default' == fields[1]: + fields[1] = '' + if fields[1]: + logging.info('found %s, updating label to %s', fields[0], fields[1]) + collection.update({'id': fields[0]}, {"$set": {"label": display(fields[1])}}) + ffv_info[0] = fields[1] + else: + name = doc['id'].split(':')[2] + logging.info('found %s w/o custom label, updating to default %s', fields[0], display(name)) + collection.update({'id': fields[0]}, {"$set": {"label": display(name)}}) + ffv_info[0] = display(name) + else: + logging.info("didn't find %s", (fields[0])) + continue + + if not ((fields[2] and fields[2] != 'Default') or fields[3]): + continue + + print 'fields: %s' % (fields) + if 'Default' == fields[2]: + fields[2] = '' + order = 0 + if fields[2]: + ffv_precedence = {} + for field in fields[2].split(','): + ffv_precedence[field.split(':')[0]] = {'ffv': field.split(':')[1], 'ordinal': order} + order += 1 + collection.update({'id': fields[0]}, {"$set": {"ffv_precedence": ffv_precedence}}) + print '\tprecedence: %s' % (ffv_precedence) + ffv_map = dict([(field.split(':')[0], field.split(':')[1]) for field in fields[2].split(',')]) + ffv_info[1] = ffv_map + if fields[3]: + ffv_info[2] = fields[3] + + ffv_infos[doc['id'].split(':')[2]] = ffv_info + except Exception as e: + traceback.print_exc() + print 'problem with parsing line %d: %s--%s' % (count, line, e) + raise e def checkForFFNFile(files, dirname, names): for name in names: if ffnPattern.search(name): files += [dirname + '/' + name]; -def findFNFfiles(args, dirname, names): - dirs, topname = path.split(dirname); - _, parent = path.split(dirs); - if dirs == args.root and topname == args.dir: - checkForFFNFile(args.topfiles, dirname, names) - if args.tumor.lower() == parent.lower() and topname.startswith(args.dir): - checkForFFNFile(args.tumorfiles, dirname, names) - def main(): parser = argparse.ArgumentParser(description="Utility to add custom labels to TCGA feature matrix in MongoDB") parser.add_argument("--host", required=True, help="MongoDB host name") @@ -57,26 +158,33 @@ def main(): args.topfiles = []; args.tumorfiles = [] - path.walk - path.walk(args.root, findFNFfiles, args) + for root, _, files in os.walk(args.root, followlinks = True): + basedir = os.path.dirname(root) + namedir = os.path.basename(root) + if basedir == args.root and namedir == args.dir: + checkForFFNFile(args.topfiles, root, files) + if args.tumor.lower() == os.path.basename(basedir).lower() and namedir == args.dir: + checkForFFNFile(args.tumorfiles, root, files) + logging.info('%s %s', args.topfiles, args.tumorfiles) conn = pymongo.Connection(args.host, args.port) collection = conn[args.db]["feature_matrix"] + ffv_infos = {} if 0 == len(args.topfiles): raise ValueError('did not find a general custom file') else: for topfile in args.topfiles: - addLabels(collection, topfile); + updateLabels(collection, topfile, ffv_infos); if 0 == len(args.tumorfiles): logging.info('did not find a tumor type custom file') else: for tumorfile in args.tumorfiles: - addLabels(collection, tumorfile); + updateLabels(collection, tumorfile, ffv_infos); + updateCategoryFeatures(collection, ffv_infos) conn.close() logging.info('finished add custom labels to feature matrix') if __name__ == '__main__': main() - \ No newline at end of file diff --git a/db/scripts/generate_stats_var_lookup.js b/db/scripts/generate_stats_var_lookup.js index b81bbf1..b1791ff 100644 --- a/db/scripts/generate_stats_var_lookup.js +++ b/db/scripts/generate_stats_var_lookup.js @@ -1,21 +1,6 @@ /* usage: mongo :/ generate_stats_var_lookup.js */ -function getCounts(values) { - var retVal = {}; - retVal["total"] = 0; - retVal["valid"] = 0; - - for (var key in values) { - retVal["total"] += 1; - if (values[key] != 'NA') { - retVal["valid"]++; - } - } - - return retVal; -} - function median(values) { values.sort(function(a, b){return a-b}); var half = Math.floor(values.length/2); @@ -45,13 +30,13 @@ function getStats(type, values) { retVal["counts"]["total"]++; if (values[key] != 'NA') { retVal["counts"]["valid"]++; - } - if (values[key] in retVal["categories"]) { - retVal["categories"][values[key]]++; - } else { - count++; - retVal["categories"][values[key]] = 1; + if (values[key] in retVal["categories"]) { + retVal["categories"][values[key]]++; + } else { + count++; + retVal["categories"][values[key]] = 1; + } } if ("N" == type && "NA" != values[key]) { @@ -102,6 +87,17 @@ db.feature_matrix.find().forEach( print("processing record " + count + " " + new Date()); } + if (doc["raw_values"]) { + var stats = getStats(doc["type"], doc["raw_values"]); + db.feature_matrix.update({"id":doc["id"]}, {$set: {"raw_statistics.counts": stats["counts"]}}); + if (stats["categories"]) { + db.feature_matrix.update({"id":doc["id"]}, {$set: {"raw_statistics.categories":stats["categories"]}}) + } + if (stats["numeric"]) { + db.feature_matrix.update({"id":doc["id"]}, {$set: {"raw_statistics.numeric":stats["numeric"]}}) + } + } + var stats = getStats(doc["type"], doc["values"]); db.feature_matrix.update({"id":doc["id"]}, {$set: {"statistics.counts": stats["counts"]}}); if (stats["categories"]) { diff --git a/db/scripts/update_fm_with_ffn_lookup.js b/db/scripts/update_fm_with_ffn_lookup.js index e7a2ede..9d98074 100644 --- a/db/scripts/update_fm_with_ffn_lookup.js +++ b/db/scripts/update_fm_with_ffn_lookup.js @@ -91,116 +91,68 @@ function getLength(start, end) { return Math.round(length) + units[units.length - 1]; } -function getFMTitle(index, fields, suffix) { - var fm_title = fields[0]; - for (var i = 1; i < index; i++) { - fm_title += ' ' + fields[i]; - } - return fm_title + ': ' + suffix; -} - -function getGisticArmString(fields) { - return getFMTitle(fields.indexOf('GisticArm'), fields, 'Gistic Arm'); -} - -function getGisticString(fields) { - var gindex = fields.indexOf('Gistic'); - return getFMTitle(gindex, fields, 'Gistic') + ' ' + fields[gindex + 1] + ' ' + fields[fields.length - 1]; -} - -var debugCNVRCounts = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; -var noloc_checks = [ +var debugCNVRCounts = [0, 0, 0, 0, 0, 0]; +// Gistic and GisticArm features // N:CNVR:Xq:chrX:60600000:155270560::SKCM-All_Lymph_Node_GisticArm_d -// ⇒ Xq (SKCM-All Lymph Node: Gistic Arm) +// ⇒ Xq (Gistic Arm) // N:CNVR:Xq28:chrX:150021680:150280252::SKCM-All_Regional_Metastases_Gistic_ROI_r_amp -// ⇒ Xq28 (SKCM-All_Regional_Metastases: Gistic ROI amp) - /^x([pq]([0-9]{1,2})?)$/i, -// N:CNVR:10p:chr10:0:40200000::GBM-TP_GisticArm_d -// ⇒ 10p (GBM-TP: Gistic Arm) - /^[0-9]{1,2}[pq]$/ -]; -var loc_checks = [ +// ⇒ Xq28 Amplification (Gistic, continuous) +// N:CNVR:Xq28:chrX:150021680:150280252::SKCM-All_Regional_Metastases_Gistic_ROI_d_del +// ⇒ Xq28 Deletion (Gistic, discrete) + +// Resegmented features // N:CNVR:Xq22.2:chrX:60600000:155270560 // ⇒ chrX:60,600,000-155,270,560 (95MB, Xq22.2) - /^x([pq])/i, // N:CNVR:8q24:chr8:138862000:140900999:: // ⇒ chr8:138,862,000-140,900,999 (2MB, 8q24) - /^[0-9]{1,2}[pq]/, // N:CNVR:SRGAP2:chr1:158887000:158888999:: // ⇒ chr1:158,887,000-158,888,999 (2kb, SRGAP2) - /^[A-Z]/ -]; var getCNVRLabel = function(doc) { - for (var i = 0; i < noloc_checks.length; i++) { - if (noloc_checks[i].test(doc["locus"])) { - if (doc["code"]) { - var fields = doc["code"].split('_'); - if (-1 < doc["code"].indexOf('GisticArm')) { - debugCNVRCounts[3 * i]++; - return replaceUnderscore(doc["locus"] + ' (' + getGisticArmString(fields) + ')'); - } else { - debugCNVRCounts[3 * i + 1]++; - return replaceUnderscore(doc["locus"] + ' (' + getGisticString(fields) + ')'); - } - } else { - debugCNVRCounts[3 * i + 2]++; - var loc = getLocation(doc) - length = getLength(doc["start"], doc["end"]); - return replaceUnderscore(loc + ' (' + length + ', ' + doc["locus"] + ')'); - } - } - } - - for (var i = 0; i < loc_checks.length; i++) { - if (loc_checks[i].test(doc["locus"])) { - var loc = getLocation(doc) - if (doc["code"]) { - var fields = doc["code"].split('_'); - if (-1 < doc["code"].indexOf('GisticArm')) { - debugCNVRCounts[3 * (i + 2)]++; - return replaceUnderscore(loc + ' (' + getGisticArmString(fields) + ', ' + doc["locus"] + ')'); - } else { - debugCNVRCounts[3 * (i + 2) + 1]++; - return replaceUnderscore(loc + ' (' + getGisticString(fields) + ', ' + doc["locus"] + ')'); - } - } else { - debugCNVRCounts[3 * (i + 2) + 2]++; - var length = getLength(doc["start"], doc["end"]); - return replaceUnderscore(loc + '(' + length + ', ' + doc["locus"] + ')'); - } + if (doc["code"]) { + if (-1 < doc["code"].indexOf('GisticArm')) { + debugCNVRCounts[0]++; + return replaceUnderscore(doc["locus"] + ' (Gistic Arm)'); + } else if (-1 < doc["code"].indexOf('Gistic')) { + var fields = doc["code"].split('_'); + var gType = fields[fields.length - 1] == 'del'? 'Deletion' : 'Amplification'; + var index = fields[fields.length - 1] == 'del'? 0 : 1; + var dType = fields[fields.length - 2] == 'd'? 'discrete' : 'continuous'; + index += fields[fields.length - 2] == 'd'? 3 : 5; + debugCNVRCounts[index - 2]++; + return replaceUnderscore(doc["locus"] + ' ' + gType + ' (Gistic, ' + dType + ')'); + } else if (-1 < doc["code"].indexOf('LOH')) { + return replaceUnderscore(doc["locus"] + ' (LOH)'); + } else { + return replaceUnderscore(doc["locus"] + ' (' + doc["code"] + ')'); } + } else { + debugCNVRCounts[5]++; + var loc = getLocation(doc) + length = getLength(doc["start"], doc["end"]); + return replaceUnderscore(loc + ' (' + length + ', ' + doc["locus"] + ')'); } - print("\n!!!!!" + doc["id"] + " didn't match any expression. using default!!!!!\n"); - var loc = getLocation(doc); - var length = getLength(doc["start"], doc["end"]); - return replaceUnderscore(loc + '(' + length + ', ' + doc["locus"] + ')'); } var of = /([0-9]+)of([0-9]+)/; var getGEXPLabel = function(doc) { var retVal = doc["gene"]; - if (doc["platform"] != "micro-array") { - var tag = doc["id"].split(':')[7]; - if (tag) { - if (of.test(tag)) { - tag = tag.replace(of, "$1 of $2"); - } - retVal += ' (' + tag + ')'; - } + var code = doc["id"].split(':')[7] + if (doc["platform"] == "mRNAseq" && of.test(code)) { + retVal += ' (' + code.replace(of, "$1 of $2") + ')'; } return replaceUnderscore(retVal); } var getMIRNLabel = function(doc) { - return replaceUnderscore(doc["microRNA"] + ' (' + doc["accession_number"] + ')'); + return replaceUnderscore(doc["microRNA"]); } var mappings = new Object(); mappings['nonsilent_somatic'] = "Excluding Silent Mutations"; mappings['code_potential_somatic'] = "Protein Coding"; mappings['missense_somatic'] = "Missense"; -mappings['mnf_somatic'] = "MNF"; -mappings['mni_somatic'] = "MNI"; +mappings['mnf_somatic'] = "Missense-Nonsense-Frameshift"; +mappings['mni_somatic'] = "Missense-Nonsense-Inframe-Frameshift"; mappings['y_n_somatic'] = "All Mutations"; var getGNABLabel = function(doc) { var name = doc["gene"]; @@ -212,22 +164,23 @@ var getGNABLabel = function(doc) { } var getMETHLabel = function(doc) { - // TODO: get the list of genes (w/ corresponding distance from TSS?) + var gene = doc["id"].split(':')[2] var fields = doc["code"].split('_'); + var probe = fields[0]; var tf_loc = ''; if (1 < fields.length) { tf_loc += fields[1]; for (var i = 2; i < fields.length; i++) { - tf_loc += ' '; + tf_loc += ', '; tf_loc += fields[i]; } } if (tf_loc) { - return replaceUnderscore(doc["probe"] + ' (' + doc['id'].split(':')[2] + ', ' + tf_loc + ')'); + return replaceUnderscore(gene + ' (' + tf_loc + ') ' + probe); } else { - return replaceUnderscore(doc["code"] + ' (' + doc['id'].split(':')[2] + ')'); + return replaceUnderscore(gene + ' ' + probe); } } @@ -237,8 +190,6 @@ var endInfo2 = /^(.+)-[RMVCEG]$/ var noInfo = /(.+)\-NA/ var phosphor = /^(.+)[_\-]p([STY][0-9]+.*)$/ var getRPPALabel = function(doc) { - // TODO: get the list of genes - // remove any 'uninteresting' trailing strings var antibody = doc["antibody"]; var matchEndInfo = endInfo.exec(antibody); if (matchEndInfo) { @@ -267,13 +218,13 @@ var getRPPALabel = function(doc) { var gene = doc["id"].split(':')[2]; if (phos) { var pfields = phos.split('_'); - phos = pfields[0]; + phos = 'p' + pfields[0]; for (var i = 1; i < pfields.length; i++) { - phos += ', ' + pfields[i]; + phos += ', p' + pfields[i]; } - return replaceUnderscore(gene + ': ' + antibody + ' (' + phos + ')'); + return replaceUnderscore(gene + ' (' + phos + ')'); } else { - return replaceUnderscore(gene + ': ' + antibody); + return replaceUnderscore(gene); } } From 7a0f847a24e292eebf18d967cb9d0439b342737f Mon Sep 17 00:00:00 2001 From: Michael Miller Date: Mon, 7 Jul 2014 12:41:07 -0700 Subject: [PATCH 47/62] add 'ffn_' prefix to ffn scripts --- .../{add_fm_custom_labels.py => ffn_add_fm_custom_labels.py} | 0 ...ate_fm_with_ffn_lookup.js => ffn_update_fm_with_ffn_lookup.js} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename db/scripts/{add_fm_custom_labels.py => ffn_add_fm_custom_labels.py} (100%) rename db/scripts/{update_fm_with_ffn_lookup.js => ffn_update_fm_with_ffn_lookup.js} (100%) diff --git a/db/scripts/add_fm_custom_labels.py b/db/scripts/ffn_add_fm_custom_labels.py similarity index 100% rename from db/scripts/add_fm_custom_labels.py rename to db/scripts/ffn_add_fm_custom_labels.py diff --git a/db/scripts/update_fm_with_ffn_lookup.js b/db/scripts/ffn_update_fm_with_ffn_lookup.js similarity index 100% rename from db/scripts/update_fm_with_ffn_lookup.js rename to db/scripts/ffn_update_fm_with_ffn_lookup.js From f756af0ba4fe816e724be94c58251d1465c2f241 Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Mon, 7 Jul 2014 15:11:46 -0700 Subject: [PATCH 48/62] added scripts to manage database import workflow --- db/scripts/datawarehouse_import.py | 166 ++++++++++++++++++ .../misc_aggregate_collection_fields.js | 54 ++++++ 2 files changed, 220 insertions(+) create mode 100644 db/scripts/datawarehouse_import.py create mode 100644 db/scripts/misc_aggregate_collection_fields.js diff --git a/db/scripts/datawarehouse_import.py b/db/scripts/datawarehouse_import.py new file mode 100644 index 0000000..6087d56 --- /dev/null +++ b/db/scripts/datawarehouse_import.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python + +import argparse +import csv +import os +import pymongo +import logging +import itertools +import inspect +import json +from copy import copy +from time import time + +from utilities import configure_logging +from subprocess import call + +''' +Usage: + python datawarehouse_import.py --config_file=/path/to/local/datawarehouse_import.json + +Requires config_file to be passed a file containing a JSON structure similar to the following: + { + "databases": { + "arbitrary_db_pointer_1": { + "vendor": "mongodb", + "host": "localhost", + "port": 4321 + }, + "arbitrary_db_pointer_2": { + "vendor": "mongodb", + "host": "localhost", + "port": 1234 + } + }, + "lookups_db": "arbitrary_db_pointer_2", + "imports": [ + { + "tumor_type": "XYZ", + "database": "arbitrary_db_pointer_1", + "collections": { + "feature_matrix": "/path/to/local/fmx/file", + "mutation_summary": "/path/to/local/mut_sum/file", + "copy_number_gistic": "/path/to/local/cn_gistic/file" + }, + "annotations": { + "CNVR": "/path/to/local/fmx/annotations/cnvr_file", + "METH": "/path/to/local/fmx/annotations/meth_file", + "RPPA": "/path/to/local/fmx/annotations/rppa_file" + } + } + ] + } +''' + +def execute_python(script_path, spec): + logging.debug("(%s,%s)" % (script_path, spec)) + current_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) + + if not "database" in spec: + logging.warn("NO DATABASE CONFIGURED") + return + + db = spec["database"] + exec_str = "python %s/%s --host=%s --port=%s --db=%s --f=%s" % (current_path, script_path, db["host"], str(db["port"]), spec["tumor_type"], spec["file"]) + logging.info("\n\t%s" % exec_str) + return_code = call([exec_str], shell=True) + + logging.debug("%s %s" % (script_path, return_code)) + +def execute_javascript(script_path, spec): + logging.debug("(%s,%s)" % (script_path, spec)) + + if not "database" in spec: + logging.warn("NO DATABASE CONFIGURED") + return + + current_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) + db = spec["database"] + + eval_expr = "" + if "lookups_db" in spec: + lookup_db = spec["lookups_db"] + eval_expr = "--eval=\'var lookups_db_uri=\"%s:%s/LOOKUPS\"\'" % (lookup_db["host"], str(lookup_db["port"])) + + exec_str = "mongo --quiet --host=%s --port=%s %s %s/%s %s" % (db["host"], str(db["port"]), spec["tumor_type"], current_path, script_path, eval_expr) + logging.info("\n\t%s" % exec_str) + return_code = call([exec_str], shell=True) + logging.debug("%s %s" % (script_path, return_code)) + +def process_import(config_json): + config_dbs = config_json["databases"] + + for db_key in config_dbs: + db = config_dbs[db_key] + logging.debug("databases [%s] %s:%s:%s" % (db_key, db["vendor"], db["host"], str(db["port"]))) + + for im in config_json["imports"]: + logging.info("### STARTED:%s:%s ###" % (im["tumor_type"], im["database"])) + + db_name = im["database"] + if db_name in config_dbs: + db_inst = config_dbs[db_name] + if not db_inst is None: im["database"] = db_inst + else: + logging.warning("unknown database [%s] [%s]" % (im["tumor_type"], db_name)) + continue + + if "lookups_db" in config_json: + im["lookups_db"] = config_dbs[config_json["lookups_db"]] + + im_collections = im["collections"] + if "feature_matrix" in im_collections: + im["file"] = im_collections["feature_matrix"] + execute_python("featurematrix_insert.py", im) + execute_javascript("featurematrix_fill_tags.js", im) + execute_javascript("featurematrix_fill_unid.js", im) + execute_javascript("featurematrix_mutated_samples.js", im) + execute_javascript("featurematrix_verify_entries.js", im) + + if "mutation_summary" in im_collections: + im["file"] = im_collections["mutation_summary"] + execute_python("mutationsummary_insert.py", im) + + if "copy_number_gistic" in im_collections: + im["file"] = im_collections["copy_number_gistic"] + execute_python("copynumbergistic_insert.py", im) + + if "annotations" in im: + im_clone = copy(im) + im_clone_annot = im_clone["annotations"] + if "CNVR" in im_clone_annot: + im_clone["file"] = im_clone_annot["CNVR"] + execute_python("featurematrix_annotate_CNVR.py", im_clone) + if "METH" in im_clone_annot: + im_clone["file"] = im_clone_annot["METH"] + execute_python("featurematrix_annotate_METH.py", im_clone) + if "RPPA" in im_clone_annot: + im_clone["file"] = im_clone_annot["RPPA"] + execute_python("featurematrix_annotate_RPPA.py", im_clone) + + execute_javascript("lookups_aggregate_sample_types.js", im) + execute_javascript("lookups_aggregate_clinical_variables.js", im) + execute_javascript("misc_aggregate_collection_fields.js", im) + + logging.info("### COMPLETED:%s ###" % im["tumor_type"]) + +def main(): + parser = argparse.ArgumentParser(description="Utility to import data into the Cancer Regulome Data Warehouse") + parser.add_argument("--config_file", required=False, default="datawarehouse_import.json", help="Configuration File") + parser.add_argument("--loglevel", default="INFO", help="Logging Level") + args = parser.parse_args() + + configure_logging(args.loglevel.upper()) + + startAt = time() + logging.info("\n------------------\nSTART\n------------------") + + logging.debug("config_file: %s" % args.config_file) + process_import(json.load(open(args.config_file))) + + endAt = time() + durAt = round(endAt - startAt, 3) + logging.info("\n------------------\nCOMPLETED in %s sec(s)\n------------------" % str(durAt)) + +if __name__ == "__main__": + main() diff --git a/db/scripts/misc_aggregate_collection_fields.js b/db/scripts/misc_aggregate_collection_fields.js new file mode 100644 index 0000000..9302fd3 --- /dev/null +++ b/db/scripts/misc_aggregate_collection_fields.js @@ -0,0 +1,54 @@ +/* + - Reads all collections in the database, and extract all fields, preparing a lookup collection + - Writes to "collection_fields" collection + + - Documents in collection will look like this: + { "_id" : , "value" : } + + Usage: + + mongo --host=$HOST $DB_NAME mapreduce_collection_fields.js + + // TODO : Organize output better + */ + +var mapFn = function () { + for (var field in this) { + if (field === "_id") continue; + emit(field, null); + } +}; + +var reduceFn = function (field) { + return field; +}; + +var finalizeFn = function() { + return target_collection; +}; + +var db_name = db["_name"]; +var pretty_print = function(msg) { + var dtFmt = (new Date()).toLocaleFormat("%Y-%m-%d %H:%M:%S,000"); + print("[INFO] " + dtFmt + " - misc_aggregate_collection_fields(" + db_name + ") - " + msg); +}; + +db["collection_fields"].drop(); + +db.getCollectionNames().forEach(function (collection_name) { + if (collection_name === "system.indexes") return; + if (collection_name === "collection_fields") return; + + pretty_print(collection_name); + db[collection_name].mapReduce(mapFn, reduceFn, { + "out": { + "merge": "collection_fields" + }, + "scope": { + "target_collection": collection_name + }, + "finalize": finalizeFn + }); +}); + +pretty_print("unique fields captured:" + db["collection_fields"].count()); From 80e9c1a2b103b362ae5415996106093f7f26d926 Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Mon, 7 Jul 2014 15:14:21 -0700 Subject: [PATCH 49/62] fixed dependency --- db/scripts/datawarehouse_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/scripts/datawarehouse_import.py b/db/scripts/datawarehouse_import.py index 6087d56..211ade4 100644 --- a/db/scripts/datawarehouse_import.py +++ b/db/scripts/datawarehouse_import.py @@ -146,7 +146,7 @@ def process_import(config_json): def main(): parser = argparse.ArgumentParser(description="Utility to import data into the Cancer Regulome Data Warehouse") - parser.add_argument("--config_file", required=False, default="datawarehouse_import.json", help="Configuration File") + parser.add_argument("--config_file", required=True, help="Configuration File") parser.add_argument("--loglevel", default="INFO", help="Logging Level") args = parser.parse_args() From 0902db3c365d1e8bed9449cf1dd5efb1d79d5c32 Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Mon, 7 Jul 2014 15:16:08 -0700 Subject: [PATCH 50/62] renaming/dropping --- db/scripts/featurematrix_tags_grouped.js | 39 --------------- db/scripts/featurematrix_tags_lookup.js | 50 ------------------- db/scripts/featurematrix_verify_entries.js | 50 +++++++++++++++++++ ...mx.js => misc_rename_protein_field_fmx.js} | 0 db/scripts/verify_entries.js | 44 ---------------- 5 files changed, 50 insertions(+), 133 deletions(-) delete mode 100644 db/scripts/featurematrix_tags_grouped.js delete mode 100644 db/scripts/featurematrix_tags_lookup.js create mode 100644 db/scripts/featurematrix_verify_entries.js rename db/scripts/{rename_protein_field_in_fmx.js => misc_rename_protein_field_fmx.js} (100%) delete mode 100644 db/scripts/verify_entries.js diff --git a/db/scripts/featurematrix_tags_grouped.js b/db/scripts/featurematrix_tags_grouped.js deleted file mode 100644 index 4afb06e..0000000 --- a/db/scripts/featurematrix_tags_grouped.js +++ /dev/null @@ -1,39 +0,0 @@ -/* -- Reads from "feature_tags" collection in LOOKUPS database -- Writes to map reduce output to "feature_tags_grouped" collection - -Usage: - - mongo --host=$HOST $DB_NAME featurematrix_tags_grouped.js - -Example of the type of features targeted for aggregation by this script. Must be run on each tumor_type feature matrix independently. -{ - "_id" : ObjectId("53a0a595044458b47fe9b0a9"), - "tag" : "10p15.1", - "feature_id" : "C:CNVR:10p15.1:chr10:5610350:5628570::BLCA-TP_Gistic_ROI_d_amp", - "source" : "CNVR", - "db_name" : "BLCA" -} - */ - -var result = db.feature_tags.aggregate([ - { - "$group": { - "_id": "$tag", - "tumor_types": { - "$addToSet": "$db_name" - } - } - } -]); -print("aggregate completed"); - -result["result"].forEach(function(d) { - db.feature_tags_grouped.insert({ - "tag": d["_id"], - "tumor_types": d["tumor_types"] - }); -}); -print("insert completed"); - -print("feature_tags_grouped:" + db.feature_tags_grouped.count()); diff --git a/db/scripts/featurematrix_tags_lookup.js b/db/scripts/featurematrix_tags_lookup.js deleted file mode 100644 index 7d3b366..0000000 --- a/db/scripts/featurematrix_tags_lookup.js +++ /dev/null @@ -1,50 +0,0 @@ -/* -- Reads from "feature_matrix" collection -- Aggregates tags and writes individual entries per tag to lookups database - -Usage: - - mongo --host=$HOST $DB_NAME featurematrix_tags_lookup.js --eval="var lookupsDbUri='hostname:port/LOOKUPS';" - -Example of the type of features targeted for aggregation by this script. Must be run on each tumor_type feature matrix independently. -{ - "_id" : ObjectId("52d484d43d8a1d038104f407"), - "id" : "B:GNAB:A1BG:chr19:58858172:58874214:-:y_n_somatic", - "strand" : "-", - "tags" : [ "KLHDC9", "PFDN2", "ARHGAP30", "PVRL4", "SRGAP2", "1q23.3" ] - "end" : 58874214, - "start" : 58858172, - "source" : "GNAB", - "chr" : "19", - "code" : "y_n_somatic", - "type" : "B" -} - */ - -var query = { - "$and": [ - { "tags": { "$exists": true } }, - { "tags": { "$nin": ["NO_MATCH"] } } - ] -}; - -var lookupsDb = connect(lookupsDbUri); -var db_name = db["_name"]; - -db.feature_matrix.find(query).forEach(function(d) { - var feature_tags = d["tags"]; - if (feature_tags === undefined || feature_tags.length <= 0) return; - - var feature_id = d["id"]; - var source = d["source"]; - feature_tags.forEach(function(tag) { - if (tag && tag != "") { - lookupsDb["feature_tags"].insert({ - "tag": tag, - "feature_id": feature_id, - "source": source, - "db_name": db_name - }); - } - }); -}); diff --git a/db/scripts/featurematrix_verify_entries.js b/db/scripts/featurematrix_verify_entries.js new file mode 100644 index 0000000..8968dc0 --- /dev/null +++ b/db/scripts/featurematrix_verify_entries.js @@ -0,0 +1,50 @@ +var exists = { "$exists": true }; +var not_exists = { "$exists": false }; + +var cnt_gexp_exists = db["feature_matrix"].find({ "gene": exists, "source":"GEXP" }).count(); +var cnt_gexp_not_exists = db["feature_matrix"].find({ "gene": not_exists, "source":"GEXP" }).count(); + +var cnt_gnab_exists = db["feature_matrix"].find({ "gene": exists, "source":"GNAB" }).count(); +var cnt_gnab_not_exists = db["feature_matrix"].find({ "gene": not_exists, "source":"GNAB" }).count(); + +var cnt_rppa_antibody_exists = db["feature_matrix"].find({ "antibody": exists, "source":"RPPA" }).count(); +var cnt_rppa_antibody_not_exists = db["feature_matrix"].find({ "antibody": not_exists, "source":"RPPA" }).count(); + +var cnt_rppa_exists = db["feature_matrix"].find({ "refGenes": exists, "source":"RPPA" }).count(); +var cnt_rppa_not_exists = db["feature_matrix"].find({ "refGenes": not_exists, "source":"RPPA" }).count(); + +var cnt_meth_probe_exists = db["feature_matrix"].find({ "probe": exists, "source":"METH" }).count(); +var cnt_meth_probe_not_exists = db["feature_matrix"].find({ "probe": not_exists, "source":"METH" }).count(); + +var cnt_meth_exists = db["feature_matrix"].find({ "refGenes": exists, "source":"METH" }).count(); +var cnt_meth_not_exists = db["feature_matrix"].find({ "refGenes": not_exists, "source":"METH" }).count(); + +var cnt_mirn_exists = db["feature_matrix"].find({ "microRNA": exists, "source":"MIRN" }).count(); +var cnt_mirn_not_exists = db["feature_matrix"].find({ "microRNA": not_exists, "source":"MIRN" }).count(); + +var cnt_cnvr_exists = db["feature_matrix"].find({ "refGenes": exists, "source":"CNVR" }).count(); +var cnt_cnvr_not_exists = db["feature_matrix"].find({ "refGenes": not_exists, "source":"CNVR" }).count(); + +var cnt_tags_exists = db["feature_matrix"].find({ "tags": exists }).count(); +var cnt_tags_not_exists = db["feature_matrix"].find({ "tags": not_exists }).count(); + +var cnt_label_exists = db["feature_matrix"].find({ "label": exists }).count(); +var cnt_label_not_exists = db["feature_matrix"].find({ "label": not_exists }).count(); + +var db_name = db["_name"]; +var pretty_print = function(msg) { + var dtFmt = (new Date()).toLocaleFormat("%Y-%m-%d %H:%M:%S,000"); + print("[INFO] " + dtFmt + " - featurematrix_verify_entries(" + db_name + ") - " + msg); +}; + +pretty_print("header: :: ['count exists'/'count not exists']"); +pretty_print("GEXP:gene [" + cnt_gexp_exists + "/" + cnt_gexp_not_exists + "]"); +pretty_print("GNAB:gene [" + cnt_gnab_exists + "/" + cnt_gnab_not_exists + "]"); +pretty_print("RPPA:antibody [" + cnt_rppa_antibody_exists + "/" + cnt_rppa_antibody_not_exists + "]"); +pretty_print("RPPA:refGenes [" + cnt_rppa_exists + "/" + cnt_rppa_not_exists + "]"); +pretty_print("METH:probe [" + cnt_meth_probe_exists + "/" + cnt_meth_probe_not_exists + "]"); +pretty_print("METH:refGenes [" + cnt_meth_exists + "/" + cnt_meth_not_exists + "]"); +pretty_print("MIRN:microRNA [" + cnt_mirn_exists + "/" + cnt_mirn_not_exists + "]"); +pretty_print("CNVR:refGenes [" + cnt_cnvr_exists + "/" + cnt_cnvr_not_exists + "]"); +pretty_print("*:tags [" + cnt_tags_exists + "/" + cnt_tags_not_exists + "]"); +pretty_print("*:label [" + cnt_label_exists + "/" + cnt_label_not_exists + "]"); \ No newline at end of file diff --git a/db/scripts/rename_protein_field_in_fmx.js b/db/scripts/misc_rename_protein_field_fmx.js similarity index 100% rename from db/scripts/rename_protein_field_in_fmx.js rename to db/scripts/misc_rename_protein_field_fmx.js diff --git a/db/scripts/verify_entries.js b/db/scripts/verify_entries.js deleted file mode 100644 index 0b12db1..0000000 --- a/db/scripts/verify_entries.js +++ /dev/null @@ -1,44 +0,0 @@ -var exists = { "$exists": true }; -var not_exists = { "$exists": false }; - -var cnt_gexp_exists = db.feature_matrix.find({ "gene": exists, "source":"GEXP" }).count(); -var cnt_gexp_not_exists = db.feature_matrix.find({ "gene": not_exists, "source":"GEXP" }).count(); - -var cnt_gnab_exists = db.feature_matrix.find({ "gene": exists, "source":"GNAB" }).count(); -var cnt_gnab_not_exists = db.feature_matrix.find({ "gene": not_exists, "source":"GNAB" }).count(); - -var cnt_rppa_antibody_exists = db.feature_matrix.find({ "antibody": exists, "source":"RPPA" }).count(); -var cnt_rppa_antibody_not_exists = db.feature_matrix.find({ "antibody": not_exists, "source":"RPPA" }).count(); - -var cnt_rppa_exists = db.feature_matrix.find({ "refGenes": exists, "source":"RPPA" }).count(); -var cnt_rppa_not_exists = db.feature_matrix.find({ "refGenes": not_exists, "source":"RPPA" }).count(); - -var cnt_meth_probe_exists = db.feature_matrix.find({ "probe": exists, "source":"METH" }).count(); -var cnt_meth_probe_not_exists = db.feature_matrix.find({ "probe": not_exists, "source":"METH" }).count(); - -var cnt_meth_exists = db.feature_matrix.find({ "refGenes": exists, "source":"METH" }).count(); -var cnt_meth_not_exists = db.feature_matrix.find({ "refGenes": not_exists, "source":"METH" }).count(); - -var cnt_mirn_exists = db.feature_matrix.find({ "microRNA": exists, "source":"MIRN" }).count(); -var cnt_mirn_not_exists = db.feature_matrix.find({ "microRNA": not_exists, "source":"MIRN" }).count(); - -var cnt_cnvr_exists = db.feature_matrix.find({ "refGenes": exists, "source":"CNVR" }).count(); -var cnt_cnvr_not_exists = db.feature_matrix.find({ "refGenes": not_exists, "source":"CNVR" }).count(); - -var cnt_tags_exists = db.feature_matrix.find({ "tags": exists }).count(); -var cnt_tags_not_exists = db.feature_matrix.find({ "tags": not_exists }).count(); - -var cnt_label_exists = db.feature_matrix.find({ "label": exists }).count(); -var cnt_label_not_exists = db.feature_matrix.find({ "label": not_exists }).count(); - -print(":: ['count exists'/'count not exists']"); -print(db + ":GEXP:gene [" + cnt_gexp_exists + "/" + cnt_gexp_not_exists + "]"); -print(db + ":GNAB:gene [" + cnt_gnab_exists + "/" + cnt_gnab_not_exists + "]"); -print(db + ":RPPA:antibody [" + cnt_rppa_antibody_exists + "/" + cnt_rppa_antibody_not_exists + "]"); -print(db + ":RPPA:refGenes [" + cnt_rppa_exists + "/" + cnt_rppa_not_exists + "]"); -print(db + ":METH:probe [" + cnt_meth_probe_exists + "/" + cnt_meth_probe_not_exists + "]"); -print(db + ":METH:refGenes [" + cnt_meth_exists + "/" + cnt_meth_not_exists + "]"); -print(db + ":MIRN:microRNA [" + cnt_mirn_exists + "/" + cnt_mirn_not_exists + "]"); -print(db + ":CNVR:refGenes [" + cnt_cnvr_exists + "/" + cnt_cnvr_not_exists + "]"); -print(db + ":*:tags [" + cnt_tags_exists + "/" + cnt_tags_not_exists + "]"); -print(db + ":*:label [" + cnt_label_exists + "/" + cnt_label_not_exists + "]"); \ No newline at end of file From 218967d396eb41ffd0bb579801269b9d936c50e6 Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Mon, 7 Jul 2014 15:18:50 -0700 Subject: [PATCH 51/62] logging updates --- db/scripts/copynumbergistic_insert.py | 4 +- ...CNVR.py => featurematrix_annotate_CNVR.py} | 6 +-- ...METH.py => featurematrix_annotate_METH.py} | 34 +++++++------- ...RPPA.py => featurematrix_annotate_RPPA.py} | 6 +-- ...ill_tags.js => featurematrix_fill_tags.js} | 4 +- ...nique_id.js => featurematrix_fill_unid.js} | 5 ++- db/scripts/featurematrix_insert.py | 4 +- ...es.js => featurematrix_mutated_samples.js} | 20 ++++++--- .../generate_clinical_variables_lookup.js | 38 ---------------- db/scripts/generate_sample_types_lookup.js | 26 ----------- .../lookups_aggregate_clinical_variables.js | 45 +++++++++++++++++++ db/scripts/lookups_aggregate_sample_types.js | 33 ++++++++++++++ .../{extract_fmx.js => misc_extract_fmx.js} | 4 +- ....js => misc_fmx_adjustvalues_GEXP_STAD.js} | 0 db/scripts/utilities.py | 2 +- 15 files changed, 128 insertions(+), 103 deletions(-) rename db/scripts/{annotate_CNVR.py => featurematrix_annotate_CNVR.py} (93%) rename db/scripts/{annotate_METH.py => featurematrix_annotate_METH.py} (80%) rename db/scripts/{annotate_RPPA.py => featurematrix_annotate_RPPA.py} (92%) rename db/scripts/{fill_tags.js => featurematrix_fill_tags.js} (83%) rename db/scripts/{featurematrix_featureunique_id.js => featurematrix_fill_unid.js} (88%) rename db/scripts/{featurematrix_mapreduce_mutated_samples.js => featurematrix_mutated_samples.js} (69%) delete mode 100644 db/scripts/generate_clinical_variables_lookup.js delete mode 100644 db/scripts/generate_sample_types_lookup.js create mode 100644 db/scripts/lookups_aggregate_clinical_variables.js create mode 100644 db/scripts/lookups_aggregate_sample_types.js rename db/scripts/{extract_fmx.js => misc_extract_fmx.js} (87%) rename db/scripts/{featurematrix_adjustvalues_GEXP_STAD.js => misc_fmx_adjustvalues_GEXP_STAD.js} (100%) diff --git a/db/scripts/copynumbergistic_insert.py b/db/scripts/copynumbergistic_insert.py index f62fbb1..96e7321 100644 --- a/db/scripts/copynumbergistic_insert.py +++ b/db/scripts/copynumbergistic_insert.py @@ -8,7 +8,7 @@ from utilities import configure_logging def extract_records(file_path): - logging.info("extract_records(%s)" % file_path) + logging.info(file_path) with open(file_path, "rb") as csvfile: csvreader = csv.reader(csvfile, delimiter="\t") @@ -30,7 +30,7 @@ def extract_records(file_path): yield record - logging.info("extract_records(%s):samples=%s" % (file_path, str(len(ids)))) + logging.info("samples=%s" % str(len(ids))) def values_dict(ids, values): result = {} diff --git a/db/scripts/annotate_CNVR.py b/db/scripts/featurematrix_annotate_CNVR.py similarity index 93% rename from db/scripts/annotate_CNVR.py rename to db/scripts/featurematrix_annotate_CNVR.py index 9036ac1..cb7f12e 100644 --- a/db/scripts/annotate_CNVR.py +++ b/db/scripts/featurematrix_annotate_CNVR.py @@ -60,11 +60,11 @@ def find_and_modify(collection, refGenes_by_id): if collection.find({ "id": id }).count() == 1: collection.find_and_modify({ "id": id }, {"$set":{ "refGenes": refGenes }}) - logging.debug("find_and_modify [%s] [%s]===%s" % (count, id, refGenes)) + logging.debug("[%s] [%s]===%s" % (count, id, refGenes)) count += 1 if count % 1000 == 0: logging.info("update [%s]" % count) - logging.info("total find_and_modify count=%s" % count) + logging.info("total count=%s" % count) def main(): parser = argparse.ArgumentParser(description="Utility to annotate features with genes (tags) in MongoDB") @@ -94,7 +94,7 @@ def main(): conn.close() - logging.info("annotate_features_with_genes(%s):complete" % args.db) + logging.info("COMPLETE") if __name__ == "__main__": main() diff --git a/db/scripts/annotate_METH.py b/db/scripts/featurematrix_annotate_METH.py similarity index 80% rename from db/scripts/annotate_METH.py rename to db/scripts/featurematrix_annotate_METH.py index 2b00042..d3285c7 100755 --- a/db/scripts/annotate_METH.py +++ b/db/scripts/featurematrix_annotate_METH.py @@ -45,40 +45,44 @@ def extract_tags_by_id(filename): r_by_id = {} for row in csvreader: if len(row) >= 3: - probe_id = row[0] - refGenes = row[3].split(";") - uniqGenes = filter(None, list(set(refGenes))) - logging.debug("extract_tags_by_id=%s:%s:%s" % (probe_id, refGenes, uniqGenes)) - if len(uniqGenes) > 0: r_by_id[probe_id] = uniqGenes + feature_id = row[0] + count = row[1] + uniqGenes = [] + if count > 0: + refGenes = row[2].split(";") + uniqGenes = filter(None, list(set(refGenes))) + + logging.debug("%s:%s:%s" % (feature_id, refGenes, uniqGenes)) + if len(uniqGenes) > 0: r_by_id[feature_id] = uniqGenes else: skipcount += 1 - logging.warning("extract_tags_by_id=skipping:%s" % skipcount) + logging.warning("skipping:%s" % skipcount) return r_by_id def find_and_modify(collection, tags_by_id): count = 0 skipcount = 0 - for doc in collection.find({ "source": "METH", "probe": { "$exists": True } }, { "probe": True }): - if "probe" in doc: - probe_id = str(doc["probe"]) - if probe_id in tags_by_id: - tags = tags_by_id[probe_id] + for doc in collection.find({ "source": "METH", "probe": { "$exists": True } }, { "values": False }): + if "id" in doc: + feature_id = str(doc["id"]) + if feature_id in tags_by_id: + tags = tags_by_id[feature_id] if not tags is None: collection.find_and_modify({ "_id": doc["_id"] }, { "$set":{ "refGenes": tags }}) else: - logging.warning("skipping:tags not found:%s" % probe_id) + logging.warning("skipping:tags not found:%s" % feature_id) skipcount += 1 else: - logging.warning("skipping:probe not found:%s" % probe_id) + logging.debug("skipping:feature_id not found:%s" % feature_id) skipcount += 1 else: - logging.warning("skipping:probe not in doc:%s" % str(doc)) + logging.warning("skipping:feature_id not in doc:%s" % str(doc)) skipcount += 1 if count > 0 and count % 1000 == 0: logging.info("update [%s]" % count) - logging.info("total find_and_modify count=%s [skip=%s]" % (count, skipcount)) + logging.info("total count=%s [skip=%s]" % (count, skipcount)) def main(): parser = argparse.ArgumentParser(description="Utility to annotate features with probe IDs (i.e. METH) to genes based on annotations file") diff --git a/db/scripts/annotate_RPPA.py b/db/scripts/featurematrix_annotate_RPPA.py similarity index 92% rename from db/scripts/annotate_RPPA.py rename to db/scripts/featurematrix_annotate_RPPA.py index 9bc49c4..9cacbc1 100755 --- a/db/scripts/annotate_RPPA.py +++ b/db/scripts/featurematrix_annotate_RPPA.py @@ -54,14 +54,14 @@ def find_and_modify(collection, tags_by_id): cnt = collection.find({ "antibody": id }).count() if cnt == 1: collection.find_and_modify({ "antibody": id }, { "$set":{ "refGenes": tags }}) - logging.debug("find_and_modify [%s] [%s]===%s" % (count, id, tags)) + logging.debug("[%s] [%s]===%s" % (count, id, tags)) count += 1 else: - logging.warning("skipping: find_and_modify [%s] [%s]===%s" % (cnt, id, tags)) + logging.warning("skipping: [%s] [%s]===%s" % (cnt, id, tags)) skipcount += 1 if count % 100 == 0: logging.info("update [%s]" % count) - logging.info("total find_and_modify count=%s [skip=%s]" % (count, skipcount)) + logging.info("total count=%s [skip=%s]" % (count, skipcount)) def main(): parser = argparse.ArgumentParser(description="Utility to annotate features with antibody IDs (i.e. RPPA) to genes based on annotations file") diff --git a/db/scripts/fill_tags.js b/db/scripts/featurematrix_fill_tags.js similarity index 83% rename from db/scripts/fill_tags.js rename to db/scripts/featurematrix_fill_tags.js index 546566c..a75b064 100644 --- a/db/scripts/fill_tags.js +++ b/db/scripts/featurematrix_fill_tags.js @@ -27,7 +27,7 @@ var update_tags = function(d) { if (tags.length <= 0) tags.push("NO_MATCH"); - db.feature_matrix.update({ "_id": d["_id"] }, { "$set": { "tags": tags } } ); + db["feature_matrix"].update({ "_id": d["_id"] }, { "$set": { "tags": tags } } ); }; -db.feature_matrix.find(query).forEach(update_tags); \ No newline at end of file +db["feature_matrix"].find(query).forEach(update_tags); \ No newline at end of file diff --git a/db/scripts/featurematrix_featureunique_id.js b/db/scripts/featurematrix_fill_unid.js similarity index 88% rename from db/scripts/featurematrix_featureunique_id.js rename to db/scripts/featurematrix_fill_unid.js index 79afdca..bfa053a 100644 --- a/db/scripts/featurematrix_featureunique_id.js +++ b/db/scripts/featurematrix_fill_unid.js @@ -4,7 +4,7 @@ Usage: - mongo --host=$HOST $DB_NAME featurematrix_featureunique_id.js + mongo --host=$HOST $DB_NAME featurematrix_fill_unid.js Example of the type of features targeted for aggregation by this script. Must be run on each tumor_type feature matrix independently. { @@ -65,7 +65,8 @@ var UNID_generators = { }; for (var src in UNID_generators) { - print("[" + db_name + "]:updating:" + src + ":" + db["feature_matrix"].find({ "source": src }).count()); + var dtFmt = (new Date()).toLocaleFormat("%Y-%m-%d %H:%M:%S,000"); + print("[INFO] " + dtFmt + " - featurematrix_fill_unid(" + db_name + "):" + src + ":" + db["feature_matrix"].find({ "source": src }).count()); var unid_generator = UNID_generators[src]; db["feature_matrix"].find({ "source": src }).forEach(function(doc) { diff --git a/db/scripts/featurematrix_insert.py b/db/scripts/featurematrix_insert.py index b812186..5b9f05a 100755 --- a/db/scripts/featurematrix_insert.py +++ b/db/scripts/featurematrix_insert.py @@ -8,7 +8,7 @@ from utilities import configure_logging def extract_features(file_path): - logging.info("extract_features(%s)" % file_path) + logging.info(file_path) with open(file_path, "rb") as csvfile: csvreader = csv.reader(csvfile, delimiter="\t") @@ -28,7 +28,7 @@ def extract_features(file_path): yield feature_object - logging.info("extract_features(%s):samples=%s" % (file_path, str(len(ids)))) + logging.info("samples=%s" % str(len(ids))) def extract_feature_dict(feature_id): feature_parts = feature_id.split(":") diff --git a/db/scripts/featurematrix_mapreduce_mutated_samples.js b/db/scripts/featurematrix_mutated_samples.js similarity index 69% rename from db/scripts/featurematrix_mapreduce_mutated_samples.js rename to db/scripts/featurematrix_mutated_samples.js index f530dd3..ced79bd 100644 --- a/db/scripts/featurematrix_mapreduce_mutated_samples.js +++ b/db/scripts/featurematrix_mutated_samples.js @@ -61,13 +61,19 @@ var query = { "gene": { "$exists": true } }; -print("matching features from feature_matrix:" + db.feature_matrix.find(query).count()); -db.feature_matrix.mapReduce(map, reduce, { "query": query, "out": "mutated_samples_mrtemp", "finalize": finalize }); +var db_name = db["_name"]; +var pretty_print = function(msg) { + var dtFmt = (new Date()).toLocaleFormat("%Y-%m-%d %H:%M:%S,000"); + print("[INFO] " + dtFmt + " - featurematrix_mutated_samples(" + db_name + ") - " + msg); +}; + +pretty_print("matching features from feature_matrix:" + db["feature_matrix"].find(query).count()); +db["feature_matrix"].mapReduce(map, reduce, { "query": query, "out": "mutated_samples_mrtemp", "finalize": finalize }); -db.mutated_samples.drop(); -db.mutated_samples_mrtemp.find().forEach(function(d) { - db.mutated_samples.insert(d.value); +db["mutated_samples"].drop(); +db["mutated_samples_mrtemp"].find().forEach(function(d) { + db["mutated_samples"].insert(d.value); }); -db.mutated_samples_mrtemp.drop(); +db["mutated_samples_mrtemp"].drop(); -print("final result in mutated_samples:" + db.mutated_samples.find().count()); +pretty_print("final result in mutated_samples:" + db["mutated_samples"].find().count()); diff --git a/db/scripts/generate_clinical_variables_lookup.js b/db/scripts/generate_clinical_variables_lookup.js deleted file mode 100644 index df3e898..0000000 --- a/db/scripts/generate_clinical_variables_lookup.js +++ /dev/null @@ -1,38 +0,0 @@ -/* -- Extracts clinical variables from "feature_matrix" collection into GLOBAL lookup collection - -Usage: - - mongo --host=$HOST $DB_NAME generate_clinical_variables_lookup.js --eval="var lookupsDbUri='hostname:port/LOOKUPS';" - */ - -var lookupsDb = connect(lookupsDbUri); -var db_name = db["_name"]; -print("[" + db_name + "]:script:started"); - -var group_variables_fn = function(doc) { - var details = { - "id": doc["id"], - "label": doc["label"], - "tumor_type": db_name, - "source": doc["source"] - }; - lookupsDb["all_clinical"].update({ - "id": doc["id"] - }, { - "$set": { "label": doc["label"] }, - "$push": { "features": details } - }, true); -}; - -print("[" + db_name + "]:initial check=LOOKUPS:" + lookupsDb["all_clinical"].count()); - -print("[" + db_name + "]:update:started:CLIN"); -db["feature_matrix"].find({ "source": "CLIN" }).forEach(group_variables_fn); -print("[" + db_name + "]:update:started:SAMP"); -db["feature_matrix"].find({ "source": "SAMP" }).forEach(group_variables_fn); -print("[" + db_name + "]:update:completed"); - -var result = lookupsDb["all_clinical"].aggregate([ { "$unwind": "$features" }, { "$group": { "_id": "$features.tumor_type", "cnt": { "$sum": 1 } } } ]); -result["result"].forEach(printjson); -print("[" + db_name + "]:script:completed"); diff --git a/db/scripts/generate_sample_types_lookup.js b/db/scripts/generate_sample_types_lookup.js deleted file mode 100644 index 3723259..0000000 --- a/db/scripts/generate_sample_types_lookup.js +++ /dev/null @@ -1,26 +0,0 @@ -/* -- Extracts clinical variables from "feature_matrix" collection into GLOBAL lookup collection - -Usage: - - mongo --host=$HOST $DB_NAME generate_sample_types_lookup.js --eval="var lookupsDbUri='hostname:port/LOOKUPS';" - */ - -var query = { "id": "C:SAMP:TNtype:::::" }; -var projection = { "_id": false }; -var tumor_type = db["name"]; - -var lookupsDb = connect(lookupsDbUri); - -print("initial check=" + tumor_type + ":" + db.feature_matrix.find(query, projection).count()); -print("initial check=LOOKUPS:" + lookupsDb.sample_types.count()); - -var eachFn = function(doc) { - doc["tumor_type"] = tumor_type; - lookupsDb.sample_types.insert(doc); -}; - -db.feature_matrix.find(query, projection).forEach(eachFn); - -print("final check=LOOKUPS:" + lookupsDb.sample_types.count()); -print("completed"); diff --git a/db/scripts/lookups_aggregate_clinical_variables.js b/db/scripts/lookups_aggregate_clinical_variables.js new file mode 100644 index 0000000..7fa33e1 --- /dev/null +++ b/db/scripts/lookups_aggregate_clinical_variables.js @@ -0,0 +1,45 @@ +/* +- Extracts clinical variables from "feature_matrix" collection into GLOBAL lookup collection + +Usage: + + mongo --host=$HOST $DB_NAME lookups_aggregate_clinical_variables.js --eval="var lookups_db_uri='hostname:port/LOOKUPS';" + */ + +var lookups_db = connect(lookups_db_uri); +var db_name = db["_name"]; +var pretty_print = function(msg) { + var dtFmt = (new Date()).toLocaleFormat("%Y-%m-%d %H:%M:%S,000"); + print("[INFO] " + dtFmt + " - lookups_aggregate_clinical_variables(" + db_name + ") - " + msg); +}; + +pretty_print("START"); + +var group_variables_fn = function(doc) { + var details = { + "id": doc["id"], + "label": doc["label"], + "tumor_type": db_name, + "source": doc["source"] + }; + lookups_db["all_clinical"].update({ + "id": doc["id"] + }, { + "$set": { "label": doc["label"] }, + "$push": { "features": details } + }, true); +}; + +pretty_print("initial check=LOOKUPS:" + lookups_db["all_clinical"].count()); + +pretty_print("update:started:CLIN"); +db["feature_matrix"].find({ "source": "CLIN" }).forEach(group_variables_fn); + +pretty_print("update:started:SAMP"); +db["feature_matrix"].find({ "source": "SAMP" }).forEach(group_variables_fn); + +pretty_print("update:completed"); + +var result = lookups_db["all_clinical"].aggregate([ { "$unwind": "$features" }, { "$group": { "_id": "$features.tumor_type", "cnt": { "$sum": 1 } } } ]); +result["result"].forEach(printjson); +pretty_print("COMPLETED"); diff --git a/db/scripts/lookups_aggregate_sample_types.js b/db/scripts/lookups_aggregate_sample_types.js new file mode 100644 index 0000000..5f6b532 --- /dev/null +++ b/db/scripts/lookups_aggregate_sample_types.js @@ -0,0 +1,33 @@ +/* +- Extracts clinical variables from "feature_matrix" collection into GLOBAL lookup collection + +Usage: + + mongo --host=$HOST $DB_NAME lookups_aggregate_sample_types.js --eval="var lookups_db_uri='hostname:port/LOOKUPS';" + */ + +var lookups_db = connect(lookups_db_uri); +var db_name = db["_name"]; +var query = { "id": "C:SAMP:TNtype:::::" }; + +var pretty_print = function(msg) { + var dtFmt = (new Date()).toLocaleFormat("%Y-%m-%d %H:%M:%S,000"); + print("[INFO] " + dtFmt + " - lookups_aggregate_sample_types(" + db_name + ") - " + msg); +}; + +pretty_print("lookups_db=" + lookups_db_uri + ":" + lookups_db); +pretty_print("initial check=" + db["feature_matrix"].find(query).count()); +pretty_print("initial check=LOOKUPS:" + lookups_db["sample_types"].count()); + +db["feature_matrix"].find(query).forEach(function(doc) { + lookups_db["sample_types"].insert({ + "id": doc["id"], + "label": doc["label"], + "source": doc["source"], + "values": doc["values"], + "tumor_type": db_name + }); +}); + +pretty_print("final check=LOOKUPS:" + lookups_db["sample_types"].count()); +pretty_print("COMPLETED"); diff --git a/db/scripts/extract_fmx.js b/db/scripts/misc_extract_fmx.js similarity index 87% rename from db/scripts/extract_fmx.js rename to db/scripts/misc_extract_fmx.js index c6215d1..00751f6 100644 --- a/db/scripts/extract_fmx.js +++ b/db/scripts/misc_extract_fmx.js @@ -47,7 +47,7 @@ var outputFn = function (selected_samples) { var query = { "source": feature_source }; // collects a limited set of samples -db.feature_matrix.find(query).limit(record_count).forEach(collectFn(sample_count, selected_samples)); +db["feature_matrix"].find(query).limit(record_count).forEach(collectFn(sample_count, selected_samples)); // outputs data for the selected samples -db.feature_matrix.find(query).limit(record_count).forEach(outputFn(selected_samples)); \ No newline at end of file +db["feature_matrix"].find(query).limit(record_count).forEach(outputFn(selected_samples)); \ No newline at end of file diff --git a/db/scripts/featurematrix_adjustvalues_GEXP_STAD.js b/db/scripts/misc_fmx_adjustvalues_GEXP_STAD.js similarity index 100% rename from db/scripts/featurematrix_adjustvalues_GEXP_STAD.js rename to db/scripts/misc_fmx_adjustvalues_GEXP_STAD.js diff --git a/db/scripts/utilities.py b/db/scripts/utilities.py index 2cd20d0..36bf972 100644 --- a/db/scripts/utilities.py +++ b/db/scripts/utilities.py @@ -8,7 +8,7 @@ def configure_logging(logging_level=logging.DEBUG): ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging_level) - formatter = logging.Formatter("[%(levelname)s] %(asctime)s - %(message)s") + formatter = logging.Formatter("[%(levelname)s] %(asctime)s - %(module)s.%(funcName)s - %(message)s") ch.setFormatter(formatter) root.addHandler(ch) \ No newline at end of file From ab5cdd75a89073c1a01424d6717c78a71d9ee1d8 Mon Sep 17 00:00:00 2001 From: kleinone Date: Mon, 7 Jul 2014 19:07:41 -0700 Subject: [PATCH 52/62] Implemented non-coding display mode. Updated SeqPeek dependency. --- app/scripts/views/seqpeek/view.js | 142 +++++++++++++++++++++++------- bower.json | 2 +- 2 files changed, 112 insertions(+), 32 deletions(-) diff --git a/app/scripts/views/seqpeek/view.js b/app/scripts/views/seqpeek/view.js index f3d859e..e06df4a 100644 --- a/app/scripts/views/seqpeek/view.js +++ b/app/scripts/views/seqpeek/view.js @@ -15,6 +15,10 @@ define([ MutationsMapTpl, MutationsMapTableTpl, SampleListCaptionTpl ) { + var DISPLAY_MODES = { + ALL: 1, + PROTEIN: 2 + }; var MINI_LOCATOR_WIDTH = 400; var MINI_LOCATOR_HEIGHT = 24; @@ -45,7 +49,7 @@ define([ } }; - var VISUALIZATION_MODE = "ALL"; + var CURRENT_MODE = DISPLAY_MODES.ALL; var MUTATION_TYPE_COLOR_MAP = { Nonsense_Mutation: "red", @@ -188,8 +192,7 @@ define([ this.genes = this.options["genes"] || []; if (!_.isEmpty(this.genes)) this.selected_gene = _.first(this.genes); - //var renderFn = _.after(1 + (2 * this.tumor_types.length), this.__load_protein_domains); - var renderFn = _.after(1 + (2 * this.tumor_types.length), this.__preprocess_data); + var renderFn = _.after(1 + (2 * this.tumor_types.length), this.__preprocess_data_and_render); this.model["mutsig"].on("load", renderFn, this); @@ -242,12 +245,9 @@ define([ this.$(".mutations_map_table").html(""); - var mutations = this.__filter_data(this.__parse_mutations()); - var mutsig_ranks = this.__filter_mutsig_data(this.__parse_mutsig()); + var mutations = this.__parse_mutations(); - var formatter = function (value) { - return parseInt(value) + "%"; - }; + var mutsig_ranks = this.__filter_mutsig_data(this.__parse_mutsig()); var data_items = _.map(this.tumor_types, function (tumor_type) { var statistics = { @@ -322,8 +322,6 @@ define([ var uniprot_id = this.gene_to_uniprot_mapping[this.selected_gene]; var protein_data = this.found_protein_domains[uniprot_id]; - var region_data_old = [ { "type": "exon", "start": 0, "end": protein_data["length"] } ]; - var all_mutations = []; _.each(this.__parse_mutations(), function(mutation_array, tumor_type) { Array.prototype.push.apply(all_mutations, mutation_array); @@ -378,10 +376,17 @@ define([ this.__render_tracks(seqpeek_data, region_data, protein_data, seqpeek_tick_track_element, seqpeek_domain_track_element); }, - __render_tracks: function(mutation_data, region_array, protein_data, seqpeek_tick_track_element, seqpeek_domain_track_element) { - console.debug("seqpeek/view.__render_tracks"); + __build_seqpeek_config: function(region_array) { + if (CURRENT_MODE == DISPLAY_MODES.PROTEIN) { + return this.__build_seqpeek_config_for_protein_view(region_array); + } + else { + return this.__build_seqpeek_config_for_genomic_view(region_array); + } + }, - var seqpeek = SeqPeekBuilder.create({ + __build_seqpeek_config_for_protein_view: function(region_array) { + return { region_data: region_array, viewport: { width: VIEWPORT_WIDTH @@ -414,6 +419,55 @@ define([ }, region_layout: { intron_width: 10, + exon_width: VIEWPORT_WIDTH + + }, + variant_layout: { + variant_width: 5.0 + }, + variant_data_location_field: AMINO_ACID_POSITION_FIELD_NAME, + variant_data_type_field: this.selected_group_by, + variant_data_source_field: "patient_id", + selection_handler: _.bind(this.__seqpeek_selection_handler, this) + }; + }, + + __build_seqpeek_config_for_genomic_view: function(region_array) { + return { + region_data: region_array, + viewport: { + width: VIEWPORT_WIDTH + }, + bar_plot_tracks: { + bar_width: 5.0, + height: VARIANT_TRACK_MAX_HEIGHT, + stem_height: SAMPLE_PLOT_TRACK_STEM_HEIGHT, + color_scheme: this.selected_bar_plot_color_by + }, + sample_plot_tracks: { + height: VARIANT_TRACK_MAX_HEIGHT, + stem_height: 30, + color_scheme: this.selected_color_by + }, + region_track: { + height: REGION_TRACK_HEIGHT + }, + protein_domain_tracks: { + source_key: "dbname", + source_order: ["PFAM", "SMART", "PROFILE"], + color_scheme: { + "PFAM": "lightgray", + "SMART": "darkgray", + "PROFILE": "gray" + } + }, + tick_track: { + height: TICK_TRACK_HEIGHT + }, + region_layout: { + intron_left_padding: 5, + intron_right_padding: 5, + intron_width: 1.0, exon_width: function(region) { return _.max([10, region.end_aa - region.start_aa]); } @@ -425,7 +479,14 @@ define([ variant_data_type_field: this.selected_group_by, variant_data_source_field: "patient_id", selection_handler: _.bind(this.__seqpeek_selection_handler, this) - }); + }; + }, + + __render_tracks: function(mutation_data, region_array, protein_data, seqpeek_tick_track_element, seqpeek_domain_track_element) { + console.debug("seqpeek/view.__render_tracks"); + + var seqpeek_config = this.__build_seqpeek_config(region_array); + var seqpeek = SeqPeekBuilder.create(seqpeek_config); _.each(mutation_data, function(track_obj) { var track_guid = "C" + vq.utils.VisUtils.guid(); @@ -728,8 +789,21 @@ define([ }, __build_regions: function(data, protein_start, protein_end) { + if (CURRENT_MODE == DISPLAY_MODES.PROTEIN) { + return this.__build_regions_protein(data, protein_start, protein_end); + } + else { + return this.__build_regions_genomic(data, protein_start, protein_end); + } + }, + + __build_regions_protein: function(data, protein_start, protein_end) { + return [ { "type": "exon", "start": 0, "end": protein_end } ]; + }, + + __build_regions_genomic: function(data, protein_start, protein_end) { var itercount = 0; - console.log("__br"); + data.sort(function(x, y) { return (parseInt(x["chromosome_position"]) - parseInt(y["chromosome_position"])); }); @@ -739,14 +813,12 @@ define([ var has_uniprot = _.has(data_point, "uniprot_id"); if (memo.last_has_uniprot === null) { - //console.log(itercount, "-"); memo.current_array.push(data_point); memo.last_has_uniprot = has_uniprot; return memo; } if (has_uniprot != memo.last_has_uniprot) { - //console.log(itercount, "|"); memo.split_array.push({ coding: memo.last_has_uniprot, data: _.clone(memo.current_array) @@ -755,7 +827,6 @@ define([ memo.current_array = [data_point]; } else { - //console.log(itercount, "*"); memo.current_array.push(data_point); } @@ -806,8 +877,9 @@ define([ return region_info.data; }, - __preprocess_data: function() { - _.each(this.model["mutations"]["by_tumor_type"], function(data, tumor_type) { + __preprocess_data_and_render: function() { + _.each(this.model["mutations"]["by_tumor_type"], function(model, tumor_type) { + var data = model.toJSON()["items"]; if (_.isArray(data)) { _.each(data, function(d) { d[COORDINATE_FIELD_NAME] = parseInt(d[COORDINATE_FIELD_NAME]); @@ -818,32 +890,40 @@ define([ }); } }, this); + + this.__load_protein_domains(); }, __filter_data: function(data_by_tumor_type) { console.debug("seqpeek/view.__filter_data:" + this.selected_gene); - var lowercase_gene = this.selected_gene.toLowerCase(); var filtered = {}; // Filter out rows that do not have the amino acid position field present, // as drawing variants based on chromosome coordinates is not currently supported. _.each(data_by_tumor_type, function(data, tumor_type) { - if (_.isArray(data)) { - filtered[tumor_type] = _.filter(data, function(item) { - return (_.has(item, "gene") && - _.isEqual(item["gene"].toLowerCase(), lowercase_gene) && - _.has(item, UNIPROT_FIELD_NAME)); - }, this); - } else { - if (_.has(data, "gene") && _.isEqual(data["gene"], lowercase_gene)) { - filtered[tumor_type] = data; - } + if (CURRENT_MODE == DISPLAY_MODES.PROTEIN) { + filtered[tumor_type] = this.__filter_mutation_data_for_protein_view(data); + } + else { + filtered[tumor_type] = this.__filter_mutation_data_for_genomic_view(data) } }); return filtered; }, + __filter_mutation_data_for_protein_view: function(data) { + var lowercase_gene = this.selected_gene.toLowerCase(); + + return _.filter(data, function(item) { + return (_.has(item, UNIPROT_FIELD_NAME && _.has(item, "gene") && _.isEqual(item["gene"].toLowerCase(), lowercase_gene))); + }, this); + }, + + __filter_mutation_data_for_genomic_view: function(data) { + return data; + }, + __filter_mutsig_data: function(data_by_tumor_type) { console.debug("seqpeek/view.__filter_mutsig_data:" + this.selected_gene); diff --git a/bower.json b/bower.json index 189b341..d15d32f 100644 --- a/bower.json +++ b/bower.json @@ -22,7 +22,7 @@ "carve": "~0.0.7", "x2js": "*", "cytoscape": "~2.2.1", - "seqpeek": "git://github.com/IlyaLab/SeqPeek.git#ced34636358c9c88e481e03439db756d32ac6669" + "seqpeek": "git://github.com/IlyaLab/SeqPeek.git#8379f5682248e547ee4e511dde7b6208d4c42dcc" }, "resolutions": { "underscore": "~1.5.2", From b0d04b46a71d58be9a1e917ce215fe212e1d0af7 Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Tue, 8 Jul 2014 11:52:12 -0700 Subject: [PATCH 53/62] reconfigured to use datawarehouse model --- app/configurations/atlas.json | 5 +- app/configurations/datamodel.json | 5 - app/scripts/models/gs/by_tumor_type.js | 47 ++++----- app/scripts/views/stacksvis/view.js | 127 +++++++++++++------------ 4 files changed, 90 insertions(+), 94 deletions(-) diff --git a/app/configurations/atlas.json b/app/configurations/atlas.json index 0827922..7d53694 100644 --- a/app/configurations/atlas.json +++ b/app/configurations/atlas.json @@ -87,7 +87,10 @@ "view": "views/stacksvis/view", "label": "Distributions", "datamodels": { - "copy_number": "datamodel/copy_number/copy_number_gistic2", + "copy_number": { + "uri": "datamodel/tcga_datawarehouse", + "url_suffix": "/copy_number_gistic" + }, "q_value": "datamodel/copy_number/copy_number_qvalue", "mutated_samples": { "uri": "datamodel/tcga_datawarehouse", diff --git a/app/configurations/datamodel.json b/app/configurations/datamodel.json index b51060b..7999ca7 100644 --- a/app/configurations/datamodel.json +++ b/app/configurations/datamodel.json @@ -149,11 +149,6 @@ "copy_number": { "label": "Copy Number Datasets", "catalog": { - "copy_number_gistic2": { - "label": "Copy Number Gistic (13sep)", - "service": "datastores/copy_number/qed_lookups/copyNumber_Gistic2_13sep", - "model": "models/gs/by_tumor_type" - }, "copy_number_qvalue": { "label": "Copy Number Q-Values", "service": "datastores/copy_number/qed_lookups/copy_number_qvalue" diff --git a/app/scripts/models/gs/by_tumor_type.js b/app/scripts/models/gs/by_tumor_type.js index 5a11eca..82c3fa4 100644 --- a/app/scripts/models/gs/by_tumor_type.js +++ b/app/scripts/models/gs/by_tumor_type.js @@ -1,43 +1,32 @@ define(["jquery", "underscore", "backbone"], function ($, _, Backbone) { return Backbone.Model.extend({ - - initialize: function (options) { - _.extend(this, options); + initialize: function(attributes, options) { + this.set(this.parse(attributes)); }, parse: function (data) { - this.set("items", data.items); + var items = data["items"]; + this.set("items", items); - if (_.isEmpty(data.items)) { - return { "ROWS": [], "COLUMNS": [], "DATA": [] }; - } + if (_.isEmpty(items)) return { "ROWS": [], "COLUMNS": [], "DATA": [] }; - var itemsByTumorType = _.groupBy(data.items, "cancer"); - var dataByTumorType = {}; - _.each(itemsByTumorType, function (items, tumor_type) { - if (_.isEmpty(data.items)) { - dataByTumorType[tumor_type] = { "ROWS": [], "COLUMNS": [], "DATA": [] }; - } else { - var ROWS = _.pluck(items, "gene"); - var COLUMNS = _.pluck(items[0].values, "id"); - var coldict = {}; - _.each(COLUMNS, function (col, idx) { - coldict[col] = idx; - }); + var ROWS = _.pluck(items, "gene"); + var COLUMNS = _.keys(_.first(items)["values"]); + var coldict = {}; + _.each(COLUMNS, function (col, idx) { + coldict[col] = idx; + }); - var DATA = _.map(items, function (data_item) { - var row_array = []; - _.each(data_item.values, function (value_obj) { - row_array[coldict[value_obj.id]] = value_obj.v; - }); - return row_array; - }); - dataByTumorType[tumor_type] = { "ROWS": ROWS, "COLUMNS": COLUMNS, "DATA": DATA }; - } + var DATA = _.map(items, function (data_item) { + var row_array = []; + _.each(data_item["values"], function (value, id) { + row_array[coldict[id]] = value; + }); + return row_array; }); - return { "BY_TUMOR_TYPE": dataByTumorType }; + return { "ROWS": ROWS, "COLUMNS": COLUMNS, "DATA": DATA }; } }); }); diff --git a/app/scripts/views/stacksvis/view.js b/app/scripts/views/stacksvis/view.js index 0446754..d1726bf 100644 --- a/app/scripts/views/stacksvis/view.js +++ b/app/scripts/views/stacksvis/view.js @@ -1,7 +1,7 @@ -define(["jquery", "underscore", "backbone", "stacksvis", +define(["jquery", "underscore", "backbone", "stacksvis", "models/gs/by_tumor_type", "hbs!templates/stacksvis/container", "hbs!templates/gs/q_values_ampdel", "colorbrewer", "d3"], - function ($, _, Backbone, StacksVis, Tpl, QValueTpl) { + function ($, _, Backbone, StacksVis, TransformModel, Tpl, QValueTpl) { return Backbone.View.extend({ "events": { @@ -14,9 +14,13 @@ define(["jquery", "underscore", "backbone", "stacksvis", }, "initialize": function () { - this.options["models"]["copy_number"].on("load", this.__render_copy_number, this); this.options["models"]["q_value"].on("load", this.__render_q_value, this); + _.each(this.options["models"]["copy_number"]["by_tumor_type"], function(model, tumor_type) { + model.on("load", function() { + this.__render_copy_number(tumor_type, new TransformModel(model.toJSON())); + }, this); + }, this); _.each(this.options["models"]["mutated_samples"]["by_tumor_type"], function(model, tumor_type) { model.on("load", function() { this.__render_mutated_samples(tumor_type, model); @@ -64,66 +68,67 @@ define(["jquery", "underscore", "backbone", "stacksvis", }, this); }, - __render_copy_number: function () { - this.rowLabels = _.map(this.options.genes, function (g) { - return g.toLowerCase(); // TODO: not good - }); + __render_copy_number: function (tumor_type, model) { + this.rowLabels = this.options.genes; - _.each(this.options.models["copy_number"].get("BY_TUMOR_TYPE"), function (ttModel, tumor_type) { - if (_.isEmpty(ttModel.ROWS)) return; - if (_.isEmpty(ttModel.COLUMNS)) return; - if (_.isEmpty(ttModel.DATA)) return; + var ROWS = model.get("ROWS"); + if (_.isEmpty(ROWS)) return; - var columns_by_cluster = this.__column_model(ttModel); - var data = {}; - var cbscale = colorbrewer.RdYlBu[5]; + var COLUMNS = model.get("COLUMNS"); + if (_.isEmpty(COLUMNS)) return; - var gene_row_items = {}; - _.each(this.rowLabels, function (rowLabel) { - var $statsEl = this.$el.find(".stats-" + tumor_type.toUpperCase() + "-" + rowLabel.toUpperCase()).show(); + var DATA = model.get("DATA"); + if (_.isEmpty(DATA)) return; - gene_row_items[rowLabel] = $statsEl.find(".stats-hm").selector; + var columns_by_cluster = this.__column_model(model); + var data = {}; + var cbscale = colorbrewer.RdYlBu[5]; - var row_idx = ttModel.ROWS.indexOf(rowLabel); - if (row_idx < 0) return; + var gene_row_items = {}; + _.each(this.rowLabels, function (rowLabel) { + var $statsEl = this.$el.find(".stats-" + tumor_type.toUpperCase() + "-" + rowLabel.toUpperCase()).show(); + + gene_row_items[rowLabel] = $statsEl.find(".stats-hm").selector; - _.each(ttModel.DATA[row_idx], function (cell, cellIdx) { - if (_.isString(cell.orig)) cell.orig = cell.orig.trim(); - var columnLabel = ttModel.COLUMNS[cellIdx].trim(); - if (!data[columnLabel]) data[columnLabel] = {}; - data[columnLabel][rowLabel] = { - "value": cell.value, - "row": rowLabel, - "colorscale": cbscale[cell.value], - "label": columnLabel + "\n" + rowLabel + "\n" + cell.orig - }; - }, this); - - var counts = _.countBy(ttModel.DATA[row_idx], "value"); - var totals = ttModel.DATA[row_idx].length; - var lookupPercentage = function (idx) { - var count = counts[idx]; - if (count) return (100 * count / totals).toFixed(1) + "%"; - return ""; + var row_idx = ROWS.indexOf(rowLabel); + if (row_idx < 0) return; + + _.each(DATA[row_idx], function (cell, cellIdx) { + if (_.isString(cell.orig)) cell.orig = cell.orig.trim(); + var columnLabel = COLUMNS[cellIdx].trim(); + if (!data[columnLabel]) data[columnLabel] = {}; + data[columnLabel][rowLabel] = { + "value": cell.value, + "row": rowLabel, + "colorscale": cbscale[cell.value], + "label": columnLabel + "\n" + rowLabel + "\n" + cell.orig }; - $statsEl.find(".stats-samples").html(totals); - $statsEl.find(".stats-0").html(lookupPercentage("0")); - $statsEl.find(".stats-1").html(lookupPercentage("1")); - $statsEl.find(".stats-2").html(lookupPercentage("2")); - $statsEl.find(".stats-3").html(lookupPercentage("3")); - $statsEl.find(".stats-4").html(lookupPercentage("4")); }, this); - var vis = new StacksVis(this.$el, { - "bar_width": 0.75, - "vertical_padding": 1, - "highlight_fill": colorbrewer.RdYlGn[3][2], - "columns_by_cluster": columns_by_cluster, - "row_labels": this.rowLabels, - "row_selectors": gene_row_items - }); - vis.draw({ "data": data }); + var counts = _.countBy(DATA[row_idx], "value"); + var totals = DATA[row_idx].length; + var lookupPercentage = function (idx) { + var count = counts[idx]; + if (count) return (100 * count / totals).toFixed(1) + "%"; + return ""; + }; + $statsEl.find(".stats-samples").html(totals); + $statsEl.find(".stats-0").html(lookupPercentage("0")); + $statsEl.find(".stats-1").html(lookupPercentage("1")); + $statsEl.find(".stats-2").html(lookupPercentage("2")); + $statsEl.find(".stats-3").html(lookupPercentage("3")); + $statsEl.find(".stats-4").html(lookupPercentage("4")); }, this); + + var vis = new StacksVis(this.$el, { + "bar_width": 0.75, + "vertical_padding": 1, + "highlight_fill": colorbrewer.RdYlGn[3][2], + "columns_by_cluster": columns_by_cluster, + "row_labels": this.rowLabels, + "row_selectors": gene_row_items + }); + vis.draw({ "data": data }); }, __render_mutated_samples: function(tumor_type, model) { @@ -134,7 +139,7 @@ define(["jquery", "underscore", "backbone", "stacksvis", }, this); }, - __column_model: function (ttModel) { + __column_model: function (model) { var discretizeFn = function (val) { if (_.isNumber(val)) { if (val < -1.5) return 4; // homozygous loss (less than -1.5) @@ -146,20 +151,24 @@ define(["jquery", "underscore", "backbone", "stacksvis", return val; }; - _.each(ttModel.DATA, function (outer_array, idx) { - ttModel.DATA[idx] = _.map(outer_array, function (x) { + var ROWS = model.get("ROWS") || []; + var COLUMNS = model.get("COLUMNS") || []; + var DATA = model.get("DATA") || []; + + _.each(DATA, function (outer_array, idx) { + DATA[idx] = _.map(outer_array, function (x) { return { "value": discretizeFn(x), "orig": x }; }); }); var unsorted_columns = []; - _.each(ttModel.COLUMNS, function (column_name, col_idx) { + _.each(COLUMNS, function (column_name, col_idx) { var column = { "name": column_name.trim(), "cluster": "_", "values": [] }; _.each(this.rowLabels, function (row_label) { - var row_idx = ttModel.ROWS.indexOf(row_label); + var row_idx = ROWS.indexOf(row_label); if (row_idx < 0) return; - var cell = ttModel.DATA[row_idx][col_idx]; + var cell = DATA[row_idx][col_idx]; if (_.isString(cell.orig)) { cell.orig = cell.orig.trim().toLowerCase(); } From 13823bfca487b6aa48137c10266cee9cee3568b5 Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Tue, 8 Jul 2014 11:58:06 -0700 Subject: [PATCH 54/62] fixed case sensitivity --- app/scripts/views/stacksvis/view.js | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/app/scripts/views/stacksvis/view.js b/app/scripts/views/stacksvis/view.js index d1726bf..be1d5b5 100644 --- a/app/scripts/views/stacksvis/view.js +++ b/app/scripts/views/stacksvis/view.js @@ -32,9 +32,7 @@ define(["jquery", "underscore", "backbone", "stacksvis", "models/gs/by_tumor_typ this.$el.html(Tpl({ "id": Math.floor(Math.random() * 1000), "tumor_types": WebApp.UserPreferences.get("selected_tumor_types"), - "genes": _.map(this.options["genes"], function (g) { - return g.toUpperCase(); - }) + "genes": this.options["genes"] })); this.$el.find(".tooltips").tooltip({ "animation": false, "trigger": "click hover focus", "placement": "right" }); @@ -53,14 +51,14 @@ define(["jquery", "underscore", "backbone", "stacksvis", "models/gs/by_tumor_typ _.each(WebApp.UserPreferences.get("selected_tumor_types"), function (tumor_type_obj) { var items_per_gene = _.groupBy(items_per_tumor_type[tumor_type_obj.id], "gene"); _.each(this.options["genes"], function (gene) { - var gene_items = items_per_gene[gene] || items_per_gene[gene.toLowerCase()]; + var gene_items = items_per_gene[gene]; if (!gene_items || !_.isArray(gene_items)) return; _.each(gene_items, function (gene_item) { gene_item[gene_item["type"]] = true; // binarize for template use }); - var $qvalues = this.$el.find(".stats-" + tumor_type_obj.id.toUpperCase() + "-" + gene.toUpperCase()).show(); + var $qvalues = this.$el.find(".stats-" + tumor_type_obj["id"] + "-" + gene).show(); $qvalues.find(".q-values").html(QValueTpl({"items": _.sortBy(gene_items, "type")})); $qvalues.find(".tooltips").tooltip({ "animation": false, "trigger": "click hover focus", "placement": "top" }); }, this); @@ -86,7 +84,7 @@ define(["jquery", "underscore", "backbone", "stacksvis", "models/gs/by_tumor_typ var gene_row_items = {}; _.each(this.rowLabels, function (rowLabel) { - var $statsEl = this.$el.find(".stats-" + tumor_type.toUpperCase() + "-" + rowLabel.toUpperCase()).show(); + var $statsEl = this.$el.find(".stats-" + tumor_type + "-" + rowLabel).show(); gene_row_items[rowLabel] = $statsEl.find(".stats-hm").selector; @@ -98,9 +96,9 @@ define(["jquery", "underscore", "backbone", "stacksvis", "models/gs/by_tumor_typ var columnLabel = COLUMNS[cellIdx].trim(); if (!data[columnLabel]) data[columnLabel] = {}; data[columnLabel][rowLabel] = { - "value": cell.value, + "value": cell["value"], "row": rowLabel, - "colorscale": cbscale[cell.value], + "colorscale": cbscale[cell["value"]], "label": columnLabel + "\n" + rowLabel + "\n" + cell.orig }; }, this); @@ -134,7 +132,7 @@ define(["jquery", "underscore", "backbone", "stacksvis", "models/gs/by_tumor_typ __render_mutated_samples: function(tumor_type, model) { _.each(model.get("items"), function(item) { var gene = item["gene"]; - var $mutEl = this.$el.find(".stats-" + tumor_type.toUpperCase() + "-" + gene.toUpperCase()).show(); + var $mutEl = this.$el.find(".stats-" + tumor_type + "-" + gene).show(); $mutEl.find(".stats-mutations").html(item["numberOf"]); }, this); }, @@ -169,10 +167,8 @@ define(["jquery", "underscore", "backbone", "stacksvis", "models/gs/by_tumor_typ if (row_idx < 0) return; var cell = DATA[row_idx][col_idx]; - if (_.isString(cell.orig)) { - cell.orig = cell.orig.trim().toLowerCase(); - } - column.values.push(cell.value); + if (_.isString(cell["orig"])) cell["orig"] = cell["orig"].trim(); + column["values"].push(cell["value"]); }, this); unsorted_columns.push(column); }, this); From 2f4cad98ba26f6308975e878be6cafb6fe9441ef Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Tue, 8 Jul 2014 12:04:34 -0700 Subject: [PATCH 55/62] refactored function --- app/scripts/views/stacksvis/view.js | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/app/scripts/views/stacksvis/view.js b/app/scripts/views/stacksvis/view.js index be1d5b5..98615c7 100644 --- a/app/scripts/views/stacksvis/view.js +++ b/app/scripts/views/stacksvis/view.js @@ -138,26 +138,15 @@ define(["jquery", "underscore", "backbone", "stacksvis", "models/gs/by_tumor_typ }, __column_model: function (model) { - var discretizeFn = function (val) { - if (_.isNumber(val)) { - if (val < -1.5) return 4; // homozygous loss (less than -1.5) - if (val < -0.5) return 3; // heterozygous loss (between -0.5 and -1.4999) - if (val < 0.5) return 2; // diploid (between 0.5 and -0.4999) - if (val < 1.5) return 1; // gain (between 1.5 and 0.49999) - return 0; // amplification // greater than 1.5 - } - return val; - }; - var ROWS = model.get("ROWS") || []; var COLUMNS = model.get("COLUMNS") || []; var DATA = model.get("DATA") || []; _.each(DATA, function (outer_array, idx) { DATA[idx] = _.map(outer_array, function (x) { - return { "value": discretizeFn(x), "orig": x }; - }); - }); + return { "value": this.__discretize(x), "orig": x }; + }, this); + }, this); var unsorted_columns = []; _.each(COLUMNS, function (column_name, col_idx) { @@ -185,6 +174,17 @@ define(["jquery", "underscore", "backbone", "stacksvis", "models/gs/by_tumor_typ }); return columns_by_cluster; + }, + + __discretize: function (val) { + if (_.isNumber(val)) { + if (val < -1.5) return 4; // homozygous loss (less than -1.5) + if (val < -0.5) return 3; // heterozygous loss (between -0.5 and -1.4999) + if (val < 0.5) return 2; // diploid (between 0.5 and -0.4999) + if (val < 1.5) return 1; // gain (between 1.5 and 0.49999) + return 0; // amplification // greater than 1.5 + } + return val; } }); }); From e4161dce091ee75ad207090119d0b79d48080147 Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Tue, 8 Jul 2014 12:12:18 -0700 Subject: [PATCH 56/62] null check --- app/index.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/index.html b/app/index.html index 5268de1..af1af51 100644 --- a/app/index.html +++ b/app/index.html @@ -65,7 +65,10 @@
    Browser Display Dimensions
    var increment = function () { progress += 10; try { - document.getElementById("progressbar").style.width = progress + "%"; + var progressbarEl = document.getElementById("progressbar"); + if (!progressbarEl) return; + + progressbarEl.style.width = progress + "%"; if (progress < 100) setTimeout(increment, 300); } catch (e) { console.error(e); From b89b3e4665472a6419d8859bffae963f8f33c052 Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Tue, 8 Jul 2014 12:28:37 -0700 Subject: [PATCH 57/62] fixed color by mappings --- app/scripts/views/fmx_distributions/view.js | 38 ++++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/app/scripts/views/fmx_distributions/view.js b/app/scripts/views/fmx_distributions/view.js index a69dfbb..ead3df3 100644 --- a/app/scripts/views/fmx_distributions/view.js +++ b/app/scripts/views/fmx_distributions/view.js @@ -222,23 +222,8 @@ define(["jquery", "underscore", "backbone", __aggregate: function(tumor_type, model) { console.debug("fmx-dist.__aggregate(" + tumor_type + ")"); _.each(model.get("items"), function(item) { - var unid_or_id = item["unid"] || item["id"]; - var a_f_by_id = this.aggregate_features_by_id[unid_or_id]; - if (!a_f_by_id) a_f_by_id = this.aggregate_features_by_id[unid_or_id] = {}; - - if (_.has(a_f_by_id, tumor_type)) { - var existing = a_f_by_id[tumor_type]; - var overlap_values = _.extend({}, existing["values"], item["values"]); - _.each(_.keys(overlap_values), function(key) { - var value = overlap_values[key]; - if (_.isEqual(value, "NA")) value = item["values"][key]; - if (_.isEqual(value, "NA")) value = existing["values"][key]; - overlap_values[key] = value; - }); - a_f_by_id[tumor_type] = _.extend({}, existing, item, { "values": overlap_values }); - } else { - a_f_by_id[tumor_type] = item; - } + this.__aggregate_features(item["unid"], tumor_type, item); + this.__aggregate_features(item["id"], tumor_type, item); var omit_values = _.omit(_.extend({}, item), "values"); this.feature_definitions_by_id[item["unid"]] = omit_values; @@ -246,6 +231,25 @@ define(["jquery", "underscore", "backbone", }, this); }, + __aggregate_features: function(unid_or_id, tumor_type, item) { + var a_f_by_id = this.aggregate_features_by_id[unid_or_id]; + if (!a_f_by_id) a_f_by_id = this.aggregate_features_by_id[unid_or_id] = {}; + + if (_.has(a_f_by_id, tumor_type)) { + var existing = a_f_by_id[tumor_type]; + var overlap_values = _.extend({}, existing["values"], item["values"]); + _.each(_.keys(overlap_values), function(key) { + var value = overlap_values[key]; + if (_.isEqual(value, "NA")) value = item["values"][key]; + if (_.isEqual(value, "NA")) value = existing["values"][key]; + overlap_values[key] = value; + }); + a_f_by_id[tumor_type] = _.extend({}, existing, item, { "values": overlap_values }); + } else { + a_f_by_id[tumor_type] = item; + } + }, + __aggregate_sample_types: function(tumor_type) { if (this.sample_types_lookup[tumor_type]) return; From 745e8083ed26a5412570da4ad7d10e6f5aaae675 Mon Sep 17 00:00:00 2001 From: kleinone Date: Tue, 8 Jul 2014 17:52:04 -0700 Subject: [PATCH 58/62] Fixes for genomic mode region generation. Added button for switching between protein and genomic view. Updated SeqPeek dependency. --- .../templates/seqpeek/mutations_map.hbs | 3 + app/scripts/views/seqpeek/view.js | 208 ++++++++++-------- bower.json | 2 +- 3 files changed, 126 insertions(+), 87 deletions(-) diff --git a/app/scripts/templates/seqpeek/mutations_map.hbs b/app/scripts/templates/seqpeek/mutations_map.hbs index 62f9c6f..d28d5ac 100644 --- a/app/scripts/templates/seqpeek/mutations_map.hbs +++ b/app/scripts/templates/seqpeek/mutations_map.hbs @@ -51,5 +51,8 @@
    • +
  • + +
    • diff --git a/app/scripts/views/seqpeek/view.js b/app/scripts/views/seqpeek/view.js index e06df4a..060ae8e 100644 --- a/app/scripts/views/seqpeek/view.js +++ b/app/scripts/views/seqpeek/view.js @@ -41,7 +41,7 @@ define([ var DNA_CHANGE_FIELD_NAME = "dna_change"; var UNIPROT_FIELD_NAME = "uniprot_id"; - var GROUP_BY_CATEGORIES = { + var GROUP_BY_CATEGORIES_FOR_PROTEIN_VIEW = { "Mutation Type": TYPE_FIELD_NAME, "DNA Change": DNA_CHANGE_FIELD_NAME, "Protein Change": function(data_row) { @@ -49,7 +49,13 @@ define([ } }; - var CURRENT_MODE = DISPLAY_MODES.ALL; + var GROUP_BY_CATEGORIES_FOR_GENOMIC_VIEW = { + "Mutation Type": TYPE_FIELD_NAME, + "DNA Change": DNA_CHANGE_FIELD_NAME, + "Protein Change": function(data_row) { + return data_row[AMINO_ACID_MUTATION_FIELD_NAME] + "-" + data_row[AMINO_ACID_WILDTYPE_FIELD_NAME]; + } + }; var MUTATION_TYPE_COLOR_MAP = { Nonsense_Mutation: "red", @@ -102,7 +108,8 @@ define([ "click .dropdown-menu.group_by_selector a": function(e) { var group_by = $(e.target).data("id"); - this.selected_group_by = GROUP_BY_CATEGORIES[group_by]; + + this.selected_group_by = this.__get_current_group_by(group_by); this.selected_bar_plot_color_by = COLOR_BY_CATEGORIES_FOR_BAR_PLOT[group_by]; console.debug("seqpeek/group-by-selector:" + group_by); @@ -143,6 +150,17 @@ define([ this.__render(); }, + "click .btn.seqpeek-toggle-genomic": function(e) { + if (this.current_view_mode == DISPLAY_MODES.PROTEIN) { + this.current_view_mode = DISPLAY_MODES.ALL; + } + else { + this.current_view_mode = DISPLAY_MODES.PROTEIN; + } + + this.__preprocess_data_and_render(); + }, + "click .add-new-list": function() { this.__store_sample_list(); } @@ -151,13 +169,15 @@ define([ initialize: function () { this.model = this.options["models"]; - this.selected_group_by = GROUP_BY_CATEGORIES["Mutation Type"]; + this.selected_group_by = this.__get_current_group_by("Mutation Type"); this.selected_color_by = COLOR_BY_CATEGORIES["Mutation Type"]; this.selected_bar_plot_color_by = COLOR_BY_CATEGORIES_FOR_BAR_PLOT["Mutation Type"]; this.sample_track_type = "sample_plot"; this.sample_track_type_user_setting = null; + this.current_view_mode = DISPLAY_MODES.PROTEIN; + this.selected_patient_ids = []; this.samplelists = WebApp.getItemSets(); @@ -172,6 +192,15 @@ define([ this.samplelists.on("remove", this.__update_stored_samplelists, this); }, + __get_current_group_by: function(group_by_key) { + if (this.current_view_mode == DISPLAY_MODES.PROTEIN) { + return GROUP_BY_CATEGORIES_FOR_PROTEIN_VIEW[group_by_key]; + } + else { + return GROUP_BY_CATEGORIES_FOR_GENOMIC_VIEW[group_by_key]; + } + }, + __update_gene_dropdown_labels: function(gene_to_uniprot_mapping) { _.each(this.genes, function(gene_label) { var $el = this.$el.find(".seqpeek-gene-selector a[data-id=" + gene_label + "]"); @@ -207,7 +236,7 @@ define([ "selected_gene": this.selected_gene, "genes": this.genes, "selected_group_by": "Mutation Type", - "group_by_categories": _.keys(GROUP_BY_CATEGORIES), + "group_by_categories": _.keys(GROUP_BY_CATEGORIES_FOR_PROTEIN_VIEW), "color_by_categories": _.keys(COLOR_BY_CATEGORIES) })); @@ -245,7 +274,7 @@ define([ this.$(".mutations_map_table").html(""); - var mutations = this.__parse_mutations(); + var mutations = this.__filter_data(); var mutsig_ranks = this.__filter_mutsig_data(this.__parse_mutsig()); @@ -323,7 +352,7 @@ define([ var protein_data = this.found_protein_domains[uniprot_id]; var all_mutations = []; - _.each(this.__parse_mutations(), function(mutation_array, tumor_type) { + _.each(this.__filter_data(), function(mutation_array, tumor_type) { Array.prototype.push.apply(all_mutations, mutation_array); }); @@ -335,7 +364,8 @@ define([ seqpeek_data.push({ variants: variants, - tumor_type: tumor_type + tumor_type: tumor_type, + is_summary_track: false }); }, this); @@ -354,6 +384,7 @@ define([ percentOf: "NA" }})); + _.each(seqpeek_data, function(track_obj) { track_obj.target_element = _.first(this.$("#seqpeek-row-" + track_obj.tumor_type)) }, this); @@ -377,7 +408,7 @@ define([ }, __build_seqpeek_config: function(region_array) { - if (CURRENT_MODE == DISPLAY_MODES.PROTEIN) { + if (this.current_view_mode == DISPLAY_MODES.PROTEIN) { return this.__build_seqpeek_config_for_protein_view(region_array); } else { @@ -465,9 +496,7 @@ define([ height: TICK_TRACK_HEIGHT }, region_layout: { - intron_left_padding: 5, - intron_right_padding: 5, - intron_width: 1.0, + intron_width: 50.0, exon_width: function(region) { return _.max([10, region.end_aa - region.start_aa]); } @@ -517,12 +546,10 @@ define([ guid: track_guid, hovercard_content: { "Protein location": function(d) { - if (d["type"] == "exon") { - return d["start_aa"] + " - " + d["end_aa"]; - } - else { - return d["start"] + " - " + d["end"]; - } + return d["start_aa"] + " - " + d["end_aa"]; + }, + "Genomic coordinates": function(d) { + return d["start"] + " - " + d["end"]; }, "Protein length": function () { return protein_data["length"]; @@ -556,49 +583,51 @@ define([ seqpeek.addTickTrackToElement(tick_track_g); - var protein_domain_track_guid = "C" + vq.utils.VisUtils.guid(); - var protein_domain_track_g = d3.select(seqpeek_domain_track_element) - .append("svg") + if (this.current_view_mode == DISPLAY_MODES.PROTEIN) { + var protein_domain_track_guid = "C" + vq.utils.VisUtils.guid(); + var protein_domain_track_g = d3.select(seqpeek_domain_track_element) + .append("svg") .attr("width", TRACK_SVG_WIDTH) .attr("height", PROTEIN_DOMAIN_TRACK_HEIGHT) .attr("id", protein_domain_track_guid) .style("pointer-events", "none") - .append("svg:g") + .append("svg:g") .call(this.__set_track_g_position); - seqpeek.addProteinDomainTrackToElement(protein_data["matches"], protein_domain_track_g, { - guid: protein_domain_track_guid, - hovercard_content: { - "DB": function(d) { - return d.dbname; - }, - "EVD": function(d) { - return d.evd; - }, - "ID": function(d) { - return d.id; - }, - "Name": function(d) { - return d.name; - }, - "Status": function(d) { - return d.status; + seqpeek.addProteinDomainTrackToElement(protein_data["matches"], protein_domain_track_g, { + guid: protein_domain_track_guid, + hovercard_content: { + "DB": function (d) { + return d.dbname; + }, + "EVD": function (d) { + return d.evd; + }, + "ID": function (d) { + return d.id; + }, + "Name": function (d) { + return d.name; + }, + "Status": function (d) { + return d.status; + }, + "LOC": function (d) { + return d.start + " - " + d.end; + } }, - "LOC": function(d) { - return d.start + " - " + d.end; - } - }, - hovercard_links: { - "InterPro Domain Entry": { - label: 'InterPro', - url: '/', - href: function(param) { - var ipr_id = param["ipr"]["id"]; - return "http://www.ebi.ac.uk/interpro/entry/" + ipr_id; + hovercard_links: { + "InterPro Domain Entry": { + label: 'InterPro', + url: '/', + href: function (param) { + var ipr_id = param["ipr"]["id"]; + return "http://www.ebi.ac.uk/interpro/entry/" + ipr_id; + } } } - } - }); + }); + } seqpeek.createInstances(); @@ -767,7 +796,7 @@ define([ return d["uniprot_id"]; } } - }); + }, track_obj.is_summary_track); } else { return seqpeek_builder.addBarPlotTrackWithArrayData(track_obj.variants, track_target_svg, { @@ -784,12 +813,12 @@ define([ } }, max_samples_in_location: this.maximum_samples_in_location - }); + }, track_obj.is_summary_track); } }, __build_regions: function(data, protein_start, protein_end) { - if (CURRENT_MODE == DISPLAY_MODES.PROTEIN) { + if (this.current_view_mode == DISPLAY_MODES.PROTEIN) { return this.__build_regions_protein(data, protein_start, protein_end); } else { @@ -840,36 +869,46 @@ define([ last_has_uniprot: null }); - var region_info = _.reduce(split.split_array, function(memo, split_item) { + var get_first_start_coord = function(item) { + return parseInt(_.first(item["data"])["chromosome_position"]); + }; + + var region_info = _.reduce(split.split_array, function(memo, split_item, index, data_array) { var data = split_item.data; var first = _.first(data); var last = _.last(data); - var start_aa; - var end_aa; + var region; - var start_coord = parseInt(first["chromosome_position"]); - var end_coord = parseInt(last["chromosome_position"]); + var start_coord = get_first_start_coord(split_item); - var region = { - type: split_item.coding ? "exon": "noncoding", - start: start_coord, - end: end_coord, - start_aa: start_aa, - end_aa: end_aa - }; + var end_coord = parseInt(last["chromosome_position"]); if (split_item.coding) { - region.start_aa = parseInt(first["amino_acid_position"]); - region.end_aa = parseInt(last["amino_acid_position"]); - + region = { + type: "exon", + start: start_coord, + end: end_coord, + start_aa: parseInt(first["amino_acid_position"]), + end_aa: parseInt(last["amino_acid_position"]) + }; + } + else { + region = { + type: "noncoding", + start: index == 0 ? start_coord : memo.previous_end_coord + 1, + end: index == (data_array.length - 1) ? end_coord : (get_first_start_coord(data_array[index+1]) - 0) + }; } + memo.previous_end_coord = end_coord; + memo.data.push(region); return memo; }, { + previous_end_coord: null, data: [], x_position: 0 }); @@ -891,7 +930,12 @@ define([ } }, this); - this.__load_protein_domains(); + if (this.current_view_mode == DISPLAY_MODES.PROTEIN) { + this.__load_protein_domains(); + } + else { + this.__render(); + } }, __filter_data: function(data_by_tumor_type) { @@ -901,14 +945,15 @@ define([ // Filter out rows that do not have the amino acid position field present, // as drawing variants based on chromosome coordinates is not currently supported. - _.each(data_by_tumor_type, function(data, tumor_type) { - if (CURRENT_MODE == DISPLAY_MODES.PROTEIN) { + _.each(this.model["mutations"]["by_tumor_type"], function(model, tumor_type) { + var data = model.get("items"); + if (this.current_view_mode == DISPLAY_MODES.PROTEIN) { filtered[tumor_type] = this.__filter_mutation_data_for_protein_view(data); } else { filtered[tumor_type] = this.__filter_mutation_data_for_genomic_view(data) } - }); + }, this); return filtered; }, @@ -916,7 +961,7 @@ define([ var lowercase_gene = this.selected_gene.toLowerCase(); return _.filter(data, function(item) { - return (_.has(item, UNIPROT_FIELD_NAME && _.has(item, "gene") && _.isEqual(item["gene"].toLowerCase(), lowercase_gene))); + return (_.has(item, UNIPROT_FIELD_NAME) && _.has(item, "gene") && _.isEqual(item["gene"].toLowerCase(), lowercase_gene)); }, this); }, @@ -954,20 +999,11 @@ define([ return { variants: all_variants, tumor_type: "COMBINED", - track_type: "bar_plot" + track_type: "bar_plot", + is_summary_track: true }; }, - __parse_mutations: function () { - console.debug("seqpeek/view.__parse_mutations"); - - var data = {}; - _.each(this.model["mutations"]["by_tumor_type"], function(model, tumor_type) { - data[tumor_type] = model.get("items"); - }, this); - return data; - }, - __parse_mutsig: function () { console.debug("seqpeek/view.__parse_mutsig"); return _.reduce(this.model["mutsig"].get("items"), function (memo, feature) { diff --git a/bower.json b/bower.json index d15d32f..8bce952 100644 --- a/bower.json +++ b/bower.json @@ -22,7 +22,7 @@ "carve": "~0.0.7", "x2js": "*", "cytoscape": "~2.2.1", - "seqpeek": "git://github.com/IlyaLab/SeqPeek.git#8379f5682248e547ee4e511dde7b6208d4c42dcc" + "seqpeek": "git://github.com/IlyaLab/SeqPeek.git#9241a8c955c83f28335b311075ddfa54e5bc57c0" }, "resolutions": { "underscore": "~1.5.2", From 427b256aa6e2bfd3a983bde7cfd235cfbcf7cc0f Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Wed, 9 Jul 2014 12:24:13 -0700 Subject: [PATCH 59/62] added mutsig import scripts --- db/scripts/datawarehouse_import.py | 8 +++- db/scripts/mutsigrankings_insert.py | 66 +++++++++++++++++++++++++++++ db/scripts/mutsigrankings_top20.js | 22 ++++++++++ 3 files changed, 95 insertions(+), 1 deletion(-) create mode 100755 db/scripts/mutsigrankings_insert.py create mode 100644 db/scripts/mutsigrankings_top20.js diff --git a/db/scripts/datawarehouse_import.py b/db/scripts/datawarehouse_import.py index 211ade4..91331ff 100644 --- a/db/scripts/datawarehouse_import.py +++ b/db/scripts/datawarehouse_import.py @@ -40,7 +40,8 @@ "collections": { "feature_matrix": "/path/to/local/fmx/file", "mutation_summary": "/path/to/local/mut_sum/file", - "copy_number_gistic": "/path/to/local/cn_gistic/file" + "copy_number_gistic": "/path/to/local/cn_gistic/file", + "mutsig_rankings": "/path/to/local/mutsig_rankings/sig_genes.txt" }, "annotations": { "CNVR": "/path/to/local/fmx/annotations/cnvr_file", @@ -125,6 +126,11 @@ def process_import(config_json): im["file"] = im_collections["copy_number_gistic"] execute_python("copynumbergistic_insert.py", im) + if "mutsig_rankings" in im_collections: + im["file"] = im_collections["mutsig_rankings"] + execute_python("mutsigrankings_insert.py", im) + execute_javascript("mutsigrankings_top20.js", im) + if "annotations" in im: im_clone = copy(im) im_clone_annot = im_clone["annotations"] diff --git a/db/scripts/mutsigrankings_insert.py b/db/scripts/mutsigrankings_insert.py new file mode 100755 index 0000000..2e9ccb0 --- /dev/null +++ b/db/scripts/mutsigrankings_insert.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +# this file parses the mutsig output from Firehose, rank is derived from order in file + +import argparse +import csv +import os +import pymongo +import logging + +from utilities import configure_logging + +# example data: +# gene Nnon Nsil Nflank nnon npat nsite nsil nflank nnei fMLE p score time q +# CDKN1A 47190 15860 0 18 18 17 0 0 20 1.478287e+00 3.663736e-15 9.868049e+01 1.285760e-01 4.559103e-11 +# TP53 122850 35880 0 75 64 50 1 0 4 1.600534e+00 4.996004e-15 1.827995e+02 2.969270e-01 4.559103e-11 +# RB1 370240 97630 0 19 17 17 0 0 20 6.902772e-01 1.854072e-14 8.167202e+01 1.402310e-01 1.038462e-10 +# ARID1A 580710 171080 0 38 32 36 2 0 2 1.146468e+00 2.275957e-14 1.301110e+02 1.304470e-01 1.038462e-10 +# MLL2 1376830 455130 0 40 36 40 5 0 20 1.078227e+00 9.714451e-14 1.320917e+02 2.426840e-01 3.545969e-10 +# KDM6A 402090 110110 0 32 31 26 2 0 1 1.366207e+00 2.076339e-12 1.326493e+02 1.375940e-01 6.315877e-09 +# ELF3 115570 31720 0 15 11 14 0 0 20 1.089406e+00 2.276290e-10 5.431453e+01 1.243460e-01 5.934939e-07 + +def extract_rows(file_path): + with open(file_path, "rb") as csvfile: + csvreader = csv.reader(csvfile, delimiter="\t") + headers = csvreader.next() + + rank = 0 + for line in csvreader: + rank += 1 + row_obj = { "rank": rank } + for idx in range(0, len(headers)): + header = headers[idx] + value = line[idx] + if rank % 100 == 0: logging.info("line[%s]:%s:%s:%s" % (rank, idx, header, value)) + row_obj[header] = value + yield row_obj + +def main(): + parser = argparse.ArgumentParser(description="Utility to import TCGA Firehose MutSig to MongoDB") + parser.add_argument("--host", required=True, help="MongoDB host name") + parser.add_argument("--port", required=True, type=int, help="MongoDB port") + parser.add_argument("--db", required=True, help="Database name") + parser.add_argument("--f", required=True, help="Path to mutsig sig-genes file") + parser.add_argument("--loglevel", default="INFO", help="Logging Level") + args = parser.parse_args() + + configure_logging(args.loglevel.upper()) + + logging.info("import file: %s" % args.f) + logging.info("uploading to %s:%s/%s" % (args.host, args.port, args.db)) + + conn = pymongo.Connection(args.host, args.port) + collection = conn[args.db]["mutsig_rankings"] + + count = 0 + for row in extract_rows(args.f): + collection.insert(row) + count += 1 + + logging.info("inserted count=%s" % count) + conn.close() + +if __name__ == "__main__": + main() + diff --git a/db/scripts/mutsigrankings_top20.js b/db/scripts/mutsigrankings_top20.js new file mode 100644 index 0000000..e885ca0 --- /dev/null +++ b/db/scripts/mutsigrankings_top20.js @@ -0,0 +1,22 @@ +/* +- Extracts top20 ranked genes from "mutsig_rankings" collection into GLOBAL lookup collection + +Usage: + + mongo --host=$HOST $DB_NAME mutsigrankings_top20.js + */ + +var db_name = db["_name"]; +var pretty_print = function(msg) { + var dtFmt = (new Date()).toLocaleFormat("%Y-%m-%d %H:%M:%S,000"); + print("[INFO] " + dtFmt + " - mutsigrankings_top20(" + db_name + ") - " + msg); +}; + +pretty_print("START"); + +db["mutsig_rankings_top20"].drop(); +db["mutsig_rankings"].find({ "rank": { "$lte": 20 } }).forEach(function(doc) { + db["mutsig_rankings_top20"].insert(doc); +}); + +pretty_print("COMPLETED"); From 5076aeffd9721e759b162844724bb3bc43d540e5 Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Wed, 9 Jul 2014 12:31:21 -0700 Subject: [PATCH 60/62] minor fixes --- db/scripts/mutsigrankings_insert.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db/scripts/mutsigrankings_insert.py b/db/scripts/mutsigrankings_insert.py index 2e9ccb0..f602c35 100755 --- a/db/scripts/mutsigrankings_insert.py +++ b/db/scripts/mutsigrankings_insert.py @@ -32,7 +32,7 @@ def extract_rows(file_path): for idx in range(0, len(headers)): header = headers[idx] value = line[idx] - if rank % 100 == 0: logging.info("line[%s]:%s:%s:%s" % (rank, idx, header, value)) + if rank % 100 == 0: logging.debug("line[%s]:%s:%s:%s" % (rank, idx, header, value)) row_obj[header] = value yield row_obj @@ -52,6 +52,7 @@ def main(): conn = pymongo.Connection(args.host, args.port) collection = conn[args.db]["mutsig_rankings"] + collection.drop() count = 0 for row in extract_rows(args.f): From 2f99f88b558e46b97faf09cec15e407305b8141f Mon Sep 17 00:00:00 2001 From: Hector Rovira Date: Wed, 9 Jul 2014 12:32:59 -0700 Subject: [PATCH 61/62] change count and logging --- db/scripts/mutsigrankings_insert.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/scripts/mutsigrankings_insert.py b/db/scripts/mutsigrankings_insert.py index f602c35..8800ba9 100755 --- a/db/scripts/mutsigrankings_insert.py +++ b/db/scripts/mutsigrankings_insert.py @@ -52,14 +52,14 @@ def main(): conn = pymongo.Connection(args.host, args.port) collection = conn[args.db]["mutsig_rankings"] + + logging.info("dropping collection") collection.drop() - count = 0 for row in extract_rows(args.f): collection.insert(row) - count += 1 - logging.info("inserted count=%s" % count) + logging.info("inserted count=%s" % collection.count()) conn.close() if __name__ == "__main__": From ef63a55e369c75359429bcc4cb9b48bb076104f7 Mon Sep 17 00:00:00 2001 From: Michael Miller Date: Thu, 10 Jul 2014 09:11:06 -0700 Subject: [PATCH 62/62] added extract_medians.py, a script to extract the median values for gene expression features. updated the comments of ffn_update_fm_with_ffn_lookup.js to remove 'odd' characters from copy and paste --- .gitignore | 1 + db/scripts/extract_medians.py | 58 +++++++++++++++++++++ db/scripts/ffn_update_fm_with_ffn_lookup.js | 26 ++++----- 3 files changed, 72 insertions(+), 13 deletions(-) create mode 100644 db/scripts/extract_medians.py diff --git a/.gitignore b/.gitignore index a9e5258..dfb90b9 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ dist .sass-cache app/components app/data +db/sh python app/bower_components npm-debug.log diff --git a/db/scripts/extract_medians.py b/db/scripts/extract_medians.py new file mode 100644 index 0000000..b0be704 --- /dev/null +++ b/db/scripts/extract_medians.py @@ -0,0 +1,58 @@ +''' +Created on Jun 10, 2014 + +@author: michael +''' +import argparse +import logging +import pymongo + +from utilities import configure_logging + +def outputValues(outfile, dbs, gene2tumor2median): + logging.info('\n\tstarting outputValues()') + with open(outfile, 'w') as out: + out.write('\t' + '\t'.join(dbs) + '\n') + genes = gene2tumor2median.keys() + genes.sort() + for gene in genes: + line = gene + for db in dbs: + line += '\t' + str(gene2tumor2median[gene].get(db, 'NA')) + out.write(line + '\n') + logging.info('\n\tfinished outputValues()') + +def processTumors(args): + gene2tumor2median = {} + conn = pymongo.Connection(args.host, args.port) + logging.info('\n\tstarting processTumors()') + for db in args.dbs: + logging.info('\n\t\treading %s' % (db)) + collection = conn[db]["feature_matrix"] + docs = collection.find({"source": args.platform}) + for doc in docs: + tumor2median = gene2tumor2median.setdefault(doc["label"], {}) + tumor2median[db] = doc["statistics"]["numeric"]["median"] + + conn.close() + logging.info('\n\tfinished processTumors()') + return gene2tumor2median + +def main(): + parser = argparse.ArgumentParser(description="Utility to produce a table of genes vs. tumor types w/ median values") + parser.add_argument("--host", required=True, help="MongoDB host name") + parser.add_argument("--port", required=True, type=int, help="MongoDB port") + parser.add_argument("dbs", nargs='+', help="Database names") + parser.add_argument("--out", required=True, help="where to output the table") + parser.add_argument("--platform", default="GEXP", help="feature type to find the median from") + parser.add_argument("--loglevel", default="INFO", help="Logging Level") + args = parser.parse_args() + configure_logging(args.loglevel.upper()) + + logging.info('starting extract medians:\n\t%s' % (args)) + gene2tumor2median = processTumors(args); + outputValues(args.out, args.dbs, gene2tumor2median) + logging.info('finished extract medians') + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/db/scripts/ffn_update_fm_with_ffn_lookup.js b/db/scripts/ffn_update_fm_with_ffn_lookup.js index 9d98074..af39598 100644 --- a/db/scripts/ffn_update_fm_with_ffn_lookup.js +++ b/db/scripts/ffn_update_fm_with_ffn_lookup.js @@ -8,19 +8,19 @@ function replaceUnderscore(str) { var debugCLINCounts = [0, 0, 0, 0, 0, 0]; var twoReplace = "$3: $1 vs $2"; var oneReplace = "$2: $1"; -// In some cases, the string(s) in front of the “|” are not unique/specific enough and we will need to include the feature name after the “|” as follows -// C:SAMP:I(A|X) ⇒ f(X) A -// C:SAMP:I(A,B|X) ⇒ f(X) A vs B +// In some cases, the string(s) in front of the '|' are not unique/specific enough and we will need to include the feature name after the '|' as follows +// C:SAMP:I(A|X) ==> f(X) A +// C:SAMP:I(A,B|X) ==> f(X) A vs B var checks = [ -// cases where we have G1,G2,.. drop the “G” (redundant with the word “grade”) +// cases where we have G1,G2,.. drop the 'G' (redundant with the word grade) // B:CLIN:I(G1|neoplasm_histologic_grade)::::: => neoplasm histologic grade 1 [/I\(G([0-9]+),G([0-9]+)\|(.+)\)/m, twoReplace], // B:CLIN:I(G1,G2|neoplasm_histologic_grade)::::: => neoplasm histologic grade 1 vs 2 [/I\(G([0-9]+)\|(.+)\)/m, oneReplace], -// cases where we have C1,C2,... (these are cluster #s) -- drop the “C” -// C:SAMP:I(C1,C2|X) ⇒ f(X) 1 vs 2 +// cases where we have C1,C2,... (these are cluster #s) -- drop the 'C' +// C:SAMP:I(C1,C2|X) ==> f(X) 1 vs 2 [/I\(C([0-9]+),C([0-9]+)\|(.+)\)/m, twoReplace], -// C:SAMP:I(C1|X) ⇒ f(X) 1 +// C:SAMP:I(C1|X) ==> f(X) 1 [/I\(C([0-9]+)\|(.+)\)/m, oneReplace], // pathologic_T, pathologic_N, pathologic_M, WHO_class [/I\((.+),(.+)\|(.+)\)/m, twoReplace], @@ -94,19 +94,19 @@ function getLength(start, end) { var debugCNVRCounts = [0, 0, 0, 0, 0, 0]; // Gistic and GisticArm features // N:CNVR:Xq:chrX:60600000:155270560::SKCM-All_Lymph_Node_GisticArm_d -// ⇒ Xq (Gistic Arm) +// ==> Xq (Gistic Arm) // N:CNVR:Xq28:chrX:150021680:150280252::SKCM-All_Regional_Metastases_Gistic_ROI_r_amp -// ⇒ Xq28 Amplification (Gistic, continuous) +// ==> Xq28 Amplification (Gistic, continuous) // N:CNVR:Xq28:chrX:150021680:150280252::SKCM-All_Regional_Metastases_Gistic_ROI_d_del -// ⇒ Xq28 Deletion (Gistic, discrete) +// ==> Xq28 Deletion (Gistic, discrete) // Resegmented features // N:CNVR:Xq22.2:chrX:60600000:155270560 -// ⇒ chrX:60,600,000-155,270,560 (95MB, Xq22.2) +// ==> chrX:60,600,000-155,270,560 (95MB, Xq22.2) // N:CNVR:8q24:chr8:138862000:140900999:: -// ⇒ chr8:138,862,000-140,900,999 (2MB, 8q24) +// ==> chr8:138,862,000-140,900,999 (2MB, 8q24) // N:CNVR:SRGAP2:chr1:158887000:158888999:: -// ⇒ chr1:158,887,000-158,888,999 (2kb, SRGAP2) +// ==> chr1:158,887,000-158,888,999 (2kb, SRGAP2) var getCNVRLabel = function(doc) { if (doc["code"]) { if (-1 < doc["code"].indexOf('GisticArm')) {