diff --git a/.gitignore b/.gitignore
index a9e5258..dfb90b9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,6 +20,7 @@ dist
.sass-cache
app/components
app/data
+db/sh
python
app/bower_components
npm-debug.log
diff --git a/app/configurations/atlas.json b/app/configurations/atlas.json
index 6c4ab7e..7d53694 100644
--- a/app/configurations/atlas.json
+++ b/app/configurations/atlas.json
@@ -1,6 +1,7 @@
{
"default_genelist": ["BRCA1", "RAD51", "TP53", "KRAS", "CC2D1A"],
"all_tags_url": "svc/datastores/FFN/LOOKUPS/all_tags",
+ "all_clinical_url": "svc/datastores/FFN/LOOKUPS/all_clinical",
"maps": [
{
"id": "mutations_combo",
@@ -23,14 +24,6 @@
"url_suffix": "/mutation_summary"
},
"mutsig": "datamodel/mutations/mutsig_rankings",
- "features": {
- "uri": "datamodel/tcga_datawarehouse",
- "url_suffix": "/feature_matrix",
- "base_query": {
- "source": "GNAB",
- "label": "y_n_somatic"
- }
- },
"mutated_samples": {
"uri": "datamodel/tcga_datawarehouse",
"url_suffix": "/mutated_samples"
@@ -75,6 +68,9 @@
"url_suffix": "/feature_matrix",
"query_clinical_variables": true
}
+ },
+ "feature_sources_order": {
+ "GEXP": 1
}
}
]
@@ -91,7 +87,10 @@
"view": "views/stacksvis/view",
"label": "Distributions",
"datamodels": {
- "copy_number": "datamodel/copy_number/copy_number_gistic2",
+ "copy_number": {
+ "uri": "datamodel/tcga_datawarehouse",
+ "url_suffix": "/copy_number_gistic"
+ },
"q_value": "datamodel/copy_number/copy_number_qvalue",
"mutated_samples": {
"uri": "datamodel/tcga_datawarehouse",
diff --git a/app/configurations/datamodel.json b/app/configurations/datamodel.json
index 97ec131..7999ca7 100644
--- a/app/configurations/datamodel.json
+++ b/app/configurations/datamodel.json
@@ -141,7 +141,7 @@
"STAD-20140123": {
"tumor_type": "STAD",
"service": "datastores/dev_ffn_20140520/STAD",
- "description": "This dataset was prepared from TCGA feature matrices aggregated at ISB",
+ "description": "This dataset was prepared from TCGA feature matrices aggregated at ISB. STAD gene expression levels were available from RNAseq data as RPKM only, whereas levels for other tumor types were available in terms of RSEM (June 2014). To approximate RSEM values, STAD values were linearly transformed using the equation 4.550681+ STAD *1.271340. The coefficient were obtained from regressing median gene expression levels for COAD against those for STAD. COAD was selected over other tumor types due to anatomical proximity and the relative concordance of the distributions of the COAD and STAD medians.",
"label": "STAD dataset for January 2014"
}
}
@@ -149,11 +149,6 @@
"copy_number": {
"label": "Copy Number Datasets",
"catalog": {
- "copy_number_gistic2": {
- "label": "Copy Number Gistic (13sep)",
- "service": "datastores/copy_number/qed_lookups/copyNumber_Gistic2_13sep",
- "model": "models/gs/by_tumor_type"
- },
"copy_number_qvalue": {
"label": "Copy Number Q-Values",
"service": "datastores/copy_number/qed_lookups/copy_number_qvalue"
diff --git a/app/configurations/lookups.json b/app/configurations/lookups.json
index 5d1ff06..096441e 100644
--- a/app/configurations/lookups.json
+++ b/app/configurations/lookups.json
@@ -8,9 +8,6 @@
"NA": "Other"
}
},
- "clinical_variables": {
- "url": "svc/datastores/FFN/LOOKUPS/clinical_variables"
- },
"chromosomes": {
"url": "svc/data/lookups/chromosomes",
"model": "models/annotations"
diff --git a/app/index.html b/app/index.html
index 5268de1..af1af51 100644
--- a/app/index.html
+++ b/app/index.html
@@ -65,7 +65,10 @@
Browser Display Dimensions
var increment = function () {
progress += 10;
try {
- document.getElementById("progressbar").style.width = progress + "%";
+ var progressbarEl = document.getElementById("progressbar");
+ if (!progressbarEl) return;
+
+ progressbarEl.style.width = progress + "%";
if (progress < 100) setTimeout(increment, 300);
} catch (e) {
console.error(e);
diff --git a/app/scripts/models/gs/by_tumor_type.js b/app/scripts/models/gs/by_tumor_type.js
index 5a11eca..82c3fa4 100644
--- a/app/scripts/models/gs/by_tumor_type.js
+++ b/app/scripts/models/gs/by_tumor_type.js
@@ -1,43 +1,32 @@
define(["jquery", "underscore", "backbone"],
function ($, _, Backbone) {
return Backbone.Model.extend({
-
- initialize: function (options) {
- _.extend(this, options);
+ initialize: function(attributes, options) {
+ this.set(this.parse(attributes));
},
parse: function (data) {
- this.set("items", data.items);
+ var items = data["items"];
+ this.set("items", items);
- if (_.isEmpty(data.items)) {
- return { "ROWS": [], "COLUMNS": [], "DATA": [] };
- }
+ if (_.isEmpty(items)) return { "ROWS": [], "COLUMNS": [], "DATA": [] };
- var itemsByTumorType = _.groupBy(data.items, "cancer");
- var dataByTumorType = {};
- _.each(itemsByTumorType, function (items, tumor_type) {
- if (_.isEmpty(data.items)) {
- dataByTumorType[tumor_type] = { "ROWS": [], "COLUMNS": [], "DATA": [] };
- } else {
- var ROWS = _.pluck(items, "gene");
- var COLUMNS = _.pluck(items[0].values, "id");
- var coldict = {};
- _.each(COLUMNS, function (col, idx) {
- coldict[col] = idx;
- });
+ var ROWS = _.pluck(items, "gene");
+ var COLUMNS = _.keys(_.first(items)["values"]);
+ var coldict = {};
+ _.each(COLUMNS, function (col, idx) {
+ coldict[col] = idx;
+ });
- var DATA = _.map(items, function (data_item) {
- var row_array = [];
- _.each(data_item.values, function (value_obj) {
- row_array[coldict[value_obj.id]] = value_obj.v;
- });
- return row_array;
- });
- dataByTumorType[tumor_type] = { "ROWS": ROWS, "COLUMNS": COLUMNS, "DATA": DATA };
- }
+ var DATA = _.map(items, function (data_item) {
+ var row_array = [];
+ _.each(data_item["values"], function (value, id) {
+ row_array[coldict[id]] = value;
+ });
+ return row_array;
});
- return { "BY_TUMOR_TYPE": dataByTumorType };
+ return { "ROWS": ROWS, "COLUMNS": COLUMNS, "DATA": DATA };
}
});
});
diff --git a/app/scripts/models/gs/item_set.js b/app/scripts/models/gs/item_set.js
new file mode 100644
index 0000000..e1d0ffc
--- /dev/null
+++ b/app/scripts/models/gs/item_set.js
@@ -0,0 +1,82 @@
+define([
+ "jquery",
+ "underscore",
+ "backbone"
+],
+function ($, _, Backbone
+) {
+ var URL = "svc/collections/samplelists";
+
+ return Backbone.Collection.extend({
+ "url": URL,
+
+ "model": Backbone.Model.extend({
+ idAttribute: "_id"
+ }),
+
+ initialize: function() {
+ this.on("add", this.__add_handler, this);
+ this.on("remove", this.__remove_handler, this);
+ this.on("change", this.__change_handler, this);
+ },
+
+ __add_handler: function() {
+
+ },
+
+ __remove_handler: function(model, collection, index) {
+ Backbone.sync("delete", new Backbone.Model({}), {
+ "url": URL + "/" + model["id"], "success": this.__refresh
+ });
+ },
+
+ __change_handler: function() {
+
+ },
+
+ __createModelForSync: function(model) {
+ var data = _.omit(model.toJSON(), "uri", "id", "_id");
+ return new Backbone.Model(data);
+ },
+
+ updateSampleList: function(model_id, sample_list) {
+ var model = this.get(model_id);
+ var successFn = _.bind(function() {
+ model.trigger("change");
+ }, this);
+
+ model.set({
+ samples: sample_list
+ });
+
+ this.sync("update", this.__createModelForSync(model), {
+ url: URL + "/" + model["id"],
+ success: successFn,
+ context: this
+ });
+ },
+
+ updateSampleListByUnion: function(model_id, sample_list) {
+ var sample_id_set = this.get(model_id).get("samples");
+ Array.prototype.push.apply(sample_id_set, sample_list);
+ this.updateSampleList(model_id, _.unique(sample_id_set));
+ },
+
+ addSampleList: function(label, sample_id_array) {
+ var sample_list_model = new this.model({
+ "label": label,
+ "samples": sample_id_array
+ });
+
+ var successFn = _.bind(function(response, status) {
+ this.add(_.extend(sample_list_model, {"id": response["id"]}));
+ }, this);
+
+ this.sync("create", sample_list_model, {
+ url: URL,
+ success: successFn,
+ context: this
+ });
+ }
+ });
+});
diff --git a/app/scripts/templates/datamodel_collector/container.hbs b/app/scripts/templates/datamodel_collector/container.hbs
index 42283dc..6c029b9 100644
--- a/app/scripts/templates/datamodel_collector/container.hbs
+++ b/app/scripts/templates/datamodel_collector/container.hbs
@@ -21,7 +21,7 @@
Web Service API
{{label}}
- {{description}}
+ {{{description}}}
{{#if publications}}
Related Publications
@@ -49,7 +49,7 @@
Web Service API
{{label}}
- {{description}}
+ {{{description}}}
{{#if publications}}
Related Publications
diff --git a/app/scripts/templates/fmx_distributions/container.hbs b/app/scripts/templates/fmx_distributions/container.hbs
index 95662f3..750c43e 100755
--- a/app/scripts/templates/fmx_distributions/container.hbs
+++ b/app/scripts/templates/fmx_distributions/container.hbs
@@ -68,7 +68,7 @@
Color By
-
diff --git a/app/scripts/templates/seqpeek/mutations_map_table.hbs b/app/scripts/templates/seqpeek/mutations_map_table.hbs
index 3583579..dfeb872 100644
--- a/app/scripts/templates/seqpeek/mutations_map_table.hbs
+++ b/app/scripts/templates/seqpeek/mutations_map_table.hbs
@@ -4,7 +4,7 @@
Tumor Type
MutSig Rank
Samples (#)
-
+
@@ -17,7 +17,7 @@
{{/each}}
- ALL
+ COMBINED
{{total.samples}}
diff --git a/app/scripts/templates/seqpeek/sample_list_dropdown_caption.hbs b/app/scripts/templates/seqpeek/sample_list_dropdown_caption.hbs
new file mode 100644
index 0000000..c1e48b1
--- /dev/null
+++ b/app/scripts/templates/seqpeek/sample_list_dropdown_caption.hbs
@@ -0,0 +1 @@
+ {{caption}}
\ No newline at end of file
diff --git a/app/scripts/templates/seqpeek/sample_list_operations.hbs b/app/scripts/templates/seqpeek/sample_list_operations.hbs
new file mode 100644
index 0000000..4ab1c1a
--- /dev/null
+++ b/app/scripts/templates/seqpeek/sample_list_operations.hbs
@@ -0,0 +1,32 @@
+{{#if samplelists.length}}
+ Sample Lists
+
+
+
+
+ {{#each samplelists}}
+ {{label}}
+ {{/each}}
+
+
+
+
+ {{#each samplelists}}
+
+
{{number_samples}} Samples
+
+
+
+
+ Add samples to '{{label}}'
+
+
+ {{/each}}
+
+
+
+{{else}}
+ Sample Lists
+ No sample lists stored
+{{/if}}
diff --git a/app/scripts/views/clinvarlist/control.js b/app/scripts/views/clinvarlist/control.js
index cf78251..9bf3c55 100644
--- a/app/scripts/views/clinvarlist/control.js
+++ b/app/scripts/views/clinvarlist/control.js
@@ -78,7 +78,7 @@ define(["jquery", "underscore", "backbone",
var itemizer = this.itemizers[gl_model.get("_id")] = new Itemizer({"el": $glList.find(".clinvar-selector"), "model": gl_model });
itemizer.render();
- var typeahead = new TypeAhead({ "el": $glList.find(".clin-typeahead") });
+ var typeahead = new TypeAhead({ "el": $glList.find(".clin-typeahead"), "url": this.options["url"] });
typeahead.render();
typeahead.on("typed", function(clin) {
var cv_from_model = _.map(gl_model.get("clinical_variables"), function(g) {return g;});
diff --git a/app/scripts/views/clinvarlist/typeahead.js b/app/scripts/views/clinvarlist/typeahead.js
index 5f2bcc4..4fb2f3a 100644
--- a/app/scripts/views/clinvarlist/typeahead.js
+++ b/app/scripts/views/clinvarlist/typeahead.js
@@ -1,34 +1,45 @@
define([ "jquery", "underscore", "backbone" ],
function ($, _, Backbone) {
return Backbone.View.extend({
+ "clinical_variables_by_label": {},
+
initialize: function() {
_.bindAll(this, "render", "__typed");
},
render: function() {
- var clinical_variables = _.extend({}, WebApp.Lookups.get("clinical_variables").get("items"));
- if (_.isEmpty(clinical_variables)) return this;
-
- this.clinical_variables_by_label = _.groupBy(clinical_variables, "label");
+ var url = this.options["url"] + "/search/label";
+ var label_bucket = this.clinical_variables_by_label;
this.$el.typeahead({
- source: function (q, p) {
- p(_.compact(_.flatten(_.map(q.toLowerCase().split(" "), function (qi) {
- return _.map(clinical_variables, function (item) {
- if (item["label"].toLowerCase().indexOf(qi) >= 0) return item["label"];
- return null;
- });
- }))));
- },
+ "source": function (q, p) {
+ $.ajax({
+ "url": url,
+ "data": { "term": q },
+ "traditional": true,
+ "dataType": "json",
+ "success": function (json) {
+ if (json && json["items"]) {
+ var matching_labels = _.compact(_.map(json["items"], function(item) {
+ label_bucket[item["label"]] = item;
+ return item["label"];
+ }));
- updater: this.__typed
+ if (!_.isEmpty(matching_labels)) p(matching_labels.sort());
+ }
+ }
+ });
+ },
+ "items": 16,
+ "minLength": 2,
+ "updater": this.__typed
});
return this;
},
__typed: function(clin) {
- this.trigger("typed", _.first(this.clinical_variables_by_label[clin]));
+ this.trigger("typed", this.clinical_variables_by_label[clin]);
return "";
}
});
diff --git a/app/scripts/views/fmx_distributions/view.js b/app/scripts/views/fmx_distributions/view.js
index cc52bee..ead3df3 100644
--- a/app/scripts/views/fmx_distributions/view.js
+++ b/app/scripts/views/fmx_distributions/view.js
@@ -76,7 +76,7 @@ define(["jquery", "underscore", "backbone",
this.carveVis.highlight(null).render();
} else {
var selected_item = $(e.target).data("id");
- if (selected_item) {
+ if (!_.isUndefined(selected_item)) {
console.debug("fmx-dist.highlight:" + selected_item);
this.$(".legend_items").find(".active").removeClass("active");
LI.addClass("active");
@@ -121,7 +121,9 @@ define(["jquery", "underscore", "backbone",
this.$el.html(Tpl({
"id": this.id,
"genes": this.options["genes"],
- "clinical_variables": this.options["clinical_variables"],
+ "clinical_variables": _.filter(this.options["clinical_variables"], function(cv) {
+ return (_.has(cv, "id") && !_.isEqual(cv["id"].substring(0,2), "N:"));
+ }),
"tumor_types": this.options["tumor_types"],
"sample_types": this.sample_types,
"selected_genes": this.selected_genes
@@ -209,26 +211,45 @@ define(["jquery", "underscore", "backbone",
this.__aggregate(tumor_type, this.model["gene_features"]["by_tumor_type"][tumor_type]);
this.__render_fLabel_selectors("x");
this.__render_fLabel_selectors("y");
+ this.__colorBy_variables();
},
__load_fdefs_clinvars: function (tumor_type) {
console.debug("fmx-dist.__load_fdefs_clinvars(" + tumor_type + ")");
- _.each(this.options["clinical_variables"], function(item) {
- this.feature_definitions_by_id[item.id] = _.extend({}, item);
- }, this);
this.__aggregate(tumor_type, this.model["clinical_features"]["by_tumor_type"][tumor_type]);
},
__aggregate: function(tumor_type, model) {
console.debug("fmx-dist.__aggregate(" + tumor_type + ")");
_.each(model.get("items"), function(item) {
- var a_f_by_id = this.aggregate_features_by_id[item.id];
- if (!a_f_by_id) a_f_by_id = this.aggregate_features_by_id[item.id] = {};
- a_f_by_id[tumor_type] = item;
- this.feature_definitions_by_id[item.id] = _.omit(item, "values");
+ this.__aggregate_features(item["unid"], tumor_type, item);
+ this.__aggregate_features(item["id"], tumor_type, item);
+
+ var omit_values = _.omit(_.extend({}, item), "values");
+ this.feature_definitions_by_id[item["unid"]] = omit_values;
+ this.feature_definitions_by_id[item["id"]] = omit_values;
}, this);
},
+ __aggregate_features: function(unid_or_id, tumor_type, item) {
+ var a_f_by_id = this.aggregate_features_by_id[unid_or_id];
+ if (!a_f_by_id) a_f_by_id = this.aggregate_features_by_id[unid_or_id] = {};
+
+ if (_.has(a_f_by_id, tumor_type)) {
+ var existing = a_f_by_id[tumor_type];
+ var overlap_values = _.extend({}, existing["values"], item["values"]);
+ _.each(_.keys(overlap_values), function(key) {
+ var value = overlap_values[key];
+ if (_.isEqual(value, "NA")) value = item["values"][key];
+ if (_.isEqual(value, "NA")) value = existing["values"][key];
+ overlap_values[key] = value;
+ });
+ a_f_by_id[tumor_type] = _.extend({}, existing, item, { "values": overlap_values });
+ } else {
+ a_f_by_id[tumor_type] = item;
+ }
+ },
+
__aggregate_sample_types: function(tumor_type) {
if (this.sample_types_lookup[tumor_type]) return;
@@ -254,6 +275,29 @@ define(["jquery", "underscore", "backbone",
}
},
+ __colorBy_variables: function() {
+ this.$(".color_by_selector").find(".other-variables").remove();
+
+ var features = _.values(this.feature_definitions_by_id);
+ var qualifying_features = _.filter(features, function(feature) {
+ if (!_.has(feature, "source")) return false;
+ if (feature["source"] === "CLIN") return false;
+ if (feature["source"] === "SAMP") return false;
+ if (feature["source"] === "GNAB") return _.isEqual(feature["code"], "code_potential_somatic");
+ return !_.isEqual(feature["type"], "N");
+ });
+ if (_.isEmpty(qualifying_features)) return;
+
+ this.$(".color_by_selector").append("");
+ _.each(_.sortBy(qualifying_features, "label"), function(feature) {
+ this.$(".color_by_selector").append(LineItemTpl({
+ "id": feature["unid"] || feature["id"],
+ "label": feature["label"],
+ "li_class": "other-variables"
+ }));
+ }, this);
+ },
+
__render_fLabel_selectors_genes: function(axis) {
this.$(".selected-gene-" + axis).html(this.selected_genes[axis]);
@@ -266,14 +310,23 @@ define(["jquery", "underscore", "backbone",
var feature_sources = _.map(_.keys(fd_by_gene || {}), function (source) {
var s_uid = uid++ + "-" + axis;
fdefs_uid_by_source[source] = s_uid;
- return { "uid": s_uid, "label": source.toUpperCase(), "item_class": "feature_defs" };
- });
+
+ var order_dict = this.options["feature_sources_order"] || {};
+ var order = order_dict[source.toUpperCase()] || 100;
+ return { "uid": s_uid, "label": source.toUpperCase(), "item_class": "feature_defs", "order": order };
+ }, this);
+
+ feature_sources = _.sortBy(feature_sources, "order");
$feature_selector.append(FeatureDefsTpl({"axis": axis, "feature_sources": feature_sources}));
_.each(fd_by_gene, function (features, source) {
var collapserUL = $feature_selector.find("#tab-pane-" + fdefs_uid_by_source[source]);
- _.each(_.sortBy(features, "label"), function (feature) {
- collapserUL.append(LineItemTpl({ "label": feature["label"], "id": feature["id"], "a_class": "feature-selector-" + axis }));
+ var grouped_by_unid = _.groupBy(features, "unid");
+ if (_.isEmpty(grouped_by_unid)) grouped_by_unid = _.groupBy(features, "id");
+ _.each(grouped_by_unid, function(grouped_features) {
+ var feature = _.first(grouped_features);
+ if (!feature) return;
+ collapserUL.append(LineItemTpl({ "label": feature["label"], "id": feature["unid"] || feature["id"], "a_class": "feature-selector-" + axis }));
});
});
@@ -330,10 +383,12 @@ define(["jquery", "underscore", "backbone",
var Y_feature = this.feature_definitions_by_id[this.selected_features["y"]];
var data = null;
- if (this.selected_tumor_type) {
- data = this.__visdata([this.selected_tumor_type], X_feature.id, Y_feature.id);
- } else {
- data = this.__visdata(this.options["tumor_types"], X_feature.id, Y_feature.id);
+ if (X_feature && Y_feature) {
+ if (this.selected_tumor_type) {
+ data = this.__visdata([this.selected_tumor_type], X_feature, Y_feature);
+ } else {
+ data = this.__visdata(this.options["tumor_types"], X_feature, Y_feature);
+ }
}
if (_.isEmpty(data)) {
this.latest_data = [];
@@ -396,25 +451,31 @@ define(["jquery", "underscore", "backbone",
if (_.isArray(color_by_list) && _.isArray(color_by_colors)) {
if (_.isEqual(color_by_list.length, color_by_colors.length)) {
_.each(color_by_list, function (color_by, idx) {
- this.$(".legend_items").append(LegendTpl({
- "id": color_by,
- "label": color_by,
- "color": color_by_colors[idx]
- }));
+ if (color_by) {
+ this.$(".legend_items").append(LegendTpl({
+ "id": color_by,
+ "label": color_by,
+ "color": color_by_colors[idx]
+ }));
+ }
}, this);
}
}
},
- __visdata: function (tumor_types, X_feature_id, Y_feature_id) {
+ __visdata: function (tumor_types, X_f, Y_f) {
+ var X_feature_id = X_f["unid"] || X_f["id"];
+ var Y_feature_id = Y_f["unid"] || Y_f["id"];
var data = _.map(tumor_types, function (tumor_type) {
var stl = this.sample_types_lookup[tumor_type] || {};
var X_feature_by_tumor_type = this.aggregate_features_by_id[X_feature_id] || {};
- var X_feature = X_feature_by_tumor_type[tumor_type] || {};
+ var X_feature = X_feature_by_tumor_type[tumor_type];
+ if (!X_feature) return null;
var Y_feature_by_tumor_type = this.aggregate_features_by_id[Y_feature_id] || {};
- var Y_feature = Y_feature_by_tumor_type[tumor_type] || {};
+ var Y_feature = Y_feature_by_tumor_type[tumor_type];
+ if (!Y_feature) return null;
var Cby_feature_by_tumor_type = this.aggregate_features_by_id[this.selected_color_by] || {};
var Cby_feature = Cby_feature_by_tumor_type[tumor_type] || {};
diff --git a/app/scripts/views/genes/typeahead.js b/app/scripts/views/genes/typeahead.js
index 71786c6..02def51 100644
--- a/app/scripts/views/genes/typeahead.js
+++ b/app/scripts/views/genes/typeahead.js
@@ -9,7 +9,6 @@ define([ "jquery", "underscore", "backbone" ],
var all_tags_url = this.options["url"] + "/search/tag";
this.$el.typeahead({
"source": function(q, p) {
- if (!q || q.length < 2) return;
$.ajax({
"url": all_tags_url,
"data": { "term": q },
@@ -18,11 +17,13 @@ define([ "jquery", "underscore", "backbone" ],
"success": function (json) {
if (json && json["items"]) {
var matching_tags = _.uniq(_.pluck(json["items"], "tag"));
- if (!_.isEmpty(matching_tags)) p(matching_tags)
+ if (!_.isEmpty(matching_tags)) p(matching_tags.sort());
}
}
});
},
+ "items": 16,
+ "minLength": 2,
"updater": this.__typed
});
return this;
diff --git a/app/scripts/views/gs/atlas.js b/app/scripts/views/gs/atlas.js
index 933397c..f17e1b0 100644
--- a/app/scripts/views/gs/atlas.js
+++ b/app/scripts/views/gs/atlas.js
@@ -5,14 +5,16 @@ define([
"views/gs/atlas_map",
"views/genes/control",
"views/clinvarlist/control",
+ "views/samplelist/control",
"views/gs/tumor_types_control",
"views/datamodel_collector/control",
"views/collected_maps/control",
"views/datasheets/control"
+
],
function ($, _, Backbone, AtlasTpl, MapsListContainerTpl, AtlasMapView,
- GenelistControl, ClinicalListControl, TumorTypesControl, DatamodelCollectorControl, CollectedMapsControl,
- DatasheetsControl) {
+ GenelistControl, ClinicalListControl, SampleListControl, TumorTypesControl, DatamodelCollectorControl,
+ CollectedMapsControl, DatasheetsControl) {
return Backbone.View.extend({
"datasheetsControl": new DatasheetsControl({}),
@@ -56,6 +58,7 @@ define([
this.model.set("atlas_map_views", []);
this.model.on("load", this.__init_genelist_control, this);
this.model.on("load", this.__init_clinicallist_control, this);
+ this.model.on("load", this.__init_samplelist_control, this);
this.model.on("load", this.__init_tumortypes_control, this);
this.model.on("load", this.__init_datamodel_collector, this);
// this.model.on("load", this.__init_collected_maps_control, this);
@@ -89,7 +92,7 @@ define([
},
__init_clinicallist_control: function() {
- this.clinicalListControl = new ClinicalListControl({});
+ this.clinicalListControl = new ClinicalListControl({ "url": this.model.get("all_clinical_url") });
this.clinicalListControl.on("updated", function (ev) {
console.debug("atlas.__init_clinicallist_control:updated:" + JSON.stringify(ev));
if (ev["reorder"]) {
@@ -103,6 +106,21 @@ define([
this.$el.find(".clinvarlist-container").html(this.clinicalListControl.render().el);
},
+ __init_samplelist_control: function() {
+ this.sampleListControl = new SampleListControl({});
+ this.sampleListControl.on("updated", function (ev) {
+ console.debug("atlas.__init_samplelist_control:updated:" + JSON.stringify(ev));
+ if (ev["reorder"]) {
+ console.debug("atlas.__init_samplelist_control:updated:reorder:ignore");
+ return;
+ }
+
+ this.__reload_all_maps();
+ }, this);
+
+ this.$el.find(".samplelist-container").html(this.sampleListControl.render().el);
+ },
+
__init_collected_maps_control: function() {
this.collectedMapsControl = new CollectedMapsControl({});
this.collectedMapsControl.on("selected", function (ev) {
diff --git a/app/scripts/views/samplelist/control.js b/app/scripts/views/samplelist/control.js
new file mode 100644
index 0000000..376ecd5
--- /dev/null
+++ b/app/scripts/views/samplelist/control.js
@@ -0,0 +1,65 @@
+define([
+ "jquery",
+ "underscore",
+ "backbone",
+ "hbs!templates/samplelist/container"
+],
+function ($, _, Backbone,
+ Tpl
+) {
+ return Backbone.View.extend({
+ events: {
+ "click .list-remover": function(e) {
+ var listid = $(e.target).data("id");
+ this.collection.remove(listid);
+ },
+
+ "click .list-update": function(e) {
+ var listid = $(e.target).data("id");
+ var text_content = this.$el.find("#samplelist-contents-" + listid).val();
+ var sample_ids = text_content.match(/\S+/g);
+ this.collection.updateSampleList(listid, sample_ids);
+ },
+
+ "click .list-refresh": function() {
+ _.defer(this.__refresh);
+ },
+
+ "click .add-new-list": function(e) {
+ var list_label = this.$el.find(".new-list-name").val();
+
+ if (list_label.length == 0) {
+ return;
+ }
+
+ this.collection.addSampleList(list_label, []);
+ }
+ },
+
+ initialize: function() {
+ _.bindAll(this, "render");
+
+ this.collection = WebApp.getItemSets();
+ this.collection.on("add remove change", this.render, this);
+ },
+
+ render: function() {
+ var template_data = this.collection.map(function(model) {
+ var samples = model.get("samples");
+ var text_content = samples.join("\n");
+
+ return {
+ "id": model["id"],
+ "label": model.get("label"),
+ "samples": samples,
+ "text": text_content,
+ "number_samples": samples.length
+ };
+ });
+
+ this.$el.html(Tpl({ "samplelists": _.sortBy(template_data, "sort") }));
+
+ return this;
+ }
+ });
+});
diff --git a/app/scripts/views/seqpeek/sample_list_operations_view.js b/app/scripts/views/seqpeek/sample_list_operations_view.js
new file mode 100644
index 0000000..ccb58a1
--- /dev/null
+++ b/app/scripts/views/seqpeek/sample_list_operations_view.js
@@ -0,0 +1,48 @@
+define([
+ "jquery",
+ "underscore",
+ "backbone",
+ "hbs!templates/seqpeek/sample_list_operations"
+],
+function ($, _, Backbone,
+ Tpl
+ ) {
+
+ return Backbone.View.extend({
+ events: {
+ "click .list-union-op": function(e) {
+ var listid = $(e.target).data("id");
+ this.trigger("list:union", this.collection.get(listid));
+ }
+ },
+
+ initialize: function() {
+ _.bindAll(this, "__refresh");
+
+ this.collection.on("add remove change", this.__refresh, this);
+ },
+
+ render: function() {
+ this.__refresh();
+
+ return this;
+ },
+
+ __refresh: function() {
+ var template_data = this.collection.map(function(model) {
+ var samples = model.get("samples");
+ var text_content = samples.join("\n");
+
+ return {
+ "id": model["id"],
+ "label": model.get("label"),
+ "samples": samples,
+ "text": text_content,
+ "number_samples": samples.length
+ };
+ });
+
+ this.$el.html(Tpl({ "samplelists": _.sortBy(template_data, "sort") }));
+ }
+ });
+});
diff --git a/app/scripts/views/seqpeek/view.js b/app/scripts/views/seqpeek/view.js
index fd297c5..060ae8e 100644
--- a/app/scripts/views/seqpeek/view.js
+++ b/app/scripts/views/seqpeek/view.js
@@ -3,24 +3,53 @@ define([
"models/gs/protein_domain_model",
"seqpeek/util/data_adapters",
"seqpeek/builders/builder_for_existing_elements",
+ "seqpeek/util/mini_locator",
+ "views/seqpeek/sample_list_operations_view",
"hbs!templates/seqpeek/mutations_map",
- "hbs!templates/seqpeek/mutations_map_table"
+ "hbs!templates/seqpeek/mutations_map_table",
+ "hbs!templates/seqpeek/sample_list_dropdown_caption"
],
function ($, _, Backbone, d3, vq,
- ProteinDomainModel, SeqPeekDataAdapters, SeqPeekBuilder, MutationsMapTpl, MutationsMapTableTpl) {
+ ProteinDomainModel, SeqPeekDataAdapters, SeqPeekBuilder, SeqPeekMiniLocatorFactory,
+ SampleListOperationsView,
+ MutationsMapTpl, MutationsMapTableTpl,
+ SampleListCaptionTpl
+ ) {
+ var DISPLAY_MODES = {
+ ALL: 1,
+ PROTEIN: 2
+ };
+
+ var MINI_LOCATOR_WIDTH = 400;
+ var MINI_LOCATOR_HEIGHT = 24;
+
+ var Y_AXIS_SCALE_WIDTH = 50;
+
var VARIANT_TRACK_MAX_HEIGHT = 150;
var TICK_TRACK_HEIGHT = 25;
var REGION_TRACK_HEIGHT = 10;
var PROTEIN_DOMAIN_TRACK_HEIGHT = 40;
var VIEWPORT_WIDTH = 1000;
+ var SAMPLE_PLOT_TRACK_STEM_HEIGHT = 30;
+ var TRACK_SVG_WIDTH = VIEWPORT_WIDTH + Y_AXIS_SCALE_WIDTH;
- var POSITION_FIELD_NAME = "amino_acid_position";
+ var AMINO_ACID_POSITION_FIELD_NAME = "amino_acid_position";
+ var COORDINATE_FIELD_NAME = "chromosome_position";
var TYPE_FIELD_NAME = "mutation_type";
var AMINO_ACID_MUTATION_FIELD_NAME = "amino_acid_mutation";
var AMINO_ACID_WILDTYPE_FIELD_NAME = "amino_acid_wildtype";
var DNA_CHANGE_FIELD_NAME = "dna_change";
+ var UNIPROT_FIELD_NAME = "uniprot_id";
- var GROUP_BY_CATEGORIES = {
+ var GROUP_BY_CATEGORIES_FOR_PROTEIN_VIEW = {
+ "Mutation Type": TYPE_FIELD_NAME,
+ "DNA Change": DNA_CHANGE_FIELD_NAME,
+ "Protein Change": function(data_row) {
+ return data_row[AMINO_ACID_MUTATION_FIELD_NAME] + "-" + data_row[AMINO_ACID_WILDTYPE_FIELD_NAME];
+ }
+ };
+
+ var GROUP_BY_CATEGORIES_FOR_GENOMIC_VIEW = {
"Mutation Type": TYPE_FIELD_NAME,
"DNA Change": DNA_CHANGE_FIELD_NAME,
"Protein Change": function(data_row) {
@@ -79,7 +108,8 @@ define([
"click .dropdown-menu.group_by_selector a": function(e) {
var group_by = $(e.target).data("id");
- this.selected_group_by = GROUP_BY_CATEGORIES[group_by];
+
+ this.selected_group_by = this.__get_current_group_by(group_by);
this.selected_bar_plot_color_by = COLOR_BY_CATEGORIES_FOR_BAR_PLOT[group_by];
console.debug("seqpeek/group-by-selector:" + group_by);
@@ -110,10 +140,6 @@ define([
this.__enable_seqpeek_selection();
},
- "click .btn.seqpeek-print-ids": function(e) {
- this.__print_selected_samples();
- },
-
"click .btn.seqpeek-toggle-bars": function(e) {
if (this.sample_track_type_user_setting == "bar_plot") {
this.sample_track_type_user_setting = "sample_plot";
@@ -122,20 +148,70 @@ define([
this.sample_track_type_user_setting = "bar_plot";
}
this.__render();
+ },
+
+ "click .btn.seqpeek-toggle-genomic": function(e) {
+ if (this.current_view_mode == DISPLAY_MODES.PROTEIN) {
+ this.current_view_mode = DISPLAY_MODES.ALL;
+ }
+ else {
+ this.current_view_mode = DISPLAY_MODES.PROTEIN;
+ }
+
+ this.__preprocess_data_and_render();
+ },
+
+ "click .add-new-list": function() {
+ this.__store_sample_list();
}
},
initialize: function () {
this.model = this.options["models"];
- this.selected_group_by = GROUP_BY_CATEGORIES["Mutation Type"];
+ this.selected_group_by = this.__get_current_group_by("Mutation Type");
this.selected_color_by = COLOR_BY_CATEGORIES["Mutation Type"];
this.selected_bar_plot_color_by = COLOR_BY_CATEGORIES_FOR_BAR_PLOT["Mutation Type"];
this.sample_track_type = "sample_plot";
this.sample_track_type_user_setting = null;
+ this.current_view_mode = DISPLAY_MODES.PROTEIN;
+
this.selected_patient_ids = [];
+
+ this.samplelists = WebApp.getItemSets();
+
+ this.sample_list_op_view = new SampleListOperationsView({
+ collection: this.samplelists
+ });
+
+ this.sample_list_op_view.on("list:union", this.__sample_list_union, this);
+
+ this.samplelists.on("add", this.__update_stored_samplelists, this);
+ this.samplelists.on("remove", this.__update_stored_samplelists, this);
+ },
+
+ __get_current_group_by: function(group_by_key) {
+ if (this.current_view_mode == DISPLAY_MODES.PROTEIN) {
+ return GROUP_BY_CATEGORIES_FOR_PROTEIN_VIEW[group_by_key];
+ }
+ else {
+ return GROUP_BY_CATEGORIES_FOR_GENOMIC_VIEW[group_by_key];
+ }
+ },
+
+ __update_gene_dropdown_labels: function(gene_to_uniprot_mapping) {
+ _.each(this.genes, function(gene_label) {
+ var $el = this.$el.find(".seqpeek-gene-selector a[data-id=" + gene_label + "]");
+
+ if (_.has(gene_to_uniprot_mapping, gene_label)) {
+ $el.text(gene_label);
+ }
+ else {
+ $el.text(gene_label + " NO DATA");
+ }
+ }, this);
},
render: function() {
@@ -145,7 +221,7 @@ define([
this.genes = this.options["genes"] || [];
if (!_.isEmpty(this.genes)) this.selected_gene = _.first(this.genes);
- var renderFn = _.after(1 + (2 * this.tumor_types.length), this.__load_protein_domains);
+ var renderFn = _.after(1 + (2 * this.tumor_types.length), this.__preprocess_data_and_render);
this.model["mutsig"].on("load", renderFn, this);
@@ -160,7 +236,7 @@ define([
"selected_gene": this.selected_gene,
"genes": this.genes,
"selected_group_by": "Mutation Type",
- "group_by_categories": _.keys(GROUP_BY_CATEGORIES),
+ "group_by_categories": _.keys(GROUP_BY_CATEGORIES_FOR_PROTEIN_VIEW),
"color_by_categories": _.keys(COLOR_BY_CATEGORIES)
}));
@@ -170,6 +246,26 @@ define([
})
}));
+ this.__update_sample_list_dropdown();
+
+ this.$el.find(".sample-list-operations").html(this.sample_list_op_view.render().el);
+
+ var $sample_list_dropdown = $(this.$el.find("a.sample-list-dropdown"));
+
+ // Manually open and close the sample list dialog dropdown. The dropdown
+ // will not close when clicking outside.
+ $sample_list_dropdown.on("click.dropdown.data-api", function(e) {
+ var parent = $(this.parentNode);
+ var is_open = parent.hasClass("open");
+
+ if (is_open == false) {
+ parent.addClass("open");
+ }
+ else {
+ parent.removeClass("open");
+ }
+ });
+
return this;
},
@@ -178,12 +274,9 @@ define([
this.$(".mutations_map_table").html("");
- var mutations = this.__filter_data(this.__parse_mutations());
- var mutsig_ranks = this.__filter_mutsig_data(this.__parse_mutsig());
+ var mutations = this.__filter_data();
- var formatter = function (value) {
- return parseInt(value) + "%";
- };
+ var mutsig_ranks = this.__filter_mutsig_data(this.__parse_mutsig());
var data_items = _.map(this.tumor_types, function (tumor_type) {
var statistics = {
@@ -242,10 +335,28 @@ define([
var seqpeek_data = [];
+ this.__update_gene_dropdown_labels(this.gene_to_uniprot_mapping);
+
+ if (! _.has(this.gene_to_uniprot_mapping, this.selected_gene)) {
+ this.$(".mutations_map_table").html(MutationsMapTableTpl({
+ "items": data_items,
+ "total": {
+ samples: "No data",
+ percentOf: "NA"
+ }}));
+
+ return;
+ }
+
var uniprot_id = this.gene_to_uniprot_mapping[this.selected_gene];
var protein_data = this.found_protein_domains[uniprot_id];
- var region_data = [ { "type": "exon", "start": 0, "end": protein_data["length"] } ];
+ var all_mutations = [];
+ _.each(this.__filter_data(), function(mutation_array, tumor_type) {
+ Array.prototype.push.apply(all_mutations, mutation_array);
+ });
+
+ var region_data = this.__build_regions(all_mutations, 0, protein_data["length"]);
_.each(this.tumor_types, function (tumor_type) {
var variants = mutations[tumor_type];
@@ -253,7 +364,8 @@ define([
seqpeek_data.push({
variants: variants,
- tumor_type: tumor_type
+ tumor_type: tumor_type,
+ is_summary_track: false
});
}, this);
@@ -272,6 +384,7 @@ define([
percentOf: "NA"
}}));
+
_.each(seqpeek_data, function(track_obj) {
track_obj.target_element = _.first(this.$("#seqpeek-row-" + track_obj.tumor_type))
}, this);
@@ -294,10 +407,17 @@ define([
this.__render_tracks(seqpeek_data, region_data, protein_data, seqpeek_tick_track_element, seqpeek_domain_track_element);
},
- __render_tracks: function(mutation_data, region_array, protein_data, seqpeek_tick_track_element, seqpeek_domain_track_element) {
- console.debug("seqpeek/view.__render_tracks");
+ __build_seqpeek_config: function(region_array) {
+ if (this.current_view_mode == DISPLAY_MODES.PROTEIN) {
+ return this.__build_seqpeek_config_for_protein_view(region_array);
+ }
+ else {
+ return this.__build_seqpeek_config_for_genomic_view(region_array);
+ }
+ },
- var seqpeek = SeqPeekBuilder.create({
+ __build_seqpeek_config_for_protein_view: function(region_array) {
+ return {
region_data: region_array,
viewport: {
width: VIEWPORT_WIDTH
@@ -305,7 +425,7 @@ define([
bar_plot_tracks: {
bar_width: 5.0,
height: VARIANT_TRACK_MAX_HEIGHT,
- stem_height: 30,
+ stem_height: SAMPLE_PLOT_TRACK_STEM_HEIGHT,
color_scheme: this.selected_bar_plot_color_by
},
sample_plot_tracks: {
@@ -329,41 +449,108 @@ define([
height: TICK_TRACK_HEIGHT
},
region_layout: {
- intron_width: 5,
+ intron_width: 10,
exon_width: VIEWPORT_WIDTH
+
},
variant_layout: {
variant_width: 5.0
},
- variant_data_location_field: POSITION_FIELD_NAME,
+ variant_data_location_field: AMINO_ACID_POSITION_FIELD_NAME,
variant_data_type_field: this.selected_group_by,
variant_data_source_field: "patient_id",
selection_handler: _.bind(this.__seqpeek_selection_handler, this)
- });
+ };
+ },
+
+ __build_seqpeek_config_for_genomic_view: function(region_array) {
+ return {
+ region_data: region_array,
+ viewport: {
+ width: VIEWPORT_WIDTH
+ },
+ bar_plot_tracks: {
+ bar_width: 5.0,
+ height: VARIANT_TRACK_MAX_HEIGHT,
+ stem_height: SAMPLE_PLOT_TRACK_STEM_HEIGHT,
+ color_scheme: this.selected_bar_plot_color_by
+ },
+ sample_plot_tracks: {
+ height: VARIANT_TRACK_MAX_HEIGHT,
+ stem_height: 30,
+ color_scheme: this.selected_color_by
+ },
+ region_track: {
+ height: REGION_TRACK_HEIGHT
+ },
+ protein_domain_tracks: {
+ source_key: "dbname",
+ source_order: ["PFAM", "SMART", "PROFILE"],
+ color_scheme: {
+ "PFAM": "lightgray",
+ "SMART": "darkgray",
+ "PROFILE": "gray"
+ }
+ },
+ tick_track: {
+ height: TICK_TRACK_HEIGHT
+ },
+ region_layout: {
+ intron_width: 50.0,
+ exon_width: function(region) {
+ return _.max([10, region.end_aa - region.start_aa]);
+ }
+ },
+ variant_layout: {
+ variant_width: 5.0
+ },
+ variant_data_location_field: COORDINATE_FIELD_NAME,
+ variant_data_type_field: this.selected_group_by,
+ variant_data_source_field: "patient_id",
+ selection_handler: _.bind(this.__seqpeek_selection_handler, this)
+ };
+ },
+
+ __render_tracks: function(mutation_data, region_array, protein_data, seqpeek_tick_track_element, seqpeek_domain_track_element) {
+ console.debug("seqpeek/view.__render_tracks");
+
+ var seqpeek_config = this.__build_seqpeek_config(region_array);
+ var seqpeek = SeqPeekBuilder.create(seqpeek_config);
_.each(mutation_data, function(track_obj) {
var track_guid = "C" + vq.utils.VisUtils.guid();
var track_elements_svg = d3.select(track_obj.target_element)
.append("svg")
- .attr("width", VIEWPORT_WIDTH)
+ .attr("width", TRACK_SVG_WIDTH)
.attr("height", VARIANT_TRACK_MAX_HEIGHT + REGION_TRACK_HEIGHT)
.attr("id", track_guid)
.style("pointer-events", "none");
var sample_plot_track_g = track_elements_svg
.append("g")
- .style("pointer-events", "none");
+ .style("pointer-events", "none")
+ .call(this.__set_track_g_position);
var region_track_g = track_elements_svg
+ .append("g")
+ .style("pointer-events", "none")
+ .call(this.__set_track_g_position)
.append("g")
.style("pointer-events", "none");
track_obj.track_info = this.__add_data_track(track_obj, seqpeek, track_guid, sample_plot_track_g);
track_obj.variant_track_svg = track_elements_svg;
+ track_obj.sample_plot_track_g = sample_plot_track_g;
seqpeek.addRegionScaleTrackToElement(region_track_g, {
guid: track_guid,
hovercard_content: {
+ "Protein location": function(d) {
+ return d["start_aa"] + " - " + d["end_aa"];
+ },
+ "Genomic coordinates": function(d) {
+ return d["start"] + " - " + d["end"];
+ },
"Protein length": function () {
return protein_data["length"];
},
@@ -386,55 +573,61 @@ define([
track_obj.region_track_svg = region_track_g;
}, this);
- var tick_track_svg = d3.select(seqpeek_tick_track_element)
+ var tick_track_g = d3.select(seqpeek_tick_track_element)
.append("svg")
- .attr("width", VIEWPORT_WIDTH)
- .attr("height", TICK_TRACK_HEIGHT)
- .style("pointer-events", "none");
+ .attr("width", TRACK_SVG_WIDTH)
+ .attr("height", TICK_TRACK_HEIGHT)
+ .style("pointer-events", "none")
+ .append("svg:g")
+ .call(this.__set_track_g_position);
- seqpeek.addTickTrackToElement(tick_track_svg);
+ seqpeek.addTickTrackToElement(tick_track_g);
- var protein_domain_track_guid = "C" + vq.utils.VisUtils.guid();
- var protein_domain_track_svg = d3.select(seqpeek_domain_track_element)
- .append("svg")
- .attr("width", VIEWPORT_WIDTH)
- .attr("height", PROTEIN_DOMAIN_TRACK_HEIGHT)
- .attr("id", protein_domain_track_guid)
- .style("pointer-events", "none");
-
- seqpeek.addProteinDomainTrackToElement(protein_data["matches"], protein_domain_track_svg, {
- guid: protein_domain_track_guid,
- hovercard_content: {
- "DB": function(d) {
- return d.dbname;
- },
- "EVD": function(d) {
- return d.evd;
- },
- "ID": function(d) {
- return d.id;
- },
- "Name": function(d) {
- return d.name;
- },
- "Status": function(d) {
- return d.status;
+ if (this.current_view_mode == DISPLAY_MODES.PROTEIN) {
+ var protein_domain_track_guid = "C" + vq.utils.VisUtils.guid();
+ var protein_domain_track_g = d3.select(seqpeek_domain_track_element)
+ .append("svg")
+ .attr("width", TRACK_SVG_WIDTH)
+ .attr("height", PROTEIN_DOMAIN_TRACK_HEIGHT)
+ .attr("id", protein_domain_track_guid)
+ .style("pointer-events", "none")
+ .append("svg:g")
+ .call(this.__set_track_g_position);
+
+ seqpeek.addProteinDomainTrackToElement(protein_data["matches"], protein_domain_track_g, {
+ guid: protein_domain_track_guid,
+ hovercard_content: {
+ "DB": function (d) {
+ return d.dbname;
+ },
+ "EVD": function (d) {
+ return d.evd;
+ },
+ "ID": function (d) {
+ return d.id;
+ },
+ "Name": function (d) {
+ return d.name;
+ },
+ "Status": function (d) {
+ return d.status;
+ },
+ "LOC": function (d) {
+ return d.start + " - " + d.end;
+ }
},
- "LOC": function(d) {
- return d.start + " - " + d.end;
- }
- },
- hovercard_links: {
- "InterPro Domain Entry": {
- label: 'InterPro',
- url: '/',
- href: function(param) {
- var ipr_id = param["ipr"]["id"];
- return "http://www.ebi.ac.uk/interpro/entry/" + ipr_id;
+ hovercard_links: {
+ "InterPro Domain Entry": {
+ label: 'InterPro',
+ url: '/',
+ href: function (param) {
+ var ipr_id = param["ipr"]["id"];
+ return "http://www.ebi.ac.uk/interpro/entry/" + ipr_id;
+ }
}
}
- }
- });
+ });
+ }
seqpeek.createInstances();
@@ -448,18 +641,113 @@ define([
track_obj.variant_track_svg.attr("height", total_track_height);
track_obj.region_track_svg
- .attr("transform", "translate(0," + (variant_track_height) + ")")
- });
+ .attr("transform", "translate(0," + (variant_track_height) + ")");
+
+ this.__render_scales(track_obj.variant_track_svg, total_track_height, track_instance.statistics);
+ }, this);
+ var regions_start_coordinate = seqpeek.getRegionMetadata().start_coordinate;
+ var regions_end_coordinate = seqpeek.getRegionMetadata().end_coordinate;
+
+ var mini_locator_scale = MINI_LOCATOR_WIDTH / seqpeek.getRegionMetadata().total_width;
+ this.__create_mini_locator(seqpeek.getProcessedRegionData(), seqpeek.region_layout, mini_locator_scale, regions_start_coordinate, regions_end_coordinate);
+
+ seqpeek.scrollEventCallback(_.bind(function(d) {
+ var visible_coordinates = d.visible_coordinates;
+ this.mini_locator.render(visible_coordinates[0], visible_coordinates[1]);
+ }, this));
seqpeek.render();
this.seqpeek = seqpeek;
},
+ __create_mini_locator: function(region_data, region_layout, scale, start_coordinate, end_coordinate) {
+ var $mini_locator = this.$el.find(".seqpeek-mini-locator")
+ .attr("width", MINI_LOCATOR_WIDTH)
+ .attr("height", MINI_LOCATOR_HEIGHT);
+
+ this.mini_locator = SeqPeekMiniLocatorFactory.create($mini_locator[0])
+ .data(region_data)
+ .region_layout(region_layout)
+ .scale(scale);
+
+ this.mini_locator.render(start_coordinate, end_coordinate);
+ },
+
+ __set_track_g_position: function(track_selector) {
+ track_selector
+ .attr("transform", "translate(" + Y_AXIS_SCALE_WIDTH + ",0)");
+ },
+
+ __render_scales: function(track_selector, total_track_height, track_statistics) {
+ var right = Y_AXIS_SCALE_WIDTH - 10;
+ var scale_start = -(REGION_TRACK_HEIGHT + SAMPLE_PLOT_TRACK_STEM_HEIGHT);
+
+ var axis = track_selector
+ .append("svg:g")
+ .attr("class", "y-axis")
+ .attr("transform", "translate(0," + total_track_height + ")");
+
+ axis
+ .append("svg:line")
+ .attr("y1", scale_start)
+ .attr("x1", right)
+ .attr("y2", -total_track_height)
+ .attr("x2", right)
+ .style("stroke", "black");
+
+ var domain = [
+ track_statistics.min_samples_in_location,
+ track_statistics.max_samples_in_location
+ ];
+
+ var scale = d3.scale.linear().domain(domain).range([scale_start, -total_track_height]);
+ var ticks = [
+ {
+ text: domain[0],
+ y: scale(domain[0]),
+ text_y: -5
+ },
+ {
+ text: domain[1],
+ y: scale(domain[1]) + 1,
+ text_y: +13
+ }
+ ];
+
+ var tick_g = axis
+ .selectAll(".tick")
+ .data(ticks)
+ .enter()
+ .append("svg:g")
+ .attr("class", "y-axis-tick")
+ .attr("transform", function(d) {
+ return "translate(0," + d.y + ")";
+ });
+
+ tick_g
+ .append("svg:line")
+ .attr("y1", 0.0)
+ .attr("y2", 0.0)
+ .attr("x1", right - 10)
+ .attr("x2", right)
+ .style("stroke", "black");
+ tick_g
+ .append("svg:text")
+ .attr("x", right - 15)
+ .attr("y", function(d) {
+ return d.text_y;
+ })
+ .text(function(d) {
+ return d.text;
+ })
+ .style("text-anchor", "end");
+ },
+
__find_maximum_samples_in_location: function(mutation_data) {
var track_maximums = [];
_.each(mutation_data, function(track_obj) {
- var grouped_data = SeqPeekDataAdapters.group_by_location(track_obj.variants, this.selected_group_by, POSITION_FIELD_NAME);
+ var grouped_data = SeqPeekDataAdapters.group_by_location(track_obj.variants, this.selected_group_by, COORDINATE_FIELD_NAME);
SeqPeekDataAdapters.apply_statistics(grouped_data, function() {return 'all';});
var max_number_of_samples_in_position = d3.max(grouped_data, function(data_by_location) {
@@ -477,12 +765,17 @@ define([
__add_data_track: function(track_obj, seqpeek_builder, track_guid, track_target_svg) {
var track_type = track_obj.track_type || this.sample_track_type_user_setting;
+ var variants = track_obj.variants;
+ variants.sort(function(x, y) {
+ return (parseInt(x["chromosome_position"]) - parseInt(y["chromosome_position"]));
+ });
+
if (track_type == "sample_plot") {
- return seqpeek_builder.addSamplePlotTrackWithArrayData(track_obj.variants, track_target_svg, {
+ return seqpeek_builder.addSamplePlotTrackWithArrayData(variants, track_target_svg, {
guid: track_guid,
hovercard_content: {
"Location": function (d) {
- return d[POSITION_FIELD_NAME];
+ return d[COORDINATE_FIELD_NAME];
},
"Amino Acid Mutation": function (d) {
return d[AMINO_ACID_MUTATION_FIELD_NAME];
@@ -503,7 +796,7 @@ define([
return d["uniprot_id"];
}
}
- });
+ }, track_obj.is_summary_track);
}
else {
return seqpeek_builder.addBarPlotTrackWithArrayData(track_obj.variants, track_target_svg, {
@@ -520,34 +813,162 @@ define([
}
},
max_samples_in_location: this.maximum_samples_in_location
- });
+ }, track_obj.is_summary_track);
+ }
+ },
+
+ __build_regions: function(data, protein_start, protein_end) {
+ if (this.current_view_mode == DISPLAY_MODES.PROTEIN) {
+ return this.__build_regions_protein(data, protein_start, protein_end);
+ }
+ else {
+ return this.__build_regions_genomic(data, protein_start, protein_end);
+ }
+ },
+
+ __build_regions_protein: function(data, protein_start, protein_end) {
+ return [ { "type": "exon", "start": 0, "end": protein_end } ];
+ },
+
+ __build_regions_genomic: function(data, protein_start, protein_end) {
+ var itercount = 0;
+
+ data.sort(function(x, y) {
+ return (parseInt(x["chromosome_position"]) - parseInt(y["chromosome_position"]));
+ });
+
+ var split = _.reduce(data, function(memo, data_point, index, input_array) {
+ itercount += 1;
+ var has_uniprot = _.has(data_point, "uniprot_id");
+
+ if (memo.last_has_uniprot === null) {
+ memo.current_array.push(data_point);
+ memo.last_has_uniprot = has_uniprot;
+ return memo;
+ }
+
+ if (has_uniprot != memo.last_has_uniprot) {
+ memo.split_array.push({
+ coding: memo.last_has_uniprot,
+ data: _.clone(memo.current_array)
+ });
+
+ memo.current_array = [data_point];
+ }
+ else {
+ memo.current_array.push(data_point);
+ }
+
+ memo.last_has_uniprot = has_uniprot;
+
+ return memo;
+
+ }, {
+ current_array: [],
+ split_array: [],
+ last_has_uniprot: null
+ });
+
+ var get_first_start_coord = function(item) {
+ return parseInt(_.first(item["data"])["chromosome_position"]);
+ };
+
+ var region_info = _.reduce(split.split_array, function(memo, split_item, index, data_array) {
+ var data = split_item.data;
+ var first = _.first(data);
+ var last = _.last(data);
+
+ var region;
+
+ var start_coord = get_first_start_coord(split_item);
+
+ var end_coord = parseInt(last["chromosome_position"]);
+
+ if (split_item.coding) {
+ region = {
+ type: "exon",
+ start: start_coord,
+ end: end_coord,
+ start_aa: parseInt(first["amino_acid_position"]),
+ end_aa: parseInt(last["amino_acid_position"])
+ };
+ }
+ else {
+ region = {
+ type: "noncoding",
+ start: index == 0 ? start_coord : memo.previous_end_coord + 1,
+ end: index == (data_array.length - 1) ? end_coord : (get_first_start_coord(data_array[index+1]) - 0)
+ };
+ }
+
+ memo.previous_end_coord = end_coord;
+
+ memo.data.push(region);
+
+ return memo;
+
+ }, {
+ previous_end_coord: null,
+ data: [],
+ x_position: 0
+ });
+
+ return region_info.data;
+ },
+
+ __preprocess_data_and_render: function() {
+ _.each(this.model["mutations"]["by_tumor_type"], function(model, tumor_type) {
+ var data = model.toJSON()["items"];
+ if (_.isArray(data)) {
+ _.each(data, function(d) {
+ d[COORDINATE_FIELD_NAME] = parseInt(d[COORDINATE_FIELD_NAME]);
+
+ if (_.has(d, UNIPROT_FIELD_NAME)) {
+ d[AMINO_ACID_POSITION_FIELD_NAME] = parseInt(d[AMINO_ACID_POSITION_FIELD_NAME]);
+ }
+ });
+ }
+ }, this);
+
+ if (this.current_view_mode == DISPLAY_MODES.PROTEIN) {
+ this.__load_protein_domains();
+ }
+ else {
+ this.__render();
}
},
__filter_data: function(data_by_tumor_type) {
console.debug("seqpeek/view.__filter_data:" + this.selected_gene);
- var lowercase_gene = this.selected_gene.toLowerCase();
var filtered = {};
// Filter out rows that do not have the amino acid position field present,
// as drawing variants based on chromosome coordinates is not currently supported.
- _.each(data_by_tumor_type, function(data, tumor_type) {
- if (_.isArray(data)) {
- filtered[tumor_type] = _.filter(data, function(item) {
- return (_.has(item, "gene") &&
- _.isEqual(item["gene"].toLowerCase(), lowercase_gene) &&
- _.has(item, POSITION_FIELD_NAME));
- }, this);
- } else {
- if (_.has(data, "gene") && _.isEqual(data["gene"], lowercase_gene)) {
- filtered[tumor_type] = data;
- }
+ _.each(this.model["mutations"]["by_tumor_type"], function(model, tumor_type) {
+ var data = model.get("items");
+ if (this.current_view_mode == DISPLAY_MODES.PROTEIN) {
+ filtered[tumor_type] = this.__filter_mutation_data_for_protein_view(data);
}
- });
+ else {
+ filtered[tumor_type] = this.__filter_mutation_data_for_genomic_view(data)
+ }
+ }, this);
return filtered;
},
+ __filter_mutation_data_for_protein_view: function(data) {
+ var lowercase_gene = this.selected_gene.toLowerCase();
+
+ return _.filter(data, function(item) {
+ return (_.has(item, UNIPROT_FIELD_NAME) && _.has(item, "gene") && _.isEqual(item["gene"].toLowerCase(), lowercase_gene));
+ }, this);
+ },
+
+ __filter_mutation_data_for_genomic_view: function(data) {
+ return data;
+ },
+
__filter_mutsig_data: function(data_by_tumor_type) {
console.debug("seqpeek/view.__filter_mutsig_data:" + this.selected_gene);
@@ -577,21 +998,12 @@ define([
return {
variants: all_variants,
- tumor_type: "ALL",
- track_type: "bar_plot"
+ tumor_type: "COMBINED",
+ track_type: "bar_plot",
+ is_summary_track: true
};
},
- __parse_mutations: function () {
- console.debug("seqpeek/view.__parse_mutations");
-
- var data = {};
- _.each(this.model["mutations"]["by_tumor_type"], function(model, tumor_type) {
- data[tumor_type] = model.get("items");
- }, this);
- return data;
- },
-
__parse_mutsig: function () {
console.debug("seqpeek/view.__parse_mutsig");
return _.reduce(this.model["mutsig"].get("items"), function (memo, feature) {
@@ -635,8 +1047,9 @@ define([
var gene_to_uniprot_mapping = _.reduce(items, function(memo, item) {
var gene_label = item["gene"];
- if (!_.has(memo, gene_label)) {
- memo[gene_label] = item["uniprot_id"];
+ if (!_.has(memo, gene_label) && _.has(item, UNIPROT_FIELD_NAME)) {
+ memo[gene_label] = item[UNIPROT_FIELD_NAME];
+
}
return memo;
}, {});
@@ -654,10 +1067,42 @@ define([
__seqpeek_selection_handler: function(id_list) {
this.selected_patient_ids = id_list;
+
+ this.__update_sample_list_dropdown();
+ },
+
+ __update_sample_list_dropdown: function() {
+ var num_selected = this.selected_patient_ids.length;
+ var caption;
+
+ if (num_selected == 0 || num_selected > 1) {
+ caption = num_selected + " Samples Selected";
+ }
+ else {
+ caption = "1 Sample Selected";
+ }
+
+ this.$el.find(".sample-list-dropdown").html(SampleListCaptionTpl({
+ caption: caption
+ }));
+ },
+
+ __sample_list_union: function(target_list_model) {
+ if (this.selected_patient_ids.length > 0) {
+ this.samplelists.updateSampleListByUnion(target_list_model["id"], this.selected_patient_ids);
+ }
},
- __print_selected_samples: function() {
- console.log(this.selected_patient_ids);
+ __store_sample_list: function() {
+ var list_label = this.$el.find(".new-list-name").val();
+
+ if (list_label.length == 0 || this.selected_patient_ids.length == 0) {
+ return;
+ }
+
+ this.$el.find(".new-list-name").val("");
+
+ this.samplelists.addSampleList(list_label, this.selected_patient_ids);
}
});
});
diff --git a/app/scripts/views/stacksvis/view.js b/app/scripts/views/stacksvis/view.js
index 0446754..98615c7 100644
--- a/app/scripts/views/stacksvis/view.js
+++ b/app/scripts/views/stacksvis/view.js
@@ -1,7 +1,7 @@
-define(["jquery", "underscore", "backbone", "stacksvis",
+define(["jquery", "underscore", "backbone", "stacksvis", "models/gs/by_tumor_type",
"hbs!templates/stacksvis/container", "hbs!templates/gs/q_values_ampdel",
"colorbrewer", "d3"],
- function ($, _, Backbone, StacksVis, Tpl, QValueTpl) {
+ function ($, _, Backbone, StacksVis, TransformModel, Tpl, QValueTpl) {
return Backbone.View.extend({
"events": {
@@ -14,9 +14,13 @@ define(["jquery", "underscore", "backbone", "stacksvis",
},
"initialize": function () {
- this.options["models"]["copy_number"].on("load", this.__render_copy_number, this);
this.options["models"]["q_value"].on("load", this.__render_q_value, this);
+ _.each(this.options["models"]["copy_number"]["by_tumor_type"], function(model, tumor_type) {
+ model.on("load", function() {
+ this.__render_copy_number(tumor_type, new TransformModel(model.toJSON()));
+ }, this);
+ }, this);
_.each(this.options["models"]["mutated_samples"]["by_tumor_type"], function(model, tumor_type) {
model.on("load", function() {
this.__render_mutated_samples(tumor_type, model);
@@ -28,9 +32,7 @@ define(["jquery", "underscore", "backbone", "stacksvis",
this.$el.html(Tpl({
"id": Math.floor(Math.random() * 1000),
"tumor_types": WebApp.UserPreferences.get("selected_tumor_types"),
- "genes": _.map(this.options["genes"], function (g) {
- return g.toUpperCase();
- })
+ "genes": this.options["genes"]
}));
this.$el.find(".tooltips").tooltip({ "animation": false, "trigger": "click hover focus", "placement": "right" });
@@ -49,14 +51,14 @@ define(["jquery", "underscore", "backbone", "stacksvis",
_.each(WebApp.UserPreferences.get("selected_tumor_types"), function (tumor_type_obj) {
var items_per_gene = _.groupBy(items_per_tumor_type[tumor_type_obj.id], "gene");
_.each(this.options["genes"], function (gene) {
- var gene_items = items_per_gene[gene] || items_per_gene[gene.toLowerCase()];
+ var gene_items = items_per_gene[gene];
if (!gene_items || !_.isArray(gene_items)) return;
_.each(gene_items, function (gene_item) {
gene_item[gene_item["type"]] = true; // binarize for template use
});
- var $qvalues = this.$el.find(".stats-" + tumor_type_obj.id.toUpperCase() + "-" + gene.toUpperCase()).show();
+ var $qvalues = this.$el.find(".stats-" + tumor_type_obj["id"] + "-" + gene).show();
$qvalues.find(".q-values").html(QValueTpl({"items": _.sortBy(gene_items, "type")}));
$qvalues.find(".tooltips").tooltip({ "animation": false, "trigger": "click hover focus", "placement": "top" });
}, this);
@@ -64,106 +66,98 @@ define(["jquery", "underscore", "backbone", "stacksvis",
}, this);
},
- __render_copy_number: function () {
- this.rowLabels = _.map(this.options.genes, function (g) {
- return g.toLowerCase(); // TODO: not good
- });
+ __render_copy_number: function (tumor_type, model) {
+ this.rowLabels = this.options.genes;
- _.each(this.options.models["copy_number"].get("BY_TUMOR_TYPE"), function (ttModel, tumor_type) {
- if (_.isEmpty(ttModel.ROWS)) return;
- if (_.isEmpty(ttModel.COLUMNS)) return;
- if (_.isEmpty(ttModel.DATA)) return;
+ var ROWS = model.get("ROWS");
+ if (_.isEmpty(ROWS)) return;
- var columns_by_cluster = this.__column_model(ttModel);
- var data = {};
- var cbscale = colorbrewer.RdYlBu[5];
+ var COLUMNS = model.get("COLUMNS");
+ if (_.isEmpty(COLUMNS)) return;
- var gene_row_items = {};
- _.each(this.rowLabels, function (rowLabel) {
- var $statsEl = this.$el.find(".stats-" + tumor_type.toUpperCase() + "-" + rowLabel.toUpperCase()).show();
+ var DATA = model.get("DATA");
+ if (_.isEmpty(DATA)) return;
- gene_row_items[rowLabel] = $statsEl.find(".stats-hm").selector;
+ var columns_by_cluster = this.__column_model(model);
+ var data = {};
+ var cbscale = colorbrewer.RdYlBu[5];
- var row_idx = ttModel.ROWS.indexOf(rowLabel);
- if (row_idx < 0) return;
+ var gene_row_items = {};
+ _.each(this.rowLabels, function (rowLabel) {
+ var $statsEl = this.$el.find(".stats-" + tumor_type + "-" + rowLabel).show();
+
+ gene_row_items[rowLabel] = $statsEl.find(".stats-hm").selector;
- _.each(ttModel.DATA[row_idx], function (cell, cellIdx) {
- if (_.isString(cell.orig)) cell.orig = cell.orig.trim();
- var columnLabel = ttModel.COLUMNS[cellIdx].trim();
- if (!data[columnLabel]) data[columnLabel] = {};
- data[columnLabel][rowLabel] = {
- "value": cell.value,
- "row": rowLabel,
- "colorscale": cbscale[cell.value],
- "label": columnLabel + "\n" + rowLabel + "\n" + cell.orig
- };
- }, this);
-
- var counts = _.countBy(ttModel.DATA[row_idx], "value");
- var totals = ttModel.DATA[row_idx].length;
- var lookupPercentage = function (idx) {
- var count = counts[idx];
- if (count) return (100 * count / totals).toFixed(1) + "%";
- return "";
+ var row_idx = ROWS.indexOf(rowLabel);
+ if (row_idx < 0) return;
+
+ _.each(DATA[row_idx], function (cell, cellIdx) {
+ if (_.isString(cell.orig)) cell.orig = cell.orig.trim();
+ var columnLabel = COLUMNS[cellIdx].trim();
+ if (!data[columnLabel]) data[columnLabel] = {};
+ data[columnLabel][rowLabel] = {
+ "value": cell["value"],
+ "row": rowLabel,
+ "colorscale": cbscale[cell["value"]],
+ "label": columnLabel + "\n" + rowLabel + "\n" + cell.orig
};
- $statsEl.find(".stats-samples").html(totals);
- $statsEl.find(".stats-0").html(lookupPercentage("0"));
- $statsEl.find(".stats-1").html(lookupPercentage("1"));
- $statsEl.find(".stats-2").html(lookupPercentage("2"));
- $statsEl.find(".stats-3").html(lookupPercentage("3"));
- $statsEl.find(".stats-4").html(lookupPercentage("4"));
}, this);
- var vis = new StacksVis(this.$el, {
- "bar_width": 0.75,
- "vertical_padding": 1,
- "highlight_fill": colorbrewer.RdYlGn[3][2],
- "columns_by_cluster": columns_by_cluster,
- "row_labels": this.rowLabels,
- "row_selectors": gene_row_items
- });
- vis.draw({ "data": data });
+ var counts = _.countBy(DATA[row_idx], "value");
+ var totals = DATA[row_idx].length;
+ var lookupPercentage = function (idx) {
+ var count = counts[idx];
+ if (count) return (100 * count / totals).toFixed(1) + "%";
+ return "";
+ };
+ $statsEl.find(".stats-samples").html(totals);
+ $statsEl.find(".stats-0").html(lookupPercentage("0"));
+ $statsEl.find(".stats-1").html(lookupPercentage("1"));
+ $statsEl.find(".stats-2").html(lookupPercentage("2"));
+ $statsEl.find(".stats-3").html(lookupPercentage("3"));
+ $statsEl.find(".stats-4").html(lookupPercentage("4"));
}, this);
+
+ var vis = new StacksVis(this.$el, {
+ "bar_width": 0.75,
+ "vertical_padding": 1,
+ "highlight_fill": colorbrewer.RdYlGn[3][2],
+ "columns_by_cluster": columns_by_cluster,
+ "row_labels": this.rowLabels,
+ "row_selectors": gene_row_items
+ });
+ vis.draw({ "data": data });
},
__render_mutated_samples: function(tumor_type, model) {
_.each(model.get("items"), function(item) {
var gene = item["gene"];
- var $mutEl = this.$el.find(".stats-" + tumor_type.toUpperCase() + "-" + gene.toUpperCase()).show();
+ var $mutEl = this.$el.find(".stats-" + tumor_type + "-" + gene).show();
$mutEl.find(".stats-mutations").html(item["numberOf"]);
}, this);
},
- __column_model: function (ttModel) {
- var discretizeFn = function (val) {
- if (_.isNumber(val)) {
- if (val < -1.5) return 4; // homozygous loss (less than -1.5)
- if (val < -0.5) return 3; // heterozygous loss (between -0.5 and -1.4999)
- if (val < 0.5) return 2; // diploid (between 0.5 and -0.4999)
- if (val < 1.5) return 1; // gain (between 1.5 and 0.49999)
- return 0; // amplification // greater than 1.5
- }
- return val;
- };
-
- _.each(ttModel.DATA, function (outer_array, idx) {
- ttModel.DATA[idx] = _.map(outer_array, function (x) {
- return { "value": discretizeFn(x), "orig": x };
- });
- });
+ __column_model: function (model) {
+ var ROWS = model.get("ROWS") || [];
+ var COLUMNS = model.get("COLUMNS") || [];
+ var DATA = model.get("DATA") || [];
+
+ _.each(DATA, function (outer_array, idx) {
+ DATA[idx] = _.map(outer_array, function (x) {
+ return { "value": this.__discretize(x), "orig": x };
+ }, this);
+ }, this);
var unsorted_columns = [];
- _.each(ttModel.COLUMNS, function (column_name, col_idx) {
+ _.each(COLUMNS, function (column_name, col_idx) {
var column = { "name": column_name.trim(), "cluster": "_", "values": [] };
_.each(this.rowLabels, function (row_label) {
- var row_idx = ttModel.ROWS.indexOf(row_label);
+ var row_idx = ROWS.indexOf(row_label);
if (row_idx < 0) return;
- var cell = ttModel.DATA[row_idx][col_idx];
- if (_.isString(cell.orig)) {
- cell.orig = cell.orig.trim().toLowerCase();
- }
- column.values.push(cell.value);
+ var cell = DATA[row_idx][col_idx];
+ if (_.isString(cell["orig"])) cell["orig"] = cell["orig"].trim();
+ column["values"].push(cell["value"]);
}, this);
unsorted_columns.push(column);
}, this);
@@ -180,6 +174,17 @@ define(["jquery", "underscore", "backbone", "stacksvis",
});
return columns_by_cluster;
+ },
+
+ __discretize: function (val) {
+ if (_.isNumber(val)) {
+ if (val < -1.5) return 4; // homozygous loss (less than -1.5)
+ if (val < -0.5) return 3; // heterozygous loss (between -0.5 and -1.4999)
+ if (val < 0.5) return 2; // diploid (between 0.5 and -0.4999)
+ if (val < 1.5) return 1; // gain (between 1.5 and 0.49999)
+ return 0; // amplification // greater than 1.5
+ }
+ return val;
}
});
});
diff --git a/app/scripts/webapp.js b/app/scripts/webapp.js
index fbd0573..feadbba 100755
--- a/app/scripts/webapp.js
+++ b/app/scripts/webapp.js
@@ -1,8 +1,11 @@
define(["jquery", "underscore", "backbone",
"router",
"models/sessions", "models/datamodel", "models/lookups",
- "views/items_grid_view", "views/pivot_data_view", "views/search_control"],
- function ($, _, Backbone, AppRouter, SessionsCollection, Datamodel, LookupsModel, ItemGridView, PivotDataView, SearchControl) {
+ "views/items_grid_view", "views/pivot_data_view", "views/search_control",
+ "models/gs/item_set"],
+ function ($, _, Backbone, AppRouter, SessionsCollection, Datamodel, LookupsModel, ItemGridView, PivotDataView, SearchControl,
+ GeneSpotItemSet
+ ) {
WebApp = {
Events: _.extend(Backbone.Events),
@@ -23,7 +26,8 @@ define(["jquery", "underscore", "backbone",
},
LocalSession: new Backbone.Model(), // TODO : Add Sync
UserPreferences: new Backbone.Model(),
- Search: new SearchControl()
+ Search: new SearchControl(),
+ ItemSets: new GeneSpotItemSet()
};
WebApp.initialize = function () {
@@ -70,7 +74,9 @@ define(["jquery", "underscore", "backbone",
}
});
- WebApp.LocalSession.fetch({ "url": "svc/collections/local_session" })
+ WebApp.LocalSession.fetch({ "url": "svc/collections/local_session" });
+
+ WebApp.ItemSets.fetch();
};
WebApp.alert = function(alertEl, timeout) {
@@ -80,7 +86,11 @@ define(["jquery", "underscore", "backbone",
}, timeout || 2000);
};
- _.bindAll(WebApp, "initialize");
+ WebApp.getItemSets = function() {
+ return this.ItemSets;
+ };
+
+ _.bindAll(WebApp, "initialize", "getItemSets");
return WebApp;
});
\ No newline at end of file
diff --git a/bower.json b/bower.json
index e7fff5e..8bce952 100644
--- a/bower.json
+++ b/bower.json
@@ -22,7 +22,7 @@
"carve": "~0.0.7",
"x2js": "*",
"cytoscape": "~2.2.1",
- "seqpeek": "git://github.com/IlyaLab/SeqPeek.git#b4d49cd120ede4aaf8fac796ed757e1aeaec1da0"
+ "seqpeek": "git://github.com/IlyaLab/SeqPeek.git#9241a8c955c83f28335b311075ddfa54e5bc57c0"
},
"resolutions": {
"underscore": "~1.5.2",
diff --git a/db/scripts/add_fm_custom_labels.py b/db/scripts/add_fm_custom_labels.py
deleted file mode 100644
index 3b4cd7d..0000000
--- a/db/scripts/add_fm_custom_labels.py
+++ /dev/null
@@ -1,82 +0,0 @@
-'''
-Created on Jun 4, 2014
-
-@author: michael
-'''
-import argparse
-import logging
-from os import path
-import pymongo
-import re
-
-from utilities import configure_logging
-
-ffnPattern = re.compile('FFN.*\.tsv$')
-
-def addLabels(collection, path):
- logging.info(collection)
- with open(path, 'r') as labels:
- for line in labels:
- if '#' == line[0]:
- continue
- fields = line.strip().split('\t')
- if 2 > len(fields):
- raise ValueError('did not find two fields!!!')
- if collection.find_one({'id': fields[0]}):
- logging.info('found %s, updating label to %s', fields[0], fields[1])
- collection.update({'id': fields[0]}, {"$set": {"label": fields[1]}})
- else:
- logging.info("didn't find %s", (fields[0]))
-
-def checkForFFNFile(files, dirname, names):
- for name in names:
- if ffnPattern.search(name):
- files += [dirname + '/' + name];
-
-def findFNFfiles(args, dirname, names):
- dirs, topname = path.split(dirname);
- _, parent = path.split(dirs);
- if dirs == args.root and topname == args.dir:
- checkForFFNFile(args.topfiles, dirname, names)
- if args.tumor.lower() == parent.lower() and topname.startswith(args.dir):
- checkForFFNFile(args.tumorfiles, dirname, names)
-
-def main():
- parser = argparse.ArgumentParser(description="Utility to add custom labels to TCGA feature matrix in MongoDB")
- parser.add_argument("--host", required=True, help="MongoDB host name")
- parser.add_argument("--port", required=True, type=int, help="MongoDB port")
- parser.add_argument("--db", required=True, help="Database name")
- parser.add_argument("--tumor", required=True, help="Tumor type")
- parser.add_argument("--root", required=True, help="Root path to search for FFN custom files")
- parser.add_argument("--dir", default="aux", help="directory to look for FFN custom files")
- parser.add_argument("--loglevel", default="INFO", help="Logging Level")
- args = parser.parse_args()
- configure_logging(args.loglevel.upper())
-
- logging.info('starting add custom labels to feature matrix:\n\t%s' % (args))
-
- args.topfiles = [];
- args.tumorfiles = []
- path.walk
- path.walk(args.root, findFNFfiles, args)
- logging.info('%s %s', args.topfiles, args.tumorfiles)
-
- conn = pymongo.Connection(args.host, args.port)
- collection = conn[args.db]["feature_matrix"]
- if 0 == len(args.topfiles):
- raise ValueError('did not find a general custom file')
- else:
- for topfile in args.topfiles:
- addLabels(collection, topfile);
- if 0 == len(args.tumorfiles):
- logging.info('did not find a tumor type custom file')
- else:
- for tumorfile in args.tumorfiles:
- addLabels(collection, tumorfile);
- conn.close()
-
- logging.info('finished add custom labels to feature matrix')
-
-if __name__ == '__main__':
- main()
-
\ No newline at end of file
diff --git a/db/scripts/copynumbergistic_insert.py b/db/scripts/copynumbergistic_insert.py
new file mode 100644
index 0000000..96e7321
--- /dev/null
+++ b/db/scripts/copynumbergistic_insert.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+
+import argparse
+import csv
+import pymongo
+import logging
+
+from utilities import configure_logging
+
+def extract_records(file_path):
+ logging.info(file_path)
+
+ with open(file_path, "rb") as csvfile:
+ csvreader = csv.reader(csvfile, delimiter="\t")
+ ids = csvreader.next()[4:]
+
+ for row in csvreader:
+ gene_symbol = row[0]
+ locus_id = row[1]
+ cytoband = row[2]
+ values = row[4:]
+
+ if len(values) != len(ids): raise Exception("mismatched values and features: %s (%s/%s)" % (gene_symbol, str(len(values)), str(len(ids))))
+
+ record = {}
+ record["gene"] = gene_symbol
+ record["locus"] = locus_id
+ record["cytoband"] = cytoband
+ record["values"] = values_dict(ids, values)
+
+ yield record
+
+ logging.info("samples=%s" % str(len(ids)))
+
+def values_dict(ids, values):
+ result = {}
+ for i, v in zip(ids, values):
+ trunc_i = i
+ float_v = v
+
+ if len(i) > 16: trunc_i = i[0:15]
+ if v != "NA": float_v = float(v)
+
+ result[trunc_i] = float_v
+ return result
+
+def main():
+ parser = argparse.ArgumentParser(description="Utility to import Firehose Copy Number Gistic2 data to MongoDB")
+ parser.add_argument("--host", required=True, help="MongoDB host name")
+ parser.add_argument("--port", required=True, type=int, help="MongoDB port")
+ parser.add_argument("--db", required=True, help="Database name")
+ parser.add_argument("--f", required=True, help="Path to copy number gistic2 file")
+ parser.add_argument("--loglevel", default="INFO", help="Logging Level")
+ args = parser.parse_args()
+
+ configure_logging(args.loglevel.upper())
+
+ logging.info("import file: %s" % args.f)
+ logging.info("uploading to %s:%s/%s" % (args.host, args.port, args.db))
+
+ conn = pymongo.Connection(args.host, args.port)
+ collection = conn[args.db]["copy_number_gistic"]
+
+ count = 0
+ for record in extract_records(args.f):
+ collection.insert(record)
+ count += 1
+
+ logging.info("inserted count=%s" % count)
+
+ conn.close()
+
+if __name__ == "__main__":
+ main()
+
diff --git a/db/scripts/datawarehouse_import.py b/db/scripts/datawarehouse_import.py
new file mode 100644
index 0000000..91331ff
--- /dev/null
+++ b/db/scripts/datawarehouse_import.py
@@ -0,0 +1,172 @@
+#!/usr/bin/env python
+
+import argparse
+import csv
+import os
+import pymongo
+import logging
+import itertools
+import inspect
+import json
+from copy import copy
+from time import time
+
+from utilities import configure_logging
+from subprocess import call
+
+'''
+Usage:
+ python datawarehouse_import.py --config_file=/path/to/local/datawarehouse_import.json
+
+Requires config_file to be passed a file containing a JSON structure similar to the following:
+ {
+ "databases": {
+ "arbitrary_db_pointer_1": {
+ "vendor": "mongodb",
+ "host": "localhost",
+ "port": 4321
+ },
+ "arbitrary_db_pointer_2": {
+ "vendor": "mongodb",
+ "host": "localhost",
+ "port": 1234
+ }
+ },
+ "lookups_db": "arbitrary_db_pointer_2",
+ "imports": [
+ {
+ "tumor_type": "XYZ",
+ "database": "arbitrary_db_pointer_1",
+ "collections": {
+ "feature_matrix": "/path/to/local/fmx/file",
+ "mutation_summary": "/path/to/local/mut_sum/file",
+ "copy_number_gistic": "/path/to/local/cn_gistic/file",
+ "mutsig_rankings": "/path/to/local/mutsig_rankings/sig_genes.txt"
+ },
+ "annotations": {
+ "CNVR": "/path/to/local/fmx/annotations/cnvr_file",
+ "METH": "/path/to/local/fmx/annotations/meth_file",
+ "RPPA": "/path/to/local/fmx/annotations/rppa_file"
+ }
+ }
+ ]
+ }
+'''
+
+def execute_python(script_path, spec):
+ logging.debug("(%s,%s)" % (script_path, spec))
+ current_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
+
+ if not "database" in spec:
+ logging.warn("NO DATABASE CONFIGURED")
+ return
+
+ db = spec["database"]
+ exec_str = "python %s/%s --host=%s --port=%s --db=%s --f=%s" % (current_path, script_path, db["host"], str(db["port"]), spec["tumor_type"], spec["file"])
+ logging.info("\n\t%s" % exec_str)
+ return_code = call([exec_str], shell=True)
+
+ logging.debug("%s %s" % (script_path, return_code))
+
+def execute_javascript(script_path, spec):
+ logging.debug("(%s,%s)" % (script_path, spec))
+
+ if not "database" in spec:
+ logging.warn("NO DATABASE CONFIGURED")
+ return
+
+ current_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
+ db = spec["database"]
+
+ eval_expr = ""
+ if "lookups_db" in spec:
+ lookup_db = spec["lookups_db"]
+ eval_expr = "--eval=\'var lookups_db_uri=\"%s:%s/LOOKUPS\"\'" % (lookup_db["host"], str(lookup_db["port"]))
+
+ exec_str = "mongo --quiet --host=%s --port=%s %s %s/%s %s" % (db["host"], str(db["port"]), spec["tumor_type"], current_path, script_path, eval_expr)
+ logging.info("\n\t%s" % exec_str)
+ return_code = call([exec_str], shell=True)
+ logging.debug("%s %s" % (script_path, return_code))
+
+def process_import(config_json):
+ config_dbs = config_json["databases"]
+
+ for db_key in config_dbs:
+ db = config_dbs[db_key]
+ logging.debug("databases [%s] %s:%s:%s" % (db_key, db["vendor"], db["host"], str(db["port"])))
+
+ for im in config_json["imports"]:
+ logging.info("### STARTED:%s:%s ###" % (im["tumor_type"], im["database"]))
+
+ db_name = im["database"]
+ if db_name in config_dbs:
+ db_inst = config_dbs[db_name]
+ if not db_inst is None: im["database"] = db_inst
+ else:
+ logging.warning("unknown database [%s] [%s]" % (im["tumor_type"], db_name))
+ continue
+
+ if "lookups_db" in config_json:
+ im["lookups_db"] = config_dbs[config_json["lookups_db"]]
+
+ im_collections = im["collections"]
+ if "feature_matrix" in im_collections:
+ im["file"] = im_collections["feature_matrix"]
+ execute_python("featurematrix_insert.py", im)
+ execute_javascript("featurematrix_fill_tags.js", im)
+ execute_javascript("featurematrix_fill_unid.js", im)
+ execute_javascript("featurematrix_mutated_samples.js", im)
+ execute_javascript("featurematrix_verify_entries.js", im)
+
+ if "mutation_summary" in im_collections:
+ im["file"] = im_collections["mutation_summary"]
+ execute_python("mutationsummary_insert.py", im)
+
+ if "copy_number_gistic" in im_collections:
+ im["file"] = im_collections["copy_number_gistic"]
+ execute_python("copynumbergistic_insert.py", im)
+
+ if "mutsig_rankings" in im_collections:
+ im["file"] = im_collections["mutsig_rankings"]
+ execute_python("mutsigrankings_insert.py", im)
+ execute_javascript("mutsigrankings_top20.js", im)
+
+ if "annotations" in im:
+ im_clone = copy(im)
+ im_clone_annot = im_clone["annotations"]
+ if "CNVR" in im_clone_annot:
+ im_clone["file"] = im_clone_annot["CNVR"]
+ execute_python("featurematrix_annotate_CNVR.py", im_clone)
+ if "METH" in im_clone_annot:
+ im_clone["file"] = im_clone_annot["METH"]
+ execute_python("featurematrix_annotate_METH.py", im_clone)
+ if "RPPA" in im_clone_annot:
+ im_clone["file"] = im_clone_annot["RPPA"]
+ execute_python("featurematrix_annotate_RPPA.py", im_clone)
+
+ execute_javascript("lookups_aggregate_sample_types.js", im)
+ execute_javascript("lookups_aggregate_clinical_variables.js", im)
+ execute_javascript("misc_aggregate_collection_fields.js", im)
+
+ logging.info("### COMPLETED:%s ###" % im["tumor_type"])
+
+def main():
+ parser = argparse.ArgumentParser(description="Utility to import data into the Cancer Regulome Data Warehouse")
+ parser.add_argument("--config_file", required=True, help="Configuration File")
+ parser.add_argument("--loglevel", default="INFO", help="Logging Level")
+ args = parser.parse_args()
+
+ configure_logging(args.loglevel.upper())
+
+ startAt = time()
+ logging.info("\n------------------\nSTART\n------------------")
+
+ logging.debug("config_file: %s" % args.config_file)
+ process_import(json.load(open(args.config_file)))
+
+ endAt = time()
+ durAt = round(endAt - startAt, 3)
+ logging.info("\n------------------\nCOMPLETED in %s sec(s)\n------------------" % str(durAt))
+
+if __name__ == "__main__":
+ main()
diff --git a/db/scripts/extract_medians.py b/db/scripts/extract_medians.py
new file mode 100644
index 0000000..b0be704
--- /dev/null
+++ b/db/scripts/extract_medians.py
@@ -0,0 +1,58 @@
+'''
+Created on Jun 10, 2014
+
+@author: michael
+'''
+import argparse
+import logging
+import pymongo
+
+from utilities import configure_logging
+
+def outputValues(outfile, dbs, gene2tumor2median):
+ logging.info('\n\tstarting outputValues()')
+ with open(outfile, 'w') as out:
+ out.write('\t' + '\t'.join(dbs) + '\n')
+ genes = gene2tumor2median.keys()
+ genes.sort()
+ for gene in genes:
+ line = gene
+ for db in dbs:
+ line += '\t' + str(gene2tumor2median[gene].get(db, 'NA'))
+ out.write(line + '\n')
+ logging.info('\n\tfinished outputValues()')
+
+def processTumors(args):
+ gene2tumor2median = {}
+ conn = pymongo.Connection(args.host, args.port)
+ logging.info('\n\tstarting processTumors()')
+ for db in args.dbs:
+ logging.info('\n\t\treading %s' % (db))
+ collection = conn[db]["feature_matrix"]
+ docs = collection.find({"source": args.platform})
+ for doc in docs:
+ tumor2median = gene2tumor2median.setdefault(doc["label"], {})
+ tumor2median[db] = doc["statistics"]["numeric"]["median"]
+
+ conn.close()
+ logging.info('\n\tfinished processTumors()')
+ return gene2tumor2median
+
+def main():
+ parser = argparse.ArgumentParser(description="Utility to produce a table of genes vs. tumor types w/ median values")
+ parser.add_argument("--host", required=True, help="MongoDB host name")
+ parser.add_argument("--port", required=True, type=int, help="MongoDB port")
+ parser.add_argument("dbs", nargs='+', help="Database names")
+ parser.add_argument("--out", required=True, help="where to output the table")
+ parser.add_argument("--platform", default="GEXP", help="feature type to find the median from")
+ parser.add_argument("--loglevel", default="INFO", help="Logging Level")
+ args = parser.parse_args()
+ configure_logging(args.loglevel.upper())
+
+ logging.info('starting extract medians:\n\t%s' % (args))
+ gene2tumor2median = processTumors(args);
+ outputValues(args.out, args.dbs, gene2tumor2median)
+ logging.info('finished extract medians')
+
+if __name__ == '__main__':
+ main()
\ No newline at end of file
diff --git a/db/scripts/annotate_CNVR.py b/db/scripts/featurematrix_annotate_CNVR.py
similarity index 93%
rename from db/scripts/annotate_CNVR.py
rename to db/scripts/featurematrix_annotate_CNVR.py
index 9036ac1..cb7f12e 100644
--- a/db/scripts/annotate_CNVR.py
+++ b/db/scripts/featurematrix_annotate_CNVR.py
@@ -60,11 +60,11 @@ def find_and_modify(collection, refGenes_by_id):
if collection.find({ "id": id }).count() == 1:
collection.find_and_modify({ "id": id }, {"$set":{ "refGenes": refGenes }})
- logging.debug("find_and_modify [%s] [%s]===%s" % (count, id, refGenes))
+ logging.debug("[%s] [%s]===%s" % (count, id, refGenes))
count += 1
if count % 1000 == 0: logging.info("update [%s]" % count)
- logging.info("total find_and_modify count=%s" % count)
+ logging.info("total count=%s" % count)
def main():
parser = argparse.ArgumentParser(description="Utility to annotate features with genes (tags) in MongoDB")
@@ -94,7 +94,7 @@ def main():
conn.close()
- logging.info("annotate_features_with_genes(%s):complete" % args.db)
+ logging.info("COMPLETE")
if __name__ == "__main__":
main()
diff --git a/db/scripts/annotate_METH.py b/db/scripts/featurematrix_annotate_METH.py
similarity index 80%
rename from db/scripts/annotate_METH.py
rename to db/scripts/featurematrix_annotate_METH.py
index 2b00042..d3285c7 100755
--- a/db/scripts/annotate_METH.py
+++ b/db/scripts/featurematrix_annotate_METH.py
@@ -45,40 +45,44 @@ def extract_tags_by_id(filename):
r_by_id = {}
for row in csvreader:
if len(row) >= 3:
- probe_id = row[0]
- refGenes = row[3].split(";")
- uniqGenes = filter(None, list(set(refGenes)))
- logging.debug("extract_tags_by_id=%s:%s:%s" % (probe_id, refGenes, uniqGenes))
- if len(uniqGenes) > 0: r_by_id[probe_id] = uniqGenes
+ feature_id = row[0]
+ count = row[1]
+ uniqGenes = []
+ if count > 0:
+ refGenes = row[2].split(";")
+ uniqGenes = filter(None, list(set(refGenes)))
+
+ logging.debug("%s:%s:%s" % (feature_id, refGenes, uniqGenes))
+ if len(uniqGenes) > 0: r_by_id[feature_id] = uniqGenes
else:
skipcount += 1
- logging.warning("extract_tags_by_id=skipping:%s" % skipcount)
+ logging.warning("skipping:%s" % skipcount)
return r_by_id
def find_and_modify(collection, tags_by_id):
count = 0
skipcount = 0
- for doc in collection.find({ "source": "METH", "probe": { "$exists": True } }, { "probe": True }):
- if "probe" in doc:
- probe_id = str(doc["probe"])
- if probe_id in tags_by_id:
- tags = tags_by_id[probe_id]
+ for doc in collection.find({ "source": "METH", "probe": { "$exists": True } }, { "values": False }):
+ if "id" in doc:
+ feature_id = str(doc["id"])
+ if feature_id in tags_by_id:
+ tags = tags_by_id[feature_id]
if not tags is None:
collection.find_and_modify({ "_id": doc["_id"] }, { "$set":{ "refGenes": tags }})
else:
- logging.warning("skipping:tags not found:%s" % probe_id)
+ logging.warning("skipping:tags not found:%s" % feature_id)
skipcount += 1
else:
- logging.warning("skipping:probe not found:%s" % probe_id)
+ logging.debug("skipping:feature_id not found:%s" % feature_id)
skipcount += 1
else:
- logging.warning("skipping:probe not in doc:%s" % str(doc))
+ logging.warning("skipping:feature_id not in doc:%s" % str(doc))
skipcount += 1
if count > 0 and count % 1000 == 0: logging.info("update [%s]" % count)
- logging.info("total find_and_modify count=%s [skip=%s]" % (count, skipcount))
+ logging.info("total count=%s [skip=%s]" % (count, skipcount))
def main():
parser = argparse.ArgumentParser(description="Utility to annotate features with probe IDs (i.e. METH) to genes based on annotations file")
diff --git a/db/scripts/annotate_RPPA.py b/db/scripts/featurematrix_annotate_RPPA.py
similarity index 92%
rename from db/scripts/annotate_RPPA.py
rename to db/scripts/featurematrix_annotate_RPPA.py
index 9bc49c4..9cacbc1 100755
--- a/db/scripts/annotate_RPPA.py
+++ b/db/scripts/featurematrix_annotate_RPPA.py
@@ -54,14 +54,14 @@ def find_and_modify(collection, tags_by_id):
cnt = collection.find({ "antibody": id }).count()
if cnt == 1:
collection.find_and_modify({ "antibody": id }, { "$set":{ "refGenes": tags }})
- logging.debug("find_and_modify [%s] [%s]===%s" % (count, id, tags))
+ logging.debug("[%s] [%s]===%s" % (count, id, tags))
count += 1
else:
- logging.warning("skipping: find_and_modify [%s] [%s]===%s" % (cnt, id, tags))
+ logging.warning("skipping: [%s] [%s]===%s" % (cnt, id, tags))
skipcount += 1
if count % 100 == 0: logging.info("update [%s]" % count)
- logging.info("total find_and_modify count=%s [skip=%s]" % (count, skipcount))
+ logging.info("total count=%s [skip=%s]" % (count, skipcount))
def main():
parser = argparse.ArgumentParser(description="Utility to annotate features with antibody IDs (i.e. RPPA) to genes based on annotations file")
diff --git a/db/scripts/fill_tags.js b/db/scripts/featurematrix_fill_tags.js
similarity index 83%
rename from db/scripts/fill_tags.js
rename to db/scripts/featurematrix_fill_tags.js
index 546566c..a75b064 100644
--- a/db/scripts/fill_tags.js
+++ b/db/scripts/featurematrix_fill_tags.js
@@ -27,7 +27,7 @@ var update_tags = function(d) {
if (tags.length <= 0) tags.push("NO_MATCH");
- db.feature_matrix.update({ "_id": d["_id"] }, { "$set": { "tags": tags } } );
+ db["feature_matrix"].update({ "_id": d["_id"] }, { "$set": { "tags": tags } } );
};
-db.feature_matrix.find(query).forEach(update_tags);
\ No newline at end of file
+db["feature_matrix"].find(query).forEach(update_tags);
\ No newline at end of file
diff --git a/db/scripts/featurematrix_fill_unid.js b/db/scripts/featurematrix_fill_unid.js
new file mode 100644
index 0000000..bfa053a
--- /dev/null
+++ b/db/scripts/featurematrix_fill_unid.js
@@ -0,0 +1,79 @@
+/*
+- Reads from "feature_matrix" collection
+- Updates each feature with a "unid" that more coarsely defines uniqueness
+
+Usage:
+
+ mongo --host=$HOST $DB_NAME featurematrix_fill_unid.js
+
+Example of the type of features targeted for aggregation by this script. Must be run on each tumor_type feature matrix independently.
+{
+ "_id": "537bde73ab47b3606a2b2f5f",
+ "end": 7590856,
+ "chromosome": "chr17",
+ "start": 7565097,
+ "tags": [
+ "TP53"
+ ],
+ "label": "TP53 (7157)",
+ "source": "GEXP",
+ "platform": "mRNAseq",
+ "gene": "TP53",
+ "type": "N",
+ "id": "N:GEXP:TP53:chr17:7565097:7590856:-:7157",
+ "strand": "-"
+}
+
+Uniqueness_id -> "unid": "N:GEXP:TP53:7157"
+*/
+
+var db_name = db["_name"];
+
+var UNID_builder = function(doc, key_ids) {
+ var keys = [];
+ key_ids.forEach(function(key_id) {
+ if (doc[key_id]) keys.push(doc[key_id]);
+ });
+ return keys.join(":");
+};
+
+var UNID_generators = {
+ "GEXP": function(doc) {
+ return UNID_builder(doc, ["type", "source", "gene", "code"]);
+ },
+ "GNAB": function(doc) {
+ return UNID_builder(doc, ["type", "source", "gene", "code"]);
+ },
+ "MIRN": function(doc) {
+ return UNID_builder(doc, ["type", "source", "accession_number", "code"]);
+ },
+ "CNVR": function(doc) {
+ return UNID_builder(doc, ["type", "source", "chromosome", "start", "end", "code"]);
+ },
+ "METH": function(doc) {
+ return UNID_builder(doc, ["type", "source", "probe"]);
+ },
+ "RPPA": function(doc) {
+ return UNID_builder(doc, ["type", "source", "antibody"]);
+ },
+ "CLIN": function(doc) {
+ return UNID_builder(doc, ["type", "source", "code"]);
+ },
+ "SAMP": function(doc) {
+ return UNID_builder(doc, ["type", "source", "code"]);
+ }
+};
+
+for (var src in UNID_generators) {
+ var dtFmt = (new Date()).toLocaleFormat("%Y-%m-%d %H:%M:%S,000");
+ print("[INFO] " + dtFmt + " - featurematrix_fill_unid(" + db_name + "):" + src + ":" + db["feature_matrix"].find({ "source": src }).count());
+
+ var unid_generator = UNID_generators[src];
+ db["feature_matrix"].find({ "source": src }).forEach(function(doc) {
+ db["feature_matrix"].update({
+ "_id": doc["_id"]
+ }, {
+ "$set": { "unid": unid_generator(doc) }
+ });
+ });
+}
\ No newline at end of file
diff --git a/db/scripts/insert_featurematrix.py b/db/scripts/featurematrix_insert.py
similarity index 96%
rename from db/scripts/insert_featurematrix.py
rename to db/scripts/featurematrix_insert.py
index b812186..5b9f05a 100755
--- a/db/scripts/insert_featurematrix.py
+++ b/db/scripts/featurematrix_insert.py
@@ -8,7 +8,7 @@
from utilities import configure_logging
def extract_features(file_path):
- logging.info("extract_features(%s)" % file_path)
+ logging.info(file_path)
with open(file_path, "rb") as csvfile:
csvreader = csv.reader(csvfile, delimiter="\t")
@@ -28,7 +28,7 @@ def extract_features(file_path):
yield feature_object
- logging.info("extract_features(%s):samples=%s" % (file_path, str(len(ids))))
+ logging.info("samples=%s" % str(len(ids)))
def extract_feature_dict(feature_id):
feature_parts = feature_id.split(":")
diff --git a/db/scripts/mapreduce_mutated_samples.js b/db/scripts/featurematrix_mutated_samples.js
similarity index 67%
rename from db/scripts/mapreduce_mutated_samples.js
rename to db/scripts/featurematrix_mutated_samples.js
index 055d88a..ced79bd 100644
--- a/db/scripts/mapreduce_mutated_samples.js
+++ b/db/scripts/featurematrix_mutated_samples.js
@@ -56,18 +56,24 @@ var finalize = function(key, reducedValue) {
var query = {
"source":"GNAB",
- "code":"y_n_somatic",
+ "code":"code_potential_somatic",
"type": "B",
"gene": { "$exists": true }
};
-print("matching features from feature_matrix:" + db.feature_matrix.find(query).count());
-db.feature_matrix.mapReduce(map, reduce, { "query": query, "out": "mutated_samples_mrtemp", "finalize": finalize });
+var db_name = db["_name"];
+var pretty_print = function(msg) {
+ var dtFmt = (new Date()).toLocaleFormat("%Y-%m-%d %H:%M:%S,000");
+ print("[INFO] " + dtFmt + " - featurematrix_mutated_samples(" + db_name + ") - " + msg);
+};
+
+pretty_print("matching features from feature_matrix:" + db["feature_matrix"].find(query).count());
+db["feature_matrix"].mapReduce(map, reduce, { "query": query, "out": "mutated_samples_mrtemp", "finalize": finalize });
-db.mutated_samples.drop();
-db.mutated_samples_mrtemp.find().forEach(function(d) {
- db.mutated_samples.insert(d.value);
+db["mutated_samples"].drop();
+db["mutated_samples_mrtemp"].find().forEach(function(d) {
+ db["mutated_samples"].insert(d.value);
});
-db.mutated_samples_mrtemp.drop();
+db["mutated_samples_mrtemp"].drop();
-print("final result in mutated_samples:" + db.mutated_samples.find().count());
+pretty_print("final result in mutated_samples:" + db["mutated_samples"].find().count());
diff --git a/db/scripts/featurematrix_verify_entries.js b/db/scripts/featurematrix_verify_entries.js
new file mode 100644
index 0000000..8968dc0
--- /dev/null
+++ b/db/scripts/featurematrix_verify_entries.js
@@ -0,0 +1,50 @@
+var exists = { "$exists": true };
+var not_exists = { "$exists": false };
+
+var cnt_gexp_exists = db["feature_matrix"].find({ "gene": exists, "source":"GEXP" }).count();
+var cnt_gexp_not_exists = db["feature_matrix"].find({ "gene": not_exists, "source":"GEXP" }).count();
+
+var cnt_gnab_exists = db["feature_matrix"].find({ "gene": exists, "source":"GNAB" }).count();
+var cnt_gnab_not_exists = db["feature_matrix"].find({ "gene": not_exists, "source":"GNAB" }).count();
+
+var cnt_rppa_antibody_exists = db["feature_matrix"].find({ "antibody": exists, "source":"RPPA" }).count();
+var cnt_rppa_antibody_not_exists = db["feature_matrix"].find({ "antibody": not_exists, "source":"RPPA" }).count();
+
+var cnt_rppa_exists = db["feature_matrix"].find({ "refGenes": exists, "source":"RPPA" }).count();
+var cnt_rppa_not_exists = db["feature_matrix"].find({ "refGenes": not_exists, "source":"RPPA" }).count();
+
+var cnt_meth_probe_exists = db["feature_matrix"].find({ "probe": exists, "source":"METH" }).count();
+var cnt_meth_probe_not_exists = db["feature_matrix"].find({ "probe": not_exists, "source":"METH" }).count();
+
+var cnt_meth_exists = db["feature_matrix"].find({ "refGenes": exists, "source":"METH" }).count();
+var cnt_meth_not_exists = db["feature_matrix"].find({ "refGenes": not_exists, "source":"METH" }).count();
+
+var cnt_mirn_exists = db["feature_matrix"].find({ "microRNA": exists, "source":"MIRN" }).count();
+var cnt_mirn_not_exists = db["feature_matrix"].find({ "microRNA": not_exists, "source":"MIRN" }).count();
+
+var cnt_cnvr_exists = db["feature_matrix"].find({ "refGenes": exists, "source":"CNVR" }).count();
+var cnt_cnvr_not_exists = db["feature_matrix"].find({ "refGenes": not_exists, "source":"CNVR" }).count();
+
+var cnt_tags_exists = db["feature_matrix"].find({ "tags": exists }).count();
+var cnt_tags_not_exists = db["feature_matrix"].find({ "tags": not_exists }).count();
+
+var cnt_label_exists = db["feature_matrix"].find({ "label": exists }).count();
+var cnt_label_not_exists = db["feature_matrix"].find({ "label": not_exists }).count();
+
+var db_name = db["_name"];
+var pretty_print = function(msg) {
+ var dtFmt = (new Date()).toLocaleFormat("%Y-%m-%d %H:%M:%S,000");
+ print("[INFO] " + dtFmt + " - featurematrix_verify_entries(" + db_name + ") - " + msg);
+};
+
+pretty_print("header: :: ['count exists'/'count not exists']");
+pretty_print("GEXP:gene [" + cnt_gexp_exists + "/" + cnt_gexp_not_exists + "]");
+pretty_print("GNAB:gene [" + cnt_gnab_exists + "/" + cnt_gnab_not_exists + "]");
+pretty_print("RPPA:antibody [" + cnt_rppa_antibody_exists + "/" + cnt_rppa_antibody_not_exists + "]");
+pretty_print("RPPA:refGenes [" + cnt_rppa_exists + "/" + cnt_rppa_not_exists + "]");
+pretty_print("METH:probe [" + cnt_meth_probe_exists + "/" + cnt_meth_probe_not_exists + "]");
+pretty_print("METH:refGenes [" + cnt_meth_exists + "/" + cnt_meth_not_exists + "]");
+pretty_print("MIRN:microRNA [" + cnt_mirn_exists + "/" + cnt_mirn_not_exists + "]");
+pretty_print("CNVR:refGenes [" + cnt_cnvr_exists + "/" + cnt_cnvr_not_exists + "]");
+pretty_print("*:tags [" + cnt_tags_exists + "/" + cnt_tags_not_exists + "]");
+pretty_print("*:label [" + cnt_label_exists + "/" + cnt_label_not_exists + "]");
\ No newline at end of file
diff --git a/db/scripts/ffn_add_fm_custom_labels.py b/db/scripts/ffn_add_fm_custom_labels.py
new file mode 100644
index 0000000..612e241
--- /dev/null
+++ b/db/scripts/ffn_add_fm_custom_labels.py
@@ -0,0 +1,190 @@
+'''
+Created on Jun 4, 2014
+
+usage: add_fm_custom_labels.py --host --port --tumor --root
+NOTE: --db will generally be equal to --tumor
+NOTE: --root must not end with a '/'
+
+@author: michael
+'''
+import argparse
+import logging
+import os
+import pymongo
+import re
+import traceback
+
+from utilities import configure_logging
+
+ffnPattern = re.compile('FFN.*\.tsv$')
+clusterGroupPattern = re.compile('^[CG]([0-9]+)$')
+
+def removeClusterOrGroup(name):
+ match = clusterGroupPattern.match(name)
+ if match:
+ return match.group(1)
+ else:
+ return name
+
+def display(value):
+ value = value.replace('_', ' ')
+ return value[0].upper() + value[1:]
+
+def makeLabel(label, fv, typeformat):
+ if not typeformat or typeformat == 'Default':
+ return label + ': ' + fv
+ elif typeformat == 'Append':
+ return fv + ' ' + label
+ elif typeformat == 'AppendWithParentheses':
+ return fv + ' (' + label + ')'
+ elif typeformat == 'DoNotDisplay':
+ return fv
+ elif typeformat == 'Prepend':
+ return label + ' ' + fv
+ print '''Didn't fall into any case!!!'''
+
+def updateCategoryFeatures(collection, ffv_infos):
+ catOnePattern = r'I\(([^,]+)\|%s\)'
+ catTwoPattern = r'I\((.+),(.+)\|%s\)'
+ for name, ffv_info in ffv_infos.iteritems():
+ regex = re.compile(catOnePattern % (name))
+ docsOne = collection.find({"id": regex})
+ print 'for name %s found %d docs' % (name, docsOne.count())
+ for doc in docsOne:
+ try:
+ nlabel = ffv_info[0]
+ match = regex.match(doc['id'].split(':')[2])
+ fv = ffv_info[1].get(match.group(1), removeClusterOrGroup(match.group(1)))
+ collection.update({'id': doc['id']}, {"$set": {"label": makeLabel(display(nlabel), display(fv), ffv_info[2])}})
+ except Exception as e:
+ traceback.print_exc()
+ print 'problem with updateCategoryFeatures(%s-%s): %s: %s' % (doc['id'], fv, name, ffv_info)
+ raise e
+ docsOne.close()
+
+ regex = re.compile(catTwoPattern % (name))
+ docsTwo = collection.find({"id": regex})
+ print 'for name %s found %d docs' % (name, docsTwo.count())
+ for doc in docsTwo:
+ try:
+ nlabel = ffv_info[0]
+ match = regex.match(doc['id'].split(':')[2])
+ fv1 = ffv_info[1].get(match.group(1), removeClusterOrGroup(match.group(1)))
+ fv2 = ffv_info[1].get(match.group(2), removeClusterOrGroup(match.group(2)))
+ collection.update({'id': doc['id']}, {"$set": {"label": makeLabel(display(nlabel), display(fv1) + ' vs ' + display(fv2), ffv_info[2])}})
+ except Exception as e:
+ traceback.print_exc()
+ print 'problem with updateCategoryFeatures(%s-%s vs %s): %s: %s' % (doc['id'], fv1, fv2, name, ffv_info)
+ raise e
+ docsTwo.close()
+
+
+def updateLabels(collection, path, ffv_infos):
+ logging.info('\ncollection: %s file: %s' % (collection, path))
+ count = 0
+ with open(path, 'r') as labels:
+ labels.readline()
+ for line in labels:
+ try:
+ count += 1
+ if '#' == line[0]:
+ continue
+ fields = line.strip().split('\t')
+ if 2 > len(fields):
+ raise ValueError('did not find two fields!!!')
+
+ fields += [None, None]
+ doc = collection.find_one({'id': fields[0]})
+ ffv_info = [None, {}, None]
+ if doc:
+ if 'Default' == fields[1]:
+ fields[1] = ''
+ if fields[1]:
+ logging.info('found %s, updating label to %s', fields[0], fields[1])
+ collection.update({'id': fields[0]}, {"$set": {"label": display(fields[1])}})
+ ffv_info[0] = fields[1]
+ else:
+ name = doc['id'].split(':')[2]
+ logging.info('found %s w/o custom label, updating to default %s', fields[0], display(name))
+ collection.update({'id': fields[0]}, {"$set": {"label": display(name)}})
+ ffv_info[0] = display(name)
+ else:
+ logging.info("didn't find %s", (fields[0]))
+ continue
+
+ if not ((fields[2] and fields[2] != 'Default') or fields[3]):
+ continue
+
+ print 'fields: %s' % (fields)
+ if 'Default' == fields[2]:
+ fields[2] = ''
+ order = 0
+ if fields[2]:
+ ffv_precedence = {}
+ for field in fields[2].split(','):
+ ffv_precedence[field.split(':')[0]] = {'ffv': field.split(':')[1], 'ordinal': order}
+ order += 1
+ collection.update({'id': fields[0]}, {"$set": {"ffv_precedence": ffv_precedence}})
+ print '\tprecedence: %s' % (ffv_precedence)
+ ffv_map = dict([(field.split(':')[0], field.split(':')[1]) for field in fields[2].split(',')])
+ ffv_info[1] = ffv_map
+ if fields[3]:
+ ffv_info[2] = fields[3]
+
+ ffv_infos[doc['id'].split(':')[2]] = ffv_info
+ except Exception as e:
+ traceback.print_exc()
+ print 'problem with parsing line %d: %s--%s' % (count, line, e)
+ raise e
+
+def checkForFFNFile(files, dirname, names):
+ for name in names:
+ if ffnPattern.search(name):
+ files += [dirname + '/' + name];
+
+def main():
+ parser = argparse.ArgumentParser(description="Utility to add custom labels to TCGA feature matrix in MongoDB")
+ parser.add_argument("--host", required=True, help="MongoDB host name")
+ parser.add_argument("--port", required=True, type=int, help="MongoDB port")
+ parser.add_argument("--db", required=True, help="Database name")
+ parser.add_argument("--tumor", required=True, help="Tumor type")
+ parser.add_argument("--root", required=True, help="Root path to search for FFN custom files")
+ parser.add_argument("--dir", default="aux", help="directory to look for FFN custom files")
+ parser.add_argument("--loglevel", default="INFO", help="Logging Level")
+ args = parser.parse_args()
+ configure_logging(args.loglevel.upper())
+
+ logging.info('starting add custom labels to feature matrix:\n\t%s' % (args))
+
+ args.topfiles = [];
+ args.tumorfiles = []
+ for root, _, files in os.walk(args.root, followlinks = True):
+ basedir = os.path.dirname(root)
+ namedir = os.path.basename(root)
+ if basedir == args.root and namedir == args.dir:
+ checkForFFNFile(args.topfiles, root, files)
+ if args.tumor.lower() == os.path.basename(basedir).lower() and namedir == args.dir:
+ checkForFFNFile(args.tumorfiles, root, files)
+
+ logging.info('%s %s', args.topfiles, args.tumorfiles)
+
+ conn = pymongo.Connection(args.host, args.port)
+ collection = conn[args.db]["feature_matrix"]
+ ffv_infos = {}
+ if 0 == len(args.topfiles):
+ raise ValueError('did not find a general custom file')
+ else:
+ for topfile in args.topfiles:
+ updateLabels(collection, topfile, ffv_infos);
+ if 0 == len(args.tumorfiles):
+ logging.info('did not find a tumor type custom file')
+ else:
+ for tumorfile in args.tumorfiles:
+ updateLabels(collection, tumorfile, ffv_infos);
+ updateCategoryFeatures(collection, ffv_infos)
+ conn.close()
+
+ logging.info('finished add custom labels to feature matrix')
+
+if __name__ == '__main__':
+ main()
diff --git a/db/scripts/update_fm_with_ffn_lookup.js b/db/scripts/ffn_update_fm_with_ffn_lookup.js
similarity index 57%
rename from db/scripts/update_fm_with_ffn_lookup.js
rename to db/scripts/ffn_update_fm_with_ffn_lookup.js
index e7a2ede..af39598 100644
--- a/db/scripts/update_fm_with_ffn_lookup.js
+++ b/db/scripts/ffn_update_fm_with_ffn_lookup.js
@@ -8,19 +8,19 @@ function replaceUnderscore(str) {
var debugCLINCounts = [0, 0, 0, 0, 0, 0];
var twoReplace = "$3: $1 vs $2";
var oneReplace = "$2: $1";
-// In some cases, the string(s) in front of the “|” are not unique/specific enough and we will need to include the feature name after the “|” as follows
-// C:SAMP:I(A|X) ⇒ f(X) A
-// C:SAMP:I(A,B|X) ⇒ f(X) A vs B
+// In some cases, the string(s) in front of the '|' are not unique/specific enough and we will need to include the feature name after the '|' as follows
+// C:SAMP:I(A|X) ==> f(X) A
+// C:SAMP:I(A,B|X) ==> f(X) A vs B
var checks = [
-// cases where we have G1,G2,.. drop the “G” (redundant with the word “grade”)
+// cases where we have G1,G2,.. drop the 'G' (redundant with the word grade)
// B:CLIN:I(G1|neoplasm_histologic_grade)::::: => neoplasm histologic grade 1
[/I\(G([0-9]+),G([0-9]+)\|(.+)\)/m, twoReplace],
// B:CLIN:I(G1,G2|neoplasm_histologic_grade)::::: => neoplasm histologic grade 1 vs 2
[/I\(G([0-9]+)\|(.+)\)/m, oneReplace],
-// cases where we have C1,C2,... (these are cluster #s) -- drop the “C”
-// C:SAMP:I(C1,C2|X) ⇒ f(X) 1 vs 2
+// cases where we have C1,C2,... (these are cluster #s) -- drop the 'C'
+// C:SAMP:I(C1,C2|X) ==> f(X) 1 vs 2
[/I\(C([0-9]+),C([0-9]+)\|(.+)\)/m, twoReplace],
-// C:SAMP:I(C1|X) ⇒ f(X) 1
+// C:SAMP:I(C1|X) ==> f(X) 1
[/I\(C([0-9]+)\|(.+)\)/m, oneReplace],
// pathologic_T, pathologic_N, pathologic_M, WHO_class
[/I\((.+),(.+)\|(.+)\)/m, twoReplace],
@@ -91,116 +91,68 @@ function getLength(start, end) {
return Math.round(length) + units[units.length - 1];
}
-function getFMTitle(index, fields, suffix) {
- var fm_title = fields[0];
- for (var i = 1; i < index; i++) {
- fm_title += ' ' + fields[i];
- }
- return fm_title + ': ' + suffix;
-}
-
-function getGisticArmString(fields) {
- return getFMTitle(fields.indexOf('GisticArm'), fields, 'Gistic Arm');
-}
-
-function getGisticString(fields) {
- var gindex = fields.indexOf('Gistic');
- return getFMTitle(gindex, fields, 'Gistic') + ' ' + fields[gindex + 1] + ' ' + fields[fields.length - 1];
-}
-
-var debugCNVRCounts = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
-var noloc_checks = [
+var debugCNVRCounts = [0, 0, 0, 0, 0, 0];
+// Gistic and GisticArm features
// N:CNVR:Xq:chrX:60600000:155270560::SKCM-All_Lymph_Node_GisticArm_d
-// ⇒ Xq (SKCM-All Lymph Node: Gistic Arm)
+// ==> Xq (Gistic Arm)
// N:CNVR:Xq28:chrX:150021680:150280252::SKCM-All_Regional_Metastases_Gistic_ROI_r_amp
-// ⇒ Xq28 (SKCM-All_Regional_Metastases: Gistic ROI amp)
- /^x([pq]([0-9]{1,2})?)$/i,
-// N:CNVR:10p:chr10:0:40200000::GBM-TP_GisticArm_d
-// ⇒ 10p (GBM-TP: Gistic Arm)
- /^[0-9]{1,2}[pq]$/
-];
-var loc_checks = [
+// ==> Xq28 Amplification (Gistic, continuous)
+// N:CNVR:Xq28:chrX:150021680:150280252::SKCM-All_Regional_Metastases_Gistic_ROI_d_del
+// ==> Xq28 Deletion (Gistic, discrete)
+
+// Resegmented features
// N:CNVR:Xq22.2:chrX:60600000:155270560
-// ⇒ chrX:60,600,000-155,270,560 (95MB, Xq22.2)
- /^x([pq])/i,
+// ==> chrX:60,600,000-155,270,560 (95MB, Xq22.2)
// N:CNVR:8q24:chr8:138862000:140900999::
-// ⇒ chr8:138,862,000-140,900,999 (2MB, 8q24)
- /^[0-9]{1,2}[pq]/,
+// ==> chr8:138,862,000-140,900,999 (2MB, 8q24)
// N:CNVR:SRGAP2:chr1:158887000:158888999::
-// ⇒ chr1:158,887,000-158,888,999 (2kb, SRGAP2)
- /^[A-Z]/
-];
+// ==> chr1:158,887,000-158,888,999 (2kb, SRGAP2)
var getCNVRLabel = function(doc) {
- for (var i = 0; i < noloc_checks.length; i++) {
- if (noloc_checks[i].test(doc["locus"])) {
- if (doc["code"]) {
- var fields = doc["code"].split('_');
- if (-1 < doc["code"].indexOf('GisticArm')) {
- debugCNVRCounts[3 * i]++;
- return replaceUnderscore(doc["locus"] + ' (' + getGisticArmString(fields) + ')');
- } else {
- debugCNVRCounts[3 * i + 1]++;
- return replaceUnderscore(doc["locus"] + ' (' + getGisticString(fields) + ')');
- }
- } else {
- debugCNVRCounts[3 * i + 2]++;
- var loc = getLocation(doc)
- length = getLength(doc["start"], doc["end"]);
- return replaceUnderscore(loc + ' (' + length + ', ' + doc["locus"] + ')');
- }
- }
- }
-
- for (var i = 0; i < loc_checks.length; i++) {
- if (loc_checks[i].test(doc["locus"])) {
- var loc = getLocation(doc)
- if (doc["code"]) {
- var fields = doc["code"].split('_');
- if (-1 < doc["code"].indexOf('GisticArm')) {
- debugCNVRCounts[3 * (i + 2)]++;
- return replaceUnderscore(loc + ' (' + getGisticArmString(fields) + ', ' + doc["locus"] + ')');
- } else {
- debugCNVRCounts[3 * (i + 2) + 1]++;
- return replaceUnderscore(loc + ' (' + getGisticString(fields) + ', ' + doc["locus"] + ')');
- }
- } else {
- debugCNVRCounts[3 * (i + 2) + 2]++;
- var length = getLength(doc["start"], doc["end"]);
- return replaceUnderscore(loc + '(' + length + ', ' + doc["locus"] + ')');
- }
+ if (doc["code"]) {
+ if (-1 < doc["code"].indexOf('GisticArm')) {
+ debugCNVRCounts[0]++;
+ return replaceUnderscore(doc["locus"] + ' (Gistic Arm)');
+ } else if (-1 < doc["code"].indexOf('Gistic')) {
+ var fields = doc["code"].split('_');
+ var gType = fields[fields.length - 1] == 'del'? 'Deletion' : 'Amplification';
+ var index = fields[fields.length - 1] == 'del'? 0 : 1;
+ var dType = fields[fields.length - 2] == 'd'? 'discrete' : 'continuous';
+ index += fields[fields.length - 2] == 'd'? 3 : 5;
+ debugCNVRCounts[index - 2]++;
+ return replaceUnderscore(doc["locus"] + ' ' + gType + ' (Gistic, ' + dType + ')');
+ } else if (-1 < doc["code"].indexOf('LOH')) {
+ return replaceUnderscore(doc["locus"] + ' (LOH)');
+ } else {
+ return replaceUnderscore(doc["locus"] + ' (' + doc["code"] + ')');
}
+ } else {
+ debugCNVRCounts[5]++;
+ var loc = getLocation(doc)
+ length = getLength(doc["start"], doc["end"]);
+ return replaceUnderscore(loc + ' (' + length + ', ' + doc["locus"] + ')');
}
- print("\n!!!!!" + doc["id"] + " didn't match any expression. using default!!!!!\n");
- var loc = getLocation(doc);
- var length = getLength(doc["start"], doc["end"]);
- return replaceUnderscore(loc + '(' + length + ', ' + doc["locus"] + ')');
}
var of = /([0-9]+)of([0-9]+)/;
var getGEXPLabel = function(doc) {
var retVal = doc["gene"];
- if (doc["platform"] != "micro-array") {
- var tag = doc["id"].split(':')[7];
- if (tag) {
- if (of.test(tag)) {
- tag = tag.replace(of, "$1 of $2");
- }
- retVal += ' (' + tag + ')';
- }
+ var code = doc["id"].split(':')[7]
+ if (doc["platform"] == "mRNAseq" && of.test(code)) {
+ retVal += ' (' + code.replace(of, "$1 of $2") + ')';
}
return replaceUnderscore(retVal);
}
var getMIRNLabel = function(doc) {
- return replaceUnderscore(doc["microRNA"] + ' (' + doc["accession_number"] + ')');
+ return replaceUnderscore(doc["microRNA"]);
}
var mappings = new Object();
mappings['nonsilent_somatic'] = "Excluding Silent Mutations";
mappings['code_potential_somatic'] = "Protein Coding";
mappings['missense_somatic'] = "Missense";
-mappings['mnf_somatic'] = "MNF";
-mappings['mni_somatic'] = "MNI";
+mappings['mnf_somatic'] = "Missense-Nonsense-Frameshift";
+mappings['mni_somatic'] = "Missense-Nonsense-Inframe-Frameshift";
mappings['y_n_somatic'] = "All Mutations";
var getGNABLabel = function(doc) {
var name = doc["gene"];
@@ -212,22 +164,23 @@ var getGNABLabel = function(doc) {
}
var getMETHLabel = function(doc) {
- // TODO: get the list of genes (w/ corresponding distance from TSS?)
+ var gene = doc["id"].split(':')[2]
var fields = doc["code"].split('_');
+ var probe = fields[0];
var tf_loc = '';
if (1 < fields.length) {
tf_loc += fields[1];
for (var i = 2; i < fields.length; i++) {
- tf_loc += ' ';
+ tf_loc += ', ';
tf_loc += fields[i];
}
}
if (tf_loc) {
- return replaceUnderscore(doc["probe"] + ' (' + doc['id'].split(':')[2] + ', ' + tf_loc + ')');
+ return replaceUnderscore(gene + ' (' + tf_loc + ') ' + probe);
}
else {
- return replaceUnderscore(doc["code"] + ' (' + doc['id'].split(':')[2] + ')');
+ return replaceUnderscore(gene + ' ' + probe);
}
}
@@ -237,8 +190,6 @@ var endInfo2 = /^(.+)-[RMVCEG]$/
var noInfo = /(.+)\-NA/
var phosphor = /^(.+)[_\-]p([STY][0-9]+.*)$/
var getRPPALabel = function(doc) {
- // TODO: get the list of genes
- // remove any 'uninteresting' trailing strings
var antibody = doc["antibody"];
var matchEndInfo = endInfo.exec(antibody);
if (matchEndInfo) {
@@ -267,13 +218,13 @@ var getRPPALabel = function(doc) {
var gene = doc["id"].split(':')[2];
if (phos) {
var pfields = phos.split('_');
- phos = pfields[0];
+ phos = 'p' + pfields[0];
for (var i = 1; i < pfields.length; i++) {
- phos += ', ' + pfields[i];
+ phos += ', p' + pfields[i];
}
- return replaceUnderscore(gene + ': ' + antibody + ' (' + phos + ')');
+ return replaceUnderscore(gene + ' (' + phos + ')');
} else {
- return replaceUnderscore(gene + ': ' + antibody);
+ return replaceUnderscore(gene);
}
}
diff --git a/db/scripts/generate_clinical_variables_lookup.js b/db/scripts/generate_clinical_variables_lookup.js
deleted file mode 100644
index df3e898..0000000
--- a/db/scripts/generate_clinical_variables_lookup.js
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
-- Extracts clinical variables from "feature_matrix" collection into GLOBAL lookup collection
-
-Usage:
-
- mongo --host=$HOST $DB_NAME generate_clinical_variables_lookup.js --eval="var lookupsDbUri='hostname:port/LOOKUPS';"
- */
-
-var lookupsDb = connect(lookupsDbUri);
-var db_name = db["_name"];
-print("[" + db_name + "]:script:started");
-
-var group_variables_fn = function(doc) {
- var details = {
- "id": doc["id"],
- "label": doc["label"],
- "tumor_type": db_name,
- "source": doc["source"]
- };
- lookupsDb["all_clinical"].update({
- "id": doc["id"]
- }, {
- "$set": { "label": doc["label"] },
- "$push": { "features": details }
- }, true);
-};
-
-print("[" + db_name + "]:initial check=LOOKUPS:" + lookupsDb["all_clinical"].count());
-
-print("[" + db_name + "]:update:started:CLIN");
-db["feature_matrix"].find({ "source": "CLIN" }).forEach(group_variables_fn);
-print("[" + db_name + "]:update:started:SAMP");
-db["feature_matrix"].find({ "source": "SAMP" }).forEach(group_variables_fn);
-print("[" + db_name + "]:update:completed");
-
-var result = lookupsDb["all_clinical"].aggregate([ { "$unwind": "$features" }, { "$group": { "_id": "$features.tumor_type", "cnt": { "$sum": 1 } } } ]);
-result["result"].forEach(printjson);
-print("[" + db_name + "]:script:completed");
diff --git a/db/scripts/generate_featurematrix_tags_lookup.js b/db/scripts/generate_featurematrix_tags_lookup.js
deleted file mode 100644
index 669bf01..0000000
--- a/db/scripts/generate_featurematrix_tags_lookup.js
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
-- Extracts tags from "feature_matrix" collection into GLOBAL lookup collection
-
-Usage:
-
- mongo --host=$HOST $DB_NAME generate_featurematrix_tags_lookup.js --eval="var lookupsDbUri='hostname:port/LOOKUPS';"
- */
-
-var lookupsDb = connect(lookupsDbUri);
-var db_name = db["_name"];
-
-print("[" + db_name + "]:script:started");
-print("[" + db_name + "]:initial check=LOOKUPS:" + lookupsDb["all_tags"].count());
-
-print("[" + db_name + "]:update:started");
-db["feature_matrix"].find(
- {
- "$and": [
- { "tags": { "$exists": true } },
- { "tags": { "$nin": ["NO_MATCH"] } }
- ]
- },
- {
- "tags": true,
- "id": true
- }
-).forEach(function (doc) {
- if (!doc) return;
- var tags = doc["tags"] || [];
- if (tags.length <= 0) return;
-
- for (var i = 0; i < tags.length; i++) {
- var tag = tags[i];
- if (tag) {
- var details = { "id": doc["id"], "tumor_type": db_name };
- lookupsDb["all_tags"].update({ "tag": tag }, { "$push": { "features": details } }, true);
- }
- }
-});
-print("[" + db_name + "]:update:completed");
-
-print("[" + db_name + "]:final check=LOOKUPS:" + lookupsDb["all_tags"].count());
-print("[" + db_name + "]:script:completed");
diff --git a/db/scripts/generate_sample_types_lookup.js b/db/scripts/generate_sample_types_lookup.js
deleted file mode 100644
index 3723259..0000000
--- a/db/scripts/generate_sample_types_lookup.js
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
-- Extracts clinical variables from "feature_matrix" collection into GLOBAL lookup collection
-
-Usage:
-
- mongo --host=$HOST $DB_NAME generate_sample_types_lookup.js --eval="var lookupsDbUri='hostname:port/LOOKUPS';"
- */
-
-var query = { "id": "C:SAMP:TNtype:::::" };
-var projection = { "_id": false };
-var tumor_type = db["name"];
-
-var lookupsDb = connect(lookupsDbUri);
-
-print("initial check=" + tumor_type + ":" + db.feature_matrix.find(query, projection).count());
-print("initial check=LOOKUPS:" + lookupsDb.sample_types.count());
-
-var eachFn = function(doc) {
- doc["tumor_type"] = tumor_type;
- lookupsDb.sample_types.insert(doc);
-};
-
-db.feature_matrix.find(query, projection).forEach(eachFn);
-
-print("final check=LOOKUPS:" + lookupsDb.sample_types.count());
-print("completed");
diff --git a/db/scripts/generate_stats_var_lookup.js b/db/scripts/generate_stats_var_lookup.js
index 59a7459..b1791ff 100644
--- a/db/scripts/generate_stats_var_lookup.js
+++ b/db/scripts/generate_stats_var_lookup.js
@@ -1,125 +1,113 @@
/*
- usage: mongo :/ fn_2_ffn_clin.js
+ usage: mongo :/ generate_stats_var_lookup.js
*/
-function getCounts(values) {
- var retVal = {};
- retVal["total"] = 0;
- retVal["valid"] = 0;
-
- for (var key in values) {
- retVal["total"] += 1;
- if (values[key] != 'NA') {
- retVal["valid"]++;
- }
- }
-
- return retVal;
+function median(values) {
+ values.sort(function(a, b){return a-b});
+ var half = Math.floor(values.length/2);
+ if(values.length % 2)
+ retVal = values[half];
+ else
+ retVal = (values[half-1] + values[half]) / 2.0;
+ return retVal;
}
function getStats(type, values) {
var count = 0;
- var retVal = [{}, {}, {}];
- retVal[0]["total"] = 0;
- retVal[0]["valid"] = 0;
+ var retVal = new Object();
+ retVal["counts"] = new Object();
+ retVal["counts"]["total"] = 0;
+ retVal["counts"]["valid"] = 0;
+
+ retVal["categories"] = new Object();
- retVal[2]["max"] = Number.MIN_VALUE;
- retVal[2]["min"] = Number.MAX_VALUE;
- retVal[2]["mean"] = 0;
- retVal[2]["median"] = new Array();
+ retVal["numeric"] = new Object();
+ retVal["numeric"]["max"] = -Number.MAX_VALUE;
+ retVal["numeric"]["min"] = Number.MAX_VALUE;
+ retVal["numeric"]["mean"] = 0;
+ retVal["numeric"]["median"] = new Array();
for (var key in values) {
- retVal[0]["total"]++;
+ retVal["counts"]["total"]++;
if (values[key] != 'NA') {
- retVal[0]["valid"]++;
- }
+ retVal["counts"]["valid"]++;
- if (values[key] in retVal[1]) {
- retVal[1][values[key]]++;
- } else {
- count++;
- retVal[1][values[key]] = 1;
+ if (values[key] in retVal["categories"]) {
+ retVal["categories"][values[key]]++;
+ } else {
+ count++;
+ retVal["categories"][values[key]] = 1;
+ }
}
if ("N" == type && "NA" != values[key]) {
- if (retVal[2]["max"] < values[key]) {
- retVal[2]["max"] = values[key];
+ if (retVal["numeric"]["max"] < values[key]) {
+ retVal["numeric"]["max"] = values[key];
}
- if (retVal[2]["min"] > values[key]) {
- retVal[2]["min"] = values[key];
+ if (retVal["numeric"]["min"] > values[key]) {
+ retVal["numeric"]["min"] = values[key];
}
- retVal[2]["mean"] += values[key];
- retval[2]["median"].push(values[key]);
+ retVal["numeric"]["mean"] += values[key];
+ retVal["numeric"]["median"].push(values[key]);
}
}
-
+
if (!((("C" == type || "B" == type) && count > 0) || (count > 0 && count < 21))) {
- retVal[1] = null;
+ retVal["categories"] = null;
}
-
- if ("N" != type || 0 == retVal[0]["valid"]) {
- retVal[2] = null;
+
+ if ("N" != type || 0 == retVal["counts"]["valid"]) {
+ retVal["numeric"] = null;
} else {
- retVal[2]["mean"] = retVal[2]["mean"] / retVal[0]["valid"];
+ retVal["numeric"]["mean"] = retVal["numeric"]["mean"] / retVal["counts"]["valid"];
- retVal[2]["stddev"] = 0;
+ retVal["numeric"]["stddev"] = 0;
for (var key in values) {
if (values[key] != 'NA') {
- retVal[2]["stddev"] += Math.pow((values[key] - retVal[2]["mean"]), 2);
+ retVal["numeric"]["stddev"] += Math.pow((values[key] - retVal["numeric"]["mean"]), 2);
}
}
- retVal[2]["stddev"] = Math.sqrt(retVal[2]["stddev"] / retVal[0]["valid"]);
+ retVal["numeric"]["stddev"] = Math.sqrt(retVal["numeric"]["stddev"] / retVal["counts"]["valid"]);
- retVal[2]["median"].sort(function(a, b){return a-b});
- retVal[2]["mad"] = retVal[2]["median"];
- var half = Math.floor(retVal[2]["median"].length/2);
- if(retVal[2]["median"].length % 2)
- retVal[2]["median"] = retVal[2]["median"][half];
- else
- retVal[2]["median"] = (retVal[2]["median"][half-1] + retVal[2]["median"][half]) / 2.0;
+ retVal["numeric"]["mad"] = retVal["numeric"]["median"];
+ retVal["numeric"]["median"] = median(retVal["numeric"]["median"]);
- for (i = 0; i < retVal[2]["mad"].length; i++) {
- retVal[2]["mad"][i] = Math.abs(retVal[2]["mad"][i] - retVal[2]["median"]);
- retVal[2]["mad"].sort(function(a, b){return a-b});
- var half = Math.floor(retVal[2]["mad"].length/2);
- if(retVal[2]["mad"].length % 2)
- retVal[2]["mad"] = retVal[2]["mad"][half];
- else
- retVal[2]["mad"] = (retVal[2]["mad"][half-1] + retVal[2]["mad"][half]) / 2.0;
+ for (i = 0; i < retVal["numeric"]["mad"].length; i++) {
+ retVal["numeric"]["mad"][i] = Math.abs(retVal["numeric"]["mad"][i] - retVal["numeric"]["median"]);
}
-
+ retVal["numeric"]["mad"] = median(retVal["numeric"]["mad"]);
}
-
return retVal;
};
print('start: ' + new Date());
var count = 0;
-db.getSiblingDB("skcm_stats_lookup_test").dropDatabase();
-db.getSiblingDB("skcm_stats_lookup_test").fm_stats.ensureIndex({"id": 1}, {unique: true});
db.feature_matrix.find().forEach(
function(doc) {
if (0 == (count++ % 4096)) {
print("processing record " + count + " " + new Date());
}
- sibDB = db.getSiblingDB("skcm_stats_lookup_test");
- var stats = getStats(doc["type"], doc["values"]);
- var entry = {
- "id":doc["id"],
- "counts":stats[0]
+ if (doc["raw_values"]) {
+ var stats = getStats(doc["type"], doc["raw_values"]);
+ db.feature_matrix.update({"id":doc["id"]}, {$set: {"raw_statistics.counts": stats["counts"]}});
+ if (stats["categories"]) {
+ db.feature_matrix.update({"id":doc["id"]}, {$set: {"raw_statistics.categories":stats["categories"]}})
+ }
+ if (stats["numeric"]) {
+ db.feature_matrix.update({"id":doc["id"]}, {$set: {"raw_statistics.numeric":stats["numeric"]}})
+ }
}
- sibDB.fm_stats.insert(entry);
- var cats = stats[1];
- if (null != cats) {
- sibDB.fm_stats.update({"id":doc["id"]}, {$set: {"categories":cats}})
+ var stats = getStats(doc["type"], doc["values"]);
+ db.feature_matrix.update({"id":doc["id"]}, {$set: {"statistics.counts": stats["counts"]}});
+ if (stats["categories"]) {
+ db.feature_matrix.update({"id":doc["id"]}, {$set: {"statistics.categories":stats["categories"]}})
}
-
- var numeric = stats[2];
- if (null != numeric) {
- sibDB.fm_stats.update({"id":doc["id"]}, {$set: {"numeric":numeric}})
+ if (stats["numeric"]) {
+ db.feature_matrix.update({"id":doc["id"]}, {$set: {"statistics.numeric":stats["numeric"]}})
}
}
)
+print('total count: ' + count);
print('end: ' + new Date());
diff --git a/db/scripts/lookups_aggregate_clinical_variables.js b/db/scripts/lookups_aggregate_clinical_variables.js
new file mode 100644
index 0000000..7fa33e1
--- /dev/null
+++ b/db/scripts/lookups_aggregate_clinical_variables.js
@@ -0,0 +1,45 @@
+/*
+- Extracts clinical variables from "feature_matrix" collection into GLOBAL lookup collection
+
+Usage:
+
+ mongo --host=$HOST $DB_NAME lookups_aggregate_clinical_variables.js --eval="var lookups_db_uri='hostname:port/LOOKUPS';"
+ */
+
+var lookups_db = connect(lookups_db_uri);
+var db_name = db["_name"];
+var pretty_print = function(msg) {
+ var dtFmt = (new Date()).toLocaleFormat("%Y-%m-%d %H:%M:%S,000");
+ print("[INFO] " + dtFmt + " - lookups_aggregate_clinical_variables(" + db_name + ") - " + msg);
+};
+
+pretty_print("START");
+
+var group_variables_fn = function(doc) {
+ var details = {
+ "id": doc["id"],
+ "label": doc["label"],
+ "tumor_type": db_name,
+ "source": doc["source"]
+ };
+ lookups_db["all_clinical"].update({
+ "id": doc["id"]
+ }, {
+ "$set": { "label": doc["label"] },
+ "$push": { "features": details }
+ }, true);
+};
+
+pretty_print("initial check=LOOKUPS:" + lookups_db["all_clinical"].count());
+
+pretty_print("update:started:CLIN");
+db["feature_matrix"].find({ "source": "CLIN" }).forEach(group_variables_fn);
+
+pretty_print("update:started:SAMP");
+db["feature_matrix"].find({ "source": "SAMP" }).forEach(group_variables_fn);
+
+pretty_print("update:completed");
+
+var result = lookups_db["all_clinical"].aggregate([ { "$unwind": "$features" }, { "$group": { "_id": "$features.tumor_type", "cnt": { "$sum": 1 } } } ]);
+result["result"].forEach(printjson);
+pretty_print("COMPLETED");
diff --git a/db/scripts/lookups_aggregate_sample_types.js b/db/scripts/lookups_aggregate_sample_types.js
new file mode 100644
index 0000000..5f6b532
--- /dev/null
+++ b/db/scripts/lookups_aggregate_sample_types.js
@@ -0,0 +1,33 @@
+/*
+- Extracts clinical variables from "feature_matrix" collection into GLOBAL lookup collection
+
+Usage:
+
+ mongo --host=$HOST $DB_NAME lookups_aggregate_sample_types.js --eval="var lookups_db_uri='hostname:port/LOOKUPS';"
+ */
+
+var lookups_db = connect(lookups_db_uri);
+var db_name = db["_name"];
+var query = { "id": "C:SAMP:TNtype:::::" };
+
+var pretty_print = function(msg) {
+ var dtFmt = (new Date()).toLocaleFormat("%Y-%m-%d %H:%M:%S,000");
+ print("[INFO] " + dtFmt + " - lookups_aggregate_sample_types(" + db_name + ") - " + msg);
+};
+
+pretty_print("lookups_db=" + lookups_db_uri + ":" + lookups_db);
+pretty_print("initial check=" + db["feature_matrix"].find(query).count());
+pretty_print("initial check=LOOKUPS:" + lookups_db["sample_types"].count());
+
+db["feature_matrix"].find(query).forEach(function(doc) {
+ lookups_db["sample_types"].insert({
+ "id": doc["id"],
+ "label": doc["label"],
+ "source": doc["source"],
+ "values": doc["values"],
+ "tumor_type": db_name
+ });
+});
+
+pretty_print("final check=LOOKUPS:" + lookups_db["sample_types"].count());
+pretty_print("COMPLETED");
diff --git a/db/scripts/misc_aggregate_collection_fields.js b/db/scripts/misc_aggregate_collection_fields.js
new file mode 100644
index 0000000..9302fd3
--- /dev/null
+++ b/db/scripts/misc_aggregate_collection_fields.js
@@ -0,0 +1,54 @@
+/*
+ - Reads all collections in the database, and extract all fields, preparing a lookup collection
+ - Writes to "collection_fields" collection
+
+ - Documents in collection will look like this:
+ { "_id" : , "value" : }
+
+ Usage:
+
+ mongo --host=$HOST $DB_NAME mapreduce_collection_fields.js
+
+ // TODO : Organize output better
+ */
+
+var mapFn = function () {
+ for (var field in this) {
+ if (field === "_id") continue;
+ emit(field, null);
+ }
+};
+
+var reduceFn = function (field) {
+ return field;
+};
+
+var finalizeFn = function() {
+ return target_collection;
+};
+
+var db_name = db["_name"];
+var pretty_print = function(msg) {
+ var dtFmt = (new Date()).toLocaleFormat("%Y-%m-%d %H:%M:%S,000");
+ print("[INFO] " + dtFmt + " - misc_aggregate_collection_fields(" + db_name + ") - " + msg);
+};
+
+db["collection_fields"].drop();
+
+db.getCollectionNames().forEach(function (collection_name) {
+ if (collection_name === "system.indexes") return;
+ if (collection_name === "collection_fields") return;
+
+ pretty_print(collection_name);
+ db[collection_name].mapReduce(mapFn, reduceFn, {
+ "out": {
+ "merge": "collection_fields"
+ },
+ "scope": {
+ "target_collection": collection_name
+ },
+ "finalize": finalizeFn
+ });
+});
+
+pretty_print("unique fields captured:" + db["collection_fields"].count());
diff --git a/db/scripts/extract_fmx.js b/db/scripts/misc_extract_fmx.js
similarity index 87%
rename from db/scripts/extract_fmx.js
rename to db/scripts/misc_extract_fmx.js
index c6215d1..00751f6 100644
--- a/db/scripts/extract_fmx.js
+++ b/db/scripts/misc_extract_fmx.js
@@ -47,7 +47,7 @@ var outputFn = function (selected_samples) {
var query = { "source": feature_source };
// collects a limited set of samples
-db.feature_matrix.find(query).limit(record_count).forEach(collectFn(sample_count, selected_samples));
+db["feature_matrix"].find(query).limit(record_count).forEach(collectFn(sample_count, selected_samples));
// outputs data for the selected samples
-db.feature_matrix.find(query).limit(record_count).forEach(outputFn(selected_samples));
\ No newline at end of file
+db["feature_matrix"].find(query).limit(record_count).forEach(outputFn(selected_samples));
\ No newline at end of file
diff --git a/db/scripts/misc_fmx_adjustvalues_GEXP_STAD.js b/db/scripts/misc_fmx_adjustvalues_GEXP_STAD.js
new file mode 100644
index 0000000..9534fbf
--- /dev/null
+++ b/db/scripts/misc_fmx_adjustvalues_GEXP_STAD.js
@@ -0,0 +1,41 @@
+/*
+Usage:
+ mongo --host=$HOST --port=$PORT STAD featurematrix_adjustvalues_GEXP_STAD.js
+*/
+
+// migration phase
+// moves 'values' into 'raw_values'
+// run only once
+var exists_query = { "source": "GEXP", "raw_values": { "$exists": false }, "values": { "$exists": true } };
+var count = db["feature_matrix"].find(exists_query).count();
+if (count > 0) {
+ db["feature_matrix"].update(exists_query, {"$rename": { "values": "raw_values"}}, { "multi": true });
+}
+
+// adjustment phase
+var CONSTANT_ADDITION = 4.550681;
+var CONSTANT_MULTIPLY = 1.271340;
+
+var adjust_query = { "source": "GEXP", "raw_values": { "$exists": true } };
+db["feature_matrix"].find(adjust_query).forEach(function(rec) {
+ var dict_raw_values = rec["raw_values"];
+ var dict_values = {};
+ for (var sample_id in dict_raw_values) {
+ var raw_value = dict_raw_values[sample_id];
+ if (0 != raw_value && (!raw_value || raw_value == "")) {
+ throw sample_id + "(" + raw_value + ") didn't have a valid value for " + rec["id"];
+ }
+ if (String(raw_value).toUpperCase() != "NA") {
+ var float_value = parseFloat(raw_value);
+ if (!isNaN(float_value)) {
+ float_value = (float_value * CONSTANT_MULTIPLY) + CONSTANT_ADDITION;
+ dict_values[sample_id] = float_value;
+ } else {
+ dict_values[sample_id] = "NA";
+ }
+ } else {
+ dict_values[sample_id] = "NA";
+ }
+ }
+ db["feature_matrix"].update({ "_id": rec["_id"] }, { "$set": { "values": dict_values }});
+});
\ No newline at end of file
diff --git a/db/scripts/rename_protein_field_in_fmx.js b/db/scripts/misc_rename_protein_field_fmx.js
similarity index 100%
rename from db/scripts/rename_protein_field_in_fmx.js
rename to db/scripts/misc_rename_protein_field_fmx.js
diff --git a/db/scripts/insert_mutationsummary_mongodb.py b/db/scripts/mutationsummary_insert.py
similarity index 100%
rename from db/scripts/insert_mutationsummary_mongodb.py
rename to db/scripts/mutationsummary_insert.py
diff --git a/db/scripts/mutsigrankings_insert.py b/db/scripts/mutsigrankings_insert.py
new file mode 100755
index 0000000..8800ba9
--- /dev/null
+++ b/db/scripts/mutsigrankings_insert.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+
+# this file parses the mutsig output from Firehose, rank is derived from order in file
+
+import argparse
+import csv
+import os
+import pymongo
+import logging
+
+from utilities import configure_logging
+
+# example data:
+# gene Nnon Nsil Nflank nnon npat nsite nsil nflank nnei fMLE p score time q
+# CDKN1A 47190 15860 0 18 18 17 0 0 20 1.478287e+00 3.663736e-15 9.868049e+01 1.285760e-01 4.559103e-11
+# TP53 122850 35880 0 75 64 50 1 0 4 1.600534e+00 4.996004e-15 1.827995e+02 2.969270e-01 4.559103e-11
+# RB1 370240 97630 0 19 17 17 0 0 20 6.902772e-01 1.854072e-14 8.167202e+01 1.402310e-01 1.038462e-10
+# ARID1A 580710 171080 0 38 32 36 2 0 2 1.146468e+00 2.275957e-14 1.301110e+02 1.304470e-01 1.038462e-10
+# MLL2 1376830 455130 0 40 36 40 5 0 20 1.078227e+00 9.714451e-14 1.320917e+02 2.426840e-01 3.545969e-10
+# KDM6A 402090 110110 0 32 31 26 2 0 1 1.366207e+00 2.076339e-12 1.326493e+02 1.375940e-01 6.315877e-09
+# ELF3 115570 31720 0 15 11 14 0 0 20 1.089406e+00 2.276290e-10 5.431453e+01 1.243460e-01 5.934939e-07
+
+def extract_rows(file_path):
+ with open(file_path, "rb") as csvfile:
+ csvreader = csv.reader(csvfile, delimiter="\t")
+ headers = csvreader.next()
+
+ rank = 0
+ for line in csvreader:
+ rank += 1
+ row_obj = { "rank": rank }
+ for idx in range(0, len(headers)):
+ header = headers[idx]
+ value = line[idx]
+ if rank % 100 == 0: logging.debug("line[%s]:%s:%s:%s" % (rank, idx, header, value))
+ row_obj[header] = value
+ yield row_obj
+
+def main():
+ parser = argparse.ArgumentParser(description="Utility to import TCGA Firehose MutSig to MongoDB")
+ parser.add_argument("--host", required=True, help="MongoDB host name")
+ parser.add_argument("--port", required=True, type=int, help="MongoDB port")
+ parser.add_argument("--db", required=True, help="Database name")
+ parser.add_argument("--f", required=True, help="Path to mutsig sig-genes file")
+ parser.add_argument("--loglevel", default="INFO", help="Logging Level")
+ args = parser.parse_args()
+
+ configure_logging(args.loglevel.upper())
+
+ logging.info("import file: %s" % args.f)
+ logging.info("uploading to %s:%s/%s" % (args.host, args.port, args.db))
+
+ conn = pymongo.Connection(args.host, args.port)
+ collection = conn[args.db]["mutsig_rankings"]
+
+ logging.info("dropping collection")
+ collection.drop()
+
+ for row in extract_rows(args.f):
+ collection.insert(row)
+
+ logging.info("inserted count=%s" % collection.count())
+ conn.close()
+
+if __name__ == "__main__":
+ main()
+
diff --git a/db/scripts/mutsigrankings_top20.js b/db/scripts/mutsigrankings_top20.js
new file mode 100644
index 0000000..e885ca0
--- /dev/null
+++ b/db/scripts/mutsigrankings_top20.js
@@ -0,0 +1,22 @@
+/*
+- Extracts top20 ranked genes from "mutsig_rankings" collection into GLOBAL lookup collection
+
+Usage:
+
+ mongo --host=$HOST $DB_NAME mutsigrankings_top20.js
+ */
+
+var db_name = db["_name"];
+var pretty_print = function(msg) {
+ var dtFmt = (new Date()).toLocaleFormat("%Y-%m-%d %H:%M:%S,000");
+ print("[INFO] " + dtFmt + " - mutsigrankings_top20(" + db_name + ") - " + msg);
+};
+
+pretty_print("START");
+
+db["mutsig_rankings_top20"].drop();
+db["mutsig_rankings"].find({ "rank": { "$lte": 20 } }).forEach(function(doc) {
+ db["mutsig_rankings_top20"].insert(doc);
+});
+
+pretty_print("COMPLETED");
diff --git a/db/scripts/utilities.py b/db/scripts/utilities.py
index 2cd20d0..36bf972 100644
--- a/db/scripts/utilities.py
+++ b/db/scripts/utilities.py
@@ -8,7 +8,7 @@ def configure_logging(logging_level=logging.DEBUG):
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging_level)
- formatter = logging.Formatter("[%(levelname)s] %(asctime)s - %(message)s")
+ formatter = logging.Formatter("[%(levelname)s] %(asctime)s - %(module)s.%(funcName)s - %(message)s")
ch.setFormatter(formatter)
root.addHandler(ch)
\ No newline at end of file
diff --git a/db/scripts/verify_entries.js b/db/scripts/verify_entries.js
deleted file mode 100644
index 0b12db1..0000000
--- a/db/scripts/verify_entries.js
+++ /dev/null
@@ -1,44 +0,0 @@
-var exists = { "$exists": true };
-var not_exists = { "$exists": false };
-
-var cnt_gexp_exists = db.feature_matrix.find({ "gene": exists, "source":"GEXP" }).count();
-var cnt_gexp_not_exists = db.feature_matrix.find({ "gene": not_exists, "source":"GEXP" }).count();
-
-var cnt_gnab_exists = db.feature_matrix.find({ "gene": exists, "source":"GNAB" }).count();
-var cnt_gnab_not_exists = db.feature_matrix.find({ "gene": not_exists, "source":"GNAB" }).count();
-
-var cnt_rppa_antibody_exists = db.feature_matrix.find({ "antibody": exists, "source":"RPPA" }).count();
-var cnt_rppa_antibody_not_exists = db.feature_matrix.find({ "antibody": not_exists, "source":"RPPA" }).count();
-
-var cnt_rppa_exists = db.feature_matrix.find({ "refGenes": exists, "source":"RPPA" }).count();
-var cnt_rppa_not_exists = db.feature_matrix.find({ "refGenes": not_exists, "source":"RPPA" }).count();
-
-var cnt_meth_probe_exists = db.feature_matrix.find({ "probe": exists, "source":"METH" }).count();
-var cnt_meth_probe_not_exists = db.feature_matrix.find({ "probe": not_exists, "source":"METH" }).count();
-
-var cnt_meth_exists = db.feature_matrix.find({ "refGenes": exists, "source":"METH" }).count();
-var cnt_meth_not_exists = db.feature_matrix.find({ "refGenes": not_exists, "source":"METH" }).count();
-
-var cnt_mirn_exists = db.feature_matrix.find({ "microRNA": exists, "source":"MIRN" }).count();
-var cnt_mirn_not_exists = db.feature_matrix.find({ "microRNA": not_exists, "source":"MIRN" }).count();
-
-var cnt_cnvr_exists = db.feature_matrix.find({ "refGenes": exists, "source":"CNVR" }).count();
-var cnt_cnvr_not_exists = db.feature_matrix.find({ "refGenes": not_exists, "source":"CNVR" }).count();
-
-var cnt_tags_exists = db.feature_matrix.find({ "tags": exists }).count();
-var cnt_tags_not_exists = db.feature_matrix.find({ "tags": not_exists }).count();
-
-var cnt_label_exists = db.feature_matrix.find({ "label": exists }).count();
-var cnt_label_not_exists = db.feature_matrix.find({ "label": not_exists }).count();
-
-print(":: ['count exists'/'count not exists']");
-print(db + ":GEXP:gene [" + cnt_gexp_exists + "/" + cnt_gexp_not_exists + "]");
-print(db + ":GNAB:gene [" + cnt_gnab_exists + "/" + cnt_gnab_not_exists + "]");
-print(db + ":RPPA:antibody [" + cnt_rppa_antibody_exists + "/" + cnt_rppa_antibody_not_exists + "]");
-print(db + ":RPPA:refGenes [" + cnt_rppa_exists + "/" + cnt_rppa_not_exists + "]");
-print(db + ":METH:probe [" + cnt_meth_probe_exists + "/" + cnt_meth_probe_not_exists + "]");
-print(db + ":METH:refGenes [" + cnt_meth_exists + "/" + cnt_meth_not_exists + "]");
-print(db + ":MIRN:microRNA [" + cnt_mirn_exists + "/" + cnt_mirn_not_exists + "]");
-print(db + ":CNVR:refGenes [" + cnt_cnvr_exists + "/" + cnt_cnvr_not_exists + "]");
-print(db + ":*:tags [" + cnt_tags_exists + "/" + cnt_tags_not_exists + "]");
-print(db + ":*:label [" + cnt_label_exists + "/" + cnt_label_not_exists + "]");
\ No newline at end of file