Figure3-4-5 Naive/IGS/FGS performance fgs

.gitignore

@@ -32,6 +32,10 @@ data/resources/conservation/*
 # Benchmark results
 results/
 
+# Generated manuscript assets
+manuscript_assets/figure*/
+manuscript_assets/tables/
+
 # GEO pipeline outputs
 pipelines/geo/configs/*
 !pipelines/geo/configs/*.example.yaml

README.md

@@ -251,12 +251,13 @@ During evaluation, DE rows are first restricted to Ensembl protein-coding genes
 release 115. This keeps the evaluation universe aligned with mRNA target predictors and avoids
 penalizing tools for non-coding genes they are not designed to score. Each predictor is then
 scored only on miRNA-gene pairs that exist in that predictor's standardized file. Missing pairs
-are not filled with zero for metrics. Each run writes coverage information to
+are not filled with zero for metrics or rank plots. Each run writes coverage information to
 `tables/per_experiment/coverage_per_experiment.tsv`, and the per-predictor Markdown/PDF reports
 also record total rows, scored rows, missing rows, and coverage. For per-dataset heatmaps and agreement plots,
-FuNmiRBench uses a dataset-local tie-aware rank over the scored rows. For cross-dataset
-rank-distribution plots, it keeps a separate global tie-aware rank derived from each predictor's
-full standardized file. Predictor-agreement top fractions use an exact top-k selection per
+FuNmiRBench uses a dataset-local rank over the scored rows only, normalized so 0 is the weakest
+scored pair and 1 is the strongest scored pair; tied scores receive average rank. For cross-dataset
+rank-distribution plots, it keeps a separate global average-tie rank derived from each predictor's
+full standardized file, again using only scored pairs. Predictor-agreement top fractions use an exact top-k selection per
 predictor with a deterministic tie-break instead of a quantile threshold. Combined PR, ROC, and
 GSEA comparison plots are computed on the common set of genes scored by all compared predictors.
 

funmirbench/benchmark_reports.py

@@ -287,8 +287,8 @@ def write_run_readme(
             " (exact top-k per predictor, deterministic tie-break)"
         ),
         "- Score handling: predictors are first aligned so that higher always means stronger",
-        "- Per-dataset heatmaps and agreement plots: dataset-local tie-aware dense ranking over scored rows",
-        "- Cross-dataset rank-distribution plots: global tie-aware dense ranking over each predictor's full standardized file",
+        "- Per-dataset heatmaps and agreement plots: dataset-local average-tie ranking over scored rows",
+        "- Cross-dataset rank-distribution plots: global average-tie ranking over scored rows from each predictor's full standardized file",
         "- Combined PR/ROC/GSEA comparison plots: computed on the common set of genes scored by all compared predictors",
     ]
     if protein_coding_filter and protein_coding_filter.get("enabled"):
@@ -561,7 +561,7 @@ def add_block(ax, title, lines, *, x, y, width):
             [
                 f"GT positives: {describe_gt_rule(fdr_threshold, effect_threshold)}",
                 "Predictor scores are aligned so that higher always means stronger before evaluation.",
-                "Per-dataset heatmaps and agreement plots use dataset-local tie-aware dense ranks.",
+                "Per-dataset heatmaps and agreement plots rank only scored pairs; tied scores use average rank.",
                 "Combined PR/ROC/GSEA plots use only the common set of genes scored by all compared predictors.",
                 "Top-prediction effect CDFs are optional diagnostics and are not written by default.",
             ],
@@ -776,7 +776,8 @@ def add_block(ax, title, lines, *, x, y, width):
             "positive_background_local_rank_distributions": (
                 "Positive vs background local rank distributions",
                 "Dataset-local rank distributions aggregated across datasets, split into GT positives and background genes. "
-                "Stronger predictors should push positives higher than background."
+                "Ranks are computed only among pairs scored by each predictor; missing scores are not set to zero, "
+                "and tied scores use average rank."
             ),
             "positive_background_local_rank_counts": (
                 "Positive vs background local rank counts",
@@ -786,7 +787,8 @@ def add_block(ax, title, lines, *, x, y, width):
             "positive_background_global_rank_distributions": (
                 "Positive vs background global rank distributions",
                 "Predictor-global rank distributions aggregated across datasets, split into GT positives and background genes. "
-                "This keeps each predictor on the rank scale of its full standardized file."
+                "Ranks use the predictor's full standardized file among scored pairs; missing scores are not set to zero, "
+                "and tied scores use average rank."
             ),
             "positive_background_global_rank_counts": (
                 "Positive vs background global rank counts",

funmirbench/cross_dataset.py

@@ -150,15 +150,15 @@ def _rank_distribution_metadata(rank_type):
             "title": "Positive vs background local rank distributions",
             "axis_label": "Local rank within dataset",
             "subtitle": (
-                "Dense ranks are dataset-local; GT positives use predictor colors."
+                "Ranks are dataset-local among scored pairs; ties use average rank."
             ),
         }
     if rank_type == "global":
         return {
             "title": "Positive vs background global rank distributions",
             "axis_label": "Global rank across predictor file",
             "subtitle": (
-                "Dense ranks use each full predictor file; GT positives use predictor colors."
+                "Ranks use each full predictor file among scored pairs; ties use average rank."
             ),
         }
     raise ValueError(f"Unsupported rank distribution type: {rank_type}")

funmirbench/evaluate_common.py

@@ -340,13 +340,14 @@ def _annotate_ground_truth(df, *, perturbation=None):
 
 def _rank_scale_scores(series):
     values = series.astype(float)
-    ranks = values.rank(method="dense", ascending=True)
+    ranks = values.rank(method="average", ascending=True)
+    min_rank = ranks.min(skipna=True)
     max_rank = ranks.max(skipna=True)
-    if pd.isna(max_rank):
+    if pd.isna(min_rank) or pd.isna(max_rank):
         return pd.Series(float("nan"), index=series.index)
-    if float(max_rank) <= 1.0:
+    if float(max_rank) <= float(min_rank):
         return pd.Series(1.0, index=series.index, dtype=float)
-    return (ranks - 1.0) / (float(max_rank) - 1.0)
+    return (ranks - float(min_rank)) / (float(max_rank) - float(min_rank))
 
 
 def _tool_id_from_score_col(score_col):

funmirbench/join.py

@@ -41,24 +41,34 @@ def _elapsed(start: float) -> float:
 
 def _compute_global_rank_percentile(series: pd.Series) -> pd.Series:
     values = series.astype(float)
-    ranks = values.rank(method="dense", ascending=True)
+    ranks = values.rank(method="average", ascending=True)
+    min_rank = ranks.min(skipna=True)
     max_rank = ranks.max(skipna=True)
-    if pd.isna(max_rank):
+    if pd.isna(min_rank) or pd.isna(max_rank):
         return pd.Series(float("nan"), index=series.index)
-    if float(max_rank) <= 1.0:
+    if float(max_rank) <= float(min_rank):
         return pd.Series(1.0, index=series.index, dtype=float)
-    return (ranks - 1.0) / (float(max_rank) - 1.0)
+    return (ranks - float(min_rank)) / (float(max_rank) - float(min_rank))
 
 
 def _global_rank_percentile_map(scores: pd.Series) -> dict[float, float]:
     values = pd.to_numeric(scores, errors="coerce").dropna().astype(float)
     if values.empty:
         return {}
-    ordered = pd.Series(values.unique()).sort_values(kind="mergesort").reset_index(drop=True)
-    if len(ordered) <= 1:
-        return {float(value): 1.0 for value in ordered}
-    denominator = float(len(ordered) - 1)
-    return {float(value): float(index / denominator) for index, value in enumerate(ordered)}
+    ranks = values.rank(method="average", ascending=True)
+    min_rank = ranks.min(skipna=True)
+    max_rank = ranks.max(skipna=True)
+    if pd.isna(min_rank) or pd.isna(max_rank):
+        return {}
+    if float(max_rank) <= float(min_rank):
+        return {float(value): 1.0 for value in values.unique()}
+    normalized = (ranks - float(min_rank)) / (float(max_rank) - float(min_rank))
+    return {
+        float(score): float(rank)
+        for score, rank in pd.DataFrame(
+            {"score": values, "rank": normalized}
+        ).drop_duplicates("score").itertuples(index=False)
+    }
 
 
 def _normalize_scores(scores: pd.Series, *, score_direction: str, tool_id: str) -> pd.Series:

funmirbench/run_report_pdf.py

@@ -348,19 +348,19 @@ def _plot_items(combined_outputs):
         {
             "positive_background_local_rank_distributions": (
                 "Local Rank Distributions",
-                "GT positives and background genes by dataset-local rank.",
+                "GT positives and background genes by dataset-local rank among scored pairs; tied scores use average rank.",
             ),
             "positive_background_local_rank_counts": (
                 "Local Rank Counts",
-                "Binned gene counts by dataset-local rank.",
+                "Binned gene counts by dataset-local rank among scored pairs.",
             ),
             "positive_background_global_rank_distributions": (
                 "Global Rank Distributions",
-                "GT positives and background genes by predictor-file rank.",
+                "GT positives and background genes by predictor-file rank among scored pairs; tied scores use average rank.",
             ),
             "positive_background_global_rank_counts": (
                 "Global Rank Counts",
-                "Binned gene counts by predictor-file rank.",
+                "Binned gene counts by predictor-file rank among scored pairs.",
             ),
             "positive_recovery_fraction_by_prediction_count": (
                 "GT-Positive Recovery by Prediction Count",
@@ -497,8 +497,8 @@ def write_publication_run_pdf_report(
             "Evaluation design",
             [
                 "Higher aligned scores mean stronger predicted targeting.",
-                "Dataset plots use dataset-local, tie-aware dense ranks.",
-                "Cross-dataset global ranks use each full standardized predictor file.",
+                "Dataset plots rank only scored pairs; tied scores use average rank.",
+                "Cross-dataset global ranks use scored pairs from each full standardized predictor file.",
             ],
             x=0.04,
             y=0.57,