fix: prevent markers.csv duplication in multi-sample runs (#165)

subworkflows/local/update_from_ome/main.nf

@@ -16,22 +16,32 @@ workflow UPDATE_FROM_OME {
         .dump(tag: "ch_samplesheet_meta")
         .set { samplesheet_meta }
 
-    c_sum = 0
-    agg   = channel.empty()
+    agg = channel.empty()
 
     samplesheet_meta
         .map {
             meta, image_tiles, dfp, ffp -> [meta.cycle_number, meta.channel_count]
         }
         .unique()
         .toSortedList()
-        .flatMap()
-        .map {
-            cn, cc ->
-            temp = [cn, c_sum]
-            c_sum += cc
-            return temp
+        .map { pairs ->
+            // Samples must agree on the channel count of a given cycle, otherwise a single
+            // shared markers.csv cannot describe every image and MCQUANT will later fail with
+            // an opaque "number of channels doesn't match" error. Catch it here instead.
+            pairs.groupBy { it[0] }.each { cycle, entries ->
+                def counts = entries.collect { it[1] }.unique()
+                if (counts.size() != 1) {
+                    error "Samples disagree on the channel count for cycle ${cycle} (found ${counts}). " +
+                          "A shared markers.csv cannot be built across images with different channel layouts."
+                }
+            }
+            // Cumulative per-cycle offset so each cycle's channels are numbered contiguously
+            // across the stacked image. Computed inside a single map so it does not rely on
+            // downstream operators preserving order.
+            def c_sum = 0
+            pairs.collect { cn, cc -> def temp = [cn, c_sum]; c_sum += cc; temp }
         }
+        .flatMap()
         .dump(tag: "CHANNEL_DELTA_INDEX")
         .set { agg }
 
@@ -81,31 +91,48 @@ workflow UPDATE_FROM_OME {
         }
         .map{ orig, validated ->  markersheet_template + validated + orig }
         .map{ meta -> meta - meta.subMap('id') }
-        .unique()
-        .toSortedList { a, b -> a.channel_number <=> b.channel_number }
-        .dump(tag: "ch_markersheet_meta")
-        .set { markersheet_meta }
-
-
-    markersheet_meta  // Inter sample marker setup check
-        .map {
-            entry -> [[entry.cycle_number, entry.channel_number], [entry.exposure_time, entry.exposure_time_unit]]
-        }
+        // Collapse the per-sample rows down to one shared row per channel. Deduplicating on
+        // the whole map (the previous `.unique()`) leaves duplicates whenever samples differ
+        // in any per-sample field (e.g. exposure time), inflating markers.csv beyond the image
+        // channel count (nf-core/mcmicro#165). Group on the channel identity instead.
+        .map{ meta -> [[meta.cycle_number, meta.channel_number], meta] }
         .groupTuple()
-        .map {
-            key, values ->
-                if (values.unique().size() != 1)
-                    error "Inconsistent marker exposure data across samples."
-        }
-
-    if (params.backsub) {
-        markersheet_meta
-            .map { entry ->
-                if (entry.exposure_time == null){
-                    error "Exposure time cannot be NULL if doing backsub"
+        .map{ key, rows ->
+            def (cycle_number, channel_number) = key
+            // Per-sample-varying fields are allowed to differ; ignore them when checking that
+            // samples agree on the actual marker definition for this channel.
+            def per_sample_fields = ['exposure_time', 'exposure_time_unit']
+            def core = rows.collect{ it - it.subMap(per_sample_fields) }.unique()
+            if (core.size() != 1) {
+                error "Samples disagree on the marker definition for cycle ${cycle_number}, " +
+                      "channel ${channel_number}: ${core}."
+            }
+            if (params.backsub) {
+                def exposures = rows.collect{ [it.exposure_time, it.exposure_time_unit] }.unique()
+                if (exposures.size() != 1) {
+                    error "Samples report inconsistent exposure for cycle ${cycle_number}, " +
+                          "channel ${channel_number} (${exposures}); required for background subtraction."
+                }
+                if (rows.any{ it.exposure_time == null }) {
+                    error "Exposure time cannot be null for cycle ${cycle_number}, " +
+                          "channel ${channel_number} when doing backsub."
                 }
             }
-    }
+            rows.first()
+        }
+        .toSortedList { a, b -> a.channel_number <=> b.channel_number }
+        .map { rows ->
+            // Defense in depth: the shared markers.csv must have exactly one contiguously
+            // numbered row per image channel, or MCQUANT will reject it downstream.
+            def channels = rows.collect { it.channel_number }
+            if (channels != (1..rows.size()).toList()) {
+                error "Constructed markers.csv has ${rows.size()} rows numbered ${channels}; " +
+                      "expected a contiguous 1..N with one row per image channel."
+            }
+            rows
+        }
+        .dump(tag: "ch_markersheet_meta")
+        .set { markersheet_meta }
 
     emit:
     samplesheet = samplesheet_meta

subworkflows/local/update_from_ome/tests/fixtures/cc_A_cyc1.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<OME xmlns="http://www.openmicroscopy.org/Schemas/OME/2016-06">
+  <Image ID="Image:0" Name="img">
+    <Pixels ID="Pixels:0" DimensionOrder="XYCZT" Type="uint16"
+            SizeX="1024" SizeY="1024" SizeC="2" SizeZ="1" SizeT="1"
+            PhysicalSizeX="0.325" PhysicalSizeXUnit="um"
+            PhysicalSizeY="0.325" PhysicalSizeYUnit="um"
+            BigEndian="false" Interleaved="false" SignificantBits="16">
+      <Plane TheC="0" TheZ="0" TheT="0" ExposureTime="1.0" ExposureTimeUnit="s"/><Plane TheC="1" TheZ="0" TheT="0" ExposureTime="1.0" ExposureTimeUnit="s"/>
+    </Pixels>
+  </Image>
+</OME>

subworkflows/local/update_from_ome/tests/fixtures/cc_B_cyc1.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<OME xmlns="http://www.openmicroscopy.org/Schemas/OME/2016-06">
+  <Image ID="Image:0" Name="img">
+    <Pixels ID="Pixels:0" DimensionOrder="XYCZT" Type="uint16"
+            SizeX="1024" SizeY="1024" SizeC="3" SizeZ="1" SizeT="1"
+            PhysicalSizeX="0.325" PhysicalSizeXUnit="um"
+            PhysicalSizeY="0.325" PhysicalSizeYUnit="um"
+            BigEndian="false" Interleaved="false" SignificantBits="16">
+      <Plane TheC="0" TheZ="0" TheT="0" ExposureTime="1.0" ExposureTimeUnit="s"/><Plane TheC="1" TheZ="0" TheT="0" ExposureTime="1.0" ExposureTimeUnit="s"/><Plane TheC="2" TheZ="0" TheT="0" ExposureTime="1.0" ExposureTimeUnit="s"/>
+    </Pixels>
+  </Image>
+</OME>

subworkflows/local/update_from_ome/tests/fixtures/exp_A_cyc1.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<OME xmlns="http://www.openmicroscopy.org/Schemas/OME/2016-06">
+  <Image ID="Image:0" Name="img">
+    <Pixels ID="Pixels:0" DimensionOrder="XYCZT" Type="uint16"
+            SizeX="1024" SizeY="1024" SizeC="2" SizeZ="1" SizeT="1"
+            PhysicalSizeX="0.325" PhysicalSizeXUnit="um"
+            PhysicalSizeY="0.325" PhysicalSizeYUnit="um"
+            BigEndian="false" Interleaved="false" SignificantBits="16">
+      <Plane TheC="0" TheZ="0" TheT="0" ExposureTime="0.1" ExposureTimeUnit="s"/><Plane TheC="1" TheZ="0" TheT="0" ExposureTime="1.0" ExposureTimeUnit="s"/>
+    </Pixels>
+  </Image>
+</OME>

subworkflows/local/update_from_ome/tests/fixtures/exp_A_cyc2.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<OME xmlns="http://www.openmicroscopy.org/Schemas/OME/2016-06">
+  <Image ID="Image:0" Name="img">
+    <Pixels ID="Pixels:0" DimensionOrder="XYCZT" Type="uint16"
+            SizeX="1024" SizeY="1024" SizeC="2" SizeZ="1" SizeT="1"
+            PhysicalSizeX="0.325" PhysicalSizeXUnit="um"
+            PhysicalSizeY="0.325" PhysicalSizeYUnit="um"
+            BigEndian="false" Interleaved="false" SignificantBits="16">
+      <Plane TheC="0" TheZ="0" TheT="0" ExposureTime="1.0" ExposureTimeUnit="s"/><Plane TheC="1" TheZ="0" TheT="0" ExposureTime="1.0" ExposureTimeUnit="s"/>
+    </Pixels>
+  </Image>
+</OME>

subworkflows/local/update_from_ome/tests/fixtures/exp_B_cyc1.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<OME xmlns="http://www.openmicroscopy.org/Schemas/OME/2016-06">
+  <Image ID="Image:0" Name="img">
+    <Pixels ID="Pixels:0" DimensionOrder="XYCZT" Type="uint16"
+            SizeX="1024" SizeY="1024" SizeC="2" SizeZ="1" SizeT="1"
+            PhysicalSizeX="0.325" PhysicalSizeXUnit="um"
+            PhysicalSizeY="0.325" PhysicalSizeYUnit="um"
+            BigEndian="false" Interleaved="false" SignificantBits="16">
+      <Plane TheC="0" TheZ="0" TheT="0" ExposureTime="0.5" ExposureTimeUnit="s"/><Plane TheC="1" TheZ="0" TheT="0" ExposureTime="2.0" ExposureTimeUnit="s"/>
+    </Pixels>
+  </Image>
+</OME>

subworkflows/local/update_from_ome/tests/fixtures/exp_B_cyc2.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<OME xmlns="http://www.openmicroscopy.org/Schemas/OME/2016-06">
+  <Image ID="Image:0" Name="img">
+    <Pixels ID="Pixels:0" DimensionOrder="XYCZT" Type="uint16"
+            SizeX="1024" SizeY="1024" SizeC="2" SizeZ="1" SizeT="1"
+            PhysicalSizeX="0.325" PhysicalSizeXUnit="um"
+            PhysicalSizeY="0.325" PhysicalSizeYUnit="um"
+            BigEndian="false" Interleaved="false" SignificantBits="16">
+      <Plane TheC="0" TheZ="0" TheT="0" ExposureTime="3.0" ExposureTimeUnit="s"/><Plane TheC="1" TheZ="0" TheT="0" ExposureTime="1.0" ExposureTimeUnit="s"/>
+    </Pixels>
+  </Image>
+</OME>

subworkflows/local/update_from_ome/tests/main.nf.test

@@ -86,6 +86,81 @@ nextflow_workflow {
                     { assert workflow.errorReport.contains("Markersheet cycle/channel numbering") }
                 )
             }
+        },
+
+        // Regression test for nf-core/mcmicro#165: when samples were imaged with different
+        // exposure times, the shared markers.csv used to be inflated with duplicate rows
+        // (44 -> 64 in the reported case), causing MCQUANT to abort. It must instead collapse
+        // to one row per image channel.
+        test("Should not duplicate markers when samples differ in exposure time") {
+            when {
+                workflow {
+                    """
+                    def fx = "\${projectDir}/subworkflows/local/update_from_ome/tests/fixtures"
+                    // Two samples (A, B), two cycles, two channels; B reports different
+                    // exposure times than A for the same channels.
+                    input[0] = Channel.of(
+                        [[id:'A', cycle_number:1], 'tiles', [], []],
+                        [[id:'A', cycle_number:2], 'tiles', [], []],
+                        [[id:'B', cycle_number:1], 'tiles', [], []],
+                        [[id:'B', cycle_number:2], 'tiles', [], []],
+                    )
+                    input[1] = Channel.value([
+                        [channel_number: 1, cycle_number: 1, marker_name: 'DNA1'],
+                        [channel_number: 2, cycle_number: 1, marker_name: 'CD3'],
+                        [channel_number: 3, cycle_number: 2, marker_name: 'DNA2'],
+                        [channel_number: 4, cycle_number: 2, marker_name: 'CD8'],
+                    ])
+                    input[2] = Channel.of(
+                        [[id:'A', cycle_number:1], file("\${fx}/exp_A_cyc1.xml")],
+                        [[id:'A', cycle_number:2], file("\${fx}/exp_A_cyc2.xml")],
+                        [[id:'B', cycle_number:1], file("\${fx}/exp_B_cyc1.xml")],
+                        [[id:'B', cycle_number:2], file("\${fx}/exp_B_cyc2.xml")],
+                    )
+                    """
+                }
+            }
+            then {
+                assertAll(
+                    { assert workflow.success },
+                    // Exactly one row per image channel, contiguously numbered - not inflated.
+                    { assert workflow.out.markersheet[0].size() == 4 },
+                    { assert workflow.out.markersheet[0].collect { it.channel_number } == [1, 2, 3, 4] },
+                    { assert workflow.out.markersheet[0].collect { it.marker_name } == ['DNA1', 'CD3', 'DNA2', 'CD8'] },
+                )
+            }
+        },
+
+        // Regression test for nf-core/mcmicro#165: samples that disagree on a cycle's channel
+        // count cannot share one markers.csv. This must fail with a clear message instead of
+        // silently producing an inflated sheet.
+        test("Should fail clearly when samples disagree on channel count") {
+            when {
+                workflow {
+                    """
+                    def fx = "\${projectDir}/subworkflows/local/update_from_ome/tests/fixtures"
+                    // Sample A cycle 1 has 2 channels; sample B cycle 1 has 3.
+                    input[0] = Channel.of(
+                        [[id:'A', cycle_number:1], 'tiles', [], []],
+                        [[id:'B', cycle_number:1], 'tiles', [], []],
+                    )
+                    input[1] = Channel.value([
+                        [channel_number: 1, cycle_number: 1, marker_name: 'DNA1'],
+                        [channel_number: 2, cycle_number: 1, marker_name: 'CD3'],
+                    ])
+                    input[2] = Channel.of(
+                        [[id:'A', cycle_number:1], file("\${fx}/cc_A_cyc1.xml")],
+                        [[id:'B', cycle_number:1], file("\${fx}/cc_B_cyc1.xml")],
+                    )
+                    """
+                }
+            }
+            then {
+                assertAll(
+                    { assert workflow.failed },
+                    { assert workflow.errorReport.contains("channel count") }
+                )
+            }
         }
 
     ]