feat(segmentation_user_layer): add user shader for segmentation layers with support for segment properties

docs/user-guide/annotation_shaders.rst

@@ -71,7 +71,7 @@ interpolation**) will help us do that.
 
 .. code-block:: glsl
 
-  #uicontrol float red invlerp(range=[10,30], window=[0,1000])
+  #uicontrol invlerp red(range=[10,30], window=[0,1000])
   void main() {
     vec3 mycolor = vec3(red(), 0.0, 0.0);
     setColor(mycolor);
@@ -94,7 +94,7 @@ property goes from low to high.
 
 .. code-block:: glsl
 
-  #uicontrol float intensity invlerp(range=[10,30], window=[0,1000])
+  #uicontrol invlerp intensity(range=[10,30], window=[0,1000])
   #uicontrol vec3 mycolor color(default="red")
   void main() {
     setColor(intensity()*mycolor);
@@ -166,7 +166,7 @@ we might use the ``invlerp`` control again.
 
 .. code-block:: glsl
 
-  #uicontrol float intensity invlerp(range=[10,30], window=[0,1000])
+  #uicontrol invlerp intensity(range=[10,30], window=[0,1000])
   void main() {
     setColor(defaultColor());
     setPointMarkerSize(1.0 + 19.0*intensity());
@@ -177,7 +177,7 @@ control.
 
 .. code-block:: glsl
 
-  #uicontrol float intensity invlerp(range=[10,30], window=[0,1000])
+  #uicontrol invlerp intensity(range=[10,30], window=[0,1000])
   #uicontrol float maxsize slider(min=1.0, max=50.0, step=1.0)
   void main() {
     setColor(defaultColor());

docs/user-guide/index.rst

@@ -12,10 +12,13 @@ Topics
   to segmentation layers.
 - :doc:`annotation_shaders` — writing custom GLSL shader code to drive
   the visual appearance of annotations from their properties.
+- :doc:`segmentation_shaders` — writing custom GLSL shader code to color
+  segmentation layers from segment ids and segment properties.
 
 .. toctree::
    :hidden:
 
    navigation
    annotations
    annotation_shaders
+   segmentation_shaders

docs/user-guide/segmentation_shaders.rst

@@ -0,0 +1,199 @@
+Segmentation Shaders
+--------------------
+
+Segmentation layers can use custom GLSL code to change how segments are colored.
+The technical reference for the shader API is in
+`the segmentation rendering guide <https://github.com/google/neuroglancer/blob/master/src/layer/segmentation/rendering.md>`_.
+This page is a gentler introduction to the same ideas.
+
+The Segment Color Function
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A segmentation shader defines a ``segmentColor`` function. The default shader
+keeps Neuroglancer's existing segment color unchanged:
+
+.. code-block:: text
+
+  vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) {
+    return color;
+  }
+
+The ``color`` argument is the color Neuroglancer would normally use for the
+segment. It may come from the segment color hash, the layer's default segment
+color, or an explicit color assigned to that segment. The ``isStated`` argument
+is ``true`` when ``color`` came from an explicit segment color. The
+``hasProperties`` argument is ``true`` when Neuroglancer found segment property
+data for the current segment.
+
+To make every visible segment red, return a red ``vec3``:
+
+.. code-block:: text
+
+  vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) {
+    return vec3(1.0, 0.0, 0.0);
+  }
+
+If you want to preserve explicit per-segment colors but recolor everything
+else, use ``isStated``:
+
+.. code-block:: text
+
+  vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) {
+    if (isStated) {
+      return color;
+    }
+    return vec3(0.0, 0.6, 1.0);
+  }
+
+Opacity
+~~~~~~~
+
+Use a ``vec4`` return type to adjust opacity. The alpha channel is the fourth
+component. It is multiplied by **Opacity (on)** or **Opacity (off)** in slice
+views. For meshes and skeletons, it is multiplied by **Opacity (3d)**.
+
+.. code-block:: text
+
+  vec4 segmentColor(vec4 color, bool hasProperties, bool isStated) {
+    if (isStated) {
+      return color;
+    }
+    return vec4(color.rgb, 0.35);
+  }
+
+Segment Properties
+~~~~~~~~~~~~~~~~~~
+
+Segment properties can drive color choices. If a segment property map is
+available, a shader can read tags and properties directly by name:
+
+.. code-block:: text
+
+  vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) {
+    if (!hasProperties) {
+      return vec3(0.5, 0.5, 0.5);
+    }
+    if (tag("axon")) {
+      return vec3(1.0, 0.4, 0.0);
+    }
+    if (prop("size") > 100u) {
+      return vec3(1.0, 1.0, 0.0);
+    }
+    return color;
+  }
+
+``tag("axon")`` returns ``true`` when the current segment has that tag.
+``prop("size")`` reads a numerical property named ``size``. For string
+properties, compare the value with a string literal:
+
+.. code-block:: text
+
+  vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) {
+    if (!hasProperties) {
+      return vec3(0.5, 0.5, 0.5);
+    }
+    if (prop("class") == "interneuron") {
+      return vec3(0.0, 1.0, 0.6);
+    }
+    return color;
+  }
+
+Property Controls
+~~~~~~~~~~~~~~~~~
+
+A ``property`` UI control lets the user choose which segment property a shader
+uses. The control can be filtered to tags, numerical properties, or string
+properties.
+
+.. code-block:: text
+
+  #uicontrol property selectedTag(type="tag")
+  #uicontrol property selectedSize(type="number")
+  #uicontrol property selectedClass(type="string")
+
+  vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) {
+    if (!hasProperties) {
+      return vec3(0.5, 0.5, 0.5);
+    }
+    if (selectedTag) {
+      return vec3(1.0, 0.0, 0.0);
+    }
+    if (selectedSize > 100u) {
+      return vec3(1.0, 1.0, 0.0);
+    }
+    if (selectedClass == "interneuron") {
+      return vec3(0.0, 1.0, 0.6);
+    }
+    return color;
+  }
+
+A ``type="tag"`` control lets the user choose an individual tag and evaluates
+to a ``bool`` indicating whether the current segment has that tag. A
+``type="string"`` control lets the user choose a string property and evaluates
+to that property's value, which can be compared with a string literal.
+``type="number"`` and ``type="numerical"`` both select numerical properties;
+the control's GLSL type matches the selected property's data type.
+
+Data Mapping
+~~~~~~~~~~~~
+
+For continuous numerical properties, an ``invlerp`` control maps a property
+range to ``0-1``. The user can adjust the selected property and range from the
+shader UI.
+
+.. code-block:: glsl
+
+  #uicontrol invlerp intensity(property="size", range=[0, 1000])
+
+  vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) {
+    if (!hasProperties) {
+      return vec3(0.5, 0.5, 0.5);
+    }
+    return intensity() * vec3(1.0, 0.2, 0.0);
+  }
+
+If ``property`` is omitted, Neuroglancer selects the first available numerical
+segment property as the default. ``range`` controls the data values mapped to
+``0`` and ``1``. ``window`` can be added to control the range shown by the UI
+widget.
+
+Colormaps
+~~~~~~~~~
+
+The remapped value from an ``invlerp`` control can be passed to a colormap:
+
+.. code-block:: glsl
+
+  #uicontrol invlerp intensity(property="size", range=[0, 1000])
+
+  vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) {
+    if (!hasProperties) {
+      return vec3(0.5, 0.5, 0.5);
+    }
+    if (isStated) {
+      return color;
+    }
+    return colormapJet(intensity());
+  }
+
+This pattern is useful when a numerical segment property should control color
+continuously while still respecting explicitly assigned segment colors.
+
+Relationship to the Skeleton Shader
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A segmentation layer may have both a segment color shader and a skeleton user
+shader. The segment color shader determines the per-segment color used by slice
+views, meshes, and skeletons.
+
+The default skeleton shader preserves the result of the segment color shader:
+
+.. code-block:: glsl
+
+  void main() {
+    emitDefault();
+  }
+
+Here, ``emitDefault()`` emits the color and alpha returned by the segment color
+shader. A custom skeleton shader can also access that result through
+``vec4 segmentColor()``, or emit its own color.

python/neuroglancer/viewer_state.py

@@ -988,6 +988,9 @@ def skeleton_shader(self, shader):
     segment_default_color = segmentDefaultColor = wrapped_property(
         "segmentDefaultColor", optional(str)
     )
+    segment_color_shader = segmentColorShader = wrapped_property(
+        "segmentColorShader", optional(str)
+    )
 
     @property
     def segment_html_color_dict(self):

python/tests/skeleton_options_test.py

@@ -23,12 +23,13 @@
 
 
 class SkeletonSource(neuroglancer.skeleton.SkeletonSource):
-    def __init__(self):
+    def __init__(self, vertex_position=(0, 0, 0)):
         super().__init__(dimensions=dimensions)
+        self.vertex_position = vertex_position
 
     def get_skeleton(self, object_id):
         return neuroglancer.skeleton.Skeleton(
-            vertex_positions=[[0, 0, 0]],
+            vertex_positions=[self.vertex_position],
             edges=[[0, 0]],
         )
 
@@ -55,20 +56,52 @@ def test_skeleton_options(webdriver):
         s.layers[0].skeleton_rendering.shader_controls["color"] = "#f00"
         s.show_axis_lines = False
     screenshot = webdriver.viewer.screenshot(size=[10, 10]).screenshot
+    opaque_pixels_2d = screenshot.image_pixels.copy()
     np.testing.assert_array_equal(
-        screenshot.image_pixels,
+        opaque_pixels_2d,
         np.tile(np.array([255, 0, 0, 255], dtype=np.uint8), (10, 10, 1)),
     )
 
+    with webdriver.viewer.txn() as s:
+        s.layers[0].source[0].subsources["default"] = False
+    background_pixels_2d = webdriver.viewer.screenshot(
+        size=[10, 10]
+    ).screenshot.image_pixels.copy()
+
+    with webdriver.viewer.txn() as s:
+        s.layers[0].source[0].subsources["default"] = True
+        s.layers[0].object_alpha = 0.5
+    screenshot = webdriver.viewer.screenshot(size=[10, 10]).screenshot
+    pixels = screenshot.image_pixels
+    np.testing.assert_allclose(
+        pixels[..., :3],
+        opaque_pixels_2d[..., :3] * 0.5 + background_pixels_2d[..., :3] * 0.5,
+        atol=2,
+    )
+    np.testing.assert_array_equal(pixels[..., 3], 255)
+
     with webdriver.viewer.txn() as s:
         s.layout = "3d"
+        s.layers[0].object_alpha = 1.0
         s.layers[0].skeleton_rendering.line_width3d = 100
     screenshot = webdriver.viewer.screenshot(size=[10, 10]).screenshot
+    opaque_pixels = screenshot.image_pixels.copy()
     np.testing.assert_array_equal(
-        screenshot.image_pixels,
+        opaque_pixels,
         np.tile(np.array([255, 0, 0, 255], dtype=np.uint8), (10, 10, 1)),
     )
 
+    # Perspective OIT requires premultiplied RGB, so reducing opacity must
+    # reduce the red intensity rather than leave it saturated or brighten it.
+    with webdriver.viewer.txn() as s:
+        s.layers[0].object_alpha = 0.5
+    screenshot = webdriver.viewer.screenshot(size=[10, 10]).screenshot
+    pixels = screenshot.image_pixels
+    np.testing.assert_allclose(pixels[..., 0], opaque_pixels[..., 0] * 0.5, atol=2)
+    np.testing.assert_array_equal(pixels[..., 1], 0)
+    np.testing.assert_array_equal(pixels[..., 2], 0)
+    np.testing.assert_array_equal(pixels[..., 3], 255)
+
     with webdriver.viewer.txn() as s:
         s.layers[0].source[0].subsources["default"] = False
 
@@ -77,3 +110,51 @@ def test_skeleton_options(webdriver):
         screenshot.image_pixels,
         np.tile(np.array([0, 0, 0, 255], dtype=np.uint8), (10, 10, 1)),
     )
+
+
+def test_segment_color_shader_alpha_makes_skeleton_transparent(webdriver):
+    image_data = np.zeros((20, 20, 20), dtype=np.uint8)
+    image_data[1:19, 1:19, 11:20] = 255
+
+    with webdriver.viewer.txn() as s:
+        s.dimensions = dimensions
+        s.layers.append(
+            name="image",
+            layer=neuroglancer.ImageLayer(
+                source=neuroglancer.LocalVolume(data=image_data, dimensions=dimensions),
+                volume_rendering_mode="Max",
+                shader="""
+#uicontrol invlerp normalized
+void main() {
+  emitRGBA(vec4(0.0, 0.0, normalized(), 1.0));
+}
+""",
+            ),
+        )
+        s.layers.append(
+            name="skeleton",
+            layer=neuroglancer.SegmentationLayer(
+                source=SkeletonSource((10, 10, 5)),
+                segments=[1],
+                segment_color_shader="""
+vec4 segmentColor(vec4 color, bool hasProperties, bool isStated) {
+  return vec4(color.rgb, 0.5);
+}
+""",
+                skeleton_shader="""
+void main() {
+  emitRGB(vec3(1.0, 0.0, 0.0));
+}
+""",
+            ),
+        )
+        s.layers["skeleton"].skeleton_rendering.line_width3d = 100
+        s.layout = "3d"
+        s.show_axis_lines = False
+        s.position = [10, 10, 10]
+        s.projection_scale = 15
+
+    webdriver.sync()
+    pixels = webdriver.viewer.screenshot(size=[10, 10]).screenshot.image_pixels
+    assert np.all(pixels[..., 0] > 0)
+    assert np.all(pixels[..., 2] > 0)