#6151 - Add support for gamma agreement

inception/inception-agreement/src/main/java/de/tudarmstadt/ukp/clarin/webanno/agreement/AgreementSummary.java

@@ -28,6 +28,7 @@
 import java.util.Set;
 import java.util.stream.DoubleStream;
 
+import de.tudarmstadt.ukp.clarin.webanno.agreement.results.aligning.FullAligningAgreementResult;
 import de.tudarmstadt.ukp.clarin.webanno.agreement.results.coding.FullCodingAgreementResult;
 import de.tudarmstadt.ukp.clarin.webanno.agreement.results.unitizing.FullUnitizingAgreementResult;
 import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationFeature;
@@ -122,6 +123,10 @@ public static AgreementSummary of(Serializable aResult)
             return new AgreementSummary(result);
         }
 
+        if (aResult instanceof FullAligningAgreementResult result) {
+            return new AgreementSummary(result);
+        }
+
         throw new IllegalArgumentException(
                 "Unsupported result type: [" + aResult.getClass().getName() + "]");
     }
@@ -152,6 +157,18 @@ public AgreementSummary(FullUnitizingAgreementResult aResult)
         pluralitySets = -1;
     }
 
+    public AgreementSummary(FullAligningAgreementResult aResult)
+    {
+        this((FullAgreementResult_ImplBase<?>) aResult);
+
+        incompleteSetsByLabel = -1;
+        incompleteSetsByPosition = -1;
+        relevantSetCount = -1;
+        completeSetCount = -1;
+        usedSetCount = -1;
+        pluralitySets = -1;
+    }
+
     public AgreementSummary(FullCodingAgreementResult aResult)
     {
         this((FullAgreementResult_ImplBase<?>) aResult);

inception/inception-agreement/src/main/java/de/tudarmstadt/ukp/clarin/webanno/agreement/config/AgreementAutoConfiguration.java

@@ -23,6 +23,7 @@
 import de.tudarmstadt.ukp.clarin.webanno.agreement.AgreementServiceImpl;
 import de.tudarmstadt.ukp.clarin.webanno.agreement.measures.cohenkappa.CohenKappaAgreementMeasureSupport;
 import de.tudarmstadt.ukp.clarin.webanno.agreement.measures.fleisskappa.FleissKappaAgreementMeasureSupport;
+import de.tudarmstadt.ukp.clarin.webanno.agreement.measures.gamma.GammaAgreementMeasureSupport;
 import de.tudarmstadt.ukp.clarin.webanno.agreement.measures.gwetac1.GwetAC1AgreementMeasureSupport;
 import de.tudarmstadt.ukp.clarin.webanno.agreement.measures.gwetac2.GwetAC2AgreementMeasureSupport;
 import de.tudarmstadt.ukp.clarin.webanno.agreement.measures.krippendorffalpha.KrippendorffAlphaAgreementMeasureSupport;
@@ -71,6 +72,12 @@ public KrippendorffAlphaUnitizingAgreementMeasureSupport krippendorffAlphaUnitiz
         return new KrippendorffAlphaUnitizingAgreementMeasureSupport();
     }
 
+    @Bean
+    public GammaAgreementMeasureSupport gammaAgreementMeasureSupport()
+    {
+        return new GammaAgreementMeasureSupport();
+    }
+
     @Bean
     public GwetAC1AgreementMeasureSupport gwetAC1AgreementMeasureSupport(
             AnnotationSchemaService aAnnotationService, DiffAdapterRegistry aDiffAdapterRegistry)

inception/inception-agreement/src/main/java/de/tudarmstadt/ukp/clarin/webanno/agreement/measures/gamma/GammaAgreementMeasure.java

@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Technische Universität Darmstadt under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The Technische Universität Darmstadt
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package de.tudarmstadt.ukp.clarin.webanno.agreement.measures.gamma;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonList;
+import static org.apache.commons.lang3.StringUtils.isNotEmpty;
+
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.uima.cas.CAS;
+import org.apache.uima.cas.Feature;
+import org.apache.uima.fit.util.FSUtil;
+import org.apache.uima.jcas.tcas.Annotation;
+import org.dkpro.statistics.agreement.InsufficientDataException;
+import org.dkpro.statistics.agreement.aligning.AlignableAnnotationUnit;
+import org.dkpro.statistics.agreement.aligning.AligningAnnotationStudy;
+import org.dkpro.statistics.agreement.aligning.GammaAgreement;
+import org.dkpro.statistics.agreement.aligning.data.AnnotationSet;
+import org.dkpro.statistics.agreement.aligning.data.Rater;
+import org.dkpro.statistics.agreement.aligning.disorder.StatisticalContinuumDisorderSampler;
+import org.dkpro.statistics.agreement.aligning.dissimilarity.CombinedCategoricalDissimilarity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import de.tudarmstadt.ukp.clarin.webanno.agreement.measures.AgreementMeasure_ImplBase;
+import de.tudarmstadt.ukp.clarin.webanno.agreement.measures.DefaultAgreementTraits;
+import de.tudarmstadt.ukp.clarin.webanno.agreement.results.aligning.FullAligningAgreementResult;
+import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationFeature;
+import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer;
+
+public class GammaAgreementMeasure
+    extends AgreementMeasure_ImplBase<//
+            FullAligningAgreementResult, //
+            DefaultAgreementTraits>
+{
+    private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+    private static final Set<String> POSITION_VALUE = Set.of(POSITION);
+
+    // Fixed seed for the Monte-Carlo chance model so that repeated calculations on the same data
+    // yield the same agreement value.
+    private static final long SEED = 428_984_162_539_017_403L;
+
+    public GammaAgreementMeasure(AnnotationLayer aLayer, AnnotationFeature aFeature,
+            DefaultAgreementTraits aTraits)
+    {
+        super(aLayer, aFeature, aTraits);
+    }
+
+    @Override
+    public FullAligningAgreementResult getAgreement(Map<String, CAS> aCasMap)
+    {
+        var typeName = getLayer().getName();
+        var categoryFeature = getFeature() != null ? getFeature().getName() : POSITION;
+
+        var study = new AligningAnnotationStudy();
+        var raterIdx = 0;
+        for (var entry : aCasMap.entrySet()) {
+            var cas = entry.getValue();
+            if (cas == null) {
+                // If a user has never worked on a source document, its CAS is null here - we
+                // skip it.
+                continue;
+            }
+
+            var t = cas.getTypeSystem().getType(typeName);
+            if (t == null) {
+                // CAS not upgraded yet
+                continue;
+            }
+
+            var rater = new Rater(entry.getKey(), raterIdx++);
+            var f = getFeature() != null ? t.getFeatureByBaseName(getFeature().getName()) : null;
+            for (var ann : cas.<Annotation> select(t)) {
+                if (ann.getBegin() >= ann.getEnd()) {
+                    LOG.trace("[{}] Not adding zero-width unit at [{}-{}]", entry.getKey(),
+                            ann.getBegin(), ann.getEnd());
+                    continue;
+                }
+
+                for (var value : getValues(ann, f)) {
+                    // A null or empty value is represented as an unlabelled unit (empty feature
+                    // map) which the disorder sampler treats as a category of its own. Keeping
+                    // null and empty equivalent is consistent with the coding measures.
+                    Map<String, String> features = isNotEmpty(value)
+                            ? Map.of(categoryFeature, value)
+                            : emptyMap();
+                    study.addUnit(new AlignableAnnotationUnit(rater, (String) null, ann.getBegin(),
+                            ann.getEnd(), features));
+                }
+            }
+        }
+
+        LOG.trace("Raters in study: {}", study.getRaterCount());
+        LOG.trace("Units in study : {}", study.getUnitCount());
+
+        var featureName = getFeature() != null ? getFeature().getName() : null;
+        var result = new FullAligningAgreementResult(typeName, featureName, study,
+                new ArrayList<>(aCasMap.keySet()), getTraits().isExcludeIncomplete());
+
+        if (result.isEmpty() || study.getRaterCount() < 2) {
+            result.setAgreement(Double.NaN);
+            return result;
+        }
+
+        var measure = GammaAgreement.builder() //
+                .withAnnotationSet(new AnnotationSet(study.getUnits())) //
+                .withDissimilarity(new CombinedCategoricalDissimilarity()) //
+                .withDisorderSampler(
+                        m -> new StatisticalContinuumDisorderSampler(m, categoryFeature)) //
+                .withSeed(SEED) //
+                .build();
+
+        try {
+            result.setAgreement(measure.calculateAgreement());
+        }
+        catch (InsufficientDataException e) {
+            LOG.debug("Unable to calculate gamma agreement", e);
+            result.setAgreement(Double.NaN);
+        }
+
+        return result;
+    }
+
+    private Collection<String> getValues(Annotation aAnn, Feature aFeature)
+    {
+        if (aFeature == null) {
+            return POSITION_VALUE;
+        }
+
+        var featureValue = FSUtil.getFeature(aAnn, aFeature, Object.class);
+        if (featureValue instanceof Collection<?> collectionValue) {
+            return collectionValue.stream().map(GammaAgreementMeasure::toLabel).toList();
+        }
+
+        return singletonList(toLabel(featureValue));
+    }
+
+    private static String toLabel(Object aValue)
+    {
+        return aValue != null ? String.valueOf(aValue) : null;
+    }
+}

inception/inception-agreement/src/main/java/de/tudarmstadt/ukp/clarin/webanno/agreement/measures/gamma/GammaAgreementMeasureSupport.java

@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Technische Universität Darmstadt under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The Technische Universität Darmstadt
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package de.tudarmstadt.ukp.clarin.webanno.agreement.measures.gamma;
+
+import org.apache.wicket.markup.html.panel.EmptyPanel;
+import org.apache.wicket.markup.html.panel.Panel;
+import org.apache.wicket.model.IModel;
+import org.dkpro.statistics.agreement.aligning.AligningAnnotationStudy;
+
+import de.tudarmstadt.ukp.clarin.webanno.agreement.AgreementResult_ImplBase;
+import de.tudarmstadt.ukp.clarin.webanno.agreement.PairwiseAgreementResult;
+import de.tudarmstadt.ukp.clarin.webanno.agreement.PerDocumentAgreementResult;
+import de.tudarmstadt.ukp.clarin.webanno.agreement.measures.AgreementMeasure;
+import de.tudarmstadt.ukp.clarin.webanno.agreement.measures.AgreementMeasureSupport_ImplBase;
+import de.tudarmstadt.ukp.clarin.webanno.agreement.measures.DefaultAgreementTraits;
+import de.tudarmstadt.ukp.clarin.webanno.agreement.results.aligning.FullAligningAgreementResult;
+import de.tudarmstadt.ukp.clarin.webanno.agreement.results.perdoc.PerDocumentAgreementTable;
+import de.tudarmstadt.ukp.clarin.webanno.agreement.results.unitizing.PairwiseUnitizingAgreementTable;
+import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationFeature;
+import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer;
+import de.tudarmstadt.ukp.inception.annotation.layer.span.api.SpanLayerSupport;
+
+public class GammaAgreementMeasureSupport
+    extends AgreementMeasureSupport_ImplBase<//
+            DefaultAgreementTraits, //
+            FullAligningAgreementResult, //
+            AligningAnnotationStudy>
+{
+    public static final String ID = "Gamma";
+
+    @Override
+    public String getId()
+    {
+        return ID;
+    }
+
+    @Override
+    public String getName()
+    {
+        return "Gamma (aligning / character offsets)";
+    }
+
+    @Override
+    public boolean accepts(AnnotationLayer aLayer, AnnotationFeature aFeature)
+    {
+        return SpanLayerSupport.TYPE.equals(aLayer.getType());
+    }
+
+    @Override
+    public AgreementMeasure<FullAligningAgreementResult> createMeasure(AnnotationLayer aLayer,
+            AnnotationFeature aFeature, DefaultAgreementTraits aTraits)
+    {
+        return new GammaAgreementMeasure(aLayer, aFeature, aTraits);
+    }
+
+    @Override
+    public Panel createResultsPanel(String aId, IModel<? extends AgreementResult_ImplBase> aResults,
+            DefaultAgreementTraits aTraits)
+    {
+        if (aResults.getObject() instanceof PairwiseAgreementResult) {
+            return new PairwiseUnitizingAgreementTable(aId, (IModel) aResults, aTraits);
+        }
+
+        if (aResults.getObject() instanceof PerDocumentAgreementResult) {
+            return new PerDocumentAgreementTable(aId, (IModel) aResults, aTraits);
+        }
+
+        return new EmptyPanel(aId);
+    }
+
+    @Override
+    public boolean isSupportingMoreThanTwoRaters()
+    {
+        return true;
+    }
+}

inception/inception-agreement/src/main/java/de/tudarmstadt/ukp/clarin/webanno/agreement/results/aligning/FullAligningAgreementResult.java

@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Technische Universität Darmstadt under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The Technische Universität Darmstadt
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package de.tudarmstadt.ukp.clarin.webanno.agreement.results.aligning;
+
+import static org.apache.commons.lang3.StringUtils.isNotEmpty;
+
+import java.util.List;
+
+import org.dkpro.statistics.agreement.aligning.AlignableAnnotationUnit;
+import org.dkpro.statistics.agreement.aligning.AligningAnnotationStudy;
+
+import de.tudarmstadt.ukp.clarin.webanno.agreement.FullAgreementResult_ImplBase;
+
+public class FullAligningAgreementResult
+    extends FullAgreementResult_ImplBase<AligningAnnotationStudy>
+{
+    private static final long serialVersionUID = -3672534728485229289L;
+
+    public FullAligningAgreementResult(String aType, String aFeature,
+            AligningAnnotationStudy aStudy, List<String> aCasGroupIds, boolean aExcludeIncomplete)
+    {
+        super(aType, aFeature, aStudy, aCasGroupIds, aExcludeIncomplete);
+    }
+
+    @Override
+    public boolean isAllNull(String aRater)
+    {
+        return study.getUnits().stream() //
+                .filter(u -> aRater.equals(u.getRater().getName())) //
+                .noneMatch(FullAligningAgreementResult::hasLabel);
+    }
+
+    @Override
+    public long getNonNullCount(String aRater)
+    {
+        return study.getUnits().stream() //
+                .filter(u -> aRater.equals(u.getRater().getName())) //
+                .filter(FullAligningAgreementResult::hasLabel) //
+                .count();
+    }
+
+    @Override
+    public long getItemCount(String aRater)
+    {
+        return study.getUnits().stream() //
+                .filter(u -> aRater.equals(u.getRater().getName())) //
+                .count();
+    }
+
+    @Override
+    public boolean isEmpty()
+    {
+        return study.getUnitCount() == 0;
+    }
+
+    private static boolean hasLabel(AlignableAnnotationUnit aUnit)
+    {
+        return aUnit.getFeatureNames().stream() //
+                .map(aUnit::getFeatureValue) //
+                .anyMatch(v -> isNotEmpty(v));
+    }
+}