Enhancement#434 group concat multilingual strings

jopa-impl/src/main/java/cz/cvut/kbss/jopa/query/sparql/loader/AttributeBasedRowsToAxiomsQueryResultLoader.java

@@ -205,10 +205,7 @@ private static Assertion attributeToAssertion(FieldSpecification<?, ?> fs) {
     private Stream<Value<?>> extractValue(ResultRow row,
                                           QueryVariableMapping variableMapping) throws OntoDriverException {
         if (variableMapping.canGroupConcat()) {
-            final String values = row.getString(variableMapping.attributeVar());
-            // This does not handle blank nodes
-            return Stream.of(values.split(AttributeEnumeratingSparqlAssemblyModifier.GROUP_CONCAT_SEPARATOR))
-                         .map(URI::create).map(Value::new);
+            return variableMapping.readGroupConcatValue(row);
         } else {
             return Stream.of(new Value<>(row.getObject(variableMapping.attributeVar())));
         }

jopa-impl/src/main/java/cz/cvut/kbss/jopa/query/sparql/loader/AttributeEnumeratingSparqlAssemblyModifier.java

@@ -113,14 +113,12 @@ private void modifyProjection(TokenQueryParameter<?> firstProjected, TokenStream
         final StringBuilder groupBy = new StringBuilder(" GROUP BY " + firstProjected.getIdentifierAsQueryString());
         boolean hasGroupConcat = false;
         for (QueryVariableMapping varMapping : queryMod.variables()) {
-            final String variable = "?" + varMapping.attributeVar();
-            // PERF: GROUP_CONCAT prevents combinatorial blowup of the number of rows when properties have multiple values
+            // PERF: GROUP_CONCAT may prevent combinatorial blowup of the number of rows
             if (varMapping.canGroupConcat()) {
-                projection.append(" (GROUP_CONCAT(DISTINCT ").append(variable).append("; SEPARATOR='")
-                          .append(GROUP_CONCAT_SEPARATOR).append("') AS ").append(variable).append("_gc)");
-                varMapping.setAttributeVar(varMapping.attributeVar() + "_gc");
+                projection.append(' ').append(varMapping.generateGroupConcat());
                 hasGroupConcat = true;
             } else {
+                final String variable = "?" + varMapping.attributeVar();
                 projection.append(' ').append(variable);
                 groupBy.append(' ').append(variable);
             }

jopa-impl/src/main/java/cz/cvut/kbss/jopa/query/sparql/loader/GroupConcatQueryModifier.java

@@ -0,0 +1,40 @@
+package cz.cvut.kbss.jopa.query.sparql.loader;
+
+import cz.cvut.kbss.ontodriver.exception.OntoDriverException;
+import cz.cvut.kbss.ontodriver.iteration.ResultRow;
+import cz.cvut.kbss.ontodriver.model.Value;
+
+import java.util.stream.Stream;
+
+/**
+ * Supports modifying a query to project GROUP_CONCAT for a given attribute.
+ * <p>
+ * Both generating the GROUP_CONCAT projection and reading the values from the result row are supported.
+ */
+abstract class GroupConcatQueryModifier {
+
+    static final String GROUP_CONCAT_SEPARATOR = "\u001F";
+    static final String GROUP_CONCAT_SUFFIX = "_gc";
+
+    final QueryVariableMapping variableMapping;
+
+    GroupConcatQueryModifier(QueryVariableMapping variableMapping) {
+        this.variableMapping = variableMapping;
+    }
+
+    /**
+     * Generates a {@literal GROUP_CONCAT} pattern for query projection.
+     *
+     * @return Pattern for query projection
+     */
+    abstract String generateGroupConcat();
+
+    /**
+     * Reads and extracts values created by a {@literal GROUP_CONCAT} pattern.
+     *
+     * @param row Row to extract values from
+     * @return Extracted values
+     * @throws OntoDriverException If reading the values fails
+     */
+    abstract Stream<Value<?>> readValue(ResultRow row) throws OntoDriverException;
+}

jopa-impl/src/main/java/cz/cvut/kbss/jopa/query/sparql/loader/IriGroupConcatQueryModifier.java

@@ -0,0 +1,34 @@
+package cz.cvut.kbss.jopa.query.sparql.loader;
+
+import cz.cvut.kbss.ontodriver.exception.OntoDriverException;
+import cz.cvut.kbss.ontodriver.iteration.ResultRow;
+import cz.cvut.kbss.ontodriver.model.Value;
+
+import java.net.URI;
+import java.util.stream.Stream;
+
+/**
+ * Supports modifying a query to project GROUP_CONCAT for a plural attribute whose values are identifiers, and for a
+ * types attributes.
+ */
+class IriGroupConcatQueryModifier extends GroupConcatQueryModifier {
+
+    IriGroupConcatQueryModifier(QueryVariableMapping variableMapping) {
+        super(variableMapping);
+    }
+
+    @Override
+    String generateGroupConcat() {
+        return "(GROUP_CONCAT(DISTINCT ?" + variableMapping.attributeVar() + "; SEPARATOR='" +
+                GROUP_CONCAT_SEPARATOR + "') AS ?" + variableMapping.attributeVar() + GROUP_CONCAT_SUFFIX + ")";
+    }
+
+    @Override
+    Stream<Value<?>> readValue(ResultRow row) throws OntoDriverException {
+        assert row.isBound(variableMapping.attributeVar());
+        final String values = row.getString(variableMapping.attributeVar());
+        // This does not handle blank nodes
+        return Stream.of(values.split(AttributeEnumeratingSparqlAssemblyModifier.GROUP_CONCAT_SEPARATOR))
+                     .map(URI::create).map(Value::new);
+    }
+}

jopa-impl/src/main/java/cz/cvut/kbss/jopa/query/sparql/loader/MultilingualStringGroupConcatQueryModifier.java

@@ -0,0 +1,51 @@
+package cz.cvut.kbss.jopa.query.sparql.loader;
+
+import cz.cvut.kbss.ontodriver.exception.OntoDriverException;
+import cz.cvut.kbss.ontodriver.iteration.ResultRow;
+import cz.cvut.kbss.ontodriver.model.LangString;
+import cz.cvut.kbss.ontodriver.model.Value;
+
+import java.util.stream.Stream;
+
+/**
+ * Supports generating GROUP_CONCAT projection for multilingual string attributes.
+ */
+class MultilingualStringGroupConcatQueryModifier extends GroupConcatQueryModifier {
+
+    MultilingualStringGroupConcatQueryModifier(QueryVariableMapping variableMapping) {
+        super(variableMapping);
+    }
+
+    @Override
+    String generateGroupConcat() {
+        // (GROUP_CONCAT(DISTINCT CONCAT(STR(?var), "@", LANG(?var)); SEPARATOR='') AS ?var_gc)
+        final String queryVar = "?" + variableMapping.attributeVar();
+        return "(GROUP_CONCAT(DISTINCT CONCAT(STR(" + queryVar + "), \"@\", " +
+                "LANG(" + queryVar + ")); SEPARATOR='" + GROUP_CONCAT_SEPARATOR + "') " +
+                "AS " + queryVar + GROUP_CONCAT_SUFFIX + ")";
+    }
+
+    @Override
+    Stream<Value<?>> readValue(ResultRow row) throws OntoDriverException {
+        assert row.isBound(variableMapping.attributeVar());
+        final String values = row.getString(variableMapping.attributeVar());
+        if (values.isEmpty()) {
+            return Stream.empty();
+        }
+        return Stream.of(values.split(GROUP_CONCAT_SEPARATOR))
+                     .map(MultilingualStringGroupConcatQueryModifier::toLangString)
+                     .map(Value::new);
+    }
+
+    private static LangString toLangString(String token) {
+        // The token is produced by CONCAT(STR(?var), "@", LANG(?var)).
+        // The lexical value may itself contain '@', so split on the last occurrence.
+        final int sep = token.lastIndexOf('@');
+        if (sep < 0) {
+            return new LangString(token);
+        }
+        final String value = token.substring(0, sep);
+        final String language = token.substring(sep + 1);
+        return language.isEmpty() ? new LangString(value) : new LangString(value, language);
+    }
+}

jopa-impl/src/main/java/cz/cvut/kbss/jopa/query/sparql/loader/QueryVariableMapping.java

@@ -1,54 +1,82 @@
 package cz.cvut.kbss.jopa.query.sparql.loader;
 
+import cz.cvut.kbss.jopa.model.MultilingualString;
 import cz.cvut.kbss.jopa.model.metamodel.FieldSpecification;
 import cz.cvut.kbss.jopa.model.metamodel.PluralAttribute;
 import cz.cvut.kbss.jopa.model.metamodel.TypesSpecification;
 import cz.cvut.kbss.jopa.utils.IdentifierTransformer;
+import cz.cvut.kbss.ontodriver.exception.OntoDriverException;
+import cz.cvut.kbss.ontodriver.iteration.ResultRow;
+import cz.cvut.kbss.ontodriver.model.Value;
 
 import java.util.Objects;
+import java.util.stream.Stream;
 
 public final class QueryVariableMapping {
     private final String subjectVar;
     private String attributeVar;
     private final FieldSpecification<?, ?> attribute;
 
-    private boolean canGroupConcat;
+    private GroupConcatQueryModifier groupConcatModifier;
 
     public QueryVariableMapping(String subjectVar, String attributeVar, FieldSpecification<?, ?> attribute) {
         this.subjectVar = subjectVar;
         this.attributeVar = attributeVar;
         this.attribute = attribute;
-        this.canGroupConcat = resolveCanGroupConcat();
+        this.groupConcatModifier = resolveCanGroupConcat();
     }
 
-    private boolean resolveCanGroupConcat() {
-        return isPluralPlainIdentifierAttribute() || attribute instanceof TypesSpecification<?, ?>;
+    private GroupConcatQueryModifier resolveCanGroupConcat() {
+        if (attribute == null) {
+            return null;
+        }
+        if (isPluralPlainIdentifierAttribute() || attribute instanceof TypesSpecification<?, ?>) {
+            return new IriGroupConcatQueryModifier(this);
+        }
+        if (isMultilingualStringAttribute()) {
+            return new MultilingualStringGroupConcatQueryModifier(this);
+        }
+        return null;
     }
 
-    public boolean isPluralPlainIdentifierAttribute() {
-        if (attribute == null || !attribute.isMappedAttribute() || !attribute.isCollection()) {
+    private boolean isPluralPlainIdentifierAttribute() {
+        if (!attribute.isMappedAttribute() || !attribute.isCollection()) {
             return false;
         }
         final PluralAttribute<?, ?, ?> att = (PluralAttribute<?, ?, ?>) attribute;
         return att.isAssociation() && IdentifierTransformer.isValidIdentifierType(att.getBindableJavaType());
     }
 
-    public boolean isTypes() {
-        return attribute == null || attribute instanceof TypesSpecification<?, ?>;
+    private boolean isMultilingualStringAttribute() {
+        return attribute.getJavaType().equals(MultilingualString.class)
+                || attribute.isCollection() && ((PluralAttribute<?, ?, ?>) attribute).getBindableJavaType()
+                                                                                     .equals(MultilingualString.class);
     }
 
     public String subjectVar() {return subjectVar;}
 
     public String attributeVar() {return attributeVar;}
 
-    public void setAttributeVar(String attributeVar) {this.attributeVar = attributeVar;}
-
     public FieldSpecification<?, ?> attribute() {return attribute;}
 
-    public boolean canGroupConcat() {return canGroupConcat;}
+    public boolean canGroupConcat() {
+        return groupConcatModifier != null;
+    }
 
     public void preventGroupConcat() {
-        canGroupConcat = false;
+        this.groupConcatModifier = null;
+    }
+
+    public String generateGroupConcat() {
+        assert canGroupConcat();
+        final String result = groupConcatModifier.generateGroupConcat();
+        this.attributeVar = attributeVar + GroupConcatQueryModifier.GROUP_CONCAT_SUFFIX;
+        return result;
+    }
+
+    public Stream<Value<?>> readGroupConcatValue(ResultRow row) throws OntoDriverException {
+        assert canGroupConcat();
+        return groupConcatModifier.readValue(row);
     }
 
     @Override

jopa-impl/src/test/java/cz/cvut/kbss/jopa/query/sparql/loader/AttributeBasedRowsToAxiomsQueryResultLoaderTest.java

@@ -19,6 +19,7 @@
 
 import cz.cvut.kbss.jopa.environment.OWLClassA;
 import cz.cvut.kbss.jopa.environment.OWLClassD;
+import cz.cvut.kbss.jopa.environment.OWLClassU;
 import cz.cvut.kbss.jopa.environment.Phone;
 import cz.cvut.kbss.jopa.environment.Vocabulary;
 import cz.cvut.kbss.jopa.environment.utils.Generators;
@@ -39,6 +40,7 @@
 import cz.cvut.kbss.ontodriver.model.Assertion;
 import cz.cvut.kbss.ontodriver.model.Axiom;
 import cz.cvut.kbss.ontodriver.model.AxiomImpl;
+import cz.cvut.kbss.ontodriver.model.LangString;
 import cz.cvut.kbss.ontodriver.model.NamedResource;
 import cz.cvut.kbss.ontodriver.model.Value;
 import org.junit.jupiter.api.BeforeEach;
@@ -331,4 +333,36 @@ void loadResultAutomaticallyAddsClassAssertionForDirectEntityMappingWithoutProje
         verify(uow).readObjectFromAxioms(eq(Phone.class), captor.capture(), any(AxiomBasedLoadingConfigGroup.class));
         assertThat(captor.getValue(), hasItem(new AxiomImpl<>(NamedResource.create(phone.getUri()), Assertion.createClassAssertion(false), new Value<>(URI.create(Vocabulary.c_Phone)))));
     }
+
+    @Test
+    void loadResultSupportsGroupConcatenatedMultilingualStrings() throws Exception {
+        final EntityGraph<OWLClassU> fetchGraph = new EntityGraphImpl<>(metamodelMocks.forOwlClassU()
+                                                                                      .entityType(), metamodel);
+        fetchGraph.addAttributeNodes("singularStringAtt");
+        final AttributeBasedRowsToAxiomsQueryResultLoader<OWLClassU> sut = new AttributeBasedRowsToAxiomsQueryResultLoader<>(uow, OWLClassU.class, descriptor, fetchGraph);
+        final OWLClassU instance = new OWLClassU(Generators.createIndividualIdentifier());
+        final ResultRow row = mock(ResultRow.class);
+        when(row.getColumnCount()).thenReturn(2);
+        when(row.getColumnNames()).thenReturn(List.of("x", "x_singularStringAtt"));
+        when(row.getObject(0, URI.class)).thenReturn(instance.getId());
+        when(row.isBound("x")).thenReturn(true);
+        when(row.getObject("x", URI.class)).thenReturn(instance.getId());
+        when(row.isBound("x_singularStringAtt")).thenReturn(true);
+        final String concatenated = "value-en@en"
+                + AttributeEnumeratingSparqlAssemblyModifier.GROUP_CONCAT_SEPARATOR + "hodnota-cs@cs";
+        when(row.getString("x_singularStringAtt")).thenReturn(concatenated);
+        when(uow.readObjectFromAxioms(eq(OWLClassU.class), anyCollection(), any(AxiomBasedLoadingConfigGroup.class))).thenReturn(instance);
+
+        final Optional<OWLClassU> intermediate = sut.loadResult(row);
+        assertFalse(intermediate.isPresent());
+        final Optional<OWLClassU> result = sut.loadLastPending();
+        assertTrue(result.isPresent());
+        final ArgumentCaptor<Collection<Axiom<?>>> captor = ArgumentCaptor.forClass(Collection.class);
+        verify(uow).readObjectFromAxioms(eq(OWLClassU.class), captor.capture(), any(AxiomBasedLoadingConfigGroup.class));
+        captor.getValue().forEach(ax -> assertEquals(instance.getId(), ax.getSubject().getIdentifier()));
+        assertTrue(captor.getValue().stream()
+                         .anyMatch(ax -> ax.getValue().getValue().equals(new LangString("value-en", "en"))));
+        assertTrue(captor.getValue().stream()
+                         .anyMatch(ax -> ax.getValue().getValue().equals(new LangString("hodnota-cs", "cs"))));
+    }
 }

jopa-impl/src/test/java/cz/cvut/kbss/jopa/query/sparql/loader/AttributeEnumeratingSparqlAssemblyModifierTest.java

@@ -19,6 +19,7 @@
 
 import cz.cvut.kbss.jopa.environment.OWLClassJ;
 import cz.cvut.kbss.jopa.environment.OWLClassP;
+import cz.cvut.kbss.jopa.environment.OWLClassU;
 import cz.cvut.kbss.jopa.environment.Vocabulary;
 import cz.cvut.kbss.jopa.environment.utils.Generators;
 import cz.cvut.kbss.jopa.environment.utils.MetamodelMocks;
@@ -251,6 +252,36 @@ void modifyGroupConcatenatesOnlyMostSpecificEligibleAttributeToPreserveGrouping(
         assertThat(result, containsString("GROUP BY ?x ?x_owlClassA"));
     }
 
+    @Test
+    void modifyGroupConcatenatesSingularMultilingualStringAttributeValues() {
+        final TokenStreamSparqlQueryHolder holder = parser.parseQuery("SELECT ?x WHERE { ?x a ?type . }");
+        final EntityGraph<OWLClassU> fetchGraph = new EntityGraphImpl<>(metamodelMocks.forOwlClassU()
+                                                                                      .entityType(), metamodel);
+        fetchGraph.addAttributeNodes("singularStringAtt", "modified");
+        final AttributeEnumeratingSparqlAssemblyModifier sut = createSut(metamodelMocks.forOwlClassU()
+                                                                                       .entityType(), new EntityDescriptor(), fetchGraph);
+        holder.setAssemblyModifier(sut);
+
+        final String result = holder.assembleQuery();
+        assertThat(result, containsString("(GROUP_CONCAT(DISTINCT CONCAT(STR(?x_singularStringAtt), \"@\", LANG(?x_singularStringAtt)); SEPARATOR='" + GROUP_CONCAT_SEPARATOR + "') AS ?x_singularStringAtt_gc)"));
+        assertThat(result, containsString("GROUP BY ?x ?x_modified"));
+    }
+
+    @Test
+    void modifyGroupConcatenatesPluralMultilingualStringAttributeValues() {
+        final TokenStreamSparqlQueryHolder holder = parser.parseQuery("SELECT ?x WHERE { ?x a ?type . }");
+        final EntityGraph<OWLClassU> fetchGraph = new EntityGraphImpl<>(metamodelMocks.forOwlClassU()
+                                                                                      .entityType(), metamodel);
+        fetchGraph.addAttributeNodes("pluralStringAtt", "modified");
+        final AttributeEnumeratingSparqlAssemblyModifier sut = createSut(metamodelMocks.forOwlClassU()
+                                                                                       .entityType(), new EntityDescriptor(), fetchGraph);
+        holder.setAssemblyModifier(sut);
+
+        final String result = holder.assembleQuery();
+        assertThat(result, containsString("(GROUP_CONCAT(DISTINCT CONCAT(STR(?x_pluralStringAtt), \"@\", LANG(?x_pluralStringAtt)); SEPARATOR='" + GROUP_CONCAT_SEPARATOR + "') AS ?x_pluralStringAtt_gc)"));
+        assertThat(result, containsString("GROUP BY ?x ?x_modified"));
+    }
+
     /**
      * Simplified Hamcrest matcher that checks if a SPARQL query projects a specific variable. The matcher extracts the
      * projection part of the query (everything before the WHERE clause) and checks if it contains the specified

jopa-impl/src/test/java/cz/cvut/kbss/jopa/query/sparql/loader/EntityMappingQueryModifierTest.java

@@ -98,7 +98,7 @@ void modifyBindsEntityTypeAndGeneratesQueryVariableMappingForItWhenEntityClassHa
         final EntityMappingQueryModifier.QueryModification result = sut.modify(fetchGraph, "x");
         assertEquals(List.of("x_stringAttribute", "x_types"), result.variableNames());
         final Optional<QueryVariableMapping> typesMapping = result.variables().stream()
-                                                                  .filter(QueryVariableMapping::isTypes)
+                                                                  .filter(qvm -> "x_types".equals(qvm.attributeVar()))
                                                                   .findFirst();
         assertTrue(typesMapping.isPresent());
         assertNull(typesMapping.get().attribute());

jopa-impl/src/test/java/cz/cvut/kbss/jopa/sessions/cache/TtlCacheManagerTest.java

@@ -19,6 +19,7 @@
 
 import cz.cvut.kbss.jopa.model.JOPAPersistenceProperties;
 import cz.cvut.kbss.jopa.model.descriptors.Descriptor;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
@@ -41,6 +42,11 @@ public void setUp() throws Exception {
         this.manager = new TtlCacheManager(Collections.emptyMap());
     }
 
+    @AfterEach
+    public void tearDown() {
+        manager.close();
+    }
+
     @Test
     public void testConstructorNumberInvalid() throws Exception {
         final Map<String, String> m = new HashMap<>();

jopa-integration-tests-owlapi/src/test/java/cz/cvut/kbss/jopa/test/query/owlapi/TypedQueryTest.java

@@ -130,4 +130,9 @@ public void optimizedEntityLoadingWorksForEntityClassesWithoutTypesField() {
     public void querySupportsFetchGraphs() {
         // OWL2Query does not support OPTIONAL
     }
+
+    @Override
+    public void querySupportsGroupConcatMultilingualStringsBasedOnFetchGraph() {
+        // OWL2Query does not support OPTIONAL and GROUP_CONCAT
+    }
 }