Add sparql to-jelly/from-jelly commands (#310)

* Use snapshots of Jelly-JVM

Will be needed to add SPARQL support. We will move back to stable releases before the next cli release.

* Add sparql to-jelly/from-jelly commands

This wasn't too bad to integrate.

Author: Ostrzyciel

.github/workflows/aot-test.yml

@@ -81,6 +81,19 @@ jobs:
           ./jelly-cli \
             rdf validate out.jelly --compare-to-rdf-file in.nt || exit 1
 
+          # Test SPARQL result set conversions
+          echo '{"head":{"vars":["a"]},"results":{"bindings":[{"a":{"type":"uri","value":"http://e.org/x"}}]}}' > in.srj
+          ./jelly-cli \
+            sparql to-jelly in.srj > out.jellys && \
+            [ -s out.jellys ] || exit 1
+          ./jelly-cli \
+            sparql from-jelly --out-format=csv out.jellys | grep 'http://e.org/x' || exit 1
+          # ASK results take a different code path than bindings
+          echo '{"head":{},"boolean":true}' | \
+            ./jelly-cli sparql to-jelly --in-format=json > ask.jellys && \
+            [ -s ask.jellys ] || exit 1
+          ./jelly-cli sparql from-jelly ask.jellys | grep 'true' || exit 1
+
       - name: Upload binary
         uses: actions/upload-artifact@v4
         with:

README.md

@@ -104,6 +104,24 @@ jelly-cli rdf validate input.jelly
 
 You can also check whether the Jelly file has been encoded using specific stream options or is equivalent to another RDF file, with the use of additional options to this command.
 
+### Convert SPARQL results to and from Jelly-SPARQL
+
+Jelly-SPARQL is a columnar format for SPARQL query results. To convert results to it, run:
+
+```shell
+jelly-cli sparql to-jelly results.srj > results.jellys
+```
+
+And to convert back:
+
+```shell
+jelly-cli sparql from-jelly results.jellys --out-format=csv > results.csv
+```
+
+Both commands handle SELECT results (bindings) and ASK results (a boolean). All standard result formats (JSON, XML, CSV and TSV) are supported, plus a plain text table (`text`) for output only.
+
+Jelly-SPARQL is an experimental draft and the format may still change.
+
 ### General tips
 
 Use the `--help` option to learn more about all the available settings:
@@ -114,6 +132,8 @@ jelly-cli rdf from-jelly --help
 jelly-cli rdf transcode --help
 jelly-cli rdf inspect --help
 jelly-cli rdf validate --help
+jelly-cli sparql to-jelly --help
+jelly-cli sparql from-jelly --help
 ```
 
 And use the `--debug` option to get more information about any exceptions you encounter.

build.sbt

@@ -8,10 +8,10 @@ ThisBuild / scalaVersion := scalaV
 Global / lintUnusedKeysOnLoad := false
 
 resolvers +=
-  "Sonatype OSS Snapshots" at "https://s01.oss.sonatype.org/content/repositories/snapshots"
+  "Sonatype OSS Snapshots" at "https://central.sonatype.com/repository/maven-snapshots"
 
 lazy val jenaV = "6.2.0"
-lazy val jellyV = "3.7.3"
+lazy val jellyV = "3.7.3+33-28c9f700-SNAPSHOT"
 lazy val graalvmV = "25.2.4"
 
 addCommandAlias("fixAll", "scalafixAll; scalafmtAll")
@@ -67,6 +67,8 @@ lazy val root = (project in file("."))
       "org.apache.jena" % "jena-arq" % jenaV,
       // Jelly-JVM 3.7.x pins Jena 5.6.x as a dependency, we must exclude it, because we use Jena 6.x.
       ("eu.neverblink.jelly" % "jelly-jena" % jellyV).excludeAll(ExclusionRule("org.apache.jena")),
+      ("eu.neverblink.jelly" % "jelly-jena-sparql" % jellyV)
+        .excludeAll(ExclusionRule("org.apache.jena")),
       "eu.neverblink.jelly" % "jelly-core-protos-google" % jellyV,
       "com.github.alexarchambault" %% "case-app" % "2.1.0",
       "org.scalatest" %% "scalatest" % "3.2.20" % "test,test-serial",

src/main/java/eu/neverblink/jelly/cli/graal/GraalSubstitutes.java

@@ -7,6 +7,7 @@
 import com.google.protobuf.TextFormat;
 import com.oracle.svm.core.annotate.*;
 
+import java.io.File;
 import java.net.URI;
 import java.nio.charset.Charset;
 import java.util.UUID;
@@ -83,6 +84,24 @@ public static String createFreshId() {
     }
 }
 
+/**
+ * Jena's data bags spill to a temporary file once they outgrow their in-memory threshold, and name
+ * that file with a secure random UUID. The SPARQL results JSON reader buffers rows in a data bag
+ * when it has to read past the bindings to find the header, which drags secure random number
+ * generation back into the binary.
+ * <p>
+ * The file name only has to be unique, so a pseudo-random UUID does the job here.
+ */
+@TargetClass(className = "org.apache.jena.atlas.data.AbstractDataBag")
+final class AbstractDataBagSubstitute {
+    @Substitute
+    protected File getNewTemporaryFile() {
+        ThreadLocalRandom r = ThreadLocalRandom.current();
+        File sysTempDir = new File(System.getProperty("java.io.tmpdir"));
+        return new File(sysTempDir, "DataBag-" + new UUID(r.nextLong(), r.nextLong()) + ".tmp");
+    }
+}
+
 /**
  * Disable UTF-32LE support in JSON parsers, which we don't need.
  * This allows us to avoid including all charsets in the native image, which saves quite a bit of space.

src/main/scala/eu/neverblink/jelly/cli/App.scala

@@ -3,7 +3,9 @@ package eu.neverblink.jelly.cli
 import caseapp.*
 import eu.neverblink.jelly.cli.command.*
 import eu.neverblink.jelly.cli.command.rdf.*
+import eu.neverblink.jelly.cli.command.sparql.*
 import eu.neverblink.jelly.cli.util.jena.riot.CliRiot
+import eu.neverblink.jelly.convert.jena.sparql.JellySparqlLanguage
 import org.apache.jena.sys.JenaSystem
 
 /** Main entrypoint.
@@ -14,6 +16,9 @@ object App extends CommandsEntryPoint:
   JenaSystem.init()
   // Initialize the CLI Riot parsers
   CliRiot.initialize()
+  // JenaSystem.init() already does this via the subsystem lifecycle, but that relies on service
+  // discovery, which we'd rather not depend on in native-image builds. The call is idempotent.
+  JellySparqlLanguage.register()
 
   override def enableCompletionsCommand: Boolean = true
 
@@ -28,4 +33,6 @@ object App extends CommandsEntryPoint:
     RdfTranscode,
     RdfInspect,
     RdfValidate,
+    SparqlFromJelly,
+    SparqlToJelly,
   )

src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlFromJelly.scala

@@ -0,0 +1,50 @@
+package eu.neverblink.jelly.cli.command.sparql
+
+import caseapp.*
+import eu.neverblink.jelly.cli.*
+import eu.neverblink.jelly.cli.command.sparql.util.SparqlFormat
+
+object SparqlFromJellyPrint:
+  val validFormats: List[SparqlFormat] = SparqlFormat.writeable
+  val defaultFormat: SparqlFormat = SparqlFormat.Json
+  lazy val helpMsg: String = SparqlFormat.helpMsg(validFormats, defaultFormat)
+
+@HelpMessage(
+  "Translates a Jelly-SPARQL stream to a different SPARQL result set format. \n" +
+    "If no input file is specified, the input is read from stdin.\n" +
+    "If no output file is specified, the output is written to stdout.\n" +
+    "Both SELECT results (bindings) and ASK results (a boolean) are supported.\n" +
+    "If an error is detected, the program will exit with a non-zero code.\n" +
+    "Otherwise, the program will exit with code 0.",
+)
+@ArgsName("<file-to-convert>")
+case class SparqlFromJellyOptions(
+    @Recurse
+    common: JellyCommandOptions = JellyCommandOptions(),
+    @HelpMessage(
+      "Output file to write the SPARQL results to. If not specified, the output is written to stdout.",
+    )
+    @ExtraName("to") outputFile: Option[String] = None,
+    @HelpMessage(
+      "Format the Jelly-SPARQL stream should be translated to. " +
+        "If not explicitly specified, but output file supplied, the format is inferred from the file name. " +
+        SparqlFromJellyPrint.helpMsg,
+    )
+    @ExtraName("out-format") outputFormat: Option[String] = None,
+) extends HasJellyCommandOptions
+
+object SparqlFromJelly extends SparqlSerDesCommand[SparqlFromJellyOptions]:
+
+  override def names: List[List[String]] = List(
+    List("sparql", "from-jelly"),
+  )
+
+  override val validFormats: List[SparqlFormat] = SparqlFromJellyPrint.validFormats
+
+  override val defaultFormat: SparqlFormat = SparqlFromJellyPrint.defaultFormat
+
+  override def doRun(options: SparqlFromJellyOptions, remainingArgs: RemainingArgs): Unit =
+    val inputFile = remainingArgs.remaining.headOption
+    val outputFormat = resolveFormat(options.outputFormat, options.outputFile)
+    val (inputStream, outputStream) = getIoStreamsFromOptions(inputFile, options.outputFile)
+    convert(SparqlFormat.JellySparql, outputFormat, inputStream, outputStream)

src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesCommand.scala

@@ -0,0 +1,74 @@
+package eu.neverblink.jelly.cli.command.sparql
+
+import caseapp.*
+import com.google.protobuf.InvalidProtocolBufferException
+import eu.neverblink.jelly.cli.*
+import eu.neverblink.jelly.cli.command.sparql.util.SparqlFormat
+import eu.neverblink.jelly.core.{RdfProtoDeserializationError, RdfProtoSerializationError}
+import org.apache.jena.riot.{RIOT, RiotException}
+import org.apache.jena.riot.resultset.{ResultSetReaderRegistry, ResultSetWriterRegistry}
+
+import java.io.{InputStream, OutputStream}
+
+/** Common logic for the two SPARQL result set conversion commands.
+  */
+abstract class SparqlSerDesCommand[T <: HasJellyCommandOptions: {Parser, Help}]
+    extends JellyCommand[T]:
+
+  override final def group = "sparql"
+
+  /** Formats the user can pick from for the non-Jelly side of the conversion. */
+  val validFormats: List[SparqlFormat]
+
+  /** Format assumed when the user gives neither an explicit format nor a recognizable file name. */
+  val defaultFormat: SparqlFormat
+
+  /** Picks the non-Jelly format.
+    *
+    * @throws InvalidFormatSpecified
+    *   if the user asked for a format this command cannot handle
+    */
+  final def resolveFormat(format: Option[String], fileName: Option[String]): SparqlFormat =
+    format match
+      case Some(name) =>
+        SparqlFormat.find(name).filter(validFormats.contains).getOrElse {
+          throw InvalidFormatSpecified(name, SparqlFormat.validFormatsString(validFormats))
+        }
+      case None =>
+        fileName
+          .flatMap(SparqlFormat.inferFormat)
+          .filter(validFormats.contains)
+          .getOrElse(defaultFormat)
+
+  /** Reads a result set in one format and writes it back out in another.
+    *
+    * Both SELECT results (bindings) and ASK results (a single boolean) are handled.
+    */
+  final def convert(
+      from: SparqlFormat,
+      to: SparqlFormat,
+      inputStream: InputStream,
+      outputStream: OutputStream,
+  ): Unit =
+    try {
+      val context = RIOT.getContext.copy()
+      val reader = ResultSetReaderRegistry.getFactory(from.jenaLang).create(from.jenaLang)
+      val writer = ResultSetWriterRegistry.getFactory(to.jenaLang).create(to.jenaLang)
+      val result = reader.readAny(inputStream, context)
+      if result.isBoolean then
+        writer.write(outputStream, result.getBooleanResult.booleanValue, context)
+      else writer.write(outputStream, result.getResultSet, context)
+      outputStream.flush()
+    } catch
+      // The Jelly RowSet reader wraps I/O errors (including protobuf ones) in a RiotException,
+      // so unwrap it to report a malformed Jelly file the same way the rdf commands do.
+      case e: RiotException =>
+        e.getCause match
+          case cause: InvalidProtocolBufferException => throw InvalidJellyFile(cause)
+          case _ => throw JenaRiotException(e)
+      case e: InvalidProtocolBufferException =>
+        throw InvalidJellyFile(e)
+      case e: RdfProtoDeserializationError =>
+        throw JellyDeserializationError(e.getMessage)
+      case e: RdfProtoSerializationError =>
+        throw JellySerializationError(e.getMessage)

src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlToJelly.scala

@@ -0,0 +1,50 @@
+package eu.neverblink.jelly.cli.command.sparql
+
+import caseapp.*
+import eu.neverblink.jelly.cli.*
+import eu.neverblink.jelly.cli.command.sparql.util.SparqlFormat
+
+object SparqlToJellyPrint:
+  val validFormats: List[SparqlFormat] = SparqlFormat.readable
+  val defaultFormat: SparqlFormat = SparqlFormat.Json
+  lazy val helpMsg: String = SparqlFormat.helpMsg(validFormats, defaultFormat)
+
+@HelpMessage(
+  "Translates SPARQL query results to a Jelly-SPARQL stream. \n" +
+    "If no input file is specified, the input is read from stdin.\n" +
+    "If no output file is specified, the output is written to stdout.\n" +
+    "Both SELECT results (bindings) and ASK results (a boolean) are supported.\n" +
+    "If an error is detected, the program will exit with a non-zero code.\n" +
+    "Otherwise, the program will exit with code 0.",
+)
+@ArgsName("<file-to-convert>")
+case class SparqlToJellyOptions(
+    @Recurse
+    common: JellyCommandOptions = JellyCommandOptions(),
+    @HelpMessage(
+      "Output file to write the Jelly-SPARQL to. If not specified, the output is written to stdout.",
+    )
+    @ExtraName("to") outputFile: Option[String] = None,
+    @HelpMessage(
+      "Format of the SPARQL results that should be translated to Jelly. " +
+        "If not explicitly specified, but input file supplied, the format is inferred from the file name. " +
+        SparqlToJellyPrint.helpMsg,
+    )
+    @ExtraName("in-format") inputFormat: Option[String] = None,
+) extends HasJellyCommandOptions
+
+object SparqlToJelly extends SparqlSerDesCommand[SparqlToJellyOptions]:
+
+  override def names: List[List[String]] = List(
+    List("sparql", "to-jelly"),
+  )
+
+  override val validFormats: List[SparqlFormat] = SparqlToJellyPrint.validFormats
+
+  override val defaultFormat: SparqlFormat = SparqlToJellyPrint.defaultFormat
+
+  override def doRun(options: SparqlToJellyOptions, remainingArgs: RemainingArgs): Unit =
+    val inputFile = remainingArgs.remaining.headOption
+    val inputFormat = resolveFormat(options.inputFormat, inputFile)
+    val (inputStream, outputStream) = getIoStreamsFromOptions(inputFile, options.outputFile)
+    convert(inputFormat, SparqlFormat.JellySparql, inputStream, outputStream)