Sync db-server SHACL validation from deployment repo

deploy/shared/db-server/Dockerfile

@@ -8,18 +8,29 @@ ENV GRAPHDB_INSTALL_DIR=/opt/graphdb/dist
 
 WORKDIR ${GRAPHDB_HOME}
 
-# Install libs related to RDF processing
+# Install libs related to RDF processing. Both deps come from pip (not the OS
+# package manager), so pip owns them cleanly and --break-system-packages only
+# bypasses PEP 668 without clobbering any OS package:
+#   - rdflib  — used directly by the import helper scripts (get-rdf-*.py),
+#               listed explicitly so it doesn't rely on pyshacl pulling it in;
+#   - pyshacl — SHACL validation.
 ### for arm64
 RUN if command -v apt >/dev/null; then \
     apt update && \
-    apt install -y python3-rdflib && \
-    apt install -y liburi-perl; \
+    apt install -y python3-pip && \
+    apt install -y liburi-perl && \
+    pip install --no-cache-dir --break-system-packages \
+        rdflib \
+        pyshacl; \
 fi
 
 ### for amd64
 RUN if command -v apk >/dev/null; then \
-    apk add py3-rdflib && \
-    apk add perl-uri; \
+    apk add py3-pip && \
+    apk add perl-uri && \
+    pip install --no-cache-dir --break-system-packages \
+        rdflib \
+        pyshacl; \
 fi
 
 

deploy/shared/db-server/bin/get-rdf-subject-by-type.py → deploy/shared/db-server/bin/get-rdf-subject-iri-by-type.py

@@ -10,9 +10,11 @@ def check_params():
     if len(sys.argv) != 3:
         log(f"""Illegal number of parameters.
 
-Script returns single subject of triple matching the pattern '?result a <type-uri>' 
+Script returns single subject of triple matching the pattern '?result a <type-uri>'
 from the file specified by <rdf-file-path>.
 
+The subject is printed as a bare IRI string (no angle brackets).
+
 Usage: {sys.argv[0]} <rdf-file-path> <type-uri>
 
 Example: {sys.argv[0]} "./init-data/forms/example-1.ttl" "http://onto.fel.cvut.cz/ontologies/form/form-template"
@@ -69,7 +71,8 @@ def main():
 
     for row in results:
         subject = row[0]
-        print(subject.n3())
+        # Print the bare IRI (not n3 <...>), ready to use as a graph context.
+        print(subject)
 
 if __name__ == "__main__":
     main()

deploy/shared/db-server/bin/repo-init.sh

@@ -31,6 +31,48 @@ wait_for_graphdb() {
     return 0
 }
 
+#
+# Generic SHACL validation of import data.
+#
+# Convention: any data folder (let's call it PARENT) may contain a `shapes/` subfolder. When it does:
+#   - every file in that `shapes/` subfolder must be named *.shapes.ttl
+#   - VALIDATION_CANDIDATES are all *.ttl files recursively in PARENT ignoring any `shapes` subfolders
+#   - VALIDATION_CANDIDATES are validated against all shapes files from PARENT/`shapes` subfolder
+#
+# Validation is advisory only: failures are logged as warnings and deployment
+# proceeds regardless.
+#
+validate_shapes_folder() {
+    local SHAPES_DIR="$1"
+    local PARENT_DIR="`dirname "$SHAPES_DIR"`"
+
+    # Enforce the naming convention for files living in a shapes folder.
+    find "$SHAPES_DIR" -maxdepth 1 -type f ! -name '*.shapes.ttl' | while read NON_SHAPE; do
+        echo "WARNING: file in shapes folder is not named *.shapes.ttl and is ignored: $NON_SHAPE"
+    done
+
+    local SHAPE_FILES="`find "$SHAPES_DIR" -maxdepth 1 -type f -name '*.shapes.ttl' | sort`"
+    if [ -z "$SHAPE_FILES" ]; then
+        echo "INFO: shapes folder $SHAPES_DIR has no *.shapes.ttl files; nothing to validate."
+        return 0
+    fi
+
+    # Validation candidates: all *.ttl files recursively under the parent folder,
+    # ignoring any `shapes` subfolders (where the shape files themselves live).
+    local TARGET_FILES="`find "$PARENT_DIR" -type f -name '*.ttl' -not -path '*/shapes/*' | sort`"
+    if [ -z "$TARGET_FILES" ]; then
+        echo "INFO: no .ttl files to validate in $PARENT_DIR."
+        return 0
+    fi
+
+    echo "$SHAPE_FILES" | while read SHAPE_FILE; do
+        echo "INFO: Validating data files in $PARENT_DIR against shapes $SHAPE_FILE ..."
+        if ! $SCRIPT_DIR/validate-shacl.sh -s "$SHAPE_FILE" $TARGET_FILES; then
+            echo "WARNING: SHACL validation reported failures for shapes $SHAPE_FILE (see report above). Continuing with deployment."
+        fi
+    done
+}
+
 ############
 ### MAIN ###
 ############
@@ -68,13 +110,18 @@ done
 DATA_DIR=/root/graphdb-import
 cd /
 
+echo "INFO: *** Validating import data against SHACL shapes folders ***"
+find "$DATA_DIR" -type d -name shapes | sort | while read SHAPES_DIR; do
+    validate_shapes_folder "$SHAPES_DIR"
+done
+
 for DIR in ${DATA_DIR}/*/; do
     REPO_NAME="`basename ${DIR}`"
 
     echo "INFO: Updating data in repository $REPO_NAME ..."
 
     find ${DATA_DIR}/${REPO_NAME} -name '*-form.ttl' | while read DATA_FILE; do
-	CONTEXT=`$SCRIPT_DIR/get-rdf-subject-by-type.py $DATA_FILE 'http://onto.fel.cvut.cz/ontologies/form/form-template' | sed 's/[<>]//g'`
+	CONTEXT=`$SCRIPT_DIR/get-rdf-subject-iri-by-type.py $DATA_FILE 'http://onto.fel.cvut.cz/ontologies/form/form-template'` || continue
 
 	echo "INFO: Replacing context ${CONTEXT} with form template from file ${DATA_FILE}."
 	$SCRIPT_DIR/rdf4j-deploy-context.sh -R -C 'text/turtle' -s http://localhost:7200 -r ${REPO_NAME} -c ${CONTEXT} ${DATA_FILE}

deploy/shared/db-server/bin/validate-shacl.sh

@@ -0,0 +1,134 @@
+#!/bin/bash
+
+#
+# Validates RDF data files against a SHACL shapes graph using pyshacl.
+#
+# Runs `pyshacl` directly, so it is meant to be executed INSIDE the db-server
+# (GraphDB) container, where pyshacl is installed by the Dockerfile. This lets
+# import data be checked with the same toolchain that deploys it, before any
+# data reaches GraphDB.
+#
+# This script is generic (not tied to forms). repo-init.sh calls it for every
+# `shapes/` folder it finds under the import tree; it can also be run by hand:
+#
+#   docker compose exec db-server \
+#     /opt/graphdb/dist/bin/validate-shacl.sh \
+#       -s /root/graphdb-import/record-manager-formgen/forms/shapes/form-template.shapes.ttl \
+#       /root/graphdb-import/record-manager-formgen/forms/example-1-form.ttl
+#
+# Exit code: 0 if every data file conforms, 1 if any has violations or errors,
+#            2 on usage / missing-file errors.
+#
+
+set -u
+
+INFERENCE="none"
+ONT_GRAPH=""
+SHAPES=""
+QUIET=false
+
+print_usage() {
+  cat <<EOF
+SHACL validator (runs inside the db-server container).
+
+Usage:
+  $0 -s <SHAPES.ttl> [-e <ONTOLOGY.ttl>] [-i <none|rdfs|owlrl|both>] [-q] DATA_FILES...
+
+Options:
+  -s SHAPES     SHACL shapes graph to validate against (required).
+  -e ONTOLOGY   Extra ontology / data graph passed to pyshacl (--ont-graph),
+                useful when shapes rely on owl:imports or class hierarchies.
+  -i INFERENCE  Inferencing before validation: none (default), rdfs, owlrl, both.
+  -q            Quiet: print only the PASS/FAIL line per file (no violation report).
+  -h            Show this help.
+
+DATA_FILES are the RDF files to validate (one or more). The data format is
+auto-detected from each file's extension by pyshacl.
+
+Examples:
+  $0 -s shapes/form-template.shapes.ttl forms/example-1-form.ttl forms/vita-form.ttl
+EOF
+}
+
+while getopts "s:e:i:qh" arg; do
+  case $arg in
+    s) SHAPES=$OPTARG ;;
+    e) ONT_GRAPH=$OPTARG ;;
+    i) INFERENCE=$OPTARG ;;
+    q) QUIET=true ;;
+    h) print_usage; exit 0 ;;
+    *) print_usage; exit 2 ;;
+  esac
+done
+shift $((OPTIND - 1))
+
+if ! command -v pyshacl >/dev/null 2>&1; then
+  echo "ERROR: pyshacl not found on PATH. Run this inside the db-server container," >&2
+  echo "       e.g.  docker compose exec db-server $0 ..." >&2
+  exit 2
+fi
+
+if [ -z "$SHAPES" ]; then
+  echo "ERROR: a SHACL shapes file is required (-s)." >&2
+  print_usage >&2
+  exit 2
+fi
+if [ ! -f "$SHAPES" ]; then
+  echo "ERROR: shapes file not found: $SHAPES" >&2
+  exit 2
+fi
+if [ -n "$ONT_GRAPH" ] && [ ! -f "$ONT_GRAPH" ]; then
+  echo "ERROR: ontology file not found: $ONT_GRAPH" >&2
+  exit 2
+fi
+if [ "$#" -eq 0 ]; then
+  echo "ERROR: no data files given." >&2
+  print_usage >&2
+  exit 2
+fi
+
+ONT_ARGS=()
+[ -n "$ONT_GRAPH" ] && ONT_ARGS=(-e "$ONT_GRAPH")
+
+echo "INFO: *** SHACL VALIDATION ***"
+echo "INFO:     - shapes:    $SHAPES"
+[ -n "$ONT_GRAPH" ] && echo "INFO:     - ontology:  $ONT_GRAPH"
+echo "INFO:     - inference: $INFERENCE"
+echo "INFO:     - files:     $#"
+echo
+
+FAILED=0
+TOTAL=0
+for DATA in "$@"; do
+  TOTAL=$((TOTAL + 1))
+  if [ ! -f "$DATA" ]; then
+    echo "FAIL  $DATA  (file not found)"
+    FAILED=$((FAILED + 1))
+    continue
+  fi
+
+  OUTPUT="$(pyshacl \
+    -s "$SHAPES" \
+    "${ONT_ARGS[@]}" \
+    -i "$INFERENCE" \
+    -f human \
+    "$DATA" 2>&1)"
+  STATUS=$?
+
+  if [ "$STATUS" -eq 0 ]; then
+    echo "PASS  $DATA"
+  else
+    echo "FAIL  $DATA"
+    [ "$QUIET" = false ] && echo "$OUTPUT" | sed 's/^/          /'
+    FAILED=$((FAILED + 1))
+  fi
+done
+
+echo
+if [ "$FAILED" -eq 0 ]; then
+  echo "INFO: all $TOTAL file(s) conform."
+  exit 0
+else
+  echo "ERROR: $FAILED of $TOTAL file(s) failed validation."
+  exit 1
+fi

deploy/shared/db-server/import/record-manager-formgen/forms/shapes/form-template.shapes.ttl

@@ -0,0 +1,37 @@
+# SHACL shapes for the Record Manager "form template" RDF pattern.
+#
+# A resource is recognized as a form template by Record Manager
+# (FormGenService.getFormTemplates / query/findFormTemplates.rq) only when ALL of:
+#
+#   1. it lives in a named graph whose IRI equals the template IRI;
+#   2. it is typed form:form-template
+#      (http://onto.fel.cvut.cz/ontologies/form/form-template);
+#   3. it has an rdfs:label;
+#   4. it has an rdfs:comment.
+#
+# This shape enforces requirements (2)-(4) — the triple-level pattern.
+# Requirement (1), the named-graph constraint, cannot be expressed in core SHACL
+# (validation runs over the merged data graph, not over graph names); it is enforced
+# by the consuming query findFormTemplates.rq. See doc/form-templates.md.
+
+@prefix sh:   <http://www.w3.org/ns/shacl#> .
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+@prefix form: <http://onto.fel.cvut.cz/ontologies/form/> .
+
+form:FormTemplateShape
+    a sh:NodeShape ;
+    rdfs:label "Form template shape" ;
+    rdfs:comment "Validates the triple-level pattern required for a resource to be recognized as a form template." ;
+    # Targets every resource typed form:form-template; unrelated subjects are ignored,
+    # so the shape can be run against arbitrary TTL files.
+    sh:targetClass form:form-template ;
+    sh:property [
+        sh:path     rdfs:label ;
+        sh:minCount 1 ;
+        sh:message  "A form template must have at least one rdfs:label; otherwise it is silently dropped from the form template listing." ;
+    ] ;
+    sh:property [
+        sh:path     rdfs:comment ;
+        sh:minCount 1 ;
+        sh:message  "A form template must have at least one rdfs:comment; otherwise it is silently dropped from the form template listing." ;
+    ] .