diff --git a/deploy/shared/db-server/Dockerfile b/deploy/shared/db-server/Dockerfile index ee37b33f..4967d3e9 100644 --- a/deploy/shared/db-server/Dockerfile +++ b/deploy/shared/db-server/Dockerfile @@ -8,18 +8,29 @@ ENV GRAPHDB_INSTALL_DIR=/opt/graphdb/dist WORKDIR ${GRAPHDB_HOME} -# Install libs related to RDF processing +# Install libs related to RDF processing. Both deps come from pip (not the OS +# package manager), so pip owns them cleanly and --break-system-packages only +# bypasses PEP 668 without clobbering any OS package: +# - rdflib — used directly by the import helper scripts (get-rdf-*.py), +# listed explicitly so it doesn't rely on pyshacl pulling it in; +# - pyshacl — SHACL validation. ### for arm64 RUN if command -v apt >/dev/null; then \ apt update && \ - apt install -y python3-rdflib && \ - apt install -y liburi-perl; \ + apt install -y python3-pip && \ + apt install -y liburi-perl && \ + pip install --no-cache-dir --break-system-packages \ + rdflib \ + pyshacl; \ fi ### for amd64 RUN if command -v apk >/dev/null; then \ - apk add py3-rdflib && \ - apk add perl-uri; \ + apk add py3-pip && \ + apk add perl-uri && \ + pip install --no-cache-dir --break-system-packages \ + rdflib \ + pyshacl; \ fi diff --git a/deploy/shared/db-server/bin/get-rdf-subject-by-type.py b/deploy/shared/db-server/bin/get-rdf-subject-iri-by-type.py similarity index 91% rename from deploy/shared/db-server/bin/get-rdf-subject-by-type.py rename to deploy/shared/db-server/bin/get-rdf-subject-iri-by-type.py index d8675b77..a49c1b9d 100755 --- a/deploy/shared/db-server/bin/get-rdf-subject-by-type.py +++ b/deploy/shared/db-server/bin/get-rdf-subject-iri-by-type.py @@ -10,9 +10,11 @@ def check_params(): if len(sys.argv) != 3: log(f"""Illegal number of parameters. -Script returns single subject of triple matching the pattern '?result a ' +Script returns single subject of triple matching the pattern '?result a ' from the file specified by . +The subject is printed as a bare IRI string (no angle brackets). + Usage: {sys.argv[0]} Example: {sys.argv[0]} "./init-data/forms/example-1.ttl" "http://onto.fel.cvut.cz/ontologies/form/form-template" @@ -69,7 +71,8 @@ def main(): for row in results: subject = row[0] - print(subject.n3()) + # Print the bare IRI (not n3 <...>), ready to use as a graph context. + print(subject) if __name__ == "__main__": main() diff --git a/deploy/shared/db-server/bin/repo-init.sh b/deploy/shared/db-server/bin/repo-init.sh index 29074f46..06b10677 100755 --- a/deploy/shared/db-server/bin/repo-init.sh +++ b/deploy/shared/db-server/bin/repo-init.sh @@ -31,6 +31,48 @@ wait_for_graphdb() { return 0 } +# +# Generic SHACL validation of import data. +# +# Convention: any data folder (let's call it PARENT) may contain a `shapes/` subfolder. When it does: +# - every file in that `shapes/` subfolder must be named *.shapes.ttl +# - VALIDATION_CANDIDATES are all *.ttl files recursively in PARENT ignoring any `shapes` subfolders +# - VALIDATION_CANDIDATES are validated against all shapes files from PARENT/`shapes` subfolder +# +# Validation is advisory only: failures are logged as warnings and deployment +# proceeds regardless. +# +validate_shapes_folder() { + local SHAPES_DIR="$1" + local PARENT_DIR="`dirname "$SHAPES_DIR"`" + + # Enforce the naming convention for files living in a shapes folder. + find "$SHAPES_DIR" -maxdepth 1 -type f ! -name '*.shapes.ttl' | while read NON_SHAPE; do + echo "WARNING: file in shapes folder is not named *.shapes.ttl and is ignored: $NON_SHAPE" + done + + local SHAPE_FILES="`find "$SHAPES_DIR" -maxdepth 1 -type f -name '*.shapes.ttl' | sort`" + if [ -z "$SHAPE_FILES" ]; then + echo "INFO: shapes folder $SHAPES_DIR has no *.shapes.ttl files; nothing to validate." + return 0 + fi + + # Validation candidates: all *.ttl files recursively under the parent folder, + # ignoring any `shapes` subfolders (where the shape files themselves live). + local TARGET_FILES="`find "$PARENT_DIR" -type f -name '*.ttl' -not -path '*/shapes/*' | sort`" + if [ -z "$TARGET_FILES" ]; then + echo "INFO: no .ttl files to validate in $PARENT_DIR." + return 0 + fi + + echo "$SHAPE_FILES" | while read SHAPE_FILE; do + echo "INFO: Validating data files in $PARENT_DIR against shapes $SHAPE_FILE ..." + if ! $SCRIPT_DIR/validate-shacl.sh -s "$SHAPE_FILE" $TARGET_FILES; then + echo "WARNING: SHACL validation reported failures for shapes $SHAPE_FILE (see report above). Continuing with deployment." + fi + done +} + ############ ### MAIN ### ############ @@ -68,13 +110,18 @@ done DATA_DIR=/root/graphdb-import cd / +echo "INFO: *** Validating import data against SHACL shapes folders ***" +find "$DATA_DIR" -type d -name shapes | sort | while read SHAPES_DIR; do + validate_shapes_folder "$SHAPES_DIR" +done + for DIR in ${DATA_DIR}/*/; do REPO_NAME="`basename ${DIR}`" echo "INFO: Updating data in repository $REPO_NAME ..." find ${DATA_DIR}/${REPO_NAME} -name '*-form.ttl' | while read DATA_FILE; do - CONTEXT=`$SCRIPT_DIR/get-rdf-subject-by-type.py $DATA_FILE 'http://onto.fel.cvut.cz/ontologies/form/form-template' | sed 's/[<>]//g'` + CONTEXT=`$SCRIPT_DIR/get-rdf-subject-iri-by-type.py $DATA_FILE 'http://onto.fel.cvut.cz/ontologies/form/form-template'` || continue echo "INFO: Replacing context ${CONTEXT} with form template from file ${DATA_FILE}." $SCRIPT_DIR/rdf4j-deploy-context.sh -R -C 'text/turtle' -s http://localhost:7200 -r ${REPO_NAME} -c ${CONTEXT} ${DATA_FILE} diff --git a/deploy/shared/db-server/bin/validate-shacl.sh b/deploy/shared/db-server/bin/validate-shacl.sh new file mode 100755 index 00000000..5c8a4688 --- /dev/null +++ b/deploy/shared/db-server/bin/validate-shacl.sh @@ -0,0 +1,134 @@ +#!/bin/bash + +# +# Validates RDF data files against a SHACL shapes graph using pyshacl. +# +# Runs `pyshacl` directly, so it is meant to be executed INSIDE the db-server +# (GraphDB) container, where pyshacl is installed by the Dockerfile. This lets +# import data be checked with the same toolchain that deploys it, before any +# data reaches GraphDB. +# +# This script is generic (not tied to forms). repo-init.sh calls it for every +# `shapes/` folder it finds under the import tree; it can also be run by hand: +# +# docker compose exec db-server \ +# /opt/graphdb/dist/bin/validate-shacl.sh \ +# -s /root/graphdb-import/record-manager-formgen/forms/shapes/form-template.shapes.ttl \ +# /root/graphdb-import/record-manager-formgen/forms/example-1-form.ttl +# +# Exit code: 0 if every data file conforms, 1 if any has violations or errors, +# 2 on usage / missing-file errors. +# + +set -u + +INFERENCE="none" +ONT_GRAPH="" +SHAPES="" +QUIET=false + +print_usage() { + cat < [-e ] [-i ] [-q] DATA_FILES... + +Options: + -s SHAPES SHACL shapes graph to validate against (required). + -e ONTOLOGY Extra ontology / data graph passed to pyshacl (--ont-graph), + useful when shapes rely on owl:imports or class hierarchies. + -i INFERENCE Inferencing before validation: none (default), rdfs, owlrl, both. + -q Quiet: print only the PASS/FAIL line per file (no violation report). + -h Show this help. + +DATA_FILES are the RDF files to validate (one or more). The data format is +auto-detected from each file's extension by pyshacl. + +Examples: + $0 -s shapes/form-template.shapes.ttl forms/example-1-form.ttl forms/vita-form.ttl +EOF +} + +while getopts "s:e:i:qh" arg; do + case $arg in + s) SHAPES=$OPTARG ;; + e) ONT_GRAPH=$OPTARG ;; + i) INFERENCE=$OPTARG ;; + q) QUIET=true ;; + h) print_usage; exit 0 ;; + *) print_usage; exit 2 ;; + esac +done +shift $((OPTIND - 1)) + +if ! command -v pyshacl >/dev/null 2>&1; then + echo "ERROR: pyshacl not found on PATH. Run this inside the db-server container," >&2 + echo " e.g. docker compose exec db-server $0 ..." >&2 + exit 2 +fi + +if [ -z "$SHAPES" ]; then + echo "ERROR: a SHACL shapes file is required (-s)." >&2 + print_usage >&2 + exit 2 +fi +if [ ! -f "$SHAPES" ]; then + echo "ERROR: shapes file not found: $SHAPES" >&2 + exit 2 +fi +if [ -n "$ONT_GRAPH" ] && [ ! -f "$ONT_GRAPH" ]; then + echo "ERROR: ontology file not found: $ONT_GRAPH" >&2 + exit 2 +fi +if [ "$#" -eq 0 ]; then + echo "ERROR: no data files given." >&2 + print_usage >&2 + exit 2 +fi + +ONT_ARGS=() +[ -n "$ONT_GRAPH" ] && ONT_ARGS=(-e "$ONT_GRAPH") + +echo "INFO: *** SHACL VALIDATION ***" +echo "INFO: - shapes: $SHAPES" +[ -n "$ONT_GRAPH" ] && echo "INFO: - ontology: $ONT_GRAPH" +echo "INFO: - inference: $INFERENCE" +echo "INFO: - files: $#" +echo + +FAILED=0 +TOTAL=0 +for DATA in "$@"; do + TOTAL=$((TOTAL + 1)) + if [ ! -f "$DATA" ]; then + echo "FAIL $DATA (file not found)" + FAILED=$((FAILED + 1)) + continue + fi + + OUTPUT="$(pyshacl \ + -s "$SHAPES" \ + "${ONT_ARGS[@]}" \ + -i "$INFERENCE" \ + -f human \ + "$DATA" 2>&1)" + STATUS=$? + + if [ "$STATUS" -eq 0 ]; then + echo "PASS $DATA" + else + echo "FAIL $DATA" + [ "$QUIET" = false ] && echo "$OUTPUT" | sed 's/^/ /' + FAILED=$((FAILED + 1)) + fi +done + +echo +if [ "$FAILED" -eq 0 ]; then + echo "INFO: all $TOTAL file(s) conform." + exit 0 +else + echo "ERROR: $FAILED of $TOTAL file(s) failed validation." + exit 1 +fi diff --git a/deploy/shared/db-server/import/record-manager-formgen/forms/shapes/form-template.shapes.ttl b/deploy/shared/db-server/import/record-manager-formgen/forms/shapes/form-template.shapes.ttl new file mode 100644 index 00000000..aca22ca7 --- /dev/null +++ b/deploy/shared/db-server/import/record-manager-formgen/forms/shapes/form-template.shapes.ttl @@ -0,0 +1,37 @@ +# SHACL shapes for the Record Manager "form template" RDF pattern. +# +# A resource is recognized as a form template by Record Manager +# (FormGenService.getFormTemplates / query/findFormTemplates.rq) only when ALL of: +# +# 1. it lives in a named graph whose IRI equals the template IRI; +# 2. it is typed form:form-template +# (http://onto.fel.cvut.cz/ontologies/form/form-template); +# 3. it has an rdfs:label; +# 4. it has an rdfs:comment. +# +# This shape enforces requirements (2)-(4) — the triple-level pattern. +# Requirement (1), the named-graph constraint, cannot be expressed in core SHACL +# (validation runs over the merged data graph, not over graph names); it is enforced +# by the consuming query findFormTemplates.rq. See doc/form-templates.md. + +@prefix sh: . +@prefix rdfs: . +@prefix form: . + +form:FormTemplateShape + a sh:NodeShape ; + rdfs:label "Form template shape" ; + rdfs:comment "Validates the triple-level pattern required for a resource to be recognized as a form template." ; + # Targets every resource typed form:form-template; unrelated subjects are ignored, + # so the shape can be run against arbitrary TTL files. + sh:targetClass form:form-template ; + sh:property [ + sh:path rdfs:label ; + sh:minCount 1 ; + sh:message "A form template must have at least one rdfs:label; otherwise it is silently dropped from the form template listing." ; + ] ; + sh:property [ + sh:path rdfs:comment ; + sh:minCount 1 ; + sh:message "A form template must have at least one rdfs:comment; otherwise it is silently dropped from the form template listing." ; + ] .