diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0508abb..6e5280c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,8 +26,17 @@ jobs: java-version: ${{ matrix.java-version }} cache: 'maven' + - name: Verify vendored upstream integrity + run: ./tools/export-upstream --verify + + - name: Self-check vendoring enforcement + run: ./tools/export-upstream --self-test + - name: Check Code Formatting (Spotless) run: mvn --batch-mode spotless:check + - name: Validate reactor + run: mvn --batch-mode validate + - name: Run Tests and Verifications run: mvn --batch-mode clean verify diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 5a41960..fcc2888 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -27,8 +27,17 @@ jobs: java-version: ${{ matrix.java-version }} cache: 'maven' + - name: Verify vendored upstream integrity + run: ./tools/export-upstream --verify + + - name: Self-check vendoring enforcement + run: ./tools/export-upstream --self-test + - name: Check Code Formatting (Spotless) run: mvn --batch-mode spotless:check + - name: Validate reactor + run: mvn --batch-mode validate + - name: Run Tests and Verifications run: mvn --batch-mode clean verify diff --git a/libsql-conformance/pom.xml b/libsql-conformance/pom.xml index 37a727c..f1eba3d 100644 --- a/libsql-conformance/pom.xml +++ b/libsql-conformance/pom.xml @@ -59,35 +59,58 @@ 3.3.1 - copy-xerial-test-sources - generate-test-sources + copy-docker-config + pre-integration-test copy-resources - ${project.build.directory}/generated-test-sources/xerial + ${project.build.directory}/docker-config - ${project.basedir}/../upstream-xerial_sqlite-jdbc/src/test/java + ${project.basedir}/src/test/docker-config false + + + + org.apache.maven.plugins + maven-antrun-plugin + 3.2.0 + - copy-docker-config - pre-integration-test + verify-vendored-upstream + generate-test-sources - copy-resources + run - ${project.build.directory}/docker-config - - - ${project.basedir}/src/test/docker-config - false - - + + + + + + + + + + copy-xerial-test-sources + generate-test-sources + + run + + + + + + + + diff --git a/tools/export-upstream b/tools/export-upstream index 390059e..524199c 100755 --- a/tools/export-upstream +++ b/tools/export-upstream @@ -4,31 +4,47 @@ set -euo pipefail usage() { cat <<'EOF' Usage: export-upstream [options] + export-upstream --verify [--manifest FILE] + export-upstream --self-test Export xerial/sqlite-jdbc at the given tag into upstream-xerial_sqlite-jdbc/ and update upstream-xerial_sqlite-jdbc/UPSTREAM.md. The tag is resolved to its commit SHA with the GitHub API, and both the tag and the SHA are recorded. +The selected files come from upstream-xerial_sqlite-jdbc.manifest by default. +The manifest is the single source of truth for both the export and the +libsql-conformance build. + Requires the GitHub CLI, authenticated with `gh auth login`. Examples: export-upstream 3.53.4.0 export-upstream 3.53.4.0 --manifest upstream.manifest + export-upstream --verify Options: - --manifest FILE Repo-relative paths to export, one per line. + --manifest FILE Repo-relative paths, one per line. Blank lines and lines starting with # are ignored. - --all Export the whole repository instead of the default slice. + --all Export the whole repository instead of the manifest. --force Replace a non-empty destination not created by this script. + --verify Check the vendored tree against the manifest and + CHECKSUMS.sha256 without contacting GitHub. + --self-test Exercise the verification rules on a temporary copy of + the vendored tree. -h, --help Show this help. -Default export slice when no manifest is given and --all is not used: - src/test/java - src/test/resources - LICENSE - LICENSE.zentus - NOTICE - VERSION +Default manifest: upstream-xerial_sqlite-jdbc.manifest + +Vendored divergences + CHECKSUMS.sha256 records the upstream hash of every exported file on a + "# upstream ./path" line and the current hash on the following line. + A deliberate edit keeps the upstream line, updates the current hash, and + records a rationale in the manifest: + + # divergence: -- + + --verify rejects an edit whose current hash differs from upstream without a + rationale, and rejects a rationale whose file still matches upstream. The destination must be empty, missing, or marked with .export-upstream. The script never edits UPSTREAM.md content outside its managed markers. @@ -44,36 +60,305 @@ md_cell() { printf '%s' "$1" | sed 's/|/\\|/g' } +sha256_file() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + elif command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$1" | awk '{print $1}' + else + die "sha256sum or shasum is required" + fi +} + +sha256_check() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum -c "$1" + elif command -v shasum >/dev/null 2>&1; then + shasum -a 256 -c "$1" + else + die "sha256sum or shasum is required" + fi +} + +check_manifest_path() { + case "$1" in + /*|..|../*|*/../*|*/..) + die "manifest path must be relative and stay inside the repository: $1" + ;; + esac +} + +load_manifest_files() { + local manifest=$1 + local out=$2 + local line + + [ -f "$manifest" ] || die "manifest not found: $manifest" + : > "$out" + while IFS= read -r line || [ -n "$line" ]; do + line=${line%$'\r'} + case "$line" in + ""|"#"*) + continue + ;; + esac + check_manifest_path "$line" + printf '%s\n' "$line" >> "$out" + done < "$manifest" +} + +load_divergences() { + local manifest=$1 + local out=$2 + local line rest path reason + + : > "$out" + while IFS= read -r line || [ -n "$line" ]; do + line=${line%$'\r'} + case "$line" in + "# divergence:"*) + rest=$(printf '%s' "$line" | sed -E 's/^# divergence:[[:space:]]*//') + path=${rest%% *} + reason=$(printf '%s' "$rest" | sed -E 's/^[^ ]+[[:space:]]*(--)?[[:space:]]*//') + if [ -z "$path" ] || [ -z "$reason" ]; then + die "malformed divergence rationale: $line" + fi + check_manifest_path "$path" + printf '%s\t%s\n' "$path" "$reason" >> "$out" + ;; + esac + done < "$manifest" +} + +verify_tree() { + local manifest=$1 + local entry path current upstream saved + local bad=0 + + [ -f "$dest/.export-upstream" ] || die "not an export-upstream tree: $dest" + + load_manifest_files "$manifest" "$tmp/manifest.raw" + [ -s "$tmp/manifest.raw" ] || die "manifest is empty: $manifest" + + : > "$tmp/manifest.paths" + while IFS= read -r entry; do + if [ -f "$dest/$entry" ]; then + printf '%s\n' "$entry" >> "$tmp/manifest.paths" + elif [ -d "$dest/$entry" ]; then + (cd "$dest" && find "$entry" -type f -print) + else + die "manifest path not found in $dest: $entry" + fi + done < "$tmp/manifest.raw" + + LC_ALL=C sort "$tmp/manifest.paths" -o "$tmp/manifest.paths.sorted" + if uniq -d "$tmp/manifest.paths.sorted" | grep -q .; then + die "manifest contains duplicate paths: $manifest" + fi + mv "$tmp/manifest.paths.sorted" "$tmp/manifest.paths" + + ( + cd "$dest" + find . -type f \ + ! -name CHECKSUMS.sha256 \ + ! -name .export-upstream \ + ! -name UPSTREAM.md \ + -print \ + | sed 's|^\./||' + ) | LC_ALL=C sort > "$tmp/actual.paths" + + if ! diff -u "$tmp/manifest.paths" "$tmp/actual.paths" > "$tmp/tree.diff"; then + cat "$tmp/tree.diff" >&2 + die "vendored tree does not match $manifest" + fi + + [ -f "$dest/CHECKSUMS.sha256" ] || die "missing $dest/CHECKSUMS.sha256" + if ! (cd "$dest" && sha256_check CHECKSUMS.sha256 >/dev/null); then + die "CHECKSUMS.sha256 does not match the vendored tree" + fi + + awk ' + /^# upstream / { + p = $4 + sub(/^\.\//, "", p) + if (p in upstream) { + print "DUPLICATE_UPSTREAM\t" p + bad = 1 + } + upstream[p] = $3 + next + } + /^#/ { + next + } + NF >= 2 { + p = $2 + sub(/^\.\//, "", p) + if (p in current) { + print "DUPLICATE_CURRENT\t" p + bad = 1 + } + current[p] = $1 + next + } + END { + for (p in current) { + if (!(p in upstream)) { + print "MISSING_UPSTREAM\t" p + bad = 1 + } else { + print "ENTRY\t" p "\t" current[p] "\t" upstream[p] + } + } + for (p in upstream) { + if (!(p in current)) { + print "MISSING_CURRENT\t" p + bad = 1 + } + } + exit bad + } + ' "$dest/CHECKSUMS.sha256" > "$tmp/checksums.tsv" || die "malformed CHECKSUMS.sha256" + + load_divergences "$manifest" "$tmp/divergences" + + while IFS=$'\t' read -r entry path current upstream; do + case "$entry" in + ENTRY) + if ! grep -Fqx "$path" "$tmp/manifest.paths"; then + echo "checksum entry not in manifest: $path" >&2 + bad=1 + fi + if [ "$current" != "$upstream" ] \ + && ! awk -F '\t' -v path="$path" '$1 == path { found = 1 } END { exit !found }' "$tmp/divergences"; then + echo "divergence without rationale in $manifest: $path" >&2 + bad=1 + fi + ;; + *) + echo "$entry: $path" >&2 + bad=1 + ;; + esac + done < "$tmp/checksums.tsv" + + while IFS=$'\t' read -r path saved; do + [ -n "$saved" ] || { + echo "empty divergence rationale in $manifest: $path" >&2 + bad=1 + } + if ! grep -Fqx "$path" "$tmp/manifest.paths"; then + echo "divergence path not in manifest: $path" >&2 + bad=1 + continue + fi + entry=$(awk -F '\t' -v path="$path" '$1 == "ENTRY" && $2 == path { print $3 "\t" $4 }' "$tmp/checksums.tsv") + if [ -z "$entry" ]; then + echo "divergence path has no checksums: $path" >&2 + bad=1 + continue + fi + current=${entry%%$'\t'*} + upstream=${entry#*$'\t'} + if [ "$current" = "$upstream" ]; then + echo "divergence rationale but file matches upstream: $path" >&2 + bad=1 + fi + done < "$tmp/divergences" + + [ "$bad" -eq 0 ] || die "vendored upstream verification failed" + echo "Verified $dest against $manifest" +} + +self_test_reset() { + rm -rf "$self_root/$dest" + cp -a "$dest" "$self_root/$dest" + cp "$manifest" "$self_root/$manifest" +} + +self_test_expect_pass() { + local label=$1 + + if ! "$self_root/tools/export-upstream" --verify > "$tmp/self-test.out" 2>&1; then + cat "$tmp/self-test.out" >&2 + die "self-test: $label was rejected" + fi +} + +self_test_expect_fail() { + local label=$1 + + if "$self_root/tools/export-upstream" --verify > "$tmp/self-test.out" 2>&1; then + die "self-test: $label was accepted" + fi +} + +self_test() { + local path="src/test/java/org/sqlite/Utils.java" + local file hash + + self_root="$tmp/self-test" + mkdir -p "$self_root/tools" + cp -a "$dest" "$self_root/$dest" + cp "$manifest" "$self_root/$manifest" + cp "$repo_root/tools/export-upstream" "$self_root/tools/export-upstream" + chmod +x "$self_root/tools/export-upstream" + + self_test_expect_pass "pristine tree" + + file="$self_root/$dest/$path" + printf ' ' >> "$file" + self_test_expect_fail "one-byte change" + + self_test_reset + printf 'class ExtraTest {}\n' > "$self_root/$dest/src/test/java/org/sqlite/ExtraTest.java" + self_test_expect_fail "unlisted vendored file" + + self_test_reset + rm "$self_root/$dest/$path" + self_test_expect_fail "deleted vendored file" + + self_test_reset + file="$self_root/$dest/$path" + printf ' ' >> "$file" + hash=$(sha256_file "$file") + awk -v path="./$path" -v hash="$hash" ' + /^#/ { print; next } + $2 == path { print hash " " $2; next } + { print } + ' "$self_root/$dest/CHECKSUMS.sha256" > "$self_root/$dest/CHECKSUMS.sha256.new" + mv "$self_root/$dest/CHECKSUMS.sha256.new" "$self_root/$dest/CHECKSUMS.sha256" + self_test_expect_fail "divergence without rationale" + + printf '\n# divergence: %s -- self-test rationale\n' "$path" >> "$self_root/$manifest" + self_test_expect_pass "recorded divergence" + + self_test_reset + mv "$self_root/$manifest" "$self_root/manifest.hidden" + self_test_expect_fail "missing default manifest" + + echo "Self-test passed" +} + repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) cd "$repo_root" repo="xerial/sqlite-jdbc" dest="upstream-xerial_sqlite-jdbc" upstream_md="$dest/UPSTREAM.md" -manifest="" +manifest="upstream-xerial_sqlite-jdbc.manifest" manifest_set=0 export_all=0 force=0 +verify=0 +self_test_enabled=0 +tag="" if [ "$#" -lt 1 ]; then usage exit 2 fi -case "$1" in - -h|--help) - usage - exit 0 - ;; - -*) - usage - exit 2 - ;; -esac - -tag="$1" -shift - while [ "$#" -gt 0 ]; do case "$1" in --manifest) @@ -90,16 +375,57 @@ while [ "$#" -gt 0 ]; do force=1 shift ;; + --verify) + verify=1 + shift + ;; + --self-test) + self_test_enabled=1 + shift + ;; -h|--help) usage exit 0 ;; - *) + -*) die "unknown option: $1" ;; + *) + [ -z "$tag" ] || die "unexpected argument: $1" + tag="$1" + shift + ;; esac done +if [ "$self_test_enabled" -eq 1 ]; then + [ -z "$tag" ] || die "--self-test does not take a tag" + [ "$export_all" -eq 0 ] || die "--self-test cannot be used with --all" + [ "$force" -eq 0 ] || die "--self-test cannot be used with --force" + [ "$verify" -eq 0 ] || die "--self-test cannot be used with --verify" + + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + self_test + exit 0 +fi + +if [ "$verify" -eq 1 ]; then + [ -z "$tag" ] || die "--verify does not take a tag" + [ "$export_all" -eq 0 ] || die "--verify cannot be used with --all" + [ "$force" -eq 0 ] || die "--verify cannot be used with --force" + + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + verify_tree "$manifest" + exit 0 +fi + +[ -n "$tag" ] || { + usage + exit 2 +} + if [ "$export_all" -eq 1 ] && [ "$manifest_set" -eq 1 ]; then die "--all and --manifest cannot be used together" fi @@ -139,23 +465,10 @@ if [ "$export_all" -eq 1 ]; then paths_label="whole repository" else paths=() - if [ -n "$manifest" ]; then - [ -f "$manifest" ] || die "manifest not found: $manifest" - while IFS= read -r line || [ -n "$line" ]; do - line=${line%$'\r'} - case "$line" in - ""|"#"*) - continue - ;; - /*|..|../*|*/../*|*/..) - die "manifest path must be relative and stay inside the repository: $line" - ;; - esac - paths+=("$line") - done < "$manifest" - else - paths=(src/test/java src/test/resources LICENSE LICENSE.zentus NOTICE VERSION) - fi + load_manifest_files "$manifest" "$tmp/manifest.paths" + while IFS= read -r line; do + paths+=("$line") + done < "$tmp/manifest.paths" [ "${#paths[@]}" -gt 0 ] || die "manifest is empty: $manifest" @@ -237,18 +550,27 @@ else fi mv "$upstream_md.tmp" "$upstream_md" -if command -v sha256sum >/dev/null 2>&1; then - ( - cd "$dest" - find . -type f \ - ! -name CHECKSUMS.sha256 \ - ! -name .export-upstream \ - ! -name UPSTREAM.md \ - -print0 \ - | sort -z \ - | xargs -0 -r sha256sum > CHECKSUMS.sha256 - ) -fi +( + cd "$dest" + find . -type f \ + ! -name CHECKSUMS.sha256 \ + ! -name .export-upstream \ + ! -name UPSTREAM.md \ + -print \ + | sed 's|^\./||' +) | LC_ALL=C sort > "$tmp/checksum.paths" + +{ + echo '# Generated by tools/export-upstream. Do not edit by hand except for a deliberate divergence.' + echo '# The uncommented line is the current hash; "# upstream" is the hash at the pinned tag.' + echo '# A deliberate divergence keeps the upstream line, updates the current line, and records' + echo '# "# divergence: -- " in upstream-xerial_sqlite-jdbc.manifest.' +} > "$dest/CHECKSUMS.sha256" +while IFS= read -r path; do + hash=$(sha256_file "$dest/$path") + printf '# upstream %s ./%s\n' "$hash" "$path" >> "$dest/CHECKSUMS.sha256" + printf '%s ./%s\n' "$hash" "$path" >> "$dest/CHECKSUMS.sha256" +done < "$tmp/checksum.paths" echo "Exported tag $tag ($sha) to $dest" echo "Updated $upstream_md" diff --git a/upstream-xerial_sqlite-jdbc.manifest b/upstream-xerial_sqlite-jdbc.manifest index 2381609..a839a9d 100644 --- a/upstream-xerial_sqlite-jdbc.manifest +++ b/upstream-xerial_sqlite-jdbc.manifest @@ -1,3 +1,13 @@ +# Source of truth for the vendored xerial/sqlite-jdbc tree: the conformance +# build copies these paths, and tools/export-upstream exports these paths by +# default. Run `tools/export-upstream --verify` after any change. +# +# A deliberate divergence from upstream keeps the "# upstream" hash in +# upstream-xerial_sqlite-jdbc/CHECKSUMS.sha256, updates the current hash, and +# adds a line in this file: +# # divergence: -- +# Use the reason to say which libSQL or Hrana behavior makes parity impossible. +# # First conformance batch. src/test/java/org/sqlite/Utils.java src/test/java/org/sqlite/ResultSetTest.java diff --git a/upstream-xerial_sqlite-jdbc/CHECKSUMS.sha256 b/upstream-xerial_sqlite-jdbc/CHECKSUMS.sha256 index 0240387..234b581 100644 --- a/upstream-xerial_sqlite-jdbc/CHECKSUMS.sha256 +++ b/upstream-xerial_sqlite-jdbc/CHECKSUMS.sha256 @@ -1,15 +1,34 @@ +# Generated by tools/export-upstream. Do not edit by hand except for a deliberate divergence. +# The uncommented line is the current hash; "# upstream" is the hash at the pinned tag. +# A deliberate divergence keeps the upstream line, updates the current line, and records +# "# divergence: -- " in upstream-xerial_sqlite-jdbc.manifest. +# upstream 3ddf9be5c28fe27dad143a5dc76eea25222ad1dd68934a047064e56ed2fa40c5 ./LICENSE 3ddf9be5c28fe27dad143a5dc76eea25222ad1dd68934a047064e56ed2fa40c5 ./LICENSE +# upstream 89167dab92289c7e5e2b65b044f0856b703d05e5d5e35c3548e73d9c7d2f5048 ./LICENSE.zentus 89167dab92289c7e5e2b65b044f0856b703d05e5d5e35c3548e73d9c7d2f5048 ./LICENSE.zentus +# upstream a0670cf020edf700524c43c85321c2ba027a8bace5682001db60875d4a9fe52b ./NOTICE a0670cf020edf700524c43c85321c2ba027a8bace5682001db60875d4a9fe52b ./NOTICE +# upstream 0a60829ce6edd56de46fb87303401c5c1e1e666da5631056340f5c5ce902dcdf ./VERSION +0a60829ce6edd56de46fb87303401c5c1e1e666da5631056340f5c5ce902dcdf ./VERSION +# upstream 1351fdca2644e90ac9028ed99c6096c9ecee82ccb7d45cedd9bb454887fd2c6b ./src/test/java/org/sqlite/CachedRowSetTest.java 1351fdca2644e90ac9028ed99c6096c9ecee82ccb7d45cedd9bb454887fd2c6b ./src/test/java/org/sqlite/CachedRowSetTest.java +# upstream 9fb0f2df7de66cac0a46b241c0c1cd3311a80646d0431d377097741b99053760 ./src/test/java/org/sqlite/FetchSizeTest.java 9fb0f2df7de66cac0a46b241c0c1cd3311a80646d0431d377097741b99053760 ./src/test/java/org/sqlite/FetchSizeTest.java +# upstream 6dfcecb209e01829ba0aea782b47351443e3970ad1d33ac4c3fb979e6dff1f40 ./src/test/java/org/sqlite/InsertQueryTest.java 6dfcecb209e01829ba0aea782b47351443e3970ad1d33ac4c3fb979e6dff1f40 ./src/test/java/org/sqlite/InsertQueryTest.java +# upstream bb07398946480ca44c70d03e6f8c399f53506b44c83a312e2d28aebd239c0238 ./src/test/java/org/sqlite/JSON1Test.java bb07398946480ca44c70d03e6f8c399f53506b44c83a312e2d28aebd239c0238 ./src/test/java/org/sqlite/JSON1Test.java +# upstream 242bb51330240174bab5f443aa4d8de1049e6d947374e080a636c4aa4d0049ee ./src/test/java/org/sqlite/MetadataLeakTest.java 242bb51330240174bab5f443aa4d8de1049e6d947374e080a636c4aa4d0049ee ./src/test/java/org/sqlite/MetadataLeakTest.java +# upstream 7af8ae0999a0bf26a20c96ab23131505bd79827e6af17742771e36e9d8d985f4 ./src/test/java/org/sqlite/PreparedStatementThreadTest.java 7af8ae0999a0bf26a20c96ab23131505bd79827e6af17742771e36e9d8d985f4 ./src/test/java/org/sqlite/PreparedStatementThreadTest.java +# upstream d787e71bdf2b8d12a2669eb06c432cf2a39ac77f678a92459e3dd9b6c4519a43 ./src/test/java/org/sqlite/RSMetaDataTest.java +d787e71bdf2b8d12a2669eb06c432cf2a39ac77f678a92459e3dd9b6c4519a43 ./src/test/java/org/sqlite/RSMetaDataTest.java +# upstream 6065d284933c8462cc13217992820a9261075c0e9ee126df6cbe9961d90303f9 ./src/test/java/org/sqlite/ResultSetTest.java 6065d284933c8462cc13217992820a9261075c0e9ee126df6cbe9961d90303f9 ./src/test/java/org/sqlite/ResultSetTest.java +# upstream 958c1f2ff5ca2f1257dfd847207ea793839969b234d140576b51af346697233c ./src/test/java/org/sqlite/ResultSetWithoutResultsTest.java 958c1f2ff5ca2f1257dfd847207ea793839969b234d140576b51af346697233c ./src/test/java/org/sqlite/ResultSetWithoutResultsTest.java -d787e71bdf2b8d12a2669eb06c432cf2a39ac77f678a92459e3dd9b6c4519a43 ./src/test/java/org/sqlite/RSMetaDataTest.java +# upstream ed2fb4bf7319bcba0662b83707375501337173956bae7c57dda6e6a2c46b7505 ./src/test/java/org/sqlite/TypeMapTest.java ed2fb4bf7319bcba0662b83707375501337173956bae7c57dda6e6a2c46b7505 ./src/test/java/org/sqlite/TypeMapTest.java +# upstream bef3eddca2a6b80472050a3a52d5e7246128f0644d29c24ed091709b0ed6f121 ./src/test/java/org/sqlite/Utils.java bef3eddca2a6b80472050a3a52d5e7246128f0644d29c24ed091709b0ed6f121 ./src/test/java/org/sqlite/Utils.java -0a60829ce6edd56de46fb87303401c5c1e1e666da5631056340f5c5ce902dcdf ./VERSION