diff --git a/utils/Containerfile b/utils/Containerfile index 8760e53..3e192df 100644 --- a/utils/Containerfile +++ b/utils/Containerfile @@ -1,6 +1,9 @@ # This Containerfile is inspired by the Containerfile used by Hermeto: # https://github.com/hermetoproject/hermeto/blob/main/Dockerfile +# Cosign from Red Hat Trusted Artifact Signer. +FROM registry.redhat.io/rhtas/cosign-rhel9:1.3.6@sha256:ff6a2a11b8c1dff47cb115cfa3ba5709bff5f1cf485bdaecff47f1a37cad2405 AS cosign + FROM registry.access.redhat.com/ubi10/ubi:latest as ubi ######################## @@ -12,6 +15,8 @@ RUN dnf -y install \ --nodocs \ python3 \ jq \ + curl \ + findutils \ && dnf clean all ############### @@ -31,6 +36,7 @@ FROM base LABEL maintainer="Red Hat" COPY --from=builder /venv /venv +COPY --from=cosign /usr/local/bin/cosign /usr/local/bin/cosign COPY scripts/* /usr/local/bin/ diff --git a/utils/scripts/npm-common.sh b/utils/scripts/npm-common.sh new file mode 100755 index 0000000..f5dda2a --- /dev/null +++ b/utils/scripts/npm-common.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Shared helpers for npm pulp release scripts. Source from other scripts: +# # shellcheck source=npm-common.sh +# source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/npm-common.sh" + +# Fail if the named tar member is missing, empty, or larger than max_bytes. +# Probes with head -c so we never buffer more than max+1 bytes. +assert_tar_member_size() { + local archive="${1}" + local member="${2}" + local max_bytes="${3}" + local tmp size + tmp="$(mktemp)" + # Process substitution avoids a pipefail-sensitive tar|head pipeline: + # head closes early on truncate; tar's SIGPIPE stays in the subshell. + head -c "$((max_bytes + 1))" \ + < <(tar -xOf "${archive}" "${member}" 2>/dev/null) \ + > "${tmp}" || true + size="$(wc -c < "${tmp}" | tr -d ' ')" + rm -f "${tmp}" + if [[ -z "${size}" || "${size}" -eq 0 ]]; then + return 1 + fi + if [[ "${size}" -gt "${max_bytes}" ]]; then + echo "ERROR: ${member} in $(basename "${archive}") exceeds" \ + "${max_bytes} bytes" >&2 + return 1 + fi + return 0 +} diff --git a/utils/scripts/npm-fetch-chains-provenance b/utils/scripts/npm-fetch-chains-provenance new file mode 100755 index 0000000..6224724 --- /dev/null +++ b/utils/scripts/npm-fetch-chains-provenance @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# Verify and store Tekton Chains SLSA provenance for snapshot images. +# +# Env: +# FILES_DIR Extracted files root (required) +# IMAGES_TXT File listing digest-pinned images (required) +# TRUSTED_PROVENANCE_NAMESPACES Comma-separated namespaces (default: calunga-tenant) +# +set -euo pipefail + +FILES_DIR="${FILES_DIR:?FILES_DIR required}" +IMAGES_TXT="${IMAGES_TXT:?IMAGES_TXT required}" +TRUSTED_PROVENANCE_NAMESPACES="${TRUSTED_PROVENANCE_NAMESPACES:-calunga-tenant}" + +PROVENANCE_DIR="${FILES_DIR}/chains-provenance" +DOCKER_CONFIG="$(mktemp -d)" +export DOCKER_CONFIG +on_exit() { + rm -rf "${DOCKER_CONFIG}" +} +trap 'on_exit' EXIT +mkdir -p "${PROVENANCE_DIR}" + +# Trusted Chains identity for npm snapshot images (promote or build). +readonly EXPECTED_BUILDER_ID="https://konflux-ci.dev/chains/v2" +readonly EXPECTED_BUILD_TYPE="https://tekton.dev/chains/v2/slsa" +TRUSTED_NS_JSON="$(jq -nc --arg s "${TRUSTED_PROVENANCE_NAMESPACES}" ' + $s + | split(",") + | map(gsub("^\\s+|\\s+$"; "")) + | map(select(length > 0)) +')" +if [[ "$(jq 'length' <<<"${TRUSTED_NS_JSON}")" -eq 0 ]]; then + echo "ERROR: trustedProvenanceNamespaces must list at least one namespace" >&2 + exit 1 +fi + +while read -r IMAGE; do + [[ -n "${IMAGE}" ]] || continue + echo "Fetching Chains provenance for ${IMAGE}" + + select-oci-auth "${IMAGE}" > "${DOCKER_CONFIG}/config.json" + + DIGEST="${IMAGE##*@}" + if [[ ! "${DIGEST}" =~ ^sha256:[a-fA-F0-9]{64}$ ]]; then + echo "ERROR: image must be digest-pinned (got ${IMAGE})" >&2 + exit 1 + fi + DIGEST_SAFE="${DIGEST//[:\/]/_}" + PROVENANCE_FILE="${PROVENANCE_DIR}/${DIGEST_SAFE}.json" + + COSIGN_STDERR="/tmp/cosign-stderr-${DIGEST_SAFE}.log" + COSIGN_STDOUT="/tmp/cosign-stdout-${DIGEST_SAFE}.jsonl" + # Chains uses AWS KMS for signing, not public Sigstore/Rekor, + # so transparency log and SCT verification are skipped (same as + # extract-py-artifacts). Trust is the cluster Chains key plus + # subject digest and builder/pipeline/namespace checks below. + if ! cosign verify-attestation \ + --type=slsaprovenance \ + --insecure-ignore-tlog=true \ + --insecure-ignore-sct=true \ + --key k8s://openshift-pipelines/public-key \ + "${IMAGE}" >"${COSIGN_STDOUT}" 2>"${COSIGN_STDERR}"; then + echo "ERROR: Failed to fetch Chains provenance for ${IMAGE}" >&2 + cat "${COSIGN_STDERR}" >&2 + exit 1 + fi + # Capture full cosign output (avoid SIGPIPE from head under pipefail), + # then keep a SLSA statement that: + # - has buildDefinition + # - subject digest matches this image + # - was produced by Konflux Chains with a trusted npm pipeline + # - invocationId is in an authorized tenant namespace + DIGEST_HEX="${DIGEST#sha256:}" + if ! jq -s -e \ + --arg want "${DIGEST_HEX}" \ + --arg builder "${EXPECTED_BUILDER_ID}" \ + --arg buildType "${EXPECTED_BUILD_TYPE}" \ + --argjson trustedNs "${TRUSTED_NS_JSON}" ' + map(try (.payload | @base64d | fromjson) catch empty) + | map(select( + .predicate.buildDefinition != null + and (.predicate.buildDefinition.buildType // "") == $buildType + and (.predicate.runDetails.builder.id // "") == $builder + and ( + (.predicate.buildDefinition.externalParameters.runSpec.pipelineRef.name // "") as $pn + | ($pn == "promote-npm" or $pn == "build-npm") + ) + and ( + (.predicate.runDetails.metadata.invocationId // "") as $id + | any($trustedNs[]; . as $ns | ($id | startswith($ns + "/"))) + ) + and any( + .subject[]?; + (.digest.sha256 // "") == $want + ) + )) + | .[0] // empty + | select(. != null and . != {}) + ' "${COSIGN_STDOUT}" > "${PROVENANCE_FILE}"; then + echo "ERROR: No trusted Chains SLSA provenance for ${DIGEST}" >&2 + echo " expected builder=${EXPECTED_BUILDER_ID}" >&2 + echo " expected buildType=${EXPECTED_BUILD_TYPE}" >&2 + echo " expected pipelineRef.name in {promote-npm,build-npm}" >&2 + echo " expected invocationId namespace in ${TRUSTED_NS_JSON}" >&2 + cat "${COSIGN_STDERR}" >&2 + cat "${COSIGN_STDOUT}" >&2 + exit 1 + fi + echo " Saved Chains provenance to ${PROVENANCE_FILE}" + rm -f "${COSIGN_STDERR}" "${COSIGN_STDOUT}" +done < "${IMAGES_TXT}" + +echo "Chains provenance files:" +ls -la "${PROVENANCE_DIR}" diff --git a/utils/scripts/npm-populate-release-notes b/utils/scripts/npm-populate-release-notes new file mode 100755 index 0000000..531d53f --- /dev/null +++ b/utils/scripts/npm-populate-release-notes @@ -0,0 +1,230 @@ +#!/usr/bin/env bash +# Populate releaseNotes.content.artifacts (and mapping contentType) from +# extracted npm .tgz packages under FILES_DIR. +# +# Env: +# FILES_DIR Directory tree containing *.tgz (required) +# DATA_FILE Path to data.json to update (required) +# +set -euo pipefail + +# shellcheck source=npm-common.sh +_SCRIPT_DIR="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" +# Prefer sibling when running from a checkout; fall back to PATH install layout. +if [[ -f "${_SCRIPT_DIR}/npm-common.sh" ]]; then + # shellcheck disable=SC1091 + source "${_SCRIPT_DIR}/npm-common.sh" +else + # shellcheck disable=SC1091 + source "$(command -v npm-common.sh)" +fi + +DATA_FILE="${DATA_FILE:?DATA_FILE required}" +FILES_DIR="${FILES_DIR:?FILES_DIR required}" +ARTIFACTS_JSONL="$(mktemp)" +TMP_DATA="" +on_exit() { + rm -f "${ARTIFACTS_JSONL}" + [[ -n "${TMP_DATA}" ]] && rm -f "${TMP_DATA}" +} +trap 'on_exit' EXIT +shopt -s nullglob + +# Cap metadata reads from package-controlled tar members (defense in depth). +readonly MAX_PACKAGE_JSON_BYTES=1048576 # 1 MiB +readonly MAX_SBOM_BYTES=5242880 # 5 MiB + + +# Map package.json os/cpu fields to advisory architecture/os values. +# Single unrestricted value is used; multiple or negated entries → any. +parse_npm_platform() { + local pkg_json="${1}" + jq -r ' + def one_or_any: + if . == null then "any" + elif type == "string" then + if startswith("!") then "any" else . end + elif type == "array" then + if length == 1 + and (.[0] | type == "string") + and (.[0] | startswith("!") | not) + then .[0] + else "any" end + else "any" end; + def map_cpu: + if . == "any" then "any" + elif . == "x64" or . == "x86_64" then "amd64" + elif . == "arm64" or . == "aarch64" then "arm64" + elif . == "ia32" or . == "x86" then "386" + elif . == "ppc64" then "ppc64le" + elif . == "s390x" then "s390x" + else . end; + def map_os: + if . == "any" then "any" + elif . == "linux" then "linux" + elif . == "darwin" then "macos" + elif . == "win32" then "windows" + else . end; + ((.cpu | one_or_any) | map_cpu) as $arch + | ((.os | one_or_any) | map_os) as $os + | "\($arch) \($os)" + ' <<<"${pkg_json}" +} + +while IFS= read -r -d '' tgz; do + filename="$(basename "${tgz}")" + echo "Processing tarball: ${tgz}" + + if ! assert_tar_member_size "${tgz}" "package/package.json" \ + "${MAX_PACKAGE_JSON_BYTES}"; then + echo "ERROR: package/package.json missing or oversized in ${filename}" >&2 + exit 1 + fi + pkg_json="$(tar -xOf "${tgz}" package/package.json)" || { + echo "ERROR: package/package.json missing in ${filename}" + exit 1 + } + name="$(jq -r '.name // empty' <<<"${pkg_json}")" + version="$(jq -r '.version // empty' <<<"${pkg_json}")" + if [[ -z "${name}" || -z "${version}" ]]; then + echo "ERROR: ${filename} package.json missing non-empty name/version" >&2 + exit 1 + fi + + if ! assert_tar_member_size "${tgz}" "package/sboms/redhat.spdx.json" \ + "${MAX_SBOM_BYTES}"; then + echo "ERROR: package/sboms/redhat.spdx.json missing or oversized in ${filename}" >&2 + exit 1 + fi + sbom_json="$(tar -xOf "${tgz}" package/sboms/redhat.spdx.json 2>/dev/null)" || { + echo "ERROR: package/sboms/redhat.spdx.json missing in ${filename}" + exit 1 + } + + purl="$(jq -r \ + --arg name "${name}" \ + --arg version "${version}" ' + def decode_npm_name: + gsub("%40"; "@") + | gsub("%2[Ff]"; "/") + | gsub("%2f"; "/"); + def uri_decode: + # Decode common PURL percent-escapes for name/version comparison. + gsub("%2[Bb]"; "+") + | gsub("%2b"; "+") + | gsub("%3[Aa]"; ":") + | gsub("%3a"; ":") + | gsub("%40"; "@") + | gsub("%2[Ff]"; "/") + | gsub("%2f"; "/") + | gsub("%3[Ff]"; "?") + | gsub("%3f"; "?") + | gsub("%23"; "#") + | gsub("%20"; " ") + | gsub("%25"; "%"); + def parse_npm_purl: + . as $purl + | sub("^pkg:npm/"; "") + | split("#")[0] + | split("?")[0] + | (try capture("^(?.+)@(?[^@]+)$") catch empty) as $m + | if $m == null then empty + else { + purl: $purl, + name: ($m.n | decode_npm_name | uri_decode), + version: ($m.v | uri_decode) + } end; + [.packages[]?.externalRefs[]? + | select(.referenceType == "purl") + | .referenceLocator + | select(startswith("pkg:npm/")) + | parse_npm_purl + | select(.name == $name and .version == $version) + | .purl + ] | first // empty + ' <<<"${sbom_json}")" + + if [[ -z "${purl}" ]]; then + # Canonical npm PURL with percent-encoded name/version components. + # Scope marker @ -> %40; '/' between scope and name stays unencoded. + # e.g. @scope/pkg -> pkg:npm/%40scope/pkg@version + purl="$(jq -nr --arg n "${name}" --arg v "${version}" ' + def enc: @uri; + ($n + | if startswith("@") and test("/") then + (capture("^@(?[^/]+)/(?.+)$") + | "%40\(.scope | enc)/\(.pkg | enc)") + else enc end) as $id + | "pkg:npm/\($id)@\($v | enc)" + ')" + echo " WARNING: no matching pkg:npm for ${name}@${version} in SBOM; using ${purl}" + fi + echo " Found PURL: ${purl}" + + read -r arch os <<< "$(parse_npm_platform "${pkg_json}")" + echo " Platform from package.json: arch=${arch}, os=${os}" + + jq -nc \ + --arg component "${name}" \ + --arg purl "${purl}" \ + --arg architecture "${arch}" \ + --arg os "${os}" \ + '{component: $component, purl: $purl, architecture: $architecture, os: $os}' \ + >> "${ARTIFACTS_JSONL}" +done < <(find "${FILES_DIR}" -type f -name '*.tgz' -print0 | sort -z) + +DEDUPED="$(jq -sc ' + if length == 0 then [] + else + group_by(.component + "|" + .purl + "|" + .architecture + "|" + .os) + | map(.[0]) + end +' "${ARTIFACTS_JSONL}")" +rm -f "${ARTIFACTS_JSONL}" + +ARTIFACT_COUNT="$(jq 'length' <<<"${DEDUPED}")" +if [[ "${ARTIFACT_COUNT}" -eq 0 ]]; then + echo "WARNING: No .tgz files found (infra-only / empty snapshot); continuing" +else + echo "Adding ${ARTIFACT_COUNT} artifact(s) to releaseNotes.content.artifacts" +fi + +TMP_DATA="$(mktemp)" +jq --argjson artifacts "${DEDUPED}" ' + .releaseNotes.content.artifacts = ( + ((.releaseNotes.content.artifacts // []) + $artifacts) + | group_by(.component + "|" + .purl + "|" + .architecture + "|" + .os) + | map(.[0]) + ) +' "${DATA_FILE}" > "${TMP_DATA}" && mv "${TMP_DATA}" "${DATA_FILE}" + +if [[ "${ARTIFACT_COUNT}" -gt 0 ]]; then + COMPONENT_NAMES="$(jq -r '[.[].component] | unique | .[]' <<<"${DEDUPED}")" + MAPPING_ENTRIES="[]" + while IFS= read -r comp; do + [[ -n "${comp}" ]] || continue + MAPPING_ENTRIES="$(jq -c --arg name "${comp}" \ + '. + [{name: $name, contentType: "generic"}]' \ + <<<"${MAPPING_ENTRIES}")" + done <<<"${COMPONENT_NAMES}" + jq --argjson new_comps "${MAPPING_ENTRIES}" ' + ($new_comps | map(.name)) as $names | + (.mapping.components // []) as $existing | + ($existing | map(.name)) as $existing_names | + .mapping.components = [ + $existing[] | + if (.name as $n | $names | index($n)) + and ((.contentGateway?.contentType // + .contentType // "") == "") + then . + {contentType: "generic"} + else . end + ] + [ + $new_comps[] + | select(.name as $n | + $existing_names | index($n) | not) + ] + ' "${DATA_FILE}" > "${TMP_DATA}" && mv "${TMP_DATA}" "${DATA_FILE}" +fi + +echo "Done. Updated artifacts:" +jq '.releaseNotes.content.artifacts' "${DATA_FILE}" diff --git a/utils/scripts/npm-pulp-upload b/utils/scripts/npm-pulp-upload new file mode 100755 index 0000000..94c371f --- /dev/null +++ b/utils/scripts/npm-pulp-upload @@ -0,0 +1,400 @@ +#!/usr/bin/env bash +# Upload npm .tgz packages to a Pulp npm repository (REST + curl). +# +# Env: +# FILES_DIR Directory tree containing *.tgz (required) +# PULP_BASE_URL Pulp base URL (required) +# PULP_API_ROOT API root, e.g. /api/ (default: /api/) +# PULP_DOMAIN Pulp domain (required) +# PULP_REPOSITORY npm repository name (required) +# PULP_FILE_REPOSITORY Optional file repo for *.tl-compliance.json sidecars +# +# Credentials: /etc/service-account-secret/{username,password} +# +set -euo pipefail + +FILES_DIR="${FILES_DIR:?FILES_DIR required}" +PULP_BASE_URL="${PULP_BASE_URL:?PULP_BASE_URL required}" +PULP_API_ROOT="${PULP_API_ROOT:-/api/}" +PULP_DOMAIN="${PULP_DOMAIN:?PULP_DOMAIN required}" +PULP_REPOSITORY="${PULP_REPOSITORY:?PULP_REPOSITORY required}" +PULP_FILE_REPOSITORY="${PULP_FILE_REPOSITORY:-}" + +# shellcheck source=npm-common.sh +_SCRIPT_DIR="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" +# Prefer sibling when running from a checkout; fall back to PATH install layout. +if [[ -f "${_SCRIPT_DIR}/npm-common.sh" ]]; then + # shellcheck disable=SC1091 + source "${_SCRIPT_DIR}/npm-common.sh" +else + # shellcheck disable=SC1091 + source "$(command -v npm-common.sh)" +fi + +readonly CURL_RETRY=3 +readonly CURL_MAX_TIME_QUERY_SECONDS=30 +readonly CURL_MAX_TIME_UPLOAD_SECONDS=120 +readonly CURL_MAX_TIME_SIDECAR_SECONDS=60 +readonly COMPLIANCE_LABEL_KEY="tl.compliance_level" +readonly MAX_PACKAGE_JSON_BYTES=1048576 # 1 MiB +readonly MAX_COMPLIANCE_SIDECAR_BYTES=1048576 # 1 MiB + + +# Empty / missing FILES_DIR is a successful no-op (infra-only snapshots). +# Skip Pulp auth/API entirely when there is nothing to upload. +mkdir -p "${FILES_DIR}" +ls -la "${FILES_DIR}" +tgz_count="$(find "${FILES_DIR}" -type f -name '*.tgz' | wc -l)" +if [[ "${tgz_count}" -eq 0 ]]; then + echo "No .tgz packages under ${FILES_DIR}; skipping Pulp upload (infra-only no-op)" + exit 0 +fi + +PULP_USER="$(< /etc/service-account-secret/username)" +PULP_PASS="$(< /etc/service-account-secret/password)" + +# Normalize so "/api" and "/api/" both yield ".../api/pulp/..." +PULP_API_ROOT_NORM="${PULP_API_ROOT%/}/" +PULP_QUERY_BASE="${PULP_BASE_URL}${PULP_API_ROOT_NORM}pulp/${PULP_DOMAIN}/api/v3/" +AUTH=(-u "${PULP_USER}:${PULP_PASS}") + +# Run curl and return stdout. On failure, print the response body to +# stderr so --fail-with-body payloads are visible in Tekton logs. +curl_capture() { + local out ec=0 + out="$(curl "$@")" || ec=$? + if [[ "${ec}" -ne 0 ]]; then + echo "ERROR: curl failed (exit ${ec})" >&2 + [[ -n "${out}" ]] && echo "${out}" >&2 + return "${ec}" + fi + printf '%s' "${out}" +} + +refresh_repository_version() { + local repo_json + repo_json="$(curl_capture --fail-with-body --silent \ + --retry "${CURL_RETRY}" \ + --max-time "${CURL_MAX_TIME_QUERY_SECONDS}" \ + -G \ + "${AUTH[@]}" \ + "${PULP_QUERY_BASE}repositories/npm/npm/" \ + --data-urlencode "name=${PULP_REPOSITORY}")" + PULP_REPOSITORY_VERSION="$(jq -r \ + '.results[0].latest_version_href // empty' <<<"${repo_json}")" + PULP_REPOSITORY_HREF="$(jq -r \ + '.results[0].pulp_href // empty' <<<"${repo_json}")" + if [[ -z "${PULP_REPOSITORY_HREF}" || "${PULP_REPOSITORY_HREF}" == "null" ]]; then + echo "ERROR: npm repository ${PULP_REPOSITORY} not found in domain ${PULP_DOMAIN}" >&2 + return 1 + fi + if [[ -z "${PULP_REPOSITORY_VERSION}" || "${PULP_REPOSITORY_VERSION}" == "null" ]]; then + echo "ERROR: npm repository ${PULP_REPOSITORY} has no latest_version_href" >&2 + return 1 + fi +} + +refresh_repository_version +echo "Repository: ${PULP_REPOSITORY_HREF}" +echo "Version: ${PULP_REPOSITORY_VERSION}" + +PULP_FILE_REPOSITORY_HREF="" +if [[ -n "${PULP_FILE_REPOSITORY:-}" ]]; then + file_repo_json="$(curl_capture --fail-with-body --silent \ + --retry "${CURL_RETRY}" \ + --max-time "${CURL_MAX_TIME_QUERY_SECONDS}" \ + -G \ + "${AUTH[@]}" \ + "${PULP_QUERY_BASE}repositories/file/file/" \ + --data-urlencode "name=${PULP_FILE_REPOSITORY}")" + PULP_FILE_REPOSITORY_HREF="$(jq -r \ + '.results[0].pulp_href // empty' <<<"${file_repo_json}")" + if [[ -z "${PULP_FILE_REPOSITORY_HREF}" || "${PULP_FILE_REPOSITORY_HREF}" == "null" ]]; then + echo "ERROR: file repository ${PULP_FILE_REPOSITORY} not found in domain ${PULP_DOMAIN}" >&2 + exit 1 + fi + echo "File repository (compliance sidecars): ${PULP_FILE_REPOSITORY_HREF}" +else + echo "PULP_FILE_REPOSITORY unset; skipping *.tl-compliance.json file uploads (labels still apply)" +fi + +# Returns: +# 0 -> exists with matching sha256 (sets EXISTING_CONTENT_HREF, EXISTING_LABEL) +# 2 -> not found +# 3 -> exists with different sha256 +check_package_exists_with_digest() { + local name="${1}" + local version="${2}" + local local_file="${3}" + local local_sha response count content_href artifact_href artifact_json server_sha + + EXISTING_CONTENT_HREF="" + EXISTING_LABEL="" + + # Always use the latest repository version so earlier uploads in this + # run are visible to subsequent existence checks. + refresh_repository_version + + local_sha="$(sha256sum "${local_file}" | awk '{print $1}')" + response="$(curl_capture --fail-with-body --silent \ + --retry "${CURL_RETRY}" \ + --max-time "${CURL_MAX_TIME_QUERY_SECONDS}" \ + -G \ + "${AUTH[@]}" \ + "${PULP_QUERY_BASE}content/npm/packages/" \ + --data-urlencode "name=${name}" \ + --data-urlencode "version=${version}" \ + --data-urlencode "repository_version=${PULP_REPOSITORY_VERSION}")" + count="$(jq -r '.count // (.results|length) // 0' <<<"${response}")" + if [[ "${count}" -eq 0 ]]; then + return 2 + fi + + # Require exactly one result that matches the requested name/version + # (do not trust results[0] alone if filters are loose or count > 1). + match_json="$(jq -c \ + --arg name "${name}" \ + --arg version "${version}" ' + [.results[]? + | select( + ((.name // "") == $name) + and ((.version // "") == $version) + ) + ] + | if length == 1 then .[0] + else empty end + ' <<<"${response}")" + if [[ -z "${match_json}" ]]; then + echo "ERROR: expected exactly one Pulp package for ${name}@${version}," \ + "got count=${count} with no unique name/version match" >&2 + return 1 + fi + + content_href="$(jq -r '.pulp_href // empty' <<<"${match_json}")" + EXISTING_CONTENT_HREF="${content_href}" + EXISTING_LABEL="$(jq -r \ + --arg key "${COMPLIANCE_LABEL_KEY}" \ + '.pulp_labels[$key] // empty' <<<"${match_json}")" + + artifact_href="$(jq -r '.artifact // empty' <<<"${match_json}")" + if [[ -z "${artifact_href}" || "${artifact_href}" == "null" ]]; then + # Detail fetch when list omits artifact + response="$(curl_capture --fail-with-body --silent \ + --retry "${CURL_RETRY}" \ + --max-time "${CURL_MAX_TIME_QUERY_SECONDS}" \ + "${AUTH[@]}" "${PULP_BASE_URL}${content_href}")" + artifact_href="$(jq -r '.artifact // empty' <<<"${response}")" + EXISTING_LABEL="$(jq -r \ + --arg key "${COMPLIANCE_LABEL_KEY}" \ + '.pulp_labels[$key] // empty' <<<"${response}")" + fi + if [[ -z "${artifact_href}" || "${artifact_href}" == "null" ]]; then + echo "ERROR: no artifact href for existing ${name}@${version}" >&2 + return 1 + fi + + artifact_json="$(curl_capture --fail-with-body --silent \ + --retry "${CURL_RETRY}" \ + --max-time "${CURL_MAX_TIME_QUERY_SECONDS}" \ + "${AUTH[@]}" "${PULP_BASE_URL}${artifact_href}")" + server_sha="$(jq -r '.sha256 // empty' <<<"${artifact_json}")" + if [[ -n "${server_sha}" && "${server_sha}" == "${local_sha}" ]]; then + return 0 + fi + echo "ERROR: ${name}@${version} exists in Pulp with different content" >&2 + echo " Local sha256: ${local_sha}" >&2 + echo " Remote sha256: ${server_sha:-unknown}" >&2 + return 3 +} + +apply_compliance_label() { + local content_href="${1}" + local level="${2}" + local current="${3:-}" + + [[ -n "${level}" ]] || return 0 + if [[ "${current}" == "${level}" ]]; then + echo " Label ${COMPLIANCE_LABEL_KEY}=${level} already set" + return 0 + fi + + echo " Setting ${COMPLIANCE_LABEL_KEY}=${level} on ${content_href}" + curl --fail-with-body --silent \ + --retry "${CURL_RETRY}" \ + --max-time "${CURL_MAX_TIME_QUERY_SECONDS}" \ + "${AUTH[@]}" \ + -X POST \ + -H "Content-Type: application/json" \ + -d "$(jq -nc --arg k "${COMPLIANCE_LABEL_KEY}" --arg v "${level}" \ + '{key: $k, value: $v}')" \ + "${PULP_BASE_URL}${content_href}set_label/" +} + +upload_compliance_sidecar() { + local sidecar="${1}" + local pkg_name="${2}" + local pkg_version="${3}" + local base digest safe_name safe_version relative_path + [[ -f "${sidecar}" ]] || return 0 + if [[ -z "${PULP_FILE_REPOSITORY_HREF}" ]]; then + return 0 + fi + # Collapse package-controlled metadata into a single path segment so + # '/', '\', '..', and other separators cannot manipulate the Pulp path. + safe_segment() { + local out + out="$(printf '%s' "${1}" | sed -E 's/[^A-Za-z0-9._@+-]+/_/g')" + out="${out#"${out%%[![:space:]]*}"}" + out="${out%"${out##*[![:space:]]}"}" + if [[ -z "${out}" || "${out}" == "." || "${out}" == ".." ]]; then + out="unknown" + fi + printf '%s' "${out}" + } + base="$(safe_segment "$(basename "${sidecar}")")" + digest="$(sha256sum "${sidecar}" | awk '{print $1}')" + safe_name="$(safe_segment "${pkg_name}")" + safe_version="$(safe_segment "${pkg_version}")" + relative_path="npm-tl-compliance/${safe_name}/${safe_version}/${digest}/${base}" + curl --fail-with-body --silent \ + --retry "${CURL_RETRY}" \ + --max-time "${CURL_MAX_TIME_SIDECAR_SECONDS}" \ + "${AUTH[@]}" \ + -X POST \ + -F "file=@${sidecar}" \ + --form-string "repository=${PULP_FILE_REPOSITORY_HREF}" \ + --form-string "relative_path=${relative_path}" \ + "${PULP_QUERY_BASE}content/file/files/" \ + && echo "Uploaded compliance sidecar ${sidecar} -> ${relative_path}" \ + || echo "WARNING: could not upload compliance sidecar ${sidecar}" +} + +uploaded=() +skipped_existing=() +failed=() + +while IFS= read -r -d '' file; do + if ! assert_tar_member_size "${file}" "package/package.json" \ + "${MAX_PACKAGE_JSON_BYTES}"; then + echo "ERROR: ${file} missing or oversized package/package.json" >&2 + failed+=("${file}") + continue + fi + if ! pkg_json="$(tar -xOf "${file}" package/package.json 2>/dev/null)"; then + echo "ERROR: ${file} missing package/package.json or is not a valid tarball" >&2 + failed+=("${file}") + continue + fi + name="$(jq -r '.name // empty' <<<"${pkg_json}")" + version="$(jq -r '.version // empty' <<<"${pkg_json}")" + if [[ -z "${name}" || -z "${version}" ]]; then + echo "ERROR: ${file} package.json missing non-empty name/version" >&2 + failed+=("${file}") + continue + fi + + compliance_level="" + sidecar="${file%.tgz}.tl-compliance.json" + if [[ -f "${sidecar}" ]]; then + sidecar_size="$(wc -c < "${sidecar}")" + if [[ "${sidecar_size}" -gt "${MAX_COMPLIANCE_SIDECAR_BYTES}" ]]; then + echo "ERROR: compliance sidecar ${sidecar} is ${sidecar_size} bytes" \ + "(max ${MAX_COMPLIANCE_SIDECAR_BYTES})" >&2 + failed+=("${file}") + continue + fi + if ! compliance_level="$(jq -er ' + if type != "object" then + error("sidecar root must be a JSON object") + elif (.compliance_level | type) != "string" then + error("compliance_level must be a string") + elif (.compliance_level | test("^L[123]$") | not) then + error("compliance_level must be L1, L2, or L3") + else .compliance_level end + ' "${sidecar}" 2>/tmp/sidecar-jq.err)"; then + echo "ERROR: invalid compliance sidecar ${sidecar}:" \ + "$(cat /tmp/sidecar-jq.err 2>/dev/null || true)" >&2 + failed+=("${file}") + continue + fi + fi + + echo "Checking if ${name}@${version} (${file}) exists in Pulp..." + status=0 + check_package_exists_with_digest "${name}" "${version}" "${file}" || status=$? + + if [[ "${status}" -eq 0 ]]; then + echo "Skipping upload of ${file} - identical content already in Pulp" + if [[ -n "${compliance_level}" ]]; then + if ! apply_compliance_label \ + "${EXISTING_CONTENT_HREF}" "${compliance_level}" "${EXISTING_LABEL}"; then + echo "ERROR: could not set ${COMPLIANCE_LABEL_KEY}=${compliance_level}" \ + "on existing ${name}@${version}" >&2 + failed+=("${file}") + continue + fi + fi + skipped_existing+=("${file}") + upload_compliance_sidecar "${sidecar}" "${name}" "${version}" + continue + elif [[ "${status}" -eq 3 || "${status}" -eq 1 ]]; then + failed+=("${file}") + continue + elif [[ "${status}" -ne 2 ]]; then + echo "ERROR: unexpected existence check status ${status} for ${file}" >&2 + failed+=("${file}") + continue + fi + + echo "Uploading ${file} (compliance=${compliance_level:-none})..." + # Use --form-string for text fields: -F treats a leading @ in the + # value as "read from file", which breaks scoped names (@scope/pkg). + CURL_ARGS=( + --fail-with-body + --silent + --retry "${CURL_RETRY}" + --max-time "${CURL_MAX_TIME_UPLOAD_SECONDS}" + "${AUTH[@]}" + -X POST + -F "file=@${file}" + --form-string "repository=${PULP_REPOSITORY_HREF}" + --form-string "name=${name}" + --form-string "version=${version}" + ) + if [[ -n "${compliance_level}" ]]; then + labels_json="$(jq -nc --arg lvl "${compliance_level}" \ + --arg key "${COMPLIANCE_LABEL_KEY}" \ + '{ ($key): $lvl }')" + CURL_ARGS+=(--form-string "pulp_labels=${labels_json}") + fi + + if curl "${CURL_ARGS[@]}" \ + "${PULP_QUERY_BASE}content/npm/packages/upload/"; then + echo "Uploaded ${file}" + uploaded+=("${file}") + # New content creates a new repository version; refresh so later + # existence checks in this run see the upload. + refresh_repository_version + echo " Repository version now: ${PULP_REPOSITORY_VERSION}" + upload_compliance_sidecar "${sidecar}" "${name}" "${version}" + else + echo "ERROR: Failed to upload ${file}" + failed+=("${file}") + fi +done < <(find "${FILES_DIR}" -type f -name '*.tgz' -print0 | sort -z) + +echo "" +echo "==============================" +echo " npm Pulp Upload Summary" +echo "==============================" +echo "Uploaded (${#uploaded[@]}):" +for f in "${uploaded[@]+"${uploaded[@]}"}"; do echo " - ${f}"; done +echo "Skipped - already in Pulp (${#skipped_existing[@]}):" +for f in "${skipped_existing[@]+"${skipped_existing[@]}"}"; do echo " - ${f}"; done +echo "Failed (${#failed[@]}):" +for f in "${failed[@]+"${failed[@]}"}"; do echo " - ${f}"; done + +if [[ ${#failed[@]} -gt 0 ]]; then + echo "ERROR: ${#failed[@]} upload(s) failed" + exit 1 +fi diff --git a/utils/scripts/select-oci-auth b/utils/scripts/select-oci-auth new file mode 100755 index 0000000..48f5b7f --- /dev/null +++ b/utils/scripts/select-oci-auth @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Selects the expected token from ~/.docker/config.json given an image reference. Default +# location of ~/.docker/config.json may be overriden by setting AUTHFILE +# +# Vendored from konflux-ci/build-trusted-artifacts (select-oci-auth.sh) so +# plumbing-utils can auth to private registries without release-service-utils. +# +# Usage: +# select-oci-auth +# +set -o errexit +set -o nounset +set -o pipefail + +if [ -z "${1:-}" ]; then + >&2 echo "Specify the image reference to match" + exit 1 +fi + +original_ref="$1" + +# Remove digest from image reference +ref="${original_ref/@*}" + +# Remove tag from image reference while making sure optional registry port is taken into account +ref="$(echo -n "$ref" | sed 's_/\(.*\):\(.*\)_/\1_g')" + +registry="${ref/\/*}" + +AUTHFILE="${AUTHFILE:-$HOME/.docker/config.json}" + +if [[ -f $AUTHFILE ]]; then + while true; do + token=$(< "${AUTHFILE}" jq -c '.auths["'"$ref"'"]') + if [[ "$token" != "null" && "$token" != "" ]]; then + >&2 echo "Using token for $ref" + echo -n '{"auths": {"'"$registry"'": '"$token"'}}' | jq -c . + exit 0 + fi + + if [[ "$ref" != *"/"* ]]; then + break + fi + + ref="${ref%*/*}" + done +fi + +>&2 echo "Token not found for $original_ref" + +echo -n '{"auths": {}}'