From 9d25f6f4222fc4814864c00195604cac1f97cef7 Mon Sep 17 00:00:00 2001 From: Omar Baradei Date: Wed, 10 Jun 2026 09:48:33 -0700 Subject: [PATCH 1/9] load-backup-archives: restore collections asynchronously and verify completion The synchronous Collections API RESTORE call hard-times-out after 180 seconds server-side, which large collections (release-group, url, recording) routinely exceed on modest hardware. The status check then accepted the error response anyway, because '=~ 0' is a substring match and "500" contains "0"; the extracted backup files were deleted from under the still-running background restore, leaving those collections silently empty (artist and release survived only by finishing under 180s). Submit the restore with async=, poll REQUESTSTATUS until a terminal state, fail loudly on failed/notfound, and only then clean up the request status and the extracted files. --- docker/scripts/load-backup-archives | 31 ++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/docker/scripts/load-backup-archives b/docker/scripts/load-backup-archives index fa324a8..fbca3e7 100644 --- a/docker/scripts/load-backup-archives +++ b/docker/scripts/load-backup-archives @@ -102,13 +102,38 @@ do tar -x --zstd -f "$DUMP_DIR/$dump_file" -C "$SOLR_BACKUPS_DIR" "$collection/$collection" echo "$SCRIPT_NAME: $(date): Restore the collection '$collection' from extracted backup..." - restore_output="$(curl -sS "$SOLR_BASE_URL/solr/admin/collections?action=RESTORE&collection=$collection&name=$collection&location=$SOLR_BACKUPS_DIR")" - if [[ ! "$(echo "$restore_output" | jq .responseHeader.status)" =~ 0 ]] + # Submit the restore asynchronously: the synchronous Collections API call + # hard-times-out after 180 seconds server-side, which large collections + # (release-group, url, recording) routinely exceed. The previous status + # check also accepted error responses, because `=~ 0` is a substring + # match and "500" contains "0"; the backup files were then deleted from + # under the still-running background restore, leaving the collection + # silently empty. + async_id="load-backup-$collection-$$" + restore_output="$(curl -sS "$SOLR_BASE_URL/solr/admin/collections?action=RESTORE&collection=$collection&name=$collection&location=$SOLR_BACKUPS_DIR&async=$async_id")" + if [[ "$(echo "$restore_output" | jq -r .responseHeader.status)" != 0 ]] then - echo >&2 "$SCRIPT_NAME: Fatal error while restoring the collection '$collection':" + echo >&2 "$SCRIPT_NAME: Fatal error while submitting the restore of the collection '$collection':" echo >&2 "$(echo "$restore_output" | jq .error.code): $(echo "$restore_output" | jq -r .error.msg)" exit 70 # EX_SOFTWARE fi + while true + do + sleep 10 + status_output="$(curl -sS "$SOLR_BASE_URL/solr/admin/collections?action=REQUESTSTATUS&requestid=$async_id")" + restore_state="$(echo "$status_output" | jq -r .status.state)" + case "$restore_state" in + completed) + break + ;; + failed|notfound) + echo >&2 "$SCRIPT_NAME: Fatal error while restoring the collection '$collection' (async state: $restore_state):" + echo >&2 "$(echo "$status_output" | jq -r '.status.msg // .error.msg // .')" + exit 70 # EX_SOFTWARE + ;; + esac + done + curl -sS "$SOLR_BASE_URL/solr/admin/collections?action=DELETESTATUS&requestid=$async_id" > /dev/null echo "$SCRIPT_NAME: $(date): Delete the no-longer-needed extracted backup '$collection'..." find "$SOLR_BACKUPS_DIR/$collection" -type f -delete From cbfd85cd104fc61c35516dd6935f4ae5dd477230 Mon Sep 17 00:00:00 2001 From: yvanzo Date: Wed, 1 Jul 2026 12:38:13 +0200 Subject: [PATCH 2/9] Amend 9d25f6f: Disable ShellCheck warning on echo Printing to stderr seems to be more readable like that. Reference: https://www.shellcheck.net/wiki/SC2005 --- docker/scripts/load-backup-archives | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/scripts/load-backup-archives b/docker/scripts/load-backup-archives index fbca3e7..1d448ec 100644 --- a/docker/scripts/load-backup-archives +++ b/docker/scripts/load-backup-archives @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck disable=SC2005 set -e -o pipefail -u From eabc2c51d331b7550d87c6bf6797742a1466b785 Mon Sep 17 00:00:00 2001 From: yvanzo Date: Wed, 1 Jul 2026 12:38:15 +0200 Subject: [PATCH 3/9] Amend 9d25f6f: Handle unexpected restore states MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Beyond the three restore states (`completed`, `failed`, and `notfound`) explicitly handled already, two other states (`running` and `submitted`) are silently handled too. However, it doesn’t handle unexpected restore states due to either future changes in Solr or transient network issues. This patch makes handling all known restore status explicitly, to allow handling unexpected restore states too. It will also allow reporting progress in a separate amending commit. Note that when the output of `REQUESTSTATUS` doesn’t contain any `.status.state` key, then `jq` will return the `null` string. References: https://github.com/apache/solr/blob/releases/solr/9.7.0/solr/solrj/src/java/org/apache/solr/client/solrj/response/RequestStatusState.java#L28-L41 https://solr.apache.org/guide/solr/latest/configuration-guide/collections-api.html#requeststatus https://jqlang.org/manual/#object-identifier-index --- docker/scripts/load-backup-archives | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docker/scripts/load-backup-archives b/docker/scripts/load-backup-archives index 1d448ec..6f63787 100644 --- a/docker/scripts/load-backup-archives +++ b/docker/scripts/load-backup-archives @@ -127,11 +127,24 @@ do completed) break ;; + running|submitted) + continue + ;; failed|notfound) echo >&2 "$SCRIPT_NAME: Fatal error while restoring the collection '$collection' (async state: $restore_state):" echo >&2 "$(echo "$status_output" | jq -r '.status.msg // .error.msg // .')" exit 70 # EX_SOFTWARE ;; + null) + echo >&2 "$SCRIPT_NAME: No restore state for the collection '$collection'; Solr response:" + echo >&2 "$(echo "$status_output" | jq .)" + exit 75 # EX_TEMPFAIL + ;; + *) + echo >&2 "$SCRIPT_NAME: Unknown restore state '$restore_state' for the collection '$collection'; Solr response:" + echo >&2 "$(echo "$status_output" | jq .)" + exit 76 # EX_PROTOCOL + ;; esac done curl -sS "$SOLR_BASE_URL/solr/admin/collections?action=DELETESTATUS&requestid=$async_id" > /dev/null From 882ddc8e0535a30f88e9c9a01576efdc14cc0120 Mon Sep 17 00:00:00 2001 From: yvanzo Date: Wed, 1 Jul 2026 12:38:16 +0200 Subject: [PATCH 4/9] Amend 9d25f6f: Add customizable timeout While the synchronous requests were timing out too soon due to a hard server-side time limit of 180s, it is still safer to have a time limit for the asynchronous requests, preferably client-side and customizable. This patch limits the number of polling to 360. That number can be changed through the environment variable `MUSICBRAINZ_SEARCH_MAX_POLL`. With 10s between each polling, it will stop after one hour at least. --- docker/scripts/load-backup-archives | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docker/scripts/load-backup-archives b/docker/scripts/load-backup-archives index 6f63787..4a81e09 100644 --- a/docker/scripts/load-backup-archives +++ b/docker/scripts/load-backup-archives @@ -4,6 +4,7 @@ set -e -o pipefail -u DUMP_DIR="${MUSICBRAINZ_SEARCH_DUMP_DIR:-/var/cache/musicbrainz/solr-backups}" +MAX_POLL="${MUSICBRAINZ_SEARCH_MAX_POLL:-360}" # 360 × 10s = 1h SOLR_BACKUPS_DIR="${SOLR_BACKUPS_DIR:-/var/solr/data/backups}" SOLR_BASE_URL="http://${SOLR_LOCAL_HOST:-localhost}:${SOLR_PORT:-8983}" @@ -19,6 +20,10 @@ Options: Environment: MUSICBRAINZ_SEARCH_DUMP_DIR path to directory with Solr backup tar archives (default: /var/cache/musicbrainz/solr-backups) + MUSICBRAINZ_SEARCH_MAX_POLL maximum number of polling attempts for + asynchronous requests sent to the Solr API, + with a pause of 10s between each attempt + (default: 360 attempts, that is about 1h) SOLR_BACKUPS_DIR path to directory with Solr backup files (default: /var/solr/data/backups) SOLR_LOCAL_HOST upstream Solr server listening host @@ -118,9 +123,11 @@ do echo >&2 "$(echo "$restore_output" | jq .error.code): $(echo "$restore_output" | jq -r .error.msg)" exit 70 # EX_SOFTWARE fi + poll_attempt=0 while true do sleep 10 + poll_attempt=$((poll_attempt + 1)) status_output="$(curl -sS "$SOLR_BASE_URL/solr/admin/collections?action=REQUESTSTATUS&requestid=$async_id")" restore_state="$(echo "$status_output" | jq -r .status.state)" case "$restore_state" in @@ -146,6 +153,11 @@ do exit 76 # EX_PROTOCOL ;; esac + if [[ $poll_attempt -ge $MAX_POLL ]] + then + echo >&2 "$SCRIPT_NAME: Timed out waiting for restore of the collection '$collection' (after $((MAX_POLL * 10)) seconds)" + exit 75 # EX_TEMPFAIL + fi done curl -sS "$SOLR_BASE_URL/solr/admin/collections?action=DELETESTATUS&requestid=$async_id" > /dev/null From c6c390f2f705861b3a38486a7d7ba9b59ac9da52 Mon Sep 17 00:00:00 2001 From: yvanzo Date: Wed, 1 Jul 2026 12:38:17 +0200 Subject: [PATCH 5/9] Amend 9d25f6f: Quote status check RHS for clarity Clarify that this check is a string comparison with the right-hand side. --- docker/scripts/load-backup-archives | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/scripts/load-backup-archives b/docker/scripts/load-backup-archives index 4a81e09..ce725e2 100644 --- a/docker/scripts/load-backup-archives +++ b/docker/scripts/load-backup-archives @@ -117,7 +117,7 @@ do # silently empty. async_id="load-backup-$collection-$$" restore_output="$(curl -sS "$SOLR_BASE_URL/solr/admin/collections?action=RESTORE&collection=$collection&name=$collection&location=$SOLR_BACKUPS_DIR&async=$async_id")" - if [[ "$(echo "$restore_output" | jq -r .responseHeader.status)" != 0 ]] + if [[ "$(echo "$restore_output" | jq -r .responseHeader.status)" != '0' ]] then echo >&2 "$SCRIPT_NAME: Fatal error while submitting the restore of the collection '$collection':" echo >&2 "$(echo "$restore_output" | jq .error.code): $(echo "$restore_output" | jq -r .error.msg)" From 8779a2b64d33272a3773da044e545ef04391919d Mon Sep 17 00:00:00 2001 From: yvanzo Date: Wed, 1 Jul 2026 12:38:19 +0200 Subject: [PATCH 6/9] Amend 9d25f6f: Drop duplicate/obsolete comment This comment is in git commit message already. It is about previous bugged code only. --- docker/scripts/load-backup-archives | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docker/scripts/load-backup-archives b/docker/scripts/load-backup-archives index ce725e2..afb31c0 100644 --- a/docker/scripts/load-backup-archives +++ b/docker/scripts/load-backup-archives @@ -110,11 +110,7 @@ do echo "$SCRIPT_NAME: $(date): Restore the collection '$collection' from extracted backup..." # Submit the restore asynchronously: the synchronous Collections API call # hard-times-out after 180 seconds server-side, which large collections - # (release-group, url, recording) routinely exceed. The previous status - # check also accepted error responses, because `=~ 0` is a substring - # match and "500" contains "0"; the backup files were then deleted from - # under the still-running background restore, leaving the collection - # silently empty. + # (release-group, url, recording) routinely exceed. async_id="load-backup-$collection-$$" restore_output="$(curl -sS "$SOLR_BASE_URL/solr/admin/collections?action=RESTORE&collection=$collection&name=$collection&location=$SOLR_BACKUPS_DIR&async=$async_id")" if [[ "$(echo "$restore_output" | jq -r .responseHeader.status)" != '0' ]] From 40924203c77304a8ec62b82589137fd4d0034ddf Mon Sep 17 00:00:00 2001 From: yvanzo Date: Wed, 1 Jul 2026 12:38:20 +0200 Subject: [PATCH 7/9] Amend 9d25f6f: Add timestamp to async request ID To better avoid conflicts with stalled requests from previous attempts, this patch adds the current timestamp to the asynchronous request ID. It also prefixes each number with a descriptive abbreviation. --- docker/scripts/load-backup-archives | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/scripts/load-backup-archives b/docker/scripts/load-backup-archives index afb31c0..29b4265 100644 --- a/docker/scripts/load-backup-archives +++ b/docker/scripts/load-backup-archives @@ -111,7 +111,7 @@ do # Submit the restore asynchronously: the synchronous Collections API call # hard-times-out after 180 seconds server-side, which large collections # (release-group, url, recording) routinely exceed. - async_id="load-backup-$collection-$$" + async_id="load-backup-$collection-pid$$-ts$(date +%s)" restore_output="$(curl -sS "$SOLR_BASE_URL/solr/admin/collections?action=RESTORE&collection=$collection&name=$collection&location=$SOLR_BACKUPS_DIR&async=$async_id")" if [[ "$(echo "$restore_output" | jq -r .responseHeader.status)" != '0' ]] then From 8ab1ee3ca15114bb0968b3a24501a5c98072ba89 Mon Sep 17 00:00:00 2001 From: yvanzo Date: Wed, 1 Jul 2026 12:38:22 +0200 Subject: [PATCH 8/9] Amend 9d25f6f: Warn on DELETESTATUS call failure A delete status failure is most likely not blocking, for example if the asynchronous request ID already expired. Still, it is worth warning about it to help with debugging in case of serious issues. Reference: https://solr.apache.org/guide/solr/9_7/configuration-guide/collections-api.html#deletestatus --- docker/scripts/load-backup-archives | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/scripts/load-backup-archives b/docker/scripts/load-backup-archives index 29b4265..84fc78d 100644 --- a/docker/scripts/load-backup-archives +++ b/docker/scripts/load-backup-archives @@ -155,7 +155,12 @@ do exit 75 # EX_TEMPFAIL fi done - curl -sS "$SOLR_BASE_URL/solr/admin/collections?action=DELETESTATUS&requestid=$async_id" > /dev/null + deletestatus_output="$(curl -sS "$SOLR_BASE_URL/solr/admin/collections?action=DELETESTATUS&requestid=$async_id")" + if [[ "$(echo "$deletestatus_output" | jq -r .responseHeader.status)" != '0' ]] + then + echo >&2 "$SCRIPT_NAME: Warning: failed to delete async status for '$async_id':" + echo >&2 "$(echo "$deletestatus_output" | jq -r '.error.msg // .')" + fi echo "$SCRIPT_NAME: $(date): Delete the no-longer-needed extracted backup '$collection'..." find "$SOLR_BACKUPS_DIR/$collection" -type f -delete From 61f6c1f904c8dfcae4c88d7bd99117fdbc269abf Mon Sep 17 00:00:00 2001 From: yvanzo Date: Wed, 1 Jul 2026 12:38:25 +0200 Subject: [PATCH 9/9] Amend 9d25f6f: Report progress in restore Restoring large collections such as recordings can take several minutes. To help with monitoring the progress during that time, this patch logs a progress message about every minute, actually every 6 polling attempts. --- docker/scripts/load-backup-archives | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/scripts/load-backup-archives b/docker/scripts/load-backup-archives index 84fc78d..d7f6a7b 100644 --- a/docker/scripts/load-backup-archives +++ b/docker/scripts/load-backup-archives @@ -131,7 +131,10 @@ do break ;; running|submitted) - continue + if (( poll_attempt % 6 == 1 )) + then + echo "$SCRIPT_NAME: $(date): Still waiting for the restore of the collection '$collection' to complete (state: $restore_state, attempt $poll_attempt/$MAX_POLL)..." + fi ;; failed|notfound) echo >&2 "$SCRIPT_NAME: Fatal error while restoring the collection '$collection' (async state: $restore_state):"