From e2441be91e22eb4b8dbbdbc48b95c13dc158bd41 Mon Sep 17 00:00:00 2001 From: Jeffrey Tang Date: Mon, 27 Jul 2026 14:04:36 -0500 Subject: [PATCH 01/20] update Signed-off-by: Jeffrey Tang --- examples/state-save-and-restore/README.md | 9 +++--- examples/state-save-and-restore/Taskfile.yml | 30 +++++++++++++------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/examples/state-save-and-restore/README.md b/examples/state-save-and-restore/README.md index 88b298a1c3..70265e7098 100644 --- a/examples/state-save-and-restore/README.md +++ b/examples/state-save-and-restore/README.md @@ -90,7 +90,7 @@ task restore This will: -* Stop and destroy existing network +* Stop and destroy the mirror node and consensus network * Recreate PostgreSQL database * Import database dump * Create new consensus network with same configuration @@ -171,13 +171,13 @@ The `init.sh` script sets up the PostgreSQL database with: 1. **Database Recreation**: Deploys fresh PostgreSQL and runs `init.sh` to create database structure (database, schemas, roles, users, extensions) 2. **Database Restore**: Imports database dump which drops and recreates tables with all data -3. **Stable Service Validation**: Verifies per-node service DNS names are resolvable (`network--svc..svc.cluster.local`) +3. **Fresh Network Deployment**: Regenerates consensus keys, redeploys the consensus network, and runs node setup for the new pods 4. **Restore Input Build**: Builds `./saved-states/restore-input/states///` and copies each node's state zip 5. **State Upload and Start**: Starts all nodes together with `solo consensus node start --state-file ./saved-states/restore-input` * State files are extracted to `data/saved/` * Cleanup: Only the latest/biggest round is kept, older rounds are automatically deleted to save disk space * Node ID Renaming: Directory paths containing node IDs are automatically renamed to match each target node -6. **Mirror Node**: Deploys mirror node connected to restored database and seeds initial data +6. **Mirror Node**: Redeploys the mirror node connected to the restored database 7. **Verification**: Checks that restored state matches original ## Notes @@ -187,8 +187,7 @@ The `init.sh` script sets up the PostgreSQL database with: * External PostgreSQL database provides data persistence and queryability * State restoration maintains transaction history and account balances * Mirror node will resume from the restored state point -* **Per-node State Restore**: Uses each node's own state zip and starts all nodes together on the existing network pods -* Stable per-node service names are validated before restore start +* **Per-node State Restore**: Uses each node's own state zip and starts all nodes together on a freshly redeployed network * Database dump includes all mirror node data (transactions, accounts, etc.) ### View Logs diff --git a/examples/state-save-and-restore/Taskfile.yml b/examples/state-save-and-restore/Taskfile.yml index 569836d72b..c0cad12701 100644 --- a/examples/state-save-and-restore/Taskfile.yml +++ b/examples/state-save-and-restore/Taskfile.yml @@ -51,7 +51,7 @@ tasks: - task: stop-network - task: save-state - cmd: echo "" - - cmd: echo "⏳ Waiting 10 seconds before restore..." + - cmd: echo "⏳ Waiting 10 seconds before destroy and restore..." - cmd: sleep 10 - task: restore - cmd: echo "" @@ -213,28 +213,27 @@ tasks: restore: desc: Recreate network and restore state with external database cmds: + - task: destroy-network - task: destroy-database - task: deploy-external-database - task: deploy-network-with-state - task: restore-database + - task: deploy-mirror-external - task: verify-state - cmd: echo "✅ Network and database restored!" deploy-network-with-state: - desc: Deploy network and upload saved state + desc: Deploy a fresh network and start nodes from saved state cmds: - cmd: echo "Deploying network with saved state..." - cmd: | - echo "Reusing existing consensus network pods for restore to keep service IPs stable..." - kubectl get pod network-node1-0 -n {{ .NAMESPACE }} >/dev/null - kubectl get pod network-node2-0 -n {{ .NAMESPACE }} >/dev/null + $SOLO_COMMAND keys consensus generate --gossip-keys --tls-keys --node-aliases {{ .NODE_ALIASES }} --deployment {{ .DEPLOYMENT }} + - cmd: $SOLO_COMMAND consensus network deploy --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} + - cmd: $SOLO_COMMAND consensus node setup --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} - cmd: | - echo "Validating stable per-node service DNS names..." + echo "Waiting for recreated consensus pods to exist before restore..." for node in $(echo {{ .NODE_ALIASES }} | tr ',' ' '); do - SERVICE_FQDN="network-${node}-svc.{{ .NAMESPACE }}.svc.cluster.local" - kubectl exec network-node1-0 -n {{ .NAMESPACE }} -c root-container -- \ - getent hosts "${SERVICE_FQDN}" >/dev/null - echo "✅ ${SERVICE_FQDN} is resolvable" + kubectl wait --for=create pod/network-${node}-0 -n {{ .NAMESPACE }} --timeout=300s done - cmd: | # Build the directory layout expected by Solo for per-node state restore: @@ -304,6 +303,15 @@ tasks: - cmd: $SOLO_COMMAND consensus network freeze --deployment {{ .DEPLOYMENT }} - cmd: echo "✅ Network frozen" + destroy-network: + desc: Destroy mirror node and consensus network while keeping cluster configuration + cmds: + - cmd: echo "Destroying mirror node and consensus network..." + - cmd: $SOLO_COMMAND mirror node destroy --deployment {{ .DEPLOYMENT }} --force || true + - cmd: $SOLO_COMMAND consensus node stop --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} || true + - cmd: $SOLO_COMMAND consensus network destroy --deployment {{ .DEPLOYMENT }} --force --delete-pvcs --delete-secrets -q || true + - cmd: echo "✅ Network resources destroyed" + destroy-database: desc: Destroy external database @@ -317,6 +325,8 @@ tasks: destroy: desc: Destroy cluster and clean up all resources cmds: + - task: destroy-network + - task: destroy-database - cmd: kind delete cluster --name {{ .CLUSTER_NAME }} - cmd: echo "✅ Cluster destroyed" - task: clean-state From 52a3680e8452ee921d12ecd3f12dceec07126c65 Mon Sep 17 00:00:00 2001 From: Jeffrey Tang Date: Mon, 27 Jul 2026 16:04:41 -0500 Subject: [PATCH 02/20] save Signed-off-by: Jeffrey Tang --- examples/state-save-and-restore/Taskfile.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/state-save-and-restore/Taskfile.yml b/examples/state-save-and-restore/Taskfile.yml index c0cad12701..08e08eea84 100644 --- a/examples/state-save-and-restore/Taskfile.yml +++ b/examples/state-save-and-restore/Taskfile.yml @@ -215,6 +215,7 @@ tasks: cmds: - task: destroy-network - task: destroy-database + - task: init-solo - task: deploy-external-database - task: deploy-network-with-state - task: restore-database From ad84f73aa9490107da30cd841a49f96020d90c51 Mon Sep 17 00:00:00 2001 From: Jeffrey Tang Date: Mon, 27 Jul 2026 16:45:23 -0500 Subject: [PATCH 03/20] save Signed-off-by: Jeffrey Tang --- examples/state-save-and-restore/README.md | 6 +++--- examples/state-save-and-restore/Taskfile.yml | 5 +---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/examples/state-save-and-restore/README.md b/examples/state-save-and-restore/README.md index 70265e7098..1f206963b4 100644 --- a/examples/state-save-and-restore/README.md +++ b/examples/state-save-and-restore/README.md @@ -93,7 +93,7 @@ This will: * Stop and destroy the mirror node and consensus network * Recreate PostgreSQL database * Import database dump -* Create new consensus network with same configuration +* Recreate the consensus network with the original deployment metadata and key material * Upload saved state to new nodes * Start nodes with restored state * Reconnect mirror node to database @@ -171,7 +171,7 @@ The `init.sh` script sets up the PostgreSQL database with: 1. **Database Recreation**: Deploys fresh PostgreSQL and runs `init.sh` to create database structure (database, schemas, roles, users, extensions) 2. **Database Restore**: Imports database dump which drops and recreates tables with all data -3. **Fresh Network Deployment**: Regenerates consensus keys, redeploys the consensus network, and runs node setup for the new pods +3. **Fresh Network Deployment**: Reuses the original deployment metadata and consensus key material, redeploys the consensus network, and runs node setup for the new pods 4. **Restore Input Build**: Builds `./saved-states/restore-input/states///` and copies each node's state zip 5. **State Upload and Start**: Starts all nodes together with `solo consensus node start --state-file ./saved-states/restore-input` * State files are extracted to `data/saved/` @@ -187,7 +187,7 @@ The `init.sh` script sets up the PostgreSQL database with: * External PostgreSQL database provides data persistence and queryability * State restoration maintains transaction history and account balances * Mirror node will resume from the restored state point -* **Per-node State Restore**: Uses each node's own state zip and starts all nodes together on a freshly redeployed network +* **Per-node State Restore**: Uses each node's own state zip and starts all nodes together on a freshly redeployed network with the original consensus keys * Database dump includes all mirror node data (transactions, accounts, etc.) ### View Logs diff --git a/examples/state-save-and-restore/Taskfile.yml b/examples/state-save-and-restore/Taskfile.yml index 08e08eea84..0c40b564f8 100644 --- a/examples/state-save-and-restore/Taskfile.yml +++ b/examples/state-save-and-restore/Taskfile.yml @@ -215,7 +215,6 @@ tasks: cmds: - task: destroy-network - task: destroy-database - - task: init-solo - task: deploy-external-database - task: deploy-network-with-state - task: restore-database @@ -227,8 +226,6 @@ tasks: desc: Deploy a fresh network and start nodes from saved state cmds: - cmd: echo "Deploying network with saved state..." - - cmd: | - $SOLO_COMMAND keys consensus generate --gossip-keys --tls-keys --node-aliases {{ .NODE_ALIASES }} --deployment {{ .DEPLOYMENT }} - cmd: $SOLO_COMMAND consensus network deploy --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} - cmd: $SOLO_COMMAND consensus node setup --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} - cmd: | @@ -310,7 +307,7 @@ tasks: - cmd: echo "Destroying mirror node and consensus network..." - cmd: $SOLO_COMMAND mirror node destroy --deployment {{ .DEPLOYMENT }} --force || true - cmd: $SOLO_COMMAND consensus node stop --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} || true - - cmd: $SOLO_COMMAND consensus network destroy --deployment {{ .DEPLOYMENT }} --force --delete-pvcs --delete-secrets -q || true + - cmd: $SOLO_COMMAND consensus network destroy --deployment {{ .DEPLOYMENT }} --force -q || true - cmd: echo "✅ Network resources destroyed" From 9bf5cce1713ed110cae5973c1057c62c819fbe49 Mon Sep 17 00:00:00 2001 From: Jeffrey Tang Date: Mon, 27 Jul 2026 17:27:24 -0500 Subject: [PATCH 04/20] save Signed-off-by: Jeffrey Tang --- examples/state-save-and-restore/Taskfile.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/state-save-and-restore/Taskfile.yml b/examples/state-save-and-restore/Taskfile.yml index 0c40b564f8..93044bc6eb 100644 --- a/examples/state-save-and-restore/Taskfile.yml +++ b/examples/state-save-and-restore/Taskfile.yml @@ -215,6 +215,7 @@ tasks: cmds: - task: destroy-network - task: destroy-database + - task: init-solo - task: deploy-external-database - task: deploy-network-with-state - task: restore-database From 8af597241f6f8ed7222f49bf303cfe2df9dce96c Mon Sep 17 00:00:00 2001 From: Jeffrey Tang Date: Mon, 27 Jul 2026 22:21:34 -0500 Subject: [PATCH 05/20] save Signed-off-by: Jeffrey Tang --- examples/state-save-and-restore/README.md | 16 +++--- examples/state-save-and-restore/Taskfile.yml | 57 ++++++++++++++++++++ 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/examples/state-save-and-restore/README.md b/examples/state-save-and-restore/README.md index 1f206963b4..ca548d7906 100644 --- a/examples/state-save-and-restore/README.md +++ b/examples/state-save-and-restore/README.md @@ -135,6 +135,7 @@ saved-states/ ├── state-restore-namespace/ │ ├── network-node1-0-state.zip │ └── network-node2-0-state.zip +├── mirror-passwords-secret.json └── database-dump.sql # PostgreSQL database export ``` @@ -143,6 +144,7 @@ saved-states/ * State files are named using the pod naming convention: `network--0-state.zip` * During save: All node state files are downloaded * During restore: A per-node restore input directory is built and passed to `solo consensus node start --state-file` +* Mirror database credentials are preserved in `mirror-passwords-secret.json` and restored before mirror redeploy The example also includes: @@ -166,19 +168,21 @@ The `init.sh` script sets up the PostgreSQL database with: 1. **Download State**: Uses `solo consensus state download` to download signed state from each consensus node to `~/.solo/logs//` 2. **Copy State Files**: Copies state files from `~/.solo/logs//` to `./saved-states/` directory 3. **Export Database**: Uses `pg_dump` with `--clean --if-exists` flags to export the complete database including schema and data +4. **Save Mirror Credentials**: Exports the `mirror-passwords` secret so the restored mirror deployment reuses the original DB role passwords ### State Restoration Process 1. **Database Recreation**: Deploys fresh PostgreSQL and runs `init.sh` to create database structure (database, schemas, roles, users, extensions) -2. **Database Restore**: Imports database dump which drops and recreates tables with all data -3. **Fresh Network Deployment**: Reuses the original deployment metadata and consensus key material, redeploys the consensus network, and runs node setup for the new pods -4. **Restore Input Build**: Builds `./saved-states/restore-input/states///` and copies each node's state zip -5. **State Upload and Start**: Starts all nodes together with `solo consensus node start --state-file ./saved-states/restore-input` +2. **Fresh Network Deployment**: Reuses the original deployment metadata and consensus key material, redeploys the consensus network, and runs node setup for the new pods +3. **Restore Mirror Credentials**: Restores the saved `mirror-passwords` secret so mirror components reuse the original database passwords +4. **Database Restore**: Reconciles mirror database roles from the saved secret, then imports the database dump +5. **Restore Input Build**: Builds `./saved-states/restore-input/states///` and copies each node's state zip +6. **State Upload and Start**: Starts all nodes together with `solo consensus node start --state-file ./saved-states/restore-input` * State files are extracted to `data/saved/` * Cleanup: Only the latest/biggest round is kept, older rounds are automatically deleted to save disk space * Node ID Renaming: Directory paths containing node IDs are automatically renamed to match each target node -6. **Mirror Node**: Redeploys the mirror node connected to the restored database -7. **Verification**: Checks that restored state matches original +7. **Mirror Node**: Redeploys the mirror node connected to the restored database +8. **Verification**: Checks that restored state matches original ## Notes diff --git a/examples/state-save-and-restore/Taskfile.yml b/examples/state-save-and-restore/Taskfile.yml index 93044bc6eb..0abecdb2f8 100644 --- a/examples/state-save-and-restore/Taskfile.yml +++ b/examples/state-save-and-restore/Taskfile.yml @@ -23,6 +23,7 @@ vars: # State Save Configuration STATE_SAVE_DIR: "{{ .USER_WORKING_DIR }}/saved-states" + MIRROR_PASSWORDS_SECRET_FILE: "{{ .STATE_SAVE_DIR }}/mirror-passwords-secret.json" # External Database Configuration (Optional) POSTGRES_USERNAME: "postgres" @@ -204,6 +205,17 @@ tasks: env PGPASSWORD={{ .POSTGRES_PASSWORD }} pg_dump -U {{ .POSTGRES_USERNAME }} \ --clean --if-exists \ {{ .POSTGRES_MIRROR_NODE_DATABASE_NAME }} > {{ .STATE_SAVE_DIR }}/database-dump.sql + - cmd: echo "Saving mirror passwords secret..." + - cmd: | + kubectl get secret mirror-passwords -n {{ .NAMESPACE }} -o json | \ + jq 'del( + .metadata.annotations."kubectl.kubernetes.io/last-applied-configuration", + .metadata.creationTimestamp, + .metadata.managedFields, + .metadata.resourceVersion, + .metadata.uid + )' > {{ .MIRROR_PASSWORDS_SECRET_FILE }} + - cmd: echo "✅ Mirror passwords secret exported to {{ .MIRROR_PASSWORDS_SECRET_FILE }}" - cmd: echo "✅ Database exported to {{ .STATE_SAVE_DIR }}/database-dump.sql" - cmd: echo "✅ Network state and database saved to {{ .STATE_SAVE_DIR }}" - cmd: ls -lh {{ .STATE_SAVE_DIR }} @@ -218,6 +230,7 @@ tasks: - task: init-solo - task: deploy-external-database - task: deploy-network-with-state + - task: restore-mirror-passwords-secret - task: restore-database - task: deploy-mirror-external - task: verify-state @@ -268,6 +281,39 @@ tasks: desc: Restore database from dump cmds: - cmd: echo "Restoring database from dump..." + - cmd: | + if [ -f {{ .MIRROR_PASSWORDS_SECRET_FILE }} ]; then + echo "Recreating mirror database roles from saved credentials..." + jq -r '.data | keys[]' {{ .MIRROR_PASSWORDS_SECRET_FILE }} | while read -r key; do + case "${key}" in + *USERNAME) + password_key="${key%USERNAME}PASSWORD" + username="$(jq -r --arg key "${key}" '.data[$key] | @base64d' {{ .MIRROR_PASSWORDS_SECRET_FILE }})" + password="$(jq -r --arg key "${password_key}" '.data[$key] // empty | @base64d' {{ .MIRROR_PASSWORDS_SECRET_FILE }})" + + case "${username}" in + postgres|readonly|readonlyuser|readwrite|temporary_admin|"") + continue + ;; + esac + + if [ -z "${password}" ]; then + continue + fi + + username_sql="$(printf "%s" "${username}" | sed 's/"/""/g')" + role_name_sql="$(printf "%s" "${username}" | sed "s/'/''/g")" + password_sql="$(printf "%s" "${password}" | sed "s/'/''/g")" + + kubectl exec {{ .POSTGRES_CONTAINER_NAME }} -n {{ .POSTGRES_DATABASE_NAMESPACE }} -- \ + env PGPASSWORD={{ .POSTGRES_PASSWORD }} psql -U {{ .POSTGRES_USERNAME }} -d postgres \ + -v ON_ERROR_STOP=1 \ + -c "DO \$\$ BEGIN IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '${role_name_sql}') THEN ALTER ROLE \"${username_sql}\" WITH LOGIN PASSWORD '${password_sql}'; ELSE CREATE ROLE \"${username_sql}\" WITH LOGIN PASSWORD '${password_sql}'; END IF; END \$\$;" + echo "✅ Reconciled role ${username}" + ;; + esac + done + fi - cmd: | kubectl cp {{ .STATE_SAVE_DIR }}/database-dump.sql \ {{ .POSTGRES_CONTAINER_NAME }}:/tmp/database-dump.sql -n {{ .POSTGRES_DATABASE_NAMESPACE }} @@ -277,6 +323,17 @@ tasks: -d {{ .POSTGRES_MIRROR_NODE_DATABASE_NAME }} -f /tmp/database-dump.sql - cmd: echo "✅ Database restored" + restore-mirror-passwords-secret: + desc: Restore saved mirror database credential secret + cmds: + - cmd: | + if [ ! -f {{ .MIRROR_PASSWORDS_SECRET_FILE }} ]; then + echo "⚠️ Saved mirror-passwords secret not found: {{ .MIRROR_PASSWORDS_SECRET_FILE }}" + exit 1 + fi + - cmd: kubectl apply -f {{ .MIRROR_PASSWORDS_SECRET_FILE }} + - cmd: echo "✅ Restored mirror-passwords secret" + # ==================== Verification ==================== verify-state: From 96dd7d2ff2d970318cc4ea0d5b1dc6c5edd548b8 Mon Sep 17 00:00:00 2001 From: Jeffrey Tang Date: Thu, 30 Jul 2026 09:33:18 -0500 Subject: [PATCH 06/20] save Signed-off-by: Jeffrey Tang --- examples/state-save-and-restore/Taskfile.yml | 237 +++++++++++++++--- .../scripts/generate-override-network.mjs | 148 +++++++++++ 2 files changed, 349 insertions(+), 36 deletions(-) create mode 100644 examples/state-save-and-restore/scripts/generate-override-network.mjs diff --git a/examples/state-save-and-restore/Taskfile.yml b/examples/state-save-and-restore/Taskfile.yml index 0abecdb2f8..62b06d32e4 100644 --- a/examples/state-save-and-restore/Taskfile.yml +++ b/examples/state-save-and-restore/Taskfile.yml @@ -15,6 +15,9 @@ vars: NODE_ALIASES: "node1,node2" DEPLOYMENT: "state-restore-deployment" NAMESPACE: "state-restore-namespace" + ENABLE_MIRROR_WORKFLOW: '{{ default "false" (env "ENABLE_MIRROR_WORKFLOW") }}' + + SOLO_USER_DIR: "{{ default (printf \"%s/.solo\" (env \"HOME\")) }}" # Cluster Configuration CLUSTER_NAME: "state-restore-cluster" @@ -24,6 +27,13 @@ vars: # State Save Configuration STATE_SAVE_DIR: "{{ .USER_WORKING_DIR }}/saved-states" MIRROR_PASSWORDS_SECRET_FILE: "{{ .STATE_SAVE_DIR }}/mirror-passwords-secret.json" + ORIGINAL_NETWORK_JSON_FILE: "{{ .STATE_SAVE_DIR }}/original-network.json" + GENERATED_OVERRIDE_NETWORK_JSON_FILE: "{{ .STATE_SAVE_DIR }}/override-network.json" + CURRENT_SERVICE_ENDPOINTS_FILE: "{{ .STATE_SAVE_DIR }}/current-service-endpoints.json" + OVERRIDE_NETWORK_GENERATOR_SCRIPT: "{{ .TASKFILE_DIR }}/scripts/generate-override-network.mjs" + PREPULL_IMAGES_SCRIPT: "{{ .TASKFILE_DIR }}/scripts/prepull-images.sh" + SAVED_KEYS_DIR: "{{ .STATE_SAVE_DIR }}/keys" + SOLO_CACHE_KEYS_DIR: "{{ .SOLO_USER_DIR }}/cache/keys" # External Database Configuration (Optional) POSTGRES_USERNAME: "postgres" @@ -36,8 +46,6 @@ vars: POSTGRES_CONTAINER_NAME: "{{ .POSTGRES_NAME }}-0" POSTGRES_HOST_FQDN: "{{ .POSTGRES_NAME }}.database.svc.cluster.local" - SOLO_USER_DIR: "{{ default (printf \"%s/.solo\" (env \"HOME\")) }}" - tasks: # ==================== Main Tasks ==================== @@ -52,7 +60,7 @@ tasks: - task: stop-network - task: save-state - cmd: echo "" - - cmd: echo "⏳ Waiting 10 seconds before destroy and restore..." + - cmd: echo "⏳ Waiting 10 seconds before fresh-cluster restore..." - cmd: sleep 10 - task: restore - cmd: echo "" @@ -64,13 +72,18 @@ tasks: setup: desc: Deploy initial network with external PostgreSQL database cmds: - # Run solo init to install dependencies - - cmd: $SOLO_COMMAND init --dev - task: create-cluster + - task: preload-network-images - task: init-solo - - task: deploy-external-database + - task: deploy-block-node - task: deploy-network - - task: deploy-mirror-external + - cmd: | + if [ "{{ .ENABLE_MIRROR_WORKFLOW }}" = "true" ]; then + task deploy-external-database + task deploy-mirror-external + else + echo "Skipping mirror/database setup; set ENABLE_MIRROR_WORKFLOW=true to include it." + fi - task: generate-transactions - cmd: echo "✅ Initial network with external database setup complete!" - cmd: echo "Run 'task save-state' to save state and database" @@ -94,13 +107,17 @@ tasks: kind create cluster -n {{ .CLUSTER_NAME }} fi - cmd: sleep 10 # Wait for control plane - - cmd: kubectl config set-context {{ .CONTEXT }} + - cmd: kubectl config use-context {{ .CONTEXT }} + + preload-network-images: + desc: Pre-pull and load consensus-network images into the Kind cluster + cmds: + - cmd: bash {{ .PREPULL_IMAGES_SCRIPT }} {{ .CLUSTER_NAME }} init-solo: - desc: Initialize Solo and configure cluster + desc: Connect cluster reference and configure deployment cmds: - - cmd: $SOLO_COMMAND cluster-ref config setup --cluster-ref {{ .CLUSTER_REF }} - - cmd: $SOLO_COMMAND cluster-ref config connect --cluster-ref {{ .CLUSTER_REF }} --context {{ .CONTEXT }} + - cmd: $SOLO_COMMAND cluster-ref config connect --cluster-ref {{ .CLUSTER_REF }} --context {{ .CONTEXT }} --quiet-mode - cmd: $SOLO_COMMAND deployment config create --namespace {{ .NAMESPACE }} --deployment {{ .DEPLOYMENT }} --realm 0 --shard 0 - cmd: $SOLO_COMMAND deployment cluster attach --cluster-ref {{ .CLUSTER_REF }} --deployment {{ .DEPLOYMENT }} --num-consensus-nodes {{ .NETWORK_SIZE }} @@ -110,7 +127,9 @@ tasks: desc: Deploy consensus network cmds: - cmd: $SOLO_COMMAND keys consensus generate --gossip-keys --tls-keys --node-aliases {{ .NODE_ALIASES }} --deployment {{ .DEPLOYMENT }} - - cmd: $SOLO_COMMAND consensus network deploy --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} + - cmd: | + $SOLO_COMMAND consensus network deploy --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} \ + --enable-monitoring-support false --quiet-mode --dev - cmd: $SOLO_COMMAND consensus node setup --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} - cmd: $SOLO_COMMAND consensus node start --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} - cmd: echo "✅ Consensus network deployed with {{ .NETWORK_SIZE }} nodes" @@ -201,21 +220,68 @@ tasks: done - cmd: echo "Exporting database..." - cmd: | - kubectl exec {{ .POSTGRES_CONTAINER_NAME }} -n {{ .POSTGRES_DATABASE_NAMESPACE }} -- \ - env PGPASSWORD={{ .POSTGRES_PASSWORD }} pg_dump -U {{ .POSTGRES_USERNAME }} \ - --clean --if-exists \ - {{ .POSTGRES_MIRROR_NODE_DATABASE_NAME }} > {{ .STATE_SAVE_DIR }}/database-dump.sql + if [ "{{ .ENABLE_MIRROR_WORKFLOW }}" = "true" ]; then + kubectl exec {{ .POSTGRES_CONTAINER_NAME }} -n {{ .POSTGRES_DATABASE_NAMESPACE }} -- \ + env PGPASSWORD={{ .POSTGRES_PASSWORD }} pg_dump -U {{ .POSTGRES_USERNAME }} \ + --clean --if-exists \ + {{ .POSTGRES_MIRROR_NODE_DATABASE_NAME }} > {{ .STATE_SAVE_DIR }}/database-dump.sql + else + echo "Skipping database export; mirror workflow disabled." + fi + - cmd: echo "Saving source network JSON..." + - cmd: | + kubectl exec network-node1-0 -n {{ .NAMESPACE }} -c root-container -- bash -c ' + if [ -f /opt/hgcapp/services-hedera/HapiApp2.0/output/network.json ]; then + cat /opt/hgcapp/services-hedera/HapiApp2.0/output/network.json + elif [ -f /opt/hgcapp/services-hedera/HapiApp2.0/data/config/genesis-network.json ]; then + cat /opt/hgcapp/services-hedera/HapiApp2.0/data/config/genesis-network.json + elif [ -f /opt/hgcapp/services-hedera/HapiApp2.0/data/config/.archive/genesis-network.json ]; then + cat /opt/hgcapp/services-hedera/HapiApp2.0/data/config/.archive/genesis-network.json + else + echo "No usable network JSON found in node pod" >&2 + exit 1 + fi + ' > {{ .ORIGINAL_NETWORK_JSON_FILE }} + - cmd: echo "✅ Source network JSON exported to {{ .ORIGINAL_NETWORK_JSON_FILE }}" - cmd: echo "Saving mirror passwords secret..." - cmd: | - kubectl get secret mirror-passwords -n {{ .NAMESPACE }} -o json | \ - jq 'del( - .metadata.annotations."kubectl.kubernetes.io/last-applied-configuration", - .metadata.creationTimestamp, - .metadata.managedFields, - .metadata.resourceVersion, - .metadata.uid - )' > {{ .MIRROR_PASSWORDS_SECRET_FILE }} - - cmd: echo "✅ Mirror passwords secret exported to {{ .MIRROR_PASSWORDS_SECRET_FILE }}" + if [ "{{ .ENABLE_MIRROR_WORKFLOW }}" = "true" ]; then + kubectl get secret mirror-passwords -n {{ .NAMESPACE }} -o json | \ + jq 'del( + .metadata.annotations."kubectl.kubernetes.io/last-applied-configuration", + .metadata.creationTimestamp, + .metadata.managedFields, + .metadata.resourceVersion, + .metadata.uid + )' > {{ .MIRROR_PASSWORDS_SECRET_FILE }} + echo "✅ Mirror passwords secret exported to {{ .MIRROR_PASSWORDS_SECRET_FILE }}" + else + echo "Skipping mirror secret export; mirror workflow disabled." + fi + - cmd: echo "Saving consensus node key material from Kubernetes secrets..." + - cmd: mkdir -p {{ .SAVED_KEYS_DIR }} + - cmd: | + kubectl get secret network-node-hapi-app-secrets -n {{ .NAMESPACE }} -o json | \ + jq -r '.data | to_entries[] | @base64' | \ + while IFS= read -r entry; do + key_file_name="$(printf '%s' "${entry}" | base64 --decode | jq -r '.key')" + key_file_value="$(printf '%s' "${entry}" | base64 --decode | jq -r '.value')" + printf '%s' "${key_file_value}" | base64 --decode > "{{ .SAVED_KEYS_DIR }}/${key_file_name}" + chmod 600 "{{ .SAVED_KEYS_DIR }}/${key_file_name}" + echo "✅ Saved TLS key file ${key_file_name}" + done + - cmd: | + for node in $(echo {{ .NODE_ALIASES }} | tr ',' ' '); do + kubectl get secret network-${node}-keys-secrets -n {{ .NAMESPACE }} -o json | \ + jq -r '.data | to_entries[] | @base64' | \ + while IFS= read -r entry; do + key_file_name="$(printf '%s' "${entry}" | base64 --decode | jq -r '.key')" + key_file_value="$(printf '%s' "${entry}" | base64 --decode | jq -r '.value')" + printf '%s' "${key_file_value}" | base64 --decode > "{{ .SAVED_KEYS_DIR }}/${key_file_name}" + chmod 600 "{{ .SAVED_KEYS_DIR }}/${key_file_name}" + echo "✅ Saved gossip key file ${key_file_name}" + done + done - cmd: echo "✅ Database exported to {{ .STATE_SAVE_DIR }}/database-dump.sql" - cmd: echo "✅ Network state and database saved to {{ .STATE_SAVE_DIR }}" - cmd: ls -lh {{ .STATE_SAVE_DIR }} @@ -223,23 +289,42 @@ tasks: # ==================== State Restore Tasks ==================== restore: - desc: Recreate network and restore state with external database + desc: Recreate a fresh cluster, restore state, and verify the saved network can boot again cmds: - task: destroy-network - - task: destroy-database + - cmd: | + if [ "{{ .ENABLE_MIRROR_WORKFLOW }}" = "true" ]; then + task destroy-database + else + echo "Skipping database cleanup; mirror workflow disabled." + fi + - task: destroy-cluster + - task: create-cluster + - task: preload-network-images - task: init-solo - - task: deploy-external-database + - task: restore-consensus-keys + - task: deploy-block-node - task: deploy-network-with-state - - task: restore-mirror-passwords-secret - - task: restore-database - - task: deploy-mirror-external + - cmd: | + if [ "{{ .ENABLE_MIRROR_WORKFLOW }}" = "true" ]; then + task deploy-external-database + task restore-mirror-passwords-secret + task restore-database + task deploy-mirror-external + else + echo "Skipping mirror/database restore; consensus override-network test only." + fi - task: verify-state - cmd: echo "✅ Network and database restored!" deploy-network-with-state: - desc: Deploy a fresh network and start nodes from saved state + desc: Deploy a fresh network and start nodes from saved state with override-network endpoint remapping cmds: - cmd: echo "Deploying network with saved state..." + - cmd: | + echo "This workflow validates saved-state startup on a freshly recreated cluster." + echo "It rewrites the saved roster with override-network.json so the restarted" + echo "nodes adopt the fresh cluster's current gossip service IPs." - cmd: $SOLO_COMMAND consensus network deploy --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} - cmd: $SOLO_COMMAND consensus node setup --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} - cmd: | @@ -247,6 +332,8 @@ tasks: for node in $(echo {{ .NODE_ALIASES }} | tr ',' ' '); do kubectl wait --for=create pod/network-${node}-0 -n {{ .NAMESPACE }} --timeout=300s done + - task: generate-override-network + - task: install-override-network - cmd: | # Build the directory layout expected by Solo for per-node state restore: # /states///network--0-state.zip @@ -266,6 +353,23 @@ tasks: fi cp "${SRC_STATE_FILE_PATH}" "${DEST_STATE_FILE_PATH}" + TMP_STATE_DIR="$(mktemp -d)" + unzip -q "${DEST_STATE_FILE_PATH}" -d "${TMP_STATE_DIR}" + + # Keep the downloaded signed states intact and let Solo choose the + # restore round. Only strip the top-level replay/cache directories; + # the round-local PCES files are needed for CN v0.74 to progress + # past CHECKING after it restores the selected signed state. + rm -rf "${TMP_STATE_DIR}/preconsensus-events" + rm -rf "${TMP_STATE_DIR}/saved" + rm -rf "${TMP_STATE_DIR}/swirlds-tmp" + + rm -f "${DEST_STATE_FILE_PATH}" + ORIGINAL_WORKDIR="${PWD}" + cd "${TMP_STATE_DIR}" + zip -qr "${DEST_STATE_FILE_PATH}" . + cd "${ORIGINAL_WORKDIR}" + rm -rf "${TMP_STATE_DIR}" echo "Prepared state for ${node}: ${DEST_STATE_FILE_PATH}" done @@ -277,6 +381,56 @@ tasks: --state-file "${RESTORE_INPUT_DIR}" - cmd: echo "✅ Nodes started with restored state" + restore-consensus-keys: + desc: Restore saved consensus key material into Solo cache + cmds: + - cmd: | + if [ ! -d {{ .SAVED_KEYS_DIR }} ]; then + echo "⚠️ Saved key directory not found: {{ .SAVED_KEYS_DIR }}" + exit 1 + fi + - cmd: mkdir -p "{{ .SOLO_CACHE_KEYS_DIR }}" + - cmd: find "{{ .SOLO_CACHE_KEYS_DIR }}" -mindepth 1 -maxdepth 1 -exec rm -rf {} + + - cmd: cp "{{ .SAVED_KEYS_DIR }}"/* "{{ .SOLO_CACHE_KEYS_DIR }}/" + - cmd: chmod 600 "{{ .SOLO_CACHE_KEYS_DIR }}"/* + - cmd: echo "✅ Restored consensus key material to {{ .SOLO_CACHE_KEYS_DIR }}" + + generate-override-network: + desc: Generate override-network.json using the fresh cluster service IPs + cmds: + - cmd: | + if [ ! -f {{ .ORIGINAL_NETWORK_JSON_FILE }} ]; then + echo "⚠️ Saved source network JSON not found: {{ .ORIGINAL_NETWORK_JSON_FILE }}" + exit 1 + fi + - cmd: | + kubectl get service -n {{ .NAMESPACE }} -o json > {{ .CURRENT_SERVICE_ENDPOINTS_FILE }} + - cmd: | + node {{ .OVERRIDE_NETWORK_GENERATOR_SCRIPT }} \ + --source-network {{ .ORIGINAL_NETWORK_JSON_FILE }} \ + --services-json {{ .CURRENT_SERVICE_ENDPOINTS_FILE }} \ + --node-aliases {{ .NODE_ALIASES }} \ + --output {{ .GENERATED_OVERRIDE_NETWORK_JSON_FILE }} + - cmd: echo "✅ Generated override-network.json at {{ .GENERATED_OVERRIDE_NETWORK_JSON_FILE }}" + + install-override-network: + desc: Copy override-network.json into each consensus node pod + cmds: + - cmd: | + if [ ! -f {{ .GENERATED_OVERRIDE_NETWORK_JSON_FILE }} ]; then + echo "⚠️ Generated override-network.json not found: {{ .GENERATED_OVERRIDE_NETWORK_JSON_FILE }}" + exit 1 + fi + - cmd: | + for node in $(echo {{ .NODE_ALIASES }} | tr ',' ' '); do + kubectl cp {{ .GENERATED_OVERRIDE_NETWORK_JSON_FILE }} \ + {{ .NAMESPACE }}/network-${node}-0:/opt/hgcapp/services-hedera/HapiApp2.0/data/config/override-network.json \ + -c root-container + kubectl exec network-${node}-0 -n {{ .NAMESPACE }} -c root-container -- \ + ls -l /opt/hgcapp/services-hedera/HapiApp2.0/data/config/override-network.json + done + - cmd: echo "✅ override-network.json copied to each consensus node pod" + restore-database: desc: Restore database from dump cmds: @@ -360,10 +514,11 @@ tasks: - cmd: echo "✅ Network frozen" destroy-network: - desc: Destroy mirror node and consensus network while keeping cluster configuration + desc: Destroy mirror node, block node, and consensus network while keeping cluster configuration cmds: - - cmd: echo "Destroying mirror node and consensus network..." + - cmd: echo "Destroying mirror node, block node, and consensus network..." - cmd: $SOLO_COMMAND mirror node destroy --deployment {{ .DEPLOYMENT }} --force || true + - cmd: $SOLO_COMMAND block node destroy --deployment {{ .DEPLOYMENT }} --force || true - cmd: $SOLO_COMMAND consensus node stop --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} || true - cmd: $SOLO_COMMAND consensus network destroy --deployment {{ .DEPLOYMENT }} --force -q || true - cmd: echo "✅ Network resources destroyed" @@ -383,12 +538,22 @@ tasks: cmds: - task: destroy-network - task: destroy-database - - cmd: kind delete cluster --name {{ .CLUSTER_NAME }} - - cmd: echo "✅ Cluster destroyed" + - task: destroy-cluster - task: clean-state + destroy-cluster: + desc: Delete the Kind cluster + cmds: + - cmd: kind delete cluster --name {{ .CLUSTER_NAME }} || true + - cmd: echo "✅ Cluster destroyed" + clean-state: desc: Remove saved state files cmds: - cmd: rm -rf {{ .STATE_SAVE_DIR }} - cmd: echo "✅ Saved state files removed" + deploy-block-node: + desc: Deploy block node so CN v0.74 does not require MinIO-backed stream storage + cmds: + - cmd: $SOLO_COMMAND block node add --deployment {{ .DEPLOYMENT }} --quiet-mode --dev + - cmd: echo "✅ Block node deployed" diff --git a/examples/state-save-and-restore/scripts/generate-override-network.mjs b/examples/state-save-and-restore/scripts/generate-override-network.mjs new file mode 100644 index 0000000000..6ffc9ca9ea --- /dev/null +++ b/examples/state-save-and-restore/scripts/generate-override-network.mjs @@ -0,0 +1,148 @@ +#!/usr/bin/env node +// SPDX-License-Identifier: Apache-2.0 + +import fs from 'node:fs'; + +class OverrideNetworkGenerator { + static main() { + const argumentsMap = this.parseArguments(process.argv.slice(2)); + const sourceNetworkPath = this.requireArgument(argumentsMap, 'source-network'); + const servicesJsonPath = this.requireArgument(argumentsMap, 'services-json'); + const nodeAliasesText = this.requireArgument(argumentsMap, 'node-aliases'); + const outputPath = this.requireArgument(argumentsMap, 'output'); + const sourceNetwork = JSON.parse(fs.readFileSync(sourceNetworkPath, 'utf8')); + const servicesDocument = JSON.parse(fs.readFileSync(servicesJsonPath, 'utf8')); + const nodeAliases = nodeAliasesText + .split(',') + .map(nodeAlias => nodeAlias.trim()) + .filter(nodeAlias => nodeAlias.length > 0); + + if (!Array.isArray(sourceNetwork.nodeMetadata)) { + throw new Error('source network JSON is missing nodeMetadata'); + } + + if (!Array.isArray(servicesDocument.items)) { + throw new Error('services JSON is missing items'); + } + + if (sourceNetwork.nodeMetadata.length < nodeAliases.length) { + throw new Error( + `source network only has ${String(sourceNetwork.nodeMetadata.length)} nodeMetadata entries for ${String(nodeAliases.length)} aliases`, + ); + } + + const rewrittenNetwork = structuredClone(sourceNetwork); + let changedEndpointCount = 0; + + for (const [nodeIndex, nodeAlias] of nodeAliases.entries()) { + const serviceName = `network-${nodeAlias}-svc`; + const service = servicesDocument.items.find(candidateService => candidateService?.metadata?.name === serviceName); + + if (!service) { + throw new Error(`service not found in services JSON: ${serviceName}`); + } + + const clusterIpAddress = service?.spec?.clusterIP; + if (typeof clusterIpAddress !== 'string' || clusterIpAddress.length === 0 || clusterIpAddress === 'None') { + throw new Error(`service ${serviceName} does not have a usable clusterIP`); + } + + const encodedIpAddress = this.encodeIpv4Address(clusterIpAddress); + const nodeMetadata = rewrittenNetwork.nodeMetadata[nodeIndex]; + changedEndpointCount += this.rewriteNodeServiceEndpoints(nodeMetadata, encodedIpAddress, clusterIpAddress, serviceName); + } + + if (changedEndpointCount === 0) { + throw new Error('override-network.json was not changed; this would not test endpoint remapping'); + } + + fs.writeFileSync(outputPath, `${JSON.stringify(rewrittenNetwork, null, 2)}\n`); + console.log(`Wrote ${outputPath} with ${String(changedEndpointCount)} rewritten endpoint entries`); + } + + static parseArguments(argumentList) { + const argumentsMap = new Map(); + + for (let argumentIndex = 0; argumentIndex < argumentList.length; argumentIndex += 1) { + const argument = argumentList[argumentIndex]; + if (!argument.startsWith('--')) { + throw new Error(`Unexpected argument: ${argument}`); + } + + const argumentName = argument.slice(2); + const argumentValue = argumentList[argumentIndex + 1]; + if (!argumentValue || argumentValue.startsWith('--')) { + throw new Error(`Missing value for --${argumentName}`); + } + + argumentsMap.set(argumentName, argumentValue); + argumentIndex += 1; + } + + return argumentsMap; + } + + static requireArgument(argumentsMap, argumentName) { + const argumentValue = argumentsMap.get(argumentName); + if (!argumentValue) { + throw new Error(`Missing required argument --${argumentName}`); + } + return argumentValue; + } + + static encodeIpv4Address(ipAddressText) { + const octets = ipAddressText.split('.').map(octetText => Number(octetText)); + + if (octets.length !== 4 || octets.some(octet => !Number.isInteger(octet) || octet < 0 || octet > 255)) { + throw new Error(`Only IPv4 addresses are supported, got: ${ipAddressText}`); + } + + return Buffer.from(octets).toString('base64'); + } + + static rewriteNodeServiceEndpoints(nodeMetadata, encodedIpAddress, clusterIpAddress, serviceName) { + if (!nodeMetadata || typeof nodeMetadata !== 'object') { + throw new Error('nodeMetadata entry is missing or invalid'); + } + + return this.rewriteServiceEndpointList(nodeMetadata?.node, encodedIpAddress, clusterIpAddress, serviceName); + } + + static rewriteServiceEndpointList(parentObject, encodedIpAddress, clusterIpAddress, serviceName) { + if (!parentObject || typeof parentObject !== 'object') { + throw new Error(`node metadata is missing for ${serviceName}`); + } + + const serviceEndpoints = parentObject.serviceEndpoint; + if (!Array.isArray(serviceEndpoints) || serviceEndpoints.length === 0) { + throw new Error(`node is missing serviceEndpoint entries for ${serviceName}`); + } + + let changedEndpointCount = 0; + + parentObject.serviceEndpoint = serviceEndpoints.map(serviceEndpoint => { + const rewrittenEndpoint = {...serviceEndpoint}; + const existingIpAddress = rewrittenEndpoint.ipAddressV4; + + rewrittenEndpoint.ipAddressV4 = encodedIpAddress; + + const endpointChanged = existingIpAddress !== encodedIpAddress; + if (endpointChanged) { + changedEndpointCount += 1; + } + + return rewrittenEndpoint; + }); + + console.log(`Updated service endpoints for ${serviceName} -> ${clusterIpAddress}`); + return changedEndpointCount; + } +} + +try { + OverrideNetworkGenerator.main(); +} catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(message); + process.exitCode = 1; +} From 16cead5c4a65de0c1f53e48a55e679ebfd502e79 Mon Sep 17 00:00:00 2001 From: Jeffrey Tang Date: Thu, 30 Jul 2026 11:42:51 -0500 Subject: [PATCH 07/20] save Signed-off-by: Jeffrey Tang --- examples/state-save-and-restore/Taskfile.yml | 68 +++++++++++++++++--- 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/examples/state-save-and-restore/Taskfile.yml b/examples/state-save-and-restore/Taskfile.yml index 62b06d32e4..8460e3f36b 100644 --- a/examples/state-save-and-restore/Taskfile.yml +++ b/examples/state-save-and-restore/Taskfile.yml @@ -13,6 +13,7 @@ vars: # Network Configuration NETWORK_SIZE: "2" NODE_ALIASES: "node1,node2" + CONSENSUS_NODE_VERSION: "v0.74.3" DEPLOYMENT: "state-restore-deployment" NAMESPACE: "state-restore-namespace" ENABLE_MIRROR_WORKFLOW: '{{ default "false" (env "ENABLE_MIRROR_WORKFLOW") }}' @@ -129,9 +130,14 @@ tasks: - cmd: $SOLO_COMMAND keys consensus generate --gossip-keys --tls-keys --node-aliases {{ .NODE_ALIASES }} --deployment {{ .DEPLOYMENT }} - cmd: | $SOLO_COMMAND consensus network deploy --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} \ + --consensus-node-version {{ .CONSENSUS_NODE_VERSION }} \ --enable-monitoring-support false --quiet-mode --dev - - cmd: $SOLO_COMMAND consensus node setup --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} - - cmd: $SOLO_COMMAND consensus node start --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} + - cmd: | + $SOLO_COMMAND consensus node setup --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} \ + --consensus-node-version {{ .CONSENSUS_NODE_VERSION }} + - cmd: | + $SOLO_COMMAND consensus node start --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} \ + --consensus-node-version {{ .CONSENSUS_NODE_VERSION }} - cmd: echo "✅ Consensus network deployed with {{ .NETWORK_SIZE }} nodes" deploy-mirror-external: @@ -325,8 +331,12 @@ tasks: echo "This workflow validates saved-state startup on a freshly recreated cluster." echo "It rewrites the saved roster with override-network.json so the restarted" echo "nodes adopt the fresh cluster's current gossip service IPs." - - cmd: $SOLO_COMMAND consensus network deploy --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} - - cmd: $SOLO_COMMAND consensus node setup --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} + - cmd: | + $SOLO_COMMAND consensus network deploy --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} \ + --consensus-node-version {{ .CONSENSUS_NODE_VERSION }} + - cmd: | + $SOLO_COMMAND consensus node setup --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} \ + --consensus-node-version {{ .CONSENSUS_NODE_VERSION }} - cmd: | echo "Waiting for recreated consensus pods to exist before restore..." for node in $(echo {{ .NODE_ALIASES }} | tr ',' ' '); do @@ -356,10 +366,44 @@ tasks: TMP_STATE_DIR="$(mktemp -d)" unzip -q "${DEST_STATE_FILE_PATH}" -d "${TMP_STATE_DIR}" - # Keep the downloaded signed states intact and let Solo choose the - # restore round. Only strip the top-level replay/cache directories; - # the round-local PCES files are needed for CN v0.74 to progress - # past CHECKING after it restores the selected signed state. + # Build a deterministic single-round archive for restore. Recent + # saved bundles can contain a later freeze round plus newer PCES + # spans; if Solo picks the latest signed non-freeze round from a + # multi-round archive, replay can trip an ISS. Keeping the earliest + # fully signed non-freeze round preserves the matching seq0 PCES + # while avoiding later replay spans that belong to newer rounds. + NODE_STATE_ROOT="$(find "${TMP_STATE_DIR}/com.hedera.services.ServicesMain" -mindepth 2 -maxdepth 2 -type d | head -n 1)" + if [ -z "${NODE_STATE_ROOT}" ]; then + echo "⚠️ Could not locate saved state rounds in ${DEST_STATE_FILE_PATH}" + exit 1 + fi + + SELECTED_ROUND="" + for round_dir in $(find "${NODE_STATE_ROOT}" -mindepth 1 -maxdepth 1 -type d | sort -n); do + metadata_file="${round_dir}/stateMetadata.txt" + if [ ! -f "${metadata_file}" ]; then + continue + fi + + freeze_state="$(awk -F: '/^FREEZE_STATE:/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2}' "${metadata_file}")" + signing_weight="$(awk -F: '/^SIGNING_WEIGHT_SUM:/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2}' "${metadata_file}")" + total_weight="$(awk -F: '/^TOTAL_WEIGHT:/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2}' "${metadata_file}")" + + if [ "${freeze_state}" = "false" ] && [ -n "${signing_weight}" ] && [ "${signing_weight}" = "${total_weight}" ]; then + SELECTED_ROUND="$(basename "${round_dir}")" + break + fi + done + + if [ -z "${SELECTED_ROUND}" ]; then + echo "⚠️ Could not find a fully signed non-freeze round in ${DEST_STATE_FILE_PATH}" + exit 1 + fi + + find "${NODE_STATE_ROOT}" -mindepth 1 -maxdepth 1 -type d ! -name "${SELECTED_ROUND}" -exec rm -rf {} + + + # Remove top-level replay/cache directories and let Solo rebuild + # them from the selected round. rm -rf "${TMP_STATE_DIR}/preconsensus-events" rm -rf "${TMP_STATE_DIR}/saved" rm -rf "${TMP_STATE_DIR}/swirlds-tmp" @@ -370,7 +414,7 @@ tasks: zip -qr "${DEST_STATE_FILE_PATH}" . cd "${ORIGINAL_WORKDIR}" rm -rf "${TMP_STATE_DIR}" - echo "Prepared state for ${node}: ${DEST_STATE_FILE_PATH}" + echo "Prepared state for ${node}: ${DEST_STATE_FILE_PATH} (round ${SELECTED_ROUND})" done echo "Prepared restore input:" @@ -378,6 +422,7 @@ tasks: # Start all nodes together so they can transition to ACTIVE as a group. $SOLO_COMMAND consensus node start --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} \ + --consensus-node-version {{ .CONSENSUS_NODE_VERSION }} \ --state-file "${RESTORE_INPUT_DIR}" - cmd: echo "✅ Nodes started with restored state" @@ -555,5 +600,8 @@ tasks: deploy-block-node: desc: Deploy block node so CN v0.74 does not require MinIO-backed stream storage cmds: - - cmd: $SOLO_COMMAND block node add --deployment {{ .DEPLOYMENT }} --quiet-mode --dev + - cmd: | + $SOLO_COMMAND block node add --deployment {{ .DEPLOYMENT }} \ + --consensus-node-version {{ .CONSENSUS_NODE_VERSION }} \ + --quiet-mode --dev - cmd: echo "✅ Block node deployed" From e29cf3ced18eef7d8aa37a5c1979f3eb94323416 Mon Sep 17 00:00:00 2001 From: Jeffrey Tang Date: Thu, 30 Jul 2026 16:32:39 -0500 Subject: [PATCH 08/20] save Signed-off-by: Jeffrey Tang --- examples/state-save-and-restore/Taskfile.yml | 163 ++++++++++-------- .../scripts/generate-override-network.mjs | 45 +++-- resources/cleanup-state-rounds.sh | 64 +------ resources/wait-for-stable-saved-state.sh | 85 +++++++++ src/commands/node/handlers.ts | 46 +++-- src/commands/node/tasks.ts | 48 +++++- src/core/network-nodes.ts | 113 +++++++++++- 7 files changed, 407 insertions(+), 157 deletions(-) create mode 100644 resources/wait-for-stable-saved-state.sh diff --git a/examples/state-save-and-restore/Taskfile.yml b/examples/state-save-and-restore/Taskfile.yml index 8460e3f36b..f5b11ab082 100644 --- a/examples/state-save-and-restore/Taskfile.yml +++ b/examples/state-save-and-restore/Taskfile.yml @@ -11,41 +11,40 @@ env: vars: # Network Configuration - NETWORK_SIZE: "2" - NODE_ALIASES: "node1,node2" - CONSENSUS_NODE_VERSION: "v0.74.3" - DEPLOYMENT: "state-restore-deployment" - NAMESPACE: "state-restore-namespace" + NETWORK_SIZE: '2' + NODE_ALIASES: 'node1,node2' + DEPLOYMENT: 'state-restore-deployment' + NAMESPACE: 'state-restore-namespace' ENABLE_MIRROR_WORKFLOW: '{{ default "false" (env "ENABLE_MIRROR_WORKFLOW") }}' - SOLO_USER_DIR: "{{ default (printf \"%s/.solo\" (env \"HOME\")) }}" + SOLO_USER_DIR: '{{ default (printf "%s/.solo" (env "HOME")) }}' # Cluster Configuration - CLUSTER_NAME: "state-restore-cluster" - CONTEXT: "kind-state-restore-cluster" - CLUSTER_REF: "kind-state-restore-cluster" + CLUSTER_NAME: 'state-restore-cluster' + CONTEXT: 'kind-state-restore-cluster' + CLUSTER_REF: 'kind-state-restore-cluster' # State Save Configuration - STATE_SAVE_DIR: "{{ .USER_WORKING_DIR }}/saved-states" - MIRROR_PASSWORDS_SECRET_FILE: "{{ .STATE_SAVE_DIR }}/mirror-passwords-secret.json" - ORIGINAL_NETWORK_JSON_FILE: "{{ .STATE_SAVE_DIR }}/original-network.json" - GENERATED_OVERRIDE_NETWORK_JSON_FILE: "{{ .STATE_SAVE_DIR }}/override-network.json" - CURRENT_SERVICE_ENDPOINTS_FILE: "{{ .STATE_SAVE_DIR }}/current-service-endpoints.json" - OVERRIDE_NETWORK_GENERATOR_SCRIPT: "{{ .TASKFILE_DIR }}/scripts/generate-override-network.mjs" - PREPULL_IMAGES_SCRIPT: "{{ .TASKFILE_DIR }}/scripts/prepull-images.sh" - SAVED_KEYS_DIR: "{{ .STATE_SAVE_DIR }}/keys" - SOLO_CACHE_KEYS_DIR: "{{ .SOLO_USER_DIR }}/cache/keys" + STATE_SAVE_DIR: '{{ .USER_WORKING_DIR }}/saved-states' + MIRROR_PASSWORDS_SECRET_FILE: '{{ .STATE_SAVE_DIR }}/mirror-passwords-secret.json' + ORIGINAL_NETWORK_JSON_FILE: '{{ .STATE_SAVE_DIR }}/original-network.json' + GENERATED_OVERRIDE_NETWORK_JSON_FILE: '{{ .STATE_SAVE_DIR }}/override-network.json' + CURRENT_SERVICE_ENDPOINTS_FILE: '{{ .STATE_SAVE_DIR }}/current-service-endpoints.json' + OVERRIDE_NETWORK_GENERATOR_SCRIPT: '{{ .TASKFILE_DIR }}/scripts/generate-override-network.mjs' + PREPULL_IMAGES_SCRIPT: '{{ .TASKFILE_DIR }}/scripts/prepull-images.sh' + SAVED_KEYS_DIR: '{{ .STATE_SAVE_DIR }}/keys' + SOLO_CACHE_KEYS_DIR: '{{ .SOLO_USER_DIR }}/cache/keys' # External Database Configuration (Optional) - POSTGRES_USERNAME: "postgres" - POSTGRES_PASSWORD: "XXXXXXXX" - POSTGRES_READONLY_USERNAME: "readonlyuser" - POSTGRES_READONLY_PASSWORD: "XXXXXXXX" - POSTGRES_MIRROR_NODE_DATABASE_NAME: "mirror_node" - POSTGRES_NAME: "my-postgresql" - POSTGRES_DATABASE_NAMESPACE: "database" - POSTGRES_CONTAINER_NAME: "{{ .POSTGRES_NAME }}-0" - POSTGRES_HOST_FQDN: "{{ .POSTGRES_NAME }}.database.svc.cluster.local" + POSTGRES_USERNAME: 'postgres' + POSTGRES_PASSWORD: 'XXXXXXXX' + POSTGRES_READONLY_USERNAME: 'readonlyuser' + POSTGRES_READONLY_PASSWORD: 'XXXXXXXX' + POSTGRES_MIRROR_NODE_DATABASE_NAME: 'mirror_node' + POSTGRES_NAME: 'my-postgresql' + POSTGRES_DATABASE_NAMESPACE: 'database' + POSTGRES_CONTAINER_NAME: '{{ .POSTGRES_NAME }}-0' + POSTGRES_HOST_FQDN: '{{ .POSTGRES_NAME }}.database.svc.cluster.local' tasks: # ==================== Main Tasks ==================== @@ -107,7 +106,7 @@ tasks: else kind create cluster -n {{ .CLUSTER_NAME }} fi - - cmd: sleep 10 # Wait for control plane + - cmd: sleep 10 # Wait for control plane - cmd: kubectl config use-context {{ .CONTEXT }} preload-network-images: @@ -130,14 +129,9 @@ tasks: - cmd: $SOLO_COMMAND keys consensus generate --gossip-keys --tls-keys --node-aliases {{ .NODE_ALIASES }} --deployment {{ .DEPLOYMENT }} - cmd: | $SOLO_COMMAND consensus network deploy --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} \ - --consensus-node-version {{ .CONSENSUS_NODE_VERSION }} \ --enable-monitoring-support false --quiet-mode --dev - - cmd: | - $SOLO_COMMAND consensus node setup --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} \ - --consensus-node-version {{ .CONSENSUS_NODE_VERSION }} - - cmd: | - $SOLO_COMMAND consensus node start --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} \ - --consensus-node-version {{ .CONSENSUS_NODE_VERSION }} + - cmd: $SOLO_COMMAND consensus node setup --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} + - cmd: $SOLO_COMMAND consensus node start --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} - cmd: echo "✅ Consensus network deployed with {{ .NETWORK_SIZE }} nodes" deploy-mirror-external: @@ -189,7 +183,6 @@ tasks: -n {{ .POSTGRES_DATABASE_NAMESPACE }} \ -- /bin/bash /tmp/init.sh "{{ .POSTGRES_USERNAME }}" "{{ .POSTGRES_READONLY_USERNAME }}" "{{ .POSTGRES_READONLY_PASSWORD }}" - # ==================== Transaction Generation ==================== generate-transactions: @@ -332,11 +325,8 @@ tasks: echo "It rewrites the saved roster with override-network.json so the restarted" echo "nodes adopt the fresh cluster's current gossip service IPs." - cmd: | - $SOLO_COMMAND consensus network deploy --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} \ - --consensus-node-version {{ .CONSENSUS_NODE_VERSION }} - - cmd: | - $SOLO_COMMAND consensus node setup --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} \ - --consensus-node-version {{ .CONSENSUS_NODE_VERSION }} + $SOLO_COMMAND consensus network deploy --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} + - cmd: $SOLO_COMMAND consensus node setup --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} - cmd: | echo "Waiting for recreated consensus pods to exist before restore..." for node in $(echo {{ .NODE_ALIASES }} | tr ',' ' '); do @@ -349,6 +339,7 @@ tasks: # /states///network--0-state.zip RESTORE_INPUT_DIR="{{ .STATE_SAVE_DIR }}/restore-input" RESTORE_STATES_DIR="${RESTORE_INPUT_DIR}/states/{{ .CLUSTER_REF }}/{{ .NAMESPACE }}" + RESTORE_ROUND="" rm -rf "${RESTORE_STATES_DIR}" mkdir -p "${RESTORE_STATES_DIR}" @@ -366,12 +357,10 @@ tasks: TMP_STATE_DIR="$(mktemp -d)" unzip -q "${DEST_STATE_FILE_PATH}" -d "${TMP_STATE_DIR}" - # Build a deterministic single-round archive for restore. Recent - # saved bundles can contain a later freeze round plus newer PCES - # spans; if Solo picks the latest signed non-freeze round from a - # multi-round archive, replay can trip an ISS. Keeping the earliest - # fully signed non-freeze round preserves the matching seq0 PCES - # while avoiding later replay spans that belong to newer rounds. + # Build a deterministic single-round archive for restore. The + # workflow now captures state from a frozen network, so prefer the + # fully signed freeze round as the exact recovery boundary. The + # download step requires this boundary for a frozen deployment. NODE_STATE_ROOT="$(find "${TMP_STATE_DIR}/com.hedera.services.ServicesMain" -mindepth 2 -maxdepth 2 -type d | head -n 1)" if [ -z "${NODE_STATE_ROOT}" ]; then echo "⚠️ Could not locate saved state rounds in ${DEST_STATE_FILE_PATH}" @@ -379,7 +368,17 @@ tasks: fi SELECTED_ROUND="" - for round_dir in $(find "${NODE_STATE_ROOT}" -mindepth 1 -maxdepth 1 -type d | sort -n); do + FALLBACK_ROUND="" + # Sort on the numeric basename rather than the full temporary path; + # otherwise `sort -n` can choose a stale round unpredictably. + for round_dir in $( + find "${NODE_STATE_ROOT}" -mindepth 1 -maxdepth 1 -type d -print | + while IFS= read -r candidate_dir; do + printf '%s\t%s\n' "$(basename "${candidate_dir}")" "${candidate_dir}" + done | + sort -n -k1,1 | + cut -f2- + ); do metadata_file="${round_dir}/stateMetadata.txt" if [ ! -f "${metadata_file}" ]; then continue @@ -389,22 +388,38 @@ tasks: signing_weight="$(awk -F: '/^SIGNING_WEIGHT_SUM:/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2}' "${metadata_file}")" total_weight="$(awk -F: '/^TOTAL_WEIGHT:/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2}' "${metadata_file}")" - if [ "${freeze_state}" = "false" ] && [ -n "${signing_weight}" ] && [ "${signing_weight}" = "${total_weight}" ]; then + if [ -n "${signing_weight}" ] && [ "${signing_weight}" = "${total_weight}" ] && [ "${freeze_state}" = "true" ]; then SELECTED_ROUND="$(basename "${round_dir}")" break fi + + if [ -n "${signing_weight}" ] && [ "${signing_weight}" = "${total_weight}" ] && [ "${freeze_state}" = "false" ]; then + FALLBACK_ROUND="$(basename "${round_dir}")" + fi done if [ -z "${SELECTED_ROUND}" ]; then - echo "⚠️ Could not find a fully signed non-freeze round in ${DEST_STATE_FILE_PATH}" + SELECTED_ROUND="${FALLBACK_ROUND}" + fi + + if [ -z "${SELECTED_ROUND}" ]; then + echo "⚠️ Could not find a fully signed round in ${DEST_STATE_FILE_PATH}" + exit 1 + fi + + if [ -n "${RESTORE_ROUND}" ] && [ "${RESTORE_ROUND}" != "${SELECTED_ROUND}" ]; then + echo "⚠️ Nodes selected different restore rounds: ${RESTORE_ROUND} and ${SELECTED_ROUND}" exit 1 fi + RESTORE_ROUND="${SELECTED_ROUND}" find "${NODE_STATE_ROOT}" -mindepth 1 -maxdepth 1 -type d ! -name "${SELECTED_ROUND}" -exec rm -rf {} + - # Remove top-level replay/cache directories and let Solo rebuild - # them from the selected round. - rm -rf "${TMP_STATE_DIR}/preconsensus-events" + # Preserve the captured PCES directory. It is part of the exact + # state/round boundary and is required for the restored platform to + # advance from OBSERVING/CHECKING. The cleanup helper preserves this + # captured replay input while removing superseded state directories. + # Remove unrelated replay/cache directories before restore. rm -rf "${TMP_STATE_DIR}/saved" rm -rf "${TMP_STATE_DIR}/swirlds-tmp" @@ -417,12 +432,24 @@ tasks: echo "Prepared state for ${node}: ${DEST_STATE_FILE_PATH} (round ${SELECTED_ROUND})" done + # Apply the endpoint transplant after the snapshot's signed round. + # If the override is staged at the snapshot round, Hedera mutates the + # roster before validating that round's state signature and reports an + # ISS. Placing it at the next round lets startup verify the snapshot + # first, then apply the fresh cluster endpoints during replay. + OVERRIDE_ROUND=$((RESTORE_ROUND + 1)) + for node in $(echo {{ .NODE_ALIASES }} | tr ',' ' '); do + kubectl exec network-${node}-0 -n {{ .NAMESPACE }} -c root-container -- \ + bash -c "mkdir -p /opt/hgcapp/services-hedera/HapiApp2.0/data/config/${OVERRIDE_ROUND} && mv /opt/hgcapp/services-hedera/HapiApp2.0/data/config/override-network.json /opt/hgcapp/services-hedera/HapiApp2.0/data/config/${OVERRIDE_ROUND}/override-network.json" + kubectl exec network-${node}-0 -n {{ .NAMESPACE }} -c root-container -- \ + ls -l /opt/hgcapp/services-hedera/HapiApp2.0/data/config/${OVERRIDE_ROUND}/override-network.json + done + echo "Prepared restore input:" find "${RESTORE_INPUT_DIR}" -maxdepth 4 -type f | sort - # Start all nodes together so they can transition to ACTIVE as a group. + # Start all nodes together so they replay the same restored boundary. $SOLO_COMMAND consensus node start --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} \ - --consensus-node-version {{ .CONSENSUS_NODE_VERSION }} \ --state-file "${RESTORE_INPUT_DIR}" - cmd: echo "✅ Nodes started with restored state" @@ -536,17 +563,19 @@ tasks: # ==================== Verification ==================== verify-state: - desc: Verify restored state by generating transactions + desc: Verify restored state without submitting transactions to a frozen network cmds: - - cmd: echo "Verifying restored state by generating transactions..." - - cmd: sleep 30 # Wait for nodes to stabilize - cmd: | - echo "Generating test transactions to verify network functionality..." - for i in {1..3}; do - $SOLO_COMMAND ledger account create --deployment {{ .DEPLOYMENT }} - sleep 2 + echo "Verifying restored frozen state..." + for node in $(echo {{ .NODE_ALIASES }} | tr ',' ' '); do + kubectl exec network-${node}-0 -n {{ .NAMESPACE }} -c root-container -- \ + sh -c 'curl -sf http://localhost:9999/metrics | grep platform_PlatformStatus | grep -v "^#" | grep -q " 6\\.0$"' + if kubectl logs network-${node}-0 -n {{ .NAMESPACE }} -c root-container --tail=500 | grep -q "Invalid State Signature"; then + echo "⚠️ Invalid state signature found in restored ${node} logs" + exit 1 + fi done - - cmd: echo "✅ State verification complete - network is processing transactions" + - cmd: echo "✅ State verification complete - both restored nodes are FREEZE_COMPLETE with no ISS" # ==================== Cleanup Tasks ==================== @@ -556,6 +585,8 @@ tasks: # freeze network before download otherwise may get error "file changed as we read it" - cmd: echo "Freezing network..." - cmd: $SOLO_COMMAND consensus network freeze --deployment {{ .DEPLOYMENT }} + - cmd: echo "Waiting for the saved state files to stabilize before download..." + - cmd: sleep 30 - cmd: echo "✅ Network frozen" destroy-network: @@ -568,7 +599,6 @@ tasks: - cmd: $SOLO_COMMAND consensus network destroy --deployment {{ .DEPLOYMENT }} --force -q || true - cmd: echo "✅ Network resources destroyed" - destroy-database: desc: Destroy external database cmds: @@ -600,8 +630,5 @@ tasks: deploy-block-node: desc: Deploy block node so CN v0.74 does not require MinIO-backed stream storage cmds: - - cmd: | - $SOLO_COMMAND block node add --deployment {{ .DEPLOYMENT }} \ - --consensus-node-version {{ .CONSENSUS_NODE_VERSION }} \ - --quiet-mode --dev + - cmd: $SOLO_COMMAND block node add --deployment {{ .DEPLOYMENT }} --quiet-mode --dev - cmd: echo "✅ Block node deployed" diff --git a/examples/state-save-and-restore/scripts/generate-override-network.mjs b/examples/state-save-and-restore/scripts/generate-override-network.mjs index 6ffc9ca9ea..e350f843b7 100644 --- a/examples/state-save-and-restore/scripts/generate-override-network.mjs +++ b/examples/state-save-and-restore/scripts/generate-override-network.mjs @@ -49,7 +49,7 @@ class OverrideNetworkGenerator { const encodedIpAddress = this.encodeIpv4Address(clusterIpAddress); const nodeMetadata = rewrittenNetwork.nodeMetadata[nodeIndex]; - changedEndpointCount += this.rewriteNodeServiceEndpoints(nodeMetadata, encodedIpAddress, clusterIpAddress, serviceName); + changedEndpointCount += this.rewriteNodeEndpoints(nodeMetadata, encodedIpAddress, clusterIpAddress, serviceName); } if (changedEndpointCount === 0) { @@ -100,31 +100,56 @@ class OverrideNetworkGenerator { return Buffer.from(octets).toString('base64'); } - static rewriteNodeServiceEndpoints(nodeMetadata, encodedIpAddress, clusterIpAddress, serviceName) { + static rewriteNodeEndpoints(nodeMetadata, encodedIpAddress, clusterIpAddress, serviceName) { if (!nodeMetadata || typeof nodeMetadata !== 'object') { throw new Error('nodeMetadata entry is missing or invalid'); } - return this.rewriteServiceEndpointList(nodeMetadata?.node, encodedIpAddress, clusterIpAddress, serviceName); + // The restored roster uses gossip endpoints before the service endpoints are + // needed. Rewrite both representations so the fresh cluster can establish + // peer sync without depending on endpoint names from the source cluster. + let changedEndpointCount = this.rewriteEndpointList( + nodeMetadata?.node, + 'gossipEndpoint', + encodedIpAddress, + clusterIpAddress, + serviceName, + ); + changedEndpointCount += this.rewriteEndpointList( + nodeMetadata?.node, + 'serviceEndpoint', + encodedIpAddress, + clusterIpAddress, + serviceName, + ); + changedEndpointCount += this.rewriteEndpointList( + nodeMetadata?.rosterEntry, + 'gossipEndpoint', + encodedIpAddress, + clusterIpAddress, + serviceName, + ); + return changedEndpointCount; } - static rewriteServiceEndpointList(parentObject, encodedIpAddress, clusterIpAddress, serviceName) { + static rewriteEndpointList(parentObject, endpointProperty, encodedIpAddress, clusterIpAddress, serviceName) { if (!parentObject || typeof parentObject !== 'object') { throw new Error(`node metadata is missing for ${serviceName}`); } - const serviceEndpoints = parentObject.serviceEndpoint; - if (!Array.isArray(serviceEndpoints) || serviceEndpoints.length === 0) { - throw new Error(`node is missing serviceEndpoint entries for ${serviceName}`); + const endpoints = parentObject[endpointProperty]; + if (!Array.isArray(endpoints) || endpoints.length === 0) { + throw new Error(`node is missing ${endpointProperty} entries for ${serviceName}`); } let changedEndpointCount = 0; - parentObject.serviceEndpoint = serviceEndpoints.map(serviceEndpoint => { - const rewrittenEndpoint = {...serviceEndpoint}; + parentObject[endpointProperty] = endpoints.map(endpoint => { + const rewrittenEndpoint = {...endpoint}; const existingIpAddress = rewrittenEndpoint.ipAddressV4; rewrittenEndpoint.ipAddressV4 = encodedIpAddress; + delete rewrittenEndpoint.domainName; const endpointChanged = existingIpAddress !== encodedIpAddress; if (endpointChanged) { @@ -134,7 +159,7 @@ class OverrideNetworkGenerator { return rewrittenEndpoint; }); - console.log(`Updated service endpoints for ${serviceName} -> ${clusterIpAddress}`); + console.log(`Updated ${endpointProperty} for ${serviceName} -> ${clusterIpAddress}`); return changedEndpointCount; } } diff --git a/resources/cleanup-state-rounds.sh b/resources/cleanup-state-rounds.sh index e3d18933ff..6a4f0ab505 100644 --- a/resources/cleanup-state-rounds.sh +++ b/resources/cleanup-state-rounds.sh @@ -14,6 +14,12 @@ HEDERA_HAPI_PATH="${1:-/opt/hgcapp/services-hedera/HapiApp2.0}" STATE_DIR="${HEDERA_HAPI_PATH}/data/saved/com.hedera.services.ServicesMain" +extract_pces_max_round() { + pces_path="$1" + pces_name=$(basename "$pces_path") + echo "$pces_name" | sed -n 's/.*_maxr\([0-9][0-9]*\)_.*/\1/p' +} + echo "Cleaning up old state rounds in ${STATE_DIR}" cd "${STATE_DIR}" || exit 0 @@ -84,64 +90,6 @@ for nodeid in */; do fi done - # Rebuild top-level PCES from the selected pre-freeze round so the event - # creator has the preconsensus events it needs to become active after - # resuming from the kept state. - if [ -d "${pces_source_round}/preconsensus-events" ]; then - top_level_pces="${HEDERA_HAPI_PATH}/data/saved/preconsensus-events" - echo " Rebuilding top-level preconsensus events from round: $pces_source_round" - rm -rf "$top_level_pces" - source_pces_dir="${pces_source_round}/preconsensus-events" - find "$source_pces_dir" -type f -name '*.pces' | while IFS= read -r pces_file; do - relative_pces_path=${pces_file#"$source_pces_dir"/} - pces_node_id=${relative_pces_path%%/*} - pces_filename=${pces_file##*/} - pces_date=${pces_filename%%T*} - - year=$(echo "$pces_date" | cut -d- -f1) - month=$(echo "$pces_date" | cut -d- -f2) - day=$(echo "$pces_date" | cut -d- -f3) - - if [ -n "$pces_node_id" ] && [ -n "$year" ] && [ -n "$month" ] && [ -n "$day" ] && [ "$pces_date" != "$pces_filename" ]; then - pces_destination_dir="${top_level_pces}/${pces_node_id}/${year}/${month}/${day}" - else - pces_destination_dir="${top_level_pces}/${pces_node_id}" - fi - - mkdir -p "$pces_destination_dir" - cp "$pces_file" "$pces_destination_dir/" - done - fi - - if [ -d "${latest_round}/preconsensus-events" ]; then - round_pces_dir="${latest_round}/preconsensus-events" - round_pces_tmp="${latest_round}/preconsensus-events.tmp" - echo " Normalizing round preconsensus events for state: $latest_round" - rm -rf "$round_pces_tmp" - mkdir -p "$round_pces_tmp" - find "$round_pces_dir" -type f -name '*.pces' | while IFS= read -r pces_file; do - relative_pces_path=${pces_file#"$round_pces_dir"/} - pces_node_id=${relative_pces_path%%/*} - pces_filename=${pces_file##*/} - pces_date=${pces_filename%%T*} - - year=$(echo "$pces_date" | cut -d- -f1) - month=$(echo "$pces_date" | cut -d- -f2) - day=$(echo "$pces_date" | cut -d- -f3) - - if [ -n "$pces_node_id" ] && [ -n "$year" ] && [ -n "$month" ] && [ -n "$day" ] && [ "$pces_date" != "$pces_filename" ]; then - pces_destination_dir="${round_pces_tmp}/${pces_node_id}/${year}/${month}/${day}" - else - pces_destination_dir="${round_pces_tmp}/${pces_node_id}" - fi - - mkdir -p "$pces_destination_dir" - cp "$pces_file" "$pces_destination_dir/" - done - rm -rf "$round_pces_dir" - mv "$round_pces_tmp" "$round_pces_dir" - fi - if [ "$pces_source_round" != "$latest_round" ] && [ -d "$pces_source_round" ]; then echo " Removing old round after PCES rebuild: $pces_source_round" rm -rf "$pces_source_round" diff --git a/resources/wait-for-stable-saved-state.sh b/resources/wait-for-stable-saved-state.sh new file mode 100644 index 0000000000..c9f54a5ad9 --- /dev/null +++ b/resources/wait-for-stable-saved-state.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +# When called from `consensus state download`, `true` means the remote config +# says the node is frozen. Prefer a fully signed freeze round, but fall back to +# a fully signed non-freeze round when this CN version does not sign freeze +# states (it reports SIGNING_WEIGHT_SUM: 0 for them). +prefer_freeze_round="${1:-false}" +saved_dir="${2:-/opt/hgcapp/services-hedera/HapiApp2.0/data/saved}" + +if command -v sha256sum >/dev/null 2>&1; then + hash_cmd=(sha256sum) +elif command -v shasum >/dev/null 2>&1; then + hash_cmd=(shasum -a 256) +elif command -v openssl >/dev/null 2>&1; then + hash_cmd=(openssl dgst -sha256) +else + echo "No SHA-256 implementation found in container" >&2 + exit 14 +fi + +if [[ ! -d "${saved_dir}" ]]; then + exit 10 +fi + +round_root="$(find "${saved_dir}/com.hedera.services.ServicesMain" -mindepth 2 -maxdepth 2 -type d 2>/dev/null | head -n 1)" +if [[ -z "${round_root}" ]]; then + exit 11 +fi + +# Prefer the newest fully signed freeze round because it is the cleanest +# recovery boundary. For stopped deployments, use the newest fully signed +# non-freeze round instead. +selected_round="" +fallback_round="" +selected_kind="none" +while IFS= read -r round_dir; do + metadata_file="${round_dir}/stateMetadata.txt" + [[ -f "${metadata_file}" ]] || continue + + freeze_state="$(awk -F: '/^FREEZE_STATE:/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2}' "${metadata_file}")" + signing_weight="$(awk -F: '/^SIGNING_WEIGHT_SUM:/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2}' "${metadata_file}")" + total_weight="$(awk -F: '/^TOTAL_WEIGHT:/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2}' "${metadata_file}")" + + if [[ -n "${signing_weight}" && "${signing_weight}" == "${total_weight}" ]]; then + if [[ "${freeze_state}" == "true" ]]; then + selected_round="$(basename "${round_dir}")" + selected_kind="freeze" + continue + fi + + fallback_round="$(basename "${round_dir}")" + fi +done < <( + find "${round_root}" -mindepth 1 -maxdepth 1 -type d -print \ + | while IFS= read -r candidate_dir; do + printf '%s\t%s\n' "$(basename "${candidate_dir}")" "${candidate_dir}" + done \ + | sort -n -k1,1 \ + | cut -f2- +) + +if [[ -z "${selected_round}" ]]; then + selected_round="${fallback_round}" + selected_kind="non-freeze" +fi + +if [[ -z "${selected_round}" ]]; then + exit 12 +fi + +if [[ "${prefer_freeze_round}" == "true" && "${selected_kind}" != "freeze" ]]; then + selected_kind="frozen-fallback" +fi + +# Fingerprint the entire saved-state tree, not just the chosen round directory, +# so the caller can detect when background flushes have stopped changing disk +# contents across consecutive polls. +find "${saved_dir}" -type f -print0 \ + | sort -z \ + | xargs -0 "${hash_cmd[@]}" \ + | "${hash_cmd[@]}" \ + | awk -v round="${selected_round}" -v kind="${selected_kind}" '{print $1, round, kind}' diff --git a/src/commands/node/handlers.ts b/src/commands/node/handlers.ts index c377238173..1c4af8ed7d 100644 --- a/src/commands/node/handlers.ts +++ b/src/commands/node/handlers.ts @@ -1074,22 +1074,28 @@ export class NodeCommandHandlers extends CommandHandler { public async start(argv: ArgvStruct): Promise { argv = addFlagsToArgv(argv, NodeFlags.START_FLAGS); const leaseWrapper: LeaseWrapper = {lease: undefined}; + const restoringState: boolean = + typeof argv[flags.stateFile.name] === 'string' && argv[flags.stateFile.name].length > 0; - await this.commandAction( - argv, - [ - this.tasks.loadConfiguration(argv, leaseWrapper, this.leaseManager), - this.tasks.initialize( - argv, - this.configs.startConfigBuilder.bind(this.configs), - leaseWrapper.lease, - true, - false, - ), - this.validateAllNodePhases({acceptedPhases: [DeploymentPhase.CONFIGURED]}), - this.tasks.identifyExistingNodes(), - this.tasks.uploadStateFiles(({config}): boolean => config.stateFile.length === 0), - this.tasks.startNodes('nodeAliases'), + const startTasks = [ + this.tasks.loadConfiguration(argv, leaseWrapper, this.leaseManager), + this.tasks.initialize(argv, this.configs.startConfigBuilder.bind(this.configs), leaseWrapper.lease, true, false), + this.validateAllNodePhases({acceptedPhases: [DeploymentPhase.CONFIGURED]}), + this.tasks.identifyExistingNodes(), + this.tasks.uploadStateFiles(({config}): boolean => config.stateFile.length === 0), + this.tasks.startNodes('nodeAliases'), + ]; + + if (restoringState) { + // A freeze-captured archive is expected to restore into FREEZE_COMPLETE. + // Do not require ACTIVE or run ACTIVE-only TSS/start-event tasks for it. + startTasks.push( + this.tasks.checkAllNodesAreFrozen('nodeAliases'), + this.tasks.checkNodeProxiesAreActive(), + this.changeAllNodePhases(DeploymentPhase.FROZEN), + ); + } else { + startTasks.push( this.tasks.checkNodesAndProxiesAreActive('nodeAliases'), this.tasks.enablePortForwarding(true), this.tasks.waitForTss(), @@ -1097,9 +1103,12 @@ export class NodeCommandHandlers extends CommandHandler { this.changeAllNodePhases(DeploymentPhase.STARTED, LedgerPhase.INITIALIZED), this.tasks.addNodeStakes(), this.tasks.emitNodeStartedEvent(), - // TODO only show this if we are not running in one-shot mode - // this.tasks.showUserMessages(), - ], + ); + } + + await this.commandAction( + argv, + startTasks, constants.LISTR_DEFAULT_OPTIONS.DEFAULT, 'Error starting node', leaseWrapper.lease, @@ -1155,6 +1164,7 @@ export class NodeCommandHandlers extends CommandHandler { this.tasks.identifyExistingNodes(), this.tasks.sendFreezeTransaction(), this.tasks.checkAllNodesAreFrozen('existingNodeAliases'), + this.tasks.waitForFrozenStateToBeSigned('existingNodeAliases'), this.tasks.stopNodes('existingNodeAliases'), this.changeAllNodePhases(DeploymentPhase.FROZEN), ], diff --git a/src/commands/node/tasks.ts b/src/commands/node/tasks.ts index ee17bae73a..d5925bcf01 100644 --- a/src/commands/node/tasks.ts +++ b/src/commands/node/tasks.ts @@ -162,6 +162,7 @@ import {SemanticVersion} from '../../business/utils/semantic-version.js'; import {DeploymentStateSchema} from '../../data/schema/model/remote/deployment-state-schema.js'; import {type BaseStateSchema} from '../../data/schema/model/remote/state/base-state-schema.js'; import {ComponentStateMetadataSchema} from '../../data/schema/model/remote/state/component-state-metadata-schema.js'; +import {ConsensusNodeStateSchema} from '../../data/schema/model/remote/state/consensus-node-state-schema.js'; import net from 'node:net'; import {type NodeConnectionsContext} from './config-interfaces/node-connections-context.js'; import {TDirectoryData} from '../../integration/kube/t-directory-data.js'; @@ -2251,6 +2252,34 @@ export class NodeCommandTasks { }; } + public waitForFrozenStateToBeSigned(nodeAliasesProperty: string): SoloListrTask { + return { + title: 'Wait for frozen state files to stabilize', + task: (context_, task): SoloListr => { + const nodeAliases: NodeAliases = context_.config[nodeAliasesProperty]; + const subTasks: SoloListrTask[] = nodeAliases.map( + (nodeAlias): SoloListrTask => ({ + title: `Wait for stable frozen state: ${chalk.yellow(nodeAlias)}`, + task: async (): Promise => { + const context: string = extractContextFromConsensusNodes( + nodeAlias, + this.remoteConfig.getConsensusNodes(), + ); + const podReference: PodReference = PodReference.of( + context_.config.namespace, + Templates.renderNetworkPodName(nodeAlias), + ); + await container + .resolve(InjectTokens.NetworkNodes) + .waitForFrozenStateToBeStable(podReference, context); + }, + }), + ); + return task.newListr(subTasks, {concurrent: true, rendererOptions: {collapseSubtasks: false}}); + }, + }; + } + public checkNodeProxiesAreActive(): SoloListrTask { return { title: 'Check node proxies are ACTIVE', @@ -3148,9 +3177,26 @@ export class NodeCommandTasks { task: async (context_): Promise => { for (const nodeAlias of context_.config.nodeAliases) { const context: string = extractContextFromConsensusNodes(nodeAlias, context_.config.consensusNodes); + const nodeComponent: ConsensusNodeStateSchema = this.remoteConfig.configuration.components.getComponent( + ComponentTypes.ConsensusNode, + Templates.renderComponentIdFromNodeAlias(nodeAlias), + ); + const deploymentPhase: DeploymentPhase = nodeComponent.metadata.phase; + + if (![DeploymentPhase.FROZEN, DeploymentPhase.STOPPED].includes(deploymentPhase)) { + this.logger.showUser( + chalk.yellow( + `Warning: node ${nodeAlias} is in phase '${deploymentPhase}'. State download is only supported when consensus nodes are frozen or stopped.`, + ), + ); + throw new SoloErrors.validation.illegalArgument( + `Consensus node ${nodeAlias} must be in phase '${DeploymentPhase.FROZEN}' or '${DeploymentPhase.STOPPED}' before downloading saved state.`, + ); + } + await container .resolve(InjectTokens.NetworkNodes) - .getStatesFromPod(context_.config.namespace, nodeAlias, context); + .getStatesFromPod(context_.config.namespace, nodeAlias, context, undefined, deploymentPhase); } }, }; diff --git a/src/core/network-nodes.ts b/src/core/network-nodes.ts index 8d7fd449fe..c7cfda9022 100644 --- a/src/core/network-nodes.ts +++ b/src/core/network-nodes.ts @@ -19,6 +19,8 @@ import {K8} from '../integration/kube/k8.js'; import {Container} from '../integration/kube/resources/container/container.js'; import {NodeStatusEnums} from './enumerations.js'; import chalk from 'chalk'; +import {DeploymentPhase} from '../data/schema/model/remote/deployment-phase.js'; +import {SoloErrors} from './errors/solo-errors.js'; /** * Class to manage network nodes @@ -139,6 +141,7 @@ export class NetworkNodes { nodeAlias: string, context?: string, baseDirectory?: string, + deploymentPhase?: DeploymentPhase, ): Promise { const pods: Pod[] = await this.k8Factory .getK8(context) @@ -149,12 +152,29 @@ export class NetworkNodes { const stateBaseDirectory: string = baseDirectory || SOLO_LOGS_DIR; const promises: Promise[] = []; for (const pod of pods) { - promises.push(this.getState(pod, namespace, stateBaseDirectory, context)); + promises.push(this.getState(pod, namespace, stateBaseDirectory, context, deploymentPhase)); } return await Promise.all(promises); } - private async getState(pod: Pod, namespace: NamespaceName, baseDirectory: string, context?: string): Promise { + /** + * Wait for a fully signed freeze state before a freeze workflow stops the node. + * A FROZEN platform status alone is not enough: stopping immediately can leave + * the archive with only a non-freeze state and misaligned PCES replay data. + */ + public async waitForFrozenStateToBeStable(podReference: PodReference, context?: string): Promise { + const containerReference: ContainerReference = ContainerReference.of(podReference, ROOT_CONTAINER); + const container: Container = this.k8Factory.getK8(context).containers().readByRef(containerReference); + await this.waitForStableSavedState(container, podReference.name.name, false); + } + + private async getState( + pod: Pod, + namespace: NamespaceName, + baseDirectory: string, + context?: string, + deploymentPhase?: DeploymentPhase, + ): Promise { const podReference: PodReference = pod.podReference; this.logger.debug(`getNodeState(${pod.podReference.name.name}): begin...`); const targetDirectory: string = PathEx.join(baseDirectory, namespace.name); @@ -167,6 +187,15 @@ export class NetworkNodes { const k8: K8 = this.k8Factory.getK8(context); const zipFileName: string = `${HEDERA_HAPI_PATH}/${podReference.name}-state.zip`; + // A frozen node should yield a freeze round; a merely stopped node may only + // have a non-freeze signed round available. + const requireFreezeRound: boolean = deploymentPhase === DeploymentPhase.FROZEN; + + await this.waitForStableSavedState( + k8.containers().readByRef(containerReference), + podReference.name.name, + requireFreezeRound, + ); // Zip doesn't have a -C flag like tar, so we use sh -c with subshell to change directory // Use the -X to archive for cross-platform compatibility @@ -187,6 +216,86 @@ export class NetworkNodes { this.logger.debug(`getNodeState(${pod.podReference.name.name}): ...end`); } + private async waitForStableSavedState( + container: Container, + podName: string, + requireFreezeRound: boolean, + ): Promise { + const maxAttempts: number = 180; + const stablePollsRequired: number = 3; + const pollDelay: Duration = Duration.ofSeconds(2); + let lastFingerprint: string | undefined; + let stablePolls: number = 0; + const scriptName: string = 'wait-for-stable-saved-state.sh'; + const sourcePath: string = PathEx.joinWithRealPath(constants.RESOURCES_DIR, scriptName); + const destinationPath: string = `${HEDERA_HAPI_PATH}/${scriptName}`; + + // Reuse a checked-in resource script so the in-pod state-selection logic is + // versioned alongside Solo and remains readable/testable outside TS strings. + await container.copyTo(sourcePath, `${HEDERA_HAPI_PATH}`); + await sleep(Duration.ofSeconds(1)); + await container.execContainer([ + 'bash', + '-c', + `sync ${HEDERA_HAPI_PATH} && chown hedera:hedera ${destinationPath} && chmod 0755 ${destinationPath}`, + ]); + + for (let attempt: number = 1; attempt <= maxAttempts; attempt++) { + try { + const rawOutput: string = await container.execContainer([ + 'bash', + '-lc', + // The script prints " " + // once it finds the best fully signed saved-state boundary currently + // persisted on disk, preferring a freeze round when requested. + `${destinationPath} ${String(requireFreezeRound)} ${HEDERA_HAPI_PATH}/data/saved`, + ]); + const output: string = rawOutput.trim(); + + const [fingerprint, round, kind] = output.split(/\s+/); + if (!fingerprint || !round || !kind) { + throw new SoloErrors.validation.illegalArgument(`Missing saved state fingerprint for pod ${podName}`); + } + + stablePolls = fingerprint === lastFingerprint ? stablePolls + 1 : 1; + lastFingerprint = fingerprint; + + this.logger.debug( + `[state-download] ${podName}: round ${round} (${kind}) stable poll ${stablePolls}/${stablePollsRequired}`, + ); + + if (kind === 'frozen-fallback') { + // A frozen deployment can expose the FROZEN platform status before a + // freeze-marked round becomes fully signed on disk. In that case, + // export the newest fully signed non-freeze round instead of waiting + // indefinitely for a freeze round that may never materialize. + this.logger.warn( + `[state-download] ${podName}: deployment is FROZEN but no fully signed freeze round exists on disk yet; using the newest fully signed non-freeze round`, + ); + } + + if (stablePolls >= stablePollsRequired) { + // One final sync narrows the gap between the successful probe and the + // subsequent zip/copy operation. + await container.execContainer('sync'); + return; + } + } catch (error) { + // The script exits non-zero until a qualifying signed round exists or the + // saved-state tree stops changing across polls. + this.logger.debug(`[state-download] ${podName}: saved state not stable yet`, error); + } + + await sleep(pollDelay); + } + + throw new SoloErrors.validation.illegalArgument( + requireFreezeRound + ? `Timed out waiting for a stable fully signed saved state on pod ${podName}. The deployment is frozen, but no signed round became stable on disk.` + : `Timed out waiting for a stable fully signed saved state on pod ${podName}. Stop or freeze the node and retry state download.`, + ); + } + public async getNetworkNodePodStatus(podReference: PodReference, context?: string): Promise { return this.k8Factory .getK8(context) From 404fe9bccb2c5cc7c2a0c4508afe9c83efc17579 Mon Sep 17 00:00:00 2001 From: Jeffrey Tang Date: Thu, 30 Jul 2026 16:50:52 -0500 Subject: [PATCH 09/20] save Signed-off-by: Jeffrey Tang --- examples/state-save-and-restore/Taskfile.yml | 2 -- resources/cleanup-state-rounds.sh | 24 ++------------------ src/commands/node/handlers.ts | 2 +- src/commands/node/tasks.ts | 2 +- 4 files changed, 4 insertions(+), 26 deletions(-) diff --git a/examples/state-save-and-restore/Taskfile.yml b/examples/state-save-and-restore/Taskfile.yml index f5b11ab082..944248e320 100644 --- a/examples/state-save-and-restore/Taskfile.yml +++ b/examples/state-save-and-restore/Taskfile.yml @@ -585,8 +585,6 @@ tasks: # freeze network before download otherwise may get error "file changed as we read it" - cmd: echo "Freezing network..." - cmd: $SOLO_COMMAND consensus network freeze --deployment {{ .DEPLOYMENT }} - - cmd: echo "Waiting for the saved state files to stabilize before download..." - - cmd: sleep 30 - cmd: echo "✅ Network frozen" destroy-network: diff --git a/resources/cleanup-state-rounds.sh b/resources/cleanup-state-rounds.sh index 6a4f0ab505..edd6fd64a8 100644 --- a/resources/cleanup-state-rounds.sh +++ b/resources/cleanup-state-rounds.sh @@ -14,12 +14,6 @@ HEDERA_HAPI_PATH="${1:-/opt/hgcapp/services-hedera/HapiApp2.0}" STATE_DIR="${HEDERA_HAPI_PATH}/data/saved/com.hedera.services.ServicesMain" -extract_pces_max_round() { - pces_path="$1" - pces_name=$(basename "$pces_path") - echo "$pces_name" | sed -n 's/.*_maxr\([0-9][0-9]*\)_.*/\1/p' -} - echo "Cleaning up old state rounds in ${STATE_DIR}" cd "${STATE_DIR}" || exit 0 @@ -41,10 +35,8 @@ for nodeid in */; do if [ -n "$rounds" ]; then latest_round="" - pces_source_round="" highest_round=$(echo "$rounds" | tail -n 1) highest_round_freeze_state="" - earliest_signed_non_freeze_round="" latest_signed_non_freeze_round="" for round in $rounds; do metadata_file="${round}/stateMetadata.txt" @@ -59,9 +51,6 @@ for nodeid in */; do fi if [ "$freeze_state" = "false" ] && [ -n "$signing_weight" ] && [ "$signing_weight" = "$total_weight" ]; then - if [ -z "$earliest_signed_non_freeze_round" ]; then - earliest_signed_non_freeze_round="$round" - fi latest_signed_non_freeze_round="$round" latest_round="$round" fi @@ -75,25 +64,16 @@ for nodeid in */; do latest_round="$highest_round" fi - if [ -z "$pces_source_round" ]; then - pces_source_round="$latest_round" - fi - round_count=$(echo "$rounds" | wc -l) - echo "Node ${nodeid}${realmShard}: Found ${round_count} rounds, keeping state: ${latest_round}, PCES: ${pces_source_round}" + echo "Node ${nodeid}${realmShard}: Found ${round_count} rounds, keeping state: ${latest_round}" for round in $rounds; do - if [ "$round" != "$latest_round" ] && [ "$round" != "$pces_source_round" ]; then + if [ "$round" != "$latest_round" ]; then echo " Removing old round: $round" rm -rf "$round" fi done - - if [ "$pces_source_round" != "$latest_round" ] && [ -d "$pces_source_round" ]; then - echo " Removing old round after PCES rebuild: $pces_source_round" - rm -rf "$pces_source_round" - fi fi cd ../.. diff --git a/src/commands/node/handlers.ts b/src/commands/node/handlers.ts index 1c4af8ed7d..ca4c95ffca 100644 --- a/src/commands/node/handlers.ts +++ b/src/commands/node/handlers.ts @@ -1164,7 +1164,7 @@ export class NodeCommandHandlers extends CommandHandler { this.tasks.identifyExistingNodes(), this.tasks.sendFreezeTransaction(), this.tasks.checkAllNodesAreFrozen('existingNodeAliases'), - this.tasks.waitForFrozenStateToBeSigned('existingNodeAliases'), + this.tasks.waitForFrozenStateToBeStable('existingNodeAliases'), this.tasks.stopNodes('existingNodeAliases'), this.changeAllNodePhases(DeploymentPhase.FROZEN), ], diff --git a/src/commands/node/tasks.ts b/src/commands/node/tasks.ts index d5925bcf01..bf7ee9b38e 100644 --- a/src/commands/node/tasks.ts +++ b/src/commands/node/tasks.ts @@ -2252,7 +2252,7 @@ export class NodeCommandTasks { }; } - public waitForFrozenStateToBeSigned(nodeAliasesProperty: string): SoloListrTask { + public waitForFrozenStateToBeStable(nodeAliasesProperty: string): SoloListrTask { return { title: 'Wait for frozen state files to stabilize', task: (context_, task): SoloListr => { From fb9b87df1b4df1efd00e6ff34c8cd7e84c228a29 Mon Sep 17 00:00:00 2001 From: Jeffrey Tang Date: Thu, 30 Jul 2026 17:23:24 -0500 Subject: [PATCH 10/20] simplify Signed-off-by: Jeffrey Tang --- examples/state-save-and-restore/Taskfile.yml | 209 ++---------------- .../state-save-and-restore/scripts/init.sh | 92 -------- package-lock.json | 21 -- 3 files changed, 14 insertions(+), 308 deletions(-) delete mode 100644 examples/state-save-and-restore/scripts/init.sh diff --git a/examples/state-save-and-restore/Taskfile.yml b/examples/state-save-and-restore/Taskfile.yml index 944248e320..9b95eaf5ac 100644 --- a/examples/state-save-and-restore/Taskfile.yml +++ b/examples/state-save-and-restore/Taskfile.yml @@ -15,7 +15,6 @@ vars: NODE_ALIASES: 'node1,node2' DEPLOYMENT: 'state-restore-deployment' NAMESPACE: 'state-restore-namespace' - ENABLE_MIRROR_WORKFLOW: '{{ default "false" (env "ENABLE_MIRROR_WORKFLOW") }}' SOLO_USER_DIR: '{{ default (printf "%s/.solo" (env "HOME")) }}' @@ -26,26 +25,13 @@ vars: # State Save Configuration STATE_SAVE_DIR: '{{ .USER_WORKING_DIR }}/saved-states' - MIRROR_PASSWORDS_SECRET_FILE: '{{ .STATE_SAVE_DIR }}/mirror-passwords-secret.json' ORIGINAL_NETWORK_JSON_FILE: '{{ .STATE_SAVE_DIR }}/original-network.json' GENERATED_OVERRIDE_NETWORK_JSON_FILE: '{{ .STATE_SAVE_DIR }}/override-network.json' CURRENT_SERVICE_ENDPOINTS_FILE: '{{ .STATE_SAVE_DIR }}/current-service-endpoints.json' OVERRIDE_NETWORK_GENERATOR_SCRIPT: '{{ .TASKFILE_DIR }}/scripts/generate-override-network.mjs' - PREPULL_IMAGES_SCRIPT: '{{ .TASKFILE_DIR }}/scripts/prepull-images.sh' SAVED_KEYS_DIR: '{{ .STATE_SAVE_DIR }}/keys' SOLO_CACHE_KEYS_DIR: '{{ .SOLO_USER_DIR }}/cache/keys' - # External Database Configuration (Optional) - POSTGRES_USERNAME: 'postgres' - POSTGRES_PASSWORD: 'XXXXXXXX' - POSTGRES_READONLY_USERNAME: 'readonlyuser' - POSTGRES_READONLY_PASSWORD: 'XXXXXXXX' - POSTGRES_MIRROR_NODE_DATABASE_NAME: 'mirror_node' - POSTGRES_NAME: 'my-postgresql' - POSTGRES_DATABASE_NAMESPACE: 'database' - POSTGRES_CONTAINER_NAME: '{{ .POSTGRES_NAME }}-0' - POSTGRES_HOST_FQDN: '{{ .POSTGRES_NAME }}.database.svc.cluster.local' - tasks: # ==================== Main Tasks ==================== @@ -70,23 +56,17 @@ tasks: # ==================== Setup Tasks ==================== setup: - desc: Deploy initial network with external PostgreSQL database + desc: Deploy initial consensus network cmds: + # Install the Solo-managed dependency binaries before creating the cluster. + - cmd: $SOLO_COMMAND init --dev - task: create-cluster - - task: preload-network-images - task: init-solo - task: deploy-block-node - task: deploy-network - - cmd: | - if [ "{{ .ENABLE_MIRROR_WORKFLOW }}" = "true" ]; then - task deploy-external-database - task deploy-mirror-external - else - echo "Skipping mirror/database setup; set ENABLE_MIRROR_WORKFLOW=true to include it." - fi - task: generate-transactions - - cmd: echo "✅ Initial network with external database setup complete!" - - cmd: echo "Run 'task save-state' to save state and database" + - cmd: echo "✅ Initial consensus network setup complete!" + - cmd: echo "Run 'task save-state' to save consensus state" # ==================== Infrastructure Tasks ==================== @@ -109,11 +89,6 @@ tasks: - cmd: sleep 10 # Wait for control plane - cmd: kubectl config use-context {{ .CONTEXT }} - preload-network-images: - desc: Pre-pull and load consensus-network images into the Kind cluster - cmds: - - cmd: bash {{ .PREPULL_IMAGES_SCRIPT }} {{ .CLUSTER_NAME }} - init-solo: desc: Connect cluster reference and configure deployment cmds: @@ -134,55 +109,6 @@ tasks: - cmd: $SOLO_COMMAND consensus node start --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} - cmd: echo "✅ Consensus network deployed with {{ .NETWORK_SIZE }} nodes" - deploy-mirror-external: - desc: Deploy mirror node with external database - cmds: - # Solo mirror node add" - - cmd: | - $SOLO_COMMAND mirror node add --deployment {{ .DEPLOYMENT }} --use-external-database \ - --enable-ingress --external-database-host {{ .POSTGRES_HOST_FQDN }} \ - --external-database-owner-username {{ .POSTGRES_USERNAME }} --external-database-owner-password {{ .POSTGRES_PASSWORD }} \ - --external-database-read-username {{ .POSTGRES_READONLY_USERNAME }} \ - --external-database-read-password {{ .POSTGRES_READONLY_PASSWORD }} \ - --enable-ingress --pinger -q --dev - - cmd: echo "✅ Mirror node deployed with external PostgreSQL database" - - # ==================== External Database Tasks ==================== - - deploy-external-database: - desc: Deploy external PostgreSQL database - cmds: - # Install PostgreSQL using Helm" - - cmd: | - PATH="{{ .SOLO_USER_DIR }}/bin:$PATH" - helm repo add postgresql-helm https://leverages.github.io/helm - helm install {{ .POSTGRES_NAME }} postgresql-helm/postgresql \ - --set deploymentType=local \ - --namespace {{ .POSTGRES_DATABASE_NAMESPACE }} --create-namespace \ - --set postgresql.auth.password={{ .POSTGRES_PASSWORD }} - - # Wait for PostgreSQL pod to be ready" - - cmd: | - kubectl wait --for=condition=ready pod/{{ .POSTGRES_CONTAINER_NAME }} \ - -n {{ .POSTGRES_DATABASE_NAMESPACE }} --timeout=300s - - # Copy init.sql inside the database pod" - - cmd: | - kubectl cp {{ .TASKFILE_DIR }}/scripts/init.sh \ - {{ .POSTGRES_CONTAINER_NAME }}:/tmp/init.sh \ - -n {{ .POSTGRES_DATABASE_NAMESPACE }} - - # Make init.sh executable" - - cmd: | - kubectl exec -it {{ .POSTGRES_CONTAINER_NAME }} \ - -n {{ .POSTGRES_DATABASE_NAMESPACE }} -- chmod +x /tmp/init.sh - - # Execute init.sh inside the database pod" - - cmd: | - kubectl exec -it {{ .POSTGRES_CONTAINER_NAME }} \ - -n {{ .POSTGRES_DATABASE_NAMESPACE }} \ - -- /bin/bash /tmp/init.sh "{{ .POSTGRES_USERNAME }}" "{{ .POSTGRES_READONLY_USERNAME }}" "{{ .POSTGRES_READONLY_PASSWORD }}" - # ==================== Transaction Generation ==================== generate-transactions: @@ -199,7 +125,7 @@ tasks: # ==================== State Save Tasks ==================== save-state: - desc: Download consensus node state and export database + desc: Download consensus node state and save restore metadata cmds: - cmd: mkdir -p {{ .STATE_SAVE_DIR }} - cmd: echo "Downloading state from consensus nodes..." @@ -217,17 +143,10 @@ tasks: exit 1 fi done - - cmd: echo "Exporting database..." - - cmd: | - if [ "{{ .ENABLE_MIRROR_WORKFLOW }}" = "true" ]; then - kubectl exec {{ .POSTGRES_CONTAINER_NAME }} -n {{ .POSTGRES_DATABASE_NAMESPACE }} -- \ - env PGPASSWORD={{ .POSTGRES_PASSWORD }} pg_dump -U {{ .POSTGRES_USERNAME }} \ - --clean --if-exists \ - {{ .POSTGRES_MIRROR_NODE_DATABASE_NAME }} > {{ .STATE_SAVE_DIR }}/database-dump.sql - else - echo "Skipping database export; mirror workflow disabled." - fi - - cmd: echo "Saving source network JSON..." + # Save the original roster and network metadata as the template for + # override-network.json; the restore workflow replaces only endpoint IPs + # with addresses from the freshly created cluster. + - cmd: echo "Saving original network definition for endpoint override..." - cmd: | kubectl exec network-node1-0 -n {{ .NAMESPACE }} -c root-container -- bash -c ' if [ -f /opt/hgcapp/services-hedera/HapiApp2.0/output/network.json ]; then @@ -242,21 +161,6 @@ tasks: fi ' > {{ .ORIGINAL_NETWORK_JSON_FILE }} - cmd: echo "✅ Source network JSON exported to {{ .ORIGINAL_NETWORK_JSON_FILE }}" - - cmd: echo "Saving mirror passwords secret..." - - cmd: | - if [ "{{ .ENABLE_MIRROR_WORKFLOW }}" = "true" ]; then - kubectl get secret mirror-passwords -n {{ .NAMESPACE }} -o json | \ - jq 'del( - .metadata.annotations."kubectl.kubernetes.io/last-applied-configuration", - .metadata.creationTimestamp, - .metadata.managedFields, - .metadata.resourceVersion, - .metadata.uid - )' > {{ .MIRROR_PASSWORDS_SECRET_FILE }} - echo "✅ Mirror passwords secret exported to {{ .MIRROR_PASSWORDS_SECRET_FILE }}" - else - echo "Skipping mirror secret export; mirror workflow disabled." - fi - cmd: echo "Saving consensus node key material from Kubernetes secrets..." - cmd: mkdir -p {{ .SAVED_KEYS_DIR }} - cmd: | @@ -281,8 +185,7 @@ tasks: echo "✅ Saved gossip key file ${key_file_name}" done done - - cmd: echo "✅ Database exported to {{ .STATE_SAVE_DIR }}/database-dump.sql" - - cmd: echo "✅ Network state and database saved to {{ .STATE_SAVE_DIR }}" + - cmd: echo "✅ Network state and restore metadata saved to {{ .STATE_SAVE_DIR }}" - cmd: ls -lh {{ .STATE_SAVE_DIR }} # ==================== State Restore Tasks ==================== @@ -291,30 +194,14 @@ tasks: desc: Recreate a fresh cluster, restore state, and verify the saved network can boot again cmds: - task: destroy-network - - cmd: | - if [ "{{ .ENABLE_MIRROR_WORKFLOW }}" = "true" ]; then - task destroy-database - else - echo "Skipping database cleanup; mirror workflow disabled." - fi - task: destroy-cluster - task: create-cluster - - task: preload-network-images - task: init-solo - task: restore-consensus-keys - task: deploy-block-node - task: deploy-network-with-state - - cmd: | - if [ "{{ .ENABLE_MIRROR_WORKFLOW }}" = "true" ]; then - task deploy-external-database - task restore-mirror-passwords-secret - task restore-database - task deploy-mirror-external - else - echo "Skipping mirror/database restore; consensus override-network test only." - fi - task: verify-state - - cmd: echo "✅ Network and database restored!" + - cmd: echo "✅ Consensus network restored!" deploy-network-with-state: desc: Deploy a fresh network and start nodes from saved state with override-network endpoint remapping @@ -503,63 +390,6 @@ tasks: done - cmd: echo "✅ override-network.json copied to each consensus node pod" - restore-database: - desc: Restore database from dump - cmds: - - cmd: echo "Restoring database from dump..." - - cmd: | - if [ -f {{ .MIRROR_PASSWORDS_SECRET_FILE }} ]; then - echo "Recreating mirror database roles from saved credentials..." - jq -r '.data | keys[]' {{ .MIRROR_PASSWORDS_SECRET_FILE }} | while read -r key; do - case "${key}" in - *USERNAME) - password_key="${key%USERNAME}PASSWORD" - username="$(jq -r --arg key "${key}" '.data[$key] | @base64d' {{ .MIRROR_PASSWORDS_SECRET_FILE }})" - password="$(jq -r --arg key "${password_key}" '.data[$key] // empty | @base64d' {{ .MIRROR_PASSWORDS_SECRET_FILE }})" - - case "${username}" in - postgres|readonly|readonlyuser|readwrite|temporary_admin|"") - continue - ;; - esac - - if [ -z "${password}" ]; then - continue - fi - - username_sql="$(printf "%s" "${username}" | sed 's/"/""/g')" - role_name_sql="$(printf "%s" "${username}" | sed "s/'/''/g")" - password_sql="$(printf "%s" "${password}" | sed "s/'/''/g")" - - kubectl exec {{ .POSTGRES_CONTAINER_NAME }} -n {{ .POSTGRES_DATABASE_NAMESPACE }} -- \ - env PGPASSWORD={{ .POSTGRES_PASSWORD }} psql -U {{ .POSTGRES_USERNAME }} -d postgres \ - -v ON_ERROR_STOP=1 \ - -c "DO \$\$ BEGIN IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '${role_name_sql}') THEN ALTER ROLE \"${username_sql}\" WITH LOGIN PASSWORD '${password_sql}'; ELSE CREATE ROLE \"${username_sql}\" WITH LOGIN PASSWORD '${password_sql}'; END IF; END \$\$;" - echo "✅ Reconciled role ${username}" - ;; - esac - done - fi - - cmd: | - kubectl cp {{ .STATE_SAVE_DIR }}/database-dump.sql \ - {{ .POSTGRES_CONTAINER_NAME }}:/tmp/database-dump.sql -n {{ .POSTGRES_DATABASE_NAMESPACE }} - - cmd: | - kubectl exec {{ .POSTGRES_CONTAINER_NAME }} -n {{ .POSTGRES_DATABASE_NAMESPACE }} -- \ - env PGPASSWORD={{ .POSTGRES_PASSWORD }} psql -U {{ .POSTGRES_USERNAME }} \ - -d {{ .POSTGRES_MIRROR_NODE_DATABASE_NAME }} -f /tmp/database-dump.sql - - cmd: echo "✅ Database restored" - - restore-mirror-passwords-secret: - desc: Restore saved mirror database credential secret - cmds: - - cmd: | - if [ ! -f {{ .MIRROR_PASSWORDS_SECRET_FILE }} ]; then - echo "⚠️ Saved mirror-passwords secret not found: {{ .MIRROR_PASSWORDS_SECRET_FILE }}" - exit 1 - fi - - cmd: kubectl apply -f {{ .MIRROR_PASSWORDS_SECRET_FILE }} - - cmd: echo "✅ Restored mirror-passwords secret" - # ==================== Verification ==================== verify-state: @@ -588,29 +418,18 @@ tasks: - cmd: echo "✅ Network frozen" destroy-network: - desc: Destroy mirror node, block node, and consensus network while keeping cluster configuration + desc: Destroy block node and consensus network while keeping cluster configuration cmds: - - cmd: echo "Destroying mirror node, block node, and consensus network..." - - cmd: $SOLO_COMMAND mirror node destroy --deployment {{ .DEPLOYMENT }} --force || true + - cmd: echo "Destroying block node and consensus network..." - cmd: $SOLO_COMMAND block node destroy --deployment {{ .DEPLOYMENT }} --force || true - cmd: $SOLO_COMMAND consensus node stop --deployment {{ .DEPLOYMENT }} --node-aliases {{ .NODE_ALIASES }} || true - cmd: $SOLO_COMMAND consensus network destroy --deployment {{ .DEPLOYMENT }} --force -q || true - cmd: echo "✅ Network resources destroyed" - destroy-database: - desc: Destroy external database - cmds: - - cmd: | - PATH="{{ .SOLO_USER_DIR }}/bin:$PATH" - helm uninstall {{ .POSTGRES_NAME }} -n {{ .POSTGRES_DATABASE_NAMESPACE }} || true - - cmd: kubectl delete namespace {{ .POSTGRES_DATABASE_NAMESPACE }} --ignore-not-found=true || true - - cmd: echo "✅ Database destroyed" - destroy: desc: Destroy cluster and clean up all resources cmds: - task: destroy-network - - task: destroy-database - task: destroy-cluster - task: clean-state diff --git a/examples/state-save-and-restore/scripts/init.sh b/examples/state-save-and-restore/scripts/init.sh deleted file mode 100644 index 7db61b2c89..0000000000 --- a/examples/state-save-and-restore/scripts/init.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/bin/bash -set -e - -export HEDERA_MIRROR_DATABASE_NAME="mirror_node" -HEDERA_MIRROR_OWNER="$1" -HEDERA_MIRROR_READ="$2" -HEDERA_MIRROR_READ_PASSWORD="$3" - - -export HEDERA_MIRROR_GRPC_DB_HOST="localhost" - -export HEDERA_MIRROR_IMPORTER_DB_HOST="localhost" -export HEDERA_MIRROR_IMPORTER_DB_NAME="${HEDERA_MIRROR_DATABASE_NAME}" -export HEDERA_MIRROR_IMPORTER_DB_OWNER="${HEDERA_MIRROR_OWNER}" -export HEDERA_MIRROR_IMPORTER_DB_SCHEMA="public" -export HEDERA_MIRROR_IMPORTER_DB_TEMPSCHEMA="temporary" - - -PGHBACONF="/opt/bitnami/postgresql/conf/pg_hba.conf" -if [[ -f "${PGHBACONF}" ]]; then - cp "${PGHBACONF}" "${PGHBACONF}.bak" - echo "local all all trust" > "${PGHBACONF}" - pg_ctl reload -fi - -psql -d "user=postgres connect_timeout=3" \ - --set ON_ERROR_STOP=1 \ - --set "dbName=${HEDERA_MIRROR_IMPORTER_DB_NAME}" \ - --set "dbSchema=${HEDERA_MIRROR_IMPORTER_DB_SCHEMA}" \ - --set "ownerUsername=${HEDERA_MIRROR_IMPORTER_DB_OWNER}" \ - --set "tempSchema=${HEDERA_MIRROR_IMPORTER_DB_TEMPSCHEMA}" \ - --set "readUsername=${HEDERA_MIRROR_READ}" \ - --set "readPassword=${HEDERA_MIRROR_READ_PASSWORD}" <<__SQL__ - --- Create database & owner -create database :dbName with owner :ownerUsername; - --- Create roles -create role readonly; -create role readwrite in role readonly; -create role temporary_admin in role readwrite; - --- Create users -alter user :ownerUsername with createrole; - --- Grant temp schema admin privileges -grant temporary_admin to :ownerUsername; - --- Add extensions -\connect :dbName -create extension if not exists btree_gist; -create extension if not exists pg_stat_statements; -create extension if not exists pg_trgm; - --- Create schema -\connect :dbName :ownerUsername -create schema if not exists :dbSchema authorization :ownerUsername; -grant usage on schema :dbSchema to public; -revoke create on schema :dbSchema from public; - --- Create temp table schema -create schema if not exists :tempSchema authorization temporary_admin; -grant usage on schema :tempSchema to public; -revoke create on schema :tempSchema from public; - --- Create readonly user with password and grant privileges -create user :readUsername with password :'readPassword'; -grant readonly to :readUsername; - --- Grant readonly privileges -grant connect on database :dbName to readonly; -grant select on all tables in schema :dbSchema, :tempSchema to readonly; -grant select on all sequences in schema :dbSchema, :tempSchema to readonly; -grant usage on schema :dbSchema, :tempSchema to readonly; -alter default privileges in schema :dbSchema, :tempSchema grant select on tables to readonly; -alter default privileges in schema :dbSchema, :tempSchema grant select on sequences to readonly; - --- Grant readwrite privileges -grant insert, update, delete on all tables in schema :dbSchema to readwrite; -grant usage on all sequences in schema :dbSchema to readwrite; -alter default privileges in schema :dbSchema grant insert, update, delete on tables to readwrite; -alter default privileges in schema :dbSchema grant usage on sequences to readwrite; - --- Alter search path -\connect postgres postgres -alter database :dbName set search_path = :dbSchema, public, :tempSchema; -__SQL__ - -if [[ -f "${PGHBACONF}.bak" ]]; then - mv "${PGHBACONF}.bak" "${PGHBACONF}" - pg_ctl reload -fi diff --git a/package-lock.json b/package-lock.json index 370bb8b455..15bbd83347 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1424,7 +1424,6 @@ "resolved": "https://registry.npmjs.org/@inquirer/prompts/-/prompts-8.5.2.tgz", "integrity": "sha512-IYR/3C/paEVVQYQvdDlFZVjRCJVYHHON0XXMH91KO9GSxs0TdKYWlUdvfQl2EfAHDxUaN3IBffkE/BDTh5nJ6g==", "license": "MIT", - "peer": true, "dependencies": { "@inquirer/checkbox": "^5.2.1", "@inquirer/confirm": "^6.1.1", @@ -2888,7 +2887,6 @@ "integrity": "sha512-promo4eFwuiW+TfGxhi+0x3czqTYJkG8qB17ZUJiVF10Xm7NLVRSLUsfRTU/6h1e24VvRnXCx+hG7li58lkzog==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/linkify-it": "^5", "@types/mdurl": "^2" @@ -3296,7 +3294,6 @@ "integrity": "sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -3375,7 +3372,6 @@ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -3388,7 +3384,6 @@ "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -3815,7 +3810,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.10.42", "caniuse-lite": "^1.0.30001803", @@ -4063,7 +4057,6 @@ "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=18" } @@ -4574,7 +4567,6 @@ "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", "license": "MIT", - "peer": true, "dependencies": { "ms": "^2.1.3" }, @@ -5249,7 +5241,6 @@ "integrity": "sha512-DgZS62aPLXKlnxILS/AYCoRvHaZeXceIzlXPkkGGzJWSow1aEk0lbTlxUSlyjC8jcaKxAdOnTDz+o1JFSBsyjw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -5326,7 +5317,6 @@ "integrity": "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==", "dev": true, "license": "MIT", - "peer": true, "bin": { "eslint-config-prettier": "bin/cli.js" }, @@ -7904,7 +7894,6 @@ "resolved": "https://registry.npmjs.org/jsep/-/jsep-1.4.0.tgz", "integrity": "sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==", "license": "MIT", - "peer": true, "engines": { "node": ">= 10.16.0" } @@ -8072,7 +8061,6 @@ "resolved": "https://registry.npmjs.org/listr2/-/listr2-11.0.0.tgz", "integrity": "sha512-8K88S0aSrcSXdJfiZtEy5BQMnR+TyjrCGLcgAvQs6ta0NEnIm0RJ72/Pv67Jvg07cfBhDbuN74V81lSSVYEFEw==", "license": "MIT", - "peer": true, "dependencies": { "cli-truncate": "^6.1.1", "log-update": "^8.0.0", @@ -8312,7 +8300,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "argparse": "^2.0.1", "entities": "^4.5.0", @@ -9215,7 +9202,6 @@ "integrity": "sha512-nS9xOGbw2I3cjCpxwZAEJ9xK9lmJ08vEkQvLtz4du9ZrF9UrjRpeJGiIgl2Z+Qs++pmB4ecDe48Fwsh+j+j7xA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "browser-stdout": "^1.3.1", "chokidar": "^4.0.1", @@ -10286,7 +10272,6 @@ "integrity": "sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g==", "dev": true, "license": "MIT", - "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -10362,7 +10347,6 @@ "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-8.7.1.tgz", "integrity": "sha512-agdGHrXNTv0IrYscJPDou/PlEJk1c/hBZ9o/B5NH2i/nSPtPqacNxzgwf1CebXxFMjMrZH5sqv9uQuw96aGt/A==", "license": "BSD-3-Clause", - "peer": true, "dependencies": { "long": "^5.3.2" }, @@ -12141,7 +12125,6 @@ "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.2.tgz", "integrity": "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==", "license": "MIT", - "peer": true, "dependencies": { "ansi-regex": "^6.0.1" }, @@ -12512,7 +12495,6 @@ "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -12836,7 +12818,6 @@ "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -12929,7 +12910,6 @@ "integrity": "sha512-CZ4nMxWwgu1HEEFNkeaCptra9QCtkmKdgf3sWh1rl1trIhmxLilgTV4cwcbQ4wemnT4sWQN8CaKOmdYx+g2gMA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.65.0", "@typescript-eslint/types": "8.65.0", @@ -13973,7 +13953,6 @@ "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.1.tgz", "integrity": "sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==", "license": "MIT", - "peer": true, "engines": { "node": ">=10.0.0" }, From 21b6ddb0c1e687298ea82ded4bc0d8bd640e7ae9 Mon Sep 17 00:00:00 2001 From: Jeffrey Tang Date: Thu, 30 Jul 2026 18:39:18 -0500 Subject: [PATCH 11/20] save Signed-off-by: Jeffrey Tang --- examples/state-save-and-restore/Taskfile.yml | 81 +---------- src/commands/backup-restore.ts | 11 ++ src/commands/node/tasks.ts | 25 +++- src/core/network-nodes.ts | 140 +++++++++++++++++++ test/unit/core/network-nodes.test.ts | 57 ++++++++ 5 files changed, 237 insertions(+), 77 deletions(-) diff --git a/examples/state-save-and-restore/Taskfile.yml b/examples/state-save-and-restore/Taskfile.yml index 9b95eaf5ac..89f010cf0c 100644 --- a/examples/state-save-and-restore/Taskfile.yml +++ b/examples/state-save-and-restore/Taskfile.yml @@ -222,8 +222,7 @@ tasks: - task: generate-override-network - task: install-override-network - cmd: | - # Build the directory layout expected by Solo for per-node state restore: - # /states///network--0-state.zip + # Build the directory layout expected by Solo for per-node state restore. RESTORE_INPUT_DIR="{{ .STATE_SAVE_DIR }}/restore-input" RESTORE_STATES_DIR="${RESTORE_INPUT_DIR}/states/{{ .CLUSTER_REF }}/{{ .NAMESPACE }}" RESTORE_ROUND="" @@ -241,89 +240,23 @@ tasks: fi cp "${SRC_STATE_FILE_PATH}" "${DEST_STATE_FILE_PATH}" - TMP_STATE_DIR="$(mktemp -d)" - unzip -q "${DEST_STATE_FILE_PATH}" -d "${TMP_STATE_DIR}" - - # Build a deterministic single-round archive for restore. The - # workflow now captures state from a frozen network, so prefer the - # fully signed freeze round as the exact recovery boundary. The - # download step requires this boundary for a frozen deployment. - NODE_STATE_ROOT="$(find "${TMP_STATE_DIR}/com.hedera.services.ServicesMain" -mindepth 2 -maxdepth 2 -type d | head -n 1)" - if [ -z "${NODE_STATE_ROOT}" ]; then - echo "⚠️ Could not locate saved state rounds in ${DEST_STATE_FILE_PATH}" - exit 1 - fi - - SELECTED_ROUND="" - FALLBACK_ROUND="" - # Sort on the numeric basename rather than the full temporary path; - # otherwise `sort -n` can choose a stale round unpredictably. - for round_dir in $( - find "${NODE_STATE_ROOT}" -mindepth 1 -maxdepth 1 -type d -print | - while IFS= read -r candidate_dir; do - printf '%s\t%s\n' "$(basename "${candidate_dir}")" "${candidate_dir}" - done | - sort -n -k1,1 | - cut -f2- - ); do - metadata_file="${round_dir}/stateMetadata.txt" - if [ ! -f "${metadata_file}" ]; then - continue - fi - - freeze_state="$(awk -F: '/^FREEZE_STATE:/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2}' "${metadata_file}")" - signing_weight="$(awk -F: '/^SIGNING_WEIGHT_SUM:/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2}' "${metadata_file}")" - total_weight="$(awk -F: '/^TOTAL_WEIGHT:/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2}' "${metadata_file}")" - - if [ -n "${signing_weight}" ] && [ "${signing_weight}" = "${total_weight}" ] && [ "${freeze_state}" = "true" ]; then - SELECTED_ROUND="$(basename "${round_dir}")" - break - fi - - if [ -n "${signing_weight}" ] && [ "${signing_weight}" = "${total_weight}" ] && [ "${freeze_state}" = "false" ]; then - FALLBACK_ROUND="$(basename "${round_dir}")" - fi - done - + SELECTED_ROUND="$(unzip -Z1 "${DEST_STATE_FILE_PATH}" | awk -F/ '$1 == "com.hedera.services.ServicesMain" && $4 ~ /^[0-9]+$/ {print $4}' | sort -n | tail -n 1)" if [ -z "${SELECTED_ROUND}" ]; then - SELECTED_ROUND="${FALLBACK_ROUND}" - fi - - if [ -z "${SELECTED_ROUND}" ]; then - echo "⚠️ Could not find a fully signed round in ${DEST_STATE_FILE_PATH}" + echo "⚠️ Could not locate the normalized state round in ${DEST_STATE_FILE_PATH}" exit 1 fi if [ -n "${RESTORE_ROUND}" ] && [ "${RESTORE_ROUND}" != "${SELECTED_ROUND}" ]; then - echo "⚠️ Nodes selected different restore rounds: ${RESTORE_ROUND} and ${SELECTED_ROUND}" + echo "⚠️ State archives do not share the same normalized round" exit 1 fi RESTORE_ROUND="${SELECTED_ROUND}" - - find "${NODE_STATE_ROOT}" -mindepth 1 -maxdepth 1 -type d ! -name "${SELECTED_ROUND}" -exec rm -rf {} + - - # Preserve the captured PCES directory. It is part of the exact - # state/round boundary and is required for the restored platform to - # advance from OBSERVING/CHECKING. The cleanup helper preserves this - # captured replay input while removing superseded state directories. - # Remove unrelated replay/cache directories before restore. - rm -rf "${TMP_STATE_DIR}/saved" - rm -rf "${TMP_STATE_DIR}/swirlds-tmp" - - rm -f "${DEST_STATE_FILE_PATH}" - ORIGINAL_WORKDIR="${PWD}" - cd "${TMP_STATE_DIR}" - zip -qr "${DEST_STATE_FILE_PATH}" . - cd "${ORIGINAL_WORKDIR}" - rm -rf "${TMP_STATE_DIR}" echo "Prepared state for ${node}: ${DEST_STATE_FILE_PATH} (round ${SELECTED_ROUND})" done - # Apply the endpoint transplant after the snapshot's signed round. - # If the override is staged at the snapshot round, Hedera mutates the - # roster before validating that round's state signature and reports an - # ISS. Placing it at the next round lets startup verify the snapshot - # first, then apply the fresh cluster endpoints during replay. + # Apply the endpoint override after the normalized snapshot round so + # the saved state signature is verified before the fresh endpoints are + # applied during replay. OVERRIDE_ROUND=$((RESTORE_ROUND + 1)) for node in $(echo {{ .NODE_ALIASES }} | tr ',' ' '); do kubectl exec network-${node}-0 -n {{ .NAMESPACE }} -c root-container -- \ diff --git a/src/commands/backup-restore.ts b/src/commands/backup-restore.ts index 73aadf2eb7..f1adb6e8d4 100644 --- a/src/commands/backup-restore.ts +++ b/src/commands/backup-restore.ts @@ -423,6 +423,17 @@ export class BackupRestoreCommand extends BaseCommand { const statesDirectory: string = PathEx.join(outputDirectory, 'states', clusterReference); await networkNodes.getStatesFromPod(namespace, nodeAlias, context, statesDirectory); } + for (const clusterReference of new Set(consensusNodes.map((node): string => node.cluster))) { + const clusterNodes: ConsensusNode[] = consensusNodes.filter( + (node): boolean => node.cluster === clusterReference, + ); + const statesDirectory: string = PathEx.join(outputDirectory, 'states', clusterReference); + await networkNodes.normalizeDownloadedStateArchives( + namespace, + clusterNodes.map((node): NodeAlias => node.name), + statesDirectory, + ); + } task.title = `Download Node State Files: ${consensusNodes.length} node(s) completed`; }, }, diff --git a/src/commands/node/tasks.ts b/src/commands/node/tasks.ts index ad4a4faf5b..df08a24a70 100644 --- a/src/commands/node/tasks.ts +++ b/src/commands/node/tasks.ts @@ -3196,6 +3196,8 @@ export class NodeCommandTasks { return { title: 'Get node states', task: async (context_): Promise => { + const networkNodes: NetworkNodes = container.resolve(InjectTokens.NetworkNodes); + const nodePhases: DeploymentPhase[] = []; for (const nodeAlias of context_.config.nodeAliases) { const context: string = extractContextFromConsensusNodes(nodeAlias, context_.config.consensusNodes); const nodeComponent: ConsensusNodeStateSchema = this.remoteConfig.configuration.components.getComponent( @@ -3215,10 +3217,27 @@ export class NodeCommandTasks { ); } - await container - .resolve(InjectTokens.NetworkNodes) - .getStatesFromPod(context_.config.namespace, nodeAlias, context, undefined, deploymentPhase); + nodePhases.push(deploymentPhase); + await networkNodes.getStatesFromPod( + context_.config.namespace, + nodeAlias, + context, + undefined, + deploymentPhase, + ); } + + // Normalize all downloaded archives together so every node restores from + // the same signed round instead of independently selecting a boundary. + const allNodesFrozen: boolean = nodePhases.every( + (phase: DeploymentPhase): boolean => phase === DeploymentPhase.FROZEN, + ); + await networkNodes.normalizeDownloadedStateArchives( + context_.config.namespace, + context_.config.nodeAliases, + undefined, + allNodesFrozen ? DeploymentPhase.FROZEN : undefined, + ); }, }; } diff --git a/src/core/network-nodes.ts b/src/core/network-nodes.ts index c7cfda9022..e09df3b598 100644 --- a/src/core/network-nodes.ts +++ b/src/core/network-nodes.ts @@ -4,6 +4,7 @@ import {type NamespaceName} from '../types/namespace/namespace-name.js'; import {type PodReference} from '../integration/kube/resources/pod/pod-reference.js'; import {HEDERA_HAPI_PATH, LOG_CONFIG_ZIP_SUFFIX, ROOT_CONTAINER, SOLO_LOGS_DIR} from './constants.js'; import fs from 'node:fs'; +import os from 'node:os'; import {ContainerReference} from '../integration/kube/resources/container/container-reference.js'; import * as constants from './constants.js'; import {sleep} from './helpers.js'; @@ -21,6 +22,7 @@ import {NodeStatusEnums} from './enumerations.js'; import chalk from 'chalk'; import {DeploymentPhase} from '../data/schema/model/remote/deployment-phase.js'; import {SoloErrors} from './errors/solo-errors.js'; +import {Zippy} from './zippy.js'; /** * Class to manage network nodes @@ -30,9 +32,11 @@ export class NetworkNodes { public constructor( @inject(InjectTokens.SoloLogger) private readonly logger?: SoloLogger, @inject(InjectTokens.K8Factory) private readonly k8Factory?: K8Factory, + @inject(InjectTokens.Zippy) private readonly zippy?: Zippy, ) { this.logger = patchInject(logger, InjectTokens.SoloLogger, this.constructor.name); this.k8Factory = patchInject(k8Factory, InjectTokens.K8Factory, this.constructor.name); + this.zippy = patchInject(zippy, InjectTokens.Zippy, this.constructor.name); } /** @@ -157,6 +161,142 @@ export class NetworkNodes { return await Promise.all(promises); } + /** + * Normalize downloaded state archives to one common signed round. + * + * State downloads contain every round flushed before the archive is created. + * Restore must use one round that exists and is fully signed in every node + * archive; otherwise the nodes can start from different state/PCES boundaries. + */ + public async normalizeDownloadedStateArchives( + namespace: NamespaceName, + nodeAliases: string[], + baseDirectory: string = SOLO_LOGS_DIR, + deploymentPhase?: DeploymentPhase, + ): Promise { + const archivePaths: string[] = nodeAliases.map((nodeAlias: string): string => { + const archivePath: string = PathEx.join(baseDirectory, namespace.name, `network-${nodeAlias}-0-state.zip`); + if (!fs.existsSync(archivePath)) { + throw new SoloErrors.validation.illegalArgument(`State file not found: ${archivePath}`); + } + return archivePath; + }); + + const extractedDirectories: string[] = []; + try { + const roundSets: Set[] = []; + const freezeRoundSets: Set[] = []; + + for (const archivePath of archivePaths) { + const extractedDirectory: string = fs.mkdtempSync(PathEx.join(os.tmpdir(), 'solo-state-')); + extractedDirectories.push(extractedDirectory); + this.zippy.unzip(archivePath, extractedDirectory); + + const stateRoot: string = this.findSavedStateRoundRoot(extractedDirectory); + const signedRounds: Set = new Set(); + const freezeRounds: Set = new Set(); + for (const roundDirectory of fs.readdirSync(stateRoot, {withFileTypes: true})) { + if (!roundDirectory.isDirectory() || !/^\d+$/.test(roundDirectory.name)) { + continue; + } + + const metadataPath: string = PathEx.join(stateRoot, roundDirectory.name, 'stateMetadata.txt'); + if (!fs.existsSync(metadataPath)) { + continue; + } + + const metadata: string = fs.readFileSync(metadataPath, 'utf8'); + const signingWeight: string | undefined = this.readStateMetadataValue(metadata, 'SIGNING_WEIGHT_SUM'); + const totalWeight: string | undefined = this.readStateMetadataValue(metadata, 'TOTAL_WEIGHT'); + if (!signingWeight || signingWeight !== totalWeight) { + continue; + } + + signedRounds.add(roundDirectory.name); + if (this.readStateMetadataValue(metadata, 'FREEZE_STATE') === 'true') { + freezeRounds.add(roundDirectory.name); + } + } + + roundSets.push(signedRounds); + freezeRoundSets.push(freezeRounds); + } + + const commonSignedRounds: Set = this.intersectRoundSets(roundSets); + const commonFreezeRounds: Set = this.intersectRoundSets(freezeRoundSets); + const preferFreezeRound: boolean = deploymentPhase === DeploymentPhase.FROZEN; + const selectedRound: string | undefined = this.selectHighestRound( + preferFreezeRound && commonFreezeRounds.size > 0 ? commonFreezeRounds : commonSignedRounds, + ); + + if (!selectedRound) { + throw new SoloErrors.validation.illegalArgument( + `No common fully signed state round found for nodes: ${nodeAliases.join(',')}`, + ); + } + + for (const [index, extractedDirectory] of extractedDirectories.entries()) { + const stateRoot: string = this.findSavedStateRoundRoot(extractedDirectory); + for (const roundDirectory of fs.readdirSync(stateRoot, {withFileTypes: true})) { + if ( + roundDirectory.isDirectory() && + /^\d+$/.test(roundDirectory.name) && + roundDirectory.name !== selectedRound + ) { + fs.rmSync(PathEx.join(stateRoot, roundDirectory.name), {recursive: true, force: true}); + } + } + + // These transient directories are not part of the selected state/PCES boundary. + fs.rmSync(PathEx.join(extractedDirectory, 'saved'), {recursive: true, force: true}); + fs.rmSync(PathEx.join(extractedDirectory, 'swirlds-tmp'), {recursive: true, force: true}); + await this.zippy.zip(extractedDirectory, archivePaths[index]); + } + + this.logger.showUser(`Normalized state archives to common signed round ${selectedRound}`); + return selectedRound; + } finally { + for (const extractedDirectory of extractedDirectories) { + fs.rmSync(extractedDirectory, {recursive: true, force: true}); + } + } + } + + private findSavedStateRoundRoot(extractedDirectory: string): string { + const serviceDirectory: string = PathEx.join(extractedDirectory, 'com.hedera.services.ServicesMain'); + const nodeDirectory: string | undefined = fs + .readdirSync(serviceDirectory, {withFileTypes: true}) + .find((entry): boolean => entry.isDirectory())?.name; + const realmShardDirectory: string | undefined = nodeDirectory + ? fs + .readdirSync(PathEx.join(serviceDirectory, nodeDirectory), {withFileTypes: true}) + .find((entry): boolean => entry.isDirectory())?.name + : undefined; + if (!nodeDirectory || !realmShardDirectory) { + throw new SoloErrors.validation.illegalArgument(`Could not locate saved state rounds in ${extractedDirectory}`); + } + return PathEx.join(serviceDirectory, nodeDirectory, realmShardDirectory); + } + + private readStateMetadataValue(metadata: string, key: string): string | undefined { + return metadata + .split('\n') + .find((line: string): boolean => line.startsWith(`${key}:`)) + ?.slice(key.length + 1) + .trim(); + } + + private intersectRoundSets(roundSets: Set[]): Set { + const [firstSet, ...remainingSets] = roundSets; + return new Set( + [...(firstSet ?? [])].filter((round: string): boolean => remainingSets.every(set => set.has(round))), + ); + } + + private selectHighestRound(rounds: Set): string | undefined { + return [...rounds].sort((left: string, right: string): number => Number(right) - Number(left))[0]; + } + /** * Wait for a fully signed freeze state before a freeze workflow stops the node. * A FROZEN platform status alone is not enough: stopping immediately can leave diff --git a/test/unit/core/network-nodes.test.ts b/test/unit/core/network-nodes.test.ts index 7a5ffcc238..c4064f5145 100644 --- a/test/unit/core/network-nodes.test.ts +++ b/test/unit/core/network-nodes.test.ts @@ -3,6 +3,10 @@ import {expect} from 'chai'; import {afterEach, beforeEach, describe, it} from 'mocha'; import sinon from 'sinon'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import AdmZip from 'adm-zip'; import {container} from 'tsyringe-neo'; import {resetForTest} from '../../test-container.js'; @@ -11,6 +15,7 @@ import {type NetworkNodes} from '../../../src/core/network-nodes.js'; import {PodReference} from '../../../src/integration/kube/resources/pod/pod-reference.js'; import {PodName} from '../../../src/integration/kube/resources/pod/pod-name.js'; import {NamespaceName} from '../../../src/types/namespace/namespace-name.js'; +import {DeploymentPhase} from '../../../src/data/schema/model/remote/deployment-phase.js'; describe('NetworkNodes', (): void => { let networkNodes: NetworkNodes; @@ -44,4 +49,56 @@ describe('NetworkNodes', (): void => { const status: string = await networkNodes.getNetworkNodePlatformStatusName(podReference); expect(status).to.equal('UNKNOWN'); }); + + it('should normalize node archives to one common signed round', async (): Promise => { + const temporaryDirectory: string = fs.mkdtempSync(path.join(os.tmpdir(), 'network-nodes-test-')); + const namespaceDirectory: string = path.join(temporaryDirectory, 'namespace'); + fs.mkdirSync(namespaceDirectory, {recursive: true}); + + try { + for (const nodeAlias of ['node1', 'node2']) { + const sourceDirectory: string = fs.mkdtempSync(path.join(os.tmpdir(), 'state-source-')); + for (const [round, freezeState] of [ + ['100', 'false'], + ['200', 'true'], + ]) { + const roundDirectory: string = path.join( + sourceDirectory, + 'com.hedera.services.ServicesMain', + '0', + '123', + round, + ); + fs.mkdirSync(roundDirectory, {recursive: true}); + fs.writeFileSync( + path.join(roundDirectory, 'stateMetadata.txt'), + `FREEZE_STATE: ${freezeState}\nSIGNING_WEIGHT_SUM: 3\nTOTAL_WEIGHT: 3\n`, + ); + fs.writeFileSync(path.join(roundDirectory, 'preconsensus-events.pces'), 'pces'); + } + + const archive: AdmZip = new AdmZip(); + archive.addLocalFolder(sourceDirectory); + await archive.writeZipPromise(path.join(namespaceDirectory, `network-${nodeAlias}-0-state.zip`)); + fs.rmSync(sourceDirectory, {recursive: true, force: true}); + } + + await networkNodes.normalizeDownloadedStateArchives( + NamespaceName.of('namespace'), + ['node1', 'node2'], + temporaryDirectory, + DeploymentPhase.FROZEN, + ); + + for (const nodeAlias of ['node1', 'node2']) { + const archive: AdmZip = new AdmZip(path.join(namespaceDirectory, `network-${nodeAlias}-0-state.zip`)); + const entries: string[] = archive.getEntries().map(entry => entry.entryName); + expect(entries.some(entry => entry.includes('/100/'))).to.equal(false); + expect(entries.some(entry => entry.includes('/200/'))).to.equal(true); + expect(entries.some(entry => entry.endsWith('preconsensus-events.pces'))).to.equal(true); + } + } finally { + fs.rmSync(temporaryDirectory, {recursive: true, force: true}); + } + }); }); From 1265a4a6c2ae9dd6afa161a6aaf499a908850600 Mon Sep 17 00:00:00 2001 From: Jeffrey Tang Date: Thu, 30 Jul 2026 23:27:41 -0500 Subject: [PATCH 12/20] format Signed-off-by: Jeffrey Tang --- src/core/network-nodes.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/core/network-nodes.ts b/src/core/network-nodes.ts index e09df3b598..5938fce3bd 100644 --- a/src/core/network-nodes.ts +++ b/src/core/network-nodes.ts @@ -289,12 +289,14 @@ export class NetworkNodes { private intersectRoundSets(roundSets: Set[]): Set { const [firstSet, ...remainingSets] = roundSets; return new Set( - [...(firstSet ?? [])].filter((round: string): boolean => remainingSets.every(set => set.has(round))), + [...(firstSet ?? [])].filter((round: string): boolean => + remainingSets.every((set: Set): boolean => set.has(round)), + ), ); } private selectHighestRound(rounds: Set): string | undefined { - return [...rounds].sort((left: string, right: string): number => Number(right) - Number(left))[0]; + return [...rounds].toSorted((left: string, right: string): number => Number(right) - Number(left))[0]; } /** From ec0a27f8e3cb302d41924b3af6d808e6a1938f0d Mon Sep 17 00:00:00 2001 From: Jeffrey Tang Date: Fri, 31 Jul 2026 08:05:52 -0500 Subject: [PATCH 13/20] format Signed-off-by: Jeffrey Tang --- src/core/network-nodes.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/core/network-nodes.ts b/src/core/network-nodes.ts index 5938fce3bd..99a4740ce5 100644 --- a/src/core/network-nodes.ts +++ b/src/core/network-nodes.ts @@ -296,7 +296,14 @@ export class NetworkNodes { } private selectHighestRound(rounds: Set): string | undefined { - return [...rounds].toSorted((left: string, right: string): number => Number(right) - Number(left))[0]; + let highestRound: string | undefined; + for (const currentRound of rounds) { + if (highestRound === undefined || Number(currentRound) > Number(highestRound)) { + highestRound = currentRound; + } + } + + return highestRound; } /** From cf0d832cd98a17c6c255f74a3f6447ae80cdfaf9 Mon Sep 17 00:00:00 2001 From: Jeffrey Tang Date: Fri, 31 Jul 2026 09:46:26 -0500 Subject: [PATCH 14/20] extend test time Signed-off-by: Jeffrey Tang --- .github/workflows/support/e2e-test-matrix.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/support/e2e-test-matrix.json b/.github/workflows/support/e2e-test-matrix.json index 4c78d1c749..0b037ac547 100644 --- a/.github/workflows/support/e2e-test-matrix.json +++ b/.github/workflows/support/e2e-test-matrix.json @@ -22,7 +22,7 @@ "test-script": "test-e2e-node-add-local", "coverage-subdirectory": "e2e-node-add-local", "coverage-report-name": "E2E_Node_Add_Local_Tests_Coverage_Report", - "test-timeout-minutes": 30, + "test-timeout-minutes": 40, "local-java-build": true, "install-dependencies": true, "runner": "hiero-solo-linux-large", From 972838502b0442e82df8f88f0c3ea26e9459e511 Mon Sep 17 00:00:00 2001 From: Jeffrey Tang Date: Fri, 31 Jul 2026 11:58:51 -0500 Subject: [PATCH 15/20] same Signed-off-by: Jeffrey Tang --- src/commands/node/handlers.ts | 12 +++++++----- test/unit/core/network-nodes.test.ts | 8 ++++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/commands/node/handlers.ts b/src/commands/node/handlers.ts index 7e0b2e5f06..159a6d631d 100644 --- a/src/commands/node/handlers.ts +++ b/src/commands/node/handlers.ts @@ -753,7 +753,7 @@ export class NodeCommandHandlers extends CommandHandler { await this.commandAction( argv, [ - this.tasks.initialize(argv, this.configs.logsConfigBuilder.bind(this.configs), null, true, false), + this.tasks.initialize(argv, this.configs.logsConfigBuilder.bind(this.configs), undefined, true, false), this.tasks.getNodeLogsAndConfigs(undefined, outputDirectory), this.tasks.getHelmChartValues(outputDirectory), GetSoloRemoteConfigMapTask.getTask(this.k8Factory, this.logger, outputDirectory), @@ -825,7 +825,7 @@ export class NodeCommandHandlers extends CommandHandler { this.ensureInteractiveSelectionPrompt(); const selectedFromRemote: string = (await selectPrompt({ message: 'Select deployment for diagnostics logs:', - choices: remoteDeploymentNames.map((name: string) => ({name, value: name})), + choices: remoteDeploymentNames.map((name: string): {name: string; value: string} => ({name, value: name})), })) as string; this.logger.showUser(`Using selected deployment: ${selectedFromRemote}`); return selectedFromRemote; @@ -838,7 +838,9 @@ export class NodeCommandHandlers extends CommandHandler { } if (this.resolveQuietFlag(argv)) { - const deploymentNames: string = validDeployments.map((deployment: Deployment) => deployment.name).join(', '); + const deploymentNames: string = validDeployments + .map((deployment: Deployment): string => deployment.name) + .join(', '); throw new SoloErrors.system.multipleDeploymentsFound('local', deploymentNames); } @@ -869,7 +871,7 @@ export class NodeCommandHandlers extends CommandHandler { await this.commandAction( argv, [ - this.tasks.initialize(argv, this.configs.logsConfigBuilder.bind(this.configs), null, true, false), + this.tasks.initialize(argv, this.configs.logsConfigBuilder.bind(this.configs), undefined, true, false), this.tasks.getNodeLogsAndConfigs(excludeSensitiveData, outputDirectory), ...(excludeSensitiveData ? [] : [this.tasks.getHelmChartValues(outputDirectory)]), GetSoloRemoteConfigMapTask.getTask(this.k8Factory, this.logger, outputDirectory), @@ -1077,7 +1079,7 @@ export class NodeCommandHandlers extends CommandHandler { const restoringState: boolean = typeof argv[flags.stateFile.name] === 'string' && argv[flags.stateFile.name].length > 0; - const startTasks = [ + const startTasks: SoloListrTask[] = [ this.tasks.loadConfiguration(argv, leaseWrapper, this.leaseManager), this.tasks.initialize(argv, this.configs.startConfigBuilder.bind(this.configs), leaseWrapper.lease, true, false), this.validateAllNodePhases({acceptedPhases: [DeploymentPhase.CONFIGURED]}), diff --git a/test/unit/core/network-nodes.test.ts b/test/unit/core/network-nodes.test.ts index c4064f5145..aa085ddb89 100644 --- a/test/unit/core/network-nodes.test.ts +++ b/test/unit/core/network-nodes.test.ts @@ -92,10 +92,10 @@ describe('NetworkNodes', (): void => { for (const nodeAlias of ['node1', 'node2']) { const archive: AdmZip = new AdmZip(path.join(namespaceDirectory, `network-${nodeAlias}-0-state.zip`)); - const entries: string[] = archive.getEntries().map(entry => entry.entryName); - expect(entries.some(entry => entry.includes('/100/'))).to.equal(false); - expect(entries.some(entry => entry.includes('/200/'))).to.equal(true); - expect(entries.some(entry => entry.endsWith('preconsensus-events.pces'))).to.equal(true); + const entries: string[] = archive.getEntries().map((entry): string => entry.entryName); + expect(entries.some((entry: string): boolean => entry.includes('/100/'))).to.equal(false); + expect(entries.some((entry: string): boolean => entry.includes('/200/'))).to.equal(true); + expect(entries.some((entry: string): boolean => entry.endsWith('preconsensus-events.pces'))).to.equal(true); } } finally { fs.rmSync(temporaryDirectory, {recursive: true, force: true}); From 6465468d8c10636bb10b9e6d4d32c09f5794d2c5 Mon Sep 17 00:00:00 2001 From: Jeffrey Tang Date: Sat, 15 Aug 2026 11:17:01 -0500 Subject: [PATCH 16/20] fix Signed-off-by: Jeffrey Tang --- examples/state-save-and-restore/README.md | 213 ++++++++++--------- examples/state-save-and-restore/Taskfile.yml | 1 + src/commands/node/handlers.ts | 3 - src/core/constants.ts | 7 + src/core/network-nodes.ts | 63 +++--- 5 files changed, 149 insertions(+), 138 deletions(-) diff --git a/examples/state-save-and-restore/README.md b/examples/state-save-and-restore/README.md index ca548d7906..5aca094db8 100644 --- a/examples/state-save-and-restore/README.md +++ b/examples/state-save-and-restore/README.md @@ -1,16 +1,20 @@ # State Save and Restore Example -This example demonstrates how to save network state from a running Solo network, recreate a new network, and load the saved state with a mirror node using an external PostgreSQL database. +This example demonstrates how to save signed consensus-node state from a running Solo network, recreate the +cluster from scratch, and restart consensus nodes from the saved state — including rewriting gossip/service +endpoints so the restored roster works against the fresh cluster's Kubernetes service IPs. ## What it does -* Creates an initial Solo network with consensus nodes and mirror node -* Uses an external PostgreSQL database for the mirror node +* Creates an initial Solo network with consensus nodes and a block node * Runs transactions to generate state -* Downloads and saves the network state and database dump -* Destroys the initial network -* Creates a new network with the same configuration -* Restores the saved state and database to the new network +* Freezes the network and downloads signed state from each consensus node +* Saves the original network/roster definition and consensus key material needed for restore +* Destroys the network and the Kind cluster +* Recreates the cluster and consensus network from scratch +* Generates `override-network.json` from the saved roster and the fresh cluster's current service IPs +* Restarts the nodes from the saved state with the endpoint override applied +* Verifies both restored nodes reach `FREEZE_COMPLETE` with no `Invalid State Signature` ## Getting This Example @@ -32,24 +36,25 @@ Browse the source code and configuration files for this example in the [GitHub r * [kubectl](https://kubernetes.io/docs/tasks/tools/) - Kubernetes CLI * [Node.js](https://nodejs.org/) - JavaScript runtime * [Task](https://taskfile.dev/) - Task runner -* [Helm](https://helm.sh/) - Kubernetes package manager (for external database option) +* `jq` - used to extract key material from Kubernetes secrets during save/restore ## Quick Start ### Run Complete Workflow (One Command) ```bash -task # Run entire workflow: setup → save → restore +task # Run entire workflow: setup → freeze → save → restore task destroy # Cleanup when done ``` ### Step-by-Step Workflow ```bash -task setup # 1. Deploy network with external database (5-10 min) -task save-state # 2. Save state and database (2-5 min) -task restore # 3. Recreate and restore (3-5 min) -task destroy # 5. Cleanup +task setup # 1. Deploy consensus network and block node (5-10 min) +task stop-network # 2. Freeze the network so saved state is signed +task save-state # 3. Save state, roster, and key material (2-5 min) +task restore # 4. Recreate cluster and restore state (3-5 min) +task destroy # 5. Cleanup ``` ## Usage @@ -63,26 +68,27 @@ task setup This will: * Create a Kind cluster -* Deploy PostgreSQL database -* Initialize Solo -* Deploy consensus network with 3 nodes -* Deploy mirror node connected to external database +* Initialize Solo and connect the cluster reference +* Deploy a block node (so the consensus node does not require MinIO-backed stream storage) +* Deploy a consensus network with the configured number of nodes * Run sample transactions to generate state -### 2. Save Network State and Database +### 2. Freeze and Save Network State ```sh +task stop-network task save-state ``` This will: -* Download state from all consensus nodes -* Export PostgreSQL database dump -* Save both to `./saved-states/` directory -* Display saved state information +* Freeze the network so the saved state is fully signed +* Download signed state from all consensus nodes +* Save the original roster/network definition (used later to build `override-network.json`) +* Save consensus gossip and TLS key material from Kubernetes secrets +* Save everything to `./saved-states/` -### 3. Restore Network and Database +### 3. Restore Network ```sh task restore @@ -90,14 +96,14 @@ task restore This will: -* Stop and destroy the mirror node and consensus network -* Recreate PostgreSQL database -* Import database dump -* Recreate the consensus network with the original deployment metadata and key material -* Upload saved state to new nodes -* Start nodes with restored state -* Reconnect mirror node to database -* Verify the restored state +* Destroy the block node and consensus network, then delete the Kind cluster +* Recreate the Kind cluster and reconnect the cluster reference/deployment +* Restore the saved consensus key material into the Solo cache +* Redeploy the block node and a fresh consensus network (same deployment metadata and keys) +* Generate `override-network.json` from the saved roster and the fresh cluster's current service IPs, and copy it + into each consensus node pod +* Start all nodes together from the saved state with `solo consensus node start --state-file` +* Verify both nodes reach `FREEZE_COMPLETE` with no invalid state signature ### 4. Cleanup @@ -105,16 +111,17 @@ This will: task destroy ``` -This will delete the Kind cluster and clean up all resources. +This will destroy the network resources, delete the Kind cluster, and clean up saved state files. ## Available Tasks -* `default` (or just `task`) - Run complete workflow: setup → save-state → restore -* `setup` - Deploy initial network with external PostgreSQL database -* `save-state` - Download consensus node state and export database -* `restore` - Recreate network and restore state with database -* `verify-state` - Verify restored state matches original -* `destroy` - Delete cluster and clean up all resources +* `default` (or just `task`) - Run complete workflow: setup → stop-network → save-state → restore +* `setup` - Deploy initial consensus network and block node +* `stop-network` - Freeze the consensus network +* `save-state` - Download consensus node state and save restore metadata (roster + keys) +* `restore` - Recreate the cluster and restore state +* `verify-state` - Verify restored nodes reached `FREEZE_COMPLETE` with no invalid state signature +* `destroy` - Destroy network resources, delete the cluster, and clean up all resources * `clean-state` - Remove saved state files ## Customization @@ -124,7 +131,6 @@ You can adjust settings by editing the `vars:` section in `Taskfile.yml`: * `NETWORK_SIZE` - Number of consensus nodes (default: 2) * `NODE_ALIASES` - Node identifiers (default: node1,node2) * `STATE_SAVE_DIR` - Directory to save state files (default: ./saved-states) -* `POSTGRES_PASSWORD` - PostgreSQL password for external database ## State Files @@ -132,79 +138,77 @@ Saved state files are stored in `./saved-states/` with the following structure: ``` saved-states/ -├── state-restore-namespace/ -│ ├── network-node1-0-state.zip -│ └── network-node2-0-state.zip -├── mirror-passwords-secret.json -└── database-dump.sql # PostgreSQL database export +├── original-network.json # Saved roster/network definition +├── override-network.json # Generated during restore from original-network.json +├── current-service-endpoints.json # Generated during restore from `kubectl get service` +├── keys/ # Saved consensus gossip and TLS key material +├── restore-input/ +│ └── states///network--0-state.zip +└── state-restore-namespace/ + ├── network-node1-0-state.zip + └── network-node2-0-state.zip ``` **Notes:** * State files are named using the pod naming convention: `network--0-state.zip` -* During save: All node state files are downloaded -* During restore: A per-node restore input directory is built and passed to `solo consensus node start --state-file` -* Mirror database credentials are preserved in `mirror-passwords-secret.json` and restored before mirror redeploy +* During save: state is downloaded from each frozen consensus node, along with the original roster JSON and + consensus key material +* During restore: a per-node restore input directory is built and passed to `solo consensus node start --state-file` The example also includes: ``` scripts/ -└── init.sh # Database initialization script +└── generate-override-network.mjs # Rewrites gossip/service endpoints in the saved roster ``` -The `init.sh` script sets up the PostgreSQL database with: - -* mirror\_node database -* Required schemas (public, temporary) -* Roles and users (postgres, readonlyuser) -* PostgreSQL extensions (btree\_gist, pg\_stat\_statements, pg\_trgm) -* Proper permissions and grants +`generate-override-network.mjs` reads the saved `original-network.json` and the fresh cluster's current +`kubectl get service` output, rewrites each node's gossip/service endpoint IP address to the fresh cluster's +service `clusterIP`, and writes the result to `override-network.json`. ## How It Works ### State Saving Process -1. **Download State**: Uses `solo consensus state download` to download signed state from each consensus node to `~/.solo/logs//` -2. **Copy State Files**: Copies state files from `~/.solo/logs//` to `./saved-states/` directory -3. **Export Database**: Uses `pg_dump` with `--clean --if-exists` flags to export the complete database including schema and data -4. **Save Mirror Credentials**: Exports the `mirror-passwords` secret so the restored mirror deployment reuses the original DB role passwords +1. **Freeze Network**: Uses `solo consensus network freeze` so the saved state is fully signed +2. **Download State**: Uses `solo consensus state download` to download signed state from each consensus node to + `~/.solo/logs//` +3. **Copy State Files**: Copies state files from `~/.solo/logs//` to `./saved-states/` directory +4. **Save Network Definition**: Exports the roster/network JSON from a node pod to `original-network.json`, used + later to generate `override-network.json` +5. **Save Key Material**: Exports consensus gossip and TLS key material from Kubernetes secrets to `./saved-states/keys/` ### State Restoration Process -1. **Database Recreation**: Deploys fresh PostgreSQL and runs `init.sh` to create database structure (database, schemas, roles, users, extensions) -2. **Fresh Network Deployment**: Reuses the original deployment metadata and consensus key material, redeploys the consensus network, and runs node setup for the new pods -3. **Restore Mirror Credentials**: Restores the saved `mirror-passwords` secret so mirror components reuse the original database passwords -4. **Database Restore**: Reconciles mirror database roles from the saved secret, then imports the database dump -5. **Restore Input Build**: Builds `./saved-states/restore-input/states///` and copies each node's state zip +1. **Cluster Recreation**: Destroys the block node, consensus network, and Kind cluster, then recreates the + cluster and reconnects the cluster reference/deployment +2. **Key Restoration**: Restores the saved consensus key material into the Solo cache so key generation is skipped +3. **Fresh Network Deployment**: Redeploys the block node and consensus network with the original deployment + metadata, then runs node setup for the new pods +4. **Override Network Generation**: Builds `override-network.json` from the saved roster and the fresh cluster's + current service IPs, and copies it into each consensus node pod's config directory +5. **Restore Input Build**: Builds `./saved-states/restore-input/states///` and copies each + node's saved state zip 6. **State Upload and Start**: Starts all nodes together with `solo consensus node start --state-file ./saved-states/restore-input` - * State files are extracted to `data/saved/` - * Cleanup: Only the latest/biggest round is kept, older rounds are automatically deleted to save disk space - * Node ID Renaming: Directory paths containing node IDs are automatically renamed to match each target node -7. **Mirror Node**: Redeploys the mirror node connected to the restored database -8. **Verification**: Checks that restored state matches original +7. **Verification**: Checks that both restored nodes report platform status `FREEZE_COMPLETE` and that no + `Invalid State Signature` was logged ## Notes * State files can be large (several GB per node) depending on network activity * Ensure sufficient disk space in `./saved-states/` directory -* External PostgreSQL database provides data persistence and queryability -* State restoration maintains transaction history and account balances -* Mirror node will resume from the restored state point -* **Per-node State Restore**: Uses each node's own state zip and starts all nodes together on a freshly redeployed network with the original consensus keys -* Database dump includes all mirror node data (transactions, accounts, etc.) +* The network must be frozen before saving state, otherwise the state files may change while being read +* **Per-node State Restore**: Uses each node's own state zip and starts all nodes together on a freshly recreated + cluster, with `override-network.json` remapping gossip/service endpoints to the new cluster's service IPs +* Restored nodes come up in `FROZEN`/`FREEZE_COMPLETE` phase rather than `ACTIVE`, since they resume from a frozen + state rather than starting fresh ### View Logs ```bash # Consensus node logs kubectl logs -n state-restore-namespace network-node1-0 -f - -# Mirror node logs -kubectl logs -n state-restore-namespace mirror-node- -f - -# Database logs -kubectl logs -n database state-restore-postgresql-0 -f ``` ### Manual State Operations @@ -241,7 +245,8 @@ Ensure you have sufficient disk space in `./saved-states/` directory. ### Save State at Specific Time -Run `task save-state` at any point after running transactions. The state captures the network at that moment. +Run `task stop-network` then `task save-state` at any point after running transactions. The state captures the +network at that moment. ### Restore to Different Cluster @@ -269,7 +274,7 @@ task restore **State download fails**: -* Ensure nodes are running and healthy +* Ensure the network was frozen with `task stop-network` before downloading * Check pod logs: `kubectl logs -n ` * Increase timeout or download nodes sequentially @@ -277,14 +282,13 @@ task restore * Verify state files exist in `./saved-states/` * Check file permissions -* Ensure network configuration matches original +* Ensure `NETWORK_SIZE`/`NODE_ALIASES` match what was used to save state * Check state file integrity -**Database connection fails**: +**Invalid state signature after restore**: -* Verify PostgreSQL pod is ready -* Check credentials in Taskfile.yml -* Review PostgreSQL logs +* Confirm `override-network.json` was generated and copied into each pod (see `deploy-network-with-state` output) +* Confirm the saved consensus keys were restored into the Solo cache before redeploying the network **Out of disk space**: @@ -302,9 +306,6 @@ kubectl describe pod -n state-restore-namespace # Get pod logs kubectl logs -n state-restore-namespace - -# Access database shell -kubectl exec -it state-restore-postgresql-0 -n database -- psql -U postgres -d mirror_node ``` ## Example Output @@ -313,27 +314,29 @@ kubectl exec -it state-restore-postgresql-0 -n database -- psql -U postgres -d m $ task setup ✓ Create Kind cluster ✓ Initialize Solo -✓ Deploy consensus network (3 nodes) -✓ Deploy mirror node +✓ Deploy block node +✓ Deploy consensus network (2 nodes) ✓ Generate sample transactions -Network ready at: http://localhost:5551 + +$ task stop-network +✓ Network frozen $ task save-state -✓ Downloading state from node1... (2.3 GB) -✓ Downloading state from node2... (2.3 GB) -✓ Downloading state from node3... (2.3 GB) -✓ Saving metadata +✓ Saved state for node1 (network-node1-0-state.zip) +✓ Saved state for node2 (network-node2-0-state.zip) +✓ Source network JSON exported +✓ Saved consensus key material State saved to: ./saved-states/ $ task restore -✓ Stopping existing network -✓ Creating new network -✓ Uploading state to node1... -✓ Uploading state to node2... -✓ Uploading state to node3... -✓ Starting nodes with restored state -✓ Verifying restoration -State restored successfully! +✓ Network resources destroyed +✓ Cluster destroyed +✓ Restored consensus key material +✓ Block node and consensus network redeployed +✓ Generated override-network.json +✓ override-network.json copied to each consensus node pod +✓ Nodes started with restored state +✓ State verification complete - both restored nodes are FREEZE_COMPLETE with no ISS ``` *** diff --git a/examples/state-save-and-restore/Taskfile.yml b/examples/state-save-and-restore/Taskfile.yml index 89f010cf0c..0de14d87fa 100644 --- a/examples/state-save-and-restore/Taskfile.yml +++ b/examples/state-save-and-restore/Taskfile.yml @@ -377,6 +377,7 @@ tasks: cmds: - cmd: rm -rf {{ .STATE_SAVE_DIR }} - cmd: echo "✅ Saved state files removed" + deploy-block-node: desc: Deploy block node so CN v0.74 does not require MinIO-backed stream storage cmds: diff --git a/src/commands/node/handlers.ts b/src/commands/node/handlers.ts index cadbdf95ff..7424371852 100644 --- a/src/commands/node/handlers.ts +++ b/src/commands/node/handlers.ts @@ -1194,8 +1194,6 @@ export class NodeCommandHandlers extends CommandHandler { ); } else { startTasks.push( - this.tasks.checkNodesAndProxiesAreActive('nodeAliases'), - this.tasks.enablePortForwarding(true), this.tasks.checkNodesAndProxiesAreActive('nodeAliases'), this.tasks.enablePortForwarding(true), this.tasks.emitNodeStartedEvent(), @@ -1203,7 +1201,6 @@ export class NodeCommandHandlers extends CommandHandler { this.tasks.setGrpcWebEndpoint('nodeAliases', NodeSubcommandType.START), this.changeAllNodePhases(DeploymentPhase.STARTED, LedgerPhase.INITIALIZED), this.tasks.addNodeStakes(), - this.tasks.emitNodeStartedEvent(), ); } diff --git a/src/core/constants.ts b/src/core/constants.ts index af10f8e6f4..5367ce63cc 100644 --- a/src/core/constants.ts +++ b/src/core/constants.ts @@ -501,6 +501,13 @@ export const NETWORK_NODE_GRPC_READINESS_DELAY: number = export const NETWORK_NODE_GRPC_READINESS_REQUIRED_SUCCESSES: number = +getEnvironmentVariable('NETWORK_NODE_GRPC_READINESS_REQUIRED_SUCCESSES') || 3; +// Saved State Stability Checks +export const STATE_DOWNLOAD_STABLE_MAX_ATTEMPTS: number = + +getEnvironmentVariable('STATE_DOWNLOAD_STABLE_MAX_ATTEMPTS') || 180; +export const STATE_DOWNLOAD_STABLE_DELAY: number = +getEnvironmentVariable('STATE_DOWNLOAD_STABLE_DELAY') || 2000; +export const STATE_DOWNLOAD_STABLE_POLLS_REQUIRED: number = + +getEnvironmentVariable('STATE_DOWNLOAD_STABLE_POLLS_REQUIRED') || 3; + export const NETWORK_PROXY_MAX_ATTEMPTS: number = +getEnvironmentVariable('NETWORK_PROXY_MAX_ATTEMPTS') || 300; export const NETWORK_PROXY_DELAY: number = +getEnvironmentVariable('NETWORK_PROXY_DELAY') || 2000; export const PODS_READY_MAX_ATTEMPTS: number = +getEnvironmentVariable('PODS_READY_MAX_ATTEMPTS') || 300; diff --git a/src/core/network-nodes.ts b/src/core/network-nodes.ts index 99a4740ce5..8ef46a38c1 100644 --- a/src/core/network-nodes.ts +++ b/src/core/network-nodes.ts @@ -370,9 +370,9 @@ export class NetworkNodes { podName: string, requireFreezeRound: boolean, ): Promise { - const maxAttempts: number = 180; - const stablePollsRequired: number = 3; - const pollDelay: Duration = Duration.ofSeconds(2); + const maxAttempts: number = constants.STATE_DOWNLOAD_STABLE_MAX_ATTEMPTS; + const stablePollsRequired: number = constants.STATE_DOWNLOAD_STABLE_POLLS_REQUIRED; + const pollDelay: Duration = Duration.ofMillis(constants.STATE_DOWNLOAD_STABLE_DELAY); let lastFingerprint: string | undefined; let stablePolls: number = 0; const scriptName: string = 'wait-for-stable-saved-state.sh'; @@ -389,7 +389,8 @@ export class NetworkNodes { `sync ${HEDERA_HAPI_PATH} && chown hedera:hedera ${destinationPath} && chmod 0755 ${destinationPath}`, ]); - for (let attempt: number = 1; attempt <= maxAttempts; attempt++) { + let attempt: number = 0; + while (attempt < maxAttempts) { try { const rawOutput: string = await container.execContainer([ 'bash', @@ -402,32 +403,30 @@ export class NetworkNodes { const output: string = rawOutput.trim(); const [fingerprint, round, kind] = output.split(/\s+/); - if (!fingerprint || !round || !kind) { - throw new SoloErrors.validation.illegalArgument(`Missing saved state fingerprint for pod ${podName}`); - } - - stablePolls = fingerprint === lastFingerprint ? stablePolls + 1 : 1; - lastFingerprint = fingerprint; - - this.logger.debug( - `[state-download] ${podName}: round ${round} (${kind}) stable poll ${stablePolls}/${stablePollsRequired}`, - ); + if (fingerprint && round && kind) { + stablePolls = fingerprint === lastFingerprint ? stablePolls + 1 : 1; + lastFingerprint = fingerprint; - if (kind === 'frozen-fallback') { - // A frozen deployment can expose the FROZEN platform status before a - // freeze-marked round becomes fully signed on disk. In that case, - // export the newest fully signed non-freeze round instead of waiting - // indefinitely for a freeze round that may never materialize. - this.logger.warn( - `[state-download] ${podName}: deployment is FROZEN but no fully signed freeze round exists on disk yet; using the newest fully signed non-freeze round`, + this.logger.debug( + `[state-download] ${podName}: round ${round} (${kind}) stable poll ${stablePolls}/${stablePollsRequired}`, ); - } - if (stablePolls >= stablePollsRequired) { - // One final sync narrows the gap between the successful probe and the - // subsequent zip/copy operation. - await container.execContainer('sync'); - return; + if (kind === 'frozen-fallback') { + // A frozen deployment can expose the FROZEN platform status before a + // freeze-marked round becomes fully signed on disk. In that case, + // export the newest fully signed non-freeze round instead of waiting + // indefinitely for a freeze round that may never materialize. + this.logger.warn( + `[state-download] ${podName}: deployment is FROZEN but no fully signed freeze round exists on disk yet; using the newest fully signed non-freeze round`, + ); + } + + if (stablePolls >= stablePollsRequired) { + // One final sync narrows the gap between the successful probe and the + // subsequent zip/copy operation. + await container.execContainer('sync'); + return; + } } } catch (error) { // The script exits non-zero until a qualifying signed round exists or the @@ -435,13 +434,17 @@ export class NetworkNodes { this.logger.debug(`[state-download] ${podName}: saved state not stable yet`, error); } + attempt++; await sleep(pollDelay); } - throw new SoloErrors.validation.illegalArgument( + throw new SoloErrors.component.nodeNotReady( + podName, requireFreezeRound - ? `Timed out waiting for a stable fully signed saved state on pod ${podName}. The deployment is frozen, but no signed round became stable on disk.` - : `Timed out waiting for a stable fully signed saved state on pod ${podName}. Stop or freeze the node and retry state download.`, + ? 'showing a stable, fully signed freeze state on disk (the deployment is frozen, but no signed round has become stable yet)' + : 'showing a stable, fully signed saved state on disk (stop or freeze the node and retry state download)', + attempt, + maxAttempts, ); } From 785626ee3d04588d8dbfbe7d7cfc45eedf6db23c Mon Sep 17 00:00:00 2001 From: Jeffrey Tang Date: Mon, 17 Aug 2026 20:54:32 -0500 Subject: [PATCH 17/20] fix: prevent restored consensus nodes from replaying into a stale freeze config ops backup never passed each node's DeploymentPhase to state download, so the "prefer a signed freeze round" logic was dead code and always accepted the newest signed non-frozen round instead. That round's preconsensus-events archive is not bounded to it, so restoring it let the platform replay straight into a freeze transaction ordered moments later, permanently pinning restored nodes in FREEZE_COMPLETE. Fixes: - Propagate DeploymentPhase from remote config into getStatesFromPod and normalizeDownloadedStateArchives so freeze-round preference works. - Wait longer (STATE_DOWNLOAD_FREEZE_GRACE_ATTEMPTS) before falling back to a non-frozen round. - Trim preconsensus-events files to the selected round's birth round (PcesTrimmer) instead of replaying events past it, while preserving enough history for restored nodes to still create new events. Verified end-to-end against the multicluster-backup-restore example: nodes reach ACTIVE after restore, pre-restore data is intact, and new post-restore transactions are ingested by the mirror node. Signed-off-by: Jeffrey Tang --- src/commands/backup-restore.ts | 22 +++- src/core/constants.ts | 6 + src/core/network-nodes.ts | 51 ++++++-- src/core/pces-trimmer.ts | 222 +++++++++++++++++++++++++++++++++ 4 files changed, 290 insertions(+), 11 deletions(-) create mode 100644 src/core/pces-trimmer.ts diff --git a/src/commands/backup-restore.ts b/src/commands/backup-restore.ts index b65e677b46..ac3bb54745 100644 --- a/src/commands/backup-restore.ts +++ b/src/commands/backup-restore.ts @@ -38,6 +38,8 @@ import {RemoteConfig} from '../business/runtime-state/config/remote/remote-confi import {type DeploymentStateSchema} from '../data/schema/model/remote/deployment-state-schema.js'; import {type BlockNodeStateSchema} from '../data/schema/model/remote/state/block-node-state-schema.js'; import {type ConsensusNodeStateSchema} from '../data/schema/model/remote/state/consensus-node-state-schema.js'; +import {DeploymentPhase} from '../data/schema/model/remote/deployment-phase.js'; +import {ComponentTypes} from '../core/config/remote/enumerations/component-types.js'; import {type MirrorNodeStateSchema} from '../data/schema/model/remote/state/mirror-node-state-schema.js'; import {type RelayNodeStateSchema} from '../data/schema/model/remote/state/relay-node-state-schema.js'; import {type DeploymentName} from '../types/index.js'; @@ -416,22 +418,38 @@ export class BackupRestoreCommand extends BaseCommand { title: 'Download Node State Files', task: async (_, task): Promise => { const networkNodes: NetworkNodes = container.resolve(InjectTokens.NetworkNodes); + const nodePhases: Map = new Map(); for (const node of consensusNodes) { const nodeAlias: NodeAlias = node.name; const context: Context = extractContextFromConsensusNodes(nodeAlias, consensusNodes); const clusterReference: string = node.cluster; // Get cluster ref from node metadata const statesDirectory: string = PathEx.join(outputDirectory, 'states', clusterReference); - await networkNodes.getStatesFromPod(namespace, nodeAlias, context, statesDirectory); + const nodeComponent: ConsensusNodeStateSchema = this.remoteConfig.configuration.components.getComponent( + ComponentTypes.ConsensusNode, + Templates.renderComponentIdFromNodeAlias(nodeAlias), + ); + const deploymentPhase: DeploymentPhase = nodeComponent.metadata.phase; + nodePhases.set(nodeAlias, deploymentPhase); + // Passing the phase lets state download prefer a fully signed freeze round over a + // stale non-freeze round when the network was just frozen (see "Freeze network" + // above); otherwise the archive's preconsensus-events stream can outrun the selected + // round and replay straight back into the freeze when the backup is later restored. + await networkNodes.getStatesFromPod(namespace, nodeAlias, context, statesDirectory, deploymentPhase); } for (const clusterReference of new Set(consensusNodes.map((node): string => node.cluster))) { const clusterNodes: ConsensusNode[] = consensusNodes.filter( (node): boolean => node.cluster === clusterReference, ); const statesDirectory: string = PathEx.join(outputDirectory, 'states', clusterReference); + const clusterNodeAliases: NodeAlias[] = clusterNodes.map((node): NodeAlias => node.name); + const allNodesFrozen: boolean = clusterNodeAliases.every( + (nodeAlias: NodeAlias): boolean => nodePhases.get(nodeAlias) === DeploymentPhase.FROZEN, + ); await networkNodes.normalizeDownloadedStateArchives( namespace, - clusterNodes.map((node): NodeAlias => node.name), + clusterNodeAliases, statesDirectory, + allNodesFrozen ? DeploymentPhase.FROZEN : undefined, ); } task.title = `Download Node State Files: ${consensusNodes.length} node(s) completed`; diff --git a/src/core/constants.ts b/src/core/constants.ts index 5367ce63cc..da5d47d359 100644 --- a/src/core/constants.ts +++ b/src/core/constants.ts @@ -507,6 +507,12 @@ export const STATE_DOWNLOAD_STABLE_MAX_ATTEMPTS: number = export const STATE_DOWNLOAD_STABLE_DELAY: number = +getEnvironmentVariable('STATE_DOWNLOAD_STABLE_DELAY') || 2000; export const STATE_DOWNLOAD_STABLE_POLLS_REQUIRED: number = +getEnvironmentVariable('STATE_DOWNLOAD_STABLE_POLLS_REQUIRED') || 3; +// After a freeze request, the platform can report FROZEN status before the freeze round is +// fully signed on disk. A non-freeze round may look "stable" first; keep polling for this many +// additional attempts so the freeze round (once signed) is preferred over the stale fallback, +// whose bundled preconsensus-events would otherwise replay straight back into the freeze on restart. +export const STATE_DOWNLOAD_FREEZE_GRACE_ATTEMPTS: number = + +getEnvironmentVariable('STATE_DOWNLOAD_FREEZE_GRACE_ATTEMPTS') || 60; export const NETWORK_PROXY_MAX_ATTEMPTS: number = +getEnvironmentVariable('NETWORK_PROXY_MAX_ATTEMPTS') || 300; export const NETWORK_PROXY_DELAY: number = +getEnvironmentVariable('NETWORK_PROXY_DELAY') || 2000; diff --git a/src/core/network-nodes.ts b/src/core/network-nodes.ts index 8ef46a38c1..42fa38e4c0 100644 --- a/src/core/network-nodes.ts +++ b/src/core/network-nodes.ts @@ -23,6 +23,7 @@ import chalk from 'chalk'; import {DeploymentPhase} from '../data/schema/model/remote/deployment-phase.js'; import {SoloErrors} from './errors/solo-errors.js'; import {Zippy} from './zippy.js'; +import {PcesTrimmer} from './pces-trimmer.js'; /** * Class to manage network nodes @@ -250,6 +251,19 @@ export class NetworkNodes { // These transient directories are not part of the selected state/PCES boundary. fs.rmSync(PathEx.join(extractedDirectory, 'saved'), {recursive: true, force: true}); fs.rmSync(PathEx.join(extractedDirectory, 'swirlds-tmp'), {recursive: true, force: true}); + + // The top-level preconsensus-events stream is what the platform replays on the next + // restart, and it is not naturally bounded to the selected round: it can contain later + // events (e.g. a freeze transaction ordered moments after this snapshot was taken) that + // the selected round's own state does not yet reflect. Trim those out so replay cannot + // cross back into that later boundary, while keeping every event up to the selected + // round so the restored node still has other-parent candidates to build new events on + // (removing the whole stream instead would leave every node with no known events at + // all, which the platform only permits at true genesis). + PcesTrimmer.trimDirectoryToBirthRound( + PathEx.join(extractedDirectory, 'preconsensus-events'), + Number(selectedRound), + ); await this.zippy.zip(extractedDirectory, archivePaths[index]); } @@ -372,9 +386,11 @@ export class NetworkNodes { ): Promise { const maxAttempts: number = constants.STATE_DOWNLOAD_STABLE_MAX_ATTEMPTS; const stablePollsRequired: number = constants.STATE_DOWNLOAD_STABLE_POLLS_REQUIRED; + const freezeGraceAttempts: number = constants.STATE_DOWNLOAD_FREEZE_GRACE_ATTEMPTS; const pollDelay: Duration = Duration.ofMillis(constants.STATE_DOWNLOAD_STABLE_DELAY); let lastFingerprint: string | undefined; let stablePolls: number = 0; + let fallbackStableSinceAttempt: number | undefined; const scriptName: string = 'wait-for-stable-saved-state.sh'; const sourcePath: string = PathEx.joinWithRealPath(constants.RESOURCES_DIR, scriptName); const destinationPath: string = `${HEDERA_HAPI_PATH}/${scriptName}`; @@ -411,17 +427,34 @@ export class NetworkNodes { `[state-download] ${podName}: round ${round} (${kind}) stable poll ${stablePolls}/${stablePollsRequired}`, ); + const isStable: boolean = stablePolls >= stablePollsRequired; + if (kind === 'frozen-fallback') { // A frozen deployment can expose the FROZEN platform status before a - // freeze-marked round becomes fully signed on disk. In that case, - // export the newest fully signed non-freeze round instead of waiting - // indefinitely for a freeze round that may never materialize. - this.logger.warn( - `[state-download] ${podName}: deployment is FROZEN but no fully signed freeze round exists on disk yet; using the newest fully signed non-freeze round`, - ); - } - - if (stablePolls >= stablePollsRequired) { + // freeze-marked round becomes fully signed on disk. The fallback round's + // bundled preconsensus-events stream already extends past its own round + // boundary, so restoring it verbatim would replay straight back into the + // freeze on restart. Give the freeze round a grace period to finish + // signing before accepting the stale fallback. + if (isStable && fallbackStableSinceAttempt === undefined) { + fallbackStableSinceAttempt = attempt; + this.logger.warn( + `[state-download] ${podName}: deployment is FROZEN but no fully signed freeze round exists on disk yet; ` + + `waiting up to ${freezeGraceAttempts} more attempt(s) for it before falling back to the newest fully signed non-freeze round`, + ); + } + + const gracePeriodElapsed: boolean = + fallbackStableSinceAttempt !== undefined && attempt - fallbackStableSinceAttempt >= freezeGraceAttempts; + + if (isStable && gracePeriodElapsed) { + this.logger.warn( + `[state-download] ${podName}: freeze round still not fully signed after grace period; using the newest fully signed non-freeze round`, + ); + await container.execContainer('sync'); + return; + } + } else if (isStable) { // One final sync narrows the gap between the successful probe and the // subsequent zip/copy operation. await container.execContainer('sync'); diff --git a/src/core/pces-trimmer.ts b/src/core/pces-trimmer.ts new file mode 100644 index 0000000000..628464c311 --- /dev/null +++ b/src/core/pces-trimmer.ts @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: Apache-2.0 + +import fs from 'node:fs'; +import path from 'node:path'; +import {SoloErrors} from './errors/solo-errors.js'; + +const PCES_PROTOBUF_EVENTS_VERSION: number = 2; +const GOSSIP_EVENT_CORE_FIELD_NUMBER: number = 1; +const EVENT_CORE_BIRTH_ROUND_FIELD_NUMBER: number = 2; +const WIRE_TYPE_VARINT: number = 0; +const WIRE_TYPE_FIXED64: number = 1; +const WIRE_TYPE_LENGTH_DELIMITED: number = 2; +const WIRE_TYPE_FIXED32: number = 5; + +interface VarintRead { + value: bigint; + nextOffset: number; +} + +/** + * Trims preconsensus event stream (PCES) files down to a maximum birth round. + * + * A restored state snapshot at round N is already a complete, fully signed state; any + * preconsensus event with a birth round greater than N was created after that snapshot and + * should not be replayed when the snapshot is restored elsewhere (for example, replaying a + * freeze transaction ordered moments after the snapshot was taken would immediately re-freeze + * the restored node). This trims strictly at a record boundary, so no footer/checksum needs to + * be regenerated (see the PCES file format used by hiero-consensus-node's PcesFileIterator, + * which already tolerates a file ending mid-record the same way it tolerates an abrupt + * shutdown). + */ +export class PcesTrimmer { + /** + * Removes every preconsensus event with a birth round greater than maximumBirthRound from + * all .pces files found recursively under directory. A file entirely beyond the cutoff is + * deleted; a file straddling the cutoff is truncated at the first excluded record's start. + */ + public static trimDirectoryToBirthRound(directory: string, maximumBirthRound: number): void { + if (!fs.existsSync(directory)) { + return; + } + + for (const pcesFilePath of PcesTrimmer.findPcesFilesInSequenceOrder(directory)) { + PcesTrimmer.trimFileToBirthRound(pcesFilePath, maximumBirthRound); + } + } + + private static findPcesFilesInSequenceOrder(directory: string): string[] { + const filePaths: string[] = []; + PcesTrimmer.collectPcesFiles(directory, filePaths); + + // Sequence numbers are encoded as "_seqN_" in the filename and are contiguous per node; + // events must be inspected in that order regardless of how the date subdirectories nest. + filePaths.sort( + (left: string, right: string): number => + PcesTrimmer.extractSequenceNumber(left) - PcesTrimmer.extractSequenceNumber(right), + ); + return filePaths; + } + + private static collectPcesFiles(directory: string, filePaths: string[]): void { + for (const entry of fs.readdirSync(directory, {withFileTypes: true})) { + const entryPath: string = path.join(directory, entry.name); + if (entry.isDirectory()) { + PcesTrimmer.collectPcesFiles(entryPath, filePaths); + } else if (entry.isFile() && entry.name.endsWith('.pces')) { + filePaths.push(entryPath); + } + } + } + + private static extractSequenceNumber(pcesFilePath: string): number { + const match: RegExpMatchArray | null = path.basename(pcesFilePath).match(/_seq(\d+)_/); + return match ? Number(match[1]) : 0; + } + + private static trimFileToBirthRound(pcesFilePath: string, maximumBirthRound: number): void { + try { + const fileBuffer: Buffer = fs.readFileSync(pcesFilePath); + const truncateAtOffset: number | undefined = PcesTrimmer.findTruncationOffset(fileBuffer, maximumBirthRound); + + if (truncateAtOffset === undefined) { + return; + } + + if (truncateAtOffset <= 4) { + fs.rmSync(pcesFilePath, {force: true}); + return; + } + + fs.writeFileSync(pcesFilePath, fileBuffer.subarray(0, truncateAtOffset)); + } catch { + // Best-effort trim: an unexpected or unrecognized PCES layout is left untouched rather + // than risking corruption of a file the platform still needs to read on next startup. + } + } + + /** + * Returns the byte offset of the first record whose birth round exceeds maximumBirthRound, + * or undefined if the file does not need trimming (including when its header is not the + * expected version, in which case it is left untouched rather than guessed at). + */ + private static findTruncationOffset(fileBuffer: Buffer, maximumBirthRound: number): number | undefined { + if (fileBuffer.length < 4 || fileBuffer.readInt32BE(0) !== PCES_PROTOBUF_EVENTS_VERSION) { + return undefined; + } + + let recordOffset: number = 4; + while (recordOffset + 4 <= fileBuffer.length) { + const recordLength: number = fileBuffer.readInt32BE(recordOffset); + const recordStart: number = recordOffset + 4; + if (recordLength < 0 || recordStart + recordLength > fileBuffer.length) { + // A partial trailing record; nothing further to inspect. + return undefined; + } + + const recordBytes: Buffer = fileBuffer.subarray(recordStart, recordStart + recordLength); + const birthRound: number | undefined = PcesTrimmer.readGossipEventBirthRound(recordBytes); + if (birthRound !== undefined && birthRound > maximumBirthRound) { + return recordOffset; + } + + recordOffset = recordStart + recordLength; + } + + return undefined; + } + + /** + * Reads GossipEvent.event_core.birth_round (field 1, then field 2 within it) directly from + * the protobuf wire format. Only this one field is needed to decide whether an event belongs + * before or after the restore boundary, so a full protobuf runtime is not required. + */ + private static readGossipEventBirthRound(gossipEventBytes: Buffer): number | undefined { + const eventCoreBytes: Buffer | undefined = PcesTrimmer.readLengthDelimitedField( + gossipEventBytes, + GOSSIP_EVENT_CORE_FIELD_NUMBER, + ); + if (!eventCoreBytes) { + return undefined; + } + + return PcesTrimmer.readVarintField(eventCoreBytes, EVENT_CORE_BIRTH_ROUND_FIELD_NUMBER); + } + + private static readLengthDelimitedField(messageBytes: Buffer, targetFieldNumber: number): Buffer | undefined { + let offset: number = 0; + while (offset < messageBytes.length) { + const tagRead: VarintRead = PcesTrimmer.readVarint(messageBytes, offset); + const fieldNumber: number = Number(tagRead.value >> 3n); + const wireType: number = Number(tagRead.value & 0x7n); + offset = tagRead.nextOffset; + + if (fieldNumber === targetFieldNumber && wireType === WIRE_TYPE_LENGTH_DELIMITED) { + const lengthRead: VarintRead = PcesTrimmer.readVarint(messageBytes, offset); + const fieldStart: number = lengthRead.nextOffset; + return messageBytes.subarray(fieldStart, fieldStart + Number(lengthRead.value)); + } + + offset = PcesTrimmer.skipField(messageBytes, offset, wireType); + } + + return undefined; + } + + private static readVarintField(messageBytes: Buffer, targetFieldNumber: number): number | undefined { + let offset: number = 0; + while (offset < messageBytes.length) { + const tagRead: VarintRead = PcesTrimmer.readVarint(messageBytes, offset); + const fieldNumber: number = Number(tagRead.value >> 3n); + const wireType: number = Number(tagRead.value & 0x7n); + offset = tagRead.nextOffset; + + if (fieldNumber === targetFieldNumber && wireType === WIRE_TYPE_VARINT) { + return Number(PcesTrimmer.readVarint(messageBytes, offset).value); + } + + offset = PcesTrimmer.skipField(messageBytes, offset, wireType); + } + + return undefined; + } + + private static skipField(messageBytes: Buffer, offset: number, wireType: number): number { + switch (wireType) { + case WIRE_TYPE_VARINT: { + return PcesTrimmer.readVarint(messageBytes, offset).nextOffset; + } + case WIRE_TYPE_FIXED64: { + return offset + 8; + } + case WIRE_TYPE_LENGTH_DELIMITED: { + const lengthRead: VarintRead = PcesTrimmer.readVarint(messageBytes, offset); + return lengthRead.nextOffset + Number(lengthRead.value); + } + case WIRE_TYPE_FIXED32: { + return offset + 4; + } + default: { + throw new SoloErrors.internal.dataValidation('PCES protobuf wire type', '0, 1, 2, or 5', wireType); + } + } + } + + private static readVarint(buffer: Buffer, offset: number): VarintRead { + let result: bigint = 0n; + let shift: bigint = 0n; + let position: number = offset; + + while (position < buffer.length) { + const currentByte: number = buffer[position]; + position++; + result |= BigInt(currentByte & 0b0111_1111) << shift; + if ((currentByte & 0b1000_0000) === 0) { + return {value: result, nextOffset: position}; + } + shift += 7n; + } + + throw new SoloErrors.internal.dataValidation('PCES varint', 'a terminated varint', 'truncated bytes'); + } +} From 52a9b6bfc0fe61db660352fb547ebc5b51b781db Mon Sep 17 00:00:00 2001 From: Jeffrey Tang Date: Mon, 17 Aug 2026 23:29:37 -0500 Subject: [PATCH 18/20] fix: scope preconsensus-event trimming to callers that resume live The previous commit trimmed preconsensus-events unconditionally inside normalizeDownloadedStateArchives, which broke the state-save-and-restore example: unlike multicluster-backup-restore (which wants the restored network to resume ACTIVE), that example intentionally restores a frozen snapshot and expects the restored node to land back in FREEZE_COMPLETE. Its CI job passed before by relying on the very freeze-transaction replay the trim removes. - Add trimPreconsensusEventsToSelectedRound (default false) so trimming is opt-in; only config ops backup (multicluster-backup-restore) passes true. consensus state download (state-save-and-restore, node add, etc.) is unaffected. - Revert the freeze-round grace period added in the previous commit: two separate CI/local observations show the freeze round never becomes fully signed in this environment even after waiting, so it only added wasted time with no observed benefit. PcesTrimmer alone is what makes multicluster-backup-restore correct, independent of how quickly the fallback round is accepted. Signed-off-by: Jeffrey Tang --- src/commands/backup-restore.ts | 6 +++ src/core/constants.ts | 6 --- src/core/network-nodes.ts | 67 ++++++++++++++-------------------- 3 files changed, 33 insertions(+), 46 deletions(-) diff --git a/src/commands/backup-restore.ts b/src/commands/backup-restore.ts index ac3bb54745..cdc9dfc826 100644 --- a/src/commands/backup-restore.ts +++ b/src/commands/backup-restore.ts @@ -450,6 +450,12 @@ export class BackupRestoreCommand extends BaseCommand { clusterNodeAliases, statesDirectory, allNodesFrozen ? DeploymentPhase.FROZEN : undefined, + // This backup/restore workflow resumes the restored network as a live, active + // deployment rather than leaving it frozen, so any preconsensus event beyond the + // selected round (e.g. the freeze transaction the network reaches moments later) + // must be trimmed; otherwise replaying it on restart pins the node in + // FREEZE_COMPLETE instead of ACTIVE. + true, ); } task.title = `Download Node State Files: ${consensusNodes.length} node(s) completed`; diff --git a/src/core/constants.ts b/src/core/constants.ts index da5d47d359..5367ce63cc 100644 --- a/src/core/constants.ts +++ b/src/core/constants.ts @@ -507,12 +507,6 @@ export const STATE_DOWNLOAD_STABLE_MAX_ATTEMPTS: number = export const STATE_DOWNLOAD_STABLE_DELAY: number = +getEnvironmentVariable('STATE_DOWNLOAD_STABLE_DELAY') || 2000; export const STATE_DOWNLOAD_STABLE_POLLS_REQUIRED: number = +getEnvironmentVariable('STATE_DOWNLOAD_STABLE_POLLS_REQUIRED') || 3; -// After a freeze request, the platform can report FROZEN status before the freeze round is -// fully signed on disk. A non-freeze round may look "stable" first; keep polling for this many -// additional attempts so the freeze round (once signed) is preferred over the stale fallback, -// whose bundled preconsensus-events would otherwise replay straight back into the freeze on restart. -export const STATE_DOWNLOAD_FREEZE_GRACE_ATTEMPTS: number = - +getEnvironmentVariable('STATE_DOWNLOAD_FREEZE_GRACE_ATTEMPTS') || 60; export const NETWORK_PROXY_MAX_ATTEMPTS: number = +getEnvironmentVariable('NETWORK_PROXY_MAX_ATTEMPTS') || 300; export const NETWORK_PROXY_DELAY: number = +getEnvironmentVariable('NETWORK_PROXY_DELAY') || 2000; diff --git a/src/core/network-nodes.ts b/src/core/network-nodes.ts index 42fa38e4c0..a63e59a75b 100644 --- a/src/core/network-nodes.ts +++ b/src/core/network-nodes.ts @@ -174,6 +174,7 @@ export class NetworkNodes { nodeAliases: string[], baseDirectory: string = SOLO_LOGS_DIR, deploymentPhase?: DeploymentPhase, + trimPreconsensusEventsToSelectedRound: boolean = false, ): Promise { const archivePaths: string[] = nodeAliases.map((nodeAlias: string): string => { const archivePath: string = PathEx.join(baseDirectory, namespace.name, `network-${nodeAlias}-0-state.zip`); @@ -252,18 +253,23 @@ export class NetworkNodes { fs.rmSync(PathEx.join(extractedDirectory, 'saved'), {recursive: true, force: true}); fs.rmSync(PathEx.join(extractedDirectory, 'swirlds-tmp'), {recursive: true, force: true}); - // The top-level preconsensus-events stream is what the platform replays on the next - // restart, and it is not naturally bounded to the selected round: it can contain later - // events (e.g. a freeze transaction ordered moments after this snapshot was taken) that - // the selected round's own state does not yet reflect. Trim those out so replay cannot - // cross back into that later boundary, while keeping every event up to the selected - // round so the restored node still has other-parent candidates to build new events on - // (removing the whole stream instead would leave every node with no known events at - // all, which the platform only permits at true genesis). - PcesTrimmer.trimDirectoryToBirthRound( - PathEx.join(extractedDirectory, 'preconsensus-events'), - Number(selectedRound), - ); + if (trimPreconsensusEventsToSelectedRound) { + // The top-level preconsensus-events stream is what the platform replays on the next + // restart, and it is not naturally bounded to the selected round: it can contain later + // events (e.g. a freeze transaction ordered moments after this snapshot was taken) that + // the selected round's own state does not yet reflect. Trim those out so replay cannot + // cross back into that later boundary, while keeping every event up to the selected + // round so the restored node still has other-parent candidates to build new events on + // (removing the whole stream instead would leave every node with no known events at + // all, which the platform only permits at true genesis). Only opted into by callers + // that want the restored network to resume live processing (e.g. `config ops backup`); + // callers that intentionally restore a frozen snapshot rely on that same trailing + // freeze event still being present so the restored node lands back in FREEZE_COMPLETE. + PcesTrimmer.trimDirectoryToBirthRound( + PathEx.join(extractedDirectory, 'preconsensus-events'), + Number(selectedRound), + ); + } await this.zippy.zip(extractedDirectory, archivePaths[index]); } @@ -386,11 +392,9 @@ export class NetworkNodes { ): Promise { const maxAttempts: number = constants.STATE_DOWNLOAD_STABLE_MAX_ATTEMPTS; const stablePollsRequired: number = constants.STATE_DOWNLOAD_STABLE_POLLS_REQUIRED; - const freezeGraceAttempts: number = constants.STATE_DOWNLOAD_FREEZE_GRACE_ATTEMPTS; const pollDelay: Duration = Duration.ofMillis(constants.STATE_DOWNLOAD_STABLE_DELAY); let lastFingerprint: string | undefined; let stablePolls: number = 0; - let fallbackStableSinceAttempt: number | undefined; const scriptName: string = 'wait-for-stable-saved-state.sh'; const sourcePath: string = PathEx.joinWithRealPath(constants.RESOURCES_DIR, scriptName); const destinationPath: string = `${HEDERA_HAPI_PATH}/${scriptName}`; @@ -427,34 +431,17 @@ export class NetworkNodes { `[state-download] ${podName}: round ${round} (${kind}) stable poll ${stablePolls}/${stablePollsRequired}`, ); - const isStable: boolean = stablePolls >= stablePollsRequired; - if (kind === 'frozen-fallback') { // A frozen deployment can expose the FROZEN platform status before a - // freeze-marked round becomes fully signed on disk. The fallback round's - // bundled preconsensus-events stream already extends past its own round - // boundary, so restoring it verbatim would replay straight back into the - // freeze on restart. Give the freeze round a grace period to finish - // signing before accepting the stale fallback. - if (isStable && fallbackStableSinceAttempt === undefined) { - fallbackStableSinceAttempt = attempt; - this.logger.warn( - `[state-download] ${podName}: deployment is FROZEN but no fully signed freeze round exists on disk yet; ` + - `waiting up to ${freezeGraceAttempts} more attempt(s) for it before falling back to the newest fully signed non-freeze round`, - ); - } - - const gracePeriodElapsed: boolean = - fallbackStableSinceAttempt !== undefined && attempt - fallbackStableSinceAttempt >= freezeGraceAttempts; - - if (isStable && gracePeriodElapsed) { - this.logger.warn( - `[state-download] ${podName}: freeze round still not fully signed after grace period; using the newest fully signed non-freeze round`, - ); - await container.execContainer('sync'); - return; - } - } else if (isStable) { + // freeze-marked round becomes fully signed on disk. In that case, + // export the newest fully signed non-freeze round instead of waiting + // indefinitely for a freeze round that may never materialize. + this.logger.warn( + `[state-download] ${podName}: deployment is FROZEN but no fully signed freeze round exists on disk yet; using the newest fully signed non-freeze round`, + ); + } + + if (stablePolls >= stablePollsRequired) { // One final sync narrows the gap between the successful probe and the // subsequent zip/copy operation. await container.execContainer('sync'); From 0a2a4f79442e2c8ab646bd3c03f82289693691d0 Mon Sep 17 00:00:00 2001 From: Jeffrey Tang Date: Tue, 18 Aug 2026 09:18:50 -0500 Subject: [PATCH 19/20] format Signed-off-by: Jeffrey Tang --- src/commands/node/handlers.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/commands/node/handlers.ts b/src/commands/node/handlers.ts index 121405872a..c8b91f7509 100644 --- a/src/commands/node/handlers.ts +++ b/src/commands/node/handlers.ts @@ -24,7 +24,6 @@ import {type NodeDestroyContext} from './config-interfaces/node-destroy-context. import {type NodeAddContext} from './config-interfaces/node-add-context.js'; import {type NodeUpdateContext} from './config-interfaces/node-update-context.js'; import {type NodeUpgradeContext} from './config-interfaces/node-upgrade-context.js'; -import {type NodeFreezeContext} from './config-interfaces/node-freeze-context.js'; import {ComponentTypes} from '../../core/config/remote/enumerations/component-types.js'; import {DeploymentPhase} from '../../data/schema/model/remote/deployment-phase.js'; import {Templates} from '../../core/templates.js'; From aaabc8fa1dd23ebce11544585f6f2f58118d408a Mon Sep 17 00:00:00 2001 From: Jeffrey Tang Date: Mon, 24 Aug 2026 14:27:08 -0500 Subject: [PATCH 20/20] address PR review feedback: tests, perf, and cleanup - Add unit tests for PcesTrimmer covering: an unchanged file below the cutoff, a record ending exactly at the buffer boundary, a straddling file that gets truncated, multi-byte varint birth rounds, the truncateAtOffset <= 4 whole-file-delete path, an unrecognized version header, nested directories, a missing directory, and a malformed record (left untouched rather than guessed at). - wait-for-stable-saved-state.sh: fingerprint saved-state files by size+mtime instead of hashing file contents, avoiding repeated full-content I/O on every 2s poll (up to 180 times) while the node is trying to quiesce. - Remove the redundant showUser warning in getNodeStateFiles(); the thrown SoloErrors.validation.illegalArgument already carries the same message and is shown to the user. - Name the FREEZE_COMPLETE enum value in a comment next to the magic "6.0" platform_PlatformStatus check in the state-save-and-restore Taskfile. Signed-off-by: Jeffrey Tang --- examples/state-save-and-restore/Taskfile.yml | 1 + resources/wait-for-stable-saved-state.sh | 10 +- src/commands/node/tasks.ts | 5 - test/unit/core/pces-trimmer.test.ts | 162 +++++++++++++++++++ 4 files changed, 169 insertions(+), 9 deletions(-) create mode 100644 test/unit/core/pces-trimmer.test.ts diff --git a/examples/state-save-and-restore/Taskfile.yml b/examples/state-save-and-restore/Taskfile.yml index 0de14d87fa..6e93c5252c 100644 --- a/examples/state-save-and-restore/Taskfile.yml +++ b/examples/state-save-and-restore/Taskfile.yml @@ -331,6 +331,7 @@ tasks: - cmd: | echo "Verifying restored frozen state..." for node in $(echo {{ .NODE_ALIASES }} | tr ',' ' '); do + # platform_PlatformStatus value 6 == FREEZE_COMPLETE (see NodeStatusCodes in src/core/enumerations.ts) kubectl exec network-${node}-0 -n {{ .NAMESPACE }} -c root-container -- \ sh -c 'curl -sf http://localhost:9999/metrics | grep platform_PlatformStatus | grep -v "^#" | grep -q " 6\\.0$"' if kubectl logs network-${node}-0 -n {{ .NAMESPACE }} -c root-container --tail=500 | grep -q "Invalid State Signature"; then diff --git a/resources/wait-for-stable-saved-state.sh b/resources/wait-for-stable-saved-state.sh index c9f54a5ad9..1786795036 100644 --- a/resources/wait-for-stable-saved-state.sh +++ b/resources/wait-for-stable-saved-state.sh @@ -77,9 +77,11 @@ fi # Fingerprint the entire saved-state tree, not just the chosen round directory, # so the caller can detect when background flushes have stopped changing disk -# contents across consecutive polls. -find "${saved_dir}" -type f -print0 \ - | sort -z \ - | xargs -0 "${hash_cmd[@]}" \ +# contents across consecutive polls. Fingerprint size+mtime per file rather than +# hashing file contents: this poll runs up to 180 times, and re-reading every byte +# under data/saved on each pass adds real I/O load exactly while the node is trying +# to quiesce, for no benefit over the much cheaper metadata comparison. +find "${saved_dir}" -type f -printf '%s %T@ %p\n' \ + | sort \ | "${hash_cmd[@]}" \ | awk -v round="${selected_round}" -v kind="${selected_kind}" '{print $1, round, kind}' diff --git a/src/commands/node/tasks.ts b/src/commands/node/tasks.ts index 5f9164d30f..a128541c8c 100644 --- a/src/commands/node/tasks.ts +++ b/src/commands/node/tasks.ts @@ -3344,11 +3344,6 @@ export class NodeCommandTasks { const deploymentPhase: DeploymentPhase = nodeComponent.metadata.phase; if (![DeploymentPhase.FROZEN, DeploymentPhase.STOPPED].includes(deploymentPhase)) { - this.logger.showUser( - chalk.yellow( - `Warning: node ${nodeAlias} is in phase '${deploymentPhase}'. State download is only supported when consensus nodes are frozen or stopped.`, - ), - ); throw new SoloErrors.validation.illegalArgument( `Consensus node ${nodeAlias} must be in phase '${DeploymentPhase.FROZEN}' or '${DeploymentPhase.STOPPED}' before downloading saved state.`, ); diff --git a/test/unit/core/pces-trimmer.test.ts b/test/unit/core/pces-trimmer.test.ts new file mode 100644 index 0000000000..343510e557 --- /dev/null +++ b/test/unit/core/pces-trimmer.test.ts @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: Apache-2.0 + +import {expect} from 'chai'; +import {afterEach, beforeEach, describe, it} from 'mocha'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import {PcesTrimmer} from '../../../src/core/pces-trimmer.js'; +import {PathEx} from '../../../src/business/utils/path-ex.js'; + +const PROTOBUF_EVENTS_VERSION: number = 2; +// Protobuf field tags: (field_number << 3) | wire_type. EventCore.birth_round is field 2, +// wire type 0 (varint); GossipEvent.event_core is field 1, wire type 2 (length-delimited). +const EVENT_CORE_BIRTH_ROUND_TAG: number = 16; +const GOSSIP_EVENT_EVENT_CORE_TAG: number = 10; + +function encodeVarint(value: number): Buffer { + const bytes: number[] = []; + let remaining: number = value; + while (remaining > 0b0111_1111) { + bytes.push((remaining & 0b0111_1111) | 0b1000_0000); + remaining >>>= 7; + } + bytes.push(remaining); + return Buffer.from(bytes); +} + +/** Builds a minimal GossipEvent protobuf payload carrying only event_core.birth_round. */ +function buildGossipEventBytes(birthRound: number): Buffer { + const birthRoundVarint: Buffer = encodeVarint(birthRound); + const eventCoreBytes: Buffer = Buffer.concat([Buffer.from([EVENT_CORE_BIRTH_ROUND_TAG]), birthRoundVarint]); + return Buffer.concat([ + Buffer.from([GOSSIP_EVENT_EVENT_CORE_TAG]), + encodeVarint(eventCoreBytes.length), + eventCoreBytes, + ]); +} + +/** Builds a well-formed PCES file (header + length-prefixed records) for the given birth rounds. */ +function buildPcesFileBuffer(birthRounds: number[], version: number = PROTOBUF_EVENTS_VERSION): Buffer { + const header: Buffer = Buffer.alloc(4); + header.writeInt32BE(version, 0); + + const records: Buffer[] = birthRounds.map((birthRound: number): Buffer => { + const gossipEventBytes: Buffer = buildGossipEventBytes(birthRound); + const lengthPrefix: Buffer = Buffer.alloc(4); + lengthPrefix.writeInt32BE(gossipEventBytes.length, 0); + return Buffer.concat([lengthPrefix, gossipEventBytes]); + }); + + return Buffer.concat([header, ...records]); +} + +describe('PcesTrimmer', (): void => { + let temporaryDirectory: string; + + beforeEach((): void => { + temporaryDirectory = fs.mkdtempSync(PathEx.join(os.tmpdir(), 'pces-trimmer-')); + }); + + afterEach((): void => { + fs.rmSync(temporaryDirectory, {recursive: true, force: true}); + }); + + function writePcesFile(fileName: string, buffer: Buffer): string { + const filePath: string = path.join(temporaryDirectory, fileName); + fs.writeFileSync(filePath, buffer); + return filePath; + } + + it('leaves a file unchanged when every event is at or below the cutoff round', (): void => { + const originalBuffer: Buffer = buildPcesFileBuffer([10, 20, 30]); + const filePath: string = writePcesFile('a_seq0_minr1_maxr30_orgn0.pces', originalBuffer); + + PcesTrimmer.trimDirectoryToBirthRound(temporaryDirectory, 30); + + expect(fs.readFileSync(filePath)).to.deep.equal(originalBuffer); + }); + + it('leaves a single-record file unchanged when the record ends exactly at the end of the file', (): void => { + const originalBuffer: Buffer = buildPcesFileBuffer([5]); + const filePath: string = writePcesFile('a_seq0_minr1_maxr5_orgn0.pces', originalBuffer); + + PcesTrimmer.trimDirectoryToBirthRound(temporaryDirectory, 5); + + expect(fs.readFileSync(filePath)).to.deep.equal(originalBuffer); + }); + + it('truncates a file at the first record whose birth round exceeds the cutoff', (): void => { + const filePath: string = writePcesFile('a_seq0_minr1_maxr40_orgn0.pces', buildPcesFileBuffer([10, 20, 30, 40])); + + PcesTrimmer.trimDirectoryToBirthRound(temporaryDirectory, 25); + + expect(fs.readFileSync(filePath)).to.deep.equal(buildPcesFileBuffer([10, 20])); + }); + + it('correctly parses multi-byte varints when truncating past a birth round above 127', (): void => { + const filePath: string = writePcesFile( + 'a_seq0_minr1_maxr500_orgn0.pces', + buildPcesFileBuffer([100, 200, 300, 400]), + ); + + PcesTrimmer.trimDirectoryToBirthRound(temporaryDirectory, 250); + + expect(fs.readFileSync(filePath)).to.deep.equal(buildPcesFileBuffer([100, 200])); + }); + + it('deletes a file entirely when even its first record exceeds the cutoff', (): void => { + const filePath: string = writePcesFile('a_seq0_minr50_maxr60_orgn0.pces', buildPcesFileBuffer([50, 60])); + + PcesTrimmer.trimDirectoryToBirthRound(temporaryDirectory, 10); + + expect(fs.existsSync(filePath)).to.be.false; + }); + + it('leaves a file untouched when its version header is not the recognized PROTOBUF_EVENTS version', (): void => { + const originalBuffer: Buffer = buildPcesFileBuffer([10, 999], 99); + const filePath: string = writePcesFile('a_seq0_minr1_maxr999_orgn0.pces', originalBuffer); + + PcesTrimmer.trimDirectoryToBirthRound(temporaryDirectory, 10); + + expect(fs.readFileSync(filePath)).to.deep.equal(originalBuffer); + }); + + it('trims each file in a nested directory tree independently, in sequence order', (): void => { + const nestedDirectory: string = path.join(temporaryDirectory, '2026', '08', '17'); + fs.mkdirSync(nestedDirectory, {recursive: true}); + + const belowCutoffBuffer: Buffer = buildPcesFileBuffer([1, 2]); + const belowCutoffPath: string = path.join(nestedDirectory, 'a_seq0_minr1_maxr2_orgn0.pces'); + fs.writeFileSync(belowCutoffPath, belowCutoffBuffer); + + const straddlingPath: string = path.join(nestedDirectory, 'b_seq1_minr2_maxr50_orgn0.pces'); + fs.writeFileSync(straddlingPath, buildPcesFileBuffer([2, 3, 40])); + + PcesTrimmer.trimDirectoryToBirthRound(temporaryDirectory, 3); + + expect(fs.readFileSync(belowCutoffPath)).to.deep.equal(belowCutoffBuffer); + expect(fs.readFileSync(straddlingPath)).to.deep.equal(buildPcesFileBuffer([2, 3])); + }); + + it('does nothing when the directory does not exist', (): void => { + const missingDirectory: string = path.join(temporaryDirectory, 'does-not-exist'); + + expect((): void => PcesTrimmer.trimDirectoryToBirthRound(missingDirectory, 10)).to.not.throw(); + }); + + it('leaves a file untouched when a record is malformed instead of risking an incorrect truncation', (): void => { + const header: Buffer = Buffer.alloc(4); + header.writeInt32BE(PROTOBUF_EVENTS_VERSION, 0); + // A length prefix declaring more bytes than actually follow it. + const corruptLengthPrefix: Buffer = Buffer.alloc(4); + corruptLengthPrefix.writeInt32BE(1000, 0); + const originalBuffer: Buffer = Buffer.concat([header, corruptLengthPrefix, Buffer.from([1, 2, 3])]); + const filePath: string = writePcesFile('a_seq0_minr1_maxr10_orgn0.pces', originalBuffer); + + PcesTrimmer.trimDirectoryToBirthRound(temporaryDirectory, 10); + + expect(fs.readFileSync(filePath)).to.deep.equal(originalBuffer); + }); +});