feat(relayer): add Terraform/Ansible lifecycle service

.gitignore

@@ -35,6 +35,7 @@ validator-manager.json
 
 # Local notes
 notes/
+docs/audits/2026-07-terraform-ansible-live-test-bugs.md
 
 # Claude Code config (local only)
 CLAUDE.md
@@ -55,11 +56,16 @@ Thumbs.db
 
 # Local build/test caches
 .cache/
+__pycache__/
+*.py[cod]
 
 # Secrets (never commit these)
 *.pem
 *.key
 !*.key.pub
 
+# Fetched relayer backups (plaintext keystore password inside)
+backups/
+
 # Helm vendored dependency tarballs (rebuild with: helm dependency build)
 kubernetes/helm/*/charts/

Makefile

@@ -9,11 +9,15 @@
 #   make destroy    - Tear down everything
 
 SHELL := /bin/bash
-.PHONY: setup doctor infra infra-plan deploy configure-l1 status create-l1 deploy-blockscout safe reset-l1 destroy clean logs rolling-restart health-checks monitoring faucet upgrade graph-node erpc icm-relayer init-validator-manager initialize-validator-manager primary-infra primary-infra-plan primary-deploy primary-status primary-destroy backup-keys restore-keys prepare-migration migrate-validator create-snapshot restore-snapshot list-snapshots k8s-help k8s-help-l1 k8s-help-primary k8s-l1 k8s-primary k8s-kind k8s-l1-deploy k8s-l1-wait k8s-l1-create k8s-l1-configure k8s-l1-status k8s-primary-deploy k8s-primary-wait k8s-primary-status k8s-monitoring k8s-icm-relayer k8s-erpc k8s-faucet k8s-blockscout k8s-graph-node k8s-safe k8s-backup-keys k8s-health-checks k8s-init-validator-manager k8s-reset-l1 k8s-cleanup lint validate-config-layout validate test-unit test-incremental test test-e2e-l1 test-e2e-primary test-e2e-l1-dry test-e2e-primary-dry test-e2e-dry check-primary-cloud help help-l1 help-primary help-all
+.PHONY: setup doctor infra infra-plan deploy configure-l1 status create-l1 deploy-blockscout safe reset-l1 destroy clean logs rolling-restart health-checks monitoring faucet upgrade graph-node erpc icm-relayer relayer-prereqs relayer-doctor relayer-prepare relayer-authorize relayer relayer-access relayer-status relayer-logs relayer-upgrade relayer-backup relayer-restore relayer-reset-scanner relayer-remove init-validator-manager initialize-validator-manager primary-infra primary-infra-plan primary-deploy primary-status primary-destroy backup-keys restore-keys prepare-migration migrate-validator create-snapshot restore-snapshot list-snapshots k8s-help k8s-help-l1 k8s-help-primary k8s-l1 k8s-primary k8s-kind k8s-l1-deploy k8s-l1-wait k8s-l1-create k8s-l1-configure k8s-l1-status k8s-primary-deploy k8s-primary-wait k8s-primary-status k8s-monitoring k8s-icm-relayer k8s-erpc k8s-faucet k8s-blockscout k8s-graph-node k8s-safe k8s-backup-keys k8s-health-checks k8s-init-validator-manager k8s-reset-l1 k8s-cleanup lint validate-config-layout validate test-unit test-incremental test test-e2e-l1 test-e2e-primary test-e2e-l1-dry test-e2e-primary-dry test-e2e-dry check-primary-cloud help help-l1 help-primary help-all
 
 # Default cloud provider
 CLOUD ?= aws
 NETWORK ?= fuji
+# Resolve the newest non-prerelease from the official repository. Until the
+# first production release exists, use this reviewed prerelease explicitly.
+RELAYER_VERSION ?= official-latest
+RELAYER_PRERELEASE_FALLBACK ?= v0.1.0-rc.8
 AUTO_APPROVE ?= false
 TF_INIT_RETRIES ?= 3
 SKIP_TERRAFORM_VALIDATE ?= false
@@ -226,6 +230,49 @@ icm-relayer:
 		-e "relayer_private_key=$(RELAYER_KEY)" \
 		$(if $(NETWORK),-e "icm_relayer_network=$(NETWORK)",)
 
+#
+# ACP-77/99 Validator-Lifecycle Relayer (relayerd)
+#
+relayer-prereqs:
+	@./scripts/l1/relayer.sh prereqs
+
+relayer-doctor:
+	@RELAYER_VERSION="$(RELAYER_VERSION)" RELAYER_PRERELEASE_FALLBACK="$(RELAYER_PRERELEASE_FALLBACK)" ./scripts/l1/relayer.sh doctor
+
+relayer-prepare:
+	@RELAYER_VERSION="$(RELAYER_VERSION)" RELAYER_PRERELEASE_FALLBACK="$(RELAYER_PRERELEASE_FALLBACK)" ./scripts/l1/relayer.sh prepare
+
+relayer-authorize:
+	@./scripts/l1/relayer.sh authorize
+
+relayer:
+	@RELAYER_VERSION="$(RELAYER_VERSION)" RELAYER_PRERELEASE_FALLBACK="$(RELAYER_PRERELEASE_FALLBACK)" ./scripts/l1/relayer.sh install
+
+relayer-access:
+	@./scripts/l1/relayer.sh access
+
+relayer-status:
+	@./scripts/l1/relayer.sh status
+
+relayer-logs:
+	@./scripts/l1/relayer.sh logs
+
+relayer-upgrade:
+	@RELAYER_VERSION="$(RELAYER_VERSION)" RELAYER_PRERELEASE_FALLBACK="$(RELAYER_PRERELEASE_FALLBACK)" ./scripts/l1/relayer.sh upgrade
+
+relayer-backup:
+	@./scripts/l1/relayer.sh backup
+
+relayer-restore:
+	@if [ -z "$(BACKUP)" ]; then echo "Usage: make relayer-restore BACKUP=/absolute/path/to/relayer-*.tar.gz"; exit 2; fi
+	@BACKUP="$(BACKUP)" RELAYER_VERSION="$(RELAYER_VERSION)" RELAYER_PRERELEASE_FALLBACK="$(RELAYER_PRERELEASE_FALLBACK)" ./scripts/l1/relayer.sh restore
+
+relayer-reset-scanner:
+	@./scripts/l1/relayer.sh reset-scanner
+
+relayer-remove:
+	@PURGE="$(PURGE)" ./scripts/l1/relayer.sh remove
+
 #
 # Validator Manager
 #
@@ -247,8 +294,15 @@ initialize-validator-manager:
 		-e "conversion_tx=$(CONVERSION_TX)" \
 		-e "proxy_address=$(PROXY_ADDRESS)" \
 		-e "evm_chain_id=$(EVM_CHAIN_ID)" \
+		-e "network=$(NETWORK)" \
 		$(if $(MANAGER_TYPE),-e "manager_type=$(MANAGER_TYPE)",) \
-		$(if $(ICM_CONTRACTS_PATH),-e "icm_contracts_path=$(ICM_CONTRACTS_PATH)",)
+		$(if $(ICM_CONTRACTS_PATH),-e "icm_contracts_path=$(ICM_CONTRACTS_PATH)",) \
+		$(if $(CONVERSION_ID),-e "conversion_id=$(CONVERSION_ID)",) \
+		$(if $(VALIDATOR_MESSAGES_LIBRARY),-e "validator_messages_library=$(VALIDATOR_MESSAGES_LIBRARY)",) \
+		$(if $(VALIDATOR_MANAGER_IMPLEMENTATION),-e "validator_manager_implementation=$(VALIDATOR_MANAGER_IMPLEMENTATION)",) \
+		$(if $(filter true,$(USE_LOCAL_SIG_AGG)),-e "use_local_sig_agg=true",) \
+		$(if $(SIG_AGG_URL),-e "sig_agg_url=$(SIG_AGG_URL)",) \
+		$(if $(filter true,$(PREFLIGHT_ONLY)),-e "preflight_only=true",)
 
 #
 # Primary Network Validators
@@ -666,10 +720,16 @@ test-e2e-dry: test-e2e-l1-dry test-e2e-primary-dry
 	@echo "✓ E2E dry-run checks passed"
 
 test-unit:
+	python3 -m unittest tests/test_relayer_scanner_start.py
 	@echo "Running unit tests..."
 	@cd tools/create-l1 && go test ./...
 	@cd tools/initialize-validator-manager && go test ./...
 	@cd tools/initialize-validator-manager/cmd/init_valset && go test ./...
+	@./tests/relayer-static.sh
+	@./tests/relayer-release-capability-fixtures.sh
+	@./tests/relayer-protocol-privacy-fixtures.sh
+	@./tests/relayer-doctor-fixtures.sh
+	@./tests/safe-deploy-contracts.sh
 	@echo "✓ Unit tests passed"
 
 test-incremental: lint validate test-unit test-e2e-dry
@@ -733,6 +793,18 @@ help-l1:
 	@echo "  make graph-node CHAIN_ID=... [NETWORK_NAME=...]"
 	@echo "  make erpc CHAIN_ID=... EVM_CHAIN_ID=...          (standalone re-deploy)"
 	@echo "  make icm-relayer SUBNET_ID=... CHAIN_ID=... RELAYER_KEY=0x..."
+	@echo "  make relayer-prereqs                              (install local operator software)"
+	@echo "  make relayer-doctor                               (read-only comprehensive diagnostics)"
+	@echo "  make relayer-prepare                              (stage the permanent P2P identity)"
+	@echo "  make relayer-authorize                            (authorize Terraform/Ansible-managed private validators)"
+	@echo "  Manual/external authorization: docs/l1/RELAYER-AUTHORIZATION.md"
+	@echo "  make relayer                                      (discover and install/reapply after PoAManager init)"
+	@echo "  make relayer-access | make relayer-status | make relayer-logs"
+	@echo "  make relayer-backup"
+	@echo "  make relayer-restore BACKUP=/absolute/path/to/relayer-....tar.gz"
+	@echo "  make relayer-reset-scanner                        (same-L1 rescan; requires zero operation records)"
+	@echo "  make relayer-upgrade [RELAYER_VERSION=vX.Y.Z]"
+	@echo "  make relayer-remove [PURGE=true]"
 	@echo "  make safe [CHAIN_ID=... EVM_CHAIN_ID=...]  (auto-detects from l1.env)"
 	@echo ""
 	@echo "Ops:"
@@ -829,6 +901,20 @@ help-all:
 	@echo "  make graph-node        Deploy The Graph Node for indexing"
 	@echo "  make erpc              Deploy eRPC load balancer"
 	@echo "  make icm-relayer       Deploy ICM Relayer for cross-chain messaging"
+	@echo "  make relayer-prereqs   Install local Relayer operator software"
+	@echo "  make relayer-doctor    Run read-only comprehensive diagnostics"
+	@echo "  make relayer-prepare   Stage the permanent P2P identity before private authorization"
+	@echo "  make relayer-authorize Authorize Terraform/Ansible-managed protocol-private validators"
+	@echo "  Manual/external authorization: docs/l1/RELAYER-AUTHORIZATION.md"
+	@echo "  make relayer           Auto-discover and install/reapply daemon + console on rpc[0]"
+	@echo "  make relayer-access    Forward the Relayer console and L1 RPC"
+	@echo "  make relayer-status    Show a quick runtime snapshot"
+	@echo "  make relayer-logs      Follow Relayer logs"
+	@echo "  make relayer-backup    Create a retained consistent backup"
+	@echo "  make relayer-restore BACKUP=/absolute/path/to/relayer-....tar.gz"
+	@echo "  make relayer-reset-scanner  Same-L1 rescan; backup, then reset cursor/queue only when no records exist"
+	@echo "  make relayer-upgrade [RELAYER_VERSION=vX.Y.Z]"
+	@echo "  make relayer-remove [PURGE=true]"
 	@echo ""
 	@echo "Validator Manager:"
 	@echo "  make init-validator-manager      Build the validator manager tool"

README.md

@@ -7,7 +7,7 @@ Production infrastructure automation for Avalanche blockchains.
 | | **Launch an Avalanche L1** | **Run Primary Network Validators** |
 |---|---|---|
 | **What** | Deploy your own L1 blockchain with validators, RPC nodes, and a full service stack | Operate production Avalanche Primary Network validators |
-| **Features** | Validators, archive + pruned RPC, eRPC load balancer (built-in), monitoring, block explorer, faucet, The Graph, ICM Relayer, Safe | Validators, staking key backup, database snapshots, near-zero-downtime migration, monitoring |
+| **Features** | Validators, archive + pruned RPC, eRPC load balancer (built-in), monitoring, block explorer, faucet, The Graph, ICM Relayer, validator-lifecycle Relayer (Terraform/Ansible), Safe | Validators, staking key backup, database snapshots, near-zero-downtime migration, monitoring |
 | **Clouds** | AWS, GCP, Azure | AWS |
 | **Guide** | [L1 Deployment](docs/l1/DEPLOYMENT.md) | [Primary Network](docs/primary-network/DEPLOYMENT.md) |
 
@@ -37,12 +37,15 @@ docs/         Guides and reference docs
 
 ## Documentation
 
+- [Documentation index](docs/README.md) — Deployment, operations, troubleshooting, security, and services
 - [L1 Deployment](docs/l1/DEPLOYMENT.md) — Full L1 runbook
 - [Primary Network](docs/primary-network/DEPLOYMENT.md) — Validator deployment, snapshots, migration
-- [Add-ons](docs/l1/ADD-ONS.md) — Blockscout, faucet, eRPC, The Graph, ICM Relayer, Safe
+- [Add-ons](docs/l1/ADD-ONS.md) — Blockscout, faucet, eRPC, The Graph, ICM Relayer, validator-lifecycle Relayer, Safe
 - [Operations](docs/OPERATIONS.md) — Upgrades, health checks, monitoring, rolling restarts
 - [Troubleshooting](docs/TROUBLESHOOTING.md) — Common issues and fixes
 - [Kubernetes](kubernetes/README.md) — Helm-based deployment alternative
+- [Managed validator-lifecycle Relayer](docs/l1/RELAYER.md) — Terraform/Ansible doctor, installation, access, funding, recovery, upgrades, and removal
+- [Protocol-private Relayer authorization](docs/l1/RELAYER-AUTHORIZATION.md) — Managed Terraform/Ansible automation and manual/external validator configuration
 - [Security](SECURITY.md) — Security expectations and guidelines
 
 ## Command Help