|
| 1 | +#!/usr/bin/env bash |
| 2 | +# |
| 3 | +# run-tests.sh — Apply the AWS profile for real, against an emulated API |
| 4 | +# |
| 5 | +# Usage: |
| 6 | +# ./tests/cloud-apply-emulated/run-tests.sh |
| 7 | +# |
| 8 | +# Takes a couple of minutes. Costs nothing and creates nothing outside a |
| 9 | +# local process. |
| 10 | +# |
| 11 | +# WHY THIS EXISTS |
| 12 | +# |
| 13 | +# Between "terraform validate" and "a real apply" there is a gap nothing |
| 14 | +# here was filling. |
| 15 | +# |
| 16 | +# `terraform test` runs against mocked providers. A mock answers from a |
| 17 | +# fixture: it never exercises the provider's request building, never |
| 18 | +# rejects a value the API would reject, and never has an opinion about the |
| 19 | +# order things are created in. It is the same shape of tool as the shims |
| 20 | +# elsewhere in this repository, with the same blind spot -- it agrees with |
| 21 | +# whatever the configuration says. |
| 22 | +# |
| 23 | +# This runs a real `terraform apply`, through the real AWS provider, |
| 24 | +# against an implementation of the AWS API (moto). Every request is |
| 25 | +# actually built, sent, and answered. That settles a set of questions the |
| 26 | +# mocked tests structurally cannot: |
| 27 | +# |
| 28 | +# - does the configuration apply at all, end to end, in one pass |
| 29 | +# - does every reference resolve, in an order Terraform can satisfy |
| 30 | +# - does the AMI data source match anything |
| 31 | +# - does any resource carry a value the API refuses |
| 32 | +# - does `terraform destroy` actually remove what was made |
| 33 | +# |
| 34 | +# WHAT A GREEN RUN DOES NOT MEAN |
| 35 | +# |
| 36 | +# An emulator implements the API, not the service. Nothing boots. No |
| 37 | +# health check runs. No autoscaling group replaces anything. KMS returns |
| 38 | +# plausible responses without performing cryptography. |
| 39 | +# |
| 40 | +# So this is evidence the configuration is applyable, and it is NOT |
| 41 | +# evidence the cluster works. It does not shorten the list in |
| 42 | +# docs/cloud-apply.md; it removes from that list the questions that never |
| 43 | +# needed a credit card in the first place. |
| 44 | +# |
| 45 | +# Requirements: terraform, python3 with moto[server], curl |
| 46 | + |
| 47 | +set -uo pipefail |
| 48 | + |
| 49 | +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 50 | +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" |
| 51 | +TF_DIR="${REPO_ROOT}/terraform/aws" |
| 52 | +OVERRIDE_SRC="${SCRIPT_DIR}/provider_override.tf" |
| 53 | +OVERRIDE_DST="${TF_DIR}/zz_emulated_override.tf" |
| 54 | +ENDPOINT="http://localhost:5000" |
| 55 | + |
| 56 | +WORK="$(mktemp -d)" |
| 57 | +MOTO_PID="" |
| 58 | + |
| 59 | +PASS=0 |
| 60 | +FAIL=0 |
| 61 | + |
| 62 | +green() { printf '\033[32m%s\033[0m\n' "$*"; } |
| 63 | +red() { printf '\033[31m%s\033[0m\n' "$*"; } |
| 64 | +info() { printf '\033[36m%s\033[0m\n' "$*"; } |
| 65 | + |
| 66 | +ok() { PASS=$((PASS + 1)); green " PASS $1"; } |
| 67 | +bad() { FAIL=$((FAIL + 1)); red " FAIL $1"; [[ -n "${2:-}" ]] && printf ' %s\n' "$2"; return 0; } |
| 68 | + |
| 69 | +cleanup() { |
| 70 | + local rc=$? |
| 71 | + # Destroy before stopping the emulator, or the provider has nothing to |
| 72 | + # talk to and the run leaves state behind for the next one. |
| 73 | + if [[ -f "${TF_DIR}/terraform.tfstate" ]]; then |
| 74 | + info "Destroying the emulated stack..." |
| 75 | + terraform -chdir="$TF_DIR" destroy -auto-approve -input=false \ |
| 76 | + >"${WORK}/destroy.log" 2>&1 || red " (destroy failed; see ${WORK}/destroy.log)" |
| 77 | + fi |
| 78 | + [[ -n "$MOTO_PID" ]] && kill "$MOTO_PID" 2>/dev/null |
| 79 | + rm -f "$OVERRIDE_DST" |
| 80 | + rm -rf "${TF_DIR}/.terraform" "${TF_DIR}/terraform.tfstate" \ |
| 81 | + "${TF_DIR}/terraform.tfstate.backup" "${TF_DIR}/.terraform.lock.hcl.bak" |
| 82 | + rm -rf "$WORK" |
| 83 | + exit "$rc" |
| 84 | +} |
| 85 | +trap cleanup EXIT INT TERM |
| 86 | + |
| 87 | +for dep in terraform python3 curl; do |
| 88 | + command -v "$dep" >/dev/null 2>&1 || { red "ERROR: ${dep} not found on PATH"; exit 1; } |
| 89 | +done |
| 90 | +python3 -c "import moto" 2>/dev/null || { red "ERROR: moto is not installed (pip install 'moto[server]')"; exit 1; } |
| 91 | +[[ -f "$OVERRIDE_SRC" ]] || { red "ERROR: missing ${OVERRIDE_SRC}"; exit 1; } |
| 92 | + |
| 93 | +# --------------------------------------------------------------------------- |
| 94 | +info "" |
| 95 | +info "=== Starting the emulated AWS API ===" |
| 96 | +# --------------------------------------------------------------------------- |
| 97 | +# moto stopped pre-seeding AWS managed policies in v5, so attaching |
| 98 | +# AmazonSSMManagedInstanceCore -- which iam.tf does, correctly, to give |
| 99 | +# operators SSM Session Manager instead of an open port 22 -- fails with |
| 100 | +# NoSuchEntity unless they are loaded. That is an emulator gap rather |
| 101 | +# than anything wrong with the profile, and this is the switch for it. |
| 102 | +export MOTO_IAM_LOAD_MANAGED_POLICIES=true |
| 103 | + |
| 104 | +python3 -m moto.server -p 5000 >"${WORK}/moto.log" 2>&1 & |
| 105 | +MOTO_PID=$! |
| 106 | + |
| 107 | +for _ in $(seq 1 30); do |
| 108 | + curl -fsS -o /dev/null "${ENDPOINT}/" 2>/dev/null && break |
| 109 | + sleep 1 |
| 110 | +done |
| 111 | + |
| 112 | +if curl -fsS -o /dev/null "${ENDPOINT}/" 2>/dev/null; then |
| 113 | + ok "the emulator is answering on ${ENDPOINT}" |
| 114 | +else |
| 115 | + bad "the emulator is answering on ${ENDPOINT}" "$(tail -5 "${WORK}/moto.log")" |
| 116 | + exit 1 |
| 117 | +fi |
| 118 | + |
| 119 | +cp "$OVERRIDE_SRC" "$OVERRIDE_DST" |
| 120 | + |
| 121 | +# --------------------------------------------------------------------------- |
| 122 | +info "" |
| 123 | +info "=== terraform apply, for real, against the emulator ===" |
| 124 | +# --------------------------------------------------------------------------- |
| 125 | +if terraform -chdir="$TF_DIR" init -backend=false -input=false >"${WORK}/init.log" 2>&1; then |
| 126 | + ok "terraform init" |
| 127 | +else |
| 128 | + bad "terraform init" "$(tail -15 "${WORK}/init.log")" |
| 129 | + exit 1 |
| 130 | +fi |
| 131 | + |
| 132 | +# The apply is the assertion. Everything below reads what it produced. |
| 133 | +if terraform -chdir="$TF_DIR" apply -auto-approve -input=false >"${WORK}/apply.log" 2>&1; then |
| 134 | + ok "the AWS profile applies end to end" |
| 135 | +else |
| 136 | + bad "the AWS profile applies end to end" "$(grep -iE 'error|Error:' "${WORK}/apply.log" | head -12)" |
| 137 | + # Everything after this reads state that does not exist. |
| 138 | + printf '\n=== Results ===\n' |
| 139 | + printf 'passed: %d\nfailed: %d\n' "$PASS" "$FAIL" |
| 140 | + red "FAILED" |
| 141 | + exit 1 |
| 142 | +fi |
| 143 | + |
| 144 | +STATE="${WORK}/state.json" |
| 145 | +terraform -chdir="$TF_DIR" show -json > "$STATE" 2>/dev/null |
| 146 | + |
| 147 | +count_type() { |
| 148 | + python3 -c " |
| 149 | +import json,sys |
| 150 | +d=json.load(open(sys.argv[1])) |
| 151 | +res=d.get('values',{}).get('root_module',{}).get('resources',[]) |
| 152 | +print(sum(1 for r in res if r.get('type')==sys.argv[2])) |
| 153 | +" "$STATE" "$1" 2>/dev/null || echo 0 |
| 154 | +} |
| 155 | + |
| 156 | +attr_of() { |
| 157 | + python3 -c " |
| 158 | +import json,sys |
| 159 | +d=json.load(open(sys.argv[1])) |
| 160 | +res=d.get('values',{}).get('root_module',{}).get('resources',[]) |
| 161 | +for r in res: |
| 162 | + if r.get('type')==sys.argv[2] and r.get('name')==sys.argv[3]: |
| 163 | + v=r.get('values',{}) |
| 164 | + for part in sys.argv[4].split('.'): |
| 165 | + if isinstance(v,list): |
| 166 | + v=v[int(part)] if v else None |
| 167 | + else: |
| 168 | + v=(v or {}).get(part) |
| 169 | + print(v if v is not None else '') |
| 170 | + break |
| 171 | +" "$STATE" "$1" "$2" "$3" 2>/dev/null || echo "" |
| 172 | +} |
| 173 | + |
| 174 | +# --------------------------------------------------------------------------- |
| 175 | +info "" |
| 176 | +info "=== What the apply actually produced ===" |
| 177 | +# --------------------------------------------------------------------------- |
| 178 | +# The data source is the first thing a real API can refute: a filter that |
| 179 | +# matches nothing fails the plan, and no mocked provider would notice. |
| 180 | +AMI="$(attr_of aws_launch_template vault image_id)" |
| 181 | +if [[ "$AMI" == ami-* ]]; then |
| 182 | + ok "the AMI filter resolved against the API (${AMI})" |
| 183 | +else |
| 184 | + bad "the AMI filter resolved against the API" "launch template image_id is '${AMI}'" |
| 185 | +fi |
| 186 | + |
| 187 | +# Two roles, not one: the node role in iam.tf, and the role network.tf |
| 188 | +# creates so VPC flow logs can publish to CloudWatch. Expecting 1 was |
| 189 | +# wrong about the profile rather than a finding about it. |
| 190 | +for pair in "aws_subnet:6" "aws_nat_gateway:3" "aws_kms_key:1" "aws_s3_bucket:1" "aws_autoscaling_group:1" "aws_lb_target_group:1" "aws_iam_role:2" "aws_iam_instance_profile:1" "aws_launch_template:1"; do |
| 191 | + t="${pair%%:*}"; want="${pair##*:}" |
| 192 | + got="$(count_type "$t")" |
| 193 | + if [[ "$got" == "$want" ]]; then |
| 194 | + ok "${t}: ${got}" |
| 195 | + else |
| 196 | + bad "${t}: expected ${want}" "got ${got}" |
| 197 | + fi |
| 198 | +done |
| 199 | + |
| 200 | +# Quorum arithmetic, asserted against what was actually created rather |
| 201 | +# than against the plan. |
| 202 | +MIN="$(attr_of aws_autoscaling_group vault min_size)" |
| 203 | +MAX="$(attr_of aws_autoscaling_group vault max_size)" |
| 204 | +DES="$(attr_of aws_autoscaling_group vault desired_capacity)" |
| 205 | +if [[ "$MIN" == "$MAX" && "$MAX" == "$DES" && -n "$DES" ]]; then |
| 206 | + ok "the autoscaling group is pinned at ${DES} (min = max = desired)" |
| 207 | +else |
| 208 | + bad "the autoscaling group is pinned" "min=${MIN} max=${MAX} desired=${DES}" |
| 209 | +fi |
| 210 | + |
| 211 | +# The health check that keeps standbys in the pool. A 429 is a healthy |
| 212 | +# standby; accepting only 200 would route everything at the leader. |
| 213 | +MATCHER="$(attr_of aws_lb_target_group vault health_check.0.matcher)" |
| 214 | +if [[ "$MATCHER" == "200,429" ]]; then |
| 215 | + ok "the target group accepts 200 and 429 from standbys" |
| 216 | +else |
| 217 | + bad "the target group accepts 200 and 429 from standbys" "matcher is '${MATCHER}'" |
| 218 | +fi |
| 219 | + |
| 220 | +# The bucket the snapshots go to has to be encrypted with the cluster's own |
| 221 | +# key, not the account default. |
| 222 | +if grep -q "aws_s3_bucket_server_side_encryption_configuration" "$STATE"; then |
| 223 | + ok "the snapshot bucket carries a server-side encryption configuration" |
| 224 | +else |
| 225 | + bad "the snapshot bucket carries a server-side encryption configuration" |
| 226 | +fi |
| 227 | + |
| 228 | +# --------------------------------------------------------------------------- |
| 229 | +info "" |
| 230 | +info "=== And it can be taken back down ===" |
| 231 | +# --------------------------------------------------------------------------- |
| 232 | +# A profile that applies but cannot be destroyed is a profile that bills |
| 233 | +# forever on a real account. Worth knowing here rather than there. |
| 234 | +if terraform -chdir="$TF_DIR" destroy -auto-approve -input=false >"${WORK}/destroy1.log" 2>&1; then |
| 235 | + ok "terraform destroy removes everything it made" |
| 236 | + rm -f "${TF_DIR}/terraform.tfstate" |
| 237 | +else |
| 238 | + bad "terraform destroy removes everything it made" \ |
| 239 | + "$(grep -iE 'error|Error:' "${WORK}/destroy1.log" | head -8)" |
| 240 | +fi |
| 241 | + |
| 242 | +# --------------------------------------------------------------------------- |
| 243 | +printf '\n=== Results ===\n' |
| 244 | +# --------------------------------------------------------------------------- |
| 245 | +printf 'passed: %d\nfailed: %d\n' "$PASS" "$FAIL" |
| 246 | +if [[ "$FAIL" -gt 0 ]]; then red "FAILED"; exit 1; fi |
| 247 | +green "All ${PASS} assertions passed." |
0 commit comments