diff --git a/hunter/ansible/roles/build-image/tasks/main.yml b/hunter/ansible/roles/build-image/tasks/main.yml index 2e73e57..deb4846 100644 --- a/hunter/ansible/roles/build-image/tasks/main.yml +++ b/hunter/ansible/roles/build-image/tasks/main.yml @@ -17,7 +17,7 @@ register: docker_gid - name: Build magenta-arthel base image - command: docker compose -f docker-compose-build.yml build --build-arg DOCKER_GID={{ docker_gid.stdout }} + command: docker compose -f docker-compose-build.yml build {{ '--no-cache' if docker_no_cache | default(false) else '' }} --build-arg DOCKER_GID={{ docker_gid.stdout }} args: chdir: "{{ base_dir }}/maybelle-config/hunter" diff --git a/hunter/ansible/roles/services/tasks/main.yml b/hunter/ansible/roles/services/tasks/main.yml index 3e4159f..71d2e2b 100644 --- a/hunter/ansible/roles/services/tasks/main.yml +++ b/hunter/ansible/roles/services/tasks/main.yml @@ -1,4 +1,12 @@ --- +- name: Clone or update magenta repository for services + git: + repo: https://github.com/magent-cryptograss/magenta.git + dest: "{{ base_dir }}/magenta-source" + version: production + force: yes + update: yes + - name: Create .env file with secrets for services copy: dest: "{{ base_dir }}/magenta-source/.env" @@ -6,6 +14,12 @@ POSTGRES_PASSWORD={{ postgres_password }} mode: '0600' +- name: Build shared services images + command: docker compose -f docker-compose.services.yml build {{ '--no-cache' if docker_no_cache | default(false) else '' }} + args: + chdir: "{{ base_dir }}/magenta-source" + when: docker_no_cache | default(false) + - name: Start shared services (MCP server, Memory Lane, Watcher) command: docker compose -f docker-compose.services.yml up -d --build --force-recreate args: diff --git a/maybelle/ansible/maybelle.yml b/maybelle/ansible/maybelle.yml index 53064f5..ff277be 100644 --- a/maybelle/ansible/maybelle.yml +++ b/maybelle/ansible/maybelle.yml @@ -33,9 +33,23 @@ - software-properties-common - docker.io - ansible + - python3-pip + - python3-venv state: present update_cache: yes + - name: Create ops tools venv + command: python3 -m venv /opt/magenta-ops + args: + creates: /opt/magenta-ops/bin/python + + - name: Install ops dependencies in venv + pip: + name: + - psycopg2-binary + - pyyaml + virtualenv: /opt/magenta-ops + - name: Create Jenkins home directory file: path: "{{ jenkins_home }}" @@ -127,12 +141,6 @@ include_role: name: caddy - - name: Generate SSH key for root - openssh_keypair: - path: "~/.ssh/id_ed25519" - type: ed25519 - state: present - - name: Create .ssh directory for jenkins user file: path: "{{ jenkins_home }}/.ssh" @@ -166,6 +174,14 @@ owner: root group: root + - name: Deploy NFS SSH key from vault for production deployment + copy: + content: "{{ jenkins_nfs_ssh_key | b64decode }}" + dest: "/var/jenkins_home/.ssh/id_ed25519_nfs" + mode: '0600' + owner: 1000 + group: 1000 + - name: Add hunter to known_hosts ansible.builtin.shell: cmd: "ssh-keyscan -H hunter.cryptograss.live >> /root/.ssh/known_hosts" @@ -203,11 +219,6 @@ group: 1000 mode: '0644' - - name: Cronjob to rsync production builds to web servers - ansible.builtin.cron: - name: "push sites to prod" - job: "rsync -vah --progress --delete /var/jenkins_home/www/builds/production/* jmyles_justinholmescom@ssh.nyc1.nearlyfreespeech.net: >> ~/sites-rsync.log 2>&1" - - name: Create Jenkins job definitions directory file: path: "{{ jenkins_home }}/casc_configs/jobs" @@ -252,15 +263,6 @@ owner: www-data group: www-data - - name: Slurp public SSH key # TODO: This is only useful as the jenkins user, which doesn't have a key yet. Better to custody the private key and copy it. #230 - ansible.builtin.slurp: - src: ~/.ssh/id_ed25519.pub - register: ssh_trinket - - - name: Print SSH trinket - ansible.builtin.debug: - msg: "{{ ssh_trinket['content'] | b64decode }}" - handlers: - name: Restart Jenkins container docker_container: diff --git a/maybelle/scripts/deploy-hunter-remote.py b/maybelle/scripts/deploy-hunter-remote.py index 82f5cbd..3323361 100755 --- a/maybelle/scripts/deploy-hunter-remote.py +++ b/maybelle/scripts/deploy-hunter-remote.py @@ -9,6 +9,7 @@ import sys import tempfile import os +import argparse from pathlib import Path from datetime import datetime @@ -95,7 +96,7 @@ def cleanup_ssh_agent(): subprocess.run(['ssh-agent', '-k'], capture_output=True) -def deploy_hunter(backup_file, vault_password): +def deploy_hunter(backup_file, vault_password, no_cache=False): """Run ansible deployment on hunter FROM maybelle""" print("\n" + "=" * 60) print("DEPLOYING HUNTER") @@ -138,16 +139,29 @@ def deploy_hunter(backup_file, vault_password): try: # Build ansible command using the temp vault file # Run from hunter/ansible directory so ansible.cfg is found + extra_vars = [] if backup_file: print(f"Using database backup: {backup_file}\n") - ansible_cmd = f"cd /root/maybelle-config/hunter/ansible && ansible-playbook --vault-password-file={vault_file_path} -i inventory.yml playbook.yml -e db_backup_file=/var/jenkins_home/hunter-db-backups/{backup_file}" + extra_vars.append(f"db_backup_file=/var/jenkins_home/hunter-db-backups/{backup_file}") else: print("Skipping database restoration\n") - ansible_cmd = f"cd /root/maybelle-config/hunter/ansible && ansible-playbook --vault-password-file={vault_file_path} -i inventory.yml playbook.yml" - # Run ansible FROM maybelle (ansible SSHs to hunter using our forwarded agent) + if no_cache: + print("Docker build cache disabled\n") + extra_vars.append("docker_no_cache=true") + + extra_vars_str = " -e ".join(extra_vars) + if extra_vars_str: + extra_vars_str = f" -e {extra_vars_str}" + + ansible_cmd = f"cd /root/maybelle-config/hunter/ansible && ansible-playbook --vault-password-file={vault_file_path} -i inventory.yml playbook.yml{extra_vars_str}" + + # Run ansible with SSH keepalive to prevent connection drops result = subprocess.run( - ['ssh', '-A', '-t', 'root@maybelle.cryptograss.live', ansible_cmd], + ['ssh', '-A', '-t', + '-o', 'ServerAliveInterval=30', + '-o', 'ServerAliveCountMax=10', + 'root@maybelle.cryptograss.live', ansible_cmd], check=False ) finally: @@ -183,6 +197,11 @@ def get_vault_password(): def main(): + parser = argparse.ArgumentParser(description='Deploy hunter via maybelle') + parser.add_argument('--no-cache', action='store_true', + help='Disable Docker build cache (use when magenta code has changed)') + args = parser.parse_args() + print("=" * 60) print("DEPLOY HUNTER VIA MAYBELLE") print("=" * 60) @@ -220,7 +239,7 @@ def main(): try: # Deploy - deploy_hunter(backup_file, vault_password) + deploy_hunter(backup_file, vault_password, no_cache=args.no_cache) print("\n✓ SUCCESS") except Exception as e: diff --git a/maybelle/scripts/redact_secrets_remote.py b/maybelle/scripts/redact_secrets_remote.py new file mode 100644 index 0000000..9c64597 --- /dev/null +++ b/maybelle/scripts/redact_secrets_remote.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +""" +Redact secrets from magenta database - run from maybelle. + +This script connects directly to the PostgreSQL database on hunter +and redacts vault secrets from message content. + +Usage (from maybelle): + export ANSIBLE_VAULT_PASSWORD=$(cat ~/.vault_pass) + python scripts/redact_secrets_remote.py --dry-run --verbose + python scripts/redact_secrets_remote.py # actually redact +""" + +import os +import sys +import json +import argparse +import psycopg2 +from pathlib import Path + +# Add project root for security module +sys.path.insert(0, str(Path(__file__).parent.parent)) +from security.secrets_filter import SecretsFilter + + +def main(): + parser = argparse.ArgumentParser(description='Redact secrets from magenta database') + parser.add_argument( + '--vault-path', + type=str, + default=os.path.expanduser('~/workspace/maybelle-config/group_vars/all/vault.yml'), + help='Path to Ansible vault file' + ) + parser.add_argument( + '--db-host', + type=str, + default='hunter.cryptograss.live', + help='Database host' + ) + parser.add_argument( + '--db-port', + type=int, + default=5432, + help='Database port' + ) + parser.add_argument( + '--db-name', + type=str, + default='magenta_memory', + help='Database name' + ) + parser.add_argument( + '--db-user', + type=str, + default='magent', + help='Database user' + ) + parser.add_argument( + '--dry-run', + action='store_true', + help='Show what would be redacted without making changes' + ) + parser.add_argument( + '--verbose', + action='store_true', + help='Show detailed output' + ) + parser.add_argument( + '--limit', + type=int, + default=None, + help='Limit number of messages to process' + ) + + args = parser.parse_args() + + # Get vault password + vault_password = os.environ.get('ANSIBLE_VAULT_PASSWORD') + if not vault_password: + print("ERROR: ANSIBLE_VAULT_PASSWORD environment variable not set") + print("Run: export ANSIBLE_VAULT_PASSWORD=$(cat ~/.vault_pass)") + sys.exit(1) + + # Get database password from vault + db_password = os.environ.get('MAGENTA_DB_PASSWORD') + if not db_password: + print("ERROR: MAGENTA_DB_PASSWORD environment variable not set") + print("Set it to the memory_lane_postgres_password from vault") + sys.exit(1) + + # Initialize secrets filter + print(f"Loading secrets from vault: {args.vault_path}") + secrets_filter = SecretsFilter(vault_path=args.vault_path, vault_password=vault_password) + + if not secrets_filter.secrets: + print("ERROR: No secrets loaded from vault") + sys.exit(1) + + print(f"Loaded {len(secrets_filter.secrets)} secrets to scrub") + + # Connect to database + print(f"Connecting to {args.db_host}:{args.db_port}/{args.db_name}...") + conn = psycopg2.connect( + host=args.db_host, + port=args.db_port, + dbname=args.db_name, + user=args.db_user, + password=db_password + ) + + try: + with conn.cursor() as cur: + # Count messages + cur.execute("SELECT COUNT(*) FROM conversations_message") + total = cur.fetchone()[0] + print(f"Total messages in database: {total}") + + # Query messages + query = "SELECT id, sender_id, content, created_at FROM conversations_message ORDER BY created_at" + if args.limit: + query += f" LIMIT {args.limit}" + + cur.execute(query) + + redacted_count = 0 + updates = [] + + for row in cur: + msg_id, sender_id, content, created_at = row + + # Scrub the content + scrubbed_content = secrets_filter.scrub_json(content) + + # Check if anything changed + if scrubbed_content != content: + redacted_count += 1 + + if args.verbose: + print(f"\n--- Message {msg_id} ---") + print(f"Sender: {sender_id}") + print(f"Created: {created_at}") + orig_str = json.dumps(content)[:200] if content else '' + scrub_str = json.dumps(scrubbed_content)[:200] if scrubbed_content else '' + print(f"Original: {orig_str}...") + print(f"Scrubbed: {scrub_str}...") + + if not args.dry_run: + updates.append((json.dumps(scrubbed_content), str(msg_id))) + + # Apply updates + if not args.dry_run and updates: + print(f"\nApplying {len(updates)} updates...") + with conn.cursor() as update_cur: + for scrubbed_json, msg_id in updates: + update_cur.execute( + "UPDATE conversations_message SET content = %s WHERE id = %s", + (scrubbed_json, msg_id) + ) + conn.commit() + print(f"Successfully redacted secrets from {redacted_count} messages") + elif args.dry_run: + print(f"\nDRY RUN: Would redact secrets from {redacted_count} messages") + else: + print("No messages contained secrets to redact") + + print(f"\nMessages processed: {args.limit or total}") + print(f"Messages with secrets: {redacted_count}") + + finally: + conn.close() + + +if __name__ == '__main__': + main() diff --git a/maybelle/scripts/redact_secrets_via_maybelle.py b/maybelle/scripts/redact_secrets_via_maybelle.py new file mode 100644 index 0000000..ca5d4e4 --- /dev/null +++ b/maybelle/scripts/redact_secrets_via_maybelle.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +""" +Redact secrets from magenta database - run from your laptop via maybelle. + +This script SSHes to maybelle, where it: +1. Uses the local vault password to decrypt secrets +2. Connects to hunter's PostgreSQL database +3. Redacts vault secrets from message content + +Usage (from laptop): + export ANSIBLE_VAULT_PASSWORD='your-vault-password' + # or + export ANSIBLE_VAULT_PASSWORD_FILE=~/.vault_pass + + python scripts/redact_secrets_via_maybelle.py --dry-run --verbose + python scripts/redact_secrets_via_maybelle.py # actually redact +""" + +import subprocess +import sys +import os +import argparse +from pathlib import Path + + +def run_ssh(host, command, capture_output=False, check=True): + """Run SSH command""" + result = subprocess.run( + ['ssh', host, command], + capture_output=capture_output, + text=True, + check=check + ) + return result + + +def get_vault_password(): + """Get vault password from environment or file""" + password = os.environ.get('ANSIBLE_VAULT_PASSWORD') + if password: + return password + + password_file = os.environ.get('ANSIBLE_VAULT_PASSWORD_FILE') + if password_file: + try: + with open(os.path.expanduser(password_file), 'r') as f: + return f.read().strip() + except FileNotFoundError: + raise Exception(f"Vault password file not found: {password_file}") + + raise Exception("Neither ANSIBLE_VAULT_PASSWORD nor ANSIBLE_VAULT_PASSWORD_FILE is set") + + +def main(): + parser = argparse.ArgumentParser(description='Redact secrets from magenta database via maybelle') + parser.add_argument( + '--dry-run', + action='store_true', + help='Show what would be redacted without making changes' + ) + parser.add_argument( + '--verbose', + action='store_true', + help='Show detailed output' + ) + parser.add_argument( + '--limit', + type=int, + default=None, + help='Limit number of messages to process' + ) + + args = parser.parse_args() + + print("=" * 60) + print("REDACT SECRETS VIA MAYBELLE") + print("=" * 60) + + # Get vault password + try: + vault_password = get_vault_password() + print("✓ Vault password found") + except Exception as e: + print(f"\n✗ ERROR: {e}") + print("\nSet one of:") + print(" export ANSIBLE_VAULT_PASSWORD='your-password'") + print(" export ANSIBLE_VAULT_PASSWORD_FILE='~/.vault_pass'") + sys.exit(1) + + # Ensure maybelle-config repo is up to date on maybelle + print("\nUpdating maybelle-config repository on maybelle...") + repo_setup = ''' + if [ ! -d /root/maybelle-config ]; then + git clone https://github.com/cryptograss/maybelle-config.git /root/maybelle-config + fi + cd /root/maybelle-config + git fetch origin + git checkout production || git checkout -b production origin/production + git reset --hard origin/production + ''' + run_ssh('root@maybelle.cryptograss.live', repo_setup) + print("✓ Repository updated") + + # Create temp vault password file on maybelle + print("\nSetting up vault password on maybelle...") + vault_file_path = f'/tmp/vault_pass_{os.getpid()}' + + # Escape single quotes in password for shell + escaped_password = vault_password.replace("'", "'\\''") + write_vault = f"echo '{escaped_password}' > {vault_file_path} && chmod 600 {vault_file_path}" + run_ssh('root@maybelle.cryptograss.live', write_vault) + print("✓ Vault password configured") + + try: + # Build the command to run on maybelle + # The script will decrypt vault to get DB password, then connect to hunter + + # Build args + script_args = [] + if args.dry_run: + script_args.append('--dry-run') + if args.verbose: + script_args.append('--verbose') + if args.limit: + script_args.append(f'--limit {args.limit}') + + args_str = ' '.join(script_args) + + # Run redaction script on maybelle + # First, get the DB password from vault + redact_cmd = f''' + cd /root/maybelle-config + + # Get DB password from vault + export MAGENTA_DB_PASSWORD=$(ansible-vault view --vault-password-file={vault_file_path} /root/maybelle-config/secrets/vault.yml 2>/dev/null | grep memory_lane_postgres_password | awk '{{print $2}}') + + if [ -z "$MAGENTA_DB_PASSWORD" ]; then + echo "ERROR: Could not extract DB password from vault" + exit 1 + fi + + # Run the redaction script using ops venv + /opt/magenta-ops/bin/python maybelle/scripts/redact_secrets_remote.py {args_str} + ''' + + print("\nRunning redaction on maybelle...") + print("-" * 60) + + result = subprocess.run( + ['ssh', '-t', 'root@maybelle.cryptograss.live', redact_cmd], + check=False + ) + + print("-" * 60) + + if result.returncode != 0: + print(f"\n✗ Redaction failed with exit code {result.returncode}") + sys.exit(1) + + print("\n✓ Redaction complete") + + finally: + # Clean up vault password file + print("\nCleaning up...") + run_ssh('root@maybelle.cryptograss.live', f'rm -f {vault_file_path}', check=False) + print("✓ Vault password removed from maybelle") + + +if __name__ == '__main__': + main()