From adf2f4d5afc268f3f6ec4a00734de766326b3874 Mon Sep 17 00:00:00 2001 From: magent-cryptograss Date: Wed, 19 Nov 2025 20:49:13 +0000 Subject: [PATCH 01/14] Deploy NFS SSH key and prune unused tasks - Add NFS SSH key deployment for Jenkins production rsync - Remove unused root SSH key generation - Remove unused slurp/print SSH trinket tasks - Remove cronjob (now handled by Jenkins pipeline) --- maybelle/ansible/maybelle.yml | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/maybelle/ansible/maybelle.yml b/maybelle/ansible/maybelle.yml index 53064f5..15dc789 100644 --- a/maybelle/ansible/maybelle.yml +++ b/maybelle/ansible/maybelle.yml @@ -127,12 +127,6 @@ include_role: name: caddy - - name: Generate SSH key for root - openssh_keypair: - path: "~/.ssh/id_ed25519" - type: ed25519 - state: present - - name: Create .ssh directory for jenkins user file: path: "{{ jenkins_home }}/.ssh" @@ -166,6 +160,14 @@ owner: root group: root + - name: Deploy NFS SSH key from vault for production deployment + copy: + content: "{{ jenkins_nfs_ssh_key | b64decode }}" + dest: "/var/jenkins_home/.ssh/id_ed25519_nfs" + mode: '0600' + owner: 1000 + group: 1000 + - name: Add hunter to known_hosts ansible.builtin.shell: cmd: "ssh-keyscan -H hunter.cryptograss.live >> /root/.ssh/known_hosts" @@ -203,11 +205,6 @@ group: 1000 mode: '0644' - - name: Cronjob to rsync production builds to web servers - ansible.builtin.cron: - name: "push sites to prod" - job: "rsync -vah --progress --delete /var/jenkins_home/www/builds/production/* jmyles_justinholmescom@ssh.nyc1.nearlyfreespeech.net: >> ~/sites-rsync.log 2>&1" - - name: Create Jenkins job definitions directory file: path: "{{ jenkins_home }}/casc_configs/jobs" @@ -252,15 +249,6 @@ owner: www-data group: www-data - - name: Slurp public SSH key # TODO: This is only useful as the jenkins user, which doesn't have a key yet. Better to custody the private key and copy it. #230 - ansible.builtin.slurp: - src: ~/.ssh/id_ed25519.pub - register: ssh_trinket - - - name: Print SSH trinket - ansible.builtin.debug: - msg: "{{ ssh_trinket['content'] | b64decode }}" - handlers: - name: Restart Jenkins container docker_container: From 8626507349ebfd93a1437e01b6a5634c356a15f5 Mon Sep 17 00:00:00 2001 From: magent-cryptograss Date: Wed, 19 Nov 2025 23:59:04 +0000 Subject: [PATCH 02/14] Add --no-cache flag to deploy-hunter-remote script Use when magenta code has changed and Docker needs to re-clone the repo instead of using cached layers. Usage: python deploy-hunter-remote.py --no-cache --- .../ansible/roles/build-image/tasks/main.yml | 2 +- maybelle/scripts/deploy-hunter-remote.py | 24 +++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/hunter/ansible/roles/build-image/tasks/main.yml b/hunter/ansible/roles/build-image/tasks/main.yml index 2e73e57..deb4846 100644 --- a/hunter/ansible/roles/build-image/tasks/main.yml +++ b/hunter/ansible/roles/build-image/tasks/main.yml @@ -17,7 +17,7 @@ register: docker_gid - name: Build magenta-arthel base image - command: docker compose -f docker-compose-build.yml build --build-arg DOCKER_GID={{ docker_gid.stdout }} + command: docker compose -f docker-compose-build.yml build {{ '--no-cache' if docker_no_cache | default(false) else '' }} --build-arg DOCKER_GID={{ docker_gid.stdout }} args: chdir: "{{ base_dir }}/maybelle-config/hunter" diff --git a/maybelle/scripts/deploy-hunter-remote.py b/maybelle/scripts/deploy-hunter-remote.py index 82f5cbd..2e05296 100755 --- a/maybelle/scripts/deploy-hunter-remote.py +++ b/maybelle/scripts/deploy-hunter-remote.py @@ -9,6 +9,7 @@ import sys import tempfile import os +import argparse from pathlib import Path from datetime import datetime @@ -95,7 +96,7 @@ def cleanup_ssh_agent(): subprocess.run(['ssh-agent', '-k'], capture_output=True) -def deploy_hunter(backup_file, vault_password): +def deploy_hunter(backup_file, vault_password, no_cache=False): """Run ansible deployment on hunter FROM maybelle""" print("\n" + "=" * 60) print("DEPLOYING HUNTER") @@ -138,12 +139,22 @@ def deploy_hunter(backup_file, vault_password): try: # Build ansible command using the temp vault file # Run from hunter/ansible directory so ansible.cfg is found + extra_vars = [] if backup_file: print(f"Using database backup: {backup_file}\n") - ansible_cmd = f"cd /root/maybelle-config/hunter/ansible && ansible-playbook --vault-password-file={vault_file_path} -i inventory.yml playbook.yml -e db_backup_file=/var/jenkins_home/hunter-db-backups/{backup_file}" + extra_vars.append(f"db_backup_file=/var/jenkins_home/hunter-db-backups/{backup_file}") else: print("Skipping database restoration\n") - ansible_cmd = f"cd /root/maybelle-config/hunter/ansible && ansible-playbook --vault-password-file={vault_file_path} -i inventory.yml playbook.yml" + + if no_cache: + print("Docker build cache disabled\n") + extra_vars.append("docker_no_cache=true") + + extra_vars_str = " -e ".join(extra_vars) + if extra_vars_str: + extra_vars_str = f" -e {extra_vars_str}" + + ansible_cmd = f"cd /root/maybelle-config/hunter/ansible && ansible-playbook --vault-password-file={vault_file_path} -i inventory.yml playbook.yml{extra_vars_str}" # Run ansible FROM maybelle (ansible SSHs to hunter using our forwarded agent) result = subprocess.run( @@ -183,6 +194,11 @@ def get_vault_password(): def main(): + parser = argparse.ArgumentParser(description='Deploy hunter via maybelle') + parser.add_argument('--no-cache', action='store_true', + help='Disable Docker build cache (use when magenta code has changed)') + args = parser.parse_args() + print("=" * 60) print("DEPLOY HUNTER VIA MAYBELLE") print("=" * 60) @@ -220,7 +236,7 @@ def main(): try: # Deploy - deploy_hunter(backup_file, vault_password) + deploy_hunter(backup_file, vault_password, no_cache=args.no_cache) print("\n✓ SUCCESS") except Exception as e: From 096d4d593816f964e916b5b12b92f4c291e3e0d8 Mon Sep 17 00:00:00 2001 From: magent-cryptograss Date: Thu, 20 Nov 2025 00:35:29 +0000 Subject: [PATCH 03/14] Add --no-cache support for services build --- hunter/ansible/roles/services/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hunter/ansible/roles/services/tasks/main.yml b/hunter/ansible/roles/services/tasks/main.yml index 3e4159f..7c07ef9 100644 --- a/hunter/ansible/roles/services/tasks/main.yml +++ b/hunter/ansible/roles/services/tasks/main.yml @@ -7,7 +7,7 @@ mode: '0600' - name: Start shared services (MCP server, Memory Lane, Watcher) - command: docker compose -f docker-compose.services.yml up -d --build --force-recreate + command: docker compose -f docker-compose.services.yml up -d --build {{ '--no-cache' if docker_no_cache | default(false) else '' }} --force-recreate args: chdir: "{{ base_dir }}/magenta-source" From 35137a5ab9e6481f9a915ae4f82427f6bab6403a Mon Sep 17 00:00:00 2001 From: magent-cryptograss Date: Thu, 20 Nov 2025 01:17:58 +0000 Subject: [PATCH 04/14] Use tmux for ansible deployment to survive connection drops Now runs ansible in a tmux session on maybelle instead of directly over SSH. If connection drops, the deployment continues and you can reconnect or check logs. --- maybelle/scripts/deploy-hunter-remote.py | 43 ++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/maybelle/scripts/deploy-hunter-remote.py b/maybelle/scripts/deploy-hunter-remote.py index 2e05296..3472d10 100755 --- a/maybelle/scripts/deploy-hunter-remote.py +++ b/maybelle/scripts/deploy-hunter-remote.py @@ -156,11 +156,50 @@ def deploy_hunter(backup_file, vault_password, no_cache=False): ansible_cmd = f"cd /root/maybelle-config/hunter/ansible && ansible-playbook --vault-password-file={vault_file_path} -i inventory.yml playbook.yml{extra_vars_str}" - # Run ansible FROM maybelle (ansible SSHs to hunter using our forwarded agent) + # Run ansible in tmux session on maybelle so it survives connection drops + session_name = f"hunter-deploy-{os.getpid()}" + log_file = f"/tmp/hunter-deploy-{os.getpid()}.log" + + # Create tmux session running ansible, logging to file + tmux_cmd = f"tmux new-session -d -s {session_name} 'set -o pipefail; {ansible_cmd} 2>&1 | tee {log_file}; echo EXIT_CODE=$? >> {log_file}'" + run_ssh('root@maybelle.cryptograss.live', tmux_cmd, forward_agent=True) + + print(f"Ansible running in tmux session '{session_name}' on maybelle") + print(f"Log file: {log_file}") + print("\nAttaching to session (Ctrl-B D to detach without stopping)...\n") + + # Attach to the session so user can watch result = subprocess.run( - ['ssh', '-A', '-t', 'root@maybelle.cryptograss.live', ansible_cmd], + ['ssh', '-A', '-t', 'root@maybelle.cryptograss.live', f'tmux attach -t {session_name}'], check=False ) + + # If we got disconnected, session may still be running + if result.returncode != 0: + print("\nConnection lost. Checking if deployment is still running...") + check_result = run_ssh( + 'root@maybelle.cryptograss.live', + f'tmux has-session -t {session_name} 2>/dev/null && echo "RUNNING" || echo "FINISHED"', + capture_output=True + ) + if "RUNNING" in check_result.stdout: + print(f"\nDeployment still running in tmux session '{session_name}'") + print(f"Reconnect with: ssh -A root@maybelle.cryptograss.live tmux attach -t {session_name}") + print(f"Or check logs: ssh root@maybelle.cryptograss.live cat {log_file}") + return + else: + # Session finished, check exit code from log + exit_check = run_ssh( + 'root@maybelle.cryptograss.live', + f'grep "EXIT_CODE=" {log_file} | tail -1', + capture_output=True + ) + if "EXIT_CODE=0" in exit_check.stdout: + print("Deployment completed successfully!") + result.returncode = 0 + else: + print(f"Deployment may have failed. Check log: {log_file}") + result.returncode = 1 finally: # Clean up vault password file run_ssh('root@maybelle.cryptograss.live', f'rm -f {vault_file_path}', forward_agent=True, check=False) From 37073756292d16ff729e9833c4e245472345bd31 Mon Sep 17 00:00:00 2001 From: magent-cryptograss Date: Thu, 20 Nov 2025 01:27:55 +0000 Subject: [PATCH 05/14] Revert to direct SSH with keepalive (tmux breaks agent forwarding) --- maybelle/scripts/deploy-hunter-remote.py | 46 +++--------------------- 1 file changed, 5 insertions(+), 41 deletions(-) diff --git a/maybelle/scripts/deploy-hunter-remote.py b/maybelle/scripts/deploy-hunter-remote.py index 3472d10..3323361 100755 --- a/maybelle/scripts/deploy-hunter-remote.py +++ b/maybelle/scripts/deploy-hunter-remote.py @@ -156,50 +156,14 @@ def deploy_hunter(backup_file, vault_password, no_cache=False): ansible_cmd = f"cd /root/maybelle-config/hunter/ansible && ansible-playbook --vault-password-file={vault_file_path} -i inventory.yml playbook.yml{extra_vars_str}" - # Run ansible in tmux session on maybelle so it survives connection drops - session_name = f"hunter-deploy-{os.getpid()}" - log_file = f"/tmp/hunter-deploy-{os.getpid()}.log" - - # Create tmux session running ansible, logging to file - tmux_cmd = f"tmux new-session -d -s {session_name} 'set -o pipefail; {ansible_cmd} 2>&1 | tee {log_file}; echo EXIT_CODE=$? >> {log_file}'" - run_ssh('root@maybelle.cryptograss.live', tmux_cmd, forward_agent=True) - - print(f"Ansible running in tmux session '{session_name}' on maybelle") - print(f"Log file: {log_file}") - print("\nAttaching to session (Ctrl-B D to detach without stopping)...\n") - - # Attach to the session so user can watch + # Run ansible with SSH keepalive to prevent connection drops result = subprocess.run( - ['ssh', '-A', '-t', 'root@maybelle.cryptograss.live', f'tmux attach -t {session_name}'], + ['ssh', '-A', '-t', + '-o', 'ServerAliveInterval=30', + '-o', 'ServerAliveCountMax=10', + 'root@maybelle.cryptograss.live', ansible_cmd], check=False ) - - # If we got disconnected, session may still be running - if result.returncode != 0: - print("\nConnection lost. Checking if deployment is still running...") - check_result = run_ssh( - 'root@maybelle.cryptograss.live', - f'tmux has-session -t {session_name} 2>/dev/null && echo "RUNNING" || echo "FINISHED"', - capture_output=True - ) - if "RUNNING" in check_result.stdout: - print(f"\nDeployment still running in tmux session '{session_name}'") - print(f"Reconnect with: ssh -A root@maybelle.cryptograss.live tmux attach -t {session_name}") - print(f"Or check logs: ssh root@maybelle.cryptograss.live cat {log_file}") - return - else: - # Session finished, check exit code from log - exit_check = run_ssh( - 'root@maybelle.cryptograss.live', - f'grep "EXIT_CODE=" {log_file} | tail -1', - capture_output=True - ) - if "EXIT_CODE=0" in exit_check.stdout: - print("Deployment completed successfully!") - result.returncode = 0 - else: - print(f"Deployment may have failed. Check log: {log_file}") - result.returncode = 1 finally: # Clean up vault password file run_ssh('root@maybelle.cryptograss.live', f'rm -f {vault_file_path}', forward_agent=True, check=False) From 59c15732faff1cca49a08e0c4c4d03bdd0e02107 Mon Sep 17 00:00:00 2001 From: magent-cryptograss Date: Thu, 20 Nov 2025 01:37:16 +0000 Subject: [PATCH 06/14] Fix: separate build --no-cache from up command for services --- hunter/ansible/roles/services/tasks/main.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hunter/ansible/roles/services/tasks/main.yml b/hunter/ansible/roles/services/tasks/main.yml index 7c07ef9..e3def04 100644 --- a/hunter/ansible/roles/services/tasks/main.yml +++ b/hunter/ansible/roles/services/tasks/main.yml @@ -6,8 +6,14 @@ POSTGRES_PASSWORD={{ postgres_password }} mode: '0600' +- name: Build shared services images + command: docker compose -f docker-compose.services.yml build {{ '--no-cache' if docker_no_cache | default(false) else '' }} + args: + chdir: "{{ base_dir }}/magenta-source" + when: docker_no_cache | default(false) + - name: Start shared services (MCP server, Memory Lane, Watcher) - command: docker compose -f docker-compose.services.yml up -d --build {{ '--no-cache' if docker_no_cache | default(false) else '' }} --force-recreate + command: docker compose -f docker-compose.services.yml up -d --build --force-recreate args: chdir: "{{ base_dir }}/magenta-source" From e2132f7d611351e519164db7c609a13f98e2a3d8 Mon Sep 17 00:00:00 2001 From: magent-cryptograss Date: Thu, 20 Nov 2025 01:55:37 +0000 Subject: [PATCH 07/14] Add git clone/update for magenta-source before services build --- hunter/ansible/roles/services/tasks/main.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hunter/ansible/roles/services/tasks/main.yml b/hunter/ansible/roles/services/tasks/main.yml index e3def04..71d2e2b 100644 --- a/hunter/ansible/roles/services/tasks/main.yml +++ b/hunter/ansible/roles/services/tasks/main.yml @@ -1,4 +1,12 @@ --- +- name: Clone or update magenta repository for services + git: + repo: https://github.com/magent-cryptograss/magenta.git + dest: "{{ base_dir }}/magenta-source" + version: production + force: yes + update: yes + - name: Create .env file with secrets for services copy: dest: "{{ base_dir }}/magenta-source/.env" From c1d35bc62ee51fd0f611de4a326672e33411dfc1 Mon Sep 17 00:00:00 2001 From: magent-cryptograss Date: Thu, 20 Nov 2025 04:38:44 +0000 Subject: [PATCH 08/14] Add scripts to redact vault secrets from magenta database --- maybelle/scripts/redact_secrets_remote.py | 174 ++++++++++++++++++ .../scripts/redact_secrets_via_maybelle.py | 174 ++++++++++++++++++ 2 files changed, 348 insertions(+) create mode 100644 maybelle/scripts/redact_secrets_remote.py create mode 100644 maybelle/scripts/redact_secrets_via_maybelle.py diff --git a/maybelle/scripts/redact_secrets_remote.py b/maybelle/scripts/redact_secrets_remote.py new file mode 100644 index 0000000..9c64597 --- /dev/null +++ b/maybelle/scripts/redact_secrets_remote.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +""" +Redact secrets from magenta database - run from maybelle. + +This script connects directly to the PostgreSQL database on hunter +and redacts vault secrets from message content. + +Usage (from maybelle): + export ANSIBLE_VAULT_PASSWORD=$(cat ~/.vault_pass) + python scripts/redact_secrets_remote.py --dry-run --verbose + python scripts/redact_secrets_remote.py # actually redact +""" + +import os +import sys +import json +import argparse +import psycopg2 +from pathlib import Path + +# Add project root for security module +sys.path.insert(0, str(Path(__file__).parent.parent)) +from security.secrets_filter import SecretsFilter + + +def main(): + parser = argparse.ArgumentParser(description='Redact secrets from magenta database') + parser.add_argument( + '--vault-path', + type=str, + default=os.path.expanduser('~/workspace/maybelle-config/group_vars/all/vault.yml'), + help='Path to Ansible vault file' + ) + parser.add_argument( + '--db-host', + type=str, + default='hunter.cryptograss.live', + help='Database host' + ) + parser.add_argument( + '--db-port', + type=int, + default=5432, + help='Database port' + ) + parser.add_argument( + '--db-name', + type=str, + default='magenta_memory', + help='Database name' + ) + parser.add_argument( + '--db-user', + type=str, + default='magent', + help='Database user' + ) + parser.add_argument( + '--dry-run', + action='store_true', + help='Show what would be redacted without making changes' + ) + parser.add_argument( + '--verbose', + action='store_true', + help='Show detailed output' + ) + parser.add_argument( + '--limit', + type=int, + default=None, + help='Limit number of messages to process' + ) + + args = parser.parse_args() + + # Get vault password + vault_password = os.environ.get('ANSIBLE_VAULT_PASSWORD') + if not vault_password: + print("ERROR: ANSIBLE_VAULT_PASSWORD environment variable not set") + print("Run: export ANSIBLE_VAULT_PASSWORD=$(cat ~/.vault_pass)") + sys.exit(1) + + # Get database password from vault + db_password = os.environ.get('MAGENTA_DB_PASSWORD') + if not db_password: + print("ERROR: MAGENTA_DB_PASSWORD environment variable not set") + print("Set it to the memory_lane_postgres_password from vault") + sys.exit(1) + + # Initialize secrets filter + print(f"Loading secrets from vault: {args.vault_path}") + secrets_filter = SecretsFilter(vault_path=args.vault_path, vault_password=vault_password) + + if not secrets_filter.secrets: + print("ERROR: No secrets loaded from vault") + sys.exit(1) + + print(f"Loaded {len(secrets_filter.secrets)} secrets to scrub") + + # Connect to database + print(f"Connecting to {args.db_host}:{args.db_port}/{args.db_name}...") + conn = psycopg2.connect( + host=args.db_host, + port=args.db_port, + dbname=args.db_name, + user=args.db_user, + password=db_password + ) + + try: + with conn.cursor() as cur: + # Count messages + cur.execute("SELECT COUNT(*) FROM conversations_message") + total = cur.fetchone()[0] + print(f"Total messages in database: {total}") + + # Query messages + query = "SELECT id, sender_id, content, created_at FROM conversations_message ORDER BY created_at" + if args.limit: + query += f" LIMIT {args.limit}" + + cur.execute(query) + + redacted_count = 0 + updates = [] + + for row in cur: + msg_id, sender_id, content, created_at = row + + # Scrub the content + scrubbed_content = secrets_filter.scrub_json(content) + + # Check if anything changed + if scrubbed_content != content: + redacted_count += 1 + + if args.verbose: + print(f"\n--- Message {msg_id} ---") + print(f"Sender: {sender_id}") + print(f"Created: {created_at}") + orig_str = json.dumps(content)[:200] if content else '' + scrub_str = json.dumps(scrubbed_content)[:200] if scrubbed_content else '' + print(f"Original: {orig_str}...") + print(f"Scrubbed: {scrub_str}...") + + if not args.dry_run: + updates.append((json.dumps(scrubbed_content), str(msg_id))) + + # Apply updates + if not args.dry_run and updates: + print(f"\nApplying {len(updates)} updates...") + with conn.cursor() as update_cur: + for scrubbed_json, msg_id in updates: + update_cur.execute( + "UPDATE conversations_message SET content = %s WHERE id = %s", + (scrubbed_json, msg_id) + ) + conn.commit() + print(f"Successfully redacted secrets from {redacted_count} messages") + elif args.dry_run: + print(f"\nDRY RUN: Would redact secrets from {redacted_count} messages") + else: + print("No messages contained secrets to redact") + + print(f"\nMessages processed: {args.limit or total}") + print(f"Messages with secrets: {redacted_count}") + + finally: + conn.close() + + +if __name__ == '__main__': + main() diff --git a/maybelle/scripts/redact_secrets_via_maybelle.py b/maybelle/scripts/redact_secrets_via_maybelle.py new file mode 100644 index 0000000..9c4e53f --- /dev/null +++ b/maybelle/scripts/redact_secrets_via_maybelle.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +""" +Redact secrets from magenta database - run from your laptop via maybelle. + +This script SSHes to maybelle, where it: +1. Uses the local vault password to decrypt secrets +2. Connects to hunter's PostgreSQL database +3. Redacts vault secrets from message content + +Usage (from laptop): + export ANSIBLE_VAULT_PASSWORD='your-vault-password' + # or + export ANSIBLE_VAULT_PASSWORD_FILE=~/.vault_pass + + python scripts/redact_secrets_via_maybelle.py --dry-run --verbose + python scripts/redact_secrets_via_maybelle.py # actually redact +""" + +import subprocess +import sys +import os +import argparse +from pathlib import Path + + +def run_ssh(host, command, capture_output=False, check=True): + """Run SSH command""" + result = subprocess.run( + ['ssh', host, command], + capture_output=capture_output, + text=True, + check=check + ) + return result + + +def get_vault_password(): + """Get vault password from environment or file""" + password = os.environ.get('ANSIBLE_VAULT_PASSWORD') + if password: + return password + + password_file = os.environ.get('ANSIBLE_VAULT_PASSWORD_FILE') + if password_file: + try: + with open(os.path.expanduser(password_file), 'r') as f: + return f.read().strip() + except FileNotFoundError: + raise Exception(f"Vault password file not found: {password_file}") + + raise Exception("Neither ANSIBLE_VAULT_PASSWORD nor ANSIBLE_VAULT_PASSWORD_FILE is set") + + +def main(): + parser = argparse.ArgumentParser(description='Redact secrets from magenta database via maybelle') + parser.add_argument( + '--dry-run', + action='store_true', + help='Show what would be redacted without making changes' + ) + parser.add_argument( + '--verbose', + action='store_true', + help='Show detailed output' + ) + parser.add_argument( + '--limit', + type=int, + default=None, + help='Limit number of messages to process' + ) + + args = parser.parse_args() + + print("=" * 60) + print("REDACT SECRETS VIA MAYBELLE") + print("=" * 60) + + # Get vault password + try: + vault_password = get_vault_password() + print("✓ Vault password found") + except Exception as e: + print(f"\n✗ ERROR: {e}") + print("\nSet one of:") + print(" export ANSIBLE_VAULT_PASSWORD='your-password'") + print(" export ANSIBLE_VAULT_PASSWORD_FILE='~/.vault_pass'") + sys.exit(1) + + # Ensure magenta repo is up to date on maybelle + print("\nUpdating magenta repository on maybelle...") + repo_setup = ''' + if [ ! -d /root/magenta ]; then + git clone https://github.com/magent-cryptograss/magenta.git /root/magenta + fi + cd /root/magenta + git fetch origin + git checkout main + git reset --hard origin/main + ''' + run_ssh('root@maybelle.cryptograss.live', repo_setup) + print("✓ Repository updated") + + # Create temp vault password file on maybelle + print("\nSetting up vault password on maybelle...") + vault_file_path = f'/tmp/vault_pass_{os.getpid()}' + + # Escape single quotes in password for shell + escaped_password = vault_password.replace("'", "'\\''") + write_vault = f"echo '{escaped_password}' > {vault_file_path} && chmod 600 {vault_file_path}" + run_ssh('root@maybelle.cryptograss.live', write_vault) + print("✓ Vault password configured") + + try: + # Build the command to run on maybelle + # The script will decrypt vault to get DB password, then connect to hunter + + # Build args + script_args = [] + if args.dry_run: + script_args.append('--dry-run') + if args.verbose: + script_args.append('--verbose') + if args.limit: + script_args.append(f'--limit {args.limit}') + + args_str = ' '.join(script_args) + + # Run redaction script on maybelle + # First, get the DB password from vault + redact_cmd = f''' + cd /root/magenta + + # Install dependencies if needed + pip3 install -q psycopg2-binary pyyaml 2>/dev/null || true + + # Get DB password from vault + export ANSIBLE_VAULT_PASSWORD=$(cat {vault_file_path}) + export MAGENTA_DB_PASSWORD=$(ansible-vault view /root/maybelle-config/group_vars/all/vault.yml 2>/dev/null | grep memory_lane_postgres_password | awk '{{print $2}}') + + if [ -z "$MAGENTA_DB_PASSWORD" ]; then + echo "ERROR: Could not extract DB password from vault" + exit 1 + fi + + # Run the redaction script + python3 scripts/redact_secrets_remote.py {args_str} + ''' + + print("\nRunning redaction on maybelle...") + print("-" * 60) + + result = subprocess.run( + ['ssh', '-t', 'root@maybelle.cryptograss.live', redact_cmd], + check=False + ) + + print("-" * 60) + + if result.returncode != 0: + print(f"\n✗ Redaction failed with exit code {result.returncode}") + sys.exit(1) + + print("\n✓ Redaction complete") + + finally: + # Clean up vault password file + print("\nCleaning up...") + run_ssh('root@maybelle.cryptograss.live', f'rm -f {vault_file_path}', check=False) + print("✓ Vault password removed from maybelle") + + +if __name__ == '__main__': + main() From d8117f708e591e1dc1a7bb65875f5505fa731ed7 Mon Sep 17 00:00:00 2001 From: magent-cryptograss Date: Thu, 20 Nov 2025 05:48:27 +0000 Subject: [PATCH 09/14] Fix vault path and ansible-vault invocation in redact script --- maybelle/scripts/redact_secrets_via_maybelle.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/maybelle/scripts/redact_secrets_via_maybelle.py b/maybelle/scripts/redact_secrets_via_maybelle.py index 9c4e53f..a4dc993 100644 --- a/maybelle/scripts/redact_secrets_via_maybelle.py +++ b/maybelle/scripts/redact_secrets_via_maybelle.py @@ -135,8 +135,7 @@ def main(): pip3 install -q psycopg2-binary pyyaml 2>/dev/null || true # Get DB password from vault - export ANSIBLE_VAULT_PASSWORD=$(cat {vault_file_path}) - export MAGENTA_DB_PASSWORD=$(ansible-vault view /root/maybelle-config/group_vars/all/vault.yml 2>/dev/null | grep memory_lane_postgres_password | awk '{{print $2}}') + export MAGENTA_DB_PASSWORD=$(ansible-vault view --vault-password-file={vault_file_path} /root/maybelle-config/secrets/vault.yml 2>/dev/null | grep memory_lane_postgres_password | awk '{{print $2}}') if [ -z "$MAGENTA_DB_PASSWORD" ]; then echo "ERROR: Could not extract DB password from vault" From 45455327faa82b4f1f7d778669bde79f5623378b Mon Sep 17 00:00:00 2001 From: magent-cryptograss Date: Thu, 20 Nov 2025 05:50:24 +0000 Subject: [PATCH 10/14] Fix repo paths - use maybelle-config instead of magenta --- maybelle/scripts/redact_secrets_via_maybelle.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/maybelle/scripts/redact_secrets_via_maybelle.py b/maybelle/scripts/redact_secrets_via_maybelle.py index a4dc993..c0374cc 100644 --- a/maybelle/scripts/redact_secrets_via_maybelle.py +++ b/maybelle/scripts/redact_secrets_via_maybelle.py @@ -87,13 +87,13 @@ def main(): print(" export ANSIBLE_VAULT_PASSWORD_FILE='~/.vault_pass'") sys.exit(1) - # Ensure magenta repo is up to date on maybelle - print("\nUpdating magenta repository on maybelle...") + # Ensure maybelle-config repo is up to date on maybelle + print("\nUpdating maybelle-config repository on maybelle...") repo_setup = ''' - if [ ! -d /root/magenta ]; then - git clone https://github.com/magent-cryptograss/magenta.git /root/magenta + if [ ! -d /root/maybelle-config ]; then + git clone https://github.com/cryptograss/maybelle-config.git /root/maybelle-config fi - cd /root/magenta + cd /root/maybelle-config git fetch origin git checkout main git reset --hard origin/main @@ -129,7 +129,7 @@ def main(): # Run redaction script on maybelle # First, get the DB password from vault redact_cmd = f''' - cd /root/magenta + cd /root/maybelle-config # Install dependencies if needed pip3 install -q psycopg2-binary pyyaml 2>/dev/null || true @@ -143,7 +143,7 @@ def main(): fi # Run the redaction script - python3 scripts/redact_secrets_remote.py {args_str} + python3 maybelle/scripts/redact_secrets_remote.py {args_str} ''' print("\nRunning redaction on maybelle...") From 4b46e929ec8992489e11bf05382fdb4fe027da2c Mon Sep 17 00:00:00 2001 From: magent-cryptograss Date: Thu, 20 Nov 2025 05:55:08 +0000 Subject: [PATCH 11/14] Temporarily checkout redact-secrets-script branch for testing --- maybelle/scripts/redact_secrets_via_maybelle.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maybelle/scripts/redact_secrets_via_maybelle.py b/maybelle/scripts/redact_secrets_via_maybelle.py index c0374cc..981452b 100644 --- a/maybelle/scripts/redact_secrets_via_maybelle.py +++ b/maybelle/scripts/redact_secrets_via_maybelle.py @@ -95,8 +95,8 @@ def main(): fi cd /root/maybelle-config git fetch origin - git checkout main - git reset --hard origin/main + git checkout redact-secrets-script + git reset --hard origin/redact-secrets-script ''' run_ssh('root@maybelle.cryptograss.live', repo_setup) print("✓ Repository updated") From f94985c496528d5bb5169ceb9db1d3e2013d0efb Mon Sep 17 00:00:00 2001 From: magent-cryptograss Date: Thu, 20 Nov 2025 05:57:31 +0000 Subject: [PATCH 12/14] Use production branch for deployment --- maybelle/scripts/redact_secrets_via_maybelle.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maybelle/scripts/redact_secrets_via_maybelle.py b/maybelle/scripts/redact_secrets_via_maybelle.py index 981452b..3bdd803 100644 --- a/maybelle/scripts/redact_secrets_via_maybelle.py +++ b/maybelle/scripts/redact_secrets_via_maybelle.py @@ -95,8 +95,8 @@ def main(): fi cd /root/maybelle-config git fetch origin - git checkout redact-secrets-script - git reset --hard origin/redact-secrets-script + git checkout production || git checkout -b production origin/production + git reset --hard origin/production ''' run_ssh('root@maybelle.cryptograss.live', repo_setup) print("✓ Repository updated") From e60864b0392edabd0c8af564467a432a661d5b51 Mon Sep 17 00:00:00 2001 From: magent-cryptograss Date: Thu, 20 Nov 2025 05:59:29 +0000 Subject: [PATCH 13/14] Show pip install output for debugging --- maybelle/scripts/redact_secrets_via_maybelle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maybelle/scripts/redact_secrets_via_maybelle.py b/maybelle/scripts/redact_secrets_via_maybelle.py index 3bdd803..c69048a 100644 --- a/maybelle/scripts/redact_secrets_via_maybelle.py +++ b/maybelle/scripts/redact_secrets_via_maybelle.py @@ -132,7 +132,7 @@ def main(): cd /root/maybelle-config # Install dependencies if needed - pip3 install -q psycopg2-binary pyyaml 2>/dev/null || true + pip3 install psycopg2-binary pyyaml # Get DB password from vault export MAGENTA_DB_PASSWORD=$(ansible-vault view --vault-password-file={vault_file_path} /root/maybelle-config/secrets/vault.yml 2>/dev/null | grep memory_lane_postgres_password | awk '{{print $2}}') From 00f6ceabca546ae30b9251654531b0ee6a991773 Mon Sep 17 00:00:00 2001 From: magent-cryptograss Date: Thu, 20 Nov 2025 06:03:35 +0000 Subject: [PATCH 14/14] Add ops venv to maybelle playbook with psycopg2 and pyyaml --- maybelle/ansible/maybelle.yml | 14 ++++++++++++++ maybelle/scripts/redact_secrets_via_maybelle.py | 7 ++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/maybelle/ansible/maybelle.yml b/maybelle/ansible/maybelle.yml index 15dc789..ff277be 100644 --- a/maybelle/ansible/maybelle.yml +++ b/maybelle/ansible/maybelle.yml @@ -33,9 +33,23 @@ - software-properties-common - docker.io - ansible + - python3-pip + - python3-venv state: present update_cache: yes + - name: Create ops tools venv + command: python3 -m venv /opt/magenta-ops + args: + creates: /opt/magenta-ops/bin/python + + - name: Install ops dependencies in venv + pip: + name: + - psycopg2-binary + - pyyaml + virtualenv: /opt/magenta-ops + - name: Create Jenkins home directory file: path: "{{ jenkins_home }}" diff --git a/maybelle/scripts/redact_secrets_via_maybelle.py b/maybelle/scripts/redact_secrets_via_maybelle.py index c69048a..ca5d4e4 100644 --- a/maybelle/scripts/redact_secrets_via_maybelle.py +++ b/maybelle/scripts/redact_secrets_via_maybelle.py @@ -131,9 +131,6 @@ def main(): redact_cmd = f''' cd /root/maybelle-config - # Install dependencies if needed - pip3 install psycopg2-binary pyyaml - # Get DB password from vault export MAGENTA_DB_PASSWORD=$(ansible-vault view --vault-password-file={vault_file_path} /root/maybelle-config/secrets/vault.yml 2>/dev/null | grep memory_lane_postgres_password | awk '{{print $2}}') @@ -142,8 +139,8 @@ def main(): exit 1 fi - # Run the redaction script - python3 maybelle/scripts/redact_secrets_remote.py {args_str} + # Run the redaction script using ops venv + /opt/magenta-ops/bin/python maybelle/scripts/redact_secrets_remote.py {args_str} ''' print("\nRunning redaction on maybelle...")