Add scripts to redact vault secrets from magenta database

hunter/ansible/roles/build-image/tasks/main.yml

@@ -17,7 +17,7 @@
   register: docker_gid
 
 - name: Build magenta-arthel base image
-  command: docker compose -f docker-compose-build.yml build --build-arg DOCKER_GID={{ docker_gid.stdout }}
+  command: docker compose -f docker-compose-build.yml build {{ '--no-cache' if docker_no_cache | default(false) else '' }} --build-arg DOCKER_GID={{ docker_gid.stdout }}
   args:
     chdir: "{{ base_dir }}/maybelle-config/hunter"
 

hunter/ansible/roles/services/tasks/main.yml

@@ -1,11 +1,25 @@
 ---
+- name: Clone or update magenta repository for services
+  git:
+    repo: https://github.com/magent-cryptograss/magenta.git
+    dest: "{{ base_dir }}/magenta-source"
+    version: production
+    force: yes
+    update: yes
+
 - name: Create .env file with secrets for services
   copy:
     dest: "{{ base_dir }}/magenta-source/.env"
     content: |
       POSTGRES_PASSWORD={{ postgres_password }}
     mode: '0600'
 
+- name: Build shared services images
+  command: docker compose -f docker-compose.services.yml build {{ '--no-cache' if docker_no_cache | default(false) else '' }}
+  args:
+    chdir: "{{ base_dir }}/magenta-source"
+  when: docker_no_cache | default(false)
+
 - name: Start shared services (MCP server, Memory Lane, Watcher)
   command: docker compose -f docker-compose.services.yml up -d --build --force-recreate
   args:

maybelle/ansible/maybelle.yml

@@ -33,9 +33,23 @@
           - software-properties-common
           - docker.io
           - ansible
+          - python3-pip
+          - python3-venv
         state: present
         update_cache: yes
 
+    - name: Create ops tools venv
+      command: python3 -m venv /opt/magenta-ops
+      args:
+        creates: /opt/magenta-ops/bin/python
+
+    - name: Install ops dependencies in venv
+      pip:
+        name:
+          - psycopg2-binary
+          - pyyaml
+        virtualenv: /opt/magenta-ops
+
     - name: Create Jenkins home directory
       file:
         path: "{{ jenkins_home }}"
@@ -127,12 +141,6 @@
       include_role:
         name: caddy
 
-    - name: Generate SSH key for root
-      openssh_keypair:
-        path: "~/.ssh/id_ed25519"
-        type: ed25519
-        state: present
-
     - name: Create .ssh directory for jenkins user
       file:
         path: "{{ jenkins_home }}/.ssh"
@@ -166,6 +174,14 @@
         owner: root
         group: root
 
+    - name: Deploy NFS SSH key from vault for production deployment
+      copy:
+        content: "{{ jenkins_nfs_ssh_key | b64decode }}"
+        dest: "/var/jenkins_home/.ssh/id_ed25519_nfs"
+        mode: '0600'
+        owner: 1000
+        group: 1000
+
     - name: Add hunter to known_hosts
       ansible.builtin.shell:
         cmd: "ssh-keyscan -H hunter.cryptograss.live >> /root/.ssh/known_hosts"
@@ -203,11 +219,6 @@
         group: 1000
         mode: '0644'
 
-    - name: Cronjob to rsync production builds to web servers
-      ansible.builtin.cron:
-        name: "push sites to prod"
-        job: "rsync -vah --progress --delete /var/jenkins_home/www/builds/production/* jmyles_justinholmescom@ssh.nyc1.nearlyfreespeech.net: >> ~/sites-rsync.log 2>&1"
-
     - name: Create Jenkins job definitions directory
       file:
         path: "{{ jenkins_home }}/casc_configs/jobs"
@@ -252,15 +263,6 @@
         owner: www-data
         group: www-data
 
-    - name: Slurp public SSH key  # TODO: This is only useful as the jenkins user, which doesn't have a key yet.  Better to custody the private key and copy it.  #230
-      ansible.builtin.slurp:
-        src: ~/.ssh/id_ed25519.pub
-      register: ssh_trinket
-
-    - name: Print SSH trinket
-      ansible.builtin.debug:
-        msg: "{{ ssh_trinket['content'] | b64decode }}"
-
   handlers:
     - name: Restart Jenkins container
       docker_container:

maybelle/scripts/deploy-hunter-remote.py

@@ -9,6 +9,7 @@
 import sys
 import tempfile
 import os
+import argparse
 from pathlib import Path
 from datetime import datetime
 
@@ -95,7 +96,7 @@ def cleanup_ssh_agent():
     subprocess.run(['ssh-agent', '-k'], capture_output=True)
 
 
-def deploy_hunter(backup_file, vault_password):
+def deploy_hunter(backup_file, vault_password, no_cache=False):
     """Run ansible deployment on hunter FROM maybelle"""
     print("\n" + "=" * 60)
     print("DEPLOYING HUNTER")
@@ -138,16 +139,29 @@ def deploy_hunter(backup_file, vault_password):
     try:
         # Build ansible command using the temp vault file
         # Run from hunter/ansible directory so ansible.cfg is found
+        extra_vars = []
         if backup_file:
             print(f"Using database backup: {backup_file}\n")
-            ansible_cmd = f"cd /root/maybelle-config/hunter/ansible && ansible-playbook --vault-password-file={vault_file_path} -i inventory.yml playbook.yml -e db_backup_file=/var/jenkins_home/hunter-db-backups/{backup_file}"
+            extra_vars.append(f"db_backup_file=/var/jenkins_home/hunter-db-backups/{backup_file}")
         else:
             print("Skipping database restoration\n")
-            ansible_cmd = f"cd /root/maybelle-config/hunter/ansible && ansible-playbook --vault-password-file={vault_file_path} -i inventory.yml playbook.yml"
 
-        # Run ansible FROM maybelle (ansible SSHs to hunter using our forwarded agent)
+        if no_cache:
+            print("Docker build cache disabled\n")
+            extra_vars.append("docker_no_cache=true")
+
+        extra_vars_str = " -e ".join(extra_vars)
+        if extra_vars_str:
+            extra_vars_str = f" -e {extra_vars_str}"
+
+        ansible_cmd = f"cd /root/maybelle-config/hunter/ansible && ansible-playbook --vault-password-file={vault_file_path} -i inventory.yml playbook.yml{extra_vars_str}"
+
+        # Run ansible with SSH keepalive to prevent connection drops
         result = subprocess.run(
-            ['ssh', '-A', '-t', 'root@maybelle.cryptograss.live', ansible_cmd],
+            ['ssh', '-A', '-t',
+             '-o', 'ServerAliveInterval=30',
+             '-o', 'ServerAliveCountMax=10',
+             'root@maybelle.cryptograss.live', ansible_cmd],
             check=False
         )
     finally:
@@ -183,6 +197,11 @@ def get_vault_password():
 
 
 def main():
+    parser = argparse.ArgumentParser(description='Deploy hunter via maybelle')
+    parser.add_argument('--no-cache', action='store_true',
+                        help='Disable Docker build cache (use when magenta code has changed)')
+    args = parser.parse_args()
+
     print("=" * 60)
     print("DEPLOY HUNTER VIA MAYBELLE")
     print("=" * 60)
@@ -220,7 +239,7 @@ def main():
 
     try:
         # Deploy
-        deploy_hunter(backup_file, vault_password)
+        deploy_hunter(backup_file, vault_password, no_cache=args.no_cache)
         print("\n✓ SUCCESS")
 
     except Exception as e:

maybelle/scripts/redact_secrets_remote.py

@@ -0,0 +1,174 @@
+#!/usr/bin/env python3
+"""
+Redact secrets from magenta database - run from maybelle.
+
+This script connects directly to the PostgreSQL database on hunter
+and redacts vault secrets from message content.
+
+Usage (from maybelle):
+    export ANSIBLE_VAULT_PASSWORD=$(cat ~/.vault_pass)
+    python scripts/redact_secrets_remote.py --dry-run --verbose
+    python scripts/redact_secrets_remote.py  # actually redact
+"""
+
+import os
+import sys
+import json
+import argparse
+import psycopg2
+from pathlib import Path
+
+# Add project root for security module
+sys.path.insert(0, str(Path(__file__).parent.parent))
+from security.secrets_filter import SecretsFilter
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Redact secrets from magenta database')
+    parser.add_argument(
+        '--vault-path',
+        type=str,
+        default=os.path.expanduser('~/workspace/maybelle-config/group_vars/all/vault.yml'),
+        help='Path to Ansible vault file'
+    )
+    parser.add_argument(
+        '--db-host',
+        type=str,
+        default='hunter.cryptograss.live',
+        help='Database host'
+    )
+    parser.add_argument(
+        '--db-port',
+        type=int,
+        default=5432,
+        help='Database port'
+    )
+    parser.add_argument(
+        '--db-name',
+        type=str,
+        default='magenta_memory',
+        help='Database name'
+    )
+    parser.add_argument(
+        '--db-user',
+        type=str,
+        default='magent',
+        help='Database user'
+    )
+    parser.add_argument(
+        '--dry-run',
+        action='store_true',
+        help='Show what would be redacted without making changes'
+    )
+    parser.add_argument(
+        '--verbose',
+        action='store_true',
+        help='Show detailed output'
+    )
+    parser.add_argument(
+        '--limit',
+        type=int,
+        default=None,
+        help='Limit number of messages to process'
+    )
+
+    args = parser.parse_args()
+
+    # Get vault password
+    vault_password = os.environ.get('ANSIBLE_VAULT_PASSWORD')
+    if not vault_password:
+        print("ERROR: ANSIBLE_VAULT_PASSWORD environment variable not set")
+        print("Run: export ANSIBLE_VAULT_PASSWORD=$(cat ~/.vault_pass)")
+        sys.exit(1)
+
+    # Get database password from vault
+    db_password = os.environ.get('MAGENTA_DB_PASSWORD')
+    if not db_password:
+        print("ERROR: MAGENTA_DB_PASSWORD environment variable not set")
+        print("Set it to the memory_lane_postgres_password from vault")
+        sys.exit(1)
+
+    # Initialize secrets filter
+    print(f"Loading secrets from vault: {args.vault_path}")
+    secrets_filter = SecretsFilter(vault_path=args.vault_path, vault_password=vault_password)
+
+    if not secrets_filter.secrets:
+        print("ERROR: No secrets loaded from vault")
+        sys.exit(1)
+
+    print(f"Loaded {len(secrets_filter.secrets)} secrets to scrub")
+
+    # Connect to database
+    print(f"Connecting to {args.db_host}:{args.db_port}/{args.db_name}...")
+    conn = psycopg2.connect(
+        host=args.db_host,
+        port=args.db_port,
+        dbname=args.db_name,
+        user=args.db_user,
+        password=db_password
+    )
+
+    try:
+        with conn.cursor() as cur:
+            # Count messages
+            cur.execute("SELECT COUNT(*) FROM conversations_message")
+            total = cur.fetchone()[0]
+            print(f"Total messages in database: {total}")
+
+            # Query messages
+            query = "SELECT id, sender_id, content, created_at FROM conversations_message ORDER BY created_at"
+            if args.limit:
+                query += f" LIMIT {args.limit}"
+
+            cur.execute(query)
+
+            redacted_count = 0
+            updates = []
+
+            for row in cur:
+                msg_id, sender_id, content, created_at = row
+
+                # Scrub the content
+                scrubbed_content = secrets_filter.scrub_json(content)
+
+                # Check if anything changed
+                if scrubbed_content != content:
+                    redacted_count += 1
+
+                    if args.verbose:
+                        print(f"\n--- Message {msg_id} ---")
+                        print(f"Sender: {sender_id}")
+                        print(f"Created: {created_at}")
+                        orig_str = json.dumps(content)[:200] if content else ''
+                        scrub_str = json.dumps(scrubbed_content)[:200] if scrubbed_content else ''
+                        print(f"Original: {orig_str}...")
+                        print(f"Scrubbed: {scrub_str}...")
+
+                    if not args.dry_run:
+                        updates.append((json.dumps(scrubbed_content), str(msg_id)))
+
+            # Apply updates
+            if not args.dry_run and updates:
+                print(f"\nApplying {len(updates)} updates...")
+                with conn.cursor() as update_cur:
+                    for scrubbed_json, msg_id in updates:
+                        update_cur.execute(
+                            "UPDATE conversations_message SET content = %s WHERE id = %s",
+                            (scrubbed_json, msg_id)
+                        )
+                conn.commit()
+                print(f"Successfully redacted secrets from {redacted_count} messages")
+            elif args.dry_run:
+                print(f"\nDRY RUN: Would redact secrets from {redacted_count} messages")
+            else:
+                print("No messages contained secrets to redact")
+
+            print(f"\nMessages processed: {args.limit or total}")
+            print(f"Messages with secrets: {redacted_count}")
+
+    finally:
+        conn.close()
+
+
+if __name__ == '__main__':
+    main()