From b4007bbdd41cf08287d7e0621cc65377d5ac58c0 Mon Sep 17 00:00:00 2001 From: Justin Holmes Date: Fri, 5 Dec 2025 04:31:39 +0000 Subject: [PATCH 1/6] Add ASCII art conversion tools to container - jp2a: JPEG to ASCII converter - caca-utils: libcaca tools including img2txt - libaa-bin: AAlib binary tools For creating ASCII art portraits from reference photos. --- hunter/packages.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hunter/packages.txt b/hunter/packages.txt index bd63937..35ef2ea 100644 --- a/hunter/packages.txt +++ b/hunter/packages.txt @@ -39,3 +39,6 @@ tmux screen mosh imagemagick +jp2a +caca-utils +libaa-bin From c8de5d001a26909cf435f7ab309e10c8d3211af5 Mon Sep 17 00:00:00 2001 From: Justin Holmes Date: Wed, 17 Dec 2025 17:35:58 +0000 Subject: [PATCH 2/6] Add PickiPedia secrets deployment to maybelle --- maybelle/ansible/maybelle.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/maybelle/ansible/maybelle.yml b/maybelle/ansible/maybelle.yml index b7e8ca2..b747fe3 100644 --- a/maybelle/ansible/maybelle.yml +++ b/maybelle/ansible/maybelle.yml @@ -339,6 +339,32 @@ owner: 1000 group: 1000 + - name: Create PickiPedia secrets directory + file: + path: "{{ jenkins_home }}/secrets/pickipedia" + state: directory + mode: '0755' + owner: 1000 + group: 1000 + + - name: Deploy PickiPedia LocalSettings.local.php from vault + copy: + content: | + Date: Wed, 17 Dec 2025 18:03:03 +0000 Subject: [PATCH 3/6] Add pickipedia clone and preview config to hunter containers --- hunter/container_startup.py | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/hunter/container_startup.py b/hunter/container_startup.py index 0efe46a..4e555d1 100644 --- a/hunter/container_startup.py +++ b/hunter/container_startup.py @@ -131,6 +131,47 @@ def setup_workspace(): run_install=False ) + # Clone pickipedia (MediaWiki knowledge base) + ensure_repo_cloned( + "https://github.com/cryptograss/pickipedia.git", + workspace / "pickipedia", + user='magent', + run_install=False + ) + + # Create pickipedia config for local preview if it doesn't exist + pickipedia_dir = workspace / "pickipedia" + if pickipedia_dir.exists(): + # Create .env + pickipedia_env = pickipedia_dir / ".env" + if not pickipedia_env.exists(): + pickipedia_env.write_text("""# PickiPedia local preview settings +MEDIAWIKI_VERSION=1.43.0 +WIKI_PORT=4005 +DB_NAME=pickipedia +DB_USER=pickipedia +DB_PASSWORD=pickipedia_dev +DB_ROOT_PASSWORD=root_dev +""") + run_command(f"chown magent:magent {pickipedia_env}") + logger.info("✓ Created pickipedia .env for local preview") + + # Create LocalSettings.local.php for docker-compose preview + local_settings = pickipedia_dir / "LocalSettings.local.php" + if not local_settings.exists(): + local_settings.write_text(""" Date: Wed, 17 Dec 2025 18:03:28 +0000 Subject: [PATCH 4/6] Add COMPOSE_PROJECT_NAME to avoid container conflicts between users --- hunter/container_startup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hunter/container_startup.py b/hunter/container_startup.py index 4e555d1..a529785 100644 --- a/hunter/container_startup.py +++ b/hunter/container_startup.py @@ -142,12 +142,14 @@ def setup_workspace(): # Create pickipedia config for local preview if it doesn't exist pickipedia_dir = workspace / "pickipedia" if pickipedia_dir.exists(): - # Create .env + # Create .env with user-specific settings pickipedia_env = pickipedia_dir / ".env" + dev_name = os.environ.get('DEVELOPER_NAME', 'dev') if not pickipedia_env.exists(): - pickipedia_env.write_text("""# PickiPedia local preview settings + pickipedia_env.write_text(f"""# PickiPedia local preview settings for {dev_name} MEDIAWIKI_VERSION=1.43.0 WIKI_PORT=4005 +COMPOSE_PROJECT_NAME=pickipedia-{dev_name} DB_NAME=pickipedia DB_USER=pickipedia DB_PASSWORD=pickipedia_dev From a0e5fb6edb1c8f7b6968a49fe7474086bceb709d Mon Sep 17 00:00:00 2001 From: Justin Holmes Date: Wed, 17 Dec 2025 18:17:17 +0000 Subject: [PATCH 5/6] Add PickiPedia daily backup from NFS and hunter preview infrastructure - maybelle: Daily cron to mysqldump from NFS and sync to hunter - hunter: Mount backup directory in user containers - container_startup: Clone pickipedia repo, create preview configs and load-backup.sh helper Usage on hunter: cd ~/workspace/pickipedia docker-compose up -d ./load-backup.sh # Load production data into preview --- hunter/ansible/playbook.yml | 6 ++ .../templates/docker-compose.yml.j2 | 2 + hunter/container_startup.py | 43 ++++++++++++ maybelle/ansible/maybelle.yml | 65 +++++++++++++++++++ 4 files changed, 116 insertions(+) diff --git a/hunter/ansible/playbook.yml b/hunter/ansible/playbook.yml index 14c7e83..a19890d 100644 --- a/hunter/ansible/playbook.yml +++ b/hunter/ansible/playbook.yml @@ -19,6 +19,12 @@ state: directory mode: '0755' + - name: Create pickipedia backups directory (synced from maybelle) + file: + path: /opt/magenta/pickipedia-backups + state: directory + mode: '0755' + - name: Include Security role (SSH hardening, fail2ban, firewall) include_role: name: security diff --git a/hunter/ansible/roles/user-instance/templates/docker-compose.yml.j2 b/hunter/ansible/roles/user-instance/templates/docker-compose.yml.j2 index 070a7d0..410bae9 100644 --- a/hunter/ansible/roles/user-instance/templates/docker-compose.yml.j2 +++ b/hunter/ansible/roles/user-instance/templates/docker-compose.yml.j2 @@ -20,6 +20,8 @@ services: # SSH keys mounted separately (read-only for security) - {{ base_dir }}/{{ user.name }}/.ssh:/home/magent/.ssh:ro - /var/run/docker.sock:/var/run/docker.sock + # PickiPedia backups (synced from maybelle daily) + - /opt/magenta/pickipedia-backups:/opt/magenta/pickipedia-backups:ro networks: - magenta-net environment: diff --git a/hunter/container_startup.py b/hunter/container_startup.py index a529785..f8fd727 100644 --- a/hunter/container_startup.py +++ b/hunter/container_startup.py @@ -174,6 +174,49 @@ def setup_workspace(): run_command(f"chown magent:magent {local_settings}") logger.info("✓ Created pickipedia LocalSettings.local.php for preview") + # Create helper script to load production backup into preview + load_backup_script = pickipedia_dir / "load-backup.sh" + if not load_backup_script.exists(): + load_backup_script.write_text("""#!/bin/bash +# Load latest PickiPedia backup into preview MySQL +# Run this after 'docker-compose up -d' to populate with production data + +set -e + +BACKUP_DIR="/opt/magenta/pickipedia-backups" # Synced from maybelle daily +LATEST_BACKUP=$(ls -t ${BACKUP_DIR}/pickipedia_*.sql.gz 2>/dev/null | head -1) + +if [ -z "$LATEST_BACKUP" ]; then + echo "No backup found in $BACKUP_DIR" + echo "Backups are created daily at 3:30am on maybelle" + exit 1 +fi + +echo "Loading backup: $LATEST_BACKUP" + +# Get container name from compose project +CONTAINER=$(docker ps --filter "name=pickipedia.*db" --format "{{.Names}}" | head -1) +if [ -z "$CONTAINER" ]; then + echo "MySQL container not running. Start with: docker-compose up -d" + exit 1 +fi + +# Wait for MySQL to be ready +echo "Waiting for MySQL..." +until docker exec "$CONTAINER" mysqladmin ping -h localhost --silent 2>/dev/null; do + sleep 1 +done + +# Load the backup +echo "Loading data (this may take a moment)..." +gunzip -c "$LATEST_BACKUP" | docker exec -i "$CONTAINER" mysql -u pickipedia -ppickipedia_dev pickipedia + +echo "Done! PickiPedia preview now has production data." +""") + run_command(f"chown magent:magent {load_backup_script}") + run_command(f"chmod +x {load_backup_script}") + logger.info("✓ Created pickipedia load-backup.sh script") + def setup_host_files(): """Set up SSH keys and other host-provided config.""" diff --git a/maybelle/ansible/maybelle.yml b/maybelle/ansible/maybelle.yml index b747fe3..462566d 100644 --- a/maybelle/ansible/maybelle.yml +++ b/maybelle/ansible/maybelle.yml @@ -692,6 +692,71 @@ minute: "0" job: "/usr/local/bin/backup-to-nfs.sh" + # PickiPedia backup - pull MySQL dump from NFS for hunter preview + - name: Create pickipedia backup directory + file: + path: /mnt/persist/pickipedia/backups + state: directory + mode: '0755' + + - name: Create pickipedia backup script + copy: + dest: /usr/local/bin/backup-pickipedia.sh + mode: '0750' + content: | + #!/bin/bash + # Daily PickiPedia MySQL backup from NFS + # Pulls dump via SSH for use in hunter preview environments + LOG_FILE="/var/log/pickipedia-backup.log" + BACKUP_DIR="/mnt/persist/pickipedia/backups" + SSH_KEY="/root/.ssh/id_ed25519_nfs" + NFS_HOST="ssh.nyc1.nearlyfreespeech.net" + NFS_USER="jmyles_pickipedia" + + echo "$(date): Starting PickiPedia backup" >> "$LOG_FILE" + + # Run mysqldump on NFS and pipe back + BACKUP_FILE="$BACKUP_DIR/pickipedia_$(date +%Y%m%d).sql.gz" + if ssh -i "$SSH_KEY" "${NFS_USER}@${NFS_HOST}" \ + "mysqldump -h pickipedia.db pickipedia" 2>> "$LOG_FILE" \ + | gzip > "$BACKUP_FILE"; then + echo "$(date): Backup successful: $BACKUP_FILE ($(stat -c%s "$BACKUP_FILE") bytes)" >> "$LOG_FILE" + # Keep last 7 days of backups + find "$BACKUP_DIR" -name "pickipedia_*.sql.gz" -mtime +7 -delete + + # Sync to hunter for preview environments + echo "$(date): Syncing to hunter..." >> "$LOG_FILE" + rsync -avz "$BACKUP_DIR"/ root@hunter.cryptograss.live:/opt/magenta/pickipedia-backups/ >> "$LOG_FILE" 2>&1 || \ + echo "$(date): WARNING - sync to hunter failed" >> "$LOG_FILE" + else + echo "$(date): Backup FAILED" >> "$LOG_FILE" + rm -f "$BACKUP_FILE" # Remove partial file + exit 1 + fi + + - name: Create pickipedia backup log file + file: + path: /var/log/pickipedia-backup.log + state: touch + mode: '0644' + + - name: Schedule daily pickipedia backup + ansible.builtin.cron: + name: "pickipedia backup from nfs" + hour: "3" + minute: "30" + job: "/usr/local/bin/backup-pickipedia.sh" + + - name: Add NFS pickipedia site to SSH config + blockinfile: + path: /root/.ssh/config + marker: "# {mark} PICKIPEDIA NFS" + block: | + Host nfs-pickipedia + HostName ssh.nyc1.nearlyfreespeech.net + IdentityFile ~/.ssh/id_ed25519_nfs + User jmyles_pickipedia + # Security: remove vault password after deploy completes # The encrypted vault.yml is in a public repo, so vault password on disk is a risk # Re-run chapter-1.sh to restore it if needed for future deploys From 382072bb06934bfbc4cb5ac05f93ecc1ad64d7d6 Mon Sep 17 00:00:00 2001 From: Justin Holmes Date: Wed, 17 Dec 2025 18:36:47 +0000 Subject: [PATCH 6/6] Add Jenkins job definition for pickipedia --- maybelle/jobs/pickipedia.groovy | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 maybelle/jobs/pickipedia.groovy diff --git a/maybelle/jobs/pickipedia.groovy b/maybelle/jobs/pickipedia.groovy new file mode 100644 index 0000000..1402e5f --- /dev/null +++ b/maybelle/jobs/pickipedia.groovy @@ -0,0 +1,20 @@ +pipelineJob('pickipedia') { + definition { + cpsScm { + scm { + git { + remote { + url('https://github.com/cryptograss/pickipedia.git') + credentials('github-token') + } + branch('*/main') + } + } + scriptPath('Jenkinsfile') + } + } + triggers { + // Poll every 5 minutes for changes + cron('*/5 * * * *') + } +}