mirror_projects.py

#!/usr/bin/env python3
"""
mirror_projects.py

A script to mirror all GitLab projects (including commits, branches, and tags) to a GitHub profile.
This tool uses the GitLab API to list projects and the GitHub API to verify (or create) repositories,
and then employs Git commands to perform a full mirror (using git clone --mirror and git push --mirror).

Before using this script:
  - Install dependencies: pip install requests python-dotenv
  - Have Git installed and available in your PATH.
  - Generate your GitLab and GitHub Personal Access Tokens (PATs) with the necessary API permissions.
  - Either create a .env file with your credentials or provide them as command-line arguments.

Usage:
  python mirror_projects.py [options]

  # Using .env file (recommended)
  python mirror_projects.py

  # Or with command-line arguments
  python mirror_projects.py \
      --gitlab-token YOUR_GITLAB_TOKEN \
      --github-token YOUR_GITHUB_TOKEN \
      --github-username YOUR_GITHUB_USERNAME \
      [--gitlab-url https://gitlab.com] \
      [--mirror-dir ./mirror_repos] \
      [--private] \
      [--dry-run]

Options:
  --gitlab-token       Your GitLab Personal Access Token (or set GITLAB_TOKEN in .env)
  --github-token       Your GitHub Personal Access Token (or set GITHUB_TOKEN in .env)
  --github-username    Your GitHub username (or set GITHUB_USERNAME in .env)
  --gitlab-url         Base URL for GitLab (default: "https://gitlab.com" or GITLAB_URL in .env)
  --mirror-dir         Local directory to store mirrored repositories (default: "./mirror_repos" or MIRROR_DIR in .env)
  --private            Flag to create GitHub repositories as private (default is public unless specified or PRIVATE_REPOS=true in .env)
  --dry-run            Simulate the mirroring process without making actual changes (or set DRY_RUN=true in .env)

Author: [Your Name]
License: MIT
"""

import os
import sys
import requests
import subprocess
import argparse
import logging
from urllib.parse import urljoin
from typing import List, Dict, Any, Optional
import json
from pathlib import Path
from dotenv import load_dotenv

# ------------------------------------------------------------------------------
# Constants
# ------------------------------------------------------------------------------
GITLAB_DEFAULT_URL = "https://gitlab.com"
DEFAULT_MIRROR_DIR = "./mirror_repos"
GITLAB_API_PATH = "/api/v4/projects"
GITHUB_API_URL = "https://api.github.com"
PAGINATION_PER_PAGE = 100

# ------------------------------------------------------------------------------
# Config management
# ------------------------------------------------------------------------------
def load_config():
    """
    Load configuration from .env file if it exists.
    
    Returns:
        dict: Configuration values from .env file
    """
    # Look for .env file in current directory
    env_path = Path('.') / '.env'
    load_dotenv(dotenv_path=env_path)
    
    config = {
        'gitlab_token': os.getenv('GITLAB_TOKEN'),
        'github_token': os.getenv('GITHUB_TOKEN'),
        'github_username': os.getenv('GITHUB_USERNAME'),
        'gitlab_url': os.getenv('GITLAB_URL', GITLAB_DEFAULT_URL),
        'mirror_dir': os.getenv('MIRROR_DIR', DEFAULT_MIRROR_DIR),
        'private': os.getenv('PRIVATE_REPOS', '').lower() == 'true',
        'dry_run': os.getenv('DRY_RUN', '').lower() == 'true',
        'debug': os.getenv('DEBUG', '').lower() == 'true'
    }
    
    return config

# ------------------------------------------------------------------------------
# Logging Setup
# ------------------------------------------------------------------------------
def setup_logging(debug: bool = False) -> None:
    """
    Configure logging format and output.
    
    Args:
        debug: If True, set log level to DEBUG, otherwise INFO
    """
    log_level = logging.DEBUG if debug else logging.INFO
    logging.basicConfig(
        level=log_level,
        format="%(asctime)s [%(levelname)s] %(message)s",
        handlers=[logging.StreamHandler(sys.stdout)]
    )

# ------------------------------------------------------------------------------
# GitLab API Interaction
# ------------------------------------------------------------------------------
def get_gitlab_projects(gitlab_token: str, gitlab_url: str) -> List[Dict[str, Any]]:
    """
    Retrieve all GitLab projects where the authenticated user is a member.
    
    Uses pagination to fetch all projects.
    
    Args:
        gitlab_token: GitLab Personal Access Token
        gitlab_url: Base URL for GitLab instance
        
    Returns:
        List of project dictionaries from GitLab API
        
    Raises:
        SystemExit: If API request fails
    """
    logging.info("Fetching GitLab projects...")
    headers = {'Private-Token': gitlab_token}
    projects = []
    page = 1

    while True:
        url = f"{gitlab_url}{GITLAB_API_PATH}"
        params = {'membership': True, 'page': page, 'per_page': PAGINATION_PER_PAGE}
        
        try:
            response = requests.get(url, headers=headers, params=params)
            response.raise_for_status()
        except requests.exceptions.RequestException as e:
            logging.error(f"Error fetching GitLab projects: {e}")
            sys.exit(1)

        data = response.json()
        if not data:
            break

        projects.extend(data)
        logging.info(f"Fetched page {page} with {len(data)} projects")
        page += 1

    logging.info(f"Total projects fetched: {len(projects)}")
    return projects

# ------------------------------------------------------------------------------
# GitHub API Interaction
# ------------------------------------------------------------------------------
def check_github_repo_exists(github_username: str, repo_name: str, github_token: str) -> bool:
    """
    Check if a GitHub repository exists in the user's account.

    Args:
        github_username: GitHub username
        repo_name: Repository name to check
        github_token: GitHub Personal Access Token
        
    Returns:
        True if repository exists, False otherwise
        
    Raises:
        SystemExit: If API request fails (except for 404 Not Found)
    """
    logging.info(f"Checking if GitHub repository '{repo_name}' exists...")
    url = f"{GITHUB_API_URL}/repos/{github_username}/{repo_name}"
    headers = {
        'Authorization': f'token {github_token}',
        'Accept': 'application/vnd.github.v3+json'
    }
    
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            logging.info("Repository exists.")
            return True
        elif response.status_code == 404:
            logging.info("Repository does not exist.")
            return False
        else:
            response.raise_for_status()
    except requests.exceptions.RequestException as e:
        logging.error(f"Error checking GitHub repository: {e}")
        sys.exit(1)

def create_github_repo(github_username: str, repo_name: str, github_token: str, private: bool = True) -> bool:
    """
    Create a GitHub repository for the provided repo_name.

    Args:
        github_username: GitHub username
        repo_name: Repository name to create
        github_token: GitHub Personal Access Token
        private: If True, create private repository; otherwise public
        
    Returns:
        True if repository was created successfully
        
    Raises:
        SystemExit: If API request fails
    """
    logging.info(f"Creating GitHub repository '{repo_name}'...")
    url = f"{GITHUB_API_URL}/user/repos"
    headers = {
        'Authorization': f'token {github_token}',
        'Accept': 'application/vnd.github.v3+json'
    }
    data = {
        'name': repo_name,
        'private': private
    }
    
    try:
        response = requests.post(url, headers=headers, json=data)
        response.raise_for_status()
        logging.info("Repository created successfully.")
        return True
    except requests.exceptions.RequestException as e:
        logging.error(f"Error creating GitHub repository: {e}")
        sys.exit(1)

# ------------------------------------------------------------------------------
# Git Operations for Mirroring
# ------------------------------------------------------------------------------
def mirror_repo(gitlab_repo_url: str, github_repo_url: str, mirror_dir: str, dry_run: bool = False) -> None:
    """
    Mirror a repository from GitLab to GitHub.

    If a local mirror clone exists, the script updates it; otherwise, it clones it fresh.
    Then, it pushes all refs to the GitHub remote.
    
    Args:
        gitlab_repo_url: URL of the GitLab repository
        github_repo_url: URL of the GitHub repository
        mirror_dir: Local directory to store repository mirrors
        dry_run: If True, log actions without executing Git commands
    """
    repo_name = os.path.basename(gitlab_repo_url).replace(".git", "")
    repo_path = os.path.join(mirror_dir, repo_name)

    # Update or clone the repository from GitLab
    if os.path.exists(repo_path):
        logging.info(f"Local mirror for '{repo_name}' exists. Fetching latest changes...")
        if not dry_run:
            try:
                subprocess.run(['git', '-C', repo_path, 'fetch', '--all'], check=True)
            except subprocess.CalledProcessError as e:
                logging.error(f"Error fetching updates in {repo_name}: {e}")
                return
        else:
            logging.debug(f"[DRY RUN] Would run: git -C {repo_path} fetch --all")
    else:
        logging.info(f"Cloning repository '{repo_name}' from GitLab...")
        if not dry_run:
            try:
                subprocess.run(['git', 'clone', '--mirror', gitlab_repo_url, repo_path], check=True)
            except subprocess.CalledProcessError as e:
                logging.error(f"Error cloning repository {repo_name}: {e}")
                return
        else:
            logging.debug(f"[DRY RUN] Would run: git clone --mirror {gitlab_repo_url} {repo_path}")

    # Add GitHub remote if it doesn't already exist
    if not dry_run:
        try:
            remotes_output = subprocess.check_output(['git', '-C', repo_path, 'remote']).decode().split()
            if 'github' not in remotes_output:
                logging.info(f"Adding GitHub remote for '{repo_name}'...")
                subprocess.run(['git', '-C', repo_path, 'remote', 'add', 'github', github_repo_url], check=True)
            else:
                logging.info(f"GitHub remote already exists for '{repo_name}'.")
        except subprocess.CalledProcessError as e:
            logging.error(f"Error adding GitHub remote for {repo_name}: {e}")
            return
    else:
        logging.debug(f"[DRY RUN] Would check and possibly add GitHub remote for {repo_name}")

    # Push mirror to GitHub
    logging.info(f"Pushing mirror of '{repo_name}' to GitHub...")
    if not dry_run:
        try:
            subprocess.run(['git', '-C', repo_path, 'push', '--mirror', 'github'], check=True)
            logging.info(f"Successfully mirrored '{repo_name}' to GitHub.")
        except subprocess.CalledProcessError as e:
            logging.error(f"Error pushing repository {repo_name} to GitHub: {e}")
            return
    else:
        logging.debug(f"[DRY RUN] Would run: git -C {repo_path} push --mirror github")
        logging.info(f"[DRY RUN] Would mirror '{repo_name}' to GitHub.")

# ------------------------------------------------------------------------------
# Main Function
# ------------------------------------------------------------------------------
def main():
    # Load config from .env file if it exists
    config = load_config()
    
    # Set up argument parser
    parser = argparse.ArgumentParser(description="Mirror all GitLab projects to GitHub")
    parser.add_argument('--gitlab-token', help='Your GitLab Personal Access Token')
    parser.add_argument('--github-token', help='Your GitHub Personal Access Token')
    parser.add_argument('--github-username', help='Your GitHub username')
    parser.add_argument('--gitlab-url', help=f'Base URL for GitLab (default: {GITLAB_DEFAULT_URL})')
    parser.add_argument('--mirror-dir', help=f'Local directory to store mirrored repositories (default: {DEFAULT_MIRROR_DIR})')
    parser.add_argument('--private', action='store_true', help='Create GitHub repositories as private')
    parser.add_argument('--dry-run', action='store_true', help='Simulate the mirroring process without making actual changes')
    parser.add_argument('--debug', action='store_true', help='Enable debug logging')
    args = parser.parse_args()
    
    # Merge command-line arguments with config (command-line takes precedence)
    if args.gitlab_token:
        config['gitlab_token'] = args.gitlab_token
    if args.github_token:
        config['github_token'] = args.github_token
    if args.github_username:
        config['github_username'] = args.github_username
    if args.gitlab_url:
        config['gitlab_url'] = args.gitlab_url
    if args.mirror_dir:
        config['mirror_dir'] = args.mirror_dir
    if args.private:
        config['private'] = True
    if args.dry_run:
        config['dry_run'] = True
    if args.debug:
        config['debug'] = True
    
    # Validate required configuration
    missing_values = []
    if not config['gitlab_token']:
        missing_values.append('GitLab token (use --gitlab-token or set GITLAB_TOKEN in .env)')
    if not config['github_token']:
        missing_values.append('GitHub token (use --github-token or set GITHUB_TOKEN in .env)')
    if not config['github_username']:
        missing_values.append('GitHub username (use --github-username or set GITHUB_USERNAME in .env)')
    
    if missing_values:
        print("Error: Missing required configuration values:")
        for value in missing_values:
            print(f"  - {value}")
        print("\nPlease create a .env file or provide these values as command-line arguments.")
        print("See the README.md or run 'python mirror_projects.py --help' for more information.")
        sys.exit(1)

    # Setup logging with debug flag if enabled
    setup_logging(config['debug'])

    if config['dry_run']:
        logging.info("DRY RUN MODE ENABLED - No changes will be made")

    # Ensure the local mirror directory exists
    if not config['dry_run']:
        os.makedirs(config['mirror_dir'], exist_ok=True)
    else:
        logging.debug(f"[DRY RUN] Would create directory: {config['mirror_dir']}")

    # Retrieve GitLab projects
    if not config['dry_run']:
        projects = get_gitlab_projects(config['gitlab_token'], config['gitlab_url'])
    else:
        logging.debug("[DRY RUN] Would fetch GitLab projects")
        # Create mock data for dry run
        projects = [
            {
                'id': 1,
                'name': 'Sample Project 1',
                'path': 'sample-project-1',
                'http_url_to_repo': f"{config['gitlab_url']}/user/sample-project-1.git"
            },
            {
                'id': 2,
                'name': 'Sample Project 2',
                'path': 'sample-project-2',
                'http_url_to_repo': f"{config['gitlab_url']}/user/sample-project-2.git"
            }
        ]
        logging.info(f"[DRY RUN] Using {len(projects)} sample projects for simulation")

    if not projects:
        logging.info("No GitLab projects found. Exiting.")
        sys.exit(0)

    # Process each project
    for project in projects:
        repo_name = project['path']  # This can be adjusted if you prefer project['name']
        gitlab_repo_url = project['http_url_to_repo']  # URL used for cloning from GitLab

        # Construct the GitHub repository URL with embedded token for Git push operations.
        # (Be cautious with logging or storing URLs containing tokens.)
        github_repo_url = f"https://{config['github_username']}:{config['github_token']}@github.com/{config['github_username']}/{repo_name}.git"

        # Check existence on GitHub and create repository if needed
        if not config['dry_run']:
            exists = check_github_repo_exists(config['github_username'], repo_name, config['github_token'])
            if not exists:
                create_github_repo(config['github_username'], repo_name, config['github_token'], private=config['private'])
        else:
            logging.debug(f"[DRY RUN] Would check if GitHub repository '{repo_name}' exists")
            logging.debug(f"[DRY RUN] Would create GitHub repository '{repo_name}' if it doesn't exist")

        # Mirror the repository
        mirror_repo(gitlab_repo_url, github_repo_url, config['mirror_dir'], dry_run=config['dry_run'])

    logging.info("Mirroring operation completed.")

if __name__ == "__main__":
    main()