diff --git a/.github/workflows/build-executables.yml b/.github/workflows/build-executables.yml new file mode 100644 index 00000000..5daa61d4 --- /dev/null +++ b/.github/workflows/build-executables.yml @@ -0,0 +1,155 @@ +name: Build Executables + +on: + push: + tags: + - 'v*' + workflow_dispatch: + +jobs: + build: + name: Build on ${{ matrix.os }} - ${{ matrix.arch }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + # Windows builds + - os: windows-latest + arch: x64 + artifact_name: sonar-reports-windows-x86_64.exe + binary_name: sonar-reports.exe + + # Linux builds + - os: ubuntu-latest + arch: x64 + artifact_name: sonar-reports-linux-x86_64 + binary_name: sonar-reports + + - os: ubuntu-latest + arch: arm64 + artifact_name: sonar-reports-linux-arm64 + binary_name: sonar-reports + + # macOS builds + - os: macos-latest + arch: x64 + artifact_name: sonar-reports-macos-x86_64 + binary_name: sonar-reports + + - os: macos-latest + arch: arm64 + artifact_name: sonar-reports-macos-arm64 + binary_name: sonar-reports + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-dev.txt + + - name: Build with PyInstaller (Linux ARM64) + if: matrix.os == 'ubuntu-latest' && matrix.arch == 'arm64' + run: | + # For ARM64 on Linux, we need to use cross-compilation or native ARM runner + # This is a placeholder - you may need to use QEMU or native ARM runners + sudo apt-get update + sudo apt-get install -y qemu-user-static + pyinstaller sonar-reports.spec --target-arch aarch64 + + - name: Build with PyInstaller (Windows) + if: matrix.os == 'windows-latest' + run: | + pyinstaller sonar-reports.spec + + - name: Build with PyInstaller (macOS) + if: matrix.os == 'macos-latest' + run: | + pyinstaller sonar-reports.spec --target-arch ${{ matrix.arch == 'arm64' && 'arm64' || 'x86_64' }} + + - name: Build with PyInstaller (Linux x64) + if: matrix.os == 'ubuntu-latest' && matrix.arch == 'x64' + run: | + pyinstaller sonar-reports.spec + + - name: Rename binary (Unix) + if: matrix.os != 'windows-latest' + run: | + mv dist/${{ matrix.binary_name }} dist/${{ matrix.artifact_name }} + + - name: Rename binary (Windows) + if: matrix.os == 'windows-latest' + run: | + move dist\${{ matrix.binary_name }} dist\${{ matrix.artifact_name }} + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.artifact_name }} + path: dist/${{ matrix.artifact_name }} + retention-days: 90 + + # Windows ARM build (separate job as it requires special setup) + build-windows-arm: + name: Build on Windows ARM64 + runs-on: windows-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-dev.txt + + - name: Build with PyInstaller for ARM64 + run: | + pyinstaller sonar-reports.spec --target-arch arm64 + + - name: Rename binary + run: | + move dist\sonar-reports.exe dist\sonar-reports-windows-arm64.exe + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: sonar-reports-windows-arm64.exe + path: dist/sonar-reports-windows-arm64.exe + retention-days: 90 + + release: + name: Create Release + needs: [build, build-windows-arm] + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: dist/ + + - name: Create Release + uses: softprops/action-gh-release@v1 + with: + files: dist/**/* + draft: false + prerelease: false + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml new file mode 100644 index 00000000..cafb54ef --- /dev/null +++ b/.github/workflows/build-windows.yml @@ -0,0 +1,49 @@ +name: Build Windows Executable + +on: + workflow_dispatch: + push: + branches: [main, feature/build-executables] + +jobs: + build-windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + pip install -r requirements.txt + pip install -r requirements-dev.txt + + - name: Build Windows executable + run: | + pyinstaller sonar-reports.spec + move dist\sonar-reports.exe dist\sonar-reports-windows-x86_64.exe + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: windows-executable + path: dist/sonar-reports-windows-x86_64.exe + + test-windows: + needs: build-windows + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + + - name: Download executable + uses: actions/download-artifact@v4 + with: + name: windows-executable + path: dist/ + + - name: Test executable + run: | + dist\sonar-reports-windows-x86_64.exe --help diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..94356b27 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,329 @@ +name: Build and Release + +on: + push: + branches: + - main + workflow_dispatch: + inputs: + version: + description: 'Version tag (e.g., v1.0.0)' + required: true + type: string + +jobs: + build-windows: + name: Build Windows x86_64 + runs-on: windows-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-dev.txt + + - name: Build Windows executable + run: | + pyinstaller sonar-reports.spec + move dist\sonar-reports.exe dist\sonar-reports-windows-x86_64.exe + + - name: Test executable + run: | + dist\sonar-reports-windows-x86_64.exe --help + + - name: Generate checksum + run: | + certutil -hashfile dist\sonar-reports-windows-x86_64.exe SHA256 > dist\sonar-reports-windows-x86_64.exe.sha256 + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: windows-x86_64 + path: | + dist/sonar-reports-windows-x86_64.exe + dist/sonar-reports-windows-x86_64.exe.sha256 + + build-linux: + name: Build Linux x86_64 + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-dev.txt + + - name: Build Linux executable + run: | + pyinstaller sonar-reports.spec + mv dist/sonar-reports dist/sonar-reports-linux-x86_64 + + - name: Test executable + run: | + ./dist/sonar-reports-linux-x86_64 --help + + - name: Generate checksum + run: | + sha256sum dist/sonar-reports-linux-x86_64 > dist/sonar-reports-linux-x86_64.sha256 + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: linux-x86_64 + path: | + dist/sonar-reports-linux-x86_64 + dist/sonar-reports-linux-x86_64.sha256 + + build-macos-intel: + name: Build macOS x86_64 (Intel) + runs-on: macos-13 # Intel runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-dev.txt + + - name: Build macOS Intel executable + run: | + pyinstaller sonar-reports.spec + mv dist/sonar-reports dist/sonar-reports-macos-x86_64 + + - name: Test executable + run: | + ./dist/sonar-reports-macos-x86_64 --help + + - name: Generate checksum + run: | + shasum -a 256 dist/sonar-reports-macos-x86_64 > dist/sonar-reports-macos-x86_64.sha256 + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: macos-x86_64 + path: | + dist/sonar-reports-macos-x86_64 + dist/sonar-reports-macos-x86_64.sha256 + + build-macos-arm: + name: Build macOS ARM64 (Apple Silicon) + runs-on: macos-latest # ARM64 runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-dev.txt + + - name: Build macOS ARM64 executable + run: | + pyinstaller sonar-reports.spec + mv dist/sonar-reports dist/sonar-reports-macos-arm64 + + - name: Test executable + run: | + ./dist/sonar-reports-macos-arm64 --help + + - name: Generate checksum + run: | + shasum -a 256 dist/sonar-reports-macos-arm64 > dist/sonar-reports-macos-arm64.sha256 + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: macos-arm64 + path: | + dist/sonar-reports-macos-arm64 + dist/sonar-reports-macos-arm64.sha256 + + build-windows-arm: + name: Build Windows ARM64 + runs-on: windows-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-dev.txt + + - name: Build Windows ARM64 executable + run: | + pyinstaller sonar-reports.spec --target-arch arm64 + move dist\sonar-reports.exe dist\sonar-reports-windows-arm64.exe + + - name: Generate checksum + run: | + certutil -hashfile dist\sonar-reports-windows-arm64.exe SHA256 > dist\sonar-reports-windows-arm64.exe.sha256 + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: windows-arm64 + path: | + dist/sonar-reports-windows-arm64.exe + dist/sonar-reports-windows-arm64.exe.sha256 + + build-linux-arm: + name: Build Linux ARM64 + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build Linux ARM64 executable with Docker + run: | + docker buildx build --platform linux/arm64 -f docker/Dockerfile.linux-build -t sonar-reports-linux-arm-builder --load . + docker run --rm --platform linux/arm64 -v "$(pwd)/dist:/output" sonar-reports-linux-arm-builder sh -c "cp /app/dist/sonar-reports-linux-x86_64 /output/sonar-reports-linux-arm64" + + - name: Make executable + run: chmod +x dist/sonar-reports-linux-arm64 + + - name: Generate checksum + run: | + sha256sum dist/sonar-reports-linux-arm64 > dist/sonar-reports-linux-arm64.sha256 + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: linux-arm64 + path: | + dist/sonar-reports-linux-arm64 + dist/sonar-reports-linux-arm64.sha256 + + release: + name: Create GitHub Release + needs: [build-windows, build-windows-arm, build-linux, build-linux-arm, build-macos-intel, build-macos-arm] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts/ + + - name: Display structure of downloaded files + run: ls -R artifacts/ + + - name: Prepare release assets + run: | + mkdir -p release-assets + find artifacts/ -type f -exec cp {} release-assets/ \; + ls -lah release-assets/ + + - name: Generate release notes + id: release_notes + run: | + cat > release-notes.md << 'EOF' + ## SonarQube to SonarCloud Migration Tool + + Download the appropriate binary for your platform: + + ### Windows + - `sonar-reports-windows-x86_64.exe` - Windows 64-bit (Intel/AMD) + - `sonar-reports-windows-arm64.exe` - Windows ARM64 (Surface/Qualcomm) + + ### Linux + - `sonar-reports-linux-x86_64` - Linux 64-bit (Intel/AMD) + - `sonar-reports-linux-arm64` - Linux ARM64 (Raspberry Pi, AWS Graviton) + + ### macOS + - `sonar-reports-macos-arm64` - macOS Apple Silicon (M1/M2/M3/M4) + - `sonar-reports-macos-x86_64` - macOS Intel + + ### Verification + Each binary includes a `.sha256` checksum file for verification: + + **Linux/macOS:** + ```bash + shasum -a 256 -c sonar-reports-*.sha256 + ``` + + **Windows (PowerShell):** + ```powershell + $hash = (Get-FileHash sonar-reports-windows-x86_64.exe -Algorithm SHA256).Hash + $expected = (Get-Content sonar-reports-windows-x86_64.exe.sha256).Split()[0] + if ($hash -eq $expected) { Write-Host "✓ Checksum verified" } else { Write-Host "✗ Checksum mismatch" } + ``` + + ### Quick Start + + 1. Download the binary for your platform + 2. Create a configuration file (see [README.md](https://github.com/${{ github.repository }}/blob/main/README.md)) + 3. Run: `./sonar-reports- full-migrate migration-config.json` + + For detailed instructions, see the [README](https://github.com/${{ github.repository }}/blob/main/README.md). + EOF + echo "notes_file=release-notes.md" >> $GITHUB_OUTPUT + + - name: Determine tag name + id: tag_name + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "tag=${{ inputs.version }}" >> $GITHUB_OUTPUT + else + echo "tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT + fi + + - name: Create Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.tag_name.outputs.tag }} + name: Release ${{ steps.tag_name.outputs.tag }} + body_path: release-notes.md + files: release-assets/* + draft: false + prerelease: false + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 8aa310c4..061e9901 100644 --- a/.gitignore +++ b/.gitignore @@ -31,7 +31,8 @@ wheels/ # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest -*.spec +# Keep our main spec file but ignore generated ones +# !sonar-reports.spec # Installer logs pip-log.txt @@ -116,4 +117,10 @@ sonarqube-export/ *.csv *.json *migration-files* -DEBUG.md \ No newline at end of file +files + +# Config files with secrets (keep example files in examples/) +!examples/*.example.json +config.json +*-config.json +my-*.json \ No newline at end of file diff --git a/README.md b/README.md index 981282b8..0daaf3a2 100644 --- a/README.md +++ b/README.md @@ -1,283 +1,480 @@ -# SonarQube to SonarCloud Migration Guide +# SonarQube to SonarCloud Migration Tool -This guide explains how to use the automated migration script to migrate your SonarQube Server projects to SonarCloud. +Migrate your SonarQube Server projects to SonarCloud with a single command. No Python or Docker installation required. -## Prerequisites +## Table of Contents -1. **Docker** installed and running -2. **SonarQube Server** access with admin token -3. **SonarCloud** account with: - - Enterprise subscription - - Admin token with enterprise-level permissions - - Target organization created and added to your enterprise +- [Quick Start](#quick-start) +- [What Gets Migrated](#what-gets-migrated) +- [Alternative Methods](#alternative-methods) +- [Post-Migration Steps](#post-migration-steps) +- [Troubleshooting](#troubleshooting) +- [Advanced Usage](#advanced-usage) +- [Security](#security) + +--- ## Quick Start -### 1. Configure the Script +The simplest way to migrate - download the executable, create a config file, and run one command. + +### Prerequisites + +- **SonarQube Server** with admin token +- **SonarCloud** account with: + - Enterprise subscription + - Admin token (enterprise-level permissions) + - Target organization created and added to your enterprise + +### Step 1: Download the Executable + +Download the pre-built executable for your platform from the [Releases page](https://github.com/YOUR_REPO/releases): -Edit `execute_full_migration.sh` and update these variables at the top: +- **macOS (Apple Silicon)**: `sonar-reports-macos-arm64` +- **macOS (Intel)**: `sonar-reports-macos-x86_64` +- **Linux (x86_64)**: `sonar-reports-linux-x86_64` +- **Windows (x86_64)**: `sonar-reports-windows-x86_64.exe` +Make the executable runnable (macOS/Linux): ```bash -SONARQUBE_URL="http://localhost:9000" # Your SonarQube Server URL -SONARQUBE_TOKEN="your-sonarqube-token-here" # Admin token for SonarQube -SONARCLOUD_URL="https://sonarcloud.io/" # SonarCloud URL -SONARCLOUD_TOKEN="your-sonarcloud-token-here" # Admin token for SonarCloud -SONARCLOUD_ENTERPRISE_KEY="your-enterprise-key" # Your SonarCloud Enterprise key -SONARCLOUD_ORG_KEY="your-target-org" # Target organization in SonarCloud -EXPORT_DIR="./files" # Directory for exported data -CONCURRENCY=10 # Number of concurrent API requests -TIMEOUT=60 # Request timeout in seconds +chmod +x sonar-reports-* ``` -### 2. Run the Migration +### Step 2: Create Configuration File ```bash -chmod +x execute_full_migration.sh -./execute_full_migration.sh +cp examples/migration-config.example.json migration-config.json ``` -The script will: -1. ✅ Validate configuration -2. ✅ Build the Docker image -3. ✅ Extract all data from SonarQube Server -4. ✅ Analyze and structure projects into organizations -5. ✅ Automatically update organizations.csv with your SonarCloud org key -6. ✅ Generate mappings for groups, profiles, gates, etc. -7. ✅ Migrate everything to SonarCloud -8. ✅ Verify the migration was successful - -## What Gets Migrated? - -### ✅ Projects -- Project configurations -- Project settings -- Quality profiles -- Quality gates -- New code definitions -- Tags - -### ✅ Users & Permissions -- User groups -- User permissions -- Group permissions -- Permission templates - -### ✅ Quality Configuration -- Quality profiles (with custom rules) -- Quality gates (with conditions) -- Rule customizations - -### ✅ Portfolios -- Portfolio structure -- Portfolio projects -- Portfolio permissions - -### ⚠️ What Is NOT Migrated? +Edit `migration-config.json` with your credentials: + +```json +{ + "sonarqube": { + "url": "http://localhost:9000", + "token": "YOUR_SONARQUBE_ADMIN_TOKEN" + }, + "sonarcloud": { + "url": "https://sonarcloud.io/", + "token": "YOUR_SONARCLOUD_ADMIN_TOKEN", + "enterprise_key": "YOUR_ENTERPRISE_KEY", + "org_key": "YOUR_TARGET_ORG_KEY" + }, + "settings": { + "export_directory": "./files", + "concurrency": 10, + "timeout": 60 + } +} +``` + +### Step 3: Run Migration + +Run the executable with your config file: + +**macOS/Linux:** +```bash +./sonar-reports- full-migrate migration-config.json +``` + +**Windows:** +```cmd +sonar-reports-windows-x86_64.exe full-migrate migration-config.json +``` + +Replace `` with your downloaded executable name (e.g., `macos-arm64`, `macos-x86_64`, or `linux-x86_64`). + +That's it! The tool automatically: +1. ✅ Extracts all data from SonarQube Server +2. ✅ Generates organization structure +3. ✅ Creates mappings (profiles, gates, groups) +4. ✅ Migrates everything to SonarCloud +5. ✅ Verifies the migration + +--- + +## What Gets Migrated + +### ✅ Migrated Items + +- **Projects**: Configurations, settings, tags, new code definitions +- **Quality Profiles**: Including custom rules +- **Quality Gates**: Including all conditions +- **Users & Permissions**: Groups, permissions, templates +- **Portfolios**: Structure, projects, permissions + +### ⚠️ NOT Migrated + - Historical analysis data - Issues and their history - Code coverage history - Security hotspots - Source code (you'll need to re-scan) -## Manual Steps (Alternative) +--- -If you prefer to run commands manually: +## Alternative Methods -### Step 1: Build Docker Image +### Option 1: Build the Executable Yourself + +If you prefer to build the executable from source: + +**macOS/Linux:** ```bash -docker build -t sonar-reports:local . +git clone https://github.com/YOUR_REPO/sonar-reports.git +cd sonar-reports +./scripts/build.sh ``` -### Step 2: Extract from SonarQube -```bash -# Note: Use host.docker.internal instead of localhost for Docker to access host -docker run --rm \ - -v "$(pwd)/files:/app/files" \ - sonar-reports:local extract \ - http://host.docker.internal:9000 \ - \ - --export_directory=/app/files \ - --concurrency=10 \ - --timeout=60 +**Windows:** +```cmd +git clone https://github.com/YOUR_REPO/sonar-reports.git +cd sonar-reports +scripts\build.bat ``` -### Step 3: Generate Structure +**Docker (Linux x86_64):** ```bash -docker run --rm \ - -v "$(pwd)/files:/app/files" \ - sonar-reports:local structure \ - --export_directory=/app/files +git clone https://github.com/YOUR_REPO/sonar-reports.git +cd sonar-reports +docker buildx build --platform linux/amd64 -f docker/Dockerfile.linux-build -t sonar-reports-linux-builder --load . +docker run --rm -v "$(pwd)/dist:/output" sonar-reports-linux-builder cp /app/dist/sonar-reports-linux-x86_64 /output/ ``` -### Step 4: Edit organizations.csv -Open `files/organizations.csv` and add your SonarCloud organization key in the `sonarcloud_org_key` column (second column). +The built executable will be in the `dist/` directory. Then follow steps 2-3 from the Quick Start guide. -Example: -```csv -sonarqube_org_key,sonarcloud_org_key,server_url,alm,url,is_cloud,project_count -http://host.docker.internal:9000/,your-org-key,http://host.docker.internal:9000/,,,false,28 -``` +### Option 2: Binary with Shell Script -### Step 5: Generate Mappings -```bash -docker run --rm \ - -v "$(pwd)/files:/app/files" \ - sonar-reports:local mappings \ - --export_directory=/app/files -``` +Use a shell script wrapper for the binary: -### Step 6: Run Migration ```bash -docker run --rm \ - -v "$(pwd)/files:/app/files" \ - sonar-reports:local migrate \ - \ - \ - --url=https://sonarcloud.io/ \ - --export_directory=/app/files \ - --concurrency=10 +./scripts/execute_migration_with_binary.sh migration-config.json ``` -## Troubleshooting +This script automatically detects your platform and runs the appropriate binary. + +### Option 3: Docker (No Installation) -### Migration Fails or Errors Occur +If you prefer Docker over building the binary: -1. **Check the migration requests log**: +1. **Configure the script:** ```bash - ls -lah files/*/requests.log - tail -100 files/*/requests.log + # Edit scripts/execute_full_migration.sh with your values + SONARQUBE_URL="http://localhost:9000" + SONARQUBE_TOKEN="your-token" + SONARCLOUD_URL="https://sonarcloud.io/" + SONARCLOUD_TOKEN="your-token" + SONARCLOUD_ENTERPRISE_KEY="your-enterprise-key" + SONARCLOUD_ORG_KEY="your-org" ``` -2. **Check extract data**: +2. **Run:** ```bash - ls -lah files/ - cat files/organizations.csv - cat files/projects.csv + chmod +x scripts/execute_full_migration.sh + ./scripts/execute_full_migration.sh ``` -3. **Resume a failed migration**: - If migration fails partway through, you can resume it: +### Option 4: Python CLI + +For maximum control, use the Python CLI directly: + +1. **Install dependencies:** ```bash - docker run --rm \ - -v "$(pwd)/files:/app/files" \ - sonar-reports:local migrate \ - \ - \ - --url=https://sonarcloud.io/ \ - --export_directory=/app/files \ - --run_id= + pip install -r requirements.txt ``` - Find the run ID from the directory names in `files/` (e.g., `1770304645`) +2. **Run commands individually:** + ```bash + # Extract + python src/main.py extract http://localhost:9000 YOUR_TOKEN --export_directory ./files -### No Projects Extracted + # Structure + python src/main.py structure --export_directory ./files -If the extraction completes but no projects are found: -- Verify your SonarQube token has admin permissions -- Check that projects exist in your SonarQube instance -- Review the requests.log file in `files//requests.log` for API errors -- Ensure you're using `host.docker.internal` instead of `localhost` in Docker commands + # Edit organizations.csv to add your SonarCloud org key -### Authentication Errors + # Mappings + python src/main.py mappings --export_directory ./files + + # Migrate + python src/main.py migrate YOUR_SONARCLOUD_TOKEN YOUR_ENTERPRISE_KEY --export_directory ./files + ``` -- Verify tokens are valid and haven't expired -- Ensure tokens have the required permissions: - - SonarQube: Admin or global analysis permissions - - SonarCloud: Enterprise-level admin permissions +### Option 5: Using Config Files with Individual Commands -### API Rate Limiting +You can use JSON config files with any command: -If you hit rate limits: -- Reduce `CONCURRENCY` to 5 or lower -- Increase `TIMEOUT` to 120 seconds +```bash +# Extract with config +./dist/sonar-reports-macos-arm64 extract --config extract-config.json + +# Migrate with config +./dist/sonar-reports-macos-arm64 migrate --config migrate-config.json +``` + +See [docs/CONFIG.md](docs/CONFIG.md) for configuration file documentation. + +--- ## Post-Migration Steps ### 1. Verify Projects -Visit your SonarCloud organization and verify all projects are present: + +Visit your SonarCloud organization: ``` -https://sonarcloud.io/organizations//projects +https://sonarcloud.io/organizations/YOUR_ORG/projects ``` -You can also verify via API: +Or verify via API: ```bash -curl -H "Authorization: Bearer " \ - "https://sonarcloud.io/api/projects/search?organization=&ps=500" | jq +curl -H "Authorization: Bearer YOUR_TOKEN" \ + "https://sonarcloud.io/api/projects/search?organization=YOUR_ORG&ps=500" | jq ``` ### 2. Check Quality Gates -Verify quality gates were migrated correctly: + ``` -https://sonarcloud.io/organizations//quality_gates +https://sonarcloud.io/organizations/YOUR_ORG/quality_gates ``` ### 3. Verify Quality Profiles -Check that quality profiles and custom rules were migrated: + ``` -https://sonarcloud.io/organizations//quality_profiles +https://sonarcloud.io/organizations/YOUR_ORG/quality_profiles ``` -### 4. Re-scan Projects -After migration, you'll need to run new analyses to populate code and issues: +### 4. Re-scan Your Projects + +Historical analysis data is NOT migrated. You need to run new scans: + +**Maven:** ```bash -# Example for a Maven project mvn sonar:sonar \ - -Dsonar.projectKey= \ - -Dsonar.organization= \ + -Dsonar.projectKey=YOUR_PROJECT_KEY \ + -Dsonar.organization=YOUR_ORG \ + -Dsonar.host.url=https://sonarcloud.io \ + -Dsonar.token=YOUR_TOKEN +``` + +**Gradle:** +```bash +./gradlew sonar \ + -Dsonar.projectKey=YOUR_PROJECT_KEY \ + -Dsonar.organization=YOUR_ORG \ -Dsonar.host.url=https://sonarcloud.io \ - -Dsonar.token= + -Dsonar.token=YOUR_TOKEN ``` +**Other scanners:** See [SonarCloud documentation](https://docs.sonarcloud.io/) + ### 5. Configure DevOps Integration -Set up ALM (GitHub, Azure DevOps, GitLab, Bitbucket) integrations for automatic analysis. + +Set up automatic analysis by configuring ALM integration: +- GitHub +- Azure DevOps +- GitLab +- Bitbucket + +See [SonarCloud ALM Integration](https://docs.sonarcloud.io/advanced-setup/ci-based-analysis/) + +--- + +## Troubleshooting + +### Migration Fails During Extract + +**Check logs:** +```bash +tail -100 files/*/requests.log +``` + +**Common issues:** +- SonarQube URL not accessible +- Token lacks admin permissions +- Docker: use `host.docker.internal` instead of `localhost` + +### Migration Fails During Upload + +**Common issues:** +- SonarCloud token lacks enterprise admin permissions +- Organization not added to enterprise +- Incorrect organization key + +**Review logs:** +```bash +ls -lah files/*/requests.log +``` + +### No Projects Extracted + +- Verify SonarQube token has admin permissions +- Check that projects exist in SonarQube +- Review `files//requests.log` for API errors + +### Authentication Errors + +- Verify tokens are valid and not expired +- Required permissions: + - **SonarQube**: Admin or global analysis permissions + - **SonarCloud**: Enterprise-level admin permissions + +### API Rate Limiting + +Reduce concurrency in your config file: +```json +{ + "settings": { + "concurrency": 5, + "timeout": 120 + } +} +``` + +### Resume Failed Migration + +If migration fails partway through: + +```bash +# Find the run ID from directory names in files/ +ls files/ + +# Resume with run_id +./dist/sonar-reports-macos-arm64 migrate --config migrate-config.json --run_id= +``` + +--- ## Advanced Usage ### Migrate Specific Organizations Only -Edit the `organizations.csv` file after Step 3 and remove rows for organizations you don't want to migrate. + +1. Run extract and structure commands +2. Edit `files/organizations.csv` and remove rows you don't want +3. Run mappings and migrate commands ### Migrate Specific Tasks Only -Use the `--target_task` option: + ```bash -docker run --rm \ - -v $(pwd)/sonarqube-export:/app/files \ - sonar-reports:local migrate \ - \ - --url=https://sonarcloud.io/ \ - --export_directory=/app/files \ +./dist/sonar-reports-macos-arm64 migrate \ + YOUR_TOKEN YOUR_ENTERPRISE_KEY \ + --export_directory=./files \ --target_task=createProjects ``` ### Extract Specific Data Only + ```bash -docker run --rm \ - -v $(pwd)/sonarqube-export:/app/files \ - sonar-reports:local extract \ - \ - --export_directory=/app/files \ +./dist/sonar-reports-macos-arm64 extract \ + http://localhost:9000 YOUR_TOKEN \ + --export_directory=./files \ --target_task=getProjects ``` -## Support +### Custom Export Directory -For issues or questions: -1. Check the log files in `files/*/requests.log` -2. Review the [README.rst](README.rst) for detailed CLI command documentation -3. Use `execute_full_migration.sh` for a fully automated migration -4. Open an issue on the project repository +Specify a different directory: +```json +{ + "settings": { + "export_directory": "/path/to/your/directory" + } +} +``` + +### Adjust Concurrency and Timeout -## Script Files +For slower connections or rate limiting: +```json +{ + "settings": { + "concurrency": 5, + "timeout": 120 + } +} +``` + +### Docker Testing + +Test the Linux binary in a clean container environment: + +```bash +# Build the test container +docker build -f docker/Dockerfile.test-linux -t sonar-reports-test . + +# Run a test migration (requires SonarQube accessible from Docker) +docker run --rm \ + -v "$(pwd):/app" \ + -w /app \ + --add-host=host.docker.internal:host-gateway \ + sonar-reports-test +``` -- **execute_full_migration.sh** - Automated migration script (recommended) -- **README.rst** - Technical documentation for manual CLI usage -- **MIGRATION_GUIDE.md** - Detailed migration guide (deprecated - use this README instead) +**Note:** When running in Docker, use `host.docker.internal` instead of `localhost` in your config file to access services on the host machine. -## Security Notes +--- + +## Security ⚠️ **Important Security Considerations:** -- Never commit tokens or credentials to version control -- Store tokens securely (e.g., environment variables, secret managers) -- Tokens have full admin access - protect them accordingly +### Never Commit Secrets + +- Never commit `migration-config.json` to version control +- Config files with secrets are already in `.gitignore` +- Only `.example.json` files should be committed + +### Protect Your Tokens + +- Store tokens securely (environment variables, secret managers) +- Tokens have full admin access - treat them as highly sensitive - Consider using temporary tokens that expire after migration -- Review and audit what data is being migrated + +### Use File Permissions + +Restrict access to config files: +```bash +chmod 600 migration-config.json +``` + +### Environment Variables + +For CI/CD, use environment variables: +```bash +export SONAR_TOKEN="your-token" +cat > migration-config.json < **Note:** Docker is optional for most platforms. GitHub Actions uses native Python builds for Windows (x86_64/ARM64), Linux x86_64, and macOS (x86_64/ARM64). Docker is only required for Linux ARM64 cross-compilation. + +## Available Dockerfiles + +### Dockerfile.linux-build + +Builds Linux binaries (x86_64 or ARM64) using PyInstaller in a containerized environment. + +**When to use:** +- Building Linux ARM64 binaries (required - used in CI) +- Optional local builds if you prefer Docker over native Python +- Building Linux binaries on non-Linux systems + +**Usage (x86_64):** +```bash +# Build the builder image +docker buildx build --platform linux/amd64 -f docker/Dockerfile.linux-build -t sonar-reports-linux-builder --load . + +# Extract the binary +docker run --rm -v "$(pwd)/dist:/output" sonar-reports-linux-builder cp /app/dist/sonar-reports-linux-x86_64 /output/ +``` + +**Usage (ARM64):** +```bash +# Build the ARM64 builder image (requires QEMU) +docker buildx build --platform linux/arm64 -f docker/Dockerfile.linux-build -t sonar-reports-linux-arm-builder --load . + +# Extract the binary +docker run --rm --platform linux/arm64 -v "$(pwd)/dist:/output" sonar-reports-linux-arm-builder sh -c "cp /app/dist/sonar-reports-linux-x86_64 /output/sonar-reports-linux-arm64" +``` + +**Output:** +- Binary: `dist/sonar-reports-linux-{x86_64,arm64}` +- Size: ~19MB +- Architecture: Linux ELF 64-bit + +### Dockerfile.test-linux + +Tests the Linux binary in a clean Debian environment to verify it works correctly. + +**Usage:** +```bash +# Build the test image +docker build -f docker/Dockerfile.test-linux -t sonar-reports-test . + +# Run a test migration +docker run --rm \ + -v "$(pwd):/app" \ + -w /app \ + --add-host=host.docker.internal:host-gateway \ + sonar-reports-test +``` + +**Notes:** +- Use `host.docker.internal` in your config file to access services on the host machine (like SonarQube running on localhost:9000) +- The test container includes a sample `migration-config.json` for testing + +## CI/CD Usage + +GitHub Actions (`.github/workflows/release.yml`) build strategy: + +| Platform | Build Method | Uses Docker? | +|----------|-------------|--------------| +| Windows x86_64 | Native Python on `windows-latest` | ❌ No | +| Windows ARM64 | Native Python on `windows-latest` with `--target-arch arm64` | ❌ No | +| Linux x86_64 | Native Python on `ubuntu-latest` | ❌ No | +| **Linux ARM64** | Docker + QEMU on `ubuntu-latest` | ✅ **Yes** (required) | +| macOS x86_64 | Native Python on `macos-13` | ❌ No | +| macOS ARM64 | Native Python on `macos-latest` | ❌ No | + +**Why Docker for Linux ARM64?** +- GitHub Actions doesn't provide native ARM64 Linux runners +- Docker with QEMU emulation enables cross-compilation +- All other platforms use native builds for speed and simplicity + +## Development Tips + +### Build Locally for Testing + +```bash +# Build Linux binary +cd /path/to/sonar-reports +docker buildx build --platform linux/amd64 -f docker/Dockerfile.linux-build -t sonar-reports-linux-builder --load . +docker run --rm -v "$(pwd)/dist:/output" sonar-reports-linux-builder cp /app/dist/sonar-reports-linux-x86_64 /output/ + +# Test the binary +chmod +x dist/sonar-reports-linux-x86_64 +./dist/sonar-reports-linux-x86_64 --help +``` + +### Test in Clean Environment + +```bash +# Test that the binary has all dependencies bundled +docker run --rm --platform linux/amd64 \ + -v "$(pwd)/dist:/app" \ + -w /app \ + debian:stable-slim \ + /app/sonar-reports-linux-x86_64 --help +``` + +### Configuration for Docker + +When running migrations inside Docker, your config file needs to use `host.docker.internal`: + +```json +{ + "sonarqube": { + "url": "http://host.docker.internal:9000", + "token": "your-token" + }, + "sonarcloud": { + "url": "https://sonarcloud.io/", + "token": "your-token", + "enterprise_key": "your-enterprise-key", + "org_key": "your-org" + }, + "settings": { + "export_directory": "./files", + "concurrency": 10, + "timeout": 60 + } +} +``` + +## Platform Support + +All platforms are fully supported in the GitHub Actions release workflow: + +- ✅ **Windows x86_64** - Native build (no Docker) +- ✅ **Windows ARM64** - Native build (no Docker) +- ✅ **Linux x86_64** - Native build (no Docker) +- ✅ **Linux ARM64** - Docker build with QEMU (Docker required) +- ✅ **macOS x86_64** (Intel) - Native build (no Docker) +- ✅ **macOS ARM64** (Apple Silicon) - Native build (no Docker) + +## When to Use Docker + +**Required:** +- Building Linux ARM64 binaries (GitHub Actions uses this for releases) + +**Optional:** +- Local Linux x86_64 builds if you prefer Docker +- Testing binaries in clean environments +- Building Linux binaries on Windows/macOS + +**Not needed:** +- Most GitHub Actions builds (use native Python) +- Local builds on Linux/macOS/Windows with Python installed + +## Maintenance + +When updating these Dockerfiles: + +1. Test the build locally first +2. Verify the binary works in a clean container +3. Update this README if usage changes +4. Update `.github/workflows/release.yml` if needed +5. Ensure Linux ARM64 builds still work in CI diff --git a/docs/BUILD.md b/docs/BUILD.md new file mode 100644 index 00000000..c54ced70 --- /dev/null +++ b/docs/BUILD.md @@ -0,0 +1,172 @@ +# Building Executables + +This document explains how to build standalone executables for the sonar-reports tool. + +## Supported Platforms + +The tool can be built for the following platforms: + +- **Windows** + - x86_64 (Intel/AMD 64-bit) + - ARM64 + +- **Linux** + - x86_64 (Intel/AMD 64-bit) + - ARM64 (aarch64) + +- **macOS** + - x86_64 (Intel) + - ARM64 (Apple Silicon) + +## Automated Builds (GitHub Actions) + +The easiest way to get executables is through automated builds: + +1. Push a tag to trigger a release build: + ```bash + git tag -a v1.0.0 -m "Release version 1.0.0" + git push origin v1.0.0 + ``` + +2. GitHub Actions will automatically build executables for all platforms + +3. Download the executables from the GitHub Releases page + +Alternatively, you can manually trigger a build from the Actions tab in GitHub. + +## Local Builds + +### Prerequisites + +- Python 3.11 or higher +- pip package manager + +### Building on Unix-like Systems (Linux, macOS) + +```bash +# Run the build script +./scripts/build.sh +``` + +The script will: +1. Install all required dependencies +2. Detect your OS and architecture +3. Build the executable using PyInstaller +4. Place the binary in the `dist/` directory + +The output binary will be named according to your platform: +- macOS ARM64: `sonar-reports-macos-arm64` +- macOS Intel: `sonar-reports-macos-x86_64` +- Linux ARM64: `sonar-reports-linux-arm64` +- Linux x86_64: `sonar-reports-linux-x86_64` + +### Building on Windows + +```cmd +# Run the build script +scripts\build.bat +``` + +The script will: +1. Install all required dependencies +2. Detect your architecture +3. Build the executable using PyInstaller +4. Place the binary in the `dist\` directory + +The output binary will be named according to your platform: +- Windows x86_64: `sonar-reports-windows-x86_64.exe` +- Windows ARM64: `sonar-reports-windows-arm64.exe` + +## Manual Build + +If you prefer to build manually: + +1. Install dependencies: + ```bash + pip install -r requirements.txt + pip install -r requirements-dev.txt + ``` + +2. Build with PyInstaller: + ```bash + pyinstaller sonar-reports.spec + ``` + +3. The executable will be in the `dist/` directory + +### Cross-compilation Notes + +- **Windows ARM64**: Can be built on x86_64 Windows using `pyinstaller --target-arch arm64` +- **macOS Universal**: Can be built on macOS using `pyinstaller --target-arch universal2` +- **Linux ARM64**: Requires ARM64 Linux or QEMU user-mode emulation + +## Using the Executable + +Once built, you can use the executable just like the Python script: + +```bash +# Extract from SonarQube +./dist/sonar-reports-macos-arm64 extract http://localhost:9000 YOUR_TOKEN + +# Using config file (see below) +./dist/sonar-reports-macos-arm64 extract --config config.json +``` + +## Configuration File Support + +Instead of passing command-line arguments, you can use a JSON configuration file: + +```json +{ + "url": "http://localhost:9000", + "token": "YOUR_SONARQUBE_TOKEN", + "export_directory": "./files", + "concurrency": 10, + "timeout": 60 +} +``` + +Then run: +```bash +./dist/sonar-reports-macos-arm64 extract --config config.json +``` + +See [CONFIG.md](CONFIG.md) for detailed configuration file documentation. + +## Troubleshooting + +### Build Fails with Missing Modules + +If the build fails due to missing modules, you may need to update the `hiddenimports` list in [sonar-reports.spec](../sonar-reports.spec:21). + +### Executable is Too Large + +The executable size can be reduced by: +- Removing UPX compression (change `upx=True` to `upx=False` in the spec file) +- Excluding unnecessary modules in the spec file +- Using `--onefile` mode (default) + +### Executable Doesn't Run + +- Ensure you're running the correct version for your platform +- On macOS/Linux, make sure the file is executable: `chmod +x sonar-reports-*` +- On macOS, you may need to allow the app in System Preferences > Security & Privacy +- Check antivirus software - it may be blocking the executable + +### macOS "Unverified Developer" Warning + +macOS may show a warning about unverified developers. To run the executable: + +1. Right-click the executable and select "Open" +2. Click "Open" in the dialog + +Or from the command line: +```bash +xattr -d com.apple.quarantine sonar-reports-macos-arm64 +``` + +## Distribution + +The executables are standalone and don't require Python or any dependencies to be installed. You can distribute them to users who can run them directly. + +**Important**: Always include configuration documentation with distributed executables so users know how to set up their credentials and endpoints. diff --git a/docs/CONFIG.md b/docs/CONFIG.md new file mode 100644 index 00000000..080b92fb --- /dev/null +++ b/docs/CONFIG.md @@ -0,0 +1,271 @@ +# Configuration File Documentation + +This document explains how to use JSON configuration files with sonar-reports instead of command-line arguments. + +## Why Use Configuration Files? + +Configuration files offer several advantages: + +- **Security**: Keep sensitive tokens in a file that's not in version control +- **Convenience**: Avoid typing long command-line arguments repeatedly +- **Reproducibility**: Share configurations easily across teams +- **Clarity**: All settings in one place, easy to review and modify + +## Quick Start + +1. Copy an example config file: + ```bash + cp config-extract.example.json my-config.json + ``` + +2. Edit the file with your values: + ```json + { + "url": "http://localhost:9000", + "token": "squ_your_actual_token_here", + "export_directory": "./files", + "concurrency": 10, + "timeout": 60 + } + ``` + +3. Use it with the executable: + ```bash + ./sonar-reports extract --config my-config.json + ``` + +## Configuration File Format + +Configuration files are standard JSON format. All fields are optional - you can specify only the values you need, and use command-line arguments for the rest. + +### For Extract Command + +Create a file named `extract-config.json`: + +```json +{ + "url": "http://localhost:9000", + "token": "YOUR_SONARQUBE_TOKEN", + "export_directory": "./files", + "extract_type": "all", + "concurrency": 10, + "timeout": 60, + "pem_file_path": null, + "key_file_path": null, + "cert_password": null, + "target_task": null, + "extract_id": null +} +``` + +Usage: +```bash +./sonar-reports extract --config extract-config.json +``` + +### For Migrate Command + +Create a file named `migrate-config.json`: + +```json +{ + "token": "YOUR_SONARCLOUD_TOKEN", + "enterprise_key": "YOUR_ENTERPRISE_KEY", + "url": "https://sonarcloud.io/", + "edition": "enterprise", + "export_directory": "./files", + "concurrency": 10, + "run_id": null, + "target_task": null +} +``` + +Usage: +```bash +./sonar-reports migrate --config migrate-config.json +``` + +## Configuration Parameters + +### Extract Command Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `url` | string | **required** | SonarQube server URL | +| `token` | string | **required** | Admin user token for SonarQube | +| `export_directory` | string | `/app/files/` | Directory to output export files | +| `extract_type` | string | `all` | Type of extract to run | +| `concurrency` | number | `25` | Maximum concurrent requests | +| `timeout` | number | `60` | Request timeout in seconds | +| `pem_file_path` | string | `null` | Path to client certificate PEM file | +| `key_file_path` | string | `null` | Path to client certificate key file | +| `cert_password` | string | `null` | Password for client certificate | +| `target_task` | string | `null` | Target task to complete | +| `extract_id` | string | `null` | ID of extract to resume | + +### Migrate Command Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `token` | string | **required** | SonarCloud admin token | +| `enterprise_key` | string | **required** | SonarCloud enterprise key | +| `url` | string | `https://sonarcloud.io/` | SonarCloud URL | +| `edition` | string | `enterprise` | SonarCloud license edition | +| `export_directory` | string | `/app/files/` | Directory containing exports | +| `concurrency` | number | `25` | Maximum concurrent requests | +| `run_id` | string | `null` | ID of run to resume | +| `target_task` | string | `null` | Specific migration task | + +## Combining Config Files and CLI Arguments + +Command-line arguments **override** config file values. This allows you to: + +- Keep common settings in a config file +- Override specific values on the command line + +Example: +```bash +# Config file has concurrency: 10 +./sonar-reports extract --config my-config.json --concurrency 5 + +# This will use concurrency of 5, overriding the config file +``` + +## Security Best Practices + +### 1. Never Commit Tokens to Version Control + +Add your config files to `.gitignore`: + +```bash +echo "config.json" >> .gitignore +echo "*-config.json" >> .gitignore +echo "my-*.json" >> .gitignore +``` + +### 2. Use Environment Variables + +You can reference environment variables in your workflow: + +```bash +# Set token as environment variable +export SONAR_TOKEN="squ_your_token" + +# Create config file dynamically +cat > config.json < full-migrate migration-config.json", + + "sonarqube": { + "url": "http://localhost:9000", + "token": "YOUR_SONARQUBE_ADMIN_TOKEN_HERE" + }, + + "sonarcloud": { + "url": "https://sonarcloud.io/", + "token": "YOUR_SONARCLOUD_ADMIN_TOKEN_HERE", + "enterprise_key": "YOUR_ENTERPRISE_KEY_HERE", + "org_key": "YOUR_TARGET_ORGANIZATION_KEY_HERE" + }, + + "settings": { + "export_directory": "./files", + "concurrency": 10, + "timeout": 60 + } +} diff --git a/requirements-dev.txt b/requirements-dev.txt index 7b80c3d0..1e13aee3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,3 +4,4 @@ pytest-asyncio pytest-cov respx jsonschema +pyinstaller diff --git a/scripts/build.bat b/scripts/build.bat new file mode 100644 index 00000000..45cf749f --- /dev/null +++ b/scripts/build.bat @@ -0,0 +1,40 @@ +@echo off +REM Build script for Windows + +echo Building sonar-reports executable... + +REM Check if Python is installed +python --version >nul 2>&1 +if %errorlevel% neq 0 ( + echo Error: Python is not installed + exit /b 1 +) + +REM Install dependencies +echo Installing dependencies... +pip install -r requirements.txt +pip install -r requirements-dev.txt + +REM Detect architecture +set ARCH=%PROCESSOR_ARCHITECTURE% +echo Detected Architecture: %ARCH% + +REM Build with PyInstaller +echo Building with PyInstaller... +if "%ARCH%"=="ARM64" ( + pyinstaller sonar-reports.spec --target-arch arm64 + set OUTPUT_NAME=sonar-reports-windows-arm64.exe +) else ( + pyinstaller sonar-reports.spec + set OUTPUT_NAME=sonar-reports-windows-x86_64.exe +) + +REM Rename the binary +echo Renaming binary to %OUTPUT_NAME%... +move dist\sonar-reports.exe "dist\%OUTPUT_NAME%" + +echo Build complete! Binary available at: dist\%OUTPUT_NAME% +echo. +echo You can now run: dist\%OUTPUT_NAME% --help + +pause diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 00000000..1349d050 --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Build script for Unix-like systems (Linux, macOS) + +set -e + +echo "Building sonar-reports executable..." + +# Check if Python is installed +if ! command -v python3 &> /dev/null; then + echo "Error: Python 3 is not installed" + exit 1 +fi + +# Create virtual environment if it doesn't exist +if [ ! -d "venv" ]; then + echo "Creating virtual environment..." + python3 -m venv venv +fi + +# Activate virtual environment +echo "Activating virtual environment..." +source venv/bin/activate + +# Install dependencies +echo "Installing dependencies..." +pip install --upgrade pip +pip install -r requirements.txt +pip install -r requirements-dev.txt + +# Detect architecture +ARCH=$(uname -m) +OS=$(uname -s) + +echo "Detected OS: $OS" +echo "Detected Architecture: $ARCH" + +# Build with PyInstaller +echo "Building with PyInstaller..." +pyinstaller sonar-reports.spec + +# Determine output name based on platform +if [[ "$OS" == "Darwin" ]]; then + # macOS + if [[ "$ARCH" == "arm64" ]]; then + OUTPUT_NAME="sonar-reports-macos-arm64" + else + OUTPUT_NAME="sonar-reports-macos-x86_64" + fi +elif [[ "$OS" == "Linux" ]]; then + # Linux + if [[ "$ARCH" == "aarch64" ]] || [[ "$ARCH" == "arm64" ]]; then + OUTPUT_NAME="sonar-reports-linux-arm64" + else + OUTPUT_NAME="sonar-reports-linux-x86_64" + fi +else + echo "Unsupported operating system: $OS" + exit 1 +fi + +# Rename the binary +echo "Renaming binary to $OUTPUT_NAME..." +mv dist/sonar-reports "dist/$OUTPUT_NAME" + +# Deactivate virtual environment +deactivate + +echo "Build complete! Binary available at: dist/$OUTPUT_NAME" +echo "" +echo "You can now run: ./dist/$OUTPUT_NAME --help" diff --git a/execute_full_migration.sh b/scripts/execute_full_migration.sh similarity index 100% rename from execute_full_migration.sh rename to scripts/execute_full_migration.sh diff --git a/scripts/execute_migration_with_binary.sh b/scripts/execute_migration_with_binary.sh new file mode 100755 index 00000000..ce934634 --- /dev/null +++ b/scripts/execute_migration_with_binary.sh @@ -0,0 +1,262 @@ +#!/bin/bash + +################################################################################ +# SonarQube to SonarCloud Migration Script (Using Binary Executable) +# +# This script automates the complete migration process from SonarQube Server +# to SonarCloud using the standalone binary executable. +# +# Usage: +# 1. Build the binary: ./scripts/build.sh +# 2. Create migration-config.json from examples/migration-config.example.json +# 3. Run: ./scripts/execute_migration_with_binary.sh migration-config.json +################################################################################ + +set -e # Exit on any error + +# ============================================================================= +# Configuration +# ============================================================================= + +# Detect platform and set binary name +OS=$(uname -s) +ARCH=$(uname -m) + +if [[ "$OS" == "Darwin" ]]; then + if [[ "$ARCH" == "arm64" ]]; then + BINARY="./dist/sonar-reports-macos-arm64" + else + BINARY="./dist/sonar-reports-macos-x86_64" + fi +elif [[ "$OS" == "Linux" ]]; then + if [[ "$ARCH" == "aarch64" ]] || [[ "$ARCH" == "arm64" ]]; then + BINARY="./dist/sonar-reports-linux-arm64" + else + BINARY="./dist/sonar-reports-linux-x86_64" + fi +else + echo "Error: Unsupported operating system: $OS" + exit 1 +fi + +# Check if binary exists +if [ ! -f "$BINARY" ]; then + echo "Error: Binary not found at $BINARY" + echo "Please build the binary first using: ./scripts/build.sh" + exit 1 +fi + +# Get config file from argument or use default +CONFIG_FILE="${1:-migration-config.json}" + +if [ ! -f "$CONFIG_FILE" ]; then + echo "Error: Config file not found: $CONFIG_FILE" + echo "" + echo "Usage: $0 [config-file.json]" + echo "" + echo "Example:" + echo " 1. Copy the example: cp examples/migration-config.example.json migration-config.json" + echo " 2. Edit migration-config.json with your values" + echo " 3. Run: $0 migration-config.json" + exit 1 +fi + +# ============================================================================= +# Colors and helper functions +# ============================================================================= + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +print_step() { + echo -e "\n${GREEN}===================================================${NC}" + echo -e "${GREEN}$1${NC}" + echo -e "${GREEN}===================================================${NC}\n" +} + +print_error() { + echo -e "${RED}ERROR: $1${NC}" >&2 +} + +print_warning() { + echo -e "${YELLOW}WARNING: $1${NC}" +} + +# ============================================================================= +# Load configuration from JSON +# ============================================================================= + +print_step "Loading Configuration" + +# Extract values from JSON using built-in tools (python3 or jq) +if command -v jq &> /dev/null; then + # Use jq if available + SONARQUBE_URL=$(jq -r '.sonarqube.url' "$CONFIG_FILE") + SONARQUBE_TOKEN=$(jq -r '.sonarqube.token' "$CONFIG_FILE") + SONARCLOUD_URL=$(jq -r '.sonarcloud.url' "$CONFIG_FILE") + SONARCLOUD_TOKEN=$(jq -r '.sonarcloud.token' "$CONFIG_FILE") + SONARCLOUD_ENTERPRISE_KEY=$(jq -r '.sonarcloud.enterprise_key' "$CONFIG_FILE") + SONARCLOUD_ORG_KEY=$(jq -r '.sonarcloud.org_key' "$CONFIG_FILE") + EXPORT_DIR=$(jq -r '.settings.export_directory' "$CONFIG_FILE") + CONCURRENCY=$(jq -r '.settings.concurrency' "$CONFIG_FILE") + TIMEOUT=$(jq -r '.settings.timeout' "$CONFIG_FILE") +elif command -v python3 &> /dev/null; then + # Fallback to python3 + read -r SONARQUBE_URL SONARQUBE_TOKEN SONARCLOUD_URL SONARCLOUD_TOKEN SONARCLOUD_ENTERPRISE_KEY SONARCLOUD_ORG_KEY EXPORT_DIR CONCURRENCY TIMEOUT <<< $(python3 -c " +import json +with open('$CONFIG_FILE') as f: + config = json.load(f) +print(config['sonarqube']['url'], + config['sonarqube']['token'], + config['sonarcloud']['url'], + config['sonarcloud']['token'], + config['sonarcloud']['enterprise_key'], + config['sonarcloud']['org_key'], + config['settings']['export_directory'], + config['settings']['concurrency'], + config['settings']['timeout']) +") +else + print_error "Neither jq nor python3 is available. Please install one of them." + exit 1 +fi + +# Create export directory if it doesn't exist +mkdir -p "$EXPORT_DIR" + +# Get absolute path for export directory +EXPORT_DIR_ABS=$(cd "$EXPORT_DIR" && pwd) + +print_step "Migration Configuration" +echo "Binary: $BINARY" +echo "Config File: $CONFIG_FILE" +echo "SonarQube URL: $SONARQUBE_URL" +echo "SonarCloud URL: $SONARCLOUD_URL" +echo "SonarCloud Enterprise: $SONARCLOUD_ENTERPRISE_KEY" +echo "SonarCloud Organization: $SONARCLOUD_ORG_KEY" +echo "Export Directory: $EXPORT_DIR_ABS" +echo "Concurrency: $CONCURRENCY" +echo "Timeout: ${TIMEOUT}s" + +# ============================================================================= +# Step 1: Extract Data from SonarQube +# ============================================================================= +print_step "Step 1: Extracting Data from SonarQube" + +if ! "$BINARY" extract \ + "$SONARQUBE_URL" \ + "$SONARQUBE_TOKEN" \ + --export_directory="$EXPORT_DIR_ABS" \ + --concurrency="$CONCURRENCY" \ + --timeout="$TIMEOUT"; then + print_error "Failed to extract data from SonarQube" + exit 1 +fi + +echo -e "${GREEN}✓ Data extracted successfully${NC}" + +# ============================================================================= +# Step 2: Generate Organization Structure +# ============================================================================= +print_step "Step 2: Generating Organization Structure" + +if ! "$BINARY" structure --export_directory="$EXPORT_DIR_ABS"; then + print_error "Failed to generate organization structure" + exit 1 +fi + +echo -e "${GREEN}✓ Organization structure generated${NC}" + +# ============================================================================= +# Step 3: Update organizations.csv with SonarCloud Org Key +# ============================================================================= +print_step "Step 3: Updating organizations.csv" + +ORGS_FILE="${EXPORT_DIR_ABS}/organizations.csv" + +if [ ! -f "$ORGS_FILE" ]; then + print_error "organizations.csv not found at $ORGS_FILE" + exit 1 +fi + +# Backup the original file +cp "$ORGS_FILE" "${ORGS_FILE}.backup" + +# Update the sonarcloud_org_key column (second column) in all data rows +awk -v org_key="$SONARCLOUD_ORG_KEY" 'BEGIN {FS=OFS=","} + NR==1 {print; next} # Print header as-is + {$2=org_key; print} # Update second column and print +' "${ORGS_FILE}.backup" > "$ORGS_FILE" + +echo "Updated organizations.csv:" +cat "$ORGS_FILE" +echo -e "${GREEN}✓ organizations.csv updated with SonarCloud org key${NC}" + +# ============================================================================= +# Step 4: Generate Mappings +# ============================================================================= +print_step "Step 4: Generating Mappings" + +if ! "$BINARY" mappings --export_directory="$EXPORT_DIR_ABS"; then + print_error "Failed to generate mappings" + exit 1 +fi + +echo -e "${GREEN}✓ Mappings generated successfully${NC}" +echo "Generated mapping files:" +ls -lh "${EXPORT_DIR_ABS}"/*.csv + +# ============================================================================= +# Step 5: Run Migration to SonarCloud +# ============================================================================= +print_step "Step 5: Migrating to SonarCloud" + +print_warning "This step may take several minutes depending on the number of projects..." + +if ! "$BINARY" migrate \ + "$SONARCLOUD_TOKEN" \ + "$SONARCLOUD_ENTERPRISE_KEY" \ + --url="$SONARCLOUD_URL" \ + --export_directory="$EXPORT_DIR_ABS" \ + --concurrency="$CONCURRENCY"; then + print_error "Migration failed" + exit 1 +fi + +echo -e "${GREEN}✓ Migration completed successfully${NC}" + +# ============================================================================= +# Step 6: Verify Migration +# ============================================================================= +print_step "Step 6: Verifying Migration" + +SONARCLOUD_BASE_URL="${SONARCLOUD_URL%/}" + +echo "Fetching projects from SonarCloud..." +PROJECT_COUNT=$(curl -s -H "Authorization: Bearer $SONARCLOUD_TOKEN" \ + "${SONARCLOUD_BASE_URL}/api/projects/search?organization=${SONARCLOUD_ORG_KEY}&ps=500" | \ + python3 -c "import sys, json; print(len(json.load(sys.stdin).get('components', [])))" 2>/dev/null || echo "0") + +echo -e "${GREEN}✓ Found $PROJECT_COUNT projects in SonarCloud${NC}" + +# ============================================================================= +# Migration Complete +# ============================================================================= +print_step "Migration Complete!" + +echo "Summary:" +echo " • Projects migrated: $PROJECT_COUNT" +echo " • Export data location: $EXPORT_DIR_ABS" +echo " • Migration logs: $EXPORT_DIR_ABS/*/requests.log" +echo "" +echo "View your projects at:" +echo " ${SONARCLOUD_BASE_URL}/organizations/${SONARCLOUD_ORG_KEY}/projects" +echo "" +echo -e "${YELLOW}IMPORTANT:${NC}" +echo " • Historical analysis data, issues, and code coverage were NOT migrated" +echo " • You need to re-scan your projects to populate code and issues" +echo " • Configure DevOps integrations (GitHub, Azure DevOps, etc.) for automatic analysis" +echo "" +echo -e "${GREEN}Migration completed successfully!${NC}" diff --git a/sonar-reports.spec b/sonar-reports.spec new file mode 100644 index 00000000..eefc8306 --- /dev/null +++ b/sonar-reports.spec @@ -0,0 +1,70 @@ +# -*- mode: python ; coding: utf-8 -*- +from PyInstaller.utils.hooks import collect_data_files, collect_submodules + +# Collect all submodules from src directory +hiddenimports = collect_submodules('src') +hiddenimports += [ + 'httpx', + 'click', + 'markdown', + 'tenacity', + 'nacl', + 'ruamel.yaml', + 'bashlex', + 'lxml', + # Explicitly include operations modules that are dynamically imported + 'operations.set_key', + 'operations.apply_filter', + 'operations.expand_list', + 'operations.extend_list', + 'operations.get_date', + 'operations.join_string', + 'operations.match_devops_platform', + 'operations.process_array', + 'operations.http_request', + 'operations.http_request.base', + 'operations.http_request.get', +] + +# Collect data files (if any) +# Include the tasks directory with all JSON configuration files +import os +tasks_dir = os.path.join('src', 'tasks') +datas = [(tasks_dir, 'tasks')] + +a = Analysis( + ['src/main.py'], + pathex=[], + binaries=[], + datas=datas, + hiddenimports=hiddenimports, + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=[], + noarchive=False, + optimize=0, +) + +pyz = PYZ(a.pure) + +exe = EXE( + pyz, + a.scripts, + a.binaries, + a.datas, + [], + name='sonar-reports', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + upx_exclude=[], + runtime_tmpdir=None, + console=True, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, +) diff --git a/src/config.py b/src/config.py new file mode 100644 index 00000000..97bfe655 --- /dev/null +++ b/src/config.py @@ -0,0 +1,78 @@ +"""Configuration file loader for sonar-reports""" +import json +import os +from typing import Dict, Any, Optional + + +def load_config_file(config_path: str) -> Dict[str, Any]: + """Load configuration from a JSON file. + + Args: + config_path: Path to the JSON configuration file + + Returns: + Dictionary containing configuration values + + Raises: + FileNotFoundError: If the config file doesn't exist + json.JSONDecodeError: If the config file is not valid JSON + """ + if not os.path.exists(config_path): + raise FileNotFoundError(f"Configuration file not found: {config_path}") + + with open(config_path, 'r') as f: + config = json.load(f) + + return config + + +def merge_config_with_cli(config: Dict[str, Any], cli_args: Dict[str, Any]) -> Dict[str, Any]: + """Merge configuration file values with CLI arguments. + + CLI arguments take precedence over config file values. + + Args: + config: Configuration from JSON file + cli_args: Arguments provided via CLI + + Returns: + Merged configuration dictionary + """ + merged = config.copy() + + # CLI args override config file values + for key, value in cli_args.items(): + if value is not None: + merged[key] = value + + return merged + + +def get_config_value(config: Dict[str, Any], key: str, default: Any = None) -> Any: + """Get a configuration value with fallback to default. + + Args: + config: Configuration dictionary + key: Key to lookup + default: Default value if key not found + + Returns: + Configuration value or default + """ + return config.get(key, default) + + +def validate_required_keys(config: Dict[str, Any], required_keys: list) -> None: + """Validate that required configuration keys are present. + + Args: + config: Configuration dictionary + required_keys: List of required key names + + Raises: + ValueError: If any required key is missing + """ + missing_keys = [key for key in required_keys if key not in config or config[key] is None] + + if missing_keys: + raise ValueError(f"Missing required configuration keys: {', '.join(missing_keys)}") diff --git a/src/main.py b/src/main.py index 7cd0ee55..3a4fdf30 100644 --- a/src/main.py +++ b/src/main.py @@ -13,6 +13,7 @@ from validate import validate_migration from importlib import import_module from pipelines.process import update_pipelines +from config import load_config_file, merge_config_with_cli @click.group() @@ -21,19 +22,20 @@ def cli(): @cli.command() -@click.argument('url') -@click.argument('token') +@click.argument('url', required=False) +@click.argument('token', required=False) +@click.option('--config', 'config_file', help="Path to JSON configuration file") @click.option('--pem_file_path', help="Path to client certificate pem file") @click.option('--key_file_path', help="Path to client certificate key file") @click.option('--cert_password', help="Password for client certificate") -@click.option('--export_directory', default='/app/files/', help="Root Directory to output the export") -@click.option('--extract_type', default='all', help='Type of Extract to run') -@click.option('--concurrency', default=25, help='Maximum number of concurrent requests') -@click.option('--timeout', default=60, help='Number of seconds before a request will timeout') +@click.option('--export_directory', default=None, help="Root Directory to output the export") +@click.option('--extract_type', default=None, help='Type of Extract to run') +@click.option('--concurrency', default=None, type=int, help='Maximum number of concurrent requests') +@click.option('--timeout', default=None, type=int, help='Number of seconds before a request will timeout') @click.option('--extract_id', help='ID of an extract to resume in case of failures. Extract will start by retrying last completed task') @click.option('--target_task', help='Target Task to complete. All dependent tasks will be included') -def extract(url, token, export_directory: str, extract_type, pem_file_path, key_file_path, cert_password, target_task, +def extract(url, token, config_file, export_directory: str, extract_type, pem_file_path, key_file_path, cert_password, target_task, concurrency, timeout, extract_id): """Extracts data from a SonarQube Server instance and stores it in the export directory as new line delimited json files @@ -41,7 +43,57 @@ def extract(url, token, export_directory: str, extract_type, pem_file_path, key_ TOKEN is an admin user token to the SonarQube instance + You can also use --config to specify a JSON configuration file instead of command-line arguments. """ + # Load config file if provided + if config_file: + try: + config = load_config_file(config_file) + # Merge CLI args with config file (CLI takes precedence) + cli_args = { + 'url': url, + 'token': token, + 'export_directory': export_directory, + 'extract_type': extract_type, + 'pem_file_path': pem_file_path, + 'key_file_path': key_file_path, + 'cert_password': cert_password, + 'target_task': target_task, + 'concurrency': concurrency, + 'timeout': timeout, + 'extract_id': extract_id, + } + config = merge_config_with_cli(config, cli_args) + url = config.get('url') + token = config.get('token') + export_directory = config.get('export_directory', '/app/files/') + extract_type = config.get('extract_type', 'all') + pem_file_path = config.get('pem_file_path') + key_file_path = config.get('key_file_path') + cert_password = config.get('cert_password') + target_task = config.get('target_task') + concurrency = config.get('concurrency', 25) + timeout = config.get('timeout', 60) + extract_id = config.get('extract_id') + except (FileNotFoundError, json.JSONDecodeError) as e: + click.echo(f"Error loading config file: {e}") + return + else: + # Set defaults if not using config file + if export_directory is None: + export_directory = '/app/files/' + if extract_type is None: + extract_type = 'all' + if concurrency is None: + concurrency = 25 + if timeout is None: + timeout = 60 + + # Validate required arguments + if not url or not token: + click.echo("Error: URL and TOKEN are required (either as arguments or in config file)") + return + if not url.endswith('/'): url = f"{url}/" if extract_id is None: @@ -148,23 +200,69 @@ def mappings(export_directory): @cli.command() -@click.argument('token') -@click.argument('enterprise_key') -@click.option('--edition', default='enterprise') -@click.option('--url', default='https://sonarcloud.io/') +@click.argument('token', required=False) +@click.argument('enterprise_key', required=False) +@click.option('--config', 'config_file', help="Path to JSON configuration file") +@click.option('--edition', default=None) +@click.option('--url', default=None) @click.option('--run_id', default=None, help='ID of a run to resume in case of failures. Migration will resume by retrying the last completed task') -@click.option('--concurrency', default=25, help='Maximum number of concurrent requests') -@click.option('--export_directory', default='/app/files/', +@click.option('--concurrency', default=None, type=int, help='Maximum number of concurrent requests') +@click.option('--export_directory', default=None, help="Root Directory containing all of the SonarQube exports") @click.option('--target_task', help='Name of a specific migration task to complete. All dependent tasks will be be included') -def migrate(token, edition, url, enterprise_key, concurrency, run_id, export_directory, target_task): +def migrate(token, edition, url, enterprise_key, concurrency, run_id, export_directory, target_task, config_file): """Migrate SonarQube Server configurations to SonarQube Cloud. User must run the structure and mappings command and add the SonarQube Cloud organization keys to the organizations.csv in order to start the migration TOKEN is a user token that has admin permissions at the enterprise level and all organizations ENTERPRISE_KEY is the key of the SonarQube Cloud enterprise. All migrating organizations must be added to the enterprise + + You can also use --config to specify a JSON configuration file instead of command-line arguments. """ + # Load config file if provided + if config_file: + try: + config = load_config_file(config_file) + # Merge CLI args with config file (CLI takes precedence) + cli_args = { + 'token': token, + 'enterprise_key': enterprise_key, + 'edition': edition, + 'url': url, + 'run_id': run_id, + 'concurrency': concurrency, + 'export_directory': export_directory, + 'target_task': target_task, + } + config = merge_config_with_cli(config, cli_args) + token = config.get('token') + enterprise_key = config.get('enterprise_key') + edition = config.get('edition', 'enterprise') + url = config.get('url', 'https://sonarcloud.io/') + run_id = config.get('run_id') + concurrency = config.get('concurrency', 25) + export_directory = config.get('export_directory', '/app/files/') + target_task = config.get('target_task') + except (FileNotFoundError, json.JSONDecodeError) as e: + click.echo(f"Error loading config file: {e}") + return + else: + # Set defaults if not using config file + if edition is None: + edition = 'enterprise' + if url is None: + url = 'https://sonarcloud.io/' + if concurrency is None: + concurrency = 25 + if export_directory is None: + export_directory = '/app/files/' + + # Validate required arguments + if not token or not enterprise_key: + click.echo("Error: TOKEN and ENTERPRISE_KEY are required (either as arguments or in config file)") + return + create_plan = False configure_client(url=url, cert=None, server_version="cloud", token=token) api_url = url.replace('https://', 'https://api.') @@ -295,5 +393,203 @@ def pipelines(secrets_file, sonar_token, sonar_url, input_directory, output_dire click.echo(f"Repositories Updated: {list(results.keys())}") +@cli.command() +@click.argument('config_file') +def full_migrate(config_file): + """Complete end-to-end migration from SonarQube to SonarCloud using a single config file. + + CONFIG_FILE is a JSON file containing all configuration for the migration. + + This command will automatically: + 1. Extract data from SonarQube Server + 2. Generate organization structure + 3. Update organizations with your SonarCloud org key + 4. Generate all mappings + 5. Migrate everything to SonarCloud + + Example config file: + { + "sonarqube": { + "url": "http://localhost:9000", + "token": "YOUR_TOKEN" + }, + "sonarcloud": { + "url": "https://sonarcloud.io/", + "token": "YOUR_TOKEN", + "enterprise_key": "YOUR_ENTERPRISE", + "org_key": "YOUR_ORG" + }, + "settings": { + "export_directory": "./files", + "concurrency": 10, + "timeout": 60 + } + } + """ + try: + config = load_config_file(config_file) + except (FileNotFoundError, json.JSONDecodeError) as e: + click.echo(f"Error loading config file: {e}") + return + + # Extract configuration values + try: + sonarqube_url = config['sonarqube']['url'] + sonarqube_token = config['sonarqube']['token'] + sonarcloud_url = config['sonarcloud']['url'] + sonarcloud_token = config['sonarcloud']['token'] + sonarcloud_enterprise_key = config['sonarcloud']['enterprise_key'] + sonarcloud_org_key = config['sonarcloud']['org_key'] + export_directory = config.get('settings', {}).get('export_directory', './files') + concurrency = config.get('settings', {}).get('concurrency', 10) + timeout = config.get('settings', {}).get('timeout', 60) + except KeyError as e: + click.echo(f"Error: Missing required configuration key: {e}") + click.echo("\nRequired structure:") + click.echo(" sonarqube.url, sonarqube.token") + click.echo(" sonarcloud.url, sonarcloud.token, sonarcloud.enterprise_key, sonarcloud.org_key") + return + + # Ensure URLs end with / + if not sonarqube_url.endswith('/'): + sonarqube_url = f"{sonarqube_url}/" + if not sonarcloud_url.endswith('/'): + sonarcloud_url = f"{sonarcloud_url}/" + + # Create export directory + os.makedirs(export_directory, exist_ok=True) + export_dir_abs = os.path.abspath(export_directory) + + click.echo("=" * 60) + click.echo("SonarQube to SonarCloud Full Migration") + click.echo("=" * 60) + click.echo(f"SonarQube URL: {sonarqube_url}") + click.echo(f"SonarCloud URL: {sonarcloud_url}") + click.echo(f"SonarCloud Org: {sonarcloud_org_key}") + click.echo(f"Export Directory: {export_dir_abs}") + click.echo(f"Concurrency: {concurrency}") + click.echo("=" * 60) + click.echo() + + # Step 1: Extract from SonarQube + click.echo("Step 1/5: Extracting data from SonarQube...") + extract_id = str(int(datetime.now(UTC).timestamp())) + cert = configure_client_cert(None, None, None) + server_version, edition = get_server_details(url=sonarqube_url, cert=cert, token=sonarqube_token) + extract_directory = os.path.join(export_dir_abs, extract_id + '/') + os.makedirs(extract_directory, exist_ok=True) + configure_logger(name='http_request', level='INFO', output_file=os.path.join(extract_directory, 'requests.log')) + configure_client(url=sonarqube_url, cert=cert, server_version=server_version, token=sonarqube_token, + concurrency=concurrency, timeout=timeout) + + configs = get_available_task_configs(client_version=server_version, edition=edition) + target_tasks = list([k for k in configs.keys() if k.startswith('get')]) + plan = generate_task_plan(target_tasks=target_tasks, task_configs=configs) + + with open(os.path.join(extract_directory, 'extract.json'), 'wt') as f: + json.dump(dict(plan=plan, version=server_version, edition=edition, url=sonarqube_url, + target_tasks=target_tasks, available_configs=list(configs.keys()), + run_id=extract_id), f) + + execute_plan(execution_plan=plan, inputs=dict(url=sonarqube_url), concurrency=concurrency, + task_configs=configs, output_directory=export_dir_abs, current_run_id=extract_id, + run_ids={extract_id}) + click.echo("✓ Data extracted successfully\n") + + # Step 2: Generate structure + click.echo("Step 2/5: Generating organization structure...") + from structure import map_organization_structure, map_project_structure + extract_mapping = get_unique_extracts(directory=export_dir_abs) + bindings, projects = map_project_structure(export_directory=export_dir_abs, extract_mapping=extract_mapping) + organizations = map_organization_structure(bindings) + export_csv(directory=export_dir_abs, name='organizations', data=organizations) + export_csv(directory=export_dir_abs, name='projects', data=projects) + click.echo("✓ Organization structure generated\n") + + # Step 3: Update organizations.csv with SonarCloud org key + click.echo("Step 3/5: Updating organizations with SonarCloud org key...") + orgs_file = os.path.join(export_dir_abs, 'organizations.csv') + if not os.path.exists(orgs_file): + click.echo(f"Error: organizations.csv not found at {orgs_file}") + return + + # Read and update organizations + orgs = load_csv(directory=export_dir_abs, filename='organizations.csv') + for org in orgs: + org['sonarcloud_org_key'] = sonarcloud_org_key + export_csv(directory=export_dir_abs, name='organizations', data=orgs) + click.echo(f"✓ Updated {len(orgs)} organization(s) with org key: {sonarcloud_org_key}\n") + + # Step 4: Generate mappings + click.echo("Step 4/5: Generating mappings...") + from structure import map_templates, map_groups, map_profiles, map_gates, map_portfolios + projects = load_csv(directory=export_dir_abs, filename='projects.csv') + project_org_mapping = {p['server_url'] + p['key']: p['sonarqube_org_key'] for p in projects} + + mapping = dict( + templates=map_templates(project_org_mapping=project_org_mapping, extract_mapping=extract_mapping, + export_directory=export_dir_abs), + profiles=map_profiles(extract_mapping=extract_mapping, project_org_mapping=project_org_mapping, + export_directory=export_dir_abs), + gates=map_gates(project_org_mapping=project_org_mapping, extract_mapping=extract_mapping, + export_directory=export_dir_abs), + portfolios=map_portfolios(export_directory=export_dir_abs, extract_mapping=extract_mapping) + ) + mapping['groups'] = map_groups(project_org_mapping=project_org_mapping, profiles=mapping['profiles'], + extract_mapping=extract_mapping, export_directory=export_dir_abs, + templates=mapping['templates']) + + for k, v in mapping.items(): + export_csv(directory=export_dir_abs, name=k, data=v) + click.echo("✓ Mappings generated successfully\n") + + # Step 5: Migrate to SonarCloud + click.echo("Step 5/5: Migrating to SonarCloud...") + click.echo("This may take several minutes depending on the number of projects...") + + run_id = str(int(datetime.now(UTC).timestamp())) + run_dir, completed = validate_migration(directory=export_dir_abs, run_id=run_id) + + configure_client(url=sonarcloud_url, cert=None, server_version="cloud", token=sonarcloud_token) + api_url = sonarcloud_url.replace('https://', 'https://api.') + configure_client(url=api_url, cert=None, server_version="cloud", token=sonarcloud_token) + + configure_logger(name='http_request', level='INFO', output_file=os.path.join(run_dir, 'requests.log')) + + configs = get_available_task_configs(client_version='cloud', edition='enterprise') + target_tasks = list([k for k in configs.keys() if not any([k.startswith(i) for i in ['get', 'delete', 'reset']])]) + completed = completed.union(set(MIGRATION_TASKS)) + + plan = generate_task_plan(target_tasks=target_tasks, task_configs=configs, completed=completed) + + with open(os.path.join(run_dir, 'plan.json'), 'wt') as f: + json.dump(dict(plan=plan, version='cloud', edition='enterprise', completed=list(completed), + url=sonarcloud_url, target_tasks=target_tasks, + available_configs=list(configs.keys()), run_id=run_id), f) + + execute_plan(execution_plan=plan, inputs=dict(url=sonarcloud_url, api_url=api_url, + enterprise_key=sonarcloud_enterprise_key), concurrency=concurrency, + task_configs=configs, output_directory=export_dir_abs, current_run_id=run_id, + run_ids=set(extract_mapping.values()).union({run_id})) + + click.echo("✓ Migration completed successfully\n") + + # Summary + click.echo("=" * 60) + click.echo("Migration Complete!") + click.echo("=" * 60) + click.echo(f"Export data: {export_dir_abs}") + click.echo(f"Migration logs: {run_dir}/requests.log") + click.echo() + click.echo("View your projects at:") + click.echo(f" {sonarcloud_url.rstrip('/')}/organizations/{sonarcloud_org_key}/projects") + click.echo() + click.echo("IMPORTANT:") + click.echo(" • Historical analysis data, issues, and code coverage were NOT migrated") + click.echo(" • You need to re-scan your projects to populate code and issues") + click.echo(" • Configure DevOps integrations for automatic analysis") + click.echo() + + if __name__ == '__main__': cli() diff --git a/src/plan.py b/src/plan.py index f6ef35fd..7ef7e574 100644 --- a/src/plan.py +++ b/src/plan.py @@ -1,7 +1,15 @@ import os import json +import sys -TASK_DIR = os.path.join(os.path.dirname(__file__), './tasks/') +# Handle both development and PyInstaller bundled execution +if getattr(sys, 'frozen', False): + # Running as compiled executable + base_path = sys._MEIPASS + TASK_DIR = os.path.join(base_path, 'tasks') +else: + # Running as source code + TASK_DIR = os.path.join(os.path.dirname(__file__), './tasks/') def get_sonarqube_config(client_version, root_dir, edition, files):