diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 57f6d888d..fc4918302 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -1,15 +1,56 @@ name: Validate and Process EvalAI Challenge on: - push: + push: {} + pull_request: + types: [opened, synchronize, reopened, edited] branches: - - challenge + - 'challenge' permissions: contents: read issues: write jobs: + detect-and-validate-challenge-branch: + runs-on: ubuntu-latest + outputs: + is_challenge_branch: ${{ steps.branch-check.outputs.is_challenge_branch }} + challenge_branch: ${{ steps.branch-check.outputs.challenge_branch }} + steps: + - name: Detect and validate challenge branch + id: branch-check + run: | + # Get the actual branch name + if [ "${{ github.event_name }}" == "pull_request" ]; then + BRANCH="${{ github.head_ref }}" + else + BRANCH="${{ github.ref_name }}" + fi + + echo "🔍 Analyzing branch: $BRANCH" + echo "challenge_branch=$BRANCH" >> $GITHUB_OUTPUT + + # Check if this is a challenge branch + if [[ "$BRANCH" =~ ^challenge(-.*)?$ ]]; then + echo "is_challenge_branch=true" >> $GITHUB_OUTPUT + echo "✅ Valid challenge branch detected: $BRANCH" + + # Extract branch suffix for versioning + if [[ "$BRANCH" =~ ^challenge-(.+)$ ]]; then + SUFFIX="${BASH_REMATCH[1]}" + echo "branch_suffix=$SUFFIX" >> $GITHUB_OUTPUT + echo "📋 Branch suffix: $SUFFIX" + else + echo "branch_suffix=main" >> $GITHUB_OUTPUT + echo "📋 Main challenge branch (no suffix)" + fi + else + echo "is_challenge_branch=false" >> $GITHUB_OUTPUT + echo "â„šī¸ Not a challenge branch: $BRANCH" + echo "â­ī¸ Challenge processing will be skipped" + fi + validate-host-config: runs-on: ubuntu-latest outputs: @@ -74,7 +115,7 @@ jobs: - name: Checkout challenge branch uses: actions/checkout@v3 with: - ref: challenge + ref: ${{ github.ref_name }} - name: Set up Python uses: actions/setup-python@v4 @@ -88,7 +129,7 @@ jobs: - name: Validate challenge run: | - python3 github/challenge_processing_script.py + python3 github/challenge_processing_script.py ${GITHUB_REF#refs/heads/} env: IS_VALIDATION: 'True' GITHUB_CONTEXT: ${{ toJson(github) }} @@ -96,8 +137,8 @@ jobs: - name: Create or update challenge run: | - python3 github/challenge_processing_script.py - if: ${{ success() }} + python3 github/challenge_processing_script.py ${GITHUB_REF#refs/heads/} + if: ${{ github.event_name == 'push' && success() }} env: IS_VALIDATION: 'False' GITHUB_CONTEXT: ${{ toJson(github) }} diff --git a/README.md b/README.md index 458cc8fd9..14d799f3d 100644 --- a/README.md +++ b/README.md @@ -48,18 +48,22 @@ If you are looking for a simple challenge configuration that you can replicate t 5. Create a branch with name `challenge` in the forked repository from the `master` branch. Note: Only changes in `challenge` branch will be synchronized with challenge on EvalAI. +You may maintain multiple versions by using additional branches whose names start with `challenge-` (e.g. `challenge-2024`, `challenge-v2`). +Note: The CI pipeline only runs for branches that match the pattern `challenge` or `challenge-*`. Any other branch name will be ignored. + +If you trigger the processing script manually and do **not** pass a branch argument, it will assume the branch name is `challenge` by default. 6. Add `evalai_user_auth_token` and `host_team_pk` in `github/host_config.json`. 7. Read [EvalAI challenge creation documentation](https://evalai.readthedocs.io/en/latest/configuration.html) to know more about how you want to structure your challenge. Once you are ready, start making changes in the yaml file, HTML templates, evaluation script according to your need. -8. Commit the changes and push the `challenge` branch in the repository and wait for the build to complete. View the [logs of your build](https://docs.github.com/en/free-pro-team@latest/actions/managing-workflow-runs/using-workflow-run-logs#viewing-logs-to-diagnose-failures). +8. Commit the changes and push the relevant `challenge*` branch to the repository and wait for the build to complete. View the [logs of your build](https://docs.github.com/en/free-pro-team@latest/actions/managing-workflow-runs/using-workflow-run-logs#viewing-logs-to-diagnose-failures). -9. If challenge config contains errors then a `issue` will be opened automatically in the repository with the errors otherwise the challenge will be created on EvalAI. +9. If the challenge config contains errors, a GitHub **Issue** will be opened automatically in the repository with the error details; otherwise the challenge will be created on EvalAI. 10. Go to [Hosted Challenges](https://eval.ai/web/hosted-challenges) to view your challenge. The challenge will be publicly available once EvalAI admin approves the challenge. -11. To update the challenge on EvalAI, make changes in the repository and push on `challenge` branch and wait for the build to complete. +11. To update the challenge on EvalAI, make changes in the repository and push to the corresponding `challenge*` branch and wait for the build to complete. ## Add custom dependencies for evaluation (Optional) To add custom dependency packages in the evaluation script, refer to [this guide](./evaluation_script/dependency-installation.md). diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index b0c88bb98..9504f12ec 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -3,6 +3,9 @@ import os import requests import sys +import argparse +import re +import config from config import * from utils import ( @@ -32,6 +35,20 @@ CHALLENGE_HOST_TEAM_PK = None EVALAI_HOST_URL = None +parser = argparse.ArgumentParser( + description="Validate or create/update challenge on EvalAI" +) +parser.add_argument("branch_name", nargs="?", default=None, help="Name of the git branch whose configuration is being processed") + +args = parser.parse_args() + +# Determine effective branch name (default to "challenge" if none provided) +branch_name = args.branch_name if args.branch_name else "challenge" + +# Enforce branch naming convention +if not re.match(r"^challenge(-.*)?$", branch_name): + print("Error: Branch name must start with 'challenge' (e.g., 'challenge', 'challenge-2024').") + sys.exit(1) if __name__ == "__main__": @@ -43,6 +60,10 @@ else: sys.exit(1) + + # Update the global config path for zip file creation + config.CHALLENGE_CONFIG_FILE_PATH = "challenge_config.yaml" + # Fetching the url if VALIDATION_STEP == "True": url = "{}{}".format( @@ -62,7 +83,11 @@ zip_file = open(CHALLENGE_ZIP_FILE_PATH, "rb") file = {"zip_configuration": zip_file} + # Add the branch name (if provided) so that EvalAI can distinguish between multiple + # versions of the challenge present in the same repository. data = {"GITHUB_REPOSITORY": GITHUB_REPOSITORY} + if branch_name: + data["BRANCH_NAME"] = branch_name try: response = requests.post(url, data=data, headers=headers, files=file)