[Closed PR for Fix #119] Implement feature to allow multiple versions of Challenges via git branches

.github/workflows/validate-and-process.yml

@@ -1,15 +1,56 @@
 name: Validate and Process EvalAI Challenge
 
 on:
-  push:
+  push: {}
+  pull_request:
+    types: [opened, synchronize, reopened, edited]
     branches:
-      - challenge
+      - 'challenge'
 
 permissions:
   contents: read
   issues: write
 
 jobs:
+  detect-and-validate-challenge-branch:
+    runs-on: ubuntu-latest
+    outputs:
+      is_challenge_branch: ${{ steps.branch-check.outputs.is_challenge_branch }}
+      challenge_branch: ${{ steps.branch-check.outputs.challenge_branch }}
+    steps:
+      - name: Detect and validate challenge branch
+        id: branch-check
+        run: |
+          # Get the actual branch name
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            BRANCH="${{ github.head_ref }}"
+          else
+            BRANCH="${{ github.ref_name }}"
+          fi
+
+          echo "🔍 Analyzing branch: $BRANCH"
+          echo "challenge_branch=$BRANCH" >> $GITHUB_OUTPUT
+
+          # Check if this is a challenge branch
+          if [[ "$BRANCH" =~ ^challenge(-.*)?$ ]]; then
+            echo "is_challenge_branch=true" >> $GITHUB_OUTPUT
+            echo "✅ Valid challenge branch detected: $BRANCH"
+
+            # Extract branch suffix for versioning
+            if [[ "$BRANCH" =~ ^challenge-(.+)$ ]]; then
+              SUFFIX="${BASH_REMATCH[1]}"
+              echo "branch_suffix=$SUFFIX" >> $GITHUB_OUTPUT
+              echo "📋 Branch suffix: $SUFFIX"
+            else
+              echo "branch_suffix=main" >> $GITHUB_OUTPUT
+              echo "📋 Main challenge branch (no suffix)"
+            fi
+          else
+            echo "is_challenge_branch=false" >> $GITHUB_OUTPUT
+            echo "ℹ️ Not a challenge branch: $BRANCH"
+            echo "⏭️ Challenge processing will be skipped"
+          fi
+
   validate-host-config:
     runs-on: ubuntu-latest
     outputs:
@@ -74,7 +115,7 @@ jobs:
       - name: Checkout challenge branch
         uses: actions/checkout@v3
         with:
-          ref: challenge
+          ref: ${{ github.ref_name }}
 
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -88,16 +129,16 @@ jobs:
 
       - name: Validate challenge
         run: |
-          python3 github/challenge_processing_script.py
+          python3 github/challenge_processing_script.py ${GITHUB_REF#refs/heads/}
         env:
           IS_VALIDATION: 'True'
           GITHUB_CONTEXT: ${{ toJson(github) }}
           GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }}
 
       - name: Create or update challenge
         run: |
-          python3 github/challenge_processing_script.py
-        if: ${{ success() }}
+          python3 github/challenge_processing_script.py ${GITHUB_REF#refs/heads/}
+        if: ${{ github.event_name == 'push' && success() }}
         env:
           IS_VALIDATION: 'False'
           GITHUB_CONTEXT: ${{ toJson(github) }}

README.md

@@ -48,18 +48,22 @@ If you are looking for a simple challenge configuration that you can replicate t
 
 5. Create a branch with name `challenge` in the forked repository from the `master` branch.
 <span style="color:purple">Note: Only changes in `challenge` branch will be synchronized with challenge on EvalAI.</span>
+You may maintain multiple versions by using additional branches whose names start with `challenge-` (e.g. `challenge-2024`, `challenge-v2`).
+<span style="color:purple">Note: The CI pipeline only runs for branches that match the pattern `challenge` or `challenge-*`. Any other branch name will be ignored.</span>
+
+If you trigger the processing script manually and do **not** pass a branch argument, it will assume the branch name is `challenge` by default.
 
 6. Add `evalai_user_auth_token` and `host_team_pk` in `github/host_config.json`.
 
 7. Read [EvalAI challenge creation documentation](https://evalai.readthedocs.io/en/latest/configuration.html) to know more about how you want to structure your challenge. Once you are ready, start making changes in the yaml file, HTML templates, evaluation script according to your need.
 
-8. Commit the changes and push the `challenge` branch in the repository and wait for the build to complete. View the [logs of your build](https://docs.github.com/en/free-pro-team@latest/actions/managing-workflow-runs/using-workflow-run-logs#viewing-logs-to-diagnose-failures).
+8. Commit the changes and push the relevant `challenge*` branch to the repository and wait for the build to complete. View the [logs of your build](https://docs.github.com/en/free-pro-team@latest/actions/managing-workflow-runs/using-workflow-run-logs#viewing-logs-to-diagnose-failures).
 
-9. If challenge config contains errors then a `issue` will be opened automatically in the repository with the errors otherwise the challenge will be created on EvalAI.
+9. If the challenge config contains errors, a GitHub **Issue** will be opened automatically in the repository with the error details; otherwise the challenge will be created on EvalAI.
 
 10. Go to [Hosted Challenges](https://eval.ai/web/hosted-challenges) to view your challenge. The challenge will be publicly available once EvalAI admin approves the challenge.
 
-11. To update the challenge on EvalAI, make changes in the repository and push on `challenge` branch and wait for the build to complete.
+11. To update the challenge on EvalAI, make changes in the repository and push to the corresponding `challenge*` branch and wait for the build to complete.
 
 ## Add custom dependencies for evaluation (Optional)
 To add custom dependency packages in the evaluation script, refer to [this guide](./evaluation_script/dependency-installation.md).

github/challenge_processing_script.py

@@ -3,6 +3,9 @@
 import os
 import requests
 import sys
+import argparse
+import re
+import config
 
 from config import *
 from utils import (
@@ -32,6 +35,20 @@
 CHALLENGE_HOST_TEAM_PK = None
 EVALAI_HOST_URL = None
 
+parser = argparse.ArgumentParser(
+    description="Validate or create/update challenge on EvalAI"
+)
+parser.add_argument("branch_name", nargs="?", default=None, help="Name of the git branch whose configuration is being processed")
+
+args = parser.parse_args()
+
+# Determine effective branch name (default to "challenge" if none provided)
+branch_name = args.branch_name if args.branch_name else "challenge"
+
+# Enforce branch naming convention
+if not re.match(r"^challenge(-.*)?$", branch_name):
+    print("Error: Branch name must start with 'challenge' (e.g., 'challenge', 'challenge-2024').")
+    sys.exit(1)
 
 if __name__ == "__main__":
 
@@ -43,6 +60,10 @@
     else:
         sys.exit(1)
 
+
+    # Update the global config path for zip file creation
+    config.CHALLENGE_CONFIG_FILE_PATH = "challenge_config.yaml"
+
     # Fetching the url
     if VALIDATION_STEP == "True":
         url = "{}{}".format(
@@ -62,7 +83,11 @@
     zip_file = open(CHALLENGE_ZIP_FILE_PATH, "rb")
     file = {"zip_configuration": zip_file}
 
+    # Add the branch name (if provided) so that EvalAI can distinguish between multiple
+    # versions of the challenge present in the same repository.
     data = {"GITHUB_REPOSITORY": GITHUB_REPOSITORY}
+    if branch_name:
+        data["BRANCH_NAME"] = branch_name
 
     try:
         response = requests.post(url, data=data, headers=headers, files=file)