diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml
index 57f6d888d..fc4918302 100644
--- a/.github/workflows/validate-and-process.yml
+++ b/.github/workflows/validate-and-process.yml
@@ -1,15 +1,56 @@
name: Validate and Process EvalAI Challenge
on:
- push:
+ push: {}
+ pull_request:
+ types: [opened, synchronize, reopened, edited]
branches:
- - challenge
+ - 'challenge'
permissions:
contents: read
issues: write
jobs:
+ detect-and-validate-challenge-branch:
+ runs-on: ubuntu-latest
+ outputs:
+ is_challenge_branch: ${{ steps.branch-check.outputs.is_challenge_branch }}
+ challenge_branch: ${{ steps.branch-check.outputs.challenge_branch }}
+ steps:
+ - name: Detect and validate challenge branch
+ id: branch-check
+ run: |
+ # Get the actual branch name
+ if [ "${{ github.event_name }}" == "pull_request" ]; then
+ BRANCH="${{ github.head_ref }}"
+ else
+ BRANCH="${{ github.ref_name }}"
+ fi
+
+ echo "đ Analyzing branch: $BRANCH"
+ echo "challenge_branch=$BRANCH" >> $GITHUB_OUTPUT
+
+ # Check if this is a challenge branch
+ if [[ "$BRANCH" =~ ^challenge(-.*)?$ ]]; then
+ echo "is_challenge_branch=true" >> $GITHUB_OUTPUT
+ echo "â
Valid challenge branch detected: $BRANCH"
+
+ # Extract branch suffix for versioning
+ if [[ "$BRANCH" =~ ^challenge-(.+)$ ]]; then
+ SUFFIX="${BASH_REMATCH[1]}"
+ echo "branch_suffix=$SUFFIX" >> $GITHUB_OUTPUT
+ echo "đ Branch suffix: $SUFFIX"
+ else
+ echo "branch_suffix=main" >> $GITHUB_OUTPUT
+ echo "đ Main challenge branch (no suffix)"
+ fi
+ else
+ echo "is_challenge_branch=false" >> $GITHUB_OUTPUT
+ echo "âšī¸ Not a challenge branch: $BRANCH"
+ echo "âī¸ Challenge processing will be skipped"
+ fi
+
validate-host-config:
runs-on: ubuntu-latest
outputs:
@@ -74,7 +115,7 @@ jobs:
- name: Checkout challenge branch
uses: actions/checkout@v3
with:
- ref: challenge
+ ref: ${{ github.ref_name }}
- name: Set up Python
uses: actions/setup-python@v4
@@ -88,7 +129,7 @@ jobs:
- name: Validate challenge
run: |
- python3 github/challenge_processing_script.py
+ python3 github/challenge_processing_script.py ${GITHUB_REF#refs/heads/}
env:
IS_VALIDATION: 'True'
GITHUB_CONTEXT: ${{ toJson(github) }}
@@ -96,8 +137,8 @@ jobs:
- name: Create or update challenge
run: |
- python3 github/challenge_processing_script.py
- if: ${{ success() }}
+ python3 github/challenge_processing_script.py ${GITHUB_REF#refs/heads/}
+ if: ${{ github.event_name == 'push' && success() }}
env:
IS_VALIDATION: 'False'
GITHUB_CONTEXT: ${{ toJson(github) }}
diff --git a/README.md b/README.md
index 458cc8fd9..14d799f3d 100644
--- a/README.md
+++ b/README.md
@@ -48,18 +48,22 @@ If you are looking for a simple challenge configuration that you can replicate t
5. Create a branch with name `challenge` in the forked repository from the `master` branch.
Note: Only changes in `challenge` branch will be synchronized with challenge on EvalAI.
+You may maintain multiple versions by using additional branches whose names start with `challenge-` (e.g. `challenge-2024`, `challenge-v2`).
+Note: The CI pipeline only runs for branches that match the pattern `challenge` or `challenge-*`. Any other branch name will be ignored.
+
+If you trigger the processing script manually and do **not** pass a branch argument, it will assume the branch name is `challenge` by default.
6. Add `evalai_user_auth_token` and `host_team_pk` in `github/host_config.json`.
7. Read [EvalAI challenge creation documentation](https://evalai.readthedocs.io/en/latest/configuration.html) to know more about how you want to structure your challenge. Once you are ready, start making changes in the yaml file, HTML templates, evaluation script according to your need.
-8. Commit the changes and push the `challenge` branch in the repository and wait for the build to complete. View the [logs of your build](https://docs.github.com/en/free-pro-team@latest/actions/managing-workflow-runs/using-workflow-run-logs#viewing-logs-to-diagnose-failures).
+8. Commit the changes and push the relevant `challenge*` branch to the repository and wait for the build to complete. View the [logs of your build](https://docs.github.com/en/free-pro-team@latest/actions/managing-workflow-runs/using-workflow-run-logs#viewing-logs-to-diagnose-failures).
-9. If challenge config contains errors then a `issue` will be opened automatically in the repository with the errors otherwise the challenge will be created on EvalAI.
+9. If the challenge config contains errors, a GitHub **Issue** will be opened automatically in the repository with the error details; otherwise the challenge will be created on EvalAI.
10. Go to [Hosted Challenges](https://eval.ai/web/hosted-challenges) to view your challenge. The challenge will be publicly available once EvalAI admin approves the challenge.
-11. To update the challenge on EvalAI, make changes in the repository and push on `challenge` branch and wait for the build to complete.
+11. To update the challenge on EvalAI, make changes in the repository and push to the corresponding `challenge*` branch and wait for the build to complete.
## Add custom dependencies for evaluation (Optional)
To add custom dependency packages in the evaluation script, refer to [this guide](./evaluation_script/dependency-installation.md).
diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py
index b0c88bb98..9504f12ec 100644
--- a/github/challenge_processing_script.py
+++ b/github/challenge_processing_script.py
@@ -3,6 +3,9 @@
import os
import requests
import sys
+import argparse
+import re
+import config
from config import *
from utils import (
@@ -32,6 +35,20 @@
CHALLENGE_HOST_TEAM_PK = None
EVALAI_HOST_URL = None
+parser = argparse.ArgumentParser(
+ description="Validate or create/update challenge on EvalAI"
+)
+parser.add_argument("branch_name", nargs="?", default=None, help="Name of the git branch whose configuration is being processed")
+
+args = parser.parse_args()
+
+# Determine effective branch name (default to "challenge" if none provided)
+branch_name = args.branch_name if args.branch_name else "challenge"
+
+# Enforce branch naming convention
+if not re.match(r"^challenge(-.*)?$", branch_name):
+ print("Error: Branch name must start with 'challenge' (e.g., 'challenge', 'challenge-2024').")
+ sys.exit(1)
if __name__ == "__main__":
@@ -43,6 +60,10 @@
else:
sys.exit(1)
+
+ # Update the global config path for zip file creation
+ config.CHALLENGE_CONFIG_FILE_PATH = "challenge_config.yaml"
+
# Fetching the url
if VALIDATION_STEP == "True":
url = "{}{}".format(
@@ -62,7 +83,11 @@
zip_file = open(CHALLENGE_ZIP_FILE_PATH, "rb")
file = {"zip_configuration": zip_file}
+ # Add the branch name (if provided) so that EvalAI can distinguish between multiple
+ # versions of the challenge present in the same repository.
data = {"GITHUB_REPOSITORY": GITHUB_REPOSITORY}
+ if branch_name:
+ data["BRANCH_NAME"] = branch_name
try:
response = requests.post(url, data=data, headers=headers, files=file)