From c34489cf54c6859db374dff4fb32e48f558cd8ee Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Sun, 22 Jun 2025 20:56:05 +0530 Subject: [PATCH 1/7] Modify challenge processing --- .github/workflows/validate-and-process.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 57f6d888d..f1d6c0ba4 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -1,9 +1,9 @@ name: Validate and Process EvalAI Challenge on: - push: - branches: - - challenge + push: {} + pull_request: + types: [opened, synchronize, reopened, edited] permissions: contents: read @@ -74,7 +74,7 @@ jobs: - name: Checkout challenge branch uses: actions/checkout@v3 with: - ref: challenge + ref: ${{ github.ref_name }} - name: Set up Python uses: actions/setup-python@v4 @@ -88,7 +88,7 @@ jobs: - name: Validate challenge run: | - python3 github/challenge_processing_script.py + python3 github/challenge_processing_script.py ${GITHUB_REF#refs/heads/} env: IS_VALIDATION: 'True' GITHUB_CONTEXT: ${{ toJson(github) }} @@ -96,8 +96,8 @@ jobs: - name: Create or update challenge run: | - python3 github/challenge_processing_script.py - if: ${{ success() }} + python3 github/challenge_processing_script.py ${GITHUB_REF#refs/heads/} + if: ${{ github.event_name == 'push' && success() }} env: IS_VALIDATION: 'False' GITHUB_CONTEXT: ${{ toJson(github) }} From 32a0be74fa5874b74e46b2eaef72d3d4f741bb7b Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Mon, 23 Jun 2025 16:17:00 +0530 Subject: [PATCH 2/7] Modify script to handle arguments for diff challenge versions --- github/challenge_processing_script.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index b0c88bb98..e62ee599b 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -3,6 +3,7 @@ import os import requests import sys +import argparse from config import * from utils import ( @@ -32,6 +33,15 @@ CHALLENGE_HOST_TEAM_PK = None EVALAI_HOST_URL = None +parser = argparse.ArgumentParser( + description="Validate or create/update challenge on EvalAI" +) +parser.add_argument("branch_name", nargs="?", default=None, help="Name of the git branch whose configuration is being processed") + +args = parser.parse_args() + + + if __name__ == "__main__": @@ -62,7 +72,11 @@ zip_file = open(CHALLENGE_ZIP_FILE_PATH, "rb") file = {"zip_configuration": zip_file} + # Add the branch name (if provided) so that EvalAI can distinguish between multiple + # versions of the challenge present in the same repository. data = {"GITHUB_REPOSITORY": GITHUB_REPOSITORY} + if args.branch_name: + data["BRANCH_NAME"] = args.branch_name try: response = requests.post(url, data=data, headers=headers, files=file) From 013e7602f61d390196b76d5754863921129fc0cd Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Mon, 23 Jun 2025 18:28:05 +0530 Subject: [PATCH 3/7] Update scripts --- .github/workflows/validate-and-process.yml | 42 ++++++++++++++++++++++ github/challenge_processing_script.py | 32 +++++++++++++++-- 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index f1d6c0ba4..10b19ce36 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -4,12 +4,54 @@ on: push: {} pull_request: types: [opened, synchronize, reopened, edited] + branches: + - 'challenge' permissions: contents: read issues: write jobs: + detect-and-validate-challenge-branch: + runs-on: ubuntu-latest + outputs: + is_challenge_branch: ${{ steps.branch-check.outputs.is_challenge_branch }} + challenge_branch: ${{ steps.branch-check.outputs.challenge_branch }} + branch_suffix: ${{ steps.branch-check.outputs.branch_suffix }} + steps: + - name: Detect and validate challenge branch + id: branch-check + run: | + # Get the actual branch name + if [ "${{ github.event_name }}" == "pull_request" ]; then + BRANCH="${{ github.head_ref }}" + else + BRANCH="${{ github.ref_name }}" + fi + + echo "🔍 Analyzing branch: $BRANCH" + echo "challenge_branch=$BRANCH" >> $GITHUB_OUTPUT + + # Check if this is a challenge branch + if [[ "$BRANCH" =~ ^challenge(-.*)?$ ]]; then + echo "is_challenge_branch=true" >> $GITHUB_OUTPUT + echo "✅ Valid challenge branch detected: $BRANCH" + + # Extract branch suffix for versioning + if [[ "$BRANCH" =~ ^challenge-(.+)$ ]]; then + SUFFIX="${BASH_REMATCH[1]}" + echo "branch_suffix=$SUFFIX" >> $GITHUB_OUTPUT + echo "📋 Branch suffix: $SUFFIX" + else + echo "branch_suffix=main" >> $GITHUB_OUTPUT + echo "📋 Main challenge branch (no suffix)" + fi + else + echo "is_challenge_branch=false" >> $GITHUB_OUTPUT + echo "ℹī¸ Not a challenge branch: $BRANCH" + echo "⏭ī¸ Challenge processing will be skipped" + fi + validate-host-config: runs-on: ubuntu-latest outputs: diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index e62ee599b..c068c20a4 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -4,6 +4,7 @@ import requests import sys import argparse +import re from config import * from utils import ( @@ -40,8 +41,28 @@ args = parser.parse_args() - - +# Enforce branch naming convention +if args.branch_name and not re.match(r"^challenge(-.*)?$", args.branch_name): + print("Error: Branch name must start with 'challenge' (e.g., 'challenge', 'challenge-2024').") + sys.exit(1) +def get_challenge_config_path(branch_name): + """ + Get the appropriate challenge config file path based on branch name + """ + if not branch_name or branch_name == "challenge": + return "challenge_config.yaml" + + # For branches like challenge-2024, challenge-v2, etc. + if branch_name.startswith("challenge-"): + suffix = branch_name.replace("challenge-", "") + branch_config = f"challenge_config_{suffix}.yaml" + + # Check if branch-specific config exists + if os.path.exists(branch_config): + return branch_config + + # Fallback to default config + return "challenge_config.yaml" if __name__ == "__main__": @@ -53,6 +74,13 @@ else: sys.exit(1) + # Get the appropriate challenge config based on branch + challenge_config_path = get_challenge_config_path(args.branch_name) + + # Update the global config path for zip file creation + import config + config.CHALLENGE_CONFIG_FILE_PATH = challenge_config_path + # Fetching the url if VALIDATION_STEP == "True": url = "{}{}".format( From 5db5d75f1b0382018fabb3935bc2d8d85fea5529 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Mon, 23 Jun 2025 19:08:37 +0530 Subject: [PATCH 4/7] update workflow --- .github/workflows/validate-and-process.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 10b19ce36..fc4918302 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -17,7 +17,6 @@ jobs: outputs: is_challenge_branch: ${{ steps.branch-check.outputs.is_challenge_branch }} challenge_branch: ${{ steps.branch-check.outputs.challenge_branch }} - branch_suffix: ${{ steps.branch-check.outputs.branch_suffix }} steps: - name: Detect and validate challenge branch id: branch-check From 07cfa1e960897825ee0cc0415a689c039dcc7dab Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Mon, 23 Jun 2025 19:19:49 +0530 Subject: [PATCH 5/7] update processing script --- github/challenge_processing_script.py | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index c068c20a4..51e2d9c98 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -5,6 +5,7 @@ import sys import argparse import re +import config from config import * from utils import ( @@ -45,24 +46,6 @@ if args.branch_name and not re.match(r"^challenge(-.*)?$", args.branch_name): print("Error: Branch name must start with 'challenge' (e.g., 'challenge', 'challenge-2024').") sys.exit(1) -def get_challenge_config_path(branch_name): - """ - Get the appropriate challenge config file path based on branch name - """ - if not branch_name or branch_name == "challenge": - return "challenge_config.yaml" - - # For branches like challenge-2024, challenge-v2, etc. - if branch_name.startswith("challenge-"): - suffix = branch_name.replace("challenge-", "") - branch_config = f"challenge_config_{suffix}.yaml" - - # Check if branch-specific config exists - if os.path.exists(branch_config): - return branch_config - - # Fallback to default config - return "challenge_config.yaml" if __name__ == "__main__": @@ -74,12 +57,9 @@ def get_challenge_config_path(branch_name): else: sys.exit(1) - # Get the appropriate challenge config based on branch - challenge_config_path = get_challenge_config_path(args.branch_name) # Update the global config path for zip file creation - import config - config.CHALLENGE_CONFIG_FILE_PATH = challenge_config_path + config.CHALLENGE_CONFIG_FILE_PATH = "challenge_config.yaml" # Fetching the url if VALIDATION_STEP == "True": From 01374411deafc0a95aa392d426b47cc412a986e6 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Mon, 23 Jun 2025 19:35:43 +0530 Subject: [PATCH 6/7] add lines to default to challenge --- github/challenge_processing_script.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index 51e2d9c98..9504f12ec 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -42,8 +42,11 @@ args = parser.parse_args() +# Determine effective branch name (default to "challenge" if none provided) +branch_name = args.branch_name if args.branch_name else "challenge" + # Enforce branch naming convention -if args.branch_name and not re.match(r"^challenge(-.*)?$", args.branch_name): +if not re.match(r"^challenge(-.*)?$", branch_name): print("Error: Branch name must start with 'challenge' (e.g., 'challenge', 'challenge-2024').") sys.exit(1) @@ -83,8 +86,8 @@ # Add the branch name (if provided) so that EvalAI can distinguish between multiple # versions of the challenge present in the same repository. data = {"GITHUB_REPOSITORY": GITHUB_REPOSITORY} - if args.branch_name: - data["BRANCH_NAME"] = args.branch_name + if branch_name: + data["BRANCH_NAME"] = branch_name try: response = requests.post(url, data=data, headers=headers, files=file) From fa2346691a89057299208d78a8edd2ae4c7a7ec9 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Mon, 23 Jun 2025 19:40:28 +0530 Subject: [PATCH 7/7] Update the README --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 458cc8fd9..14d799f3d 100644 --- a/README.md +++ b/README.md @@ -48,18 +48,22 @@ If you are looking for a simple challenge configuration that you can replicate t 5. Create a branch with name `challenge` in the forked repository from the `master` branch. Note: Only changes in `challenge` branch will be synchronized with challenge on EvalAI. +You may maintain multiple versions by using additional branches whose names start with `challenge-` (e.g. `challenge-2024`, `challenge-v2`). +Note: The CI pipeline only runs for branches that match the pattern `challenge` or `challenge-*`. Any other branch name will be ignored. + +If you trigger the processing script manually and do **not** pass a branch argument, it will assume the branch name is `challenge` by default. 6. Add `evalai_user_auth_token` and `host_team_pk` in `github/host_config.json`. 7. Read [EvalAI challenge creation documentation](https://evalai.readthedocs.io/en/latest/configuration.html) to know more about how you want to structure your challenge. Once you are ready, start making changes in the yaml file, HTML templates, evaluation script according to your need. -8. Commit the changes and push the `challenge` branch in the repository and wait for the build to complete. View the [logs of your build](https://docs.github.com/en/free-pro-team@latest/actions/managing-workflow-runs/using-workflow-run-logs#viewing-logs-to-diagnose-failures). +8. Commit the changes and push the relevant `challenge*` branch to the repository and wait for the build to complete. View the [logs of your build](https://docs.github.com/en/free-pro-team@latest/actions/managing-workflow-runs/using-workflow-run-logs#viewing-logs-to-diagnose-failures). -9. If challenge config contains errors then a `issue` will be opened automatically in the repository with the errors otherwise the challenge will be created on EvalAI. +9. If the challenge config contains errors, a GitHub **Issue** will be opened automatically in the repository with the error details; otherwise the challenge will be created on EvalAI. 10. Go to [Hosted Challenges](https://eval.ai/web/hosted-challenges) to view your challenge. The challenge will be publicly available once EvalAI admin approves the challenge. -11. To update the challenge on EvalAI, make changes in the repository and push on `challenge` branch and wait for the build to complete. +11. To update the challenge on EvalAI, make changes in the repository and push to the corresponding `challenge*` branch and wait for the build to complete. ## Add custom dependencies for evaluation (Optional) To add custom dependency packages in the evaluation script, refer to [this guide](./evaluation_script/dependency-installation.md).