From c34489cf54c6859db374dff4fb32e48f558cd8ee Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Sun, 22 Jun 2025 20:56:05 +0530 Subject: [PATCH 01/13] Modify challenge processing --- .github/workflows/validate-and-process.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 57f6d888d..f1d6c0ba4 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -1,9 +1,9 @@ name: Validate and Process EvalAI Challenge on: - push: - branches: - - challenge + push: {} + pull_request: + types: [opened, synchronize, reopened, edited] permissions: contents: read @@ -74,7 +74,7 @@ jobs: - name: Checkout challenge branch uses: actions/checkout@v3 with: - ref: challenge + ref: ${{ github.ref_name }} - name: Set up Python uses: actions/setup-python@v4 @@ -88,7 +88,7 @@ jobs: - name: Validate challenge run: | - python3 github/challenge_processing_script.py + python3 github/challenge_processing_script.py ${GITHUB_REF#refs/heads/} env: IS_VALIDATION: 'True' GITHUB_CONTEXT: ${{ toJson(github) }} @@ -96,8 +96,8 @@ jobs: - name: Create or update challenge run: | - python3 github/challenge_processing_script.py - if: ${{ success() }} + python3 github/challenge_processing_script.py ${GITHUB_REF#refs/heads/} + if: ${{ github.event_name == 'push' && success() }} env: IS_VALIDATION: 'False' GITHUB_CONTEXT: ${{ toJson(github) }} From 32a0be74fa5874b74e46b2eaef72d3d4f741bb7b Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Mon, 23 Jun 2025 16:17:00 +0530 Subject: [PATCH 02/13] Modify script to handle arguments for diff challenge versions --- github/challenge_processing_script.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index b0c88bb98..e62ee599b 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -3,6 +3,7 @@ import os import requests import sys +import argparse from config import * from utils import ( @@ -32,6 +33,15 @@ CHALLENGE_HOST_TEAM_PK = None EVALAI_HOST_URL = None +parser = argparse.ArgumentParser( + description="Validate or create/update challenge on EvalAI" +) +parser.add_argument("branch_name", nargs="?", default=None, help="Name of the git branch whose configuration is being processed") + +args = parser.parse_args() + + + if __name__ == "__main__": @@ -62,7 +72,11 @@ zip_file = open(CHALLENGE_ZIP_FILE_PATH, "rb") file = {"zip_configuration": zip_file} + # Add the branch name (if provided) so that EvalAI can distinguish between multiple + # versions of the challenge present in the same repository. data = {"GITHUB_REPOSITORY": GITHUB_REPOSITORY} + if args.branch_name: + data["BRANCH_NAME"] = args.branch_name try: response = requests.post(url, data=data, headers=headers, files=file) From 013e7602f61d390196b76d5754863921129fc0cd Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Mon, 23 Jun 2025 18:28:05 +0530 Subject: [PATCH 03/13] Update scripts --- .github/workflows/validate-and-process.yml | 42 ++++++++++++++++++++++ github/challenge_processing_script.py | 32 +++++++++++++++-- 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index f1d6c0ba4..10b19ce36 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -4,12 +4,54 @@ on: push: {} pull_request: types: [opened, synchronize, reopened, edited] + branches: + - 'challenge' permissions: contents: read issues: write jobs: + detect-and-validate-challenge-branch: + runs-on: ubuntu-latest + outputs: + is_challenge_branch: ${{ steps.branch-check.outputs.is_challenge_branch }} + challenge_branch: ${{ steps.branch-check.outputs.challenge_branch }} + branch_suffix: ${{ steps.branch-check.outputs.branch_suffix }} + steps: + - name: Detect and validate challenge branch + id: branch-check + run: | + # Get the actual branch name + if [ "${{ github.event_name }}" == "pull_request" ]; then + BRANCH="${{ github.head_ref }}" + else + BRANCH="${{ github.ref_name }}" + fi + + echo "๐Ÿ” Analyzing branch: $BRANCH" + echo "challenge_branch=$BRANCH" >> $GITHUB_OUTPUT + + # Check if this is a challenge branch + if [[ "$BRANCH" =~ ^challenge(-.*)?$ ]]; then + echo "is_challenge_branch=true" >> $GITHUB_OUTPUT + echo "โœ… Valid challenge branch detected: $BRANCH" + + # Extract branch suffix for versioning + if [[ "$BRANCH" =~ ^challenge-(.+)$ ]]; then + SUFFIX="${BASH_REMATCH[1]}" + echo "branch_suffix=$SUFFIX" >> $GITHUB_OUTPUT + echo "๐Ÿ“‹ Branch suffix: $SUFFIX" + else + echo "branch_suffix=main" >> $GITHUB_OUTPUT + echo "๐Ÿ“‹ Main challenge branch (no suffix)" + fi + else + echo "is_challenge_branch=false" >> $GITHUB_OUTPUT + echo "โ„น๏ธ Not a challenge branch: $BRANCH" + echo "โญ๏ธ Challenge processing will be skipped" + fi + validate-host-config: runs-on: ubuntu-latest outputs: diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index e62ee599b..c068c20a4 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -4,6 +4,7 @@ import requests import sys import argparse +import re from config import * from utils import ( @@ -40,8 +41,28 @@ args = parser.parse_args() - - +# Enforce branch naming convention +if args.branch_name and not re.match(r"^challenge(-.*)?$", args.branch_name): + print("Error: Branch name must start with 'challenge' (e.g., 'challenge', 'challenge-2024').") + sys.exit(1) +def get_challenge_config_path(branch_name): + """ + Get the appropriate challenge config file path based on branch name + """ + if not branch_name or branch_name == "challenge": + return "challenge_config.yaml" + + # For branches like challenge-2024, challenge-v2, etc. + if branch_name.startswith("challenge-"): + suffix = branch_name.replace("challenge-", "") + branch_config = f"challenge_config_{suffix}.yaml" + + # Check if branch-specific config exists + if os.path.exists(branch_config): + return branch_config + + # Fallback to default config + return "challenge_config.yaml" if __name__ == "__main__": @@ -53,6 +74,13 @@ else: sys.exit(1) + # Get the appropriate challenge config based on branch + challenge_config_path = get_challenge_config_path(args.branch_name) + + # Update the global config path for zip file creation + import config + config.CHALLENGE_CONFIG_FILE_PATH = challenge_config_path + # Fetching the url if VALIDATION_STEP == "True": url = "{}{}".format( From 5db5d75f1b0382018fabb3935bc2d8d85fea5529 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Mon, 23 Jun 2025 19:08:37 +0530 Subject: [PATCH 04/13] update workflow --- .github/workflows/validate-and-process.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 10b19ce36..fc4918302 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -17,7 +17,6 @@ jobs: outputs: is_challenge_branch: ${{ steps.branch-check.outputs.is_challenge_branch }} challenge_branch: ${{ steps.branch-check.outputs.challenge_branch }} - branch_suffix: ${{ steps.branch-check.outputs.branch_suffix }} steps: - name: Detect and validate challenge branch id: branch-check From 07cfa1e960897825ee0cc0415a689c039dcc7dab Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Mon, 23 Jun 2025 19:19:49 +0530 Subject: [PATCH 05/13] update processing script --- github/challenge_processing_script.py | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index c068c20a4..51e2d9c98 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -5,6 +5,7 @@ import sys import argparse import re +import config from config import * from utils import ( @@ -45,24 +46,6 @@ if args.branch_name and not re.match(r"^challenge(-.*)?$", args.branch_name): print("Error: Branch name must start with 'challenge' (e.g., 'challenge', 'challenge-2024').") sys.exit(1) -def get_challenge_config_path(branch_name): - """ - Get the appropriate challenge config file path based on branch name - """ - if not branch_name or branch_name == "challenge": - return "challenge_config.yaml" - - # For branches like challenge-2024, challenge-v2, etc. - if branch_name.startswith("challenge-"): - suffix = branch_name.replace("challenge-", "") - branch_config = f"challenge_config_{suffix}.yaml" - - # Check if branch-specific config exists - if os.path.exists(branch_config): - return branch_config - - # Fallback to default config - return "challenge_config.yaml" if __name__ == "__main__": @@ -74,12 +57,9 @@ def get_challenge_config_path(branch_name): else: sys.exit(1) - # Get the appropriate challenge config based on branch - challenge_config_path = get_challenge_config_path(args.branch_name) # Update the global config path for zip file creation - import config - config.CHALLENGE_CONFIG_FILE_PATH = challenge_config_path + config.CHALLENGE_CONFIG_FILE_PATH = "challenge_config.yaml" # Fetching the url if VALIDATION_STEP == "True": From 01374411deafc0a95aa392d426b47cc412a986e6 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Mon, 23 Jun 2025 19:35:43 +0530 Subject: [PATCH 06/13] add lines to default to challenge --- github/challenge_processing_script.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index 51e2d9c98..9504f12ec 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -42,8 +42,11 @@ args = parser.parse_args() +# Determine effective branch name (default to "challenge" if none provided) +branch_name = args.branch_name if args.branch_name else "challenge" + # Enforce branch naming convention -if args.branch_name and not re.match(r"^challenge(-.*)?$", args.branch_name): +if not re.match(r"^challenge(-.*)?$", branch_name): print("Error: Branch name must start with 'challenge' (e.g., 'challenge', 'challenge-2024').") sys.exit(1) @@ -83,8 +86,8 @@ # Add the branch name (if provided) so that EvalAI can distinguish between multiple # versions of the challenge present in the same repository. data = {"GITHUB_REPOSITORY": GITHUB_REPOSITORY} - if args.branch_name: - data["BRANCH_NAME"] = args.branch_name + if branch_name: + data["BRANCH_NAME"] = branch_name try: response = requests.post(url, data=data, headers=headers, files=file) From fa2346691a89057299208d78a8edd2ae4c7a7ec9 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Mon, 23 Jun 2025 19:40:28 +0530 Subject: [PATCH 07/13] Update the README --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 458cc8fd9..14d799f3d 100644 --- a/README.md +++ b/README.md @@ -48,18 +48,22 @@ If you are looking for a simple challenge configuration that you can replicate t 5. Create a branch with name `challenge` in the forked repository from the `master` branch. Note: Only changes in `challenge` branch will be synchronized with challenge on EvalAI. +You may maintain multiple versions by using additional branches whose names start with `challenge-` (e.g. `challenge-2024`, `challenge-v2`). +Note: The CI pipeline only runs for branches that match the pattern `challenge` or `challenge-*`. Any other branch name will be ignored. + +If you trigger the processing script manually and do **not** pass a branch argument, it will assume the branch name is `challenge` by default. 6. Add `evalai_user_auth_token` and `host_team_pk` in `github/host_config.json`. 7. Read [EvalAI challenge creation documentation](https://evalai.readthedocs.io/en/latest/configuration.html) to know more about how you want to structure your challenge. Once you are ready, start making changes in the yaml file, HTML templates, evaluation script according to your need. -8. Commit the changes and push the `challenge` branch in the repository and wait for the build to complete. View the [logs of your build](https://docs.github.com/en/free-pro-team@latest/actions/managing-workflow-runs/using-workflow-run-logs#viewing-logs-to-diagnose-failures). +8. Commit the changes and push the relevant `challenge*` branch to the repository and wait for the build to complete. View the [logs of your build](https://docs.github.com/en/free-pro-team@latest/actions/managing-workflow-runs/using-workflow-run-logs#viewing-logs-to-diagnose-failures). -9. If challenge config contains errors then a `issue` will be opened automatically in the repository with the errors otherwise the challenge will be created on EvalAI. +9. If the challenge config contains errors, a GitHub **Issue** will be opened automatically in the repository with the error details; otherwise the challenge will be created on EvalAI. 10. Go to [Hosted Challenges](https://eval.ai/web/hosted-challenges) to view your challenge. The challenge will be publicly available once EvalAI admin approves the challenge. -11. To update the challenge on EvalAI, make changes in the repository and push on `challenge` branch and wait for the build to complete. +11. To update the challenge on EvalAI, make changes in the repository and push to the corresponding `challenge*` branch and wait for the build to complete. ## Add custom dependencies for evaluation (Optional) To add custom dependency packages in the evaluation script, refer to [this guide](./evaluation_script/dependency-installation.md). From 164f9915e8ea8d1c6886fdaab7f94d412a784663 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Sat, 5 Jul 2025 23:48:08 +0530 Subject: [PATCH 08/13] Fix duplicate if statement error --- .github/workflows/validate-and-process.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index d1a2ac3f4..56397a243 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -250,12 +250,11 @@ jobs: " - name: Create or update challenge (GitHub-hosted) - if: success() && needs.validate-host-config.outputs.requires_self_hosted != 'true' + if: github.event_name == 'push' && success() && needs.validate-host-config.outputs.requires_self_hosted != 'true' run: | echo "๐Ÿš€ CREATING/UPDATING CHALLENGE" echo "==============================" python3 github/challenge_processing_script.py ${GITHUB_REF#refs/heads/} - if: ${{ github.event_name == 'push' && success() }} env: IS_VALIDATION: 'False' GITHUB_CONTEXT: ${{ toJson(github) }} From d1af8be6ac234d08ffd991a9dc6b94f196ce2a15 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Wed, 9 Jul 2025 21:00:26 +0530 Subject: [PATCH 09/13] Modify script --- .github/workflows/validate-and-process.yml | 10 +- challenge_config.yaml | 328 ++++++++++----------- github/challenge_processing_script.py | 159 +++++++++- github/requirements.txt | 1 + 4 files changed, 319 insertions(+), 179 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 56397a243..9e8674041 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -31,16 +31,16 @@ jobs: echo "๐Ÿ” Analyzing branch: $BRANCH" echo "challenge_branch=$BRANCH" >> $GITHUB_OUTPUT - # Check if this is a challenge branch - if [[ "$BRANCH" =~ ^challenge(-.*)?$ ]]; then + # Check if this is a challenge branch (challenge or challenge-YYYY-version) + if [[ "$BRANCH" =~ ^challenge(-[0-9]{4}-.*)?$ ]]; then echo "is_challenge_branch=true" >> $GITHUB_OUTPUT echo "โœ… Valid challenge branch detected: $BRANCH" # Extract branch suffix for versioning - if [[ "$BRANCH" =~ ^challenge-(.+)$ ]]; then + if [[ "$BRANCH" =~ ^challenge-([0-9]{4}-.+)$ ]]; then SUFFIX="${BASH_REMATCH[1]}" echo "branch_suffix=$SUFFIX" >> $GITHUB_OUTPUT - echo "๐Ÿ“‹ Branch suffix: $SUFFIX" + echo "๐Ÿ“‹ Branch version: $SUFFIX" else echo "branch_suffix=main" >> $GITHUB_OUTPUT echo "๐Ÿ“‹ Main challenge branch (no suffix)" @@ -48,7 +48,7 @@ jobs: else echo "is_challenge_branch=false" >> $GITHUB_OUTPUT echo "โ„น๏ธ Not a challenge branch: $BRANCH" - echo "โญ๏ธ Challenge processing will be skipped" + echo "โญ๏ธ Challenge processing will be skipped (valid patterns: 'challenge' or 'challenge-YYYY-version')" fi validate-host-config: diff --git a/challenge_config.yaml b/challenge_config.yaml index 6afc14ce4..4c0c9c1a4 100755 --- a/challenge_config.yaml +++ b/challenge_config.yaml @@ -1,164 +1,164 @@ -# If you are not sure what all these fields mean, please refer our documentation here: -# https://evalai.readthedocs.io/en/latest/configuration.html -title: Random Number Generator Challenge -short_description: Random number generation challenge for each submission -description: templates/description.html -evaluation_details: templates/evaluation_details.html -terms_and_conditions: templates/terms_and_conditions.html -image: logo.jpg -submission_guidelines: templates/submission_guidelines.html -leaderboard_description: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Cras egestas a libero nec sagittis. -evaluation_script: evaluation_script.zip -remote_evaluation: False -start_date: 2019-01-01 00:00:00 -end_date: 2099-05-31 23:59:59 -published: True -tags: - - random-number-generation - - machine-learning - - data-science - - computer-vision -leaderboard: - - id: 1 - schema: - { - "labels": ["Metric1", "Metric2", "Metric3", "Total"], - "default_order_by": "Total", - "metadata": { - "Metric1": { - "sort_ascending": True, - "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", - }, - "Metric2": { - "sort_ascending": True, - "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", - } - } - } - -challenge_phases: - - id: 1 - name: Dev Phase - description: templates/challenge_phase_1_description.html - leaderboard_public: False - is_public: True - challenge: 1 - is_active: True - max_concurrent_submissions_allowed: 3 - allowed_email_ids: [] - disable_logs: False - is_submission_public: True - start_date: 2019-01-19 00:00:00 - end_date: 2099-04-25 23:59:59 - test_annotation_file: annotations/test_annotations_devsplit.json - codename: dev - max_submissions_per_day: 5 - max_submissions_per_month: 50 - max_submissions: 50 - default_submission_meta_attributes: - - name: method_name - is_visible: True - - name: method_description - is_visible: True - - name: project_url - is_visible: True - - name: publication_url - is_visible: True - submission_meta_attributes: - - name: TextAttribute - description: Sample - type: text - required: False - - name: SingleOptionAttribute - description: Sample - type: radio - options: ["A", "B", "C"] - - name: MultipleChoiceAttribute - description: Sample - type: checkbox - options: ["alpha", "beta", "gamma"] - - name: TrueFalseField - description: Sample - type: boolean - required: True - is_restricted_to_select_one_submission: False - is_partial_submission_evaluation_enabled: False - allowed_submission_file_types: ".json, .zip, .txt, .tsv, .gz, .csv, .h5, .npy, .npz" - - id: 2 - name: Test Phase - description: templates/challenge_phase_2_description.html - leaderboard_public: True - is_public: True - challenge: 2 - is_active: True - max_concurrent_submissions_allowed: 3 - allowed_email_ids: [] - disable_logs: False - is_submission_public: True - start_date: 2019-01-01 00:00:00 - end_date: 2099-05-24 23:59:59 - test_annotation_file: annotations/test_annotations_testsplit.json - codename: test - max_submissions_per_day: 5 - max_submissions_per_month: 50 - max_submissions: 50 - default_submission_meta_attributes: - - name: method_name - is_visible: True - - name: method_description - is_visible: True - - name: project_url - is_visible: True - - name: publication_url - is_visible: True - submission_meta_attributes: - - name: TextAttribute - description: Sample - type: text - - name: SingleOptionAttribute - description: Sample - type: radio - options: ["A", "B", "C"] - - name: MultipleChoiceAttribute - description: Sample - type: checkbox - options: ["alpha", "beta", "gamma"] - - name: TrueFalseField - description: Sample - type: boolean - is_restricted_to_select_one_submission: False - is_partial_submission_evaluation_enabled: False - -dataset_splits: - - id: 1 - name: Train Split - codename: train_split - - id: 2 - name: Test Split - codename: test_split - -challenge_phase_splits: - - challenge_phase_id: 1 - leaderboard_id: 1 - dataset_split_id: 1 - visibility: 1 - leaderboard_decimal_precision: 2 - is_leaderboard_order_descending: True - show_execution_time: True - show_leaderboard_by_latest_submission: True - - challenge_phase_id: 2 - leaderboard_id: 1 - dataset_split_id: 1 - visibility: 3 - leaderboard_decimal_precision: 2 - is_leaderboard_order_descending: True - showeceution_time: False - show_leaderboard_by_latest_submission: False - - challenge_phase_id: 2 - leaderboard_id: 1 - dataset_split_id: 2 - visibility: 1 - leaderboard_decimal_precision: 2 - is_leaderboard_order_descending: True - show_execution_time: True - show_leaderboard_by_latest_submission: True +# If you are not sure what all these fields mean, please refer our documentation here: +# https://evalai.readthedocs.io/en/latest/configuration.html +title: Random Number Generator Challenge +short_description: Random number generation challenge for each submission +description: templates/description.html +evaluation_details: templates/evaluation_details.html +terms_and_conditions: templates/terms_and_conditions.html +image: logo.jpg +submission_guidelines: templates/submission_guidelines.html +leaderboard_description: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Cras egestas a libero nec sagittis. +evaluation_script: evaluation_script.zip +remote_evaluation: False +start_date: 2019-01-01 00:00:00 +end_date: 2099-05-31 23:59:59 +published: True +tags: + - random-number-generation + - machine-learning + - data-science + - computer-vision +leaderboard: + - id: 1 + schema: + { + "labels": ["Metric1", "Metric2", "Metric3", "Total"], + "default_order_by": "Total", + "metadata": { + "Metric1": { + "sort_ascending": True, + "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + }, + "Metric2": { + "sort_ascending": True, + "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + } + } + } + +challenge_phases: + - id: 1 + name: Dev Phase + description: templates/challenge_phase_1_description.html + leaderboard_public: False + is_public: True + challenge: 1 + is_active: True + max_concurrent_submissions_allowed: 3 + allowed_email_ids: [] + disable_logs: False + is_submission_public: True + start_date: 2019-01-19 00:00:00 + end_date: 2099-04-25 23:59:59 + test_annotation_file: annotations/test_annotations_devsplit.json + codename: dev + max_submissions_per_day: 5 + max_submissions_per_month: 50 + max_submissions: 50 + default_submission_meta_attributes: + - name: method_name + is_visible: True + - name: method_description + is_visible: True + - name: project_url + is_visible: True + - name: publication_url + is_visible: True + submission_meta_attributes: + - name: TextAttribute + description: Sample + type: text + required: False + - name: SingleOptionAttribute + description: Sample + type: radio + options: ["A", "B", "C"] + - name: MultipleChoiceAttribute + description: Sample + type: checkbox + options: ["alpha", "beta", "gamma"] + - name: TrueFalseField + description: Sample + type: boolean + required: True + is_restricted_to_select_one_submission: False + is_partial_submission_evaluation_enabled: False + allowed_submission_file_types: ".json, .zip, .txt, .tsv, .gz, .csv, .h5, .npy, .npz" + - id: 2 + name: Test Phase + description: templates/challenge_phase_2_description.html + leaderboard_public: True + is_public: True + challenge: 2 + is_active: True + max_concurrent_submissions_allowed: 3 + allowed_email_ids: [] + disable_logs: False + is_submission_public: True + start_date: 2019-01-01 00:00:00 + end_date: 2099-05-24 23:59:59 + test_annotation_file: annotations/test_annotations_testsplit.json + codename: test + max_submissions_per_day: 5 + max_submissions_per_month: 50 + max_submissions: 50 + default_submission_meta_attributes: + - name: method_name + is_visible: True + - name: method_description + is_visible: True + - name: project_url + is_visible: True + - name: publication_url + is_visible: True + submission_meta_attributes: + - name: TextAttribute + description: Sample + type: text + - name: SingleOptionAttribute + description: Sample + type: radio + options: ["A", "B", "C"] + - name: MultipleChoiceAttribute + description: Sample + type: checkbox + options: ["alpha", "beta", "gamma"] + - name: TrueFalseField + description: Sample + type: boolean + is_restricted_to_select_one_submission: False + is_partial_submission_evaluation_enabled: False + +dataset_splits: + - id: 1 + name: Train Split + codename: train_split + - id: 2 + name: Test Split + codename: test_split + +challenge_phase_splits: + - challenge_phase_id: 1 + leaderboard_id: 1 + dataset_split_id: 1 + visibility: 1 + leaderboard_decimal_precision: 2 + is_leaderboard_order_descending: True + show_execution_time: True + show_leaderboard_by_latest_submission: True + - challenge_phase_id: 2 + leaderboard_id: 1 + dataset_split_id: 1 + visibility: 3 + leaderboard_decimal_precision: 2 + is_leaderboard_order_descending: True + showeceution_time: False + show_leaderboard_by_latest_submission: False + - challenge_phase_id: 2 + leaderboard_id: 1 + dataset_split_id: 2 + visibility: 1 + leaderboard_decimal_precision: 2 + is_leaderboard_order_descending: True + show_execution_time: True + show_leaderboard_by_latest_submission: True \ No newline at end of file diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index ff430133b..1e85d8992 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -48,9 +48,9 @@ # Determine effective branch name (default to "challenge" if none provided) branch_name = args.branch_name if args.branch_name else "challenge" -# Enforce branch naming convention -if not re.match(r"^challenge(-.*)?$", branch_name): - print("Error: Branch name must start with 'challenge' (e.g., 'challenge', 'challenge-2024').") +# Enforce branch naming convention: "challenge" or "challenge-YYYY-version" +if not re.match(r"^challenge(-\d{4}-.*)?$", branch_name): + print("Error: Branch name must be 'challenge' or 'challenge-YYYY-version' (e.g., 'challenge', 'challenge-2024-v1', 'challenge-2025-final').") sys.exit(1) def is_localhost_url(url): @@ -89,6 +89,80 @@ def configure_requests_for_localhost(): print("INFO: SSL verification disabled for localhost development server") +def modify_challenge_title_for_versioning(branch_suffix): + """ + Modify the challenge title in challenge_config.yaml for all servers + to ensure different branch versions create separate challenges + + Arguments: + branch_suffix {str}: The branch suffix (e.g., "2025-v1") + """ + import yaml + + config_file = "challenge_config.yaml" + + try: + # Read the current config + with open(config_file, 'r') as f: + config = yaml.safe_load(f) + + # Get the original title + original_title = config.get('title', 'Challenge') + + # Check if title already has version suffix to avoid double-adding + if f" ({branch_suffix})" not in original_title: + # Add version suffix to title + config['title'] = f"{original_title} ({branch_suffix})" + print(f" ๐Ÿ“ Updated title: {config['title']}") + + # Write back the modified config + with open(config_file, 'w') as f: + yaml.dump(config, f, default_flow_style=False, sort_keys=False) + + print(f" โœ… Challenge config updated for localhost versioning") + return original_title # Return original title for cleanup + else: + print(f" โ„น๏ธ Title already has version suffix: {original_title}") + return None + + except Exception as e: + print(f" โš ๏ธ Warning: Could not modify challenge title: {e}") + print(f" โ„น๏ธ Continuing with original title...") + return None + + +def restore_challenge_title_for_versioning(original_title): + """ + Restore the original challenge title in challenge_config.yaml + + Arguments: + original_title {str}: The original title to restore + """ + if not original_title: + return + + import yaml + + config_file = "challenge_config.yaml" + + try: + # Read the current config + with open(config_file, 'r') as f: + config = yaml.safe_load(f) + + # Restore original title + config['title'] = original_title + + # Write back the restored config + with open(config_file, 'w') as f: + yaml.dump(config, f, default_flow_style=False, sort_keys=False) + + print(f" ๐Ÿ”„ Restored original title: {original_title}") + + except Exception as e: + print(f" โš ๏ธ Warning: Could not restore original title: {e}") + + if __name__ == "__main__": configs = load_host_configs(HOST_CONFIG_FILE_PATH) @@ -133,30 +207,90 @@ def configure_requests_for_localhost(): headers = get_request_header(HOST_AUTH_TOKEN) + # Add the branch name (if provided) so that EvalAI can distinguish between multiple + # versions of the challenge present in the same repository. + + # For branches with year-version format (e.g., challenge-2025-v1, challenge-2025-v2), + # create separate challenges by modifying the repository identifier + effective_repo_name = GITHUB_REPOSITORY + original_title = None # Track original title for cleanup + + if branch_name and branch_name != "challenge": + # Extract year-version suffix from branch name and append to repo name + # challenge-2025-v1 -> 2025-v1 + branch_suffix = branch_name.replace("challenge-", "") + effective_repo_name = f"{GITHUB_REPOSITORY}-{branch_suffix}" + print(f"๐Ÿ”„ Creating separate challenge for branch: {branch_name}") + print(f"๐Ÿ“‹ Effective repository name: {effective_repo_name}") + + # CRITICAL: Modify the challenge title BEFORE creating the zip file + # This ensures the server reads the modified title from the zip + print(f"๐Ÿ“ Modifying challenge title for branch versioning") + original_title = modify_challenge_title_for_versioning(branch_suffix) + # Creating the challenge zip file and storing in a dict to send to EvalAI + # IMPORTANT: This must happen AFTER title modification print(f"\n๐Ÿ“ฆ Creating challenge configuration package...") create_challenge_zip_file(CHALLENGE_ZIP_FILE_PATH, IGNORE_DIRS, IGNORE_FILES) zip_file = open(CHALLENGE_ZIP_FILE_PATH, "rb") file = {"zip_configuration": zip_file} - - # Add the branch name (if provided) so that EvalAI can distinguish between multiple - # versions of the challenge present in the same repository. - data = {"GITHUB_REPOSITORY": GITHUB_REPOSITORY} + + data = {"GITHUB_REPOSITORY": effective_repo_name} if branch_name: data["BRANCH_NAME"] = branch_name + # Debug output + print(f"๐Ÿ” Challenge identification:") + print(f" Original repo: {GITHUB_REPOSITORY}") + print(f" Effective repo: {effective_repo_name}") + print(f" Branch name: {branch_name}") + print(f" Data being sent: {data}") + + # Verify challenge title in the config file + try: + import yaml + with open("challenge_config.yaml", 'r') as f: + config = yaml.safe_load(f) + current_title = config.get('title', 'Unknown') + print(f" Current challenge title: {current_title}") + except Exception as e: + print(f" โš ๏ธ Could not read current title: {e}") + # Configure SSL verification based on whether we're using localhost verify_ssl = not is_localhost print(f"๐Ÿ”’ SSL Verification: {'Disabled (localhost)' if not verify_ssl else 'Enabled'}") try: print(f"\n๐ŸŒ Sending request to EvalAI server...") + print(f"๐Ÿ“ค Request details:") + print(f" URL: {url}") + print(f" Data: {data}") + print(f" Headers: {headers}") + response = requests.post(url, data=data, headers=headers, files=file, verify=verify_ssl) + print(f"๐Ÿ“ฅ Response received:") + print(f" Status code: {response.status_code}") + print(f" Status: {response.status_code == http.HTTPStatus.CREATED and 'CREATED' or response.status_code == http.HTTPStatus.OK and 'UPDATED' or 'OTHER'}") + if response.status_code != http.HTTPStatus.OK and response.status_code != http.HTTPStatus.CREATED: + print(f" Response content: {response.text}") response.raise_for_status() else: - print("\nโœ… Challenge processed successfully on EvalAI") + if response.status_code == http.HTTPStatus.CREATED: + print("\nโœ… NEW Challenge CREATED successfully on EvalAI") + elif response.status_code == http.HTTPStatus.OK: + print("\n๐Ÿ”„ Existing Challenge UPDATED successfully on EvalAI") + + # Try to parse response for additional info + try: + response_data = response.json() + if 'title' in response_data: + print(f" Challenge title: {response_data['title']}") + if 'id' in response_data: + print(f" Challenge ID: {response_data['id']}") + except: + print(" (Could not parse response JSON)") except requests.exceptions.ConnectionError as conn_err: # Handle connection errors specifically for localhost @@ -226,6 +360,11 @@ def configure_requests_for_localhost(): zip_file.close() os.remove(zip_file.name) + # Cleanup: restore original title if it was modified for versioning + if original_title: + print(f"\n๐Ÿงน Cleaning up title modifications...") + restore_challenge_title_for_versioning(original_title) + is_valid, errors = check_for_errors() if not is_valid: # Check if this is a localhost connection error - don't create GitHub issues for expected localhost failures @@ -262,7 +401,7 @@ def configure_requests_for_localhost(): else: add_pull_request_comment( GITHUB_AUTH_TOKEN, - os.path.basename(GITHUB_REPOSITORY), + os.path.basename(effective_repo_name), pr_number, errors, ) @@ -270,7 +409,7 @@ def configure_requests_for_localhost(): issue_title = ( "Following errors occurred while validating the challenge config:" ) - repo_name = os.path.basename(GITHUB_REPOSITORY) if GITHUB_REPOSITORY else "" + repo_name = os.path.basename(effective_repo_name) if effective_repo_name else "" create_github_repository_issue( GITHUB_AUTH_TOKEN, repo_name, diff --git a/github/requirements.txt b/github/requirements.txt index ed667b853..3640884b1 100644 --- a/github/requirements.txt +++ b/github/requirements.txt @@ -1,2 +1,3 @@ PyGithub===1.53 requests==2.32.4 +PyYAML==6.0.1 From e8507f6372dd003be746c5236426d7dd4a3a841b Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Wed, 9 Jul 2025 21:21:08 +0530 Subject: [PATCH 10/13] Add support for localhost --- .github/workflows/validate-and-process.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 9e8674041..a1b5a37df 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -243,10 +243,11 @@ jobs: -e GITHUB_CONTEXT='${{ toJson(github) }}' \ -e GITHUB_REPOSITORY='${{ github.repository }}' \ -e GITHUB_AUTH_TOKEN='${{ secrets.AUTH_TOKEN }}' \ + -e GIT_BRANCH='${{ github.ref_name }}' \ python:3.9-slim \ bash -c " pip install -r github/requirements.txt && - python3 github/challenge_processing_script.py + python3 github/challenge_processing_script.py \$GIT_BRANCH " - name: Create or update challenge (GitHub-hosted) @@ -273,8 +274,9 @@ jobs: -e GITHUB_CONTEXT='${{ toJson(github) }}' \ -e GITHUB_REPOSITORY='${{ github.repository }}' \ -e GITHUB_AUTH_TOKEN='${{ secrets.AUTH_TOKEN }}' \ + -e GIT_BRANCH='${{ github.ref_name }}' \ python:3.9-slim \ bash -c " pip install -r github/requirements.txt && - python3 github/challenge_processing_script.py + python3 github/challenge_processing_script.py \$GIT_BRANCH " From 70d380c191b08d380201c66e72b6f58db45f64c5 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Thu, 17 Jul 2025 02:46:57 +0530 Subject: [PATCH 11/13] Fix branch logic --- .github/workflows/validate-and-process.yml | 2 +- challenge_config.yaml | 155 +++++++++++---------- github/challenge_processing_script.py | 97 +++++-------- 3 files changed, 120 insertions(+), 134 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index a1b5a37df..a53bb64c7 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -262,7 +262,7 @@ jobs: GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }} - name: Create or update challenge (Self-hosted with Docker) - if: success() && needs.validate-host-config.outputs.requires_self_hosted == 'true' + if: github.event_name == 'push' && success() && needs.validate-host-config.outputs.requires_self_hosted == 'true' run: | echo "๐Ÿš€ CREATING/UPDATING CHALLENGE (Docker)" echo "========================================" diff --git a/challenge_config.yaml b/challenge_config.yaml index 4c0c9c1a4..423596821 100755 --- a/challenge_config.yaml +++ b/challenge_config.yaml @@ -1,20 +1,20 @@ # If you are not sure what all these fields mean, please refer our documentation here: # https://evalai.readthedocs.io/en/latest/configuration.html -title: Random Number Generator Challenge -short_description: Random number generation challenge for each submission +title: Advanced Multi-Phase AI Challenge +short_description: Multi-phase challenge with development and validation stages description: templates/description.html evaluation_details: templates/evaluation_details.html terms_and_conditions: templates/terms_and_conditions.html image: logo.jpg submission_guidelines: templates/submission_guidelines.html -leaderboard_description: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Cras egestas a libero nec sagittis. +leaderboard_description: This challenge features multiple phases with different evaluation criteria and submission limits. evaluation_script: evaluation_script.zip remote_evaluation: False -start_date: 2019-01-01 00:00:00 -end_date: 2099-05-31 23:59:59 +start_date: 2024-01-01 00:00:00 +end_date: 2025-12-31 23:59:59 published: True tags: - - random-number-generation + - multi-phase-challenge - machine-learning - data-science - computer-vision @@ -22,39 +22,51 @@ leaderboard: - id: 1 schema: { - "labels": ["Metric1", "Metric2", "Metric3", "Total"], - "default_order_by": "Total", + "labels": ["Accuracy", "Precision", "Recall", "F1-Score", "AUC"], + "default_order_by": "F1-Score", "metadata": { - "Metric1": { - "sort_ascending": True, - "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + "Accuracy": { + "sort_ascending": False, + "description": "Overall accuracy of the model predictions.", }, - "Metric2": { - "sort_ascending": True, - "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + "Precision": { + "sort_ascending": False, + "description": "Precision metric measuring positive prediction accuracy.", + }, + "Recall": { + "sort_ascending": False, + "description": "Recall metric measuring sensitivity of the model.", + }, + "F1-Score": { + "sort_ascending": False, + "description": "Harmonic mean of precision and recall.", + }, + "AUC": { + "sort_ascending": False, + "description": "Area Under the ROC Curve metric.", } } } challenge_phases: - id: 1 - name: Dev Phase + name: Development Phase description: templates/challenge_phase_1_description.html - leaderboard_public: False + leaderboard_public: True is_public: True challenge: 1 is_active: True - max_concurrent_submissions_allowed: 3 + max_concurrent_submissions_allowed: 5 allowed_email_ids: [] disable_logs: False is_submission_public: True - start_date: 2019-01-19 00:00:00 - end_date: 2099-04-25 23:59:59 + start_date: 2024-01-01 00:00:00 + end_date: 2024-08-31 23:59:59 test_annotation_file: annotations/test_annotations_devsplit.json - codename: dev - max_submissions_per_day: 5 - max_submissions_per_month: 50 - max_submissions: 50 + codename: development + max_submissions_per_day: 20 + max_submissions_per_month: 200 + max_submissions: 500 default_submission_meta_attributes: - name: method_name is_visible: True @@ -65,27 +77,30 @@ challenge_phases: - name: publication_url is_visible: True submission_meta_attributes: - - name: TextAttribute - description: Sample + - name: Model Architecture + description: Brief description of your model architecture type: text - required: False - - name: SingleOptionAttribute - description: Sample + required: True + - name: Pre-trained Model + description: Are you using a pre-trained model? type: radio - options: ["A", "B", "C"] - - name: MultipleChoiceAttribute - description: Sample + options: ["Yes", "No"] + required: True + - name: Training Data + description: What training data did you use? type: checkbox - options: ["alpha", "beta", "gamma"] - - name: TrueFalseField - description: Sample + options: ["ImageNet", "COCO", "Custom Dataset", "Synthetic Data", "Other"] + required: False + - name: Data Augmentation + description: Did you use data augmentation? type: boolean - required: True + required: False is_restricted_to_select_one_submission: False - is_partial_submission_evaluation_enabled: False - allowed_submission_file_types: ".json, .zip, .txt, .tsv, .gz, .csv, .h5, .npy, .npz" + is_partial_submission_evaluation_enabled: True + allowed_submission_file_types: ".json, .zip, .txt, .tsv, .gz, .csv, .h5, .npy, .npz, .pkl" + - id: 2 - name: Test Phase + name: Validation Phase description: templates/challenge_phase_2_description.html leaderboard_public: True is_public: True @@ -94,14 +109,14 @@ challenge_phases: max_concurrent_submissions_allowed: 3 allowed_email_ids: [] disable_logs: False - is_submission_public: True - start_date: 2019-01-01 00:00:00 - end_date: 2099-05-24 23:59:59 + is_submission_public: False + start_date: 2024-09-01 00:00:00 + end_date: 2024-11-30 23:59:59 test_annotation_file: annotations/test_annotations_testsplit.json - codename: test - max_submissions_per_day: 5 - max_submissions_per_month: 50 - max_submissions: 50 + codename: validation + max_submissions_per_day: 10 + max_submissions_per_month: 100 + max_submissions: 200 default_submission_meta_attributes: - name: method_name is_visible: True @@ -109,56 +124,54 @@ challenge_phases: is_visible: True - name: project_url is_visible: True - - name: publication_url - is_visible: True submission_meta_attributes: - - name: TextAttribute - description: Sample + - name: Model Version + description: Version identifier for your model type: text - - name: SingleOptionAttribute - description: Sample - type: radio - options: ["A", "B", "C"] - - name: MultipleChoiceAttribute - description: Sample + required: True + - name: Computational Resources + description: What computational resources were used? type: checkbox - options: ["alpha", "beta", "gamma"] - - name: TrueFalseField - description: Sample + options: ["CPU", "Single GPU", "Multiple GPUs", "TPU", "Cloud Computing", "Other"] + required: True + - name: Training Time + description: Approximate training time + type: radio + options: ["< 1 hour", "1-6 hours", "6-24 hours", "1-7 days", "> 1 week"] + required: False + - name: Ensemble Method + description: Did you use ensemble methods? type: boolean + required: False is_restricted_to_select_one_submission: False is_partial_submission_evaluation_enabled: False + allowed_submission_file_types: ".json, .zip, .txt, .tsv, .gz, .csv, .h5, .npy, .npz" dataset_splits: - id: 1 - name: Train Split - codename: train_split + name: Development Split + codename: dev_split - id: 2 name: Test Split codename: test_split challenge_phase_splits: + # Development Phase - uses dev split with public leaderboard - challenge_phase_id: 1 leaderboard_id: 1 dataset_split_id: 1 visibility: 1 - leaderboard_decimal_precision: 2 + leaderboard_decimal_precision: 4 is_leaderboard_order_descending: True show_execution_time: True show_leaderboard_by_latest_submission: True - - challenge_phase_id: 2 - leaderboard_id: 1 - dataset_split_id: 1 - visibility: 3 - leaderboard_decimal_precision: 2 - is_leaderboard_order_descending: True - showeceution_time: False - show_leaderboard_by_latest_submission: False + + # Validation Phase - uses test split with public leaderboard - challenge_phase_id: 2 leaderboard_id: 1 dataset_split_id: 2 visibility: 1 - leaderboard_decimal_precision: 2 + leaderboard_decimal_precision: 4 is_leaderboard_order_descending: True show_execution_time: True - show_leaderboard_by_latest_submission: True \ No newline at end of file + show_leaderboard_by_latest_submission: True diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index 1e85d8992..5579cec7d 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -9,7 +9,18 @@ import urllib3 from urllib.parse import urlparse -from config import * +from config import ( + HOST_CONFIG_FILE_PATH, + CHALLENGE_CONFIG_VALIDATION_URL, + CHALLENGE_CREATE_OR_UPDATE_URL, + EVALAI_ERROR_CODES, + API_HOST_URL, + IGNORE_DIRS, + IGNORE_FILES, + CHALLENGE_ZIP_FILE_PATH, + GITHUB_EVENT_NAME, + VALIDATION_STEP, +) from utils import ( add_pull_request_comment, check_for_errors, @@ -26,6 +37,16 @@ GITHUB_CONTEXT = json.loads(os.getenv("GITHUB_CONTEXT", "{}")) GITHUB_AUTH_TOKEN = os.getenv("GITHUB_AUTH_TOKEN") + +# START of the FIX: Explicitly read GITHUB_REPOSITORY from environment variable +GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY") +if not GITHUB_REPOSITORY: + print("FATAL: GITHUB_REPOSITORY environment variable is not set.") + print("Please ensure your GitHub Actions workflow sets this variable.") + sys.exit(1) +print(f"๐Ÿ” GITHUB_REPOSITORY from environment: {GITHUB_REPOSITORY}") +# END of the FIX + if not GITHUB_AUTH_TOKEN: print( "Please add your github access token to the repository secrets with the name AUTH_TOKEN" @@ -91,11 +112,11 @@ def configure_requests_for_localhost(): def modify_challenge_title_for_versioning(branch_suffix): """ - Modify the challenge title in challenge_config.yaml for all servers - to ensure different branch versions create separate challenges + Keep the original challenge title in challenge_config.yaml + Different branch versions will create separate challenges through repository name modification Arguments: - branch_suffix {str}: The branch suffix (e.g., "2025-v1") + branch_suffix {str}: The branch suffix (e.g., "2025-v1") - not used for title modification """ import yaml @@ -109,59 +130,16 @@ def modify_challenge_title_for_versioning(branch_suffix): # Get the original title original_title = config.get('title', 'Challenge') - # Check if title already has version suffix to avoid double-adding - if f" ({branch_suffix})" not in original_title: - # Add version suffix to title - config['title'] = f"{original_title} ({branch_suffix})" - print(f" ๐Ÿ“ Updated title: {config['title']}") - - # Write back the modified config - with open(config_file, 'w') as f: - yaml.dump(config, f, default_flow_style=False, sort_keys=False) - - print(f" โœ… Challenge config updated for localhost versioning") - return original_title # Return original title for cleanup - else: - print(f" โ„น๏ธ Title already has version suffix: {original_title}") - return None + print(f" ๐Ÿ“ Keeping original title: {original_title}") + print(f" โœ… Challenge versioning handled through repository name modification") + return None # No title modification needed except Exception as e: - print(f" โš ๏ธ Warning: Could not modify challenge title: {e}") + print(f" โš ๏ธ Warning: Could not read challenge title: {e}") print(f" โ„น๏ธ Continuing with original title...") return None -def restore_challenge_title_for_versioning(original_title): - """ - Restore the original challenge title in challenge_config.yaml - - Arguments: - original_title {str}: The original title to restore - """ - if not original_title: - return - - import yaml - - config_file = "challenge_config.yaml" - - try: - # Read the current config - with open(config_file, 'r') as f: - config = yaml.safe_load(f) - - # Restore original title - config['title'] = original_title - - # Write back the restored config - with open(config_file, 'w') as f: - yaml.dump(config, f, default_flow_style=False, sort_keys=False) - - print(f" ๐Ÿ”„ Restored original title: {original_title}") - - except Exception as e: - print(f" โš ๏ธ Warning: Could not restore original title: {e}") - if __name__ == "__main__": @@ -175,7 +153,8 @@ def restore_challenge_title_for_versioning(original_title): # Update the global config path for zip file creation - config.CHALLENGE_CONFIG_FILE_PATH = "challenge_config.yaml" + # Note: We're not importing config.* anymore, so we need to set this directly + CHALLENGE_CONFIG_FILE_PATH = "challenge_config.yaml" # Check if we're using a localhost server and configure accordingly is_localhost = is_localhost_url(EVALAI_HOST_URL) @@ -213,7 +192,6 @@ def restore_challenge_title_for_versioning(original_title): # For branches with year-version format (e.g., challenge-2025-v1, challenge-2025-v2), # create separate challenges by modifying the repository identifier effective_repo_name = GITHUB_REPOSITORY - original_title = None # Track original title for cleanup if branch_name and branch_name != "challenge": # Extract year-version suffix from branch name and append to repo name @@ -223,10 +201,10 @@ def restore_challenge_title_for_versioning(original_title): print(f"๐Ÿ”„ Creating separate challenge for branch: {branch_name}") print(f"๐Ÿ“‹ Effective repository name: {effective_repo_name}") - # CRITICAL: Modify the challenge title BEFORE creating the zip file - # This ensures the server reads the modified title from the zip - print(f"๐Ÿ“ Modifying challenge title for branch versioning") - original_title = modify_challenge_title_for_versioning(branch_suffix) + # Note: Challenge versioning is handled through repository name modification + # The title remains unchanged to keep it clean + print(f"๐Ÿ“ Challenge versioning handled through repository name") + modify_challenge_title_for_versioning(branch_suffix) # Creating the challenge zip file and storing in a dict to send to EvalAI # IMPORTANT: This must happen AFTER title modification @@ -360,11 +338,6 @@ def restore_challenge_title_for_versioning(original_title): zip_file.close() os.remove(zip_file.name) - # Cleanup: restore original title if it was modified for versioning - if original_title: - print(f"\n๐Ÿงน Cleaning up title modifications...") - restore_challenge_title_for_versioning(original_title) - is_valid, errors = check_for_errors() if not is_valid: # Check if this is a localhost connection error - don't create GitHub issues for expected localhost failures From e3b09dcb8d0bfe6c50a65db357e80df21f1b1c1a Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Thu, 17 Jul 2025 02:50:20 +0530 Subject: [PATCH 12/13] Update workflow --- .github/workflows/validate-and-process.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index a53bb64c7..964ec6204 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -1,11 +1,12 @@ name: Validate and Process EvalAI Challenge on: - push: {} - pull_request: - types: [opened, synchronize, reopened, edited] + push: branches: - 'challenge' + - 'challenge-*-*' + pull_request: + types: [opened, synchronize, reopened, edited] permissions: contents: read @@ -161,7 +162,7 @@ jobs: process-evalai-challenge: - needs: [validate-host-config, check-self-hosted-requirements] + needs: [detect-and-validate-challenge-branch, validate-host-config, check-self-hosted-requirements] if: | always() && needs.validate-host-config.outputs.is_valid == 'true' && @@ -172,7 +173,7 @@ jobs: - name: Checkout challenge branch uses: actions/checkout@v3 with: - ref: ${{ github.ref_name }} + ref: ${{ needs.detect-and-validate-challenge-branch.outputs.challenge_branch }} - name: Set up Python (GitHub-hosted only) if: needs.validate-host-config.outputs.requires_self_hosted != 'true' @@ -224,7 +225,7 @@ jobs: run: | echo "๐Ÿ” VALIDATING CHALLENGE CONFIGURATION" echo "=====================================" - python3 github/challenge_processing_script.py ${GITHUB_REF#refs/heads/} + python3 github/challenge_processing_script.py ${{ needs.detect-and-validate-challenge-branch.outputs.challenge_branch }} env: IS_VALIDATION: 'True' GITHUB_CONTEXT: ${{ toJson(github) }} @@ -243,7 +244,7 @@ jobs: -e GITHUB_CONTEXT='${{ toJson(github) }}' \ -e GITHUB_REPOSITORY='${{ github.repository }}' \ -e GITHUB_AUTH_TOKEN='${{ secrets.AUTH_TOKEN }}' \ - -e GIT_BRANCH='${{ github.ref_name }}' \ + -e GIT_BRANCH='${{ needs.detect-and-validate-challenge-branch.outputs.challenge_branch }}' \ python:3.9-slim \ bash -c " pip install -r github/requirements.txt && @@ -255,7 +256,7 @@ jobs: run: | echo "๐Ÿš€ CREATING/UPDATING CHALLENGE" echo "==============================" - python3 github/challenge_processing_script.py ${GITHUB_REF#refs/heads/} + python3 github/challenge_processing_script.py ${{ needs.detect-and-validate-challenge-branch.outputs.challenge_branch }} env: IS_VALIDATION: 'False' GITHUB_CONTEXT: ${{ toJson(github) }} @@ -274,7 +275,7 @@ jobs: -e GITHUB_CONTEXT='${{ toJson(github) }}' \ -e GITHUB_REPOSITORY='${{ github.repository }}' \ -e GITHUB_AUTH_TOKEN='${{ secrets.AUTH_TOKEN }}' \ - -e GIT_BRANCH='${{ github.ref_name }}' \ + -e GIT_BRANCH='${{ needs.detect-and-validate-challenge-branch.outputs.challenge_branch }}' \ python:3.9-slim \ bash -c " pip install -r github/requirements.txt && From 182c3f4ed9670b84bd2c18045dbbcae1776efba8 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Thu, 17 Jul 2025 02:55:56 +0530 Subject: [PATCH 13/13] Update github hosted steps and env variables --- .github/workflows/validate-and-process.yml | 3 +++ github/challenge_processing_script.py | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 964ec6204..387e17477 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -165,6 +165,7 @@ jobs: needs: [detect-and-validate-challenge-branch, validate-host-config, check-self-hosted-requirements] if: | always() && + needs.detect-and-validate-challenge-branch.outputs.is_challenge_branch == 'true' && needs.validate-host-config.outputs.is_valid == 'true' && (needs.check-self-hosted-requirements.result == 'success' || needs.check-self-hosted-requirements.result == 'skipped') runs-on: ${{ needs.validate-host-config.outputs.requires_self_hosted == 'true' && 'self-hosted' || 'ubuntu-latest' }} @@ -229,6 +230,7 @@ jobs: env: IS_VALIDATION: 'True' GITHUB_CONTEXT: ${{ toJson(github) }} + GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }} - name: Validate challenge configuration (Self-hosted with Docker) @@ -260,6 +262,7 @@ jobs: env: IS_VALIDATION: 'False' GITHUB_CONTEXT: ${{ toJson(github) }} + GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }} - name: Create or update challenge (Self-hosted with Docker) diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index 5579cec7d..72370b075 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -19,7 +19,6 @@ IGNORE_FILES, CHALLENGE_ZIP_FILE_PATH, GITHUB_EVENT_NAME, - VALIDATION_STEP, ) from utils import ( add_pull_request_comment, @@ -45,7 +44,10 @@ print("Please ensure your GitHub Actions workflow sets this variable.") sys.exit(1) print(f"๐Ÿ” GITHUB_REPOSITORY from environment: {GITHUB_REPOSITORY}") -# END of the FIX + +VALIDATION_STEP = os.getenv("IS_VALIDATION") +print(f"๐Ÿ” VALIDATION_STEP from IS_VALIDATION: {VALIDATION_STEP}") + if not GITHUB_AUTH_TOKEN: print(