From 5a8528326188a8911cbe2269c434cb76429e412a Mon Sep 17 00:00:00 2001 From: Eeshu Date: Tue, 18 Mar 2025 13:25:33 +0000 Subject: [PATCH 01/19] Add CI validation pipeline --- .github/workflows/ci.yml | 28 ++++++++++++++ validate_config.py | 83 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 validate_config.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..1581d73d8 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,28 @@ +name: EvalAI Starter CI + +on: + pull_request: + branches: [master] + push: + branches: [master] + +jobs: + validation: + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python 3.9 + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Install dependencies + run: | + pip install pyyaml jsonschema + + - name: Validate challenge configuration + run: python validate_config.py diff --git a/validate_config.py b/validate_config.py new file mode 100644 index 000000000..b170ccfaf --- /dev/null +++ b/validate_config.py @@ -0,0 +1,83 @@ +import yaml +import os +from jsonschema import validate, ValidationError +from datetime import datetime + +SCHEMA = { + "type": "object", + "properties": { + "title": {"type": "string"}, + "short_description": {"type": "string", "maxLength": 140}, + "description": {"type": "string", "pattern": "^templates/.*\\.html$"}, + "evaluation_details": {"type": "string", "pattern": "^templates/.*\\.html$"}, + "terms_and_conditions": {"type": "string", "pattern": "^templates/.*\\.html$"}, + "image": {"type": "string", "pattern": ".+\\.(jpg|jpeg|png)$"}, + "submission_guidelines": {"type": "string", "pattern": "^templates/.*\\.html$"}, + "evaluation_script": {"type": "string"}, + "leaderboard": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": {"type": "integer", "minimum": 1}, + "schema": { + "type": "object", + "properties": { + "labels": {"type": "array", "items": {"type": "string"}}, + "default_order_by": {"type": "string"}, + "metadata": {"type": "object"} + }, + "required": ["labels", "default_order_by"] + } + }, + "required": ["id", "schema"] + } + }, + "challenge_phases": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": {"type": "integer", "minimum": 1}, + "codename": {"type": "string"}, + "test_annotation_file": {"type": "string"} + }, + "required": ["id", "codename", "test_annotation_file"] + } + } + }, + "required": [ + "title", + "description", + "submission_guidelines", + "evaluation_script", + "leaderboard", + "challenge_phases" + ] +} + +def validate_config(): + try: + with open("challenge_config.yml") as f: + config = yaml.safe_load(f) + + validate(instance=config, schema=SCHEMA) + + for field in ["description", "evaluation_details", "terms_and_conditions", "submission_guidelines"]: + if not os.path.exists(config[field]): + raise ValidationError(f"Missing template file: {config[field]}") + + for phase in config["challenge_phases"]: + if not os.path.exists(phase["test_annotation_file"]): + raise ValidationError(f"Missing annotation file: {phase['test_annotation_file']}") + + print("✅ Config validation successful!") + return True + + except Exception as e: + print(f"❌ Validation failed: {str(e)}") + return False + +if __name__ == "__main__": + if not validate_config(): + exit(1) \ No newline at end of file From d084f232929b2393af9e7b4b7ebcd2b46e2204f5 Mon Sep 17 00:00:00 2001 From: Eeshu Date: Tue, 18 Mar 2025 13:36:04 +0000 Subject: [PATCH 02/19] Add CI validation pipeline --- validate_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validate_config.py b/validate_config.py index b170ccfaf..72267e67b 100644 --- a/validate_config.py +++ b/validate_config.py @@ -58,7 +58,7 @@ def validate_config(): try: - with open("challenge_config.yml") as f: + with open("challenge_config.yaml") as f: config = yaml.safe_load(f) validate(instance=config, schema=SCHEMA) From c1040f5741e012e6317b8724d9ce3fc8607623fc Mon Sep 17 00:00:00 2001 From: Eeshu Date: Tue, 18 Mar 2025 16:57:22 +0000 Subject: [PATCH 03/19] Integrate EvalAI API for server-side validation --- .github/workflows/ci.yml | 12 ++++-- validate_config.py | 83 ---------------------------------------- 2 files changed, 9 insertions(+), 86 deletions(-) delete mode 100644 validate_config.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1581d73d8..f539b4e87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,13 @@ jobs: - name: Install dependencies run: | - pip install pyyaml jsonschema + pip install requests pyyaml - - name: Validate challenge configuration - run: python validate_config.py + - name: Validate challenge configuration using EvalAI API + env: + GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HOST_AUTH_TOKEN: ${{ secrets.EVALAI_AUTH_TOKEN }} + CHALLENGE_HOST_TEAM_PK: ${{ secrets.CHALLENGE_HOST_TEAM_PK }} + EVALAI_HOST_URL: ${{ secrets.EVALAI_HOST_URL }} + run: | + python github/challenge_processing_script.py diff --git a/validate_config.py b/validate_config.py deleted file mode 100644 index 72267e67b..000000000 --- a/validate_config.py +++ /dev/null @@ -1,83 +0,0 @@ -import yaml -import os -from jsonschema import validate, ValidationError -from datetime import datetime - -SCHEMA = { - "type": "object", - "properties": { - "title": {"type": "string"}, - "short_description": {"type": "string", "maxLength": 140}, - "description": {"type": "string", "pattern": "^templates/.*\\.html$"}, - "evaluation_details": {"type": "string", "pattern": "^templates/.*\\.html$"}, - "terms_and_conditions": {"type": "string", "pattern": "^templates/.*\\.html$"}, - "image": {"type": "string", "pattern": ".+\\.(jpg|jpeg|png)$"}, - "submission_guidelines": {"type": "string", "pattern": "^templates/.*\\.html$"}, - "evaluation_script": {"type": "string"}, - "leaderboard": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": {"type": "integer", "minimum": 1}, - "schema": { - "type": "object", - "properties": { - "labels": {"type": "array", "items": {"type": "string"}}, - "default_order_by": {"type": "string"}, - "metadata": {"type": "object"} - }, - "required": ["labels", "default_order_by"] - } - }, - "required": ["id", "schema"] - } - }, - "challenge_phases": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": {"type": "integer", "minimum": 1}, - "codename": {"type": "string"}, - "test_annotation_file": {"type": "string"} - }, - "required": ["id", "codename", "test_annotation_file"] - } - } - }, - "required": [ - "title", - "description", - "submission_guidelines", - "evaluation_script", - "leaderboard", - "challenge_phases" - ] -} - -def validate_config(): - try: - with open("challenge_config.yaml") as f: - config = yaml.safe_load(f) - - validate(instance=config, schema=SCHEMA) - - for field in ["description", "evaluation_details", "terms_and_conditions", "submission_guidelines"]: - if not os.path.exists(config[field]): - raise ValidationError(f"Missing template file: {config[field]}") - - for phase in config["challenge_phases"]: - if not os.path.exists(phase["test_annotation_file"]): - raise ValidationError(f"Missing annotation file: {phase['test_annotation_file']}") - - print("✅ Config validation successful!") - return True - - except Exception as e: - print(f"❌ Validation failed: {str(e)}") - return False - -if __name__ == "__main__": - if not validate_config(): - exit(1) \ No newline at end of file From 1d7deb3828a79add24a1d8e7cc792d25fcca9329 Mon Sep 17 00:00:00 2001 From: Eeshu Date: Tue, 18 Mar 2025 16:59:55 +0000 Subject: [PATCH 04/19] Integrate EvalAI API for server-side validation --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f539b4e87..3bf76d302 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: - name: Install dependencies run: | - pip install requests pyyaml + pip install requests pyyaml PyGithub - name: Validate challenge configuration using EvalAI API env: From 586bd30c5645017ecc0452471722d27ef5c70b66 Mon Sep 17 00:00:00 2001 From: Eeshu Date: Tue, 18 Mar 2025 17:05:06 +0000 Subject: [PATCH 05/19] Fix GITHUB_CONTEXT handling in CI workflow --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bf76d302..b8d25c4f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,5 +30,6 @@ jobs: HOST_AUTH_TOKEN: ${{ secrets.EVALAI_AUTH_TOKEN }} CHALLENGE_HOST_TEAM_PK: ${{ secrets.CHALLENGE_HOST_TEAM_PK }} EVALAI_HOST_URL: ${{ secrets.EVALAI_HOST_URL }} + GITHUB_CONTEXT: ${{ toJSON(github) }} run: | python github/challenge_processing_script.py From 52ad3a6ddefb79b377ca4724ebee512633a0f866 Mon Sep 17 00:00:00 2001 From: Eeshu Date: Tue, 18 Mar 2025 17:11:50 +0000 Subject: [PATCH 06/19] Fix API URL and permissions in CI workflow --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8d25c4f2..fbf294509 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} HOST_AUTH_TOKEN: ${{ secrets.EVALAI_AUTH_TOKEN }} CHALLENGE_HOST_TEAM_PK: ${{ secrets.CHALLENGE_HOST_TEAM_PK }} - EVALAI_HOST_URL: ${{ secrets.EVALAI_HOST_URL }} + EVALAI_HOST_URL: https://${{ secrets.EVALAI_HOST_URL }} GITHUB_CONTEXT: ${{ toJSON(github) }} run: | python github/challenge_processing_script.py From cc76a87011d00e0aebb233f145d76dda7f3d2e49 Mon Sep 17 00:00:00 2001 From: Eeshu Date: Tue, 18 Mar 2025 18:11:47 +0000 Subject: [PATCH 07/19] Fix API URL and permissions in CI workflow --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fbf294509..35bf4247b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,10 +26,10 @@ jobs: - name: Validate challenge configuration using EvalAI API env: - GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_AUTH_TOKEN: ${{ secrets.PAT_TOKEN }} HOST_AUTH_TOKEN: ${{ secrets.EVALAI_AUTH_TOKEN }} CHALLENGE_HOST_TEAM_PK: ${{ secrets.CHALLENGE_HOST_TEAM_PK }} - EVALAI_HOST_URL: https://${{ secrets.EVALAI_HOST_URL }} + EVALAI_HOST_URL: ${{ secrets.EVALAI_HOST_URL }} GITHUB_CONTEXT: ${{ toJSON(github) }} run: | python github/challenge_processing_script.py From de897545bbe7ea25e91c35a91c3d3ce492a15622 Mon Sep 17 00:00:00 2001 From: Eeshu Date: Tue, 18 Mar 2025 18:21:37 +0000 Subject: [PATCH 08/19] Fix API URL and permissions in CI workflow --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35bf4247b..9d9cd6a46 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,3 +33,4 @@ jobs: GITHUB_CONTEXT: ${{ toJSON(github) }} run: | python github/challenge_processing_script.py + From f2160cd3636481210559785fb2569a7a98d2af88 Mon Sep 17 00:00:00 2001 From: Eeshu Date: Wed, 19 Mar 2025 02:47:44 +0000 Subject: [PATCH 09/19] Fix versions --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d9cd6a46..a6e23ab52 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,12 +13,12 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - - name: Set up Python 3.9 - uses: actions/setup-python@v4 + - name: Set up Python 3.13 + uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.13 - name: Install dependencies run: | From 5dccf3506c1b3e9131c1e8f32d3417be917db9fc Mon Sep 17 00:00:00 2001 From: Eeshu Date: Wed, 19 Mar 2025 05:14:55 +0000 Subject: [PATCH 10/19] Fixes done --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6e23ab52..4cbe88cf4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,11 +26,15 @@ jobs: - name: Validate challenge configuration using EvalAI API env: - GITHUB_AUTH_TOKEN: ${{ secrets.PAT_TOKEN }} + GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }} HOST_AUTH_TOKEN: ${{ secrets.EVALAI_AUTH_TOKEN }} CHALLENGE_HOST_TEAM_PK: ${{ secrets.CHALLENGE_HOST_TEAM_PK }} EVALAI_HOST_URL: ${{ secrets.EVALAI_HOST_URL }} GITHUB_CONTEXT: ${{ toJSON(github) }} run: | + echo "EVALAI_HOST_URL: ${EVALAI_HOST_URL}" + echo "CHALLENGE_HOST_TEAM_PK: ${CHALLENGE_HOST_TEAM_PK}" + echo "GITHUB_CONTEXT: ${GITHUB_CONTEXT}" + echo "GITHUB_AUTH_TOKEN: ${AUTH_TOKEN}" + echo "HOST_AUTH_TOKEN: ${EVALAI_AUTH_TOKEN}" python github/challenge_processing_script.py - From b3898846b718e00c938981995ea4b6b74516ad20 Mon Sep 17 00:00:00 2001 From: Eeshu Date: Wed, 19 Mar 2025 05:28:00 +0000 Subject: [PATCH 11/19] Fixes done --- .github/workflows/ci.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4cbe88cf4..128c6d639 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,9 +32,4 @@ jobs: EVALAI_HOST_URL: ${{ secrets.EVALAI_HOST_URL }} GITHUB_CONTEXT: ${{ toJSON(github) }} run: | - echo "EVALAI_HOST_URL: ${EVALAI_HOST_URL}" - echo "CHALLENGE_HOST_TEAM_PK: ${CHALLENGE_HOST_TEAM_PK}" - echo "GITHUB_CONTEXT: ${GITHUB_CONTEXT}" - echo "GITHUB_AUTH_TOKEN: ${AUTH_TOKEN}" - echo "HOST_AUTH_TOKEN: ${EVALAI_AUTH_TOKEN}" python github/challenge_processing_script.py From 1b8cfd1f16ef9fb90eac2189aa96ae9c2438b854 Mon Sep 17 00:00:00 2001 From: Eeshu Date: Thu, 20 Mar 2025 05:52:40 +0000 Subject: [PATCH 12/19] Fixes done --- .github/workflows/ci.yml | 14 ++ github/challenge_processing_script.py | 182 +++++++++++--------------- 2 files changed, 88 insertions(+), 108 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 128c6d639..73a729b6e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,17 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Validate environment variables + run: | + if [ -z "${{ secrets.EVALAI_HOST_URL }}" ]; then + echo "::error::EVALAI_HOST_URL secret is missing" + exit 1 + fi + if [[ ! "${{ secrets.EVALAI_HOST_URL }}" =~ ^https?:// ]]; then + echo "::error::EVALAI_HOST_URL must start with http:// or https://" + exit 1 + fi + - name: Set up Python 3.13 uses: actions/setup-python@v5 with: @@ -22,6 +33,7 @@ jobs: - name: Install dependencies run: | + python -m pip install --upgrade pip pip install requests pyyaml PyGithub - name: Validate challenge configuration using EvalAI API @@ -32,4 +44,6 @@ jobs: EVALAI_HOST_URL: ${{ secrets.EVALAI_HOST_URL }} GITHUB_CONTEXT: ${{ toJSON(github) }} run: | + echo "Validating challenge configuration..." + echo "API Endpoint: ${EVALAI_HOST_URL}/api/challenges/challenge/challenge_host_team/${CHALLENGE_HOST_TEAM_PK}/create_or_update_github_challenge/" python github/challenge_processing_script.py diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index b0c88bb98..2bc3ea409 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -8,7 +8,6 @@ from utils import ( add_pull_request_comment, check_for_errors, - check_if_merge_or_commit, check_if_pull_request, create_challenge_zip_file, create_github_repository_issue, @@ -19,126 +18,93 @@ sys.dont_write_bytecode = True -GITHUB_CONTEXT = json.loads(os.getenv("GITHUB_CONTEXT")) - -GITHUB_AUTH_TOKEN = os.getenv("GITHUB_AUTH_TOKEN") -if not GITHUB_AUTH_TOKEN: - print( - "Please add your github access token to the repository secrets with the name AUTH_TOKEN" - ) - sys.exit(1) - -HOST_AUTH_TOKEN = None -CHALLENGE_HOST_TEAM_PK = None -EVALAI_HOST_URL = None - +def validate_environment(): + """Validate required environment variables""" + required_vars = { + 'GITHUB_AUTH_TOKEN': lambda v: len(v) > 20, + 'EVALAI_HOST_URL': lambda v: v.startswith(('http://', 'https://')), + 'CHALLENGE_HOST_TEAM_PK': lambda v: v.isdigit(), + } + + for var, validator in required_vars.items(): + value = os.getenv(var) + if not value or not validator(value): + print(f"::error::Invalid {var}: {value}") + sys.exit(1) if __name__ == "__main__": - - configs = load_host_configs(HOST_CONFIG_FILE_PATH) - if configs: - HOST_AUTH_TOKEN = configs[0] - CHALLENGE_HOST_TEAM_PK = configs[1] - EVALAI_HOST_URL = configs[2] - else: - sys.exit(1) - - # Fetching the url - if VALIDATION_STEP == "True": - url = "{}{}".format( - EVALAI_HOST_URL, - CHALLENGE_CONFIG_VALIDATION_URL.format(CHALLENGE_HOST_TEAM_PK), - ) - else: - url = "{}{}".format( - EVALAI_HOST_URL, - CHALLENGE_CREATE_OR_UPDATE_URL.format(CHALLENGE_HOST_TEAM_PK), - ) - - headers = get_request_header(HOST_AUTH_TOKEN) - - # Creating the challenge zip file and storing in a dict to send to EvalAI - create_challenge_zip_file(CHALLENGE_ZIP_FILE_PATH, IGNORE_DIRS, IGNORE_FILES) - zip_file = open(CHALLENGE_ZIP_FILE_PATH, "rb") - file = {"zip_configuration": zip_file} - - data = {"GITHUB_REPOSITORY": GITHUB_REPOSITORY} - try: - response = requests.post(url, data=data, headers=headers, files=file) + # Validate environment variables first + validate_environment() + + # Load GitHub context + github_context = json.loads(os.getenv("GITHUB_CONTEXT", "{}")) + + # Load EvalAI configurations + configs = load_host_configs(HOST_CONFIG_FILE_PATH) + if not configs: + print("::error::Missing EvalAI host configurations") + sys.exit(1) - if ( - response.status_code != http.HTTPStatus.OK - and response.status_code != http.HTTPStatus.CREATED - ): - response.raise_for_status() - else: - print("\n" + response.json()["Success"]) - except requests.exceptions.HTTPError as err: - if response.status_code in EVALAI_ERROR_CODES: - is_token_valid = validate_token(response.json()) - if is_token_valid: - error = response.json()["error"] - error_message = "\nFollowing errors occurred while validating the challenge config:\n{}".format( - error - ) - print(error_message) - os.environ["CHALLENGE_ERRORS"] = error_message - else: - print( - "\nFollowing errors occurred while validating the challenge config: {}".format( - err - ) - ) - os.environ["CHALLENGE_ERRORS"] = str(err) - except Exception as e: + HOST_AUTH_TOKEN, CHALLENGE_HOST_TEAM_PK, EVALAI_HOST_URL = configs + + # Construct API URL with validation + base_url = EVALAI_HOST_URL.rstrip('/') if VALIDATION_STEP == "True": - error_message = "\nFollowing errors occurred while validating the challenge config: {}".format( - e - ) - print(error_message) - os.environ["CHALLENGE_ERRORS"] = error_message + endpoint = CHALLENGE_CONFIG_VALIDATION_URL.format(CHALLENGE_HOST_TEAM_PK) else: - error_message = "\nFollowing errors occurred while processing the challenge config: {}".format( - e + endpoint = CHALLENGE_CREATE_OR_UPDATE_URL.format(CHALLENGE_HOST_TEAM_PK) + + url = f"{base_url}/{endpoint.lstrip('/')}" + print(f"🔧 Using API endpoint: {url}") + + # Create challenge package + zip_path = create_challenge_zip_file(CHALLENGE_ZIP_FILE_PATH, IGNORE_DIRS, IGNORE_FILES) + + # Send to EvalAI + with open(zip_path, 'rb') as zip_file: + response = requests.post( + url, + headers=get_request_header(HOST_AUTH_TOKEN), + files={'zip_configuration': zip_file}, + data={'GITHUB_REPOSITORY': os.getenv("GITHUB_REPOSITORY")}, + timeout=30 ) - print(error_message) - os.environ["CHALLENGE_ERRORS"] = error_message + response.raise_for_status() + print(f"✅ Success: {response.json().get('Success', '')}") - zip_file.close() - os.remove(zip_file.name) + except requests.exceptions.RequestException as e: + error_msg = f"API Request failed: {str(e)}" + if hasattr(e, 'response') and e.response.text: + error_msg += f"\nResponse: {e.response.text[:500]}" + print(f"::error::{error_msg}") + sys.exit(1) + + except Exception as e: + print(f"::error::Unexpected error: {str(e)}") + sys.exit(1) - is_valid, errors = check_for_errors() - if not is_valid: + finally: + if 'zip_path' in locals() and os.path.exists(zip_path): + os.remove(zip_path) + + # Error reporting logic + if not check_for_errors()[0]: + repo_name = os.getenv("GITHUB_REPOSITORY", "").split('/')[-1] if VALIDATION_STEP == "True" and check_if_pull_request(): - pr_number = GITHUB_CONTEXT["event"]["number"] add_pull_request_comment( - GITHUB_AUTH_TOKEN, - os.path.basename(GITHUB_REPOSITORY), - pr_number, - errors, - ) - print( - "\nExiting the {} script after failure\n".format( - os.path.basename(__file__) - ) + os.getenv("GITHUB_AUTH_TOKEN"), + repo_name, + github_context.get("event", {}).get("number"), + os.getenv("CHALLENGE_ERRORS", "Unknown error") ) - sys.exit(1) else: - issue_title = ( - "Following errors occurred while validating the challenge config:" - ) create_github_repository_issue( - GITHUB_AUTH_TOKEN, - os.path.basename(GITHUB_REPOSITORY), - issue_title, - errors, + os.getenv("GITHUB_AUTH_TOKEN"), + repo_name, + "Challenge Configuration Error", + os.getenv("CHALLENGE_ERRORS", "Unknown error") ) - print( - "\nExiting the {} script after failure\n".format( - os.path.basename(__file__) - ) - ) - sys.exit(1) + sys.exit(1) - print("\nExiting the {} script after success\n".format(os.path.basename(__file__))) + print("🎉 Challenge processing completed successfully") \ No newline at end of file From a40dcc85ad1b5a7f69aef45668fc1e9e600043e4 Mon Sep 17 00:00:00 2001 From: Eeshu Date: Sat, 29 Mar 2025 14:40:00 +0000 Subject: [PATCH 13/19] change the script to the original form --- github/challenge_processing_script.py | 182 +++++++++++++++----------- 1 file changed, 108 insertions(+), 74 deletions(-) diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index 2bc3ea409..33b7d31c8 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -8,6 +8,7 @@ from utils import ( add_pull_request_comment, check_for_errors, + check_if_merge_or_commit, check_if_pull_request, create_challenge_zip_file, create_github_repository_issue, @@ -18,93 +19,126 @@ sys.dont_write_bytecode = True -def validate_environment(): - """Validate required environment variables""" - required_vars = { - 'GITHUB_AUTH_TOKEN': lambda v: len(v) > 20, - 'EVALAI_HOST_URL': lambda v: v.startswith(('http://', 'https://')), - 'CHALLENGE_HOST_TEAM_PK': lambda v: v.isdigit(), - } - - for var, validator in required_vars.items(): - value = os.getenv(var) - if not value or not validator(value): - print(f"::error::Invalid {var}: {value}") - sys.exit(1) +GITHUB_CONTEXT = json.loads(os.getenv("GITHUB_CONTEXT")) + +GITHUB_AUTH_TOKEN = os.getenv("GITHUB_AUTH_TOKEN") +if not GITHUB_AUTH_TOKEN: + print( + "Please add your github access token to the repository secrets with the name AUTH_TOKEN" + ) + sys.exit(1) + +HOST_AUTH_TOKEN = None +CHALLENGE_HOST_TEAM_PK = None +EVALAI_HOST_URL = None + if __name__ == "__main__": - try: - # Validate environment variables first - validate_environment() - - # Load GitHub context - github_context = json.loads(os.getenv("GITHUB_CONTEXT", "{}")) - - # Load EvalAI configurations - configs = load_host_configs(HOST_CONFIG_FILE_PATH) - if not configs: - print("::error::Missing EvalAI host configurations") - sys.exit(1) - HOST_AUTH_TOKEN, CHALLENGE_HOST_TEAM_PK, EVALAI_HOST_URL = configs + configs = load_host_configs(HOST_CONFIG_FILE_PATH) + if configs: + HOST_AUTH_TOKEN = configs[0] + CHALLENGE_HOST_TEAM_PK = configs[1] + EVALAI_HOST_URL = configs[2] + else: + sys.exit(1) - # Construct API URL with validation - base_url = EVALAI_HOST_URL.rstrip('/') - if VALIDATION_STEP == "True": - endpoint = CHALLENGE_CONFIG_VALIDATION_URL.format(CHALLENGE_HOST_TEAM_PK) + # Fetching the url + if VALIDATION_STEP == "True": + url = "{}{}".format( + EVALAI_HOST_URL, + CHALLENGE_CONFIG_VALIDATION_URL.format(CHALLENGE_HOST_TEAM_PK), + ) + else: + url = "{}{}".format( + EVALAI_HOST_URL, + CHALLENGE_CREATE_OR_UPDATE_URL.format(CHALLENGE_HOST_TEAM_PK), + ) + + headers = get_request_header(HOST_AUTH_TOKEN) + + # Creating the challenge zip file and storing in a dict to send to EvalAI + create_challenge_zip_file(CHALLENGE_ZIP_FILE_PATH, IGNORE_DIRS, IGNORE_FILES) + zip_file = open(CHALLENGE_ZIP_FILE_PATH, "rb") + file = {"zip_configuration": zip_file} + + data = {"GITHUB_REPOSITORY": GITHUB_REPOSITORY} + + try: + response = requests.post(url, data=data, headers=headers, files=file) + + if ( + response.status_code != http.HTTPStatus.OK + and response.status_code != http.HTTPStatus.CREATED + ): + response.raise_for_status() else: - endpoint = CHALLENGE_CREATE_OR_UPDATE_URL.format(CHALLENGE_HOST_TEAM_PK) - - url = f"{base_url}/{endpoint.lstrip('/')}" - print(f"🔧 Using API endpoint: {url}") - - # Create challenge package - zip_path = create_challenge_zip_file(CHALLENGE_ZIP_FILE_PATH, IGNORE_DIRS, IGNORE_FILES) - - # Send to EvalAI - with open(zip_path, 'rb') as zip_file: - response = requests.post( - url, - headers=get_request_header(HOST_AUTH_TOKEN), - files={'zip_configuration': zip_file}, - data={'GITHUB_REPOSITORY': os.getenv("GITHUB_REPOSITORY")}, - timeout=30 + print("\n" + response.json()["Success"]) + except requests.exceptions.HTTPError as err: + if response.status_code in EVALAI_ERROR_CODES: + is_token_valid = validate_token(response.json()) + if is_token_valid: + error = response.json()["error"] + error_message = "\nFollowing errors occurred while validating the challenge config:\n{}".format( + error + ) + print(error_message) + os.environ["CHALLENGE_ERRORS"] = error_message + else: + print( + "\nFollowing errors occurred while validating the challenge config: {}".format( + err + ) ) - response.raise_for_status() - print(f"✅ Success: {response.json().get('Success', '')}") - - except requests.exceptions.RequestException as e: - error_msg = f"API Request failed: {str(e)}" - if hasattr(e, 'response') and e.response.text: - error_msg += f"\nResponse: {e.response.text[:500]}" - print(f"::error::{error_msg}") - sys.exit(1) - + os.environ["CHALLENGE_ERRORS"] = str(err) except Exception as e: - print(f"::error::Unexpected error: {str(e)}") - sys.exit(1) + if VALIDATION_STEP == "True": + error_message = "\nFollowing errors occurred while validating the challenge config: {}".format( + e + ) + print(error_message) + os.environ["CHALLENGE_ERRORS"] = error_message + else: + error_message = "\nFollowing errors occurred while processing the challenge config: {}".format( + e + ) + print(error_message) + os.environ["CHALLENGE_ERRORS"] = error_message - finally: - if 'zip_path' in locals() and os.path.exists(zip_path): - os.remove(zip_path) + zip_file.close() + os.remove(zip_file.name) - # Error reporting logic - if not check_for_errors()[0]: - repo_name = os.getenv("GITHUB_REPOSITORY", "").split('/')[-1] + is_valid, errors = check_for_errors() + if not is_valid: if VALIDATION_STEP == "True" and check_if_pull_request(): + pr_number = GITHUB_CONTEXT["event"]["number"] add_pull_request_comment( - os.getenv("GITHUB_AUTH_TOKEN"), - repo_name, - github_context.get("event", {}).get("number"), - os.getenv("CHALLENGE_ERRORS", "Unknown error") + GITHUB_AUTH_TOKEN, + os.path.basename(GITHUB_REPOSITORY), + pr_number, + errors, + ) + print( + "\nExiting the {} script after failure\n".format( + os.path.basename(__file__) + ) ) + sys.exit(1) else: + issue_title = ( + "Following errors occurred while validating the challenge config:" + ) create_github_repository_issue( - os.getenv("GITHUB_AUTH_TOKEN"), - repo_name, - "Challenge Configuration Error", - os.getenv("CHALLENGE_ERRORS", "Unknown error") + GITHUB_AUTH_TOKEN, + os.path.basename(GITHUB_REPOSITORY), + issue_title, + errors, ) - sys.exit(1) + print( + "\nExiting the {} script after failure\n".format( + os.path.basename(__file__) + ) + ) + sys.exit(1) - print("🎉 Challenge processing completed successfully") \ No newline at end of file + print("\nExiting the {} script after success\n".format(os.path.basename(__file__))) \ No newline at end of file From 3b3ad4c9f6e48878851aa98c73b63b69def6140b Mon Sep 17 00:00:00 2001 From: Eeshu Date: Sat, 29 Mar 2025 15:25:14 +0000 Subject: [PATCH 14/19] added a new script for challenge config validation --- .github/workflows/ci.yml | 55 +++++++++-------------------- github/validate_challenge_config.py | 40 +++++++++++++++++++++ 2 files changed, 57 insertions(+), 38 deletions(-) create mode 100644 github/validate_challenge_config.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73a729b6e..7e0ff2e9d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: EvalAI Starter CI +name: Challenge Config Validation on: pull_request: @@ -7,43 +7,22 @@ on: branches: [master] jobs: - validation: + validate: runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Checkout code - uses: actions/checkout@v4 + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Install dependencies + run: pip install requests - - name: Validate environment variables - run: | - if [ -z "${{ secrets.EVALAI_HOST_URL }}" ]; then - echo "::error::EVALAI_HOST_URL secret is missing" - exit 1 - fi - if [[ ! "${{ secrets.EVALAI_HOST_URL }}" =~ ^https?:// ]]; then - echo "::error::EVALAI_HOST_URL must start with http:// or https://" - exit 1 - fi - - - name: Set up Python 3.13 - uses: actions/setup-python@v5 - with: - python-version: 3.13 - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install requests pyyaml PyGithub - - - name: Validate challenge configuration using EvalAI API - env: - GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }} - HOST_AUTH_TOKEN: ${{ secrets.EVALAI_AUTH_TOKEN }} - CHALLENGE_HOST_TEAM_PK: ${{ secrets.CHALLENGE_HOST_TEAM_PK }} - EVALAI_HOST_URL: ${{ secrets.EVALAI_HOST_URL }} - GITHUB_CONTEXT: ${{ toJSON(github) }} - run: | - echo "Validating challenge configuration..." - echo "API Endpoint: ${EVALAI_HOST_URL}/api/challenges/challenge/challenge_host_team/${CHALLENGE_HOST_TEAM_PK}/create_or_update_github_challenge/" - python github/challenge_processing_script.py + - name: Validate challenge config + env: + EVALAI_AUTH_TOKEN: ${{ secrets.EVALAI_AUTH_TOKEN }} + CHALLENGE_HOST_TEAM_PK: ${{ secrets.CHALLENGE_HOST_TEAM_PK }} + run: | + python github/validate_challenge_config.py \ No newline at end of file diff --git a/github/validate_challenge_config.py b/github/validate_challenge_config.py new file mode 100644 index 000000000..5abcb37f0 --- /dev/null +++ b/github/validate_challenge_config.py @@ -0,0 +1,40 @@ +import os +import requests +import sys +from config import ( + CHALLENGE_CONFIG_VALIDATION_URL, + API_HOST_URL, +) + +def validate_config(): + try: + host_team_pk = os.getenv("CHALLENGE_HOST_TEAM_PK") + auth_token = os.getenv("EVALAI_AUTH_TOKEN") + + if not host_team_pk or not host_team_pk.isdigit(): + raise ValueError("Invalid CHALLENGE_HOST_TEAM_PK") + + if not auth_token: + raise ValueError("EVALAI_AUTH_TOKEN not set") + + validation_url = f"{API_HOST_URL}{CHALLENGE_CONFIG_VALIDATION_URL.format(host_team_pk)}" + print(f"Validating config at: {validation_url}") + + headers = {"Authorization": f"Token {auth_token}"} + + response = requests.post(validation_url, headers=headers) + + if response.status_code == 200: + print("✅ Challenge config is valid!") + return True + else: + print(f"❌ Validation failed (HTTP {response.status_code}): {response.text}") + return False + + except Exception as e: + print(f"🔥 Validation error: {str(e)}") + return False + +if __name__ == "__main__": + if not validate_config(): + sys.exit(1) \ No newline at end of file From 362e31b0ca88e21d08850128e99469c6a488ccae Mon Sep 17 00:00:00 2001 From: Eeshu Date: Sat, 29 Mar 2025 15:46:44 +0000 Subject: [PATCH 15/19] added a new script for challenge config validation --- github/validate_challenge_config.py | 57 ++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 10 deletions(-) diff --git a/github/validate_challenge_config.py b/github/validate_challenge_config.py index 5abcb37f0..d89e0f22d 100644 --- a/github/validate_challenge_config.py +++ b/github/validate_challenge_config.py @@ -1,38 +1,75 @@ import os import requests import sys +import zipfile from config import ( CHALLENGE_CONFIG_VALIDATION_URL, API_HOST_URL, + IGNORE_DIRS, + IGNORE_FILES, + CHALLENGE_ZIP_FILE_PATH ) +def create_challenge_zip(): + """Create ZIP file of challenge configuration""" + try: + with zipfile.ZipFile(CHALLENGE_ZIP_FILE_PATH, 'w') as zipf: + for root, dirs, files in os.walk('.'): + # Skip ignored directories + dirs[:] = [d for d in dirs if d not in IGNORE_DIRS] + + for file in files: + if file in IGNORE_FILES: + continue + file_path = os.path.join(root, file) + zipf.write(file_path, os.path.relpath(file_path, '.')) + return True + except Exception as e: + print(f"🔥 Failed to create ZIP file: {str(e)}") + return False + def validate_config(): try: + # Validate environment variables host_team_pk = os.getenv("CHALLENGE_HOST_TEAM_PK") auth_token = os.getenv("EVALAI_AUTH_TOKEN") - if not host_team_pk or not host_team_pk.isdigit(): + if not (host_team_pk and host_team_pk.isdigit()): raise ValueError("Invalid CHALLENGE_HOST_TEAM_PK") - if not auth_token: - raise ValueError("EVALAI_AUTH_TOKEN not set") + raise ValueError("Missing EVALAI_AUTH_TOKEN") + + # Create challenge ZIP + if not create_challenge_zip(): + return False + # Build API URL validation_url = f"{API_HOST_URL}{CHALLENGE_CONFIG_VALIDATION_URL.format(host_team_pk)}" - print(f"Validating config at: {validation_url}") + print(f"🔍 Validating at: {validation_url}") + # Prepare request headers = {"Authorization": f"Token {auth_token}"} - response = requests.post(validation_url, headers=headers) - + with open(CHALLENGE_ZIP_FILE_PATH, 'rb') as zip_file: + files = {'zip_configuration': (CHALLENGE_ZIP_FILE_PATH, zip_file)} + response = requests.post(validation_url, headers=headers, files=files) + + # Clean up ZIP file + os.remove(CHALLENGE_ZIP_FILE_PATH) + + # Handle response if response.status_code == 200: - print("✅ Challenge config is valid!") + print("✅ Validation successful!") return True + elif response.status_code == 400: + print(f"❌ Validation errors:\n{response.json().get('error', 'Unknown error')}") else: - print(f"❌ Validation failed (HTTP {response.status_code}): {response.text}") - return False + print(f"⚠️ Unexpected response (HTTP {response.status_code}): {response.text}") + return False + except Exception as e: - print(f"🔥 Validation error: {str(e)}") + print(f"🔥 Validation failed: {str(e)}") return False if __name__ == "__main__": From 83ce8ebadb049df9ed1437aae3242e11a6d2e6a1 Mon Sep 17 00:00:00 2001 From: Eeshu Date: Sat, 29 Mar 2025 17:03:20 +0000 Subject: [PATCH 16/19] added a new script for challenge config validation --- github/validate_challenge_config.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/github/validate_challenge_config.py b/github/validate_challenge_config.py index d89e0f22d..f56800d5e 100644 --- a/github/validate_challenge_config.py +++ b/github/validate_challenge_config.py @@ -15,7 +15,6 @@ def create_challenge_zip(): try: with zipfile.ZipFile(CHALLENGE_ZIP_FILE_PATH, 'w') as zipf: for root, dirs, files in os.walk('.'): - # Skip ignored directories dirs[:] = [d for d in dirs if d not in IGNORE_DIRS] for file in files: @@ -30,7 +29,6 @@ def create_challenge_zip(): def validate_config(): try: - # Validate environment variables host_team_pk = os.getenv("CHALLENGE_HOST_TEAM_PK") auth_token = os.getenv("EVALAI_AUTH_TOKEN") @@ -39,25 +37,20 @@ def validate_config(): if not auth_token: raise ValueError("Missing EVALAI_AUTH_TOKEN") - # Create challenge ZIP if not create_challenge_zip(): return False - # Build API URL validation_url = f"{API_HOST_URL}{CHALLENGE_CONFIG_VALIDATION_URL.format(host_team_pk)}" print(f"🔍 Validating at: {validation_url}") - # Prepare request headers = {"Authorization": f"Token {auth_token}"} with open(CHALLENGE_ZIP_FILE_PATH, 'rb') as zip_file: files = {'zip_configuration': (CHALLENGE_ZIP_FILE_PATH, zip_file)} response = requests.post(validation_url, headers=headers, files=files) - # Clean up ZIP file os.remove(CHALLENGE_ZIP_FILE_PATH) - # Handle response if response.status_code == 200: print("✅ Validation successful!") return True From 6d4cb8aa9d5245edc5f26dd69f266bbc6489cbf4 Mon Sep 17 00:00:00 2001 From: Eeshu Date: Sun, 30 Mar 2025 05:52:42 +0000 Subject: [PATCH 17/19] added a new script for challenge config validation --- github/validate_challenge_config.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/github/validate_challenge_config.py b/github/validate_challenge_config.py index f56800d5e..1905ce927 100644 --- a/github/validate_challenge_config.py +++ b/github/validate_challenge_config.py @@ -5,28 +5,8 @@ from config import ( CHALLENGE_CONFIG_VALIDATION_URL, API_HOST_URL, - IGNORE_DIRS, - IGNORE_FILES, CHALLENGE_ZIP_FILE_PATH ) - -def create_challenge_zip(): - """Create ZIP file of challenge configuration""" - try: - with zipfile.ZipFile(CHALLENGE_ZIP_FILE_PATH, 'w') as zipf: - for root, dirs, files in os.walk('.'): - dirs[:] = [d for d in dirs if d not in IGNORE_DIRS] - - for file in files: - if file in IGNORE_FILES: - continue - file_path = os.path.join(root, file) - zipf.write(file_path, os.path.relpath(file_path, '.')) - return True - except Exception as e: - print(f"🔥 Failed to create ZIP file: {str(e)}") - return False - def validate_config(): try: host_team_pk = os.getenv("CHALLENGE_HOST_TEAM_PK") From 4cc187af5b67a5764aee70d59a2c8ed3f2a8f18f Mon Sep 17 00:00:00 2001 From: Eeshu Date: Sun, 30 Mar 2025 05:58:59 +0000 Subject: [PATCH 18/19] added a new script for challenge config validation --- github/validate_challenge_config.py | 102 ++++++++++++++++++++-------- 1 file changed, 75 insertions(+), 27 deletions(-) diff --git a/github/validate_challenge_config.py b/github/validate_challenge_config.py index 1905ce927..48bddcaf9 100644 --- a/github/validate_challenge_config.py +++ b/github/validate_challenge_config.py @@ -1,49 +1,97 @@ import os import requests import sys -import zipfile +import json from config import ( CHALLENGE_CONFIG_VALIDATION_URL, API_HOST_URL, - CHALLENGE_ZIP_FILE_PATH + IGNORE_DIRS, + IGNORE_FILES, + CHALLENGE_ZIP_FILE_PATH, + GITHUB_REPOSITORY, + EVALAI_ERROR_CODES +) +from utils import ( + add_pull_request_comment, + check_for_errors, + check_if_pull_request, + create_challenge_zip_file, + get_request_header, + validate_token, + load_host_configs ) -def validate_config(): - try: - host_team_pk = os.getenv("CHALLENGE_HOST_TEAM_PK") - auth_token = os.getenv("EVALAI_AUTH_TOKEN") - - if not (host_team_pk and host_team_pk.isdigit()): - raise ValueError("Invalid CHALLENGE_HOST_TEAM_PK") - if not auth_token: - raise ValueError("Missing EVALAI_AUTH_TOKEN") - if not create_challenge_zip(): - return False +sys.dont_write_bytecode = True - validation_url = f"{API_HOST_URL}{CHALLENGE_CONFIG_VALIDATION_URL.format(host_team_pk)}" - print(f"🔍 Validating at: {validation_url}") +def validate_config(): + try: + # Load host configurations + configs = load_host_configs("github/host_config.json") + if not configs: + raise ValueError("Failed to load host configurations") + + HOST_AUTH_TOKEN, CHALLENGE_HOST_TEAM_PK, EVALAI_HOST_URL = configs - headers = {"Authorization": f"Token {auth_token}"} + # Create challenge zip + create_challenge_zip_file(CHALLENGE_ZIP_FILE_PATH, IGNORE_DIRS, IGNORE_FILES) - with open(CHALLENGE_ZIP_FILE_PATH, 'rb') as zip_file: - files = {'zip_configuration': (CHALLENGE_ZIP_FILE_PATH, zip_file)} - response = requests.post(validation_url, headers=headers, files=files) - - os.remove(CHALLENGE_ZIP_FILE_PATH) + # Prepare API request + url = f"{API_HOST_URL}{CHALLENGE_CONFIG_VALIDATION_URL.format(CHALLENGE_HOST_TEAM_PK)}" + headers = get_request_header(HOST_AUTH_TOKEN) + + with open(CHALLENGE_ZIP_FILE_PATH, "rb") as zip_file: + files = {"zip_configuration": zip_file} + data = {"GITHUB_REPOSITORY": GITHUB_REPOSITORY} + + response = requests.post(url, headers=headers, files=files, data=data) + # Handle response if response.status_code == 200: print("✅ Validation successful!") return True - elif response.status_code == 400: - print(f"❌ Validation errors:\n{response.json().get('error', 'Unknown error')}") - else: - print(f"⚠️ Unexpected response (HTTP {response.status_code}): {response.text}") - return False + # Handle known error codes + if response.status_code in EVALAI_ERROR_CODES: + if validate_token(response.json()): + error = response.json().get("error", "Unknown error") + raise requests.exceptions.HTTPError(error) + + response.raise_for_status() + except requests.exceptions.HTTPError as err: + error_message = f"Validation failed: {str(err)}" + if response.status_code == 400: + error_message = f"Configuration errors:\n{response.json().get('error', '')}" + + handle_validation_error(error_message) + return False + except Exception as e: - print(f"🔥 Validation failed: {str(e)}") + handle_validation_error(str(e)) return False + + finally: + if os.path.exists(CHALLENGE_ZIP_FILE_PATH): + os.remove(CHALLENGE_ZIP_FILE_PATH) + +def handle_validation_error(error_message): + """Handle error reporting and notifications""" + print(f"❌ {error_message}") + + # Set environment variable for error tracking + os.environ["CHALLENGE_ERRORS"] = error_message + + # Add PR comment if applicable + if check_if_pull_request(): + GITHUB_CONTEXT = json.loads(os.getenv("GITHUB_CONTEXT", "{}")) + pr_number = GITHUB_CONTEXT.get("event", {}).get("number") + if pr_number: + add_pull_request_comment( + os.getenv("GITHUB_AUTH_TOKEN"), + os.path.basename(GITHUB_REPOSITORY), + pr_number, + error_message + ) if __name__ == "__main__": if not validate_config(): From e551a18f807cf15414e003689b51fc4ea0db8817 Mon Sep 17 00:00:00 2001 From: Eeshu Date: Sun, 30 Mar 2025 06:10:50 +0000 Subject: [PATCH 19/19] done some fixex --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e0ff2e9d..f83525659 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,11 +18,12 @@ jobs: python-version: 3.9 - name: Install dependencies - run: pip install requests + run: pip install requests PyGithub - name: Validate challenge config env: EVALAI_AUTH_TOKEN: ${{ secrets.EVALAI_AUTH_TOKEN }} CHALLENGE_HOST_TEAM_PK: ${{ secrets.CHALLENGE_HOST_TEAM_PK }} + GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }} run: | python github/validate_challenge_config.py \ No newline at end of file