From 206d96a969af15a01fcb5ebc1b0092cedb26c750 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Wed, 25 Jun 2025 18:27:29 +0530 Subject: [PATCH 01/40] adding localhost support (temporary) --- github/challenge_processing_script.py | 42 +++++++++++++++++++++++++-- github/utils.py | 17 +++++++++-- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index b0c88bb98..cd729de6c 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -3,6 +3,7 @@ import os import requests import sys +import urllib3 from config import * from utils import ( @@ -33,6 +34,34 @@ EVALAI_HOST_URL = None +def is_localhost_url(url): + """ + Check if the provided URL is a localhost URL + + Arguments: + url {str}: The URL to check + + Returns: + bool: True if it's a localhost URL, False otherwise + """ + localhost_indicators = [ + "127.0.0.1", + "localhost", + "0.0.0.0" + ] + return any(indicator in url.lower() for indicator in localhost_indicators) + + +def configure_requests_for_localhost(): + """ + Configure requests and urllib3 for localhost development servers + This disables SSL warnings for self-signed certificates commonly used in development + """ + # Disable SSL warnings for localhost development + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + print("INFO: SSL verification disabled for localhost development server") + + if __name__ == "__main__": configs = load_host_configs(HOST_CONFIG_FILE_PATH) @@ -43,6 +72,12 @@ else: sys.exit(1) + # Check if we're using a localhost server and configure accordingly + is_localhost = is_localhost_url(EVALAI_HOST_URL) + if is_localhost: + configure_requests_for_localhost() + print(f"INFO: Using localhost server: {EVALAI_HOST_URL}") + # Fetching the url if VALIDATION_STEP == "True": url = "{}{}".format( @@ -64,8 +99,11 @@ data = {"GITHUB_REPOSITORY": GITHUB_REPOSITORY} + # Configure SSL verification based on whether we're using localhost + verify_ssl = not is_localhost + try: - response = requests.post(url, data=data, headers=headers, files=file) + response = requests.post(url, data=data, headers=headers, files=file, verify=verify_ssl) if ( response.status_code != http.HTTPStatus.OK @@ -141,4 +179,4 @@ ) sys.exit(1) - print("\nExiting the {} script after success\n".format(os.path.basename(__file__))) + print("\nExiting the {} script after success\n".format(os.path.basename(__file__))) \ No newline at end of file diff --git a/github/utils.py b/github/utils.py index 27226e0aa..98f364366 100644 --- a/github/utils.py +++ b/github/utils.py @@ -148,9 +148,22 @@ def load_host_configs(config_path): host_auth_token = data["token"] challenge_host_team_pk = data["team_pk"] evalai_host_url = data["evalai_host_url"] + + # Validate and provide helpful information about URL format + if not evalai_host_url or evalai_host_url == "": + error_message = "\nPlease set a valid evalai_host_url in {}. Examples:\n- Production: https://eval.ai\n- Localhost: http://127.0.0.1:8888".format(config_path) + print(error_message) + os.environ["CHALLENGE_ERRORS"] = error_message + return False + + # Ensure URL doesn't end with trailing slash + if evalai_host_url.endswith('/'): + evalai_host_url = evalai_host_url.rstrip('/') + print("INFO: Removed trailing slash from evalai_host_url") + return [host_auth_token, challenge_host_team_pk, evalai_host_url] else: - error_message = "\nThe host config json file is not present. Please include an auth token, team_pk & evalai_host_url in it: {}".format( + error_message = "\nThe host config json file is not present. Please include an auth token, team_pk & evalai_host_url in it: {}\nFor localhost development, use: http://127.0.0.1:8888".format( config_path ) print(error_message) @@ -182,4 +195,4 @@ def validate_token(response): print(error) os.environ["CHALLENGE_ERRORS"] = error return False - return True + return True \ No newline at end of file From 1c344d2c569b62504ef6c07ae6051fcdc887d7ad Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Wed, 25 Jun 2025 19:12:02 +0530 Subject: [PATCH 02/40] modify workflow for localhost support --- .github/workflows/validate-and-process.yml | 23 ++++++++++++++++++++++ github/challenge_processing_script.py | 17 ++++++++++++++++ github/utils.py | 17 ++-------------- 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 57f6d888d..17595553b 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -14,6 +14,7 @@ jobs: runs-on: ubuntu-latest outputs: is_valid: ${{ steps.validate.outputs.is_valid }} + is_localhost: ${{ steps.validate.outputs.is_localhost }} steps: - name: Checkout code uses: actions/checkout@v3 @@ -22,6 +23,7 @@ jobs: id: validate run: | echo "is_valid=true" >> $GITHUB_OUTPUT + echo "is_localhost=false" >> $GITHUB_OUTPUT echo "" > validation_error.log if ! [ -f "github/host_config.json" ]; then @@ -34,6 +36,15 @@ jobs: TEAM_PK=$(jq -r '.team_pk' github/host_config.json) HOST_URL=$(jq -r '.evalai_host_url' github/host_config.json) + # Check if localhost is being used + if [[ "$HOST_URL" == *"127.0.0.1"* ]] || [[ "$HOST_URL" == *"localhost"* ]] || [[ "$HOST_URL" == *"0.0.0.0"* ]]; then + echo "is_localhost=true" >> $GITHUB_OUTPUT + echo "๐Ÿ  Localhost server detected: $HOST_URL" + echo "โš ๏ธ Note: GitHub Actions cannot connect to your local machine." + echo "โš ๏ธ This workflow will validate the configuration but cannot test server connectivity." + echo "โš ๏ธ Make sure your localhost server is running when testing locally." + fi + if [[ -z "$TOKEN" || "$TOKEN" == "" ]]; then echo "โŒ Invalid or missing token" | tee -a validation_error.log echo "is_valid=false" >> $GITHUB_OUTPUT @@ -86,6 +97,18 @@ jobs: python -m pip install --upgrade pip if [ -f github/requirements.txt ]; then pip install -r github/requirements.txt; fi + - name: Localhost Warning + if: needs.validate-host-config.outputs.is_localhost == 'true' + run: | + echo "๐Ÿ  LOCALHOST DETECTED" + echo "โš ๏ธ You are using a localhost server configuration." + echo "โš ๏ธ GitHub Actions cannot connect to your local machine." + echo "โš ๏ธ The following steps may fail due to connection issues." + echo "โš ๏ธ This is expected behavior for localhost configurations." + echo "" + echo "๐Ÿ“ To test locally, make sure your EvalAI server is running and use:" + echo " python3 github/challenge_processing_script.py" + - name: Validate challenge run: | python3 github/challenge_processing_script.py diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index cd729de6c..2b05a736b 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -112,6 +112,23 @@ def configure_requests_for_localhost(): response.raise_for_status() else: print("\n" + response.json()["Success"]) + except requests.exceptions.ConnectionError as conn_err: + # Handle connection errors specifically for localhost + if is_localhost: + error_message = "\n๐Ÿšจ LOCALHOST SERVER CONNECTION FAILED\n" + error_message += "โŒ Could not connect to your localhost EvalAI server at: {}\n".format(EVALAI_HOST_URL) + error_message += "\n๐Ÿ“‹ Please check the following:\n" + error_message += " 1. Is your EvalAI server running?\n" + error_message += " 2. Is it accessible at {}?\n".format(EVALAI_HOST_URL) + error_message += " 3. Check server logs for any startup errors\n" + error_message += "\n๐Ÿ’ก To start your local server, typically run:\n" + error_message += " python manage.py runserver 0.0.0.0:8888\n" + error_message += "\nOriginal error: {}".format(conn_err) + else: + error_message = "\nConnection failed to EvalAI server: {}".format(conn_err) + + print(error_message) + os.environ["CHALLENGE_ERRORS"] = error_message except requests.exceptions.HTTPError as err: if response.status_code in EVALAI_ERROR_CODES: is_token_valid = validate_token(response.json()) diff --git a/github/utils.py b/github/utils.py index 98f364366..27226e0aa 100644 --- a/github/utils.py +++ b/github/utils.py @@ -148,22 +148,9 @@ def load_host_configs(config_path): host_auth_token = data["token"] challenge_host_team_pk = data["team_pk"] evalai_host_url = data["evalai_host_url"] - - # Validate and provide helpful information about URL format - if not evalai_host_url or evalai_host_url == "": - error_message = "\nPlease set a valid evalai_host_url in {}. Examples:\n- Production: https://eval.ai\n- Localhost: http://127.0.0.1:8888".format(config_path) - print(error_message) - os.environ["CHALLENGE_ERRORS"] = error_message - return False - - # Ensure URL doesn't end with trailing slash - if evalai_host_url.endswith('/'): - evalai_host_url = evalai_host_url.rstrip('/') - print("INFO: Removed trailing slash from evalai_host_url") - return [host_auth_token, challenge_host_team_pk, evalai_host_url] else: - error_message = "\nThe host config json file is not present. Please include an auth token, team_pk & evalai_host_url in it: {}\nFor localhost development, use: http://127.0.0.1:8888".format( + error_message = "\nThe host config json file is not present. Please include an auth token, team_pk & evalai_host_url in it: {}".format( config_path ) print(error_message) @@ -195,4 +182,4 @@ def validate_token(response): print(error) os.environ["CHALLENGE_ERRORS"] = error return False - return True \ No newline at end of file + return True From 215825009f3109b25188901b2051d11bd25747e2 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Wed, 25 Jun 2025 19:25:15 +0530 Subject: [PATCH 03/40] Update workflow --- .github/workflows/validate-and-process.yml | 15 ++++++++- github/challenge_processing_script.py | 36 ++++++++++++++-------- github/host_config.json | 6 ++-- 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 17595553b..2c3ad9469 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -116,12 +116,25 @@ jobs: IS_VALIDATION: 'True' GITHUB_CONTEXT: ${{ toJson(github) }} GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }} + continue-on-error: ${{ needs.validate-host-config.outputs.is_localhost == 'true' }} - name: Create or update challenge run: | python3 github/challenge_processing_script.py - if: ${{ success() }} + if: ${{ success() && needs.validate-host-config.outputs.is_localhost != 'true' }} env: IS_VALIDATION: 'False' GITHUB_CONTEXT: ${{ toJson(github) }} GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }} + + - name: Localhost Success Message + if: needs.validate-host-config.outputs.is_localhost == 'true' + run: | + echo "โœ… LOCALHOST CONFIGURATION VALIDATED" + echo "" + echo "๐Ÿ  Your challenge configuration has been validated for localhost usage." + echo "๐Ÿš€ To test your challenge, make sure your EvalAI server is running locally and execute:" + echo " python3 github/challenge_processing_script.py" + echo "" + echo "๐Ÿ“‹ Server startup command (typical):" + echo " python manage.py runserver 0.0.0.0:8888" diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index 2b05a736b..acd0506f0 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -29,6 +29,9 @@ ) sys.exit(1) +# Clean up the GitHub token (remove any whitespace/newlines) +GITHUB_AUTH_TOKEN = GITHUB_AUTH_TOKEN.strip() + HOST_AUTH_TOKEN = None CHALLENGE_HOST_TEAM_PK = None EVALAI_HOST_URL = None @@ -165,7 +168,21 @@ def configure_requests_for_localhost(): is_valid, errors = check_for_errors() if not is_valid: - if VALIDATION_STEP == "True" and check_if_pull_request(): + # Check if this is a localhost connection error - don't create GitHub issues for expected localhost failures + is_localhost_connection_error = ( + is_localhost and + errors and + ("Connection refused" in errors or "LOCALHOST SERVER CONNECTION FAILED" in errors) + ) + + if is_localhost_connection_error: + print( + "\nโ„น๏ธ Localhost connection error detected. Skipping GitHub issue creation." + ) + print( + "This is expected when your local EvalAI server isn't running." + ) + elif VALIDATION_STEP == "True" and check_if_pull_request(): pr_number = GITHUB_CONTEXT["event"]["number"] add_pull_request_comment( GITHUB_AUTH_TOKEN, @@ -173,12 +190,6 @@ def configure_requests_for_localhost(): pr_number, errors, ) - print( - "\nExiting the {} script after failure\n".format( - os.path.basename(__file__) - ) - ) - sys.exit(1) else: issue_title = ( "Following errors occurred while validating the challenge config:" @@ -189,11 +200,12 @@ def configure_requests_for_localhost(): issue_title, errors, ) - print( - "\nExiting the {} script after failure\n".format( - os.path.basename(__file__) - ) + + print( + "\nExiting the {} script after failure\n".format( + os.path.basename(__file__) ) - sys.exit(1) + ) + sys.exit(1) print("\nExiting the {} script after success\n".format(os.path.basename(__file__))) \ No newline at end of file diff --git a/github/host_config.json b/github/host_config.json index e7ead6158..5448b2973 100644 --- a/github/host_config.json +++ b/github/host_config.json @@ -1,5 +1,5 @@ { - "token": "", - "team_pk": "", - "evalai_host_url": "" + "token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ0b2tlbl90eXBlIjoicmVmcmVzaCIsImV4cCI6MTc4MTg1Njc3MSwianRpIjoiNjJlM2Y4MjFjY2Q3NDI2ZGFkMmVmNjA3YzYxNzI1N2QiLCJ1c2VyX2lkIjozfQ.xqxynI4Zin0NxPo-VsrF-UAv-evPjv82SnYzVnxpCCw", + "team_pk": "2", + "evalai_host_url": "http://0.0.0.0:8888" } From 7f0d929f5bb4fb3916b12065c66dfbc8e9dc3a5d Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Wed, 25 Jun 2025 20:34:42 +0530 Subject: [PATCH 04/40] Update workflow --- .github/workflows/validate-and-process.yml | 44 ++++++++++------------ 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 2c3ad9469..6422b85db 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -80,7 +80,7 @@ jobs: process-evalai-challenge: needs: validate-host-config if: needs.validate-host-config.outputs.is_valid == 'true' - runs-on: ubuntu-latest + runs-on: ${{ needs.validate-host-config.outputs.is_localhost == 'true' && 'self-hosted' || 'ubuntu-latest' }} steps: - name: Checkout challenge branch uses: actions/checkout@v3 @@ -97,17 +97,26 @@ jobs: python -m pip install --upgrade pip if [ -f github/requirements.txt ]; then pip install -r github/requirements.txt; fi - - name: Localhost Warning + - name: Check Docker Compose Services if: needs.validate-host-config.outputs.is_localhost == 'true' run: | - echo "๐Ÿ  LOCALHOST DETECTED" - echo "โš ๏ธ You are using a localhost server configuration." - echo "โš ๏ธ GitHub Actions cannot connect to your local machine." - echo "โš ๏ธ The following steps may fail due to connection issues." - echo "โš ๏ธ This is expected behavior for localhost configurations." - echo "" - echo "๐Ÿ“ To test locally, make sure your EvalAI server is running and use:" - echo " python3 github/challenge_processing_script.py" + echo "๐Ÿ  LOCALHOST DETECTED - Checking Docker Compose Services" + echo "๐Ÿ“‹ Checking if EvalAI server is running on localhost:8888..." + + # Check if the server is accessible + if curl -s --connect-timeout 5 http://localhost:8888/ > /dev/null 2>&1; then + echo "โœ… EvalAI server is accessible at localhost:8888" + else + echo "โŒ EvalAI server is not accessible at localhost:8888" + echo "" + echo "๐Ÿ“ Make sure your EvalAI server is running with docker-compose:" + echo " docker-compose up -d" + echo "" + echo "๐Ÿ” Check running containers:" + docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" | grep -E "(evalai|8888)" || echo "No EvalAI containers found" + echo "" + echo "๐Ÿšจ This workflow will fail if the server is not running" + fi - name: Validate challenge run: | @@ -116,25 +125,12 @@ jobs: IS_VALIDATION: 'True' GITHUB_CONTEXT: ${{ toJson(github) }} GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }} - continue-on-error: ${{ needs.validate-host-config.outputs.is_localhost == 'true' }} - name: Create or update challenge run: | python3 github/challenge_processing_script.py - if: ${{ success() && needs.validate-host-config.outputs.is_localhost != 'true' }} + if: ${{ success() }} env: IS_VALIDATION: 'False' GITHUB_CONTEXT: ${{ toJson(github) }} GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }} - - - name: Localhost Success Message - if: needs.validate-host-config.outputs.is_localhost == 'true' - run: | - echo "โœ… LOCALHOST CONFIGURATION VALIDATED" - echo "" - echo "๐Ÿ  Your challenge configuration has been validated for localhost usage." - echo "๐Ÿš€ To test your challenge, make sure your EvalAI server is running locally and execute:" - echo " python3 github/challenge_processing_script.py" - echo "" - echo "๐Ÿ“‹ Server startup command (typical):" - echo " python manage.py runserver 0.0.0.0:8888" From 01cc9ac8cd73b331f87400a684eb10b031ad8439 Mon Sep 17 00:00:00 2001 From: ZahedR_327 Date: Wed, 25 Jun 2025 20:37:33 +0530 Subject: [PATCH 05/40] Update host_config.json --- github/host_config.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/github/host_config.json b/github/host_config.json index 5448b2973..e7ead6158 100644 --- a/github/host_config.json +++ b/github/host_config.json @@ -1,5 +1,5 @@ { - "token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ0b2tlbl90eXBlIjoicmVmcmVzaCIsImV4cCI6MTc4MTg1Njc3MSwianRpIjoiNjJlM2Y4MjFjY2Q3NDI2ZGFkMmVmNjA3YzYxNzI1N2QiLCJ1c2VyX2lkIjozfQ.xqxynI4Zin0NxPo-VsrF-UAv-evPjv82SnYzVnxpCCw", - "team_pk": "2", - "evalai_host_url": "http://0.0.0.0:8888" + "token": "", + "team_pk": "", + "evalai_host_url": "" } From 3697d43406f824bb8bc79ead01b3160a048170b3 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Fri, 27 Jun 2025 18:47:02 +0530 Subject: [PATCH 06/40] Add localhost support --- .github/workflows/validate-and-process.yml | 210 +++++++++++++++--- README.md | 50 +++++ github/challenge_processing_script.py | 225 ++++++++++++++++--- github/self_hosted_runner_setup.md | 244 +++++++++++++++++++++ 4 files changed, 667 insertions(+), 62 deletions(-) create mode 100644 github/self_hosted_runner_setup.md diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 6422b85db..8fdecbfd6 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -15,6 +15,8 @@ jobs: outputs: is_valid: ${{ steps.validate.outputs.is_valid }} is_localhost: ${{ steps.validate.outputs.is_localhost }} + host_url: ${{ steps.validate.outputs.host_url }} + requires_self_hosted: ${{ steps.validate.outputs.requires_self_hosted }} steps: - name: Checkout code uses: actions/checkout@v3 @@ -24,6 +26,7 @@ jobs: run: | echo "is_valid=true" >> $GITHUB_OUTPUT echo "is_localhost=false" >> $GITHUB_OUTPUT + echo "requires_self_hosted=false" >> $GITHUB_OUTPUT echo "" > validation_error.log if ! [ -f "github/host_config.json" ]; then @@ -36,27 +39,40 @@ jobs: TEAM_PK=$(jq -r '.team_pk' github/host_config.json) HOST_URL=$(jq -r '.evalai_host_url' github/host_config.json) - # Check if localhost is being used + echo "host_url=$HOST_URL" >> $GITHUB_OUTPUT + + # Enhanced localhost detection if [[ "$HOST_URL" == *"127.0.0.1"* ]] || [[ "$HOST_URL" == *"localhost"* ]] || [[ "$HOST_URL" == *"0.0.0.0"* ]]; then echo "is_localhost=true" >> $GITHUB_OUTPUT + echo "requires_self_hosted=true" >> $GITHUB_OUTPUT echo "๐Ÿ  Localhost server detected: $HOST_URL" - echo "โš ๏ธ Note: GitHub Actions cannot connect to your local machine." - echo "โš ๏ธ This workflow will validate the configuration but cannot test server connectivity." - echo "โš ๏ธ Make sure your localhost server is running when testing locally." + echo "๐Ÿค– Self-hosted runner required for local development" + echo "" + echo "๐Ÿ“‹ Requirements for local challenge creation:" + echo " โœ… Self-hosted GitHub Actions runner must be configured" + echo " โœ… EvalAI server must be running at: $HOST_URL" + echo " โœ… Network connectivity between runner and server" + echo "" + echo "โš ๏ธ Note: GitHub hosted runners cannot connect to localhost" + echo "โš ๏ธ This workflow will use your self-hosted runner instead" fi + # Validate required fields if [[ -z "$TOKEN" || "$TOKEN" == "" ]]; then - echo "โŒ Invalid or missing token" | tee -a validation_error.log + echo "โŒ Invalid or missing token in host_config.json" | tee -a validation_error.log + echo "๐Ÿ’ก Please replace with your actual EvalAI token" | tee -a validation_error.log echo "is_valid=false" >> $GITHUB_OUTPUT fi if [[ -z "$TEAM_PK" || "$TEAM_PK" == "" ]]; then - echo "โŒ Invalid or missing team_pk" | tee -a validation_error.log + echo "โŒ Invalid or missing team_pk in host_config.json" | tee -a validation_error.log + echo "๐Ÿ’ก Please replace with your actual team primary key" | tee -a validation_error.log echo "is_valid=false" >> $GITHUB_OUTPUT fi if [[ -z "$HOST_URL" || "$HOST_URL" == "" ]]; then - echo "โŒ Invalid or missing evalai_host_url" | tee -a validation_error.log + echo "โŒ Invalid or missing evalai_host_url in host_config.json" | tee -a validation_error.log + echo "๐Ÿ’ก Please replace with your EvalAI server URL" | tee -a validation_error.log echo "is_valid=false" >> $GITHUB_OUTPUT fi @@ -64,11 +80,12 @@ jobs: if: steps.validate.outputs.is_valid == 'false' uses: peter-evans/create-issue-from-file@v4 with: - title: "host_config.json validation failed" + title: "โŒ host_config.json validation failed" content-filepath: validation_error.log labels: | bug config + self-hosted - name: Fail job if invalid if: steps.validate.outputs.is_valid == 'false' @@ -76,17 +93,61 @@ jobs: echo "โŒ host_config.json validation failed. See issue for details." exit 1 + check-self-hosted-requirements: + needs: validate-host-config + if: needs.validate-host-config.outputs.requires_self_hosted == 'true' + runs-on: ubuntu-latest + steps: + - name: Self-hosted runner requirements check + run: | + echo "๐Ÿ  LOCAL DEVELOPMENT MODE DETECTED" + echo "==================================" + echo "" + echo "๐Ÿ“‹ To proceed with local challenge creation, ensure you have:" + echo "" + echo "1๏ธโƒฃ Self-hosted GitHub Actions runner configured:" + echo " โ€ข Download runner from: https://github.com/${{ github.repository }}/settings/actions/runners" + echo " โ€ข Install and configure on your local machine" + echo " โ€ข Start the runner service" + echo "" + echo "2๏ธโƒฃ EvalAI server running locally:" + echo " โ€ข Server URL: ${{ needs.validate-host-config.outputs.host_url }}" + echo " โ€ข Ensure it's accessible from your machine" + echo " โ€ข Verify API endpoints are responding" + echo "" + echo "3๏ธโƒฃ Network connectivity:" + echo " โ€ข Runner can reach your EvalAI server" + echo " โ€ข No firewall blocking connections" + echo "" + echo "โš ๏ธ This job will continue on your self-hosted runner..." + echo "โš ๏ธ If you don't have a self-hosted runner, the next job will fail" process-evalai-challenge: - needs: validate-host-config - if: needs.validate-host-config.outputs.is_valid == 'true' - runs-on: ${{ needs.validate-host-config.outputs.is_localhost == 'true' && 'self-hosted' || 'ubuntu-latest' }} + needs: [validate-host-config, check-self-hosted-requirements] + if: | + always() && + needs.validate-host-config.outputs.is_valid == 'true' && + (needs.check-self-hosted-requirements.result == 'success' || needs.check-self-hosted-requirements.result == 'skipped') + runs-on: ${{ needs.validate-host-config.outputs.requires_self_hosted == 'true' && 'self-hosted' || 'ubuntu-latest' }} + steps: - name: Checkout challenge branch uses: actions/checkout@v3 with: ref: challenge + - name: Environment Information + run: | + echo "๐Ÿ” ENVIRONMENT INFORMATION" + echo "=========================" + echo "Runner Type: ${{ needs.validate-host-config.outputs.requires_self_hosted == 'true' && 'Self-hosted' || 'GitHub-hosted' }}" + echo "EvalAI URL: ${{ needs.validate-host-config.outputs.host_url }}" + echo "Is Localhost: ${{ needs.validate-host-config.outputs.is_localhost }}" + echo "OS: $(uname -s)" + echo "Architecture: $(uname -m)" + echo "Working Directory: $(pwd)" + echo "" + - name: Set up Python uses: actions/setup-python@v4 with: @@ -95,31 +156,87 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - if [ -f github/requirements.txt ]; then pip install -r github/requirements.txt; fi + if [ -f github/requirements.txt ]; then + echo "๐Ÿ“ฆ Installing dependencies from github/requirements.txt" + pip install -r github/requirements.txt + else + echo "โš ๏ธ No requirements.txt found in github/ directory" + fi - - name: Check Docker Compose Services + - name: Pre-flight Local Server Health Check if: needs.validate-host-config.outputs.is_localhost == 'true' run: | - echo "๐Ÿ  LOCALHOST DETECTED - Checking Docker Compose Services" - echo "๐Ÿ“‹ Checking if EvalAI server is running on localhost:8888..." + echo "๐Ÿฅ LOCAL SERVER HEALTH CHECK" + echo "============================" - # Check if the server is accessible - if curl -s --connect-timeout 5 http://localhost:8888/ > /dev/null 2>&1; then - echo "โœ… EvalAI server is accessible at localhost:8888" + SERVER_URL="${{ needs.validate-host-config.outputs.host_url }}" + echo "๐Ÿ” Checking EvalAI server at: $SERVER_URL" + echo "" + + # Extract host and port from URL + if [[ "$SERVER_URL" =~ http://([^:/]+):([0-9]+) ]]; then + HOST="${BASH_REMATCH[1]}" + PORT="${BASH_REMATCH[2]}" + elif [[ "$SERVER_URL" =~ http://([^:/]+) ]]; then + HOST="${BASH_REMATCH[1]}" + PORT="80" else - echo "โŒ EvalAI server is not accessible at localhost:8888" + echo "โŒ Unable to parse server URL: $SERVER_URL" + exit 1 + fi + + echo "๐ŸŒ Resolved to Host: $HOST, Port: $PORT" + echo "" + + # Test basic connectivity + echo "1๏ธโƒฃ Testing basic connectivity..." + if timeout 10 nc -z "$HOST" "$PORT" 2>/dev/null; then + echo " โœ… Port $PORT is reachable on $HOST" + else + echo " โŒ Port $PORT is NOT reachable on $HOST" + echo " ๐Ÿ’ก Make sure your EvalAI server is running and listening on $HOST:$PORT" + fi + + # Test HTTP endpoint + echo "" + echo "2๏ธโƒฃ Testing HTTP endpoint..." + if curl -s --connect-timeout 10 --max-time 10 "$SERVER_URL/" > /dev/null 2>&1; then + echo " โœ… HTTP endpoint is responding at $SERVER_URL" + + # Test API endpoint + echo "" + echo "3๏ธโƒฃ Testing API endpoint..." + API_URL="$SERVER_URL/api/" + if curl -s --connect-timeout 10 --max-time 10 "$API_URL" > /dev/null 2>&1; then + echo " โœ… API endpoint is accessible at $API_URL" + else + echo " โš ๏ธ API endpoint test failed, but server is responding" + echo " ๐Ÿ’ก This might be normal if API requires authentication" + fi + else + echo " โŒ HTTP endpoint is not responding at $SERVER_URL" echo "" - echo "๐Ÿ“ Make sure your EvalAI server is running with docker-compose:" - echo " docker-compose up -d" + echo "๐Ÿšจ TROUBLESHOOTING STEPS:" + echo " 1. Verify your EvalAI server is running:" + echo " python manage.py runserver 0.0.0.0:8888" + echo " 2. Check if the server is binding to the correct interface" + echo " 3. Verify no firewall is blocking the connection" + echo " 4. Check server logs for errors" echo "" - echo "๐Ÿ” Check running containers:" - docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" | grep -E "(evalai|8888)" || echo "No EvalAI containers found" + echo " If using Docker:" + echo " docker-compose up -d" + echo " docker-compose ps" echo "" - echo "๐Ÿšจ This workflow will fail if the server is not running" + exit 1 fi + + echo "" + echo "โœ… Pre-flight health check completed successfully!" - - name: Validate challenge + - name: Validate challenge configuration run: | + echo "๐Ÿ” VALIDATING CHALLENGE CONFIGURATION" + echo "=====================================" python3 github/challenge_processing_script.py env: IS_VALIDATION: 'True' @@ -127,10 +244,51 @@ jobs: GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }} - name: Create or update challenge + if: success() run: | + echo "๐Ÿš€ CREATING/UPDATING CHALLENGE" + echo "==============================" python3 github/challenge_processing_script.py - if: ${{ success() }} env: IS_VALIDATION: 'False' GITHUB_CONTEXT: ${{ toJson(github) }} GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }} + + - name: Success Summary + if: success() + run: | + echo "" + echo "๐ŸŽ‰ CHALLENGE PROCESSING COMPLETED SUCCESSFULLY!" + echo "==============================================" + echo "" + if [[ "${{ needs.validate-host-config.outputs.is_localhost }}" == "true" ]]; then + echo "โœ… Local development mode: Challenge processed on self-hosted runner" + echo "โœ… EvalAI server: ${{ needs.validate-host-config.outputs.host_url }}" + echo "โœ… Self-hosted runner successfully connected to local server" + else + echo "โœ… Production mode: Challenge processed on GitHub-hosted runner" + echo "โœ… EvalAI server: ${{ needs.validate-host-config.outputs.host_url }}" + fi + echo "" + echo "๐Ÿ” Check your EvalAI instance for the updated challenge configuration" + + - name: Failure Summary + if: failure() + run: | + echo "" + echo "โŒ CHALLENGE PROCESSING FAILED" + echo "==============================" + echo "" + if [[ "${{ needs.validate-host-config.outputs.is_localhost }}" == "true" ]]; then + echo "๐Ÿ  Local development mode detected" + echo "๐Ÿ’ก Common issues for self-hosted runners:" + echo " โ€ข EvalAI server not running or not accessible" + echo " โ€ข Self-hosted runner not properly configured" + echo " โ€ข Network connectivity issues between runner and server" + echo " โ€ข Invalid authentication credentials" + echo "" + echo "๐Ÿ” Check the logs above for specific error details" + else + echo "โ˜๏ธ GitHub-hosted runner mode" + echo "๐Ÿ” Check the logs above for specific error details" + fi diff --git a/README.md b/README.md index 458cc8fd9..10c6ff364 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,49 @@ In order to test the evaluation script locally before uploading it to [EvalAI](h 3. Run the command `python -m worker.run` from the directory where `annotations/` `challenge_data/` and `worker/` directories are present. If the command runs successfully, then the evaluation script works locally and will work on the server as well. +## Local Development with Self-Hosted Runners + +For local EvalAI challenge development, this repository supports **self-hosted GitHub Actions runners** that can connect to your localhost EvalAI server. + +### When to use self-hosted runners: +- โœ… Developing with a local EvalAI server (`http://localhost:8888`) +- โœ… Testing challenges before deploying to production +- โœ… Rapid iteration during development +- โœ… Working with custom EvalAI configurations + +### Quick Setup: + +1. **Configure localhost in host_config.json:** + ```json + { + "token": "your_evalai_auth_token", + "team_pk": "your_team_primary_key", + "evalai_host_url": "http://localhost:8888" + } + ``` + +2. **Set up self-hosted runner:** + - Go to your repository โ†’ Settings โ†’ Actions โ†’ Runners + - Click "New self-hosted runner" + - Follow the setup instructions for your OS + - Start your EvalAI server: `python manage.py runserver 0.0.0.0:8888` + +3. **Test your setup:** + ```bash + python3 github/test_local_setup.py + ``` + +4. **Push to challenge branch:** + The workflow will automatically detect localhost and use your self-hosted runner! + +### For detailed setup instructions: +๐Ÿ“š See [Self-Hosted Runner Setup Guide](./github/self_hosted_runner_setup.md) + +### Troubleshooting: +- **Connection refused**: Make sure your EvalAI server is running and accessible +- **Wrong runner used**: Verify `host_config.json` contains localhost URL +- **Permission errors**: Check runner has appropriate file system access + ## Important Note `host_config.json` file includes default placeholders like: @@ -83,6 +126,13 @@ In order to test the evaluation script locally before uploading it to [EvalAI](h Please replace them with real values before pushing changes to avoid build errors. +**For localhost development**: Use `http://localhost:8888` or `http://127.0.0.1:8888` as the `evalai_host_url` and ensure you have a self-hosted runner configured. + ## Facing problems in creating a challenge? Please feel free to open issues on our [GitHub Repository](https://github.com/Cloud-CV/EvalAI-Starter/issues) or contact us at team@cloudcv.org if you have issues. + +For **self-hosted runner issues**, include: +- Runner logs from `_diag` folder +- Output from `python3 github/test_local_setup.py` +- Your `host_config.json` configuration (without sensitive tokens) diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index acd0506f0..20c1d5254 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -4,6 +4,9 @@ import requests import sys import urllib3 +import socket +import platform +from urllib.parse import urlparse from config import * from utils import ( @@ -55,6 +58,79 @@ def is_localhost_url(url): return any(indicator in url.lower() for indicator in localhost_indicators) +def get_runner_info(): + """ + Get information about the current runner environment + + Returns: + dict: Information about the runner + """ + runner_info = { + "is_github_actions": bool(os.getenv("GITHUB_ACTIONS")), + "runner_name": os.getenv("RUNNER_NAME", "unknown"), + "runner_os": os.getenv("RUNNER_OS", platform.system()), + "runner_arch": os.getenv("RUNNER_ARCH", platform.machine()), + "is_self_hosted": os.getenv("RUNNER_ENVIRONMENT") != "github-hosted", + "hostname": socket.gethostname(), + "platform": platform.platform(), + } + return runner_info + + +def test_server_connectivity(url, timeout=10): + """ + Test connectivity to the EvalAI server + + Arguments: + url {str}: The server URL to test + timeout {int}: Connection timeout in seconds + + Returns: + dict: Test results with status and details + """ + result = { + "success": False, + "details": [], + "error": None + } + + try: + parsed_url = urlparse(url) + host = parsed_url.hostname + port = parsed_url.port or (443 if parsed_url.scheme == 'https' else 80) + + result["details"].append(f"Testing connectivity to {host}:{port}") + + # Test socket connection + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(timeout) + socket_result = sock.connect_ex((host, port)) + sock.close() + + if socket_result == 0: + result["details"].append(f"โœ… Socket connection successful to {host}:{port}") + + # Test HTTP request + try: + response = requests.get(url, timeout=timeout, verify=not is_localhost_url(url)) + result["details"].append(f"โœ… HTTP request successful (Status: {response.status_code})") + result["success"] = True + except requests.exceptions.RequestException as e: + result["details"].append(f"โš ๏ธ HTTP request failed: {e}") + result["success"] = False # Still consider it a partial success since socket worked + + else: + result["details"].append(f"โŒ Socket connection failed to {host}:{port}") + result["success"] = False + + except Exception as e: + result["error"] = str(e) + result["details"].append(f"โŒ Connectivity test failed: {e}") + result["success"] = False + + return result + + def configure_requests_for_localhost(): """ Configure requests and urllib3 for localhost development servers @@ -65,7 +141,29 @@ def configure_requests_for_localhost(): print("INFO: SSL verification disabled for localhost development server") +def print_environment_info(): + """ + Print detailed information about the current environment + """ + runner_info = get_runner_info() + + print("\n๐Ÿ” ENVIRONMENT INFORMATION") + print("=" * 50) + print(f"GitHub Actions: {runner_info['is_github_actions']}") + print(f"Runner Type: {'Self-hosted' if runner_info['is_self_hosted'] else 'GitHub-hosted'}") + print(f"Runner Name: {runner_info['runner_name']}") + print(f"Operating System: {runner_info['runner_os']}") + print(f"Architecture: {runner_info['runner_arch']}") + print(f"Hostname: {runner_info['hostname']}") + print(f"Platform: {runner_info['platform']}") + print(f"Working Directory: {os.getcwd()}") + print(f"Python Version: {sys.version}") + print("=" * 50) + + if __name__ == "__main__": + + print_environment_info() configs = load_host_configs(HOST_CONFIG_FILE_PATH) if configs: @@ -77,25 +175,68 @@ def configure_requests_for_localhost(): # Check if we're using a localhost server and configure accordingly is_localhost = is_localhost_url(EVALAI_HOST_URL) + runner_info = get_runner_info() + + print(f"\n๐ŸŒ EvalAI Server: {EVALAI_HOST_URL}") + print(f"๐Ÿ  Localhost Mode: {is_localhost}") + print(f"๐Ÿค– Self-hosted Runner: {runner_info['is_self_hosted']}") + if is_localhost: configure_requests_for_localhost() print(f"INFO: Using localhost server: {EVALAI_HOST_URL}") + + # For localhost, perform connectivity test + print(f"\n๐Ÿ” Testing connectivity to localhost server...") + connectivity_test = test_server_connectivity(EVALAI_HOST_URL) + + for detail in connectivity_test["details"]: + print(f" {detail}") + + if not connectivity_test["success"]: + error_message = f"\n๐Ÿšจ LOCALHOST SERVER CONNECTIVITY FAILED\n" + error_message += f"โŒ Cannot reach EvalAI server at: {EVALAI_HOST_URL}\n\n" + error_message += "๐Ÿ“‹ Troubleshooting steps:\n" + error_message += " 1. Ensure your EvalAI server is running\n" + error_message += " 2. Verify the server is listening on the correct interface and port\n" + error_message += " 3. Check for firewall or network restrictions\n" + error_message += " 4. Confirm the URL in host_config.json is correct\n\n" + + if runner_info['is_self_hosted']: + error_message += "๐Ÿ’ก Self-hosted runner tips:\n" + error_message += " โ€ข Make sure the server is accessible from your runner machine\n" + error_message += " โ€ข Test manually: curl -v " + EVALAI_HOST_URL + "\n" + else: + error_message += "โš ๏ธ You're using a GitHub-hosted runner with localhost URL\n" + error_message += " This will not work. Please use a self-hosted runner for localhost development.\n" + + print(error_message) + os.environ["CHALLENGE_ERRORS"] = error_message + + # Don't exit immediately for localhost connectivity issues in validation mode + # Let the request attempt provide more specific error information + else: + print("โœ… Localhost server connectivity test passed!") # Fetching the url if VALIDATION_STEP == "True": + print(f"\n๐Ÿ” VALIDATION MODE: Validating challenge configuration...") url = "{}{}".format( EVALAI_HOST_URL, CHALLENGE_CONFIG_VALIDATION_URL.format(CHALLENGE_HOST_TEAM_PK), ) else: + print(f"\n๐Ÿš€ CREATION MODE: Creating/updating challenge...") url = "{}{}".format( EVALAI_HOST_URL, CHALLENGE_CREATE_OR_UPDATE_URL.format(CHALLENGE_HOST_TEAM_PK), ) + print(f"๐Ÿ“ก API Endpoint: {url}") + headers = get_request_header(HOST_AUTH_TOKEN) # Creating the challenge zip file and storing in a dict to send to EvalAI + print(f"\n๐Ÿ“ฆ Creating challenge configuration package...") create_challenge_zip_file(CHALLENGE_ZIP_FILE_PATH, IGNORE_DIRS, IGNORE_FILES) zip_file = open(CHALLENGE_ZIP_FILE_PATH, "rb") file = {"zip_configuration": zip_file} @@ -104,8 +245,10 @@ def configure_requests_for_localhost(): # Configure SSL verification based on whether we're using localhost verify_ssl = not is_localhost + print(f"๐Ÿ”’ SSL Verification: {'Disabled (localhost)' if not verify_ssl else 'Enabled'}") try: + print(f"\n๐ŸŒ Sending request to EvalAI server...") response = requests.post(url, data=data, headers=headers, files=file, verify=verify_ssl) if ( @@ -114,52 +257,59 @@ def configure_requests_for_localhost(): ): response.raise_for_status() else: - print("\n" + response.json()["Success"]) + success_message = response.json().get("Success", "Operation completed successfully") + print(f"\nโœ… SUCCESS: {success_message}") + except requests.exceptions.ConnectionError as conn_err: # Handle connection errors specifically for localhost if is_localhost: error_message = "\n๐Ÿšจ LOCALHOST SERVER CONNECTION FAILED\n" - error_message += "โŒ Could not connect to your localhost EvalAI server at: {}\n".format(EVALAI_HOST_URL) + error_message += f"โŒ Could not connect to your localhost EvalAI server at: {EVALAI_HOST_URL}\n" error_message += "\n๐Ÿ“‹ Please check the following:\n" error_message += " 1. Is your EvalAI server running?\n" - error_message += " 2. Is it accessible at {}?\n".format(EVALAI_HOST_URL) + error_message += f" 2. Is it accessible at {EVALAI_HOST_URL}?\n" error_message += " 3. Check server logs for any startup errors\n" + + if runner_info['is_self_hosted']: + error_message += "\n๐Ÿ’ก Self-hosted runner troubleshooting:\n" + error_message += " โ€ข Verify runner can reach the server: ping/curl test\n" + error_message += " โ€ข Check network configuration and firewall settings\n" + error_message += " โ€ข Ensure server is binding to correct interface (0.0.0.0 vs 127.0.0.1)\n" + else: + error_message += "\nโš ๏ธ CONFIGURATION ISSUE:\n" + error_message += " You're using a GitHub-hosted runner with a localhost URL.\n" + error_message += " GitHub-hosted runners cannot access your local machine.\n" + error_message += " Please set up a self-hosted runner for localhost development.\n" + error_message += "\n๐Ÿ’ก To start your local server, typically run:\n" error_message += " python manage.py runserver 0.0.0.0:8888\n" - error_message += "\nOriginal error: {}".format(conn_err) + error_message += f"\nOriginal error: {conn_err}" else: - error_message = "\nConnection failed to EvalAI server: {}".format(conn_err) + error_message = f"\nConnection failed to EvalAI server: {conn_err}" print(error_message) os.environ["CHALLENGE_ERRORS"] = error_message + except requests.exceptions.HTTPError as err: if response.status_code in EVALAI_ERROR_CODES: is_token_valid = validate_token(response.json()) if is_token_valid: error = response.json()["error"] - error_message = "\nFollowing errors occurred while validating the challenge config:\n{}".format( - error - ) + error_message = f"\nFollowing errors occurred while {'validating' if VALIDATION_STEP == 'True' else 'processing'} the challenge config:\n{error}" print(error_message) os.environ["CHALLENGE_ERRORS"] = error_message else: - print( - "\nFollowing errors occurred while validating the challenge config: {}".format( - err - ) - ) - os.environ["CHALLENGE_ERRORS"] = str(err) + error_message = f"\nHTTP Error occurred while {'validating' if VALIDATION_STEP == 'True' else 'processing'} the challenge config: {err}" + print(error_message) + os.environ["CHALLENGE_ERRORS"] = error_message + except Exception as e: if VALIDATION_STEP == "True": - error_message = "\nFollowing errors occurred while validating the challenge config: {}".format( - e - ) + error_message = f"\nFollowing errors occurred while validating the challenge config: {e}" print(error_message) os.environ["CHALLENGE_ERRORS"] = error_message else: - error_message = "\nFollowing errors occurred while processing the challenge config: {}".format( - e - ) + error_message = f"\nFollowing errors occurred while processing the challenge config: {e}" print(error_message) os.environ["CHALLENGE_ERRORS"] = error_message @@ -175,13 +325,22 @@ def configure_requests_for_localhost(): ("Connection refused" in errors or "LOCALHOST SERVER CONNECTION FAILED" in errors) ) - if is_localhost_connection_error: - print( - "\nโ„น๏ธ Localhost connection error detected. Skipping GitHub issue creation." - ) - print( - "This is expected when your local EvalAI server isn't running." - ) + # Also check if this is a GitHub-hosted runner trying to access localhost + is_github_hosted_localhost_error = ( + is_localhost and + not runner_info['is_self_hosted'] and + errors and + "Connection" in errors + ) + + if is_localhost_connection_error or is_github_hosted_localhost_error: + print("\nโ„น๏ธ Localhost connection error detected. Skipping GitHub issue creation.") + if is_github_hosted_localhost_error: + print(" This is expected when using GitHub-hosted runners with localhost URLs.") + print(" Please configure a self-hosted runner for local development.") + else: + print(" This is expected when your local EvalAI server isn't running.") + elif VALIDATION_STEP == "True" and check_if_pull_request(): pr_number = GITHUB_CONTEXT["event"]["number"] add_pull_request_comment( @@ -191,9 +350,7 @@ def configure_requests_for_localhost(): errors, ) else: - issue_title = ( - "Following errors occurred while validating the challenge config:" - ) + issue_title = f"Following errors occurred while {'validating' if VALIDATION_STEP == 'True' else 'processing'} the challenge config:" create_github_repository_issue( GITHUB_AUTH_TOKEN, os.path.basename(GITHUB_REPOSITORY), @@ -201,11 +358,7 @@ def configure_requests_for_localhost(): errors, ) - print( - "\nExiting the {} script after failure\n".format( - os.path.basename(__file__) - ) - ) + print(f"\nExiting the {os.path.basename(__file__)} script after failure\n") sys.exit(1) - print("\nExiting the {} script after success\n".format(os.path.basename(__file__))) \ No newline at end of file + print(f"\nโœ… Exiting the {os.path.basename(__file__)} script after success\n") \ No newline at end of file diff --git a/github/self_hosted_runner_setup.md b/github/self_hosted_runner_setup.md new file mode 100644 index 000000000..a63134dbf --- /dev/null +++ b/github/self_hosted_runner_setup.md @@ -0,0 +1,244 @@ +# Self-Hosted Runner Setup Guide + +This guide will help you set up a GitHub Actions self-hosted runner for local EvalAI challenge development. + +## Overview + +When developing EvalAI challenges locally, GitHub's hosted runners cannot access your localhost server. Self-hosted runners solve this by running the GitHub Actions workflow directly on your local machine, giving the workflow access to your local EvalAI server. + +## Prerequisites + +- Local EvalAI server running and accessible +- GitHub repository with appropriate permissions +- Local machine with internet connectivity + +## Step 1: Download and Configure the Runner + +### 1.1 Navigate to Runner Settings +1. Go to your GitHub repository +2. Click on **Settings** tab +3. Click on **Actions** in the left sidebar +4. Click on **Runners** +5. Click **New self-hosted runner** + +### 1.2 Download the Runner +Follow the download instructions for your operating system: + +**Linux/macOS:** +```bash +# Create a folder +mkdir actions-runner && cd actions-runner + +# Download the latest runner package +curl -o actions-runner-linux-x64-2.311.0.tar.gz -L https://github.com/actions/runner/releases/download/v2.311.0/actions-runner-linux-x64-2.311.0.tar.gz + +# Extract the installer +tar xzf ./actions-runner-linux-x64-2.311.0.tar.gz +``` + +**Windows:** +```powershell +# Create a folder under the drive root +mkdir \actions-runner ; cd \actions-runner + +# Download the latest runner package +Invoke-WebRequest -Uri https://github.com/actions/runner/releases/download/v2.311.0/actions-runner-win-x64-2.311.0.zip -OutFile actions-runner-win-x64-2.311.0.zip + +# Extract the installer +Add-Type -AssemblyName System.IO.Compression.FileSystem ; [System.IO.Compression.ZipFile]::ExtractToDirectory("$PWD\actions-runner-win-x64-2.311.0.zip", "$PWD") +``` + +### 1.3 Configure the Runner +```bash +# Configure the runner (follow the prompts) +./config.sh --url https://github.com/YOUR_USERNAME/YOUR_REPO --token YOUR_TOKEN + +# When prompted for runner group, press Enter for default +# When prompted for runner name, you can use: localhost-runner +# When prompted for labels, add: self-hosted,localhost-dev +# When prompted for work folder, press Enter for default +``` + +## Step 2: Start the Runner + +### 2.1 Run the Runner +```bash +# Start the runner +./run.sh +``` + +The runner will now listen for jobs from GitHub Actions. + +### 2.2 Run as a Service (Optional) +For persistent operation, you can install the runner as a service: + +**Linux/macOS:** +```bash +sudo ./svc.sh install +sudo ./svc.sh start +``` + +**Windows (run as Administrator):** +```powershell +.\svc.sh install +.\svc.sh start +``` + +## Step 3: Configure Your EvalAI Server + +### 3.1 Update host_config.json +Make sure your `github/host_config.json` is configured for localhost: + +```json +{ + "token": "your_evalai_auth_token", + "team_pk": "your_team_primary_key", + "evalai_host_url": "http://localhost:8888" +} +``` + +### 3.2 Start Your EvalAI Server +Ensure your local EvalAI server is running and accessible: + +```bash +# If using Django development server +python manage.py runserver 0.0.0.0:8888 + +# If using Docker +docker-compose up -d + +# Verify server is running +curl http://localhost:8888/api/ +``` + +**Important:** Make sure your server binds to `0.0.0.0` (all interfaces) rather than `127.0.0.1` (localhost only), especially if your runner is on a different machine or container. + +## Step 4: Test the Setup + +### 4.1 Trigger a Workflow +1. Push to the `challenge` branch or create a pull request +2. Go to the **Actions** tab in your repository +3. You should see the workflow running on your self-hosted runner + +### 4.2 Monitor the Workflow +The workflow will: +1. Detect localhost configuration +2. Use your self-hosted runner +3. Perform health checks on your local server +4. Process the challenge configuration + +## Troubleshooting + +### Runner Not Appearing +- Check if the runner process is still running +- Verify the registration token hasn't expired +- Check network connectivity to GitHub + +### Server Connection Issues +```bash +# Test server connectivity manually +curl -v http://localhost:8888/ +curl -v http://localhost:8888/api/ + +# Check if server is listening on correct interface +netstat -tlnp | grep 8888 +ss -tlnp | grep 8888 +``` + +### Workflow Still Using GitHub-Hosted Runner +- Verify your `host_config.json` contains a localhost URL +- Check that the runner has the correct labels +- Ensure the runner is online and available + +### Permission Issues +- Make sure the runner has appropriate file system permissions +- On Linux/macOS, avoid running as root if possible +- Check that the runner can access network resources + +## Security Considerations + +### Local Development Only +- Only use self-hosted runners for development/testing +- Don't use self-hosted runners for production workloads +- Be cautious about what code you run on your local machine + +### Network Security +- Ensure your local EvalAI server is not exposed to the internet +- Use firewall rules to restrict access if needed +- Monitor runner logs for unexpected activity + +### Token Management +- Keep your GitHub and EvalAI tokens secure +- Rotate tokens regularly +- Don't commit tokens to version control + +## Advanced Configuration + +### Multiple Runners +You can set up multiple runners for load balancing: +```bash +# Configure additional runners with different names +./config.sh --url https://github.com/YOUR_USERNAME/YOUR_REPO --token YOUR_TOKEN --name localhost-runner-2 +``` + +### Custom Labels +Add custom labels to target specific runners: +```bash +# During configuration, add custom labels +# Example: self-hosted,localhost-dev,linux,x64,gpu +``` + +Then target them in your workflow: +```yaml +runs-on: [self-hosted, localhost-dev, gpu] +``` + +### Environment Variables +Set environment variables for the runner: +```bash +# Create .env file in runner directory +echo "EVALAI_DEBUG=true" >> .env +echo "CUSTOM_CONFIG_PATH=/path/to/config" >> .env +``` + +## Best Practices + +1. **Keep Runners Updated**: Regularly update your self-hosted runners +2. **Monitor Resources**: Keep an eye on CPU, memory, and disk usage +3. **Clean Workspace**: Periodically clean the runner's work directory +4. **Log Management**: Monitor and rotate runner logs +5. **Backup Configuration**: Keep a backup of your runner configuration + +## Getting Help + +If you encounter issues: + +1. Check the runner logs in the `_diag` folder +2. Review the GitHub Actions workflow logs +3. Test EvalAI server connectivity manually +4. Check GitHub's self-hosted runner documentation +5. Open an issue in your repository with detailed error information + +## Example Complete Setup Script + +```bash +#!/bin/bash +# complete-setup.sh - Automated setup script + +set -e + +echo "๐Ÿš€ Setting up self-hosted runner for EvalAI local development" + +# Download and extract runner +mkdir -p actions-runner && cd actions-runner +curl -o actions-runner-linux-x64-2.311.0.tar.gz -L https://github.com/actions/runner/releases/download/v2.311.0/actions-runner-linux-x64-2.311.0.tar.gz +tar xzf ./actions-runner-linux-x64-2.311.0.tar.gz + +echo "๐Ÿ“ Please run the following command to configure your runner:" +echo "./config.sh --url https://github.com/YOUR_USERNAME/YOUR_REPO --token YOUR_TOKEN --name localhost-runner --labels self-hosted,localhost-dev" +echo "" +echo "Then start the runner with:" +echo "./run.sh" +``` + +This setup enables seamless local EvalAI challenge development with GitHub Actions! \ No newline at end of file From 4367b70e8563888bda43ef5c6371f7f6ae8bbe69 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Sun, 29 Jun 2025 19:58:40 +0530 Subject: [PATCH 07/40] add containerisation to workflow --- .github/workflows/validate-and-process.yml | 109 ++++++++++++++++----- 1 file changed, 85 insertions(+), 24 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 8fdecbfd6..14c6fb00a 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -110,17 +110,23 @@ jobs: echo " โ€ข Install and configure on your local machine" echo " โ€ข Start the runner service" echo "" - echo "2๏ธโƒฃ EvalAI server running locally:" + echo "2๏ธโƒฃ Docker installed and running:" + echo " โ€ข Docker Engine must be available on the runner machine" + echo " โ€ข Runner will use python:3.9-slim container (architecture independent)" + echo " โ€ข This solves compatibility issues with ARM64/macOS architectures" + echo "" + echo "3๏ธโƒฃ EvalAI server running locally:" echo " โ€ข Server URL: ${{ needs.validate-host-config.outputs.host_url }}" - echo " โ€ข Ensure it's accessible from your machine" + echo " โ€ข Ensure it's accessible from Docker containers (use host networking)" echo " โ€ข Verify API endpoints are responding" echo "" - echo "3๏ธโƒฃ Network connectivity:" - echo " โ€ข Runner can reach your EvalAI server" + echo "4๏ธโƒฃ Network connectivity:" + echo " โ€ข Docker container can reach your EvalAI server" echo " โ€ข No firewall blocking connections" + echo " โ€ข Consider using host.docker.internal for macOS/Windows" echo "" - echo "โš ๏ธ This job will continue on your self-hosted runner..." - echo "โš ๏ธ If you don't have a self-hosted runner, the next job will fail" + echo "โš ๏ธ This job will continue on your self-hosted runner with Docker..." + echo "โš ๏ธ If you don't have a self-hosted runner with Docker, the next job will fail" process-evalai-challenge: needs: [validate-host-config, check-self-hosted-requirements] @@ -130,6 +136,9 @@ jobs: (needs.check-self-hosted-requirements.result == 'success' || needs.check-self-hosted-requirements.result == 'skipped') runs-on: ${{ needs.validate-host-config.outputs.requires_self_hosted == 'true' && 'self-hosted' || 'ubuntu-latest' }} + # Use Docker container for self-hosted runners to ensure architecture independence + container: ${{ needs.validate-host-config.outputs.requires_self_hosted == 'true' && 'python:3.9-slim' || null }} + steps: - name: Checkout challenge branch uses: actions/checkout@v3 @@ -140,19 +149,30 @@ jobs: run: | echo "๐Ÿ” ENVIRONMENT INFORMATION" echo "=========================" - echo "Runner Type: ${{ needs.validate-host-config.outputs.requires_self_hosted == 'true' && 'Self-hosted' || 'GitHub-hosted' }}" + echo "Runner Type: ${{ needs.validate-host-config.outputs.requires_self_hosted == 'true' && 'Self-hosted (Docker)' || 'GitHub-hosted' }}" echo "EvalAI URL: ${{ needs.validate-host-config.outputs.host_url }}" echo "Is Localhost: ${{ needs.validate-host-config.outputs.is_localhost }}" echo "OS: $(uname -s)" echo "Architecture: $(uname -m)" echo "Working Directory: $(pwd)" + echo "Python Version: $(python3 --version)" + echo "Container: ${{ needs.validate-host-config.outputs.requires_self_hosted == 'true' && 'python:3.9-slim' || 'None' }}" echo "" - - name: Set up Python + - name: Set up Python (GitHub-hosted only) + if: needs.validate-host-config.outputs.requires_self_hosted != 'true' uses: actions/setup-python@v4 with: python-version: 3.9.21 + - name: Install system dependencies (Docker container only) + if: needs.validate-host-config.outputs.requires_self_hosted == 'true' + run: | + echo "๐Ÿ“ฆ Installing system dependencies in Docker container" + apt-get update + apt-get install -y curl netcat-traditional jq git + echo "โœ… System dependencies installed" + - name: Install dependencies run: | python -m pip install --upgrade pip @@ -166,11 +186,12 @@ jobs: - name: Pre-flight Local Server Health Check if: needs.validate-host-config.outputs.is_localhost == 'true' run: | - echo "๐Ÿฅ LOCAL SERVER HEALTH CHECK" - echo "============================" + echo "๐Ÿฅ LOCAL SERVER HEALTH CHECK (Docker Container)" + echo "===============================================" SERVER_URL="${{ needs.validate-host-config.outputs.host_url }}" echo "๐Ÿ” Checking EvalAI server at: $SERVER_URL" + echo "๐Ÿณ Running in Docker container - checking host networking" echo "" # Extract host and port from URL @@ -186,15 +207,52 @@ jobs: fi echo "๐ŸŒ Resolved to Host: $HOST, Port: $PORT" + + # Try different host variations for Docker networking + HOSTS_TO_TRY=("$HOST") + if [[ "$HOST" == "localhost" || "$HOST" == "127.0.0.1" ]]; then + HOSTS_TO_TRY+=("host.docker.internal" "172.17.0.1") + echo "๐Ÿ”„ Will try multiple host variants for Docker networking:" + printf ' โ€ข %s\n' "${HOSTS_TO_TRY[@]}" + fi echo "" - # Test basic connectivity + # Test basic connectivity with multiple host variants echo "1๏ธโƒฃ Testing basic connectivity..." - if timeout 10 nc -z "$HOST" "$PORT" 2>/dev/null; then - echo " โœ… Port $PORT is reachable on $HOST" - else - echo " โŒ Port $PORT is NOT reachable on $HOST" - echo " ๐Ÿ’ก Make sure your EvalAI server is running and listening on $HOST:$PORT" + CONNECTIVITY_SUCCESS=false + WORKING_HOST="" + + for TEST_HOST in "${HOSTS_TO_TRY[@]}"; do + echo " ๐Ÿ” Testing connection to $TEST_HOST:$PORT..." + if timeout 10 nc -z "$TEST_HOST" "$PORT" 2>/dev/null; then + echo " โœ… Port $PORT is reachable on $TEST_HOST" + CONNECTIVITY_SUCCESS=true + WORKING_HOST="$TEST_HOST" + break + else + echo " โŒ Port $PORT is NOT reachable on $TEST_HOST" + fi + done + + if [[ "$CONNECTIVITY_SUCCESS" != "true" ]]; then + echo " ๐Ÿ’ฅ Unable to connect to EvalAI server on any host variant" + echo " ๐Ÿ’ก Make sure your EvalAI server is running and accessible from Docker" + echo "" + echo "๐Ÿšจ DOCKER NETWORKING TROUBLESHOOTING:" + echo " 1. If running EvalAI locally, use: python manage.py runserver 0.0.0.0:8888" + echo " 2. Update host_config.json to use one of these URLs:" + printf " โ€ข http://host.docker.internal:%s (macOS/Windows)\n" "$PORT" + printf " โ€ข http://172.17.0.1:%s (Linux Docker bridge)\n" "$PORT" + printf " โ€ข http://0.0.0.0:%s (if server binds to all interfaces)\n" "$PORT" + echo " 3. Ensure Docker can access host network" + echo " 4. Check firewall settings" + exit 1 + fi + + # Update SERVER_URL to use working host + if [[ "$WORKING_HOST" != "$HOST" ]]; then + SERVER_URL="http://$WORKING_HOST:$PORT" + echo " ๐Ÿ”„ Updated server URL to: $SERVER_URL" fi # Test HTTP endpoint @@ -219,13 +277,14 @@ jobs: echo "๐Ÿšจ TROUBLESHOOTING STEPS:" echo " 1. Verify your EvalAI server is running:" echo " python manage.py runserver 0.0.0.0:8888" - echo " 2. Check if the server is binding to the correct interface" - echo " 3. Verify no firewall is blocking the connection" + echo " 2. Check if the server is binding to the correct interface (0.0.0.0)" + echo " 3. Update host_config.json with Docker-compatible URL" echo " 4. Check server logs for errors" echo "" - echo " If using Docker:" - echo " docker-compose up -d" - echo " docker-compose ps" + echo " Docker networking tips:" + echo " โ€ข Use host.docker.internal instead of localhost on macOS/Windows" + echo " โ€ข Use 172.17.0.1 (Docker bridge IP) on Linux" + echo " โ€ข Ensure server binds to 0.0.0.0, not just 127.0.0.1" echo "" exit 1 fi @@ -263,6 +322,7 @@ jobs: echo "" if [[ "${{ needs.validate-host-config.outputs.is_localhost }}" == "true" ]]; then echo "โœ… Local development mode: Challenge processed on self-hosted runner" + echo "โœ… Docker container: python:3.9-slim (architecture independent)" echo "โœ… EvalAI server: ${{ needs.validate-host-config.outputs.host_url }}" echo "โœ… Self-hosted runner successfully connected to local server" else @@ -280,10 +340,11 @@ jobs: echo "==============================" echo "" if [[ "${{ needs.validate-host-config.outputs.is_localhost }}" == "true" ]]; then - echo "๐Ÿ  Local development mode detected" - echo "๐Ÿ’ก Common issues for self-hosted runners:" - echo " โ€ข EvalAI server not running or not accessible" + echo "๐Ÿ  Local development mode detected (Docker container)" + echo "๐Ÿ’ก Common issues for self-hosted runners with Docker:" + echo " โ€ข EvalAI server not running or not accessible from container" echo " โ€ข Self-hosted runner not properly configured" + echo " โ€ข Docker networking issues between container and host" echo " โ€ข Network connectivity issues between runner and server" echo " โ€ข Invalid authentication credentials" echo "" From 531417727c1a3d7d42a8c65fe733aef3dc013ba6 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Sun, 29 Jun 2025 20:33:08 +0530 Subject: [PATCH 08/40] Support for macOS --- .github/workflows/validate-and-process.yml | 143 ++++++++++++++++----- github/host_config.json | 6 +- 2 files changed, 115 insertions(+), 34 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 14c6fb00a..460e3e746 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -112,8 +112,9 @@ jobs: echo "" echo "2๏ธโƒฃ Docker installed and running:" echo " โ€ข Docker Engine must be available on the runner machine" - echo " โ€ข Runner will use python:3.9-slim container (architecture independent)" + echo " โ€ข Will use 'docker run' commands with python:3.9-slim (architecture independent)" echo " โ€ข This solves compatibility issues with ARM64/macOS architectures" + echo " โ€ข Note: GitHub Actions containers only work on Linux, so we use docker run instead" echo "" echo "3๏ธโƒฃ EvalAI server running locally:" echo " โ€ข Server URL: ${{ needs.validate-host-config.outputs.host_url }}" @@ -136,8 +137,8 @@ jobs: (needs.check-self-hosted-requirements.result == 'success' || needs.check-self-hosted-requirements.result == 'skipped') runs-on: ${{ needs.validate-host-config.outputs.requires_self_hosted == 'true' && 'self-hosted' || 'ubuntu-latest' }} - # Use Docker container for self-hosted runners to ensure architecture independence - container: ${{ needs.validate-host-config.outputs.requires_self_hosted == 'true' && 'python:3.9-slim' || null }} + # Note: GitHub Actions containers only work on Linux runners + # For macOS self-hosted runners, we'll use Docker run commands instead steps: - name: Checkout challenge branch @@ -149,14 +150,18 @@ jobs: run: | echo "๐Ÿ” ENVIRONMENT INFORMATION" echo "=========================" - echo "Runner Type: ${{ needs.validate-host-config.outputs.requires_self_hosted == 'true' && 'Self-hosted (Docker)' || 'GitHub-hosted' }}" + echo "Runner Type: ${{ needs.validate-host-config.outputs.requires_self_hosted == 'true' && 'Self-hosted (macOS + Docker)' || 'GitHub-hosted' }}" echo "EvalAI URL: ${{ needs.validate-host-config.outputs.host_url }}" echo "Is Localhost: ${{ needs.validate-host-config.outputs.is_localhost }}" echo "OS: $(uname -s)" echo "Architecture: $(uname -m)" echo "Working Directory: $(pwd)" - echo "Python Version: $(python3 --version)" - echo "Container: ${{ needs.validate-host-config.outputs.requires_self_hosted == 'true' && 'python:3.9-slim' || 'None' }}" + if [[ "${{ needs.validate-host-config.outputs.requires_self_hosted }}" == "true" ]]; then + echo "Docker Version: $(docker --version)" + echo "Will use Docker containers for Python execution" + else + echo "Python Version: $(python3 --version)" + fi echo "" - name: Set up Python (GitHub-hosted only) @@ -165,15 +170,17 @@ jobs: with: python-version: 3.9.21 - - name: Install system dependencies (Docker container only) + - name: Prepare Docker environment (Self-hosted only) if: needs.validate-host-config.outputs.requires_self_hosted == 'true' run: | - echo "๐Ÿ“ฆ Installing system dependencies in Docker container" - apt-get update - apt-get install -y curl netcat-traditional jq git - echo "โœ… System dependencies installed" + echo "๐Ÿณ PREPARING DOCKER ENVIRONMENT" + echo "===============================" + echo "Pulling Python 3.9 Docker image..." + docker pull python:3.9-slim + echo "โœ… Docker image ready" - - name: Install dependencies + - name: Install dependencies (GitHub-hosted) + if: needs.validate-host-config.outputs.requires_self_hosted != 'true' run: | python -m pip install --upgrade pip if [ -f github/requirements.txt ]; then @@ -183,15 +190,47 @@ jobs: echo "โš ๏ธ No requirements.txt found in github/ directory" fi - - name: Pre-flight Local Server Health Check - if: needs.validate-host-config.outputs.is_localhost == 'true' + - name: Install dependencies (Self-hosted with Docker) + if: needs.validate-host-config.outputs.requires_self_hosted == 'true' + run: | + echo "๐Ÿ“ฆ Installing dependencies in Docker container" + docker run --rm -v "$(pwd):/workspace" -w /workspace python:3.9-slim bash -c " + pip install --upgrade pip + if [ -f github/requirements.txt ]; then + echo '๐Ÿ“ฆ Installing dependencies from github/requirements.txt' + pip install -r github/requirements.txt + else + echo 'โš ๏ธ No requirements.txt found in github/ directory' + fi + " + echo "โœ… Dependencies installed in Docker" + + - name: Pre-flight Local Server Health Check (GitHub-hosted) + if: needs.validate-host-config.outputs.is_localhost == 'true' && needs.validate-host-config.outputs.requires_self_hosted != 'true' + run: | + echo "๐Ÿฅ LOCAL SERVER HEALTH CHECK" + echo "============================" + + SERVER_URL="${{ needs.validate-host-config.outputs.host_url }}" + echo "๐Ÿ” Checking EvalAI server at: $SERVER_URL" + + # Health check logic for GitHub-hosted runners + # (This would typically not be used since GitHub-hosted can't reach localhost) + curl -s --connect-timeout 10 --max-time 10 "$SERVER_URL/" > /dev/null 2>&1 || { + echo "โŒ Cannot reach EvalAI server" + exit 1 + } + echo "โœ… Health check passed" + + - name: Pre-flight Local Server Health Check (Self-hosted with Docker) + if: needs.validate-host-config.outputs.is_localhost == 'true' && needs.validate-host-config.outputs.requires_self_hosted == 'true' run: | echo "๐Ÿฅ LOCAL SERVER HEALTH CHECK (Docker Container)" echo "===============================================" SERVER_URL="${{ needs.validate-host-config.outputs.host_url }}" echo "๐Ÿ” Checking EvalAI server at: $SERVER_URL" - echo "๐Ÿณ Running in Docker container - checking host networking" + echo "๐Ÿณ Running health check from Docker container" echo "" # Extract host and port from URL @@ -217,14 +256,14 @@ jobs: fi echo "" - # Test basic connectivity with multiple host variants - echo "1๏ธโƒฃ Testing basic connectivity..." + # Test basic connectivity with multiple host variants using Docker + echo "1๏ธโƒฃ Testing basic connectivity from Docker container..." CONNECTIVITY_SUCCESS=false WORKING_HOST="" for TEST_HOST in "${HOSTS_TO_TRY[@]}"; do echo " ๐Ÿ” Testing connection to $TEST_HOST:$PORT..." - if timeout 10 nc -z "$TEST_HOST" "$PORT" 2>/dev/null; then + if docker run --rm alpine/curl:latest sh -c "nc -z $TEST_HOST $PORT" 2>/dev/null; then echo " โœ… Port $PORT is reachable on $TEST_HOST" CONNECTIVITY_SUCCESS=true WORKING_HOST="$TEST_HOST" @@ -255,17 +294,17 @@ jobs: echo " ๐Ÿ”„ Updated server URL to: $SERVER_URL" fi - # Test HTTP endpoint + # Test HTTP endpoint using Docker echo "" - echo "2๏ธโƒฃ Testing HTTP endpoint..." - if curl -s --connect-timeout 10 --max-time 10 "$SERVER_URL/" > /dev/null 2>&1; then + echo "2๏ธโƒฃ Testing HTTP endpoint from Docker container..." + if docker run --rm alpine/curl:latest curl -s --connect-timeout 10 --max-time 10 "$SERVER_URL/" > /dev/null 2>&1; then echo " โœ… HTTP endpoint is responding at $SERVER_URL" # Test API endpoint echo "" - echo "3๏ธโƒฃ Testing API endpoint..." + echo "3๏ธโƒฃ Testing API endpoint from Docker container..." API_URL="$SERVER_URL/api/" - if curl -s --connect-timeout 10 --max-time 10 "$API_URL" > /dev/null 2>&1; then + if docker run --rm alpine/curl:latest curl -s --connect-timeout 10 --max-time 10 "$API_URL" > /dev/null 2>&1; then echo " โœ… API endpoint is accessible at $API_URL" else echo " โš ๏ธ API endpoint test failed, but server is responding" @@ -292,7 +331,8 @@ jobs: echo "" echo "โœ… Pre-flight health check completed successfully!" - - name: Validate challenge configuration + - name: Validate challenge configuration (GitHub-hosted) + if: needs.validate-host-config.outputs.requires_self_hosted != 'true' run: | echo "๐Ÿ” VALIDATING CHALLENGE CONFIGURATION" echo "=====================================" @@ -302,8 +342,25 @@ jobs: GITHUB_CONTEXT: ${{ toJson(github) }} GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }} - - name: Create or update challenge - if: success() + - name: Validate challenge configuration (Self-hosted with Docker) + if: needs.validate-host-config.outputs.requires_self_hosted == 'true' + run: | + echo "๐Ÿ” VALIDATING CHALLENGE CONFIGURATION (Docker)" + echo "===============================================" + docker run --rm \ + -v "$(pwd):/workspace" \ + -w /workspace \ + -e IS_VALIDATION='True' \ + -e GITHUB_CONTEXT='${{ toJson(github) }}' \ + -e GITHUB_AUTH_TOKEN='${{ secrets.AUTH_TOKEN }}' \ + python:3.9-slim \ + bash -c " + pip install -r github/requirements.txt && + python3 github/challenge_processing_script.py + " + + - name: Create or update challenge (GitHub-hosted) + if: success() && needs.validate-host-config.outputs.requires_self_hosted != 'true' run: | echo "๐Ÿš€ CREATING/UPDATING CHALLENGE" echo "==============================" @@ -313,6 +370,23 @@ jobs: GITHUB_CONTEXT: ${{ toJson(github) }} GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }} + - name: Create or update challenge (Self-hosted with Docker) + if: success() && needs.validate-host-config.outputs.requires_self_hosted == 'true' + run: | + echo "๐Ÿš€ CREATING/UPDATING CHALLENGE (Docker)" + echo "========================================" + docker run --rm \ + -v "$(pwd):/workspace" \ + -w /workspace \ + -e IS_VALIDATION='False' \ + -e GITHUB_CONTEXT='${{ toJson(github) }}' \ + -e GITHUB_AUTH_TOKEN='${{ secrets.AUTH_TOKEN }}' \ + python:3.9-slim \ + bash -c " + pip install -r github/requirements.txt && + python3 github/challenge_processing_script.py + " + - name: Success Summary if: success() run: | @@ -322,9 +396,10 @@ jobs: echo "" if [[ "${{ needs.validate-host-config.outputs.is_localhost }}" == "true" ]]; then echo "โœ… Local development mode: Challenge processed on self-hosted runner" - echo "โœ… Docker container: python:3.9-slim (architecture independent)" + echo "โœ… Docker containers: python:3.9-slim (architecture independent)" + echo "โœ… macOS + Docker approach: Works on ARM64 and AMD64 architectures" echo "โœ… EvalAI server: ${{ needs.validate-host-config.outputs.host_url }}" - echo "โœ… Self-hosted runner successfully connected to local server" + echo "โœ… Self-hosted runner successfully connected to local server via Docker" else echo "โœ… Production mode: Challenge processed on GitHub-hosted runner" echo "โœ… EvalAI server: ${{ needs.validate-host-config.outputs.host_url }}" @@ -340,14 +415,20 @@ jobs: echo "==============================" echo "" if [[ "${{ needs.validate-host-config.outputs.is_localhost }}" == "true" ]]; then - echo "๐Ÿ  Local development mode detected (Docker container)" + echo "๐Ÿ  Local development mode detected (macOS + Docker)" echo "๐Ÿ’ก Common issues for self-hosted runners with Docker:" - echo " โ€ข EvalAI server not running or not accessible from container" + echo " โ€ข EvalAI server not running or not accessible from Docker containers" echo " โ€ข Self-hosted runner not properly configured" + echo " โ€ข Docker not installed or not running on the runner machine" echo " โ€ข Docker networking issues between container and host" - echo " โ€ข Network connectivity issues between runner and server" + echo " โ€ข Need to use host.docker.internal instead of localhost in host_config.json" echo " โ€ข Invalid authentication credentials" echo "" + echo "๐Ÿ”ง Quick fixes to try:" + echo " โ€ข Update host_config.json URL to: http://host.docker.internal:8888" + echo " โ€ข Ensure EvalAI server runs with: python manage.py runserver 0.0.0.0:8888" + echo " โ€ข Check Docker is running: docker --version" + echo "" echo "๐Ÿ” Check the logs above for specific error details" else echo "โ˜๏ธ GitHub-hosted runner mode" diff --git a/github/host_config.json b/github/host_config.json index e7ead6158..d4e70184e 100644 --- a/github/host_config.json +++ b/github/host_config.json @@ -1,5 +1,5 @@ { - "token": "", - "team_pk": "", - "evalai_host_url": "" + "token": "", + "team_pk": "", + "evalai_host_url": "http://host.docker.internal:8888" } From df2d8b0c7237ca9f98fa2607c363a76ad8f16b56 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Sun, 29 Jun 2025 20:39:46 +0530 Subject: [PATCH 09/40] Update workflow --- .github/workflows/validate-and-process.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 460e3e746..9e11c65d6 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -194,7 +194,11 @@ jobs: if: needs.validate-host-config.outputs.requires_self_hosted == 'true' run: | echo "๐Ÿ“ฆ Installing dependencies in Docker container" - docker run --rm -v "$(pwd):/workspace" -w /workspace python:3.9-slim bash -c " + docker run --rm \ + --add-host host.docker.internal:host-gateway \ + -v "$(pwd):/workspace" \ + -w /workspace \ + python:3.9-slim bash -c " pip install --upgrade pip if [ -f github/requirements.txt ]; then echo '๐Ÿ“ฆ Installing dependencies from github/requirements.txt' @@ -263,7 +267,7 @@ jobs: for TEST_HOST in "${HOSTS_TO_TRY[@]}"; do echo " ๐Ÿ” Testing connection to $TEST_HOST:$PORT..." - if docker run --rm alpine/curl:latest sh -c "nc -z $TEST_HOST $PORT" 2>/dev/null; then + if docker run --rm --add-host host.docker.internal:host-gateway alpine/curl:latest sh -c "nc -z $TEST_HOST $PORT" 2>/dev/null; then echo " โœ… Port $PORT is reachable on $TEST_HOST" CONNECTIVITY_SUCCESS=true WORKING_HOST="$TEST_HOST" @@ -297,14 +301,14 @@ jobs: # Test HTTP endpoint using Docker echo "" echo "2๏ธโƒฃ Testing HTTP endpoint from Docker container..." - if docker run --rm alpine/curl:latest curl -s --connect-timeout 10 --max-time 10 "$SERVER_URL/" > /dev/null 2>&1; then + if docker run --rm --add-host host.docker.internal:host-gateway alpine/curl:latest curl -s --connect-timeout 10 --max-time 10 "$SERVER_URL/" > /dev/null 2>&1; then echo " โœ… HTTP endpoint is responding at $SERVER_URL" # Test API endpoint echo "" echo "3๏ธโƒฃ Testing API endpoint from Docker container..." API_URL="$SERVER_URL/api/" - if docker run --rm alpine/curl:latest curl -s --connect-timeout 10 --max-time 10 "$API_URL" > /dev/null 2>&1; then + if docker run --rm --add-host host.docker.internal:host-gateway alpine/curl:latest curl -s --connect-timeout 10 --max-time 10 "$API_URL" > /dev/null 2>&1; then echo " โœ… API endpoint is accessible at $API_URL" else echo " โš ๏ธ API endpoint test failed, but server is responding" @@ -348,6 +352,7 @@ jobs: echo "๐Ÿ” VALIDATING CHALLENGE CONFIGURATION (Docker)" echo "===============================================" docker run --rm \ + --add-host host.docker.internal:host-gateway \ -v "$(pwd):/workspace" \ -w /workspace \ -e IS_VALIDATION='True' \ @@ -376,6 +381,7 @@ jobs: echo "๐Ÿš€ CREATING/UPDATING CHALLENGE (Docker)" echo "========================================" docker run --rm \ + --add-host host.docker.internal:host-gateway \ -v "$(pwd):/workspace" \ -w /workspace \ -e IS_VALIDATION='False' \ @@ -428,6 +434,7 @@ jobs: echo " โ€ข Update host_config.json URL to: http://host.docker.internal:8888" echo " โ€ข Ensure EvalAI server runs with: python manage.py runserver 0.0.0.0:8888" echo " โ€ข Check Docker is running: docker --version" + echo " โ€ข Note: Workflow automatically adds --add-host host.docker.internal:host-gateway" echo "" echo "๐Ÿ” Check the logs above for specific error details" else From 14985bd4b3c8c30fa7a4d3a7c1d8758863d6d8dd Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Mon, 30 Jun 2025 01:46:54 +0530 Subject: [PATCH 10/40] update scripts --- .github/workflows/validate-and-process.yml | 56 +++++++++++++++------- github/challenge_processing_script.py | 7 ++- 2 files changed, 43 insertions(+), 20 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 9e11c65d6..9a1ff5323 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -41,12 +41,13 @@ jobs: echo "host_url=$HOST_URL" >> $GITHUB_OUTPUT - # Enhanced localhost detection - if [[ "$HOST_URL" == *"127.0.0.1"* ]] || [[ "$HOST_URL" == *"localhost"* ]] || [[ "$HOST_URL" == *"0.0.0.0"* ]]; then + # Enhanced localhost detection (includes Docker networking hosts) + if [[ "$HOST_URL" == *"127.0.0.1"* ]] || [[ "$HOST_URL" == *"localhost"* ]] || [[ "$HOST_URL" == *"0.0.0.0"* ]] || [[ "$HOST_URL" == *"host.docker.internal"* ]] || [[ "$HOST_URL" == *"172.17.0.1"* ]] || [[ "$HOST_URL" == *"192.168."* ]]; then echo "is_localhost=true" >> $GITHUB_OUTPUT echo "requires_self_hosted=true" >> $GITHUB_OUTPUT - echo "๐Ÿ  Localhost server detected: $HOST_URL" + echo "๐Ÿ  Localhost/Docker networking server detected: $HOST_URL" echo "๐Ÿค– Self-hosted runner required for local development" + echo "๐Ÿณ Docker networking will be used for container-to-host communication" echo "" echo "๐Ÿ“‹ Requirements for local challenge creation:" echo " โœ… Self-hosted GitHub Actions runner must be configured" @@ -116,10 +117,11 @@ jobs: echo " โ€ข This solves compatibility issues with ARM64/macOS architectures" echo " โ€ข Note: GitHub Actions containers only work on Linux, so we use docker run instead" echo "" - echo "3๏ธโƒฃ EvalAI server running locally:" + echo "3๏ธโƒฃ EvalAI server running via Docker Compose:" + echo " โ€ข Run: cd /path/to/evalai && docker-compose up --build" echo " โ€ข Server URL: ${{ needs.validate-host-config.outputs.host_url }}" - echo " โ€ข Ensure it's accessible from Docker containers (use host networking)" - echo " โ€ข Verify API endpoints are responding" + echo " โ€ข Django API: localhost:8000, Node.js frontend: localhost:8888" + echo " โ€ข Ensure it's accessible from Docker containers" echo "" echo "4๏ธโƒฃ Network connectivity:" echo " โ€ข Docker container can reach your EvalAI server" @@ -259,6 +261,12 @@ jobs: printf ' โ€ข %s\n' "${HOSTS_TO_TRY[@]}" fi echo "" + echo "๐Ÿ’ก Expected EvalAI Docker Compose setup:" + echo " โ€ข Django service: Port 8000 (backend API)" + echo " โ€ข Node.js service: Port 8888 (frontend, proxies to Django)" + echo " โ€ข Command: docker-compose up --build" + echo " โ€ข Workflow connects to: Node.js service at port 8888" + echo "" # Test basic connectivity with multiple host variants using Docker echo "1๏ธโƒฃ Testing basic connectivity from Docker container..." @@ -282,8 +290,13 @@ jobs: echo " ๐Ÿ’ก Make sure your EvalAI server is running and accessible from Docker" echo "" echo "๐Ÿšจ DOCKER NETWORKING TROUBLESHOOTING:" - echo " 1. If running EvalAI locally, use: python manage.py runserver 0.0.0.0:8888" - echo " 2. Update host_config.json to use one of these URLs:" + echo " 1. Start EvalAI using Docker Compose:" + echo " cd /path/to/your/evalai" + echo " docker-compose up --build" + echo " 2. This starts both services:" + echo " โ€ข Django API at localhost:8000" + echo " โ€ข Node.js frontend at localhost:8888 (proxies to Django)" + echo " 3. Update host_config.json to use one of these URLs:" printf " โ€ข http://host.docker.internal:%s (macOS/Windows)\n" "$PORT" printf " โ€ข http://172.17.0.1:%s (Linux Docker bridge)\n" "$PORT" printf " โ€ข http://0.0.0.0:%s (if server binds to all interfaces)\n" "$PORT" @@ -318,22 +331,27 @@ jobs: echo " โŒ HTTP endpoint is not responding at $SERVER_URL" echo "" echo "๐Ÿšจ TROUBLESHOOTING STEPS:" - echo " 1. Verify your EvalAI server is running:" - echo " python manage.py runserver 0.0.0.0:8888" - echo " 2. Check if the server is binding to the correct interface (0.0.0.0)" + echo " 1. Start EvalAI using Docker Compose:" + echo " cd /path/to/your/evalai" + echo " docker-compose up --build" + echo " 2. Verify both services are running:" + echo " โ€ข Django API: http://localhost:8000" + echo " โ€ข Node.js frontend: http://localhost:8888" echo " 3. Update host_config.json with Docker-compatible URL" - echo " 4. Check server logs for errors" + echo " 4. Check docker-compose logs for errors" echo "" echo " Docker networking tips:" echo " โ€ข Use host.docker.internal instead of localhost on macOS/Windows" echo " โ€ข Use 172.17.0.1 (Docker bridge IP) on Linux" - echo " โ€ข Ensure server binds to 0.0.0.0, not just 127.0.0.1" + echo " โ€ข Node.js service at :8888 proxies API calls to Django at :8000" echo "" exit 1 fi echo "" echo "โœ… Pre-flight health check completed successfully!" + echo "โœ… Node.js frontend service at :8888 is accessible from Docker containers" + echo "๐Ÿ’ก Node.js service will proxy API calls to Django service at :8000" - name: Validate challenge configuration (GitHub-hosted) if: needs.validate-host-config.outputs.requires_self_hosted != 'true' @@ -404,8 +422,9 @@ jobs: echo "โœ… Local development mode: Challenge processed on self-hosted runner" echo "โœ… Docker containers: python:3.9-slim (architecture independent)" echo "โœ… macOS + Docker approach: Works on ARM64 and AMD64 architectures" - echo "โœ… EvalAI server: ${{ needs.validate-host-config.outputs.host_url }}" - echo "โœ… Self-hosted runner successfully connected to local server via Docker" + echo "โœ… EvalAI Node.js service: ${{ needs.validate-host-config.outputs.host_url }}" + echo "โœ… Node.js service proxying API calls to Django service" + echo "โœ… Self-hosted runner successfully connected to Node.js service via Docker" else echo "โœ… Production mode: Challenge processed on GitHub-hosted runner" echo "โœ… EvalAI server: ${{ needs.validate-host-config.outputs.host_url }}" @@ -423,7 +442,7 @@ jobs: if [[ "${{ needs.validate-host-config.outputs.is_localhost }}" == "true" ]]; then echo "๐Ÿ  Local development mode detected (macOS + Docker)" echo "๐Ÿ’ก Common issues for self-hosted runners with Docker:" - echo " โ€ข EvalAI server not running or not accessible from Docker containers" + echo " โ€ข EvalAI Docker Compose services not running or not accessible from Docker containers" echo " โ€ข Self-hosted runner not properly configured" echo " โ€ข Docker not installed or not running on the runner machine" echo " โ€ข Docker networking issues between container and host" @@ -432,9 +451,10 @@ jobs: echo "" echo "๐Ÿ”ง Quick fixes to try:" echo " โ€ข Update host_config.json URL to: http://host.docker.internal:8888" - echo " โ€ข Ensure EvalAI server runs with: python manage.py runserver 0.0.0.0:8888" + echo " โ€ข Start EvalAI using Docker Compose:" + echo " cd /path/to/your/evalai && docker-compose up --build" echo " โ€ข Check Docker is running: docker --version" - echo " โ€ข Note: Workflow automatically adds --add-host host.docker.internal:host-gateway" + echo " โ€ข Note: Workflow connects to Node.js service at :8888, which proxies to Django at :8000" echo "" echo "๐Ÿ” Check the logs above for specific error details" else diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index 20c1d5254..b53882765 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -52,8 +52,11 @@ def is_localhost_url(url): """ localhost_indicators = [ "127.0.0.1", - "localhost", - "0.0.0.0" + "localhost", + "0.0.0.0", + "host.docker.internal", + "172.17.0.1", + "192.168." ] return any(indicator in url.lower() for indicator in localhost_indicators) From 8c72ccbddd336f51b84f6202a0130c0e91e3ab51 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Mon, 30 Jun 2025 02:53:28 +0530 Subject: [PATCH 11/40] Add pipeline for localhost support --- .github/workflows/validate-and-process.yml | 48 ++++++++++++++++++---- github/challenge_processing_script.py | 40 +++++++++++++----- github/config.py | 5 ++- 3 files changed, 73 insertions(+), 20 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 9a1ff5323..96736ea4c 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -311,25 +311,51 @@ jobs: echo " ๐Ÿ”„ Updated server URL to: $SERVER_URL" fi - # Test HTTP endpoint using Docker + # Test HTTP endpoint using Docker (with rate limiting protection) echo "" echo "2๏ธโƒฃ Testing HTTP endpoint from Docker container..." - if docker run --rm --add-host host.docker.internal:host-gateway alpine/curl:latest curl -s --connect-timeout 10 --max-time 10 "$SERVER_URL/" > /dev/null 2>&1; then - echo " โœ… HTTP endpoint is responding at $SERVER_URL" + + # Make request with better error handling + HTTP_RESPONSE=$(docker run --rm --add-host host.docker.internal:host-gateway alpine/curl:latest curl -s -w "HTTP_CODE:%{http_code}" --connect-timeout 10 --max-time 10 "$SERVER_URL/" 2>/dev/null) + HTTP_CODE=$(echo "$HTTP_RESPONSE" | grep -o "HTTP_CODE:[0-9]*" | cut -d: -f2) + + if [[ "$HTTP_CODE" == "200" || "$HTTP_CODE" == "201" || "$HTTP_CODE" == "302" || "$HTTP_CODE" == "303" ]]; then + echo " โœ… HTTP endpoint is responding at $SERVER_URL (Status: $HTTP_CODE)" + + # Wait before next request to avoid rate limiting + echo " โณ Waiting 2 seconds to avoid rate limiting..." + sleep 2 # Test API endpoint echo "" echo "3๏ธโƒฃ Testing API endpoint from Docker container..." API_URL="$SERVER_URL/api/" - if docker run --rm --add-host host.docker.internal:host-gateway alpine/curl:latest curl -s --connect-timeout 10 --max-time 10 "$API_URL" > /dev/null 2>&1; then - echo " โœ… API endpoint is accessible at $API_URL" + API_RESPONSE=$(docker run --rm --add-host host.docker.internal:host-gateway alpine/curl:latest curl -s -w "HTTP_CODE:%{http_code}" --connect-timeout 10 --max-time 10 "$API_URL" 2>/dev/null) + API_CODE=$(echo "$API_RESPONSE" | grep -o "HTTP_CODE:[0-9]*" | cut -d: -f2) + + if [[ "$API_CODE" == "200" || "$API_CODE" == "201" || "$API_CODE" == "302" || "$API_CODE" == "404" ]]; then + echo " โœ… API endpoint is accessible at $API_URL (Status: $API_CODE)" + elif [[ "$API_CODE" == "429" ]]; then + echo " โš ๏ธ Rate limited (HTTP 429) - server is responding but limiting requests" + echo " ๐Ÿ’ก This is normal for development servers with rate limiting enabled" else - echo " โš ๏ธ API endpoint test failed, but server is responding" + echo " โš ๏ธ API endpoint test failed (Status: $API_CODE), but server is responding" echo " ๐Ÿ’ก This might be normal if API requires authentication" fi - else - echo " โŒ HTTP endpoint is not responding at $SERVER_URL" - echo "" + elif [[ "$HTTP_CODE" == "429" ]]; then + echo " โš ๏ธ Rate limited (HTTP 429) at $SERVER_URL" + echo " ๐Ÿ’ก Server is responding but limiting request frequency" + echo " ๐Ÿ’ก This confirms the server is working - rate limiting is normal" + elif [[ -z "$HTTP_CODE" || "$HTTP_CODE" == "000" ]]; then + echo " โŒ HTTP endpoint is not responding at $SERVER_URL" + else + echo " โš ๏ธ HTTP endpoint returned status: $HTTP_CODE" + echo " ๐Ÿ’ก Server is responding but returned unexpected status" + fi + + # Only exit with error if server is completely unreachable + if [[ -z "$HTTP_CODE" || "$HTTP_CODE" == "000" ]]; then + echo "" echo "๐Ÿšจ TROUBLESHOOTING STEPS:" echo " 1. Start EvalAI using Docker Compose:" echo " cd /path/to/your/evalai" @@ -345,6 +371,10 @@ jobs: echo " โ€ข Use 172.17.0.1 (Docker bridge IP) on Linux" echo " โ€ข Node.js service at :8888 proxies API calls to Django at :8000" echo "" + echo " Rate limiting tips:" + echo " โ€ข HTTP 429 responses are normal - means server is working" + echo " โ€ข Workflow includes delays to avoid overwhelming your dev server" + echo "" exit 1 fi diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index b53882765..9b1babb5c 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -3,6 +3,7 @@ import os import requests import sys +import time import urllib3 import socket import platform @@ -244,7 +245,7 @@ def print_environment_info(): zip_file = open(CHALLENGE_ZIP_FILE_PATH, "rb") file = {"zip_configuration": zip_file} - data = {"GITHUB_REPOSITORY": GITHUB_REPOSITORY} + data = {"GITHUB_REPOSITORY": GITHUB_REPOSITORY or "unknown-repo"} # Configure SSL verification based on whether we're using localhost verify_ssl = not is_localhost @@ -252,6 +253,8 @@ def print_environment_info(): try: print(f"\n๐ŸŒ Sending request to EvalAI server...") + print(f"๐Ÿ’ก Adding small delay to avoid rate limiting...") + time.sleep(1) # Brief delay to avoid overwhelming the server response = requests.post(url, data=data, headers=headers, files=file, verify=verify_ssl) if ( @@ -294,7 +297,18 @@ def print_environment_info(): os.environ["CHALLENGE_ERRORS"] = error_message except requests.exceptions.HTTPError as err: - if response.status_code in EVALAI_ERROR_CODES: + if response.status_code == 429: + error_message = f"\n๐Ÿšจ RATE LIMITED (HTTP 429)\n" + error_message += f"โŒ EvalAI server is limiting request frequency\n" + error_message += f"๐Ÿ’ก This is normal for development servers - they limit requests to prevent overload\n\n" + error_message += f"๐Ÿ”ง Solutions:\n" + error_message += f" โ€ข Wait a few minutes and try again\n" + error_message += f" โ€ข Check if server has rate limiting enabled\n" + error_message += f" โ€ข Consider increasing server rate limits for development\n" + error_message += f"\nOriginal error: {err}" + print(error_message) + os.environ["CHALLENGE_ERRORS"] = error_message + elif response.status_code in EVALAI_ERROR_CODES: is_token_valid = validate_token(response.json()) if is_token_valid: error = response.json()["error"] @@ -345,18 +359,24 @@ def print_environment_info(): print(" This is expected when your local EvalAI server isn't running.") elif VALIDATION_STEP == "True" and check_if_pull_request(): - pr_number = GITHUB_CONTEXT["event"]["number"] - add_pull_request_comment( - GITHUB_AUTH_TOKEN, - os.path.basename(GITHUB_REPOSITORY), - pr_number, - errors, - ) + pr_number = GITHUB_CONTEXT.get("event", {}).get("number") + if not pr_number: + print("โš ๏ธ Warning: Could not get PR number from GITHUB_CONTEXT") + print(" Skipping pull request comment creation") + else: + repo_name = os.path.basename(GITHUB_REPOSITORY) if GITHUB_REPOSITORY else "unknown-repo" + add_pull_request_comment( + GITHUB_AUTH_TOKEN, + repo_name, + pr_number, + errors, + ) else: issue_title = f"Following errors occurred while {'validating' if VALIDATION_STEP == 'True' else 'processing'} the challenge config:" + repo_name = os.path.basename(GITHUB_REPOSITORY) if GITHUB_REPOSITORY else "unknown-repo" create_github_repository_issue( GITHUB_AUTH_TOKEN, - os.path.basename(GITHUB_REPOSITORY), + repo_name, issue_title, errors, ) diff --git a/github/config.py b/github/config.py index 738b69dcd..eb1afdfb5 100644 --- a/github/config.py +++ b/github/config.py @@ -6,7 +6,7 @@ HOST_CONFIG_FILE_PATH = "github/host_config.json" CHALLENGE_CONFIG_VALIDATION_URL = "/api/challenges/challenge/challenge_host_team/{}/validate_challenge_config/" CHALLENGE_CREATE_OR_UPDATE_URL = "/api/challenges/challenge/challenge_host_team/{}/create_or_update_github_challenge/" -EVALAI_ERROR_CODES = [400, 401, 406] +EVALAI_ERROR_CODES = [400, 401, 406, 429] API_HOST_URL = "https://eval.ai" IGNORE_DIRS = [ ".git", @@ -24,5 +24,8 @@ ] CHALLENGE_ZIP_FILE_PATH = "challenge_config.zip" GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY") +if not GITHUB_REPOSITORY: + print("โš ๏ธ Warning: GITHUB_REPOSITORY environment variable not set") + print(" This may happen when running outside GitHub Actions") GITHUB_EVENT_NAME = os.getenv("GITHUB_EVENT_NAME") VALIDATION_STEP = os.getenv("IS_VALIDATION") From 29d9f838978d9ac80e2441614e4dbd7ee210586b Mon Sep 17 00:00:00 2001 From: ZahedR_327 Date: Mon, 30 Jun 2025 03:07:22 +0530 Subject: [PATCH 12/40] Update validate-and-process.yml --- .github/workflows/validate-and-process.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 96736ea4c..6b53be3f5 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -405,6 +405,7 @@ jobs: -w /workspace \ -e IS_VALIDATION='True' \ -e GITHUB_CONTEXT='${{ toJson(github) }}' \ + -e GITHUB_REPOSITORY='${{ github.repository }}' \ -e GITHUB_AUTH_TOKEN='${{ secrets.AUTH_TOKEN }}' \ python:3.9-slim \ bash -c " From fa3269cf75960a8bc9c0b21ce16520f7be96b618 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Mon, 30 Jun 2025 15:26:21 +0530 Subject: [PATCH 13/40] Omit health checks --- .github/workflows/validate-and-process.yml | 241 --------------------- 1 file changed, 241 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 6b53be3f5..e034ff0c0 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -148,24 +148,6 @@ jobs: with: ref: challenge - - name: Environment Information - run: | - echo "๐Ÿ” ENVIRONMENT INFORMATION" - echo "=========================" - echo "Runner Type: ${{ needs.validate-host-config.outputs.requires_self_hosted == 'true' && 'Self-hosted (macOS + Docker)' || 'GitHub-hosted' }}" - echo "EvalAI URL: ${{ needs.validate-host-config.outputs.host_url }}" - echo "Is Localhost: ${{ needs.validate-host-config.outputs.is_localhost }}" - echo "OS: $(uname -s)" - echo "Architecture: $(uname -m)" - echo "Working Directory: $(pwd)" - if [[ "${{ needs.validate-host-config.outputs.requires_self_hosted }}" == "true" ]]; then - echo "Docker Version: $(docker --version)" - echo "Will use Docker containers for Python execution" - else - echo "Python Version: $(python3 --version)" - fi - echo "" - - name: Set up Python (GitHub-hosted only) if: needs.validate-host-config.outputs.requires_self_hosted != 'true' uses: actions/setup-python@v4 @@ -211,178 +193,6 @@ jobs: " echo "โœ… Dependencies installed in Docker" - - name: Pre-flight Local Server Health Check (GitHub-hosted) - if: needs.validate-host-config.outputs.is_localhost == 'true' && needs.validate-host-config.outputs.requires_self_hosted != 'true' - run: | - echo "๐Ÿฅ LOCAL SERVER HEALTH CHECK" - echo "============================" - - SERVER_URL="${{ needs.validate-host-config.outputs.host_url }}" - echo "๐Ÿ” Checking EvalAI server at: $SERVER_URL" - - # Health check logic for GitHub-hosted runners - # (This would typically not be used since GitHub-hosted can't reach localhost) - curl -s --connect-timeout 10 --max-time 10 "$SERVER_URL/" > /dev/null 2>&1 || { - echo "โŒ Cannot reach EvalAI server" - exit 1 - } - echo "โœ… Health check passed" - - - name: Pre-flight Local Server Health Check (Self-hosted with Docker) - if: needs.validate-host-config.outputs.is_localhost == 'true' && needs.validate-host-config.outputs.requires_self_hosted == 'true' - run: | - echo "๐Ÿฅ LOCAL SERVER HEALTH CHECK (Docker Container)" - echo "===============================================" - - SERVER_URL="${{ needs.validate-host-config.outputs.host_url }}" - echo "๐Ÿ” Checking EvalAI server at: $SERVER_URL" - echo "๐Ÿณ Running health check from Docker container" - echo "" - - # Extract host and port from URL - if [[ "$SERVER_URL" =~ http://([^:/]+):([0-9]+) ]]; then - HOST="${BASH_REMATCH[1]}" - PORT="${BASH_REMATCH[2]}" - elif [[ "$SERVER_URL" =~ http://([^:/]+) ]]; then - HOST="${BASH_REMATCH[1]}" - PORT="80" - else - echo "โŒ Unable to parse server URL: $SERVER_URL" - exit 1 - fi - - echo "๐ŸŒ Resolved to Host: $HOST, Port: $PORT" - - # Try different host variations for Docker networking - HOSTS_TO_TRY=("$HOST") - if [[ "$HOST" == "localhost" || "$HOST" == "127.0.0.1" ]]; then - HOSTS_TO_TRY+=("host.docker.internal" "172.17.0.1") - echo "๐Ÿ”„ Will try multiple host variants for Docker networking:" - printf ' โ€ข %s\n' "${HOSTS_TO_TRY[@]}" - fi - echo "" - echo "๐Ÿ’ก Expected EvalAI Docker Compose setup:" - echo " โ€ข Django service: Port 8000 (backend API)" - echo " โ€ข Node.js service: Port 8888 (frontend, proxies to Django)" - echo " โ€ข Command: docker-compose up --build" - echo " โ€ข Workflow connects to: Node.js service at port 8888" - echo "" - - # Test basic connectivity with multiple host variants using Docker - echo "1๏ธโƒฃ Testing basic connectivity from Docker container..." - CONNECTIVITY_SUCCESS=false - WORKING_HOST="" - - for TEST_HOST in "${HOSTS_TO_TRY[@]}"; do - echo " ๐Ÿ” Testing connection to $TEST_HOST:$PORT..." - if docker run --rm --add-host host.docker.internal:host-gateway alpine/curl:latest sh -c "nc -z $TEST_HOST $PORT" 2>/dev/null; then - echo " โœ… Port $PORT is reachable on $TEST_HOST" - CONNECTIVITY_SUCCESS=true - WORKING_HOST="$TEST_HOST" - break - else - echo " โŒ Port $PORT is NOT reachable on $TEST_HOST" - fi - done - - if [[ "$CONNECTIVITY_SUCCESS" != "true" ]]; then - echo " ๐Ÿ’ฅ Unable to connect to EvalAI server on any host variant" - echo " ๐Ÿ’ก Make sure your EvalAI server is running and accessible from Docker" - echo "" - echo "๐Ÿšจ DOCKER NETWORKING TROUBLESHOOTING:" - echo " 1. Start EvalAI using Docker Compose:" - echo " cd /path/to/your/evalai" - echo " docker-compose up --build" - echo " 2. This starts both services:" - echo " โ€ข Django API at localhost:8000" - echo " โ€ข Node.js frontend at localhost:8888 (proxies to Django)" - echo " 3. Update host_config.json to use one of these URLs:" - printf " โ€ข http://host.docker.internal:%s (macOS/Windows)\n" "$PORT" - printf " โ€ข http://172.17.0.1:%s (Linux Docker bridge)\n" "$PORT" - printf " โ€ข http://0.0.0.0:%s (if server binds to all interfaces)\n" "$PORT" - echo " 3. Ensure Docker can access host network" - echo " 4. Check firewall settings" - exit 1 - fi - - # Update SERVER_URL to use working host - if [[ "$WORKING_HOST" != "$HOST" ]]; then - SERVER_URL="http://$WORKING_HOST:$PORT" - echo " ๐Ÿ”„ Updated server URL to: $SERVER_URL" - fi - - # Test HTTP endpoint using Docker (with rate limiting protection) - echo "" - echo "2๏ธโƒฃ Testing HTTP endpoint from Docker container..." - - # Make request with better error handling - HTTP_RESPONSE=$(docker run --rm --add-host host.docker.internal:host-gateway alpine/curl:latest curl -s -w "HTTP_CODE:%{http_code}" --connect-timeout 10 --max-time 10 "$SERVER_URL/" 2>/dev/null) - HTTP_CODE=$(echo "$HTTP_RESPONSE" | grep -o "HTTP_CODE:[0-9]*" | cut -d: -f2) - - if [[ "$HTTP_CODE" == "200" || "$HTTP_CODE" == "201" || "$HTTP_CODE" == "302" || "$HTTP_CODE" == "303" ]]; then - echo " โœ… HTTP endpoint is responding at $SERVER_URL (Status: $HTTP_CODE)" - - # Wait before next request to avoid rate limiting - echo " โณ Waiting 2 seconds to avoid rate limiting..." - sleep 2 - - # Test API endpoint - echo "" - echo "3๏ธโƒฃ Testing API endpoint from Docker container..." - API_URL="$SERVER_URL/api/" - API_RESPONSE=$(docker run --rm --add-host host.docker.internal:host-gateway alpine/curl:latest curl -s -w "HTTP_CODE:%{http_code}" --connect-timeout 10 --max-time 10 "$API_URL" 2>/dev/null) - API_CODE=$(echo "$API_RESPONSE" | grep -o "HTTP_CODE:[0-9]*" | cut -d: -f2) - - if [[ "$API_CODE" == "200" || "$API_CODE" == "201" || "$API_CODE" == "302" || "$API_CODE" == "404" ]]; then - echo " โœ… API endpoint is accessible at $API_URL (Status: $API_CODE)" - elif [[ "$API_CODE" == "429" ]]; then - echo " โš ๏ธ Rate limited (HTTP 429) - server is responding but limiting requests" - echo " ๐Ÿ’ก This is normal for development servers with rate limiting enabled" - else - echo " โš ๏ธ API endpoint test failed (Status: $API_CODE), but server is responding" - echo " ๐Ÿ’ก This might be normal if API requires authentication" - fi - elif [[ "$HTTP_CODE" == "429" ]]; then - echo " โš ๏ธ Rate limited (HTTP 429) at $SERVER_URL" - echo " ๐Ÿ’ก Server is responding but limiting request frequency" - echo " ๐Ÿ’ก This confirms the server is working - rate limiting is normal" - elif [[ -z "$HTTP_CODE" || "$HTTP_CODE" == "000" ]]; then - echo " โŒ HTTP endpoint is not responding at $SERVER_URL" - else - echo " โš ๏ธ HTTP endpoint returned status: $HTTP_CODE" - echo " ๐Ÿ’ก Server is responding but returned unexpected status" - fi - - # Only exit with error if server is completely unreachable - if [[ -z "$HTTP_CODE" || "$HTTP_CODE" == "000" ]]; then - echo "" - echo "๐Ÿšจ TROUBLESHOOTING STEPS:" - echo " 1. Start EvalAI using Docker Compose:" - echo " cd /path/to/your/evalai" - echo " docker-compose up --build" - echo " 2. Verify both services are running:" - echo " โ€ข Django API: http://localhost:8000" - echo " โ€ข Node.js frontend: http://localhost:8888" - echo " 3. Update host_config.json with Docker-compatible URL" - echo " 4. Check docker-compose logs for errors" - echo "" - echo " Docker networking tips:" - echo " โ€ข Use host.docker.internal instead of localhost on macOS/Windows" - echo " โ€ข Use 172.17.0.1 (Docker bridge IP) on Linux" - echo " โ€ข Node.js service at :8888 proxies API calls to Django at :8000" - echo "" - echo " Rate limiting tips:" - echo " โ€ข HTTP 429 responses are normal - means server is working" - echo " โ€ข Workflow includes delays to avoid overwhelming your dev server" - echo "" - exit 1 - fi - - echo "" - echo "โœ… Pre-flight health check completed successfully!" - echo "โœ… Node.js frontend service at :8888 is accessible from Docker containers" - echo "๐Ÿ’ก Node.js service will proxy API calls to Django service at :8000" - - name: Validate challenge configuration (GitHub-hosted) if: needs.validate-host-config.outputs.requires_self_hosted != 'true' run: | @@ -441,54 +251,3 @@ jobs: pip install -r github/requirements.txt && python3 github/challenge_processing_script.py " - - - name: Success Summary - if: success() - run: | - echo "" - echo "๐ŸŽ‰ CHALLENGE PROCESSING COMPLETED SUCCESSFULLY!" - echo "==============================================" - echo "" - if [[ "${{ needs.validate-host-config.outputs.is_localhost }}" == "true" ]]; then - echo "โœ… Local development mode: Challenge processed on self-hosted runner" - echo "โœ… Docker containers: python:3.9-slim (architecture independent)" - echo "โœ… macOS + Docker approach: Works on ARM64 and AMD64 architectures" - echo "โœ… EvalAI Node.js service: ${{ needs.validate-host-config.outputs.host_url }}" - echo "โœ… Node.js service proxying API calls to Django service" - echo "โœ… Self-hosted runner successfully connected to Node.js service via Docker" - else - echo "โœ… Production mode: Challenge processed on GitHub-hosted runner" - echo "โœ… EvalAI server: ${{ needs.validate-host-config.outputs.host_url }}" - fi - echo "" - echo "๐Ÿ” Check your EvalAI instance for the updated challenge configuration" - - - name: Failure Summary - if: failure() - run: | - echo "" - echo "โŒ CHALLENGE PROCESSING FAILED" - echo "==============================" - echo "" - if [[ "${{ needs.validate-host-config.outputs.is_localhost }}" == "true" ]]; then - echo "๐Ÿ  Local development mode detected (macOS + Docker)" - echo "๐Ÿ’ก Common issues for self-hosted runners with Docker:" - echo " โ€ข EvalAI Docker Compose services not running or not accessible from Docker containers" - echo " โ€ข Self-hosted runner not properly configured" - echo " โ€ข Docker not installed or not running on the runner machine" - echo " โ€ข Docker networking issues between container and host" - echo " โ€ข Need to use host.docker.internal instead of localhost in host_config.json" - echo " โ€ข Invalid authentication credentials" - echo "" - echo "๐Ÿ”ง Quick fixes to try:" - echo " โ€ข Update host_config.json URL to: http://host.docker.internal:8888" - echo " โ€ข Start EvalAI using Docker Compose:" - echo " cd /path/to/your/evalai && docker-compose up --build" - echo " โ€ข Check Docker is running: docker --version" - echo " โ€ข Note: Workflow connects to Node.js service at :8888, which proxies to Django at :8000" - echo "" - echo "๐Ÿ” Check the logs above for specific error details" - else - echo "โ˜๏ธ GitHub-hosted runner mode" - echo "๐Ÿ” Check the logs above for specific error details" - fi From 56ddef3a553e106b5345bda4c5388ce683c0cf16 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Mon, 30 Jun 2025 15:32:12 +0530 Subject: [PATCH 14/40] Revert verification checks --- .github/workflows/validate-and-process.yml | 2 +- github/challenge_processing_script.py | 149 ++------------------- github/config.py | 2 +- 3 files changed, 13 insertions(+), 140 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index e034ff0c0..b031a0a3f 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -42,7 +42,7 @@ jobs: echo "host_url=$HOST_URL" >> $GITHUB_OUTPUT # Enhanced localhost detection (includes Docker networking hosts) - if [[ "$HOST_URL" == *"127.0.0.1"* ]] || [[ "$HOST_URL" == *"localhost"* ]] || [[ "$HOST_URL" == *"0.0.0.0"* ]] || [[ "$HOST_URL" == *"host.docker.internal"* ]] || [[ "$HOST_URL" == *"172.17.0.1"* ]] || [[ "$HOST_URL" == *"192.168."* ]]; then + if [[ "$HOST_URL" == *"127.0.0.1"* ]] || [[ "$HOST_URL" == *"localhost"* ]] || [[ "$HOST_URL" == *"0.0.0.0"* ]] || [[ "$HOST_URL" == *"host.docker.internal"* ]]; then echo "is_localhost=true" >> $GITHUB_OUTPUT echo "requires_self_hosted=true" >> $GITHUB_OUTPUT echo "๐Ÿ  Localhost/Docker networking server detected: $HOST_URL" diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index 9b1babb5c..a5df62ae7 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -3,10 +3,7 @@ import os import requests import sys -import time import urllib3 -import socket -import platform from urllib.parse import urlparse from config import * @@ -24,7 +21,11 @@ sys.dont_write_bytecode = True -GITHUB_CONTEXT = json.loads(os.getenv("GITHUB_CONTEXT")) +# ----------------------------------------------------------------------------- +# Environment variables provided by the workflow +# ----------------------------------------------------------------------------- + +GITHUB_CONTEXT = json.loads(os.getenv("GITHUB_CONTEXT", "{}")) GITHUB_AUTH_TOKEN = os.getenv("GITHUB_AUTH_TOKEN") if not GITHUB_AUTH_TOKEN: @@ -55,84 +56,16 @@ def is_localhost_url(url): "127.0.0.1", "localhost", "0.0.0.0", - "host.docker.internal", - "172.17.0.1", - "192.168." + "host.docker.internal" ] return any(indicator in url.lower() for indicator in localhost_indicators) def get_runner_info(): - """ - Get information about the current runner environment - - Returns: - dict: Information about the runner - """ - runner_info = { - "is_github_actions": bool(os.getenv("GITHUB_ACTIONS")), - "runner_name": os.getenv("RUNNER_NAME", "unknown"), - "runner_os": os.getenv("RUNNER_OS", platform.system()), - "runner_arch": os.getenv("RUNNER_ARCH", platform.machine()), + """Return a minimal dict about the runner (only what we need for error msgs).""" + return { "is_self_hosted": os.getenv("RUNNER_ENVIRONMENT") != "github-hosted", - "hostname": socket.gethostname(), - "platform": platform.platform(), } - return runner_info - - -def test_server_connectivity(url, timeout=10): - """ - Test connectivity to the EvalAI server - - Arguments: - url {str}: The server URL to test - timeout {int}: Connection timeout in seconds - - Returns: - dict: Test results with status and details - """ - result = { - "success": False, - "details": [], - "error": None - } - - try: - parsed_url = urlparse(url) - host = parsed_url.hostname - port = parsed_url.port or (443 if parsed_url.scheme == 'https' else 80) - - result["details"].append(f"Testing connectivity to {host}:{port}") - - # Test socket connection - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(timeout) - socket_result = sock.connect_ex((host, port)) - sock.close() - - if socket_result == 0: - result["details"].append(f"โœ… Socket connection successful to {host}:{port}") - - # Test HTTP request - try: - response = requests.get(url, timeout=timeout, verify=not is_localhost_url(url)) - result["details"].append(f"โœ… HTTP request successful (Status: {response.status_code})") - result["success"] = True - except requests.exceptions.RequestException as e: - result["details"].append(f"โš ๏ธ HTTP request failed: {e}") - result["success"] = False # Still consider it a partial success since socket worked - - else: - result["details"].append(f"โŒ Socket connection failed to {host}:{port}") - result["success"] = False - - except Exception as e: - result["error"] = str(e) - result["details"].append(f"โŒ Connectivity test failed: {e}") - result["success"] = False - - return result def configure_requests_for_localhost(): @@ -145,30 +78,8 @@ def configure_requests_for_localhost(): print("INFO: SSL verification disabled for localhost development server") -def print_environment_info(): - """ - Print detailed information about the current environment - """ - runner_info = get_runner_info() - - print("\n๐Ÿ” ENVIRONMENT INFORMATION") - print("=" * 50) - print(f"GitHub Actions: {runner_info['is_github_actions']}") - print(f"Runner Type: {'Self-hosted' if runner_info['is_self_hosted'] else 'GitHub-hosted'}") - print(f"Runner Name: {runner_info['runner_name']}") - print(f"Operating System: {runner_info['runner_os']}") - print(f"Architecture: {runner_info['runner_arch']}") - print(f"Hostname: {runner_info['hostname']}") - print(f"Platform: {runner_info['platform']}") - print(f"Working Directory: {os.getcwd()}") - print(f"Python Version: {sys.version}") - print("=" * 50) - - if __name__ == "__main__": - print_environment_info() - configs = load_host_configs(HOST_CONFIG_FILE_PATH) if configs: HOST_AUTH_TOKEN = configs[0] @@ -189,38 +100,6 @@ def print_environment_info(): configure_requests_for_localhost() print(f"INFO: Using localhost server: {EVALAI_HOST_URL}") - # For localhost, perform connectivity test - print(f"\n๐Ÿ” Testing connectivity to localhost server...") - connectivity_test = test_server_connectivity(EVALAI_HOST_URL) - - for detail in connectivity_test["details"]: - print(f" {detail}") - - if not connectivity_test["success"]: - error_message = f"\n๐Ÿšจ LOCALHOST SERVER CONNECTIVITY FAILED\n" - error_message += f"โŒ Cannot reach EvalAI server at: {EVALAI_HOST_URL}\n\n" - error_message += "๐Ÿ“‹ Troubleshooting steps:\n" - error_message += " 1. Ensure your EvalAI server is running\n" - error_message += " 2. Verify the server is listening on the correct interface and port\n" - error_message += " 3. Check for firewall or network restrictions\n" - error_message += " 4. Confirm the URL in host_config.json is correct\n\n" - - if runner_info['is_self_hosted']: - error_message += "๐Ÿ’ก Self-hosted runner tips:\n" - error_message += " โ€ข Make sure the server is accessible from your runner machine\n" - error_message += " โ€ข Test manually: curl -v " + EVALAI_HOST_URL + "\n" - else: - error_message += "โš ๏ธ You're using a GitHub-hosted runner with localhost URL\n" - error_message += " This will not work. Please use a self-hosted runner for localhost development.\n" - - print(error_message) - os.environ["CHALLENGE_ERRORS"] = error_message - - # Don't exit immediately for localhost connectivity issues in validation mode - # Let the request attempt provide more specific error information - else: - print("โœ… Localhost server connectivity test passed!") - # Fetching the url if VALIDATION_STEP == "True": print(f"\n๐Ÿ” VALIDATION MODE: Validating challenge configuration...") @@ -253,18 +132,12 @@ def print_environment_info(): try: print(f"\n๐ŸŒ Sending request to EvalAI server...") - print(f"๐Ÿ’ก Adding small delay to avoid rate limiting...") - time.sleep(1) # Brief delay to avoid overwhelming the server response = requests.post(url, data=data, headers=headers, files=file, verify=verify_ssl) - if ( - response.status_code != http.HTTPStatus.OK - and response.status_code != http.HTTPStatus.CREATED - ): + if response.status_code != http.HTTPStatus.OK and response.status_code != http.HTTPStatus.CREATED: response.raise_for_status() else: - success_message = response.json().get("Success", "Operation completed successfully") - print(f"\nโœ… SUCCESS: {success_message}") + print("\nโœ… Challenge processed successfully on EvalAI") except requests.exceptions.ConnectionError as conn_err: # Handle connection errors specifically for localhost @@ -384,4 +257,4 @@ def print_environment_info(): print(f"\nExiting the {os.path.basename(__file__)} script after failure\n") sys.exit(1) - print(f"\nโœ… Exiting the {os.path.basename(__file__)} script after success\n") \ No newline at end of file + print("\nโœ… Script completed\n") \ No newline at end of file diff --git a/github/config.py b/github/config.py index eb1afdfb5..1e789f55d 100644 --- a/github/config.py +++ b/github/config.py @@ -6,7 +6,7 @@ HOST_CONFIG_FILE_PATH = "github/host_config.json" CHALLENGE_CONFIG_VALIDATION_URL = "/api/challenges/challenge/challenge_host_team/{}/validate_challenge_config/" CHALLENGE_CREATE_OR_UPDATE_URL = "/api/challenges/challenge/challenge_host_team/{}/create_or_update_github_challenge/" -EVALAI_ERROR_CODES = [400, 401, 406, 429] +EVALAI_ERROR_CODES = [400, 401, 406] API_HOST_URL = "https://eval.ai" IGNORE_DIRS = [ ".git", From 521ac1f045c00f7c809c74bd51ce987de23983a1 Mon Sep 17 00:00:00 2001 From: ZahedR_327 Date: Mon, 30 Jun 2025 15:41:19 +0530 Subject: [PATCH 15/40] Revert host_config.json --- github/host_config.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/github/host_config.json b/github/host_config.json index d4e70184e..e7ead6158 100644 --- a/github/host_config.json +++ b/github/host_config.json @@ -1,5 +1,5 @@ { - "token": "", - "team_pk": "", - "evalai_host_url": "http://host.docker.internal:8888" + "token": "", + "team_pk": "", + "evalai_host_url": "" } From 5fd925dd5441b3acab6c3656b2e087202716d620 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Mon, 30 Jun 2025 16:25:02 +0530 Subject: [PATCH 16/40] Update README.md --- README.md | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 10c6ff364..41c4da274 100644 --- a/README.md +++ b/README.md @@ -117,22 +117,36 @@ For local EvalAI challenge development, this repository supports **self-hosted G - **Wrong runner used**: Verify `host_config.json` contains localhost URL - **Permission errors**: Check runner has appropriate file system access -## Important Note -`host_config.json` file includes default placeholders like: +## CI/CD Pipeline Overview + +This repository ships with a single GitHub Actions workflow โ€“ `.github/workflows/validate-and-process.yml`. Whenever you **push to the `challenge` branch** the following happens: + +1. **validate-host-config** (always on GitHub-hosted runner) + โ€ข Checks `github/host_config.json` for required fields (`token`, `team_pk`, `evalai_host_url`). + โ€ข Detects whether the URL is localhost and marks the build as requiring a self-hosted runner when needed. + +2. **check-self-hosted-requirements** (only when localhost detected) + โ€ข Prints a quick checklist so you don't forget to start your EvalAI docker-compose setup or runner service. + โ€ข Does **not** fail the build โ€“ it's purely informational. +3. **process-evalai-challenge** + โ€ข Runs on **GitHub-hosted** runner for remote URLs, **self-hosted** runner for localhost URLs. + โ€ข Installs Python dependencies directly (GitHub-hosted) or inside a `python:3.9-slim` Docker container (self-hosted) to keep your host OS clean. + โ€ข Executes `github/challenge_processing_script.py` twice: + โ€“ First with `IS_VALIDATION=True` (quick dry-run to surface YAML / template errors). + โ€“ Then with `IS_VALIDATION=False` to actually create / update the challenge on EvalAI. + โ€ข If validation fails, the job ends early and posts a GitHub Issue summarising the errors. + +## Important Note +`host_config.json` ships with placeholders like: - `` - `` - `` Please replace them with real values before pushing changes to avoid build errors. -**For localhost development**: Use `http://localhost:8888` or `http://127.0.0.1:8888` as the `evalai_host_url` and ensure you have a self-hosted runner configured. +**For localhost development**: Use `http://localhost:8000` , `http://127.0.0.1:8000` or `http://host.internal.docker:8000` as the `evalai_host_url` and ensure you have a self-hosted runner configured. ## Facing problems in creating a challenge? -Please feel free to open issues on our [GitHub Repository](https://github.com/Cloud-CV/EvalAI-Starter/issues) or contact us at team@cloudcv.org if you have issues. - -For **self-hosted runner issues**, include: -- Runner logs from `_diag` folder -- Output from `python3 github/test_local_setup.py` -- Your `host_config.json` configuration (without sensitive tokens) +Please feel free to open issues on our [GitHub Repository](https://github.com/Cloud-CV/EvalAI-Starter/issues) or contact us at team@cloudcv.org if you have issues. \ No newline at end of file From 544419c0168fe133303239a02ed802427733f072 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Mon, 30 Jun 2025 16:26:35 +0530 Subject: [PATCH 17/40] Update README for local dev --- README.md | 136 ++++++++++++++++++++++++++---------------------------- 1 file changed, 66 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index 41c4da274..8363e93ee 100644 --- a/README.md +++ b/README.md @@ -74,78 +74,74 @@ In order to test the evaluation script locally before uploading it to [EvalAI](h 3. Run the command `python -m worker.run` from the directory where `annotations/` `challenge_data/` and `worker/` directories are present. If the command runs successfully, then the evaluation script works locally and will work on the server as well. -## Local Development with Self-Hosted Runners - -For local EvalAI challenge development, this repository supports **self-hosted GitHub Actions runners** that can connect to your localhost EvalAI server. - -### When to use self-hosted runners: -- โœ… Developing with a local EvalAI server (`http://localhost:8888`) -- โœ… Testing challenges before deploying to production -- โœ… Rapid iteration during development -- โœ… Working with custom EvalAI configurations - -### Quick Setup: - -1. **Configure localhost in host_config.json:** - ```json - { - "token": "your_evalai_auth_token", - "team_pk": "your_team_primary_key", - "evalai_host_url": "http://localhost:8888" - } - ``` - -2. **Set up self-hosted runner:** - - Go to your repository โ†’ Settings โ†’ Actions โ†’ Runners - - Click "New self-hosted runner" - - Follow the setup instructions for your OS - - Start your EvalAI server: `python manage.py runserver 0.0.0.0:8888` - -3. **Test your setup:** - ```bash - python3 github/test_local_setup.py - ``` - -4. **Push to challenge branch:** - The workflow will automatically detect localhost and use your self-hosted runner! - -### For detailed setup instructions: -๐Ÿ“š See [Self-Hosted Runner Setup Guide](./github/self_hosted_runner_setup.md) - -### Troubleshooting: -- **Connection refused**: Make sure your EvalAI server is running and accessible -- **Wrong runner used**: Verify `host_config.json` contains localhost URL -- **Permission errors**: Check runner has appropriate file system access - -## CI/CD Pipeline Overview - -This repository ships with a single GitHub Actions workflow โ€“ `.github/workflows/validate-and-process.yml`. Whenever you **push to the `challenge` branch** the following happens: - -1. **validate-host-config** (always on GitHub-hosted runner) - โ€ข Checks `github/host_config.json` for required fields (`token`, `team_pk`, `evalai_host_url`). - โ€ข Detects whether the URL is localhost and marks the build as requiring a self-hosted runner when needed. - -2. **check-self-hosted-requirements** (only when localhost detected) - โ€ข Prints a quick checklist so you don't forget to start your EvalAI docker-compose setup or runner service. - โ€ข Does **not** fail the build โ€“ it's purely informational. - -3. **process-evalai-challenge** - โ€ข Runs on **GitHub-hosted** runner for remote URLs, **self-hosted** runner for localhost URLs. - โ€ข Installs Python dependencies directly (GitHub-hosted) or inside a `python:3.9-slim` Docker container (self-hosted) to keep your host OS clean. - โ€ข Executes `github/challenge_processing_script.py` twice: - โ€“ First with `IS_VALIDATION=True` (quick dry-run to surface YAML / template errors). - โ€“ Then with `IS_VALIDATION=False` to actually create / update the challenge on EvalAI. - โ€ข If validation fails, the job ends early and posts a GitHub Issue summarising the errors. - -## Important Note -`host_config.json` ships with placeholders like: -- `` -- `` -- `` +## Local Development with a Self-Hosted Runner -Please replace them with real values before pushing changes to avoid build errors. +For local server development, you can run the EvalAI server on your local machine and test your challenge configuration before deploying it publicly. This requires a **self-hosted GitHub Actions runner** to bridge the gap between GitHub and your local computer. + +#### Prerequisites +* A local checkout of the main [EvalAI](https://github.com/Cloud-CV/EvalAI) repository. +* Docker and Docker Compose installed and running on your machine. + +--- + +### Step-by-Step Guide for Local Development + +#### Step 1: Run the Local EvalAI Server + +First, start your local instance of EvalAI using Docker Compose. The API server, which the workflow communicates with, will be available on port `8000`. + +1. Navigate to your local `EvalAI` directory. +2. Run the server. The `--build` flag is only needed the first time or after code changes. + ```bash + docker-compose up --build + ``` + +**Note:** The command `docker-compose up` starts both the backend on port `8000` and the frontend on port `8888`. Our setup script specifically needs to talk to the backend API. + +#### Step 2: Set Up the Self-Hosted Runner + +A self-hosted runner is a small application you run on your machine that listens for jobs from your GitHub repository. + +1. In your challenge repository on GitHub, navigate to **Settings > Actions > Runners**. +2. Click **"New self-hosted runner"** and follow the on-screen instructions. +3. For detailed guidance, refer to the setup guide: **[Self-Hosted Runner Setup Guide](./github/self_hosted_runner_setup.md)**. + +Once configured and running, the runner will show as "Idle" in your GitHub settings. + +#### Step 3: Configure `host_config.json` for Localhost -**For localhost development**: Use `http://localhost:8000` , `http://127.0.0.1:8000` or `http://host.internal.docker:8000` as the `evalai_host_url` and ensure you have a self-hosted runner configured. +Update the configuration file to point to your local server. + +1. Open `github/host_config.json`. +2. Fill in your **local** EvalAI token and team ID. +3. Set `evalai_host_url` to point to your local API server on port `8000`. + + ```json + { + "token": "", + "team_pk": "", + "evalai_host_url": "http://localhost:8000" + } + ``` + * **For macOS/Windows Docker Users:** If the runner has trouble connecting to `localhost`, use `http://host.docker.internal:8000` as the `evalai_host_url`. This special DNS name resolves to the host machine's IP from within a Docker container. + +#### Step 4: Create the `challenge` Branch + + +With the server and runner active, create the `challenge` branch and create commits like you would when creating challenge using github. + + +#### Step 5: Monitor the Workflow + +1. Go to the **Actions** tab in your GitHub repository. +2. You will see the "Validate and Process EvalAI Challenge" workflow running. +3. Click on the workflow and observe the logs. You will see that the `process-evalai-challenge` job is running on your self-hosted runner and that it's using Docker to execute the scripts. + +If successful, your challenge will be created or updated on your local EvalAI instance. You can iterate quickly by simply pushing new changes. + +---- + +Please replace them with real values before pushing changes to avoid build errors. ## Facing problems in creating a challenge? From d488f26aa682bf739a9e77408fa15b24edbc1afb Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Mon, 30 Jun 2025 16:39:01 +0530 Subject: [PATCH 18/40] update readme --- github/self_hosted_runner_setup.md | 244 ----------------------------- 1 file changed, 244 deletions(-) delete mode 100644 github/self_hosted_runner_setup.md diff --git a/github/self_hosted_runner_setup.md b/github/self_hosted_runner_setup.md deleted file mode 100644 index a63134dbf..000000000 --- a/github/self_hosted_runner_setup.md +++ /dev/null @@ -1,244 +0,0 @@ -# Self-Hosted Runner Setup Guide - -This guide will help you set up a GitHub Actions self-hosted runner for local EvalAI challenge development. - -## Overview - -When developing EvalAI challenges locally, GitHub's hosted runners cannot access your localhost server. Self-hosted runners solve this by running the GitHub Actions workflow directly on your local machine, giving the workflow access to your local EvalAI server. - -## Prerequisites - -- Local EvalAI server running and accessible -- GitHub repository with appropriate permissions -- Local machine with internet connectivity - -## Step 1: Download and Configure the Runner - -### 1.1 Navigate to Runner Settings -1. Go to your GitHub repository -2. Click on **Settings** tab -3. Click on **Actions** in the left sidebar -4. Click on **Runners** -5. Click **New self-hosted runner** - -### 1.2 Download the Runner -Follow the download instructions for your operating system: - -**Linux/macOS:** -```bash -# Create a folder -mkdir actions-runner && cd actions-runner - -# Download the latest runner package -curl -o actions-runner-linux-x64-2.311.0.tar.gz -L https://github.com/actions/runner/releases/download/v2.311.0/actions-runner-linux-x64-2.311.0.tar.gz - -# Extract the installer -tar xzf ./actions-runner-linux-x64-2.311.0.tar.gz -``` - -**Windows:** -```powershell -# Create a folder under the drive root -mkdir \actions-runner ; cd \actions-runner - -# Download the latest runner package -Invoke-WebRequest -Uri https://github.com/actions/runner/releases/download/v2.311.0/actions-runner-win-x64-2.311.0.zip -OutFile actions-runner-win-x64-2.311.0.zip - -# Extract the installer -Add-Type -AssemblyName System.IO.Compression.FileSystem ; [System.IO.Compression.ZipFile]::ExtractToDirectory("$PWD\actions-runner-win-x64-2.311.0.zip", "$PWD") -``` - -### 1.3 Configure the Runner -```bash -# Configure the runner (follow the prompts) -./config.sh --url https://github.com/YOUR_USERNAME/YOUR_REPO --token YOUR_TOKEN - -# When prompted for runner group, press Enter for default -# When prompted for runner name, you can use: localhost-runner -# When prompted for labels, add: self-hosted,localhost-dev -# When prompted for work folder, press Enter for default -``` - -## Step 2: Start the Runner - -### 2.1 Run the Runner -```bash -# Start the runner -./run.sh -``` - -The runner will now listen for jobs from GitHub Actions. - -### 2.2 Run as a Service (Optional) -For persistent operation, you can install the runner as a service: - -**Linux/macOS:** -```bash -sudo ./svc.sh install -sudo ./svc.sh start -``` - -**Windows (run as Administrator):** -```powershell -.\svc.sh install -.\svc.sh start -``` - -## Step 3: Configure Your EvalAI Server - -### 3.1 Update host_config.json -Make sure your `github/host_config.json` is configured for localhost: - -```json -{ - "token": "your_evalai_auth_token", - "team_pk": "your_team_primary_key", - "evalai_host_url": "http://localhost:8888" -} -``` - -### 3.2 Start Your EvalAI Server -Ensure your local EvalAI server is running and accessible: - -```bash -# If using Django development server -python manage.py runserver 0.0.0.0:8888 - -# If using Docker -docker-compose up -d - -# Verify server is running -curl http://localhost:8888/api/ -``` - -**Important:** Make sure your server binds to `0.0.0.0` (all interfaces) rather than `127.0.0.1` (localhost only), especially if your runner is on a different machine or container. - -## Step 4: Test the Setup - -### 4.1 Trigger a Workflow -1. Push to the `challenge` branch or create a pull request -2. Go to the **Actions** tab in your repository -3. You should see the workflow running on your self-hosted runner - -### 4.2 Monitor the Workflow -The workflow will: -1. Detect localhost configuration -2. Use your self-hosted runner -3. Perform health checks on your local server -4. Process the challenge configuration - -## Troubleshooting - -### Runner Not Appearing -- Check if the runner process is still running -- Verify the registration token hasn't expired -- Check network connectivity to GitHub - -### Server Connection Issues -```bash -# Test server connectivity manually -curl -v http://localhost:8888/ -curl -v http://localhost:8888/api/ - -# Check if server is listening on correct interface -netstat -tlnp | grep 8888 -ss -tlnp | grep 8888 -``` - -### Workflow Still Using GitHub-Hosted Runner -- Verify your `host_config.json` contains a localhost URL -- Check that the runner has the correct labels -- Ensure the runner is online and available - -### Permission Issues -- Make sure the runner has appropriate file system permissions -- On Linux/macOS, avoid running as root if possible -- Check that the runner can access network resources - -## Security Considerations - -### Local Development Only -- Only use self-hosted runners for development/testing -- Don't use self-hosted runners for production workloads -- Be cautious about what code you run on your local machine - -### Network Security -- Ensure your local EvalAI server is not exposed to the internet -- Use firewall rules to restrict access if needed -- Monitor runner logs for unexpected activity - -### Token Management -- Keep your GitHub and EvalAI tokens secure -- Rotate tokens regularly -- Don't commit tokens to version control - -## Advanced Configuration - -### Multiple Runners -You can set up multiple runners for load balancing: -```bash -# Configure additional runners with different names -./config.sh --url https://github.com/YOUR_USERNAME/YOUR_REPO --token YOUR_TOKEN --name localhost-runner-2 -``` - -### Custom Labels -Add custom labels to target specific runners: -```bash -# During configuration, add custom labels -# Example: self-hosted,localhost-dev,linux,x64,gpu -``` - -Then target them in your workflow: -```yaml -runs-on: [self-hosted, localhost-dev, gpu] -``` - -### Environment Variables -Set environment variables for the runner: -```bash -# Create .env file in runner directory -echo "EVALAI_DEBUG=true" >> .env -echo "CUSTOM_CONFIG_PATH=/path/to/config" >> .env -``` - -## Best Practices - -1. **Keep Runners Updated**: Regularly update your self-hosted runners -2. **Monitor Resources**: Keep an eye on CPU, memory, and disk usage -3. **Clean Workspace**: Periodically clean the runner's work directory -4. **Log Management**: Monitor and rotate runner logs -5. **Backup Configuration**: Keep a backup of your runner configuration - -## Getting Help - -If you encounter issues: - -1. Check the runner logs in the `_diag` folder -2. Review the GitHub Actions workflow logs -3. Test EvalAI server connectivity manually -4. Check GitHub's self-hosted runner documentation -5. Open an issue in your repository with detailed error information - -## Example Complete Setup Script - -```bash -#!/bin/bash -# complete-setup.sh - Automated setup script - -set -e - -echo "๐Ÿš€ Setting up self-hosted runner for EvalAI local development" - -# Download and extract runner -mkdir -p actions-runner && cd actions-runner -curl -o actions-runner-linux-x64-2.311.0.tar.gz -L https://github.com/actions/runner/releases/download/v2.311.0/actions-runner-linux-x64-2.311.0.tar.gz -tar xzf ./actions-runner-linux-x64-2.311.0.tar.gz - -echo "๐Ÿ“ Please run the following command to configure your runner:" -echo "./config.sh --url https://github.com/YOUR_USERNAME/YOUR_REPO --token YOUR_TOKEN --name localhost-runner --labels self-hosted,localhost-dev" -echo "" -echo "Then start the runner with:" -echo "./run.sh" -``` - -This setup enables seamless local EvalAI challenge development with GitHub Actions! \ No newline at end of file From 6b3ab99892ae6c697cc59377c7c084076fc52918 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Mon, 30 Jun 2025 16:39:49 +0530 Subject: [PATCH 19/40] update readme --- README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8363e93ee..5621d7e4a 100644 --- a/README.md +++ b/README.md @@ -103,10 +103,8 @@ First, start your local instance of EvalAI using Docker Compose. The API server, A self-hosted runner is a small application you run on your machine that listens for jobs from your GitHub repository. 1. In your challenge repository on GitHub, navigate to **Settings > Actions > Runners**. -2. Click **"New self-hosted runner"** and follow the on-screen instructions. -3. For detailed guidance, refer to the setup guide: **[Self-Hosted Runner Setup Guide](./github/self_hosted_runner_setup.md)**. - -Once configured and running, the runner will show as "Idle" in your GitHub settings. +2. Click **"New self-hosted runner"** and follow the instructions. +3. Once configured and running, the runner will show as "Idle" in your GitHub settings. #### Step 3: Configure `host_config.json` for Localhost @@ -123,7 +121,7 @@ Update the configuration file to point to your local server. "evalai_host_url": "http://localhost:8000" } ``` - * **For macOS/Windows Docker Users:** If the runner has trouble connecting to `localhost`, use `http://host.docker.internal:8000` as the `evalai_host_url`. This special DNS name resolves to the host machine's IP from within a Docker container. + * **NOTE** : If the runner has trouble connecting to `localhost`, use `http://host.docker.internal:8000` as the `evalai_host_url`. This special DNS name resolves to the host machine's IP from within a Docker container. #### Step 4: Create the `challenge` Branch From 9ed9d8813fc76329fad369c4a0d423457d2952d8 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Mon, 30 Jun 2025 16:45:38 +0530 Subject: [PATCH 20/40] update README.md --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 5621d7e4a..56c555d12 100644 --- a/README.md +++ b/README.md @@ -129,15 +129,15 @@ Update the configuration file to point to your local server. With the server and runner active, create the `challenge` branch and create commits like you would when creating challenge using github. -#### Step 5: Monitor the Workflow - -1. Go to the **Actions** tab in your GitHub repository. -2. You will see the "Validate and Process EvalAI Challenge" workflow running. -3. Click on the workflow and observe the logs. You will see that the `process-evalai-challenge` job is running on your self-hosted runner and that it's using Docker to execute the scripts. - -If successful, your challenge will be created or updated on your local EvalAI instance. You can iterate quickly by simply pushing new changes. +If successful, upon merging your challenge will be created or updated on your local EvalAI instance in the [Hosted Challenges](http://127.0.0.1:8888/web/hosted-challenges) section on your local server. ---- +## Important Note +`host_config.json` file includes default placeholders like: + +- `` +- `` +- `` Please replace them with real values before pushing changes to avoid build errors. From 3a4bd3bc70a744ece4de7868674d2d2dc5a91bfa Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Mon, 30 Jun 2025 16:59:18 +0530 Subject: [PATCH 21/40] Update README.md for cleaner docs --- README.md | 78 +++++++++++++++++++++++-------------------------------- 1 file changed, 33 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 56c555d12..5b5b63864 100644 --- a/README.md +++ b/README.md @@ -76,62 +76,50 @@ In order to test the evaluation script locally before uploading it to [EvalAI](h ## Local Development with a Self-Hosted Runner -For local server development, you can run the EvalAI server on your local machine and test your challenge configuration before deploying it publicly. This requires a **self-hosted GitHub Actions runner** to bridge the gap between GitHub and your local computer. +> Use this when you want to test everything against a **local EvalAI server** before pushing to the real site. -#### Prerequisites -* A local checkout of the main [EvalAI](https://github.com/Cloud-CV/EvalAI) repository. -* Docker and Docker Compose installed and running on your machine. +1. **Spin up EvalAI locally** ---- - -### Step-by-Step Guide for Local Development - -#### Step 1: Run the Local EvalAI Server - -First, start your local instance of EvalAI using Docker Compose. The API server, which the workflow communicates with, will be available on port `8000`. - -1. Navigate to your local `EvalAI` directory. -2. Run the server. The `--build` flag is only needed the first time or after code changes. - ```bash - docker-compose up --build - ``` - -**Note:** The command `docker-compose up` starts both the backend on port `8000` and the frontend on port `8888`. Our setup script specifically needs to talk to the backend API. + ```bash + cd + docker-compose up --build # --build only the first time or after code changes + ``` -#### Step 2: Set Up the Self-Hosted Runner + *Backend API โ†’ `http://localhost:8000`, Frontend โ†’ `http://localhost:8888`.* -A self-hosted runner is a small application you run on your machine that listens for jobs from your GitHub repository. +2. **Register a self-hosted runner** -1. In your challenge repository on GitHub, navigate to **Settings > Actions > Runners**. -2. Click **"New self-hosted runner"** and follow the instructions. -3. Once configured and running, the runner will show as "Idle" in your GitHub settings. + 1. Repo โ†’ *Settings โ–ธ Actions โ–ธ Runners โ–ธ New self-hosted runner*. + 2. Download, `./config.sh --url --token `, then `./run.sh`. + 3. If you want to reconfigure a pre-existing runner for a new repository : + **Detach or reuse the runner:** + ```bash + ./config.sh remove --token # unregister + ./config.sh --url --token # attach elsewhere + ``` + OR + 1. Go to *Runners โ–ธ open menu โ–ธ Remove runner* , then paste the command shown in your local terminal to detach. + 2. Then follow steps 1 and 2 for configuring runner for new repository. -#### Step 3: Configure `host_config.json` for Localhost +3. **Point `host_config.json` to localhost** -Update the configuration file to point to your local server. + ```jsonc + { + "token": "", + "team_pk": "", + "evalai_host_url": "http://localhost:8000" + } + ``` -1. Open `github/host_config.json`. -2. Fill in your **local** EvalAI token and team ID. -3. Set `evalai_host_url` to point to your local API server on port `8000`. + *If the runner canโ€™t resolve `localhost`, use `http://host.docker.internal:8000`.* - ```json - { - "token": "", - "team_pk": "", - "evalai_host_url": "http://localhost:8000" - } - ``` - * **NOTE** : If the runner has trouble connecting to `localhost`, use `http://host.docker.internal:8000` as the `evalai_host_url`. This special DNS name resolves to the host machine's IP from within a Docker container. +4. **Create (or switch to) the `challenge` branch locally** + Commit your config / template / script changes here , as you would when creating a challenge using Github. -#### Step 4: Create the `challenge` Branch +5. **Verify the result** + Go to [Hosted Challenges](http://127.0.0.1:8888/web/hosted-challenges) on your local server and confirm your challenge appears and renders correctly. - -With the server and runner active, create the `challenge` branch and create commits like you would when creating challenge using github. - - -If successful, upon merging your challenge will be created or updated on your local EvalAI instance in the [Hosted Challenges](http://127.0.0.1:8888/web/hosted-challenges) section on your local server. - ----- +--- ## Important Note `host_config.json` file includes default placeholders like: From f86645b8ff7162a8db4c7c73db3ce5752075fcfe Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Tue, 1 Jul 2025 01:15:43 +0530 Subject: [PATCH 22/40] Update scripts --- .github/workflows/validate-and-process.yml | 17 +++----- github/challenge_processing_script.py | 50 +++++++++++----------- github/config.py | 3 -- 3 files changed, 31 insertions(+), 39 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index b031a0a3f..385aa26b4 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -49,31 +49,24 @@ jobs: echo "๐Ÿค– Self-hosted runner required for local development" echo "๐Ÿณ Docker networking will be used for container-to-host communication" echo "" - echo "๐Ÿ“‹ Requirements for local challenge creation:" - echo " โœ… Self-hosted GitHub Actions runner must be configured" - echo " โœ… EvalAI server must be running at: $HOST_URL" - echo " โœ… Network connectivity between runner and server" - echo "" + echo "๐Ÿ“‹ Requirements: self-hosted runner + running local EvalAI server" echo "โš ๏ธ Note: GitHub hosted runners cannot connect to localhost" echo "โš ๏ธ This workflow will use your self-hosted runner instead" fi # Validate required fields if [[ -z "$TOKEN" || "$TOKEN" == "" ]]; then - echo "โŒ Invalid or missing token in host_config.json" | tee -a validation_error.log - echo "๐Ÿ’ก Please replace with your actual EvalAI token" | tee -a validation_error.log + echo "โŒ Invalid or missing token" | tee -a validation_error.log echo "is_valid=false" >> $GITHUB_OUTPUT fi if [[ -z "$TEAM_PK" || "$TEAM_PK" == "" ]]; then - echo "โŒ Invalid or missing team_pk in host_config.json" | tee -a validation_error.log - echo "๐Ÿ’ก Please replace with your actual team primary key" | tee -a validation_error.log + echo "โŒ Invalid or missing team_pk" | tee -a validation_error.log echo "is_valid=false" >> $GITHUB_OUTPUT fi if [[ -z "$HOST_URL" || "$HOST_URL" == "" ]]; then - echo "โŒ Invalid or missing evalai_host_url in host_config.json" | tee -a validation_error.log - echo "๐Ÿ’ก Please replace with your EvalAI server URL" | tee -a validation_error.log + echo "โŒ Invalid or missing evalai_host_url" | tee -a validation_error.log echo "is_valid=false" >> $GITHUB_OUTPUT fi @@ -81,7 +74,7 @@ jobs: if: steps.validate.outputs.is_valid == 'false' uses: peter-evans/create-issue-from-file@v4 with: - title: "โŒ host_config.json validation failed" + title: "host_config.json validation failed" content-filepath: validation_error.log labels: | bug diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index a5df62ae7..4c11e08b2 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -169,37 +169,34 @@ def configure_requests_for_localhost(): print(error_message) os.environ["CHALLENGE_ERRORS"] = error_message - except requests.exceptions.HTTPError as err: - if response.status_code == 429: - error_message = f"\n๐Ÿšจ RATE LIMITED (HTTP 429)\n" - error_message += f"โŒ EvalAI server is limiting request frequency\n" - error_message += f"๐Ÿ’ก This is normal for development servers - they limit requests to prevent overload\n\n" - error_message += f"๐Ÿ”ง Solutions:\n" - error_message += f" โ€ข Wait a few minutes and try again\n" - error_message += f" โ€ข Check if server has rate limiting enabled\n" - error_message += f" โ€ข Consider increasing server rate limits for development\n" - error_message += f"\nOriginal error: {err}" - print(error_message) - os.environ["CHALLENGE_ERRORS"] = error_message - elif response.status_code in EVALAI_ERROR_CODES: + if response.status_code in EVALAI_ERROR_CODES: is_token_valid = validate_token(response.json()) if is_token_valid: error = response.json()["error"] - error_message = f"\nFollowing errors occurred while {'validating' if VALIDATION_STEP == 'True' else 'processing'} the challenge config:\n{error}" + error_message = "\nFollowing errors occurred while validating the challenge config:\n{}".format( + error + ) print(error_message) os.environ["CHALLENGE_ERRORS"] = error_message else: - error_message = f"\nHTTP Error occurred while {'validating' if VALIDATION_STEP == 'True' else 'processing'} the challenge config: {err}" - print(error_message) - os.environ["CHALLENGE_ERRORS"] = error_message + print( + "\nFollowing errors occurred while validating the challenge config: {}".format( + err + ) + ) + os.environ["CHALLENGE_ERRORS"] = str(err) except Exception as e: if VALIDATION_STEP == "True": - error_message = f"\nFollowing errors occurred while validating the challenge config: {e}" + error_message = "\nFollowing errors occurred while validating the challenge config: {}".format( + e + ) print(error_message) os.environ["CHALLENGE_ERRORS"] = error_message else: - error_message = f"\nFollowing errors occurred while processing the challenge config: {e}" + error_message = "\nFollowing errors occurred while processing the challenge config: {}".format( + e + ) print(error_message) os.environ["CHALLENGE_ERRORS"] = error_message @@ -245,7 +242,9 @@ def configure_requests_for_localhost(): errors, ) else: - issue_title = f"Following errors occurred while {'validating' if VALIDATION_STEP == 'True' else 'processing'} the challenge config:" + issue_title = ( + "Following errors occurred while validating the challenge config:" + ) repo_name = os.path.basename(GITHUB_REPOSITORY) if GITHUB_REPOSITORY else "unknown-repo" create_github_repository_issue( GITHUB_AUTH_TOKEN, @@ -253,8 +252,11 @@ def configure_requests_for_localhost(): issue_title, errors, ) - - print(f"\nExiting the {os.path.basename(__file__)} script after failure\n") - sys.exit(1) + print( + "\nExiting the {} script after failure\n".format( + os.path.basename(__file__) + ) + ) + sys.exit(1) - print("\nโœ… Script completed\n") \ No newline at end of file + print("\nExiting the {} script after success\n".format(os.path.basename(__file__))) \ No newline at end of file diff --git a/github/config.py b/github/config.py index 1e789f55d..738b69dcd 100644 --- a/github/config.py +++ b/github/config.py @@ -24,8 +24,5 @@ ] CHALLENGE_ZIP_FILE_PATH = "challenge_config.zip" GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY") -if not GITHUB_REPOSITORY: - print("โš ๏ธ Warning: GITHUB_REPOSITORY environment variable not set") - print(" This may happen when running outside GitHub Actions") GITHUB_EVENT_NAME = os.getenv("GITHUB_EVENT_NAME") VALIDATION_STEP = os.getenv("IS_VALIDATION") From dba6b71044b9bad27c9eb160dc3d82733a3ac858 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Tue, 1 Jul 2025 01:29:08 +0530 Subject: [PATCH 23/40] add runner checks --- .github/workflows/validate-and-process.yml | 59 +++++++++++++--------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 385aa26b4..00fc77853 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -97,32 +97,43 @@ jobs: echo "๐Ÿ  LOCAL DEVELOPMENT MODE DETECTED" echo "==================================" echo "" - echo "๐Ÿ“‹ To proceed with local challenge creation, ensure you have:" - echo "" - echo "1๏ธโƒฃ Self-hosted GitHub Actions runner configured:" - echo " โ€ข Download runner from: https://github.com/${{ github.repository }}/settings/actions/runners" - echo " โ€ข Install and configure on your local machine" - echo " โ€ข Start the runner service" - echo "" - echo "2๏ธโƒฃ Docker installed and running:" - echo " โ€ข Docker Engine must be available on the runner machine" - echo " โ€ข Will use 'docker run' commands with python:3.9-slim (architecture independent)" - echo " โ€ข This solves compatibility issues with ARM64/macOS architectures" - echo " โ€ข Note: GitHub Actions containers only work on Linux, so we use docker run instead" - echo "" - echo "3๏ธโƒฃ EvalAI server running via Docker Compose:" - echo " โ€ข Run: cd /path/to/evalai && docker-compose up --build" - echo " โ€ข Server URL: ${{ needs.validate-host-config.outputs.host_url }}" - echo " โ€ข Django API: localhost:8000, Node.js frontend: localhost:8888" - echo " โ€ข Ensure it's accessible from Docker containers" + # Check 1: Verify at least one self-hosted runner is online + echo "๐Ÿ” Checking for active self-hosted runners..." + RUNNERS_JSON=$(curl -s -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + "https://api.github.com/repos/${{ github.repository }}/actions/runners") + + ACTIVE_SELF_HOSTED=$(echo "$RUNNERS_JSON" | jq -r '.runners[] | select(.status == "online" and .labels[].name == "self-hosted") | .name' | wc -l) + + if [ "$ACTIVE_SELF_HOSTED" -eq 0 ]; then + echo "โŒ No active self-hosted runners found" + echo "๐Ÿ’ก Setup instructions:" + echo " 1. Go to: https://github.com/${{ github.repository }}/settings/actions/runners" + echo " 2. Click 'New self-hosted runner'" + echo " 3. Follow setup instructions for your OS" + echo " 4. Start the runner service" + exit 1 + else + echo "โœ… Found $ACTIVE_SELF_HOSTED active self-hosted runner(s)" + fi + + # Check 2: Test Docker availability (using a self-hosted runner briefly) echo "" - echo "4๏ธโƒฃ Network connectivity:" - echo " โ€ข Docker container can reach your EvalAI server" - echo " โ€ข No firewall blocking connections" - echo " โ€ข Consider using host.docker.internal for macOS/Windows" + echo "๐Ÿณ Testing Docker availability on self-hosted runner..." + + # We can't directly test Docker from GitHub-hosted runner, but we can check if Docker Hub is accessible + # The real Docker test will happen in the next job + if ! curl -s --connect-timeout 5 https://hub.docker.com > /dev/null; then + echo "โš ๏ธ Warning: Cannot reach Docker Hub (network issue?)" + echo " Docker pull commands may fail in the next job" + else + echo "โœ… Docker Hub is accessible" + fi + echo "" - echo "โš ๏ธ This job will continue on your self-hosted runner with Docker..." - echo "โš ๏ธ If you don't have a self-hosted runner with Docker, the next job will fail" + echo "๐Ÿ“‹ Next job will run on self-hosted runner and verify:" + echo " โ€ข Docker engine is running" + echo " โ€ข Can pull python:3.9-slim image" + echo " โ€ข EvalAI server connectivity" process-evalai-challenge: needs: [validate-host-config, check-self-hosted-requirements] From 90925492556630abc6cf193933a112f14ca8c23a Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Tue, 1 Jul 2025 01:36:46 +0530 Subject: [PATCH 24/40] Modify runner check logic --- .github/workflows/validate-and-process.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 00fc77853..92fb4c8cb 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -102,7 +102,12 @@ jobs: RUNNERS_JSON=$(curl -s -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ "https://api.github.com/repos/${{ github.repository }}/actions/runners") - ACTIVE_SELF_HOSTED=$(echo "$RUNNERS_JSON" | jq -r '.runners[] | select(.status == "online" and .labels[].name == "self-hosted") | .name' | wc -l) + # Debug: Show what we got from the API + echo "๐Ÿ› Debug: API Response:" + echo "$RUNNERS_JSON" | jq -r '.runners[] | {name: .name, status: .status, labels: [.labels[].name]}' + + # Fixed filter: Check for self-hosted runners that are online + ACTIVE_SELF_HOSTED=$(echo "$RUNNERS_JSON" | jq -r '.runners[] | select(.status == "online" and (.labels[] | .name == "self-hosted")) | .name' | wc -l) if [ "$ACTIVE_SELF_HOSTED" -eq 0 ]; then echo "โŒ No active self-hosted runners found" From 4bfae80b9c350bb9345af4865494ff4133515dd4 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Tue, 1 Jul 2025 01:59:55 +0530 Subject: [PATCH 25/40] fix challenge processing script --- .github/workflows/validate-and-process.yml | 43 ++++++++++++++-------- github/challenge_processing_script.py | 11 +++++- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 92fb4c8cb..03b2b638a 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -102,23 +102,34 @@ jobs: RUNNERS_JSON=$(curl -s -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ "https://api.github.com/repos/${{ github.repository }}/actions/runners") - # Debug: Show what we got from the API - echo "๐Ÿ› Debug: API Response:" - echo "$RUNNERS_JSON" | jq -r '.runners[] | {name: .name, status: .status, labels: [.labels[].name]}' - - # Fixed filter: Check for self-hosted runners that are online - ACTIVE_SELF_HOSTED=$(echo "$RUNNERS_JSON" | jq -r '.runners[] | select(.status == "online" and (.labels[] | .name == "self-hosted")) | .name' | wc -l) - - if [ "$ACTIVE_SELF_HOSTED" -eq 0 ]; then - echo "โŒ No active self-hosted runners found" - echo "๐Ÿ’ก Setup instructions:" - echo " 1. Go to: https://github.com/${{ github.repository }}/settings/actions/runners" - echo " 2. Click 'New self-hosted runner'" - echo " 3. Follow setup instructions for your OS" - echo " 4. Start the runner service" - exit 1 + # Check if API call was successful + if echo "$RUNNERS_JSON" | jq -e '.runners' > /dev/null 2>&1; then + # Debug: Show what we got from the API + echo "๐Ÿ› Debug: Found runners:" + echo "$RUNNERS_JSON" | jq -r '.runners[] | {name: .name, status: .status, labels: [.labels[].name]}' + + # Check for self-hosted runners that are online + ACTIVE_SELF_HOSTED=$(echo "$RUNNERS_JSON" | jq -r '.runners[] | select(.status == "online" and (.labels[] | .name == "self-hosted")) | .name' | wc -l) + + if [ "$ACTIVE_SELF_HOSTED" -eq 0 ]; then + echo "โŒ No active self-hosted runners found" + echo "๐Ÿ’ก Setup instructions:" + echo " 1. Go to: https://github.com/${{ github.repository }}/settings/actions/runners" + echo " 2. Click 'New self-hosted runner'" + echo " 3. Follow setup instructions for your OS" + echo " 4. Start the runner service" + exit 1 + else + echo "โœ… Found $ACTIVE_SELF_HOSTED active self-hosted runner(s)" + fi else - echo "โœ… Found $ACTIVE_SELF_HOSTED active self-hosted runner(s)" + echo "โš ๏ธ Cannot access runners API (insufficient permissions or API error)" + echo "๐Ÿ› Debug: API Response:" + echo "$RUNNERS_JSON" + echo "" + echo "๐Ÿ’ก Continuing without runner verification..." + echo " If you don't have a self-hosted runner, the next job will fail" + echo " Make sure GITHUB_TOKEN has 'repo' or 'actions:read' permissions" fi # Check 2: Test Docker availability (using a self-hosted runner briefly) diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index 4c11e08b2..985b4d075 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -168,7 +168,11 @@ def configure_requests_for_localhost(): print(error_message) os.environ["CHALLENGE_ERRORS"] = error_message - + + # Fail the job so CI visibly reports the problem + sys.exit(1) + + except requests.exceptions.HTTPError as err: if response.status_code in EVALAI_ERROR_CODES: is_token_valid = validate_token(response.json()) if is_token_valid: @@ -185,7 +189,7 @@ def configure_requests_for_localhost(): ) ) os.environ["CHALLENGE_ERRORS"] = str(err) - + except Exception as e: if VALIDATION_STEP == "True": error_message = "\nFollowing errors occurred while validating the challenge config: {}".format( @@ -228,6 +232,9 @@ def configure_requests_for_localhost(): else: print(" This is expected when your local EvalAI server isn't running.") + # Fail the job so CI visibly reports the problem + sys.exit(1) + elif VALIDATION_STEP == "True" and check_if_pull_request(): pr_number = GITHUB_CONTEXT.get("event", {}).get("number") if not pr_number: From 9785af8f898e4a9315f8848333b920a1e53e16ae Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Tue, 1 Jul 2025 02:06:41 +0530 Subject: [PATCH 26/40] Update runner checks --- .github/workflows/validate-and-process.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 03b2b638a..d650383ee 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -123,13 +123,10 @@ jobs: echo "โœ… Found $ACTIVE_SELF_HOSTED active self-hosted runner(s)" fi else - echo "โš ๏ธ Cannot access runners API (insufficient permissions or API error)" - echo "๐Ÿ› Debug: API Response:" - echo "$RUNNERS_JSON" - echo "" - echo "๐Ÿ’ก Continuing without runner verification..." - echo " If you don't have a self-hosted runner, the next job will fail" - echo " Make sure GITHUB_TOKEN has 'repo' or 'actions:read' permissions" + echo "โŒ Unable to verify self-hosted runners (API error or insufficient permissions)" + echo "๐Ÿ› Debug: API Response: $RUNNERS_JSON" + echo "๐Ÿ’ก Ensure the token used has 'actions:read' scope and try again." + exit 1 fi # Check 2: Test Docker availability (using a self-hosted runner briefly) From d44f9578d558d1a1560eee927a5b5be82e1aa646 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Tue, 1 Jul 2025 02:08:52 +0530 Subject: [PATCH 27/40] Update runner checks --- .github/workflows/validate-and-process.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index d650383ee..3e5dfc59f 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -124,8 +124,7 @@ jobs: fi else echo "โŒ Unable to verify self-hosted runners (API error or insufficient permissions)" - echo "๐Ÿ› Debug: API Response: $RUNNERS_JSON" - echo "๐Ÿ’ก Ensure the token used has 'actions:read' scope and try again." + echo " Please ensure the Runner is online and try again" exit 1 fi From 2b8e798b218b93138e3536034d125ace32197301 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Tue, 1 Jul 2025 02:28:59 +0530 Subject: [PATCH 28/40] Modify workflow --- .github/workflows/validate-and-process.yml | 50 +++++----------------- 1 file changed, 11 insertions(+), 39 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index 3e5dfc59f..fdf15273d 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -90,63 +90,35 @@ jobs: check-self-hosted-requirements: needs: validate-host-config if: needs.validate-host-config.outputs.requires_self_hosted == 'true' - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Self-hosted runner requirements check run: | echo "๐Ÿ  LOCAL DEVELOPMENT MODE DETECTED" echo "==================================" echo "" - # Check 1: Verify at least one self-hosted runner is online - echo "๐Ÿ” Checking for active self-hosted runners..." - RUNNERS_JSON=$(curl -s -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ - "https://api.github.com/repos/${{ github.repository }}/actions/runners") - - # Check if API call was successful - if echo "$RUNNERS_JSON" | jq -e '.runners' > /dev/null 2>&1; then - # Debug: Show what we got from the API - echo "๐Ÿ› Debug: Found runners:" - echo "$RUNNERS_JSON" | jq -r '.runners[] | {name: .name, status: .status, labels: [.labels[].name]}' - - # Check for self-hosted runners that are online - ACTIVE_SELF_HOSTED=$(echo "$RUNNERS_JSON" | jq -r '.runners[] | select(.status == "online" and (.labels[] | .name == "self-hosted")) | .name' | wc -l) - - if [ "$ACTIVE_SELF_HOSTED" -eq 0 ]; then - echo "โŒ No active self-hosted runners found" - echo "๐Ÿ’ก Setup instructions:" - echo " 1. Go to: https://github.com/${{ github.repository }}/settings/actions/runners" - echo " 2. Click 'New self-hosted runner'" - echo " 3. Follow setup instructions for your OS" - echo " 4. Start the runner service" - exit 1 - else - echo "โœ… Found $ACTIVE_SELF_HOSTED active self-hosted runner(s)" - fi - else - echo "โŒ Unable to verify self-hosted runners (API error or insufficient permissions)" - echo " Please ensure the Runner is online and try again" + # Check 1: Verify Docker is available + if ! docker info > /dev/null 2>&1; then + echo "โŒ Docker is not available or not running on this self-hosted runner" exit 1 + else + echo "โœ… Docker is available" fi - - # Check 2: Test Docker availability (using a self-hosted runner briefly) - echo "" - echo "๐Ÿณ Testing Docker availability on self-hosted runner..." - - # We can't directly test Docker from GitHub-hosted runner, but we can check if Docker Hub is accessible - # The real Docker test will happen in the next job + + # Check 2: Test Docker Hub connectivity if ! curl -s --connect-timeout 5 https://hub.docker.com > /dev/null; then echo "โš ๏ธ Warning: Cannot reach Docker Hub (network issue?)" - echo " Docker pull commands may fail in the next job" else echo "โœ… Docker Hub is accessible" fi - + echo "" - echo "๐Ÿ“‹ Next job will run on self-hosted runner and verify:" + echo "๐Ÿ“‹ Next job will run on this self-hosted runner and verify:" echo " โ€ข Docker engine is running" echo " โ€ข Can pull python:3.9-slim image" echo " โ€ข EvalAI server connectivity" + process-evalai-challenge: needs: [validate-host-config, check-self-hosted-requirements] if: | From 9aab02e76e30bd404f0b6a567d39d0f5bc7de472 Mon Sep 17 00:00:00 2001 From: ZahedR_327 Date: Tue, 1 Jul 2025 02:39:53 +0530 Subject: [PATCH 29/40] Update README.md --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5b5b63864..cd7631d36 100644 --- a/README.md +++ b/README.md @@ -107,16 +107,18 @@ In order to test the evaluation script locally before uploading it to [EvalAI](h { "token": "", "team_pk": "", - "evalai_host_url": "http://localhost:8000" + "evalai_host_url": "http://host.docker.internal:8000" } ``` + *host.docker.internal* : Docker's built-in hostname that points to the Docker host (your machine). + *8000* : Port where Backend API for the EvalAI server used to create challenges runs. - *If the runner canโ€™t resolve `localhost`, use `http://host.docker.internal:8000`.* -4. **Create (or switch to) the `challenge` branch locally** + +5. **Create (or switch to) the `challenge` branch locally** Commit your config / template / script changes here , as you would when creating a challenge using Github. -5. **Verify the result** +6. **Verify the result** Go to [Hosted Challenges](http://127.0.0.1:8888/web/hosted-challenges) on your local server and confirm your challenge appears and renders correctly. --- @@ -131,4 +133,4 @@ Please replace them with real values before pushing changes to avoid build error ## Facing problems in creating a challenge? -Please feel free to open issues on our [GitHub Repository](https://github.com/Cloud-CV/EvalAI-Starter/issues) or contact us at team@cloudcv.org if you have issues. \ No newline at end of file +Please feel free to open issues on our [GitHub Repository](https://github.com/Cloud-CV/EvalAI-Starter/issues) or contact us at team@cloudcv.org if you have issues. From 90fb5e10246aaf0fd65eb79d72958909498ab7d5 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Tue, 1 Jul 2025 02:46:06 +0530 Subject: [PATCH 30/40] Clean up challenge processing script --- github/challenge_processing_script.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index 985b4d075..0d60dae73 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -21,12 +21,7 @@ sys.dont_write_bytecode = True -# ----------------------------------------------------------------------------- -# Environment variables provided by the workflow -# ----------------------------------------------------------------------------- - GITHUB_CONTEXT = json.loads(os.getenv("GITHUB_CONTEXT", "{}")) - GITHUB_AUTH_TOKEN = os.getenv("GITHUB_AUTH_TOKEN") if not GITHUB_AUTH_TOKEN: print( @@ -36,7 +31,6 @@ # Clean up the GitHub token (remove any whitespace/newlines) GITHUB_AUTH_TOKEN = GITHUB_AUTH_TOKEN.strip() - HOST_AUTH_TOKEN = None CHALLENGE_HOST_TEAM_PK = None EVALAI_HOST_URL = None From 822b5e14c9d4763327decdf23bc7ba21756f2ee7 Mon Sep 17 00:00:00 2001 From: Zahed-Riyaz Date: Tue, 1 Jul 2025 02:55:21 +0530 Subject: [PATCH 31/40] Add github repo as variable in docker env --- .github/workflows/validate-and-process.yml | 1 + github/challenge_processing_script.py | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index fdf15273d..e395df1ed 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -233,6 +233,7 @@ jobs: -w /workspace \ -e IS_VALIDATION='False' \ -e GITHUB_CONTEXT='${{ toJson(github) }}' \ + -e GITHUB_REPOSITORY='${{ github.repository }}' \ -e GITHUB_AUTH_TOKEN='${{ secrets.AUTH_TOKEN }}' \ python:3.9-slim \ bash -c " diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index 0d60dae73..32c9aae10 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -118,7 +118,7 @@ def configure_requests_for_localhost(): zip_file = open(CHALLENGE_ZIP_FILE_PATH, "rb") file = {"zip_configuration": zip_file} - data = {"GITHUB_REPOSITORY": GITHUB_REPOSITORY or "unknown-repo"} + data = {"GITHUB_REPOSITORY": GITHUB_REPOSITORY} # Configure SSL verification based on whether we're using localhost verify_ssl = not is_localhost @@ -235,10 +235,9 @@ def configure_requests_for_localhost(): print("โš ๏ธ Warning: Could not get PR number from GITHUB_CONTEXT") print(" Skipping pull request comment creation") else: - repo_name = os.path.basename(GITHUB_REPOSITORY) if GITHUB_REPOSITORY else "unknown-repo" add_pull_request_comment( GITHUB_AUTH_TOKEN, - repo_name, + os.path.basename(GITHUB_REPOSITORY), pr_number, errors, ) @@ -246,7 +245,7 @@ def configure_requests_for_localhost(): issue_title = ( "Following errors occurred while validating the challenge config:" ) - repo_name = os.path.basename(GITHUB_REPOSITORY) if GITHUB_REPOSITORY else "unknown-repo" + repo_name = os.path.basename(GITHUB_REPOSITORY) if GITHUB_REPOSITORY else "" create_github_repository_issue( GITHUB_AUTH_TOKEN, repo_name, From 3563d37e4653ddfc695684c05ef16b592fb66c3b Mon Sep 17 00:00:00 2001 From: ZahedR_327 Date: Tue, 1 Jul 2025 03:24:07 +0530 Subject: [PATCH 32/40] Update validate-and-process.yml --- .github/workflows/validate-and-process.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/validate-and-process.yml b/.github/workflows/validate-and-process.yml index e395df1ed..9465caa35 100644 --- a/.github/workflows/validate-and-process.yml +++ b/.github/workflows/validate-and-process.yml @@ -127,9 +127,6 @@ jobs: (needs.check-self-hosted-requirements.result == 'success' || needs.check-self-hosted-requirements.result == 'skipped') runs-on: ${{ needs.validate-host-config.outputs.requires_self_hosted == 'true' && 'self-hosted' || 'ubuntu-latest' }} - # Note: GitHub Actions containers only work on Linux runners - # For macOS self-hosted runners, we'll use Docker run commands instead - steps: - name: Checkout challenge branch uses: actions/checkout@v3 From f6f10245f602cbfa2623958687812dbb92ca4078 Mon Sep 17 00:00:00 2001 From: ZahedR_327 Date: Tue, 1 Jul 2025 03:49:24 +0530 Subject: [PATCH 33/40] Update README.md --- README.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/README.md b/README.md index cd7631d36..a078f9c78 100644 --- a/README.md +++ b/README.md @@ -90,14 +90,8 @@ In order to test the evaluation script locally before uploading it to [EvalAI](h 2. **Register a self-hosted runner** 1. Repo โ†’ *Settings โ–ธ Actions โ–ธ Runners โ–ธ New self-hosted runner*. - 2. Download, `./config.sh --url --token `, then `./run.sh`. + 2. Select your architecture and paste the commands shown to install packages and configure your runners for your local machine 3. If you want to reconfigure a pre-existing runner for a new repository : - **Detach or reuse the runner:** - ```bash - ./config.sh remove --token # unregister - ./config.sh --url --token # attach elsewhere - ``` - OR 1. Go to *Runners โ–ธ open menu โ–ธ Remove runner* , then paste the command shown in your local terminal to detach. 2. Then follow steps 1 and 2 for configuring runner for new repository. From 159f61e073e93b6a5c4ff4709fda172b78338506 Mon Sep 17 00:00:00 2001 From: ZahedR_327 Date: Tue, 26 Aug 2025 03:02:09 +0530 Subject: [PATCH 34/40] Update challenge_processing_script.py --- github/challenge_processing_script.py | 392 +++++++++++++++++++++++++- 1 file changed, 386 insertions(+), 6 deletions(-) diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index 32c9aae10..7c49b5ce2 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -17,12 +17,15 @@ get_request_header, load_host_configs, validate_token, + check_sync_status, + is_localhost_url, ) sys.dont_write_bytecode = True +# GitHub token from repository secrets (used for GitHub API operations like creating issues, PR comments) GITHUB_CONTEXT = json.loads(os.getenv("GITHUB_CONTEXT", "{}")) -GITHUB_AUTH_TOKEN = os.getenv("GITHUB_AUTH_TOKEN") +GITHUB_AUTH_TOKEN = os.getenv("GITHUB_AUTH_TOKEN") # This comes from AUTH_TOKEN repository secret if not GITHUB_AUTH_TOKEN: print( "Please add your github access token to the repository secrets with the name AUTH_TOKEN" @@ -31,9 +34,15 @@ # Clean up the GitHub token (remove any whitespace/newlines) GITHUB_AUTH_TOKEN = GITHUB_AUTH_TOKEN.strip() -HOST_AUTH_TOKEN = None -CHALLENGE_HOST_TEAM_PK = None -EVALAI_HOST_URL = None + +# EvalAI configuration +HOST_AUTH_TOKEN = None # EvalAI user authentication token +CHALLENGE_HOST_TEAM_PK = None # EvalAI team ID +EVALAI_HOST_URL = None # EvalAI server URL + +# Fallback for GITHUB_BRANCH if not imported from config +if 'GITHUB_BRANCH' not in globals(): + GITHUB_BRANCH = os.getenv("GITHUB_REF_NAME") or os.getenv("GITHUB_BRANCH") or os.getenv("GITHUB_REF", "refs/heads/main").replace("refs/heads/", "") or "main" def is_localhost_url(url): @@ -72,7 +81,211 @@ def configure_requests_for_localhost(): print("INFO: SSL verification disabled for localhost development server") +def test_github_access(github_token, repository): + """ + Tests GitHub repository access to verify token permissions + """ + print(f"\n๐Ÿ” Testing GitHub repository access...") + print(f" Repository: {repository}") + print(f" Token: {github_token[:8]}...{github_token[-4:] if len(github_token) > 12 else '***'}") + + try: + from github import Github + client = Github(github_token) + + # Try to get the repository + repo = client.get_repo(repository) + print(f" โœ… Repository accessible: {repo.full_name}") + print(f" ๐Ÿ“ Default branch: {repo.default_branch}") + print(f" ๐Ÿ”’ Private: {repo.private}") + + # Check permissions + user = client.get_user() + print(f" ๐Ÿ‘ค Authenticated as: {user.login}") + + # Try to get repository contents + try: + contents = repo.get_contents("") + print(f" ๐Ÿ“„ Repository contents accessible") + except Exception as e: + print(f" โš ๏ธ Cannot read repository contents: {e}") + + return True + + except Exception as e: + print(f" โŒ GitHub access failed: {e}") + if "Bad credentials" in str(e): + print(f" ๐Ÿ’ก Token is invalid or expired") + elif "Not Found" in str(e): + print(f" ๐Ÿ’ก Repository not found or token doesn't have access") + elif "Bad credentials" in str(e): + print(f" ๐Ÿ’ก Token doesn't have sufficient permissions") + return False + + +def test_basic_connectivity(evalai_host_url): + """ + Tests basic connectivity to the EvalAI server + """ + print(f"\n๐Ÿ” Testing basic connectivity to EvalAI server...") + print(f" URL: {evalai_host_url}") + + # Check if this is a Docker environment + if "host.docker.internal" in evalai_host_url: + print(f" ๐Ÿณ Docker environment detected (host.docker.internal)") + print(f" ๐Ÿ’ก This means you're running from inside a Docker container") + print(f" ๐Ÿ’ก host.docker.internal should resolve to the host machine") + + try: + # Try to access the root or admin endpoint + response = requests.get( + f"{evalai_host_url}/", + verify=not is_localhost_url(evalai_host_url), + timeout=10 + ) + print(f" โœ… Root endpoint accessible (Status: {response.status_code})") + return True + except requests.exceptions.ConnectionError: + print(f" โŒ Connection failed - server not reachable") + if "host.docker.internal" in evalai_host_url: + print(f" ๐Ÿณ Docker troubleshooting:") + print(f" โ€ข Ensure EvalAI server is running on host machine") + print(f" โ€ข Check if server is binding to 0.0.0.0:8000 (not just 127.0.0.1)") + print(f" โ€ข Verify Docker can reach host.docker.internal") + print(f" โ€ข Try using host machine's IP address instead") + return False + except Exception as e: + print(f" โš ๏ธ Unexpected error: {e}") + return False + + +def test_api_endpoints(evalai_host_url, team_pk, host_auth_token): + """ + Tests different API endpoint patterns to find the correct one + """ + print(f"\n๐Ÿงช Testing API endpoint patterns...") + print(f" Team PK: {team_pk}") + print(f" EvalAI Host: {evalai_host_url}") + + # Different endpoint patterns to test + endpoint_patterns = [ + f"/api/v1/challenges/challenge_host_team/{team_pk}/validate_challenge_config/", + f"/api/challenges/challenge_host_team/{team_pk}/validate_challenge_config/", + f"/api/v1/challenges/{team_pk}/validate_challenge_config/", + f"/api/challenges/{team_pk}/validate_challenge_config/", + f"/api/v1/challenges/challenge/challenge_host_team/{team_pk}/validate_challenge_config/", + f"/api/challenges/challenge/challenge_host_team/{team_pk}/validate_challenge_config/", + # Additional patterns that might work + f"/api/v1/challenges/validate_challenge_config/", + f"/api/challenges/validate_challenge_config/", + f"/api/v1/challenge_host_team/{team_pk}/validate_challenge_config/", + f"/api/challenge_host_team/{team_pk}/validate_challenge_config/", + f"/api/v1/challenges/{team_pk}/validate/", + f"/api/challenges/{team_pk}/validate/", + ] + + results = {} + headers = get_request_header(host_auth_token) + + for pattern in endpoint_patterns: + test_url = f"{evalai_host_url}{pattern}" + print(f"\n Testing: {pattern}") + + try: + # Send a simple GET request to test the endpoint + response = requests.get( + test_url, + headers=headers, + verify=not is_localhost_url(evalai_host_url), + timeout=10 + ) + + status = response.status_code + results[pattern] = { + "status": status, + "url": test_url, + "accessible": status != 404 + } + + if status == 404: + print(f" โŒ 404 Not Found") + elif status == 401: + print(f" ๐Ÿ”’ 401 Unauthorized (endpoint exists but auth failed)") + elif status == 403: + print(f" ๐Ÿšซ 403 Forbidden (endpoint exists but access denied)") + elif status == 200: + print(f" โœ… 200 OK (endpoint accessible)") + else: + print(f" โš ๏ธ {status} (endpoint exists, status: {status})") + + except requests.exceptions.ConnectionError: + print(f" โŒ Connection Error") + results[pattern] = {"status": "Connection Error", "url": test_url, "accessible": False} + except Exception as e: + print(f" โŒ Error: {e}") + results[pattern] = {"status": f"Error: {e}", "url": test_url, "accessible": False} + + # Find the best endpoint + working_endpoints = [p for p, r in results.items() if r.get("accessible", False)] + + if working_endpoints: + print(f"\nโœ… Found working endpoints:") + for endpoint in working_endpoints: + print(f" โ€ข {endpoint}") + print(f"\n๐Ÿ’ก Update your config.py with one of these working patterns") + else: + print(f"\nโŒ No working endpoints found") + print(f" Check your team_pk and EvalAI server configuration") + + return results + + +def setup_one_way_sync(): + """ + Sets up one-way sync from EvalAI to GitHub + """ + print(f"\n๐Ÿ”„ Setting up one-way sync (EvalAI โ†’ GitHub)...") + print(f" Repository: {GITHUB_REPOSITORY}") + print(f" EvalAI Server: {EVALAI_HOST_URL}") + print(f" Using GitHub token: {GITHUB_AUTH_TOKEN[:8]}...{GITHUB_AUTH_TOKEN[-4:] if len(GITHUB_AUTH_TOKEN) > 12 else '***'}") + + # Test GitHub repository access + github_access = test_github_access(GITHUB_AUTH_TOKEN, GITHUB_REPOSITORY) + + if github_access: + print(f"โœ… GitHub repository access verified!") + print(f" EvalAI changes will automatically sync to GitHub") + print(f" GitHub changes will NOT sync back to EvalAI (by design)") + print("\n๐Ÿ’ก How it works:") + print(" 1. Make changes in EvalAI UI") + print(" 2. Changes are saved to database") + print(" 3. Django signal automatically triggers GitHub sync") + print(" 4. GitHub repository is updated with latest changes") + print(" 5. User gets immediate feedback (no waiting)") + + print(f"\nโš ๏ธ IMPORTANT: Ensure your EvalAI challenge has these fields configured:") + print(f" โ€ข github_repository: '{GITHUB_REPOSITORY}'") + print(f" โ€ข github_branch: '{GITHUB_BRANCH}' (actual repository default branch)") + print(f" โ€ข github_token: [your GitHub personal access token]") + print(f"\n๐Ÿ’ก These must be set in the EvalAI challenge settings for sync to work") + print(f"๐Ÿ’ก Note: Your repository uses '{GITHUB_BRANCH}' branch, not 'main'") + print(f"\n๐Ÿ”ง Backend uses Django signals for automatic sync (no Celery needed)") + print(f" โ€ข challenge_details_sync signal for challenge updates") + print(f" โ€ข challenge_phase_details_sync signal for phase updates") + + return True + else: + print(f"โŒ GitHub repository access failed!") + print(f" EvalAI โ†’ GitHub sync will not work until this is resolved") + print(f" Check your GitHub token permissions and repository access") + return False + + if __name__ == "__main__": + if GITHUB_CONTEXT["event"]["head_commit"]["message"].startswith("evalai_bot"): + print("Sync from Evalai") + sys.exit(0) + configs = load_host_configs(HOST_CONFIG_FILE_PATH) if configs: @@ -89,11 +302,26 @@ def configure_requests_for_localhost(): print(f"\n๐ŸŒ EvalAI Server: {EVALAI_HOST_URL}") print(f"๐Ÿ  Localhost Mode: {is_localhost}") print(f"๐Ÿค– Self-hosted Runner: {runner_info['is_self_hosted']}") + print(f"๐Ÿ”„ Sync Mode: One-way (EvalAI โ†’ GitHub)") + + if GITHUB_AUTH_TOKEN: + print(f" GitHub Token: {GITHUB_AUTH_TOKEN[:8]}...{GITHUB_AUTH_TOKEN[-4:] if len(GITHUB_AUTH_TOKEN) > 12 else '***'}") + print(f" Token Source: AUTH_TOKEN repository secret") + else: + print(f" GitHub Token: Not provided") + print(f" To enable: Add AUTH_TOKEN to repository secrets") if is_localhost: configure_requests_for_localhost() print(f"INFO: Using localhost server: {EVALAI_HOST_URL}") + # Setup one-way sync configuration + if GITHUB_AUTH_TOKEN: + setup_one_way_sync() + else: + print("โ„น๏ธ One-way sync not configured") + print(" Add AUTH_TOKEN to repository secrets to enable automatic GitHub sync") + # Fetching the url if VALIDATION_STEP == "True": print(f"\n๐Ÿ” VALIDATION MODE: Validating challenge configuration...") @@ -118,7 +346,15 @@ def configure_requests_for_localhost(): zip_file = open(CHALLENGE_ZIP_FILE_PATH, "rb") file = {"zip_configuration": zip_file} - data = {"GITHUB_REPOSITORY": GITHUB_REPOSITORY} + data = { + "GITHUB_REPOSITORY": GITHUB_REPOSITORY, + "GITHUB_AUTH_TOKEN": GITHUB_AUTH_TOKEN, + "GITHUB_BRANCH" : GITHUB_BRANCH + } + + # Add GitHub token for one-way sync if available + if GITHUB_AUTH_TOKEN: + data["GITHUB_TOKEN"] = GITHUB_AUTH_TOKEN # Configure SSL verification based on whether we're using localhost verify_ssl = not is_localhost @@ -133,6 +369,46 @@ def configure_requests_for_localhost(): else: print("\nโœ… Challenge processed successfully on EvalAI") + # If this was a challenge creation/update, try to get the challenge ID for sync status + if VALIDATION_STEP != "True" and GITHUB_AUTH_TOKEN: + try: + response_data = response.json() + if "id" in response_data: + challenge_id = response_data["id"] + print(f"\n๐Ÿ”„ Checking sync status for challenge {challenge_id}...") + sync_status = check_sync_status(EVALAI_HOST_URL, challenge_id, HOST_AUTH_TOKEN) + if sync_status: + print(f"โœ… Sync status retrieved: {sync_status}") + + # Additional debugging for sync issues + print(f"\n๐Ÿ” Sync Debugging Information:") + print(f" Challenge ID: {challenge_id}") + print(f" GitHub Token: {GITHUB_AUTH_TOKEN[:8]}...{GITHUB_AUTH_TOKEN[-4:] if len(GITHUB_AUTH_TOKEN) > 12 else '***'}") + print(f" Repository: {GITHUB_REPOSITORY}") + print(f" Branch: {GITHUB_BRANCH}") + + # Check if challenge has GitHub fields configured + print(f"\n๐Ÿ“‹ To enable EvalAI โ†’ GitHub sync, ensure your challenge has:") + print(f" โ€ข github_repository: '{GITHUB_REPOSITORY}'") + print(f" โ€ข github_branch: '{GITHUB_BRANCH}'") + print(f" โ€ข github_token: [your GitHub personal access token]") + print(f"\n๐Ÿ’ก Check these in your EvalAI challenge settings") + + print(f"\n๐Ÿ”ง Django Signal Sync Architecture:") + print(f" โ€ข challenge_details_sync signal triggers on Challenge updates") + print(f" โ€ข challenge_phase_details_sync signal triggers on Phase updates") + print(f" โ€ข No Celery/background tasks needed") + print(f" โ€ข Sync happens immediately in same request") + + print(f"\n๐Ÿ› If sync isn't working, check:") + print(f" โ€ข EvalAI logs for signal execution") + print(f" โ€ข Signal handlers are properly registered") + print(f" โ€ข GitHub fields are saved in challenge model") + print(f" โ€ข No errors in github_utils.py functions") + + except Exception as e: + print(f"โ„น๏ธ Could not retrieve sync status: {e}") + except requests.exceptions.ConnectionError as conn_err: # Handle connection errors specifically for localhost if is_localhost: @@ -176,6 +452,110 @@ def configure_requests_for_localhost(): ) print(error_message) os.environ["CHALLENGE_ERRORS"] = error_message + elif response.status_code == 404: + print(f"\nโŒ 404 Not Found Error") + print(f" The API endpoint was not found: {url}") + print(f" This usually means the URL structure has changed in the backend.") + print(f"\n๐Ÿ” Debugging Information:") + print(f" Team PK: {CHALLENGE_HOST_TEAM_PK}") + print(f" EvalAI Host: {EVALAI_HOST_URL}") + print(f" Full URL: {url}") + print(f"\n๐Ÿ’ก Possible Solutions:") + print(f" 1. Check if the team_pk ({CHALLENGE_HOST_TEAM_PK}) is correct") + print(f" 2. Verify the EvalAI server is running and accessible") + print(f" 3. Check if the API endpoint structure has changed") + print(f" 4. Try accessing the EvalAI admin interface to verify team ID") + + # First test basic connectivity + print(f"\n๐Ÿ” Testing basic server connectivity...") + if test_basic_connectivity(EVALAI_HOST_URL): + print(f" โœ… Server is reachable, testing API endpoints...") + + # Automatically test different endpoint patterns + print(f"\n๐Ÿงช Automatically testing different endpoint patterns...") + endpoint_results = test_api_endpoints(EVALAI_HOST_URL, CHALLENGE_HOST_TEAM_PK, HOST_AUTH_TOKEN) + + if any(r.get("accessible", False) for r in endpoint_results.values()): + print(f"\nโœ… Found working endpoints! Retrying with working pattern...") + working_endpoints = [p for p, r in endpoint_results.items() if r.get("accessible", False)] + + # Try the first working endpoint + working_endpoint = working_endpoints[0] + print(f" ๐Ÿš€ Retrying with: {working_endpoint}") + + # Update the URL and retry the request + new_url = f"{EVALAI_HOST_URL}{working_endpoint}" + print(f" ๐Ÿ“ก New API Endpoint: {new_url}") + + try: + print(f"\n๐Ÿ”„ Retrying request with working endpoint...") + response = requests.post(new_url, data=data, headers=headers, files=file, verify=verify_ssl) + + if response.status_code in [200, 201, 202]: + print("\nโœ… Challenge processed successfully on EvalAI with working endpoint!") + + # If this was a challenge creation/update, try to get the challenge ID for sync status + if VALIDATION_STEP != "True" and GITHUB_AUTH_TOKEN: + try: + response_data = response.json() + if "id" in response_data: + challenge_id = response_data["id"] + print(f"\n๐Ÿ”„ Checking sync status for challenge {challenge_id}...") + sync_status = check_sync_status(EVALAI_HOST_URL, challenge_id, HOST_AUTH_TOKEN) + if sync_status: + print(f"โœ… Sync status retrieved: {sync_status}") + except Exception as e: + print(f"โ„น๏ธ Could not retrieve sync status: {e}") + + # Success - clear errors and continue + os.environ["CHALLENGE_ERRORS"] = "False" + print(f"\n๐ŸŽ‰ Successfully processed challenge with working endpoint!") + print(f"๐Ÿ’ก Update your config.py with this working pattern:") + print(f" CHALLENGE_CONFIG_VALIDATION_URL = \"{working_endpoint}\"") + print(f" CHALLENGE_CREATE_OR_UPDATE_URL = \"{working_endpoint.replace('validate_challenge_config', 'create_or_update_github_challenge')}\"") + + # Continue with success flow + zip_file.close() + os.remove(zip_file.name) + print("\nExiting the {} script after success\n".format(os.path.basename(__file__))) + sys.exit(0) + + else: + print(f"\nโŒ Retry failed with status: {response.status_code}") + print(f" Response: {response.text}") + + except Exception as retry_err: + print(f"\nโŒ Retry failed with error: {retry_err}") + + print(f"\n๐Ÿ’ก Update your config.py with this working pattern:") + print(f" CHALLENGE_CONFIG_VALIDATION_URL = \"{working_endpoint}\"") + print(f" CHALLENGE_CREATE_OR_UPDATE_URL = \"{working_endpoint.replace('validate_challenge_config', 'create_or_update_github_challenge')}\"") + + else: + print(f"\nโŒ No working endpoints found. Please check:") + print(f" โ€ข Team PK is correct: {CHALLENGE_HOST_TEAM_PK}") + print(f" โ€ข EvalAI server is running at: {EVALAI_HOST_URL}") + print(f" โ€ข Your authentication token is valid") + else: + print(f" โŒ Server is not reachable. Please check:") + print(f" โ€ข EvalAI server is running") + print(f" โ€ข URL is correct: {EVALAI_HOST_URL}") + print(f" โ€ข Network connectivity") + + # Docker-specific suggestions + if "host.docker.internal" in EVALAI_HOST_URL: + print(f"\n๐Ÿณ Docker-specific suggestions:") + print(f" โ€ข Try using host machine's actual IP address instead of host.docker.internal") + print(f" โ€ข Ensure EvalAI server is binding to 0.0.0.0:8000, not 127.0.0.1:8000") + print(f" โ€ข Check if host.docker.internal resolves correctly in your Docker environment") + print(f" โ€ข Alternative URLs to try:") + print(f" - http://172.17.0.1:8000 (Docker bridge network gateway)") + print(f" - http://host.docker.internal:8000 (current)") + print(f" - http://localhost:8000 (if running from host)") + print(f" - http://:8000 (your actual host IP)") + + error_message = f"\n404 Not Found: API endpoint not found at {url}" + os.environ["CHALLENGE_ERRORS"] = error_message else: print( "\nFollowing errors occurred while validating the challenge config: {}".format( @@ -259,4 +639,4 @@ def configure_requests_for_localhost(): ) sys.exit(1) - print("\nExiting the {} script after success\n".format(os.path.basename(__file__))) \ No newline at end of file + print("\nExiting the {} script after success\n".format(os.path.basename(__file__))) From a3f0e7c75ec29d6be68f5fdf94b72265c5c3ea6c Mon Sep 17 00:00:00 2001 From: ZahedR_327 Date: Thu, 28 Aug 2025 13:14:06 +0530 Subject: [PATCH 35/40] Update challenge_processing_script.py --- github/challenge_processing_script.py | 407 ++------------------------ 1 file changed, 21 insertions(+), 386 deletions(-) diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index 7c49b5ce2..5ae5dd785 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -42,18 +42,12 @@ # Fallback for GITHUB_BRANCH if not imported from config if 'GITHUB_BRANCH' not in globals(): - GITHUB_BRANCH = os.getenv("GITHUB_REF_NAME") or os.getenv("GITHUB_BRANCH") or os.getenv("GITHUB_REF", "refs/heads/main").replace("refs/heads/", "") or "main" + GITHUB_BRANCH = os.getenv("GITHUB_REF_NAME") or os.getenv("GITHUB_BRANCH") or os.getenv("GITHUB_REF", "refs/heads/challenge").replace("refs/heads/", "") or "challenge" def is_localhost_url(url): """ Check if the provided URL is a localhost URL - - Arguments: - url {str}: The URL to check - - Returns: - bool: True if it's a localhost URL, False otherwise """ localhost_indicators = [ "127.0.0.1", @@ -78,215 +72,40 @@ def configure_requests_for_localhost(): """ # Disable SSL warnings for localhost development urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - print("INFO: SSL verification disabled for localhost development server") def test_github_access(github_token, repository): """ Tests GitHub repository access to verify token permissions """ - print(f"\n๐Ÿ” Testing GitHub repository access...") - print(f" Repository: {repository}") - print(f" Token: {github_token[:8]}...{github_token[-4:] if len(github_token) > 12 else '***'}") - try: from github import Github client = Github(github_token) - - # Try to get the repository - repo = client.get_repo(repository) - print(f" โœ… Repository accessible: {repo.full_name}") - print(f" ๐Ÿ“ Default branch: {repo.default_branch}") - print(f" ๐Ÿ”’ Private: {repo.private}") - - # Check permissions - user = client.get_user() - print(f" ๐Ÿ‘ค Authenticated as: {user.login}") - - # Try to get repository contents - try: - contents = repo.get_contents("") - print(f" ๐Ÿ“„ Repository contents accessible") - except Exception as e: - print(f" โš ๏ธ Cannot read repository contents: {e}") - - return True - - except Exception as e: - print(f" โŒ GitHub access failed: {e}") - if "Bad credentials" in str(e): - print(f" ๐Ÿ’ก Token is invalid or expired") - elif "Not Found" in str(e): - print(f" ๐Ÿ’ก Repository not found or token doesn't have access") - elif "Bad credentials" in str(e): - print(f" ๐Ÿ’ก Token doesn't have sufficient permissions") - return False - - -def test_basic_connectivity(evalai_host_url): - """ - Tests basic connectivity to the EvalAI server - """ - print(f"\n๐Ÿ” Testing basic connectivity to EvalAI server...") - print(f" URL: {evalai_host_url}") - - # Check if this is a Docker environment - if "host.docker.internal" in evalai_host_url: - print(f" ๐Ÿณ Docker environment detected (host.docker.internal)") - print(f" ๐Ÿ’ก This means you're running from inside a Docker container") - print(f" ๐Ÿ’ก host.docker.internal should resolve to the host machine") - - try: - # Try to access the root or admin endpoint - response = requests.get( - f"{evalai_host_url}/", - verify=not is_localhost_url(evalai_host_url), - timeout=10 - ) - print(f" โœ… Root endpoint accessible (Status: {response.status_code})") + client.get_repo(repository) return True - except requests.exceptions.ConnectionError: - print(f" โŒ Connection failed - server not reachable") - if "host.docker.internal" in evalai_host_url: - print(f" ๐Ÿณ Docker troubleshooting:") - print(f" โ€ข Ensure EvalAI server is running on host machine") - print(f" โ€ข Check if server is binding to 0.0.0.0:8000 (not just 127.0.0.1)") - print(f" โ€ข Verify Docker can reach host.docker.internal") - print(f" โ€ข Try using host machine's IP address instead") + except Exception: return False - except Exception as e: - print(f" โš ๏ธ Unexpected error: {e}") - return False - - -def test_api_endpoints(evalai_host_url, team_pk, host_auth_token): - """ - Tests different API endpoint patterns to find the correct one - """ - print(f"\n๐Ÿงช Testing API endpoint patterns...") - print(f" Team PK: {team_pk}") - print(f" EvalAI Host: {evalai_host_url}") - - # Different endpoint patterns to test - endpoint_patterns = [ - f"/api/v1/challenges/challenge_host_team/{team_pk}/validate_challenge_config/", - f"/api/challenges/challenge_host_team/{team_pk}/validate_challenge_config/", - f"/api/v1/challenges/{team_pk}/validate_challenge_config/", - f"/api/challenges/{team_pk}/validate_challenge_config/", - f"/api/v1/challenges/challenge/challenge_host_team/{team_pk}/validate_challenge_config/", - f"/api/challenges/challenge/challenge_host_team/{team_pk}/validate_challenge_config/", - # Additional patterns that might work - f"/api/v1/challenges/validate_challenge_config/", - f"/api/challenges/validate_challenge_config/", - f"/api/v1/challenge_host_team/{team_pk}/validate_challenge_config/", - f"/api/challenge_host_team/{team_pk}/validate_challenge_config/", - f"/api/v1/challenges/{team_pk}/validate/", - f"/api/challenges/{team_pk}/validate/", - ] - - results = {} - headers = get_request_header(host_auth_token) - - for pattern in endpoint_patterns: - test_url = f"{evalai_host_url}{pattern}" - print(f"\n Testing: {pattern}") - - try: - # Send a simple GET request to test the endpoint - response = requests.get( - test_url, - headers=headers, - verify=not is_localhost_url(evalai_host_url), - timeout=10 - ) - - status = response.status_code - results[pattern] = { - "status": status, - "url": test_url, - "accessible": status != 404 - } - - if status == 404: - print(f" โŒ 404 Not Found") - elif status == 401: - print(f" ๐Ÿ”’ 401 Unauthorized (endpoint exists but auth failed)") - elif status == 403: - print(f" ๐Ÿšซ 403 Forbidden (endpoint exists but access denied)") - elif status == 200: - print(f" โœ… 200 OK (endpoint accessible)") - else: - print(f" โš ๏ธ {status} (endpoint exists, status: {status})") - - except requests.exceptions.ConnectionError: - print(f" โŒ Connection Error") - results[pattern] = {"status": "Connection Error", "url": test_url, "accessible": False} - except Exception as e: - print(f" โŒ Error: {e}") - results[pattern] = {"status": f"Error: {e}", "url": test_url, "accessible": False} - - # Find the best endpoint - working_endpoints = [p for p, r in results.items() if r.get("accessible", False)] - - if working_endpoints: - print(f"\nโœ… Found working endpoints:") - for endpoint in working_endpoints: - print(f" โ€ข {endpoint}") - print(f"\n๐Ÿ’ก Update your config.py with one of these working patterns") - else: - print(f"\nโŒ No working endpoints found") - print(f" Check your team_pk and EvalAI server configuration") - - return results def setup_one_way_sync(): """ Sets up one-way sync from EvalAI to GitHub """ - print(f"\n๐Ÿ”„ Setting up one-way sync (EvalAI โ†’ GitHub)...") - print(f" Repository: {GITHUB_REPOSITORY}") - print(f" EvalAI Server: {EVALAI_HOST_URL}") - print(f" Using GitHub token: {GITHUB_AUTH_TOKEN[:8]}...{GITHUB_AUTH_TOKEN[-4:] if len(GITHUB_AUTH_TOKEN) > 12 else '***'}") - # Test GitHub repository access github_access = test_github_access(GITHUB_AUTH_TOKEN, GITHUB_REPOSITORY) if github_access: - print(f"โœ… GitHub repository access verified!") - print(f" EvalAI changes will automatically sync to GitHub") - print(f" GitHub changes will NOT sync back to EvalAI (by design)") - print("\n๐Ÿ’ก How it works:") - print(" 1. Make changes in EvalAI UI") - print(" 2. Changes are saved to database") - print(" 3. Django signal automatically triggers GitHub sync") - print(" 4. GitHub repository is updated with latest changes") - print(" 5. User gets immediate feedback (no waiting)") - - print(f"\nโš ๏ธ IMPORTANT: Ensure your EvalAI challenge has these fields configured:") - print(f" โ€ข github_repository: '{GITHUB_REPOSITORY}'") - print(f" โ€ข github_branch: '{GITHUB_BRANCH}' (actual repository default branch)") - print(f" โ€ข github_token: [your GitHub personal access token]") - print(f"\n๐Ÿ’ก These must be set in the EvalAI challenge settings for sync to work") - print(f"๐Ÿ’ก Note: Your repository uses '{GITHUB_BRANCH}' branch, not 'main'") - print(f"\n๐Ÿ”ง Backend uses Django signals for automatic sync (no Celery needed)") - print(f" โ€ข challenge_details_sync signal for challenge updates") - print(f" โ€ข challenge_phase_details_sync signal for phase updates") - return True else: - print(f"โŒ GitHub repository access failed!") - print(f" EvalAI โ†’ GitHub sync will not work until this is resolved") - print(f" Check your GitHub token permissions and repository access") + print(f"โŒ GitHub repository access failed! Ensure your token has access to {GITHUB_REPOSITORY}.") return False if __name__ == "__main__": - if GITHUB_CONTEXT["event"]["head_commit"]["message"].startswith("evalai_bot"): + if GITHUB_CONTEXT.get("event", {}).get("head_commit", {}).get("message", "").startswith("evalai_bot"): print("Sync from Evalai") sys.exit(0) - configs = load_host_configs(HOST_CONFIG_FILE_PATH) if configs: HOST_AUTH_TOKEN = configs[0] @@ -299,49 +118,30 @@ def setup_one_way_sync(): is_localhost = is_localhost_url(EVALAI_HOST_URL) runner_info = get_runner_info() - print(f"\n๐ŸŒ EvalAI Server: {EVALAI_HOST_URL}") - print(f"๐Ÿ  Localhost Mode: {is_localhost}") - print(f"๐Ÿค– Self-hosted Runner: {runner_info['is_self_hosted']}") - print(f"๐Ÿ”„ Sync Mode: One-way (EvalAI โ†’ GitHub)") - - if GITHUB_AUTH_TOKEN: - print(f" GitHub Token: {GITHUB_AUTH_TOKEN[:8]}...{GITHUB_AUTH_TOKEN[-4:] if len(GITHUB_AUTH_TOKEN) > 12 else '***'}") - print(f" Token Source: AUTH_TOKEN repository secret") - else: - print(f" GitHub Token: Not provided") - print(f" To enable: Add AUTH_TOKEN to repository secrets") - if is_localhost: configure_requests_for_localhost() - print(f"INFO: Using localhost server: {EVALAI_HOST_URL}") # Setup one-way sync configuration if GITHUB_AUTH_TOKEN: setup_one_way_sync() else: - print("โ„น๏ธ One-way sync not configured") - print(" Add AUTH_TOKEN to repository secrets to enable automatic GitHub sync") + print("โ„น๏ธ One-way sync not configured. Add AUTH_TOKEN to repository secrets.") # Fetching the url if VALIDATION_STEP == "True": - print(f"\n๐Ÿ” VALIDATION MODE: Validating challenge configuration...") url = "{}{}".format( EVALAI_HOST_URL, CHALLENGE_CONFIG_VALIDATION_URL.format(CHALLENGE_HOST_TEAM_PK), ) else: - print(f"\n๐Ÿš€ CREATION MODE: Creating/updating challenge...") url = "{}{}".format( EVALAI_HOST_URL, CHALLENGE_CREATE_OR_UPDATE_URL.format(CHALLENGE_HOST_TEAM_PK), ) - print(f"๐Ÿ“ก API Endpoint: {url}") - headers = get_request_header(HOST_AUTH_TOKEN) # Creating the challenge zip file and storing in a dict to send to EvalAI - print(f"\n๐Ÿ“ฆ Creating challenge configuration package...") create_challenge_zip_file(CHALLENGE_ZIP_FILE_PATH, IGNORE_DIRS, IGNORE_FILES) zip_file = open(CHALLENGE_ZIP_FILE_PATH, "rb") file = {"zip_configuration": zip_file} @@ -358,16 +158,14 @@ def setup_one_way_sync(): # Configure SSL verification based on whether we're using localhost verify_ssl = not is_localhost - print(f"๐Ÿ”’ SSL Verification: {'Disabled (localhost)' if not verify_ssl else 'Enabled'}") try: - print(f"\n๐ŸŒ Sending request to EvalAI server...") response = requests.post(url, data=data, headers=headers, files=file, verify=verify_ssl) if response.status_code != http.HTTPStatus.OK and response.status_code != http.HTTPStatus.CREATED: response.raise_for_status() else: - print("\nโœ… Challenge processed successfully on EvalAI") + print("โœ… Challenge processed successfully on EvalAI") # If this was a challenge creation/update, try to get the challenge ID for sync status if VALIDATION_STEP != "True" and GITHUB_AUTH_TOKEN: @@ -375,64 +173,16 @@ def setup_one_way_sync(): response_data = response.json() if "id" in response_data: challenge_id = response_data["id"] - print(f"\n๐Ÿ”„ Checking sync status for challenge {challenge_id}...") sync_status = check_sync_status(EVALAI_HOST_URL, challenge_id, HOST_AUTH_TOKEN) if sync_status: - print(f"โœ… Sync status retrieved: {sync_status}") - - # Additional debugging for sync issues - print(f"\n๐Ÿ” Sync Debugging Information:") - print(f" Challenge ID: {challenge_id}") - print(f" GitHub Token: {GITHUB_AUTH_TOKEN[:8]}...{GITHUB_AUTH_TOKEN[-4:] if len(GITHUB_AUTH_TOKEN) > 12 else '***'}") - print(f" Repository: {GITHUB_REPOSITORY}") - print(f" Branch: {GITHUB_BRANCH}") - - # Check if challenge has GitHub fields configured - print(f"\n๐Ÿ“‹ To enable EvalAI โ†’ GitHub sync, ensure your challenge has:") - print(f" โ€ข github_repository: '{GITHUB_REPOSITORY}'") - print(f" โ€ข github_branch: '{GITHUB_BRANCH}'") - print(f" โ€ข github_token: [your GitHub personal access token]") - print(f"\n๐Ÿ’ก Check these in your EvalAI challenge settings") - - print(f"\n๐Ÿ”ง Django Signal Sync Architecture:") - print(f" โ€ข challenge_details_sync signal triggers on Challenge updates") - print(f" โ€ข challenge_phase_details_sync signal triggers on Phase updates") - print(f" โ€ข No Celery/background tasks needed") - print(f" โ€ข Sync happens immediately in same request") - - print(f"\n๐Ÿ› If sync isn't working, check:") - print(f" โ€ข EvalAI logs for signal execution") - print(f" โ€ข Signal handlers are properly registered") - print(f" โ€ข GitHub fields are saved in challenge model") - print(f" โ€ข No errors in github_utils.py functions") - - except Exception as e: - print(f"โ„น๏ธ Could not retrieve sync status: {e}") + print("โœ… Sync status retrieved") + except Exception: + pass except requests.exceptions.ConnectionError as conn_err: # Handle connection errors specifically for localhost if is_localhost: - error_message = "\n๐Ÿšจ LOCALHOST SERVER CONNECTION FAILED\n" - error_message += f"โŒ Could not connect to your localhost EvalAI server at: {EVALAI_HOST_URL}\n" - error_message += "\n๐Ÿ“‹ Please check the following:\n" - error_message += " 1. Is your EvalAI server running?\n" - error_message += f" 2. Is it accessible at {EVALAI_HOST_URL}?\n" - error_message += " 3. Check server logs for any startup errors\n" - - if runner_info['is_self_hosted']: - error_message += "\n๐Ÿ’ก Self-hosted runner troubleshooting:\n" - error_message += " โ€ข Verify runner can reach the server: ping/curl test\n" - error_message += " โ€ข Check network configuration and firewall settings\n" - error_message += " โ€ข Ensure server is binding to correct interface (0.0.0.0 vs 127.0.0.1)\n" - else: - error_message += "\nโš ๏ธ CONFIGURATION ISSUE:\n" - error_message += " You're using a GitHub-hosted runner with a localhost URL.\n" - error_message += " GitHub-hosted runners cannot access your local machine.\n" - error_message += " Please set up a self-hosted runner for localhost development.\n" - - error_message += "\n๐Ÿ’ก To start your local server, typically run:\n" - error_message += " python manage.py runserver 0.0.0.0:8888\n" - error_message += f"\nOriginal error: {conn_err}" + error_message = "\nโŒ Could not connect to your localhost EvalAI server." else: error_message = f"\nConnection failed to EvalAI server: {conn_err}" @@ -446,119 +196,19 @@ def setup_one_way_sync(): if response.status_code in EVALAI_ERROR_CODES: is_token_valid = validate_token(response.json()) if is_token_valid: - error = response.json()["error"] - error_message = "\nFollowing errors occurred while validating the challenge config:\n{}".format( + error = response.json().get("error", str(err)) + error_message = "\nErrors occurred while validating the challenge config:\n{}".format( error ) print(error_message) os.environ["CHALLENGE_ERRORS"] = error_message elif response.status_code == 404: - print(f"\nโŒ 404 Not Found Error") - print(f" The API endpoint was not found: {url}") - print(f" This usually means the URL structure has changed in the backend.") - print(f"\n๐Ÿ” Debugging Information:") - print(f" Team PK: {CHALLENGE_HOST_TEAM_PK}") - print(f" EvalAI Host: {EVALAI_HOST_URL}") - print(f" Full URL: {url}") - print(f"\n๐Ÿ’ก Possible Solutions:") - print(f" 1. Check if the team_pk ({CHALLENGE_HOST_TEAM_PK}) is correct") - print(f" 2. Verify the EvalAI server is running and accessible") - print(f" 3. Check if the API endpoint structure has changed") - print(f" 4. Try accessing the EvalAI admin interface to verify team ID") - - # First test basic connectivity - print(f"\n๐Ÿ” Testing basic server connectivity...") - if test_basic_connectivity(EVALAI_HOST_URL): - print(f" โœ… Server is reachable, testing API endpoints...") - - # Automatically test different endpoint patterns - print(f"\n๐Ÿงช Automatically testing different endpoint patterns...") - endpoint_results = test_api_endpoints(EVALAI_HOST_URL, CHALLENGE_HOST_TEAM_PK, HOST_AUTH_TOKEN) - - if any(r.get("accessible", False) for r in endpoint_results.values()): - print(f"\nโœ… Found working endpoints! Retrying with working pattern...") - working_endpoints = [p for p, r in endpoint_results.items() if r.get("accessible", False)] - - # Try the first working endpoint - working_endpoint = working_endpoints[0] - print(f" ๐Ÿš€ Retrying with: {working_endpoint}") - - # Update the URL and retry the request - new_url = f"{EVALAI_HOST_URL}{working_endpoint}" - print(f" ๐Ÿ“ก New API Endpoint: {new_url}") - - try: - print(f"\n๐Ÿ”„ Retrying request with working endpoint...") - response = requests.post(new_url, data=data, headers=headers, files=file, verify=verify_ssl) - - if response.status_code in [200, 201, 202]: - print("\nโœ… Challenge processed successfully on EvalAI with working endpoint!") - - # If this was a challenge creation/update, try to get the challenge ID for sync status - if VALIDATION_STEP != "True" and GITHUB_AUTH_TOKEN: - try: - response_data = response.json() - if "id" in response_data: - challenge_id = response_data["id"] - print(f"\n๐Ÿ”„ Checking sync status for challenge {challenge_id}...") - sync_status = check_sync_status(EVALAI_HOST_URL, challenge_id, HOST_AUTH_TOKEN) - if sync_status: - print(f"โœ… Sync status retrieved: {sync_status}") - except Exception as e: - print(f"โ„น๏ธ Could not retrieve sync status: {e}") - - # Success - clear errors and continue - os.environ["CHALLENGE_ERRORS"] = "False" - print(f"\n๐ŸŽ‰ Successfully processed challenge with working endpoint!") - print(f"๐Ÿ’ก Update your config.py with this working pattern:") - print(f" CHALLENGE_CONFIG_VALIDATION_URL = \"{working_endpoint}\"") - print(f" CHALLENGE_CREATE_OR_UPDATE_URL = \"{working_endpoint.replace('validate_challenge_config', 'create_or_update_github_challenge')}\"") - - # Continue with success flow - zip_file.close() - os.remove(zip_file.name) - print("\nExiting the {} script after success\n".format(os.path.basename(__file__))) - sys.exit(0) - - else: - print(f"\nโŒ Retry failed with status: {response.status_code}") - print(f" Response: {response.text}") - - except Exception as retry_err: - print(f"\nโŒ Retry failed with error: {retry_err}") - - print(f"\n๐Ÿ’ก Update your config.py with this working pattern:") - print(f" CHALLENGE_CONFIG_VALIDATION_URL = \"{working_endpoint}\"") - print(f" CHALLENGE_CREATE_OR_UPDATE_URL = \"{working_endpoint.replace('validate_challenge_config', 'create_or_update_github_challenge')}\"") - - else: - print(f"\nโŒ No working endpoints found. Please check:") - print(f" โ€ข Team PK is correct: {CHALLENGE_HOST_TEAM_PK}") - print(f" โ€ข EvalAI server is running at: {EVALAI_HOST_URL}") - print(f" โ€ข Your authentication token is valid") - else: - print(f" โŒ Server is not reachable. Please check:") - print(f" โ€ข EvalAI server is running") - print(f" โ€ข URL is correct: {EVALAI_HOST_URL}") - print(f" โ€ข Network connectivity") - - # Docker-specific suggestions - if "host.docker.internal" in EVALAI_HOST_URL: - print(f"\n๐Ÿณ Docker-specific suggestions:") - print(f" โ€ข Try using host machine's actual IP address instead of host.docker.internal") - print(f" โ€ข Ensure EvalAI server is binding to 0.0.0.0:8000, not 127.0.0.1:8000") - print(f" โ€ข Check if host.docker.internal resolves correctly in your Docker environment") - print(f" โ€ข Alternative URLs to try:") - print(f" - http://172.17.0.1:8000 (Docker bridge network gateway)") - print(f" - http://host.docker.internal:8000 (current)") - print(f" - http://localhost:8000 (if running from host)") - print(f" - http://:8000 (your actual host IP)") - - error_message = f"\n404 Not Found: API endpoint not found at {url}" + error_message = "\n404 Not Found: API endpoint not found" + print(error_message) os.environ["CHALLENGE_ERRORS"] = error_message else: print( - "\nFollowing errors occurred while validating the challenge config: {}".format( + "\nErrors occurred while validating the challenge config: {}".format( err ) ) @@ -566,13 +216,13 @@ def setup_one_way_sync(): except Exception as e: if VALIDATION_STEP == "True": - error_message = "\nFollowing errors occurred while validating the challenge config: {}".format( + error_message = "\nErrors occurred while validating the challenge config: {}".format( e ) print(error_message) os.environ["CHALLENGE_ERRORS"] = error_message else: - error_message = "\nFollowing errors occurred while processing the challenge config: {}".format( + error_message = "\nErrors occurred while processing the challenge config: {}".format( e ) print(error_message) @@ -599,22 +249,12 @@ def setup_one_way_sync(): ) if is_localhost_connection_error or is_github_hosted_localhost_error: - print("\nโ„น๏ธ Localhost connection error detected. Skipping GitHub issue creation.") - if is_github_hosted_localhost_error: - print(" This is expected when using GitHub-hosted runners with localhost URLs.") - print(" Please configure a self-hosted runner for local development.") - else: - print(" This is expected when your local EvalAI server isn't running.") - # Fail the job so CI visibly reports the problem sys.exit(1) elif VALIDATION_STEP == "True" and check_if_pull_request(): pr_number = GITHUB_CONTEXT.get("event", {}).get("number") - if not pr_number: - print("โš ๏ธ Warning: Could not get PR number from GITHUB_CONTEXT") - print(" Skipping pull request comment creation") - else: + if pr_number: add_pull_request_comment( GITHUB_AUTH_TOKEN, os.path.basename(GITHUB_REPOSITORY), @@ -623,7 +263,7 @@ def setup_one_way_sync(): ) else: issue_title = ( - "Following errors occurred while validating the challenge config:" + "Errors occurred while validating the challenge config:" ) repo_name = os.path.basename(GITHUB_REPOSITORY) if GITHUB_REPOSITORY else "" create_github_repository_issue( @@ -632,11 +272,6 @@ def setup_one_way_sync(): issue_title, errors, ) - print( - "\nExiting the {} script after failure\n".format( - os.path.basename(__file__) - ) - ) sys.exit(1) - print("\nExiting the {} script after success\n".format(os.path.basename(__file__))) + print("Exiting the {} script after success".format(os.path.basename(__file__))) From 450c52d49b4941f98930fb68adb8ce3aa24bd4fe Mon Sep 17 00:00:00 2001 From: ZahedR_327 Date: Thu, 28 Aug 2025 13:18:09 +0530 Subject: [PATCH 36/40] Update challenge_processing_script.py --- github/challenge_processing_script.py | 32 ++++++++++++++++++--------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/github/challenge_processing_script.py b/github/challenge_processing_script.py index 5ae5dd785..d87dec774 100644 --- a/github/challenge_processing_script.py +++ b/github/challenge_processing_script.py @@ -182,15 +182,19 @@ def setup_one_way_sync(): except requests.exceptions.ConnectionError as conn_err: # Handle connection errors specifically for localhost if is_localhost: - error_message = "\nโŒ Could not connect to your localhost EvalAI server." + print("\nโ„น๏ธ Localhost connection error detected. Skipping GitHub issue creation.") + if not runner_info['is_self_hosted']: + print(" This is expected when using GitHub-hosted runners with localhost URLs.") + print(" Please configure a self-hosted runner for local development.") + else: + print(" This is expected when your local EvalAI server isn't running.") + sys.exit(1) else: error_message = f"\nConnection failed to EvalAI server: {conn_err}" - - print(error_message) - os.environ["CHALLENGE_ERRORS"] = error_message - - # Fail the job so CI visibly reports the problem - sys.exit(1) + print(error_message) + os.environ["CHALLENGE_ERRORS"] = error_message + # Fail the job so CI visibly reports the problem + sys.exit(1) except requests.exceptions.HTTPError as err: if response.status_code in EVALAI_ERROR_CODES: @@ -249,12 +253,20 @@ def setup_one_way_sync(): ) if is_localhost_connection_error or is_github_hosted_localhost_error: - # Fail the job so CI visibly reports the problem + print("\nโ„น๏ธ Localhost connection error detected. Skipping GitHub issue creation.") + if is_github_hosted_localhost_error: + print(" This is expected when using GitHub-hosted runners with localhost URLs.") + print(" Please configure a self-hosted runner for local development.") + else: + print(" This is expected when your local EvalAI server isn't running.") sys.exit(1) elif VALIDATION_STEP == "True" and check_if_pull_request(): pr_number = GITHUB_CONTEXT.get("event", {}).get("number") - if pr_number: + if not pr_number: + print("โš ๏ธ Warning: Could not get PR number from GITHUB_CONTEXT") + print(" Skipping pull request comment creation") + else: add_pull_request_comment( GITHUB_AUTH_TOKEN, os.path.basename(GITHUB_REPOSITORY), @@ -274,4 +286,4 @@ def setup_one_way_sync(): ) sys.exit(1) - print("Exiting the {} script after success".format(os.path.basename(__file__))) + print("\nExiting the {} script after success\n".format(os.path.basename(__file__))) From eb6bdbf4f385f78084da474f76c84bdcecc2d9f0 Mon Sep 17 00:00:00 2001 From: ZahedR_327 Date: Thu, 28 Aug 2025 13:21:11 +0530 Subject: [PATCH 37/40] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index a078f9c78..0bca46acf 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ If you are looking for a simple challenge configuration that you can replicate to create a challenge on EvalAI, then you are at the right place. Follow the instructions given below to get started. +> Note: EvalAI supports a sync between the Challenge UI and your GitHub repository. When you update challenge details in the EvalAI UI, changes are pushed to your repo (configured branch, typically `challenge`). When you push updates to the repo, the provided workflow validates and (re)processes the challenge on EvalAI. Ensure your challenge is configured with `github_repository`, `github_branch` (e.g., `challenge`), and `github_token`. + ## Directory Structure ``` From 1abc07c4986bd196d5f4a34ea5a9050431fa64b1 Mon Sep 17 00:00:00 2001 From: ZahedR_327 Date: Thu, 28 Aug 2025 13:21:51 +0530 Subject: [PATCH 38/40] Update README.md --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0bca46acf..7c132485f 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,6 @@ If you are looking for a simple challenge configuration that you can replicate to create a challenge on EvalAI, then you are at the right place. Follow the instructions given below to get started. -> Note: EvalAI supports a sync between the Challenge UI and your GitHub repository. When you update challenge details in the EvalAI UI, changes are pushed to your repo (configured branch, typically `challenge`). When you push updates to the repo, the provided workflow validates and (re)processes the challenge on EvalAI. Ensure your challenge is configured with `github_repository`, `github_branch` (e.g., `challenge`), and `github_token`. - ## Directory Structure ``` @@ -127,6 +125,10 @@ In order to test the evaluation script locally before uploading it to [EvalAI](h Please replace them with real values before pushing changes to avoid build errors. + +EvalAI supports a sync between the Challenge UI and your GitHub repository. When you update challenge details in the EvalAI UI, changes are pushed to your repo (configured branch, typically `challenge`). When you push updates to the repo, the provided workflow validates and (re)processes the challenge on EvalAI. Ensure your challenge is configured with `github_repository`, `github_branch` (e.g., `challenge`), and `github_token`. + + ## Facing problems in creating a challenge? Please feel free to open issues on our [GitHub Repository](https://github.com/Cloud-CV/EvalAI-Starter/issues) or contact us at team@cloudcv.org if you have issues. From d60c5ed10f0f372d57a7be093916b53e38c2127b Mon Sep 17 00:00:00 2001 From: ZahedR_327 Date: Thu, 28 Aug 2025 13:22:56 +0530 Subject: [PATCH 39/40] Update config.py --- github/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/github/config.py b/github/config.py index 738b69dcd..224a6a007 100644 --- a/github/config.py +++ b/github/config.py @@ -6,12 +6,12 @@ HOST_CONFIG_FILE_PATH = "github/host_config.json" CHALLENGE_CONFIG_VALIDATION_URL = "/api/challenges/challenge/challenge_host_team/{}/validate_challenge_config/" CHALLENGE_CREATE_OR_UPDATE_URL = "/api/challenges/challenge/challenge_host_team/{}/create_or_update_github_challenge/" +GITHUB_SYNC_STATUS_URL = "/api/v1/challenges/{}/github/sync_status/" EVALAI_ERROR_CODES = [400, 401, 406] API_HOST_URL = "https://eval.ai" IGNORE_DIRS = [ ".git", ".github", - "github", "code_upload_challenge_evaluation", "remote_challenge_evaluation", ] @@ -24,5 +24,7 @@ ] CHALLENGE_ZIP_FILE_PATH = "challenge_config.zip" GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY") +# GitHub branch - use 'challenge' as the default branch +GITHUB_BRANCH = os.getenv("GITHUB_REF_NAME") or os.getenv("GITHUB_BRANCH") or "challenge" GITHUB_EVENT_NAME = os.getenv("GITHUB_EVENT_NAME") VALIDATION_STEP = os.getenv("IS_VALIDATION") From b5fbdf7c357a6792d3a0dc9d2a6ab42e3d4415e6 Mon Sep 17 00:00:00 2001 From: ZahedR_327 Date: Thu, 28 Aug 2025 13:25:49 +0530 Subject: [PATCH 40/40] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7c132485f..0c6acbc2c 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ In order to test the evaluation script locally before uploading it to [EvalAI](h Please replace them with real values before pushing changes to avoid build errors. -EvalAI supports a sync between the Challenge UI and your GitHub repository. When you update challenge details in the EvalAI UI, changes are pushed to your repo (configured branch, typically `challenge`). When you push updates to the repo, the provided workflow validates and (re)processes the challenge on EvalAI. Ensure your challenge is configured with `github_repository`, `github_branch` (e.g., `challenge`), and `github_token`. +EvalAI supports a sync between the Challenge UI and your GitHub repository. When you update challenge details in the EvalAI UI, changes are pushed to your repo (configured branch, typically `challenge`). When you push updates to the repo, the provided workflow validates and (re)processes the challenge on EvalAI. ## Facing problems in creating a challenge?