Optimized Test Suite #9376
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Optimized Test Suite | |
| on: | |
| push: | |
| branches: [ main, develop ] | |
| paths: | |
| - 'src/**' | |
| - 'pyproject.toml' | |
| - '.github/workflows/test-optimized.yml' | |
| pull_request: | |
| branches: [ main, develop ] | |
| paths: | |
| - 'src/**' | |
| - 'pyproject.toml' | |
| - '.github/workflows/test-optimized.yml' | |
| schedule: | |
| # Nightly extended tests at 3 AM UTC | |
| - cron: '0 3 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| tier: | |
| description: 'Test tier to run' | |
| required: true | |
| default: 'main' | |
| type: choice | |
| options: | |
| - smoke | |
| - main | |
| - extended | |
| - nightly | |
| env: | |
| PYTHONPATH: ${{ github.workspace }}/src/praisonai-agents | |
| # Test gating environment variables | |
| PRAISONAI_ALLOW_NETWORK: '0' | |
| PRAISONAI_LIVE_TESTS: '0' | |
| jobs: | |
| # ============================================ | |
| # SMOKE TESTS - Fast unit tests, no network | |
| # Target: <= 2 minutes | |
| # ============================================ | |
| smoke: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 16 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Cache pip packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-smoke-${{ hashFiles('**/pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip-smoke- | |
| ${{ runner.os }}-pip- | |
| - name: Install minimal dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install pytest pytest-asyncio pytest-timeout prompt_toolkit | |
| - uses: ./.github/actions/install-monorepo-packages | |
| with: | |
| agents-extras: ".[all]" | |
| wrapper-extras: ".[chat,code]" | |
| - name: Run Smoke Tests | |
| env: | |
| PRAISONAI_TEST_TIER: smoke | |
| PRAISONAI_ALLOW_NETWORK: '0' | |
| OPENAI_MODEL_NAME: gpt-4o-mini | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| cd src/praisonai | |
| python -m pytest tests/unit/test_c5_backward_compat.py \ | |
| tests/unit/test_c7_1_boundaries.py \ | |
| tests/unit/test_c9_backward_compat.py \ | |
| tests/unit/test_c9_1_boundaries.py \ | |
| tests/unit/test_c11_browser_backward_compat.py \ | |
| tests/unit/test_c12_mcp_backward_compat.py \ | |
| tests/unit/test_c13_sandbox_backward_compat.py \ | |
| tests/unit/test_c14_deploy_backward_compat.py \ | |
| -q --tb=line --timeout=30 --maxfail=5 | |
| python -m pytest tests/unit/cli/ \ | |
| ../praisonai-bot/tests/unit/cli/test_onboard_command.py \ | |
| ../praisonai-bot/tests/unit/cli/test_onboard_non_interactive.py \ | |
| -m "not slow and not network" \ | |
| -q --tb=line \ | |
| --timeout=30 \ | |
| --ignore=tests/fixtures \ | |
| --ignore=tests/unit/cli/test_message_queue.py \ | |
| --maxfail=5 | |
| - name: Run praisonai-agents subscription-auth smoke tests | |
| env: | |
| PRAISONAI_TEST_TIER: smoke | |
| PRAISONAI_ALLOW_NETWORK: '0' | |
| OPENAI_MODEL_NAME: gpt-4o-mini | |
| OPENAI_API_KEY: 'test-key' | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| cd src/praisonai-agents | |
| python -m pytest tests/smoke/test_subscription_auth.py \ | |
| tests/unit/auth/test_agent_auth_wiring.py \ | |
| -q --tb=line --timeout=60 --maxfail=5 | |
| - name: Standalone praisonai-code import smoke | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| python -m venv /tmp/praisonai-code-standalone | |
| /tmp/praisonai-code-standalone/bin/pip install -q -e src/praisonai-agents -e src/praisonai-code | |
| /tmp/praisonai-code-standalone/bin/python -c " | |
| import praisonai_code | |
| from praisonai_code.cli.configuration.resolver import ConfigResolver | |
| from praisonai_code._version import get_package_version | |
| from praisonai_code.llm.credentials import is_configured | |
| from praisonai_code.llm.catalogue import ModelCatalogue | |
| import toml | |
| assert hasattr(ConfigResolver, '__name__') | |
| assert callable(is_configured) | |
| assert ModelCatalogue().list_models(provider='openai') | |
| import praisonai_code.cli.main | |
| import praisonai_code.cli.app | |
| from praisonai_code.tool_resolver import ToolResolver | |
| assert ToolResolver is not None | |
| print('standalone ok', get_package_version(), 'toml', toml.__version__) | |
| " | |
| /tmp/praisonai-code-standalone/bin/praisonai-code --version | |
| /tmp/praisonai-code-standalone/bin/praisonai-code run --help | |
| cd src/praisonai-code | |
| for f in praisonai_code/cli/main.py praisonai_code/cli/app.py \ | |
| praisonai_code/cli/commands/run.py praisonai_code/cli/commands/chat.py \ | |
| praisonai_code/cli/commands/code.py; do | |
| if grep -E '^from praisonai\.|^import praisonai' "$f"; then | |
| echo "FAIL: module-level wrapper import in $f" | |
| exit 1 | |
| fi | |
| done | |
| echo "C7 hot-path import gate ok" | |
| bash "$GITHUB_WORKSPACE/scripts/check_c7_imports.sh" | |
| # Standalone actions run (no wrapper); default run/chat require praisonai | |
| set +e | |
| RUN_OUT=$(timeout 90 /tmp/praisonai-code-standalone/bin/praisonai-code run --output actions "Say hello in one word" 2>&1) | |
| RUN_CODE=$? | |
| set -e | |
| echo "$RUN_OUT" | tail -20 | |
| if echo "$RUN_OUT" | grep -qE 'ImportError|ModuleNotFoundError: No module named .praisonai'; then | |
| echo "FAIL: standalone actions run hit wrapper import error" | |
| exit 1 | |
| fi | |
| echo "standalone actions run import gate ok (exit $RUN_CODE)" | |
| set +e | |
| DEFAULT_RUN=$(timeout 15 /tmp/praisonai-code-standalone/bin/praisonai-code run "hi" 2>&1) | |
| set -e | |
| echo "$DEFAULT_RUN" | tail -10 | |
| if ! echo "$DEFAULT_RUN" | grep -qi 'install praisonai'; then | |
| echo "FAIL: default run should prompt for praisonai wrapper install" | |
| exit 1 | |
| fi | |
| echo "default run wrapper gate ok" | |
| set +e | |
| CHAT_RUN=$(timeout 10 /tmp/praisonai-code-standalone/bin/praisonai-code chat 2>&1) | |
| set -e | |
| echo "$CHAT_RUN" | tail -10 | |
| if ! echo "$CHAT_RUN" | grep -qi 'install praisonai'; then | |
| echo "FAIL: chat should prompt for praisonai wrapper install" | |
| exit 1 | |
| fi | |
| echo "chat wrapper gate ok" | |
| - name: Standalone praisonai-bot smoke (C9) | |
| run: | | |
| set -euo pipefail | |
| python -m venv /tmp/praisonai-bot-standalone | |
| /tmp/praisonai-bot-standalone/bin/pip install -q -e src/praisonai-agents -e src/praisonai-bot | |
| /tmp/praisonai-bot-standalone/bin/python -c " | |
| from praisonai_bot._version import __version__ as pkg_version | |
| import praisonai_bot | |
| from praisonai_bot.bots import Bot | |
| from praisonai_bot.bots._registry import resolve_adapter | |
| from praisonai_bot.scheduler.executor import ScheduledAgentExecutor | |
| from praisonai_bot.gateway import WebSocketGateway | |
| assert praisonai_bot.__version__ == pkg_version | |
| telegram = resolve_adapter('telegram') | |
| assert telegram.__name__ == 'TelegramBot' | |
| assert telegram.__module__.startswith('praisonai_bot.bots.') | |
| assert ScheduledAgentExecutor.__module__.startswith('praisonai_bot.') | |
| print('bot standalone ok', praisonai_bot.__version__, telegram.__name__) | |
| " | |
| /tmp/praisonai-bot-standalone/bin/praisonai-bot --help | |
| bash "$GITHUB_WORKSPACE/scripts/check_c9_bot_imports.sh" | |
| bash "$GITHUB_WORKSPACE/scripts/check_bot_code_imports.sh" | |
| /tmp/praisonai-bot-standalone/bin/python -c " | |
| import praisonai_bot.cli.commands.bot as bot_cmd | |
| import praisonai_bot.cli.commands.gateway as gw_cmd | |
| print('bot command modules import ok', bot_cmd.app, gw_cmd.app) | |
| " | |
| - name: Standalone praisonai-train smoke (C10) | |
| run: | | |
| set -euo pipefail | |
| python -m venv /tmp/praisonai-train-standalone | |
| /tmp/praisonai-train-standalone/bin/pip install -q -e src/praisonai-agents -e src/praisonai-train | |
| /tmp/praisonai-train-standalone/bin/python -c " | |
| import sys | |
| from praisonai_train._version import __version__ as pkg_version | |
| import praisonai_train | |
| assert praisonai_train.__version__ == pkg_version | |
| trainer_cls = praisonai_train.AgentTrainer | |
| scenario_cls = praisonai_train.TrainingScenario | |
| assert trainer_cls.__module__.startswith('praisonai_train.') | |
| for heavy in ('torch', 'unsloth', 'trl', 'datasets'): | |
| assert heavy not in sys.modules, f'{heavy} imported eagerly' | |
| print('train standalone ok', praisonai_train.__version__, trainer_cls.__name__) | |
| " | |
| /tmp/praisonai-train-standalone/bin/praisonai-train --help | |
| /tmp/praisonai-train-standalone/bin/python -c " | |
| import praisonai_train.cli.commands.train as train_cmd | |
| print('train command module imports ok', train_cmd.app) | |
| " | |
| bash "$GITHUB_WORKSPACE/scripts/check_c10_train_imports.sh" | |
| - name: Standalone praisonai-browser smoke (C11) | |
| run: | | |
| set -euo pipefail | |
| python -m venv /tmp/praisonai-browser-standalone | |
| /tmp/praisonai-browser-standalone/bin/pip install -q -e src/praisonai-agents -e src/praisonai-browser | |
| /tmp/praisonai-browser-standalone/bin/python -c " | |
| import sys | |
| from praisonai_browser._version import __version__ as pkg_version | |
| import praisonai_browser | |
| assert praisonai_browser.__version__ == pkg_version | |
| server_cls = praisonai_browser.BrowserServer | |
| assert server_cls.__name__ == 'BrowserServer' | |
| for heavy in ('fastapi', 'uvicorn', 'playwright'): | |
| assert heavy not in sys.modules, f'{heavy} imported eagerly' | |
| print('browser standalone ok', praisonai_browser.__version__, server_cls.__name__) | |
| " | |
| /tmp/praisonai-browser-standalone/bin/praisonai-browser --help | |
| bash "$GITHUB_WORKSPACE/scripts/check_c11_browser_imports.sh" | |
| python -m pytest src/praisonai-browser/tests/browser/test_protocol_compliance.py -q -m "not network" | |
| - name: Standalone praisonai-mcp smoke (C12) | |
| run: | | |
| set -euo pipefail | |
| python -m venv /tmp/praisonai-mcp-standalone | |
| /tmp/praisonai-mcp-standalone/bin/pip install -q -e src/praisonai-agents -e src/praisonai-mcp | |
| /tmp/praisonai-mcp-standalone/bin/python -c " | |
| import sys | |
| from praisonai_mcp._version import __version__ as pkg_version | |
| import praisonai_mcp | |
| assert praisonai_mcp.__version__ == pkg_version | |
| for heavy in ('fastapi', 'uvicorn'): | |
| assert heavy not in sys.modules, f'{heavy} imported eagerly' | |
| print('mcp standalone ok', praisonai_mcp.__version__) | |
| " | |
| /tmp/praisonai-mcp-standalone/bin/praisonai-mcp --help | |
| bash "$GITHUB_WORKSPACE/scripts/check_c12_mcp_imports.sh" | |
| python -m pytest src/praisonai-mcp/tests/mcp_server -q -m "not network" | |
| - name: Standalone praisonai-sandbox smoke (C13) | |
| run: | | |
| set -euo pipefail | |
| python -m venv /tmp/praisonai-sandbox-standalone | |
| /tmp/praisonai-sandbox-standalone/bin/pip install -q -e src/praisonai-agents -e src/praisonai-sandbox | |
| /tmp/praisonai-sandbox-standalone/bin/python -c " | |
| from praisonai_sandbox._version import __version__ as pkg_version | |
| import praisonai_sandbox | |
| assert praisonai_sandbox.__version__ == pkg_version | |
| for heavy in ('docker', 'modal', 'e2b'): | |
| assert heavy not in __import__('sys').modules | |
| print('sandbox standalone ok', praisonai_sandbox.__version__) | |
| " | |
| /tmp/praisonai-sandbox-standalone/bin/praisonai-sandbox --help | |
| bash "$GITHUB_WORKSPACE/scripts/check_c13_sandbox_imports.sh" | |
| python -m pytest src/praisonai-sandbox/tests -q -m "not network" | |
| - name: Standalone praisonai-deploy smoke (C14) | |
| run: | | |
| set -euo pipefail | |
| python -m venv /tmp/praisonai-deploy-standalone | |
| /tmp/praisonai-deploy-standalone/bin/pip install -q -e src/praisonai-agents -e src/praisonai-deploy | |
| /tmp/praisonai-deploy-standalone/bin/python -c " | |
| from praisonai_deploy._version import __version__ as pkg_version | |
| import praisonai_deploy | |
| assert praisonai_deploy.__version__ == pkg_version | |
| assert 'main' not in __import__('sys').modules | |
| print('deploy standalone ok', praisonai_deploy.__version__) | |
| " | |
| /tmp/praisonai-deploy-standalone/bin/praisonai-deploy --help | |
| bash "$GITHUB_WORKSPACE/scripts/check_c14_deploy_imports.sh" | |
| python -m pytest src/praisonai-deploy/tests -q -m "not network" | |
| - name: Audit hybrid module imports (C8 full stack) | |
| run: | | |
| set -euo pipefail | |
| pip install -q -e src/praisonai-agents -e src/praisonai-code -e src/praisonai-bot -e src/praisonai-train -e src/praisonai-browser -e src/praisonai-mcp -e src/praisonai-sandbox -e src/praisonai-deploy -e src/praisonai | |
| python scripts/audit_hybrid_modules.py | |
| - name: Smoke Test Summary | |
| if: always() | |
| run: echo "✅ Smoke tests completed" | |
| # ============================================ | |
| # MAIN TESTS - Unit + safe integration | |
| # Target: <= 5 minutes | |
| # ============================================ | |
| main: | |
| needs: smoke | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| matrix: | |
| python-version: ['3.11'] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache pip packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-main-${{ hashFiles('**/pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip-main- | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install pytest pytest-asyncio pytest-timeout pytest-cov pytest-xdist | |
| - uses: ./.github/actions/install-monorepo-packages | |
| with: | |
| agents-extras: ".[all]" | |
| wrapper-extras: ".[crewai,autogen,ui,chat]" | |
| - name: Run Main Tests | |
| env: | |
| PRAISONAI_TEST_TIER: main | |
| PRAISONAI_ALLOW_NETWORK: '0' | |
| OPENAI_MODEL_NAME: gpt-4o-mini | |
| OPENAI_API_KEY: 'test-key' | |
| ANTHROPIC_API_KEY: 'test-key' | |
| GOOGLE_API_KEY: 'test-key' | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| cd src/praisonai | |
| python -m pytest tests/unit/ \ | |
| -m "not slow and not network and not local_service and not provider_anthropic and not provider_google and not provider_ollama and not provider_grok_xai and not provider_groq and not provider_cohere" \ | |
| -q --tb=line \ | |
| --timeout=60 \ | |
| --ignore=tests/fixtures \ | |
| --ignore=tests/e2e \ | |
| --ignore=tests/live \ | |
| --ignore=tests/unit/jobs \ | |
| --ignore=tests/unit/test_langflow_components.py \ | |
| --cov=praisonai \ | |
| --cov-report=xml \ | |
| -n 2 --dist loadfile \ | |
| --maxfail=10 | |
| - name: Upload coverage | |
| if: matrix.python-version == '3.11' | |
| uses: codecov/codecov-action@v5 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN || '' }} | |
| files: src/praisonai/coverage.xml | |
| flags: main-tests | |
| fail_ci_if_error: false | |
| # ============================================ | |
| # OPENAI LIVE TESTS - Real OpenAI API calls | |
| # Only runs if OPENAI_API_KEY secret is set | |
| # Safe for forks (skips gracefully if no key) | |
| # ============================================ | |
| openai-live: | |
| needs: smoke | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 8 | |
| # This job always runs but steps are conditional on secret availability | |
| steps: | |
| - name: Check if OpenAI key is available | |
| id: check-key | |
| run: | | |
| if [ -n "${{ secrets.OPENAI_API_KEY }}" ]; then | |
| echo "has_key=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "has_key=false" >> $GITHUB_OUTPUT | |
| echo "⚠️ OPENAI_API_KEY not set - skipping live tests" | |
| fi | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| if: steps.check-key.outputs.has_key == 'true' | |
| - name: Set up Python 3.11 | |
| if: steps.check-key.outputs.has_key == 'true' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Cache pip packages | |
| if: steps.check-key.outputs.has_key == 'true' | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-openai-${{ hashFiles('**/pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| if: steps.check-key.outputs.has_key == 'true' | |
| run: | | |
| pip install --upgrade pip | |
| pip install pytest pytest-asyncio pytest-timeout | |
| - uses: ./.github/actions/install-monorepo-packages | |
| if: steps.check-key.outputs.has_key == 'true' | |
| with: | |
| agents-extras: ".[all]" | |
| wrapper-extras: ".[openai]" | |
| - name: Run OpenAI Live Tests | |
| if: steps.check-key.outputs.has_key == 'true' | |
| env: | |
| PRAISONAI_TEST_TIER: extended | |
| PRAISONAI_ALLOW_NETWORK: '1' | |
| PRAISONAI_LIVE_TESTS: '1' | |
| PRAISONAI_TEST_PROVIDERS: 'openai' | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| OPENAI_MODEL_NAME: ${{ secrets.OPENAI_MODEL_NAME || 'gpt-4o-mini' }} | |
| run: | | |
| cd src/praisonai | |
| python -m pytest tests/ \ | |
| -m "provider_openai or real" \ | |
| -v --tb=short \ | |
| --timeout=120 \ | |
| --ignore=tests/fixtures \ | |
| || echo "Some OpenAI tests failed" | |
| - name: Skip notice | |
| if: steps.check-key.outputs.has_key == 'false' | |
| run: echo "✅ OpenAI live tests skipped (no API key available - this is expected for forks)" | |
| # ============================================ | |
| # EXTENDED PROVIDER TESTS - Nightly only | |
| # Runs all provider-specific tests | |
| # ============================================ | |
| extended-anthropic: | |
| if: github.event_name == 'schedule' || github.event.inputs.tier == 'extended' || github.event.inputs.tier == 'nightly' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Check if Anthropic key is available | |
| id: check-key | |
| run: | | |
| if [ -n "${{ secrets.ANTHROPIC_API_KEY }}" ]; then | |
| echo "has_key=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "has_key=false" >> $GITHUB_OUTPUT | |
| echo "⚠️ ANTHROPIC_API_KEY not set - skipping tests" | |
| fi | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| if: steps.check-key.outputs.has_key == 'true' | |
| - name: Set up Python 3.11 | |
| if: steps.check-key.outputs.has_key == 'true' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| if: steps.check-key.outputs.has_key == 'true' | |
| run: | | |
| pip install --upgrade pip | |
| pip install pytest pytest-asyncio pytest-timeout | |
| - uses: ./.github/actions/install-monorepo-packages | |
| if: steps.check-key.outputs.has_key == 'true' | |
| with: | |
| agents-extras: ".[all]" | |
| wrapper-extras: ".[anthropic]" | |
| - name: Run Anthropic Tests | |
| if: steps.check-key.outputs.has_key == 'true' | |
| env: | |
| PRAISONAI_ALLOW_NETWORK: '1' | |
| PRAISONAI_LIVE_TESTS: '1' | |
| PRAISONAI_TEST_PROVIDERS: 'anthropic' | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| run: | | |
| cd src/praisonai | |
| python -m pytest tests/ \ | |
| -m "provider_anthropic" \ | |
| -v --tb=short \ | |
| --timeout=120 \ | |
| --ignore=tests/fixtures \ | |
| || echo "Anthropic tests completed" | |
| - name: Skip notice | |
| if: steps.check-key.outputs.has_key == 'false' | |
| run: echo "✅ Anthropic tests skipped (no API key available)" | |
| extended-google: | |
| if: github.event_name == 'schedule' || github.event.inputs.tier == 'extended' || github.event.inputs.tier == 'nightly' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Check if Google key is available | |
| id: check-key | |
| env: | |
| GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} | |
| run: | | |
| if [ -n "$GOOGLE_API_KEY" ]; then | |
| echo "has_key=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "has_key=false" >> $GITHUB_OUTPUT | |
| echo "⚠️ GOOGLE_API_KEY not set - skipping tests" | |
| fi | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| if: steps.check-key.outputs.has_key == 'true' | |
| - name: Set up Python 3.11 | |
| if: steps.check-key.outputs.has_key == 'true' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| if: steps.check-key.outputs.has_key == 'true' | |
| run: | | |
| pip install --upgrade pip | |
| pip install pytest pytest-asyncio pytest-timeout | |
| - uses: ./.github/actions/install-monorepo-packages | |
| if: steps.check-key.outputs.has_key == 'true' | |
| with: | |
| agents-extras: ".[all]" | |
| wrapper-extras: ".[google]" | |
| - name: Run Google/Gemini Tests | |
| if: steps.check-key.outputs.has_key == 'true' | |
| env: | |
| PRAISONAI_ALLOW_NETWORK: '1' | |
| PRAISONAI_LIVE_TESTS: '1' | |
| PRAISONAI_TEST_PROVIDERS: 'google' | |
| GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} | |
| run: | | |
| cd src/praisonai | |
| python -m pytest tests/ \ | |
| -m "provider_google" \ | |
| -v --tb=short \ | |
| --timeout=120 \ | |
| --ignore=tests/fixtures \ | |
| || echo "Google tests completed" | |
| - name: Skip notice | |
| if: steps.check-key.outputs.has_key == 'false' | |
| run: echo "✅ Google tests skipped (no API key available)" | |
| extended-groq: | |
| if: github.event_name == 'schedule' || github.event.inputs.tier == 'extended' || github.event.inputs.tier == 'nightly' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Check if Groq key is available | |
| id: check-key | |
| env: | |
| GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} | |
| run: | | |
| if [ -n "$GROQ_API_KEY" ]; then | |
| echo "has_key=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "has_key=false" >> $GITHUB_OUTPUT | |
| echo "⚠️ GROQ_API_KEY not set - skipping tests" | |
| fi | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| if: steps.check-key.outputs.has_key == 'true' | |
| - name: Set up Python 3.11 | |
| if: steps.check-key.outputs.has_key == 'true' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| if: steps.check-key.outputs.has_key == 'true' | |
| run: | | |
| pip install --upgrade pip | |
| pip install pytest pytest-asyncio pytest-timeout | |
| - uses: ./.github/actions/install-monorepo-packages | |
| if: steps.check-key.outputs.has_key == 'true' | |
| with: | |
| agents-extras: ".[all]" | |
| - name: Run Groq Tests | |
| if: steps.check-key.outputs.has_key == 'true' | |
| env: | |
| PRAISONAI_ALLOW_NETWORK: '1' | |
| PRAISONAI_LIVE_TESTS: '1' | |
| PRAISONAI_TEST_PROVIDERS: 'groq' | |
| GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} | |
| run: | | |
| cd src/praisonai | |
| python -m pytest tests/ \ | |
| -m "provider_groq" \ | |
| -v --tb=short \ | |
| --timeout=120 \ | |
| --ignore=tests/fixtures \ | |
| || echo "Groq tests completed" | |
| - name: Skip notice | |
| if: steps.check-key.outputs.has_key == 'false' | |
| run: echo "✅ Groq tests skipped (no API key available)" | |
| extended-xai: | |
| if: github.event_name == 'schedule' || github.event.inputs.tier == 'extended' || github.event.inputs.tier == 'nightly' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Check if xAI key is available | |
| id: check-key | |
| env: | |
| XAI_API_KEY: ${{ secrets.XAI_API_KEY }} | |
| run: | | |
| if [ -n "$XAI_API_KEY" ]; then | |
| echo "has_key=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "has_key=false" >> $GITHUB_OUTPUT | |
| echo "⚠️ XAI_API_KEY not set - skipping tests" | |
| fi | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| if: steps.check-key.outputs.has_key == 'true' | |
| - name: Set up Python 3.11 | |
| if: steps.check-key.outputs.has_key == 'true' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| if: steps.check-key.outputs.has_key == 'true' | |
| run: | | |
| pip install --upgrade pip | |
| pip install pytest pytest-asyncio pytest-timeout | |
| - uses: ./.github/actions/install-monorepo-packages | |
| if: steps.check-key.outputs.has_key == 'true' | |
| with: | |
| agents-extras: ".[all]" | |
| - name: Run xAI/Grok Tests | |
| if: steps.check-key.outputs.has_key == 'true' | |
| env: | |
| PRAISONAI_ALLOW_NETWORK: '1' | |
| PRAISONAI_LIVE_TESTS: '1' | |
| PRAISONAI_TEST_PROVIDERS: 'grok_xai' | |
| XAI_API_KEY: ${{ secrets.XAI_API_KEY }} | |
| run: | | |
| cd src/praisonai | |
| python -m pytest tests/ \ | |
| -m "provider_grok_xai" \ | |
| -v --tb=short \ | |
| --timeout=120 \ | |
| --ignore=tests/fixtures \ | |
| || echo "xAI tests completed" | |
| - name: Skip notice | |
| if: steps.check-key.outputs.has_key == 'false' | |
| run: echo "✅ xAI tests skipped (no API key available)" | |
| extended-cohere: | |
| if: github.event_name == 'schedule' || github.event.inputs.tier == 'extended' || github.event.inputs.tier == 'nightly' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Check if Cohere key is available | |
| id: check-key | |
| env: | |
| COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} | |
| run: | | |
| if [ -n "$COHERE_API_KEY" ]; then | |
| echo "has_key=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "has_key=false" >> $GITHUB_OUTPUT | |
| echo "⚠️ COHERE_API_KEY not set - skipping tests" | |
| fi | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| if: steps.check-key.outputs.has_key == 'true' | |
| - name: Set up Python 3.11 | |
| if: steps.check-key.outputs.has_key == 'true' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| if: steps.check-key.outputs.has_key == 'true' | |
| run: | | |
| pip install --upgrade pip | |
| pip install pytest pytest-asyncio pytest-timeout | |
| - uses: ./.github/actions/install-monorepo-packages | |
| if: steps.check-key.outputs.has_key == 'true' | |
| with: | |
| agents-extras: ".[all]" | |
| wrapper-extras: ".[cohere]" | |
| - name: Run Cohere Tests | |
| if: steps.check-key.outputs.has_key == 'true' | |
| env: | |
| PRAISONAI_ALLOW_NETWORK: '1' | |
| PRAISONAI_LIVE_TESTS: '1' | |
| PRAISONAI_TEST_PROVIDERS: 'cohere' | |
| COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} | |
| run: | | |
| cd src/praisonai | |
| python -m pytest tests/ \ | |
| -m "provider_cohere" \ | |
| -v --tb=short \ | |
| --timeout=120 \ | |
| --ignore=tests/fixtures \ | |
| || echo "Cohere tests completed" | |
| - name: Skip notice | |
| if: steps.check-key.outputs.has_key == 'false' | |
| run: echo "✅ Cohere tests skipped (no API key available)" | |
| # ============================================ | |
| # OLLAMA LIVE TESTS - Real local model, no API key | |
| # Nightly / extended only. Installs Ollama on the runner and pulls a small | |
| # tools-capable model. Unlike the other live jobs this one needs no secret, | |
| # so its required step has no "|| echo" fallback: a failure here is a real | |
| # regression in local-model tool calling, not a missing credential. | |
| # ============================================ | |
| extended-ollama: | |
| if: github.event_name == 'schedule' || github.event.inputs.tier == 'extended' || github.event.inputs.tier == 'nightly' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| env: | |
| # Bump deliberately: this is the version everything in this job was | |
| # verified against. Latest at time of writing is v0.33.3. | |
| # No leading "v": install.sh passes ?version=$OLLAMA_VERSION and ollama.com | |
| # redirects to GitHub tag v$version (vv0.33.2 → 404 if set to v0.33.2). | |
| OLLAMA_VERSION: "0.33.2" | |
| # 522 MB, capabilities: completion, tools, thinking. | |
| OLLAMA_TEST_MODEL: qwen3:0.6b | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Cache pip packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-ollama-${{ hashFiles('**/pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Cache Ollama model blobs | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.ollama/models | |
| # No restore-keys on purpose: a partial hit from a different model would | |
| # silently test something other than OLLAMA_TEST_MODEL. | |
| key: ${{ runner.os }}-ollama-models-${{ env.OLLAMA_TEST_MODEL }} | |
| - name: Install Ollama | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| # The official installer, but VERSION-PINNED via OLLAMA_VERSION (which it | |
| # honours) so this job cannot silently change what it tests. Fetching the | |
| # release asset directly is not a workable alternative: since v0.33 the | |
| # linux bundle ships as a 1.4 GB .tar.zst with GPU runtimes bundled in, | |
| # and the installer already handles the zst/tgz split across versions. | |
| sudo apt-get update -qq && sudo apt-get install -y -qq zstd | |
| curl -fsSL https://ollama.com/install.sh | sh | |
| ollama --version | |
| - name: Start Ollama and pull model | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| ollama serve > /tmp/ollama-serve.log 2>&1 & | |
| for i in $(seq 1 30); do | |
| if curl -sf http://127.0.0.1:11434/api/tags > /dev/null; then | |
| echo "Ollama is up after ${i}s" | |
| break | |
| fi | |
| sleep 1 | |
| done | |
| curl -sf http://127.0.0.1:11434/api/tags > /dev/null | |
| ollama pull "$OLLAMA_TEST_MODEL" | |
| ollama list | |
| - name: Install dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install pytest pytest-asyncio pytest-timeout | |
| - uses: ./.github/actions/install-monorepo-packages | |
| with: | |
| agents-extras: ".[all]" | |
| - name: Run Ollama minimum contract (required) | |
| env: | |
| PRAISONAI_TEST_TIER: extended | |
| PRAISONAI_ALLOW_NETWORK: '1' | |
| PRAISONAI_LIVE_TESTS: '1' | |
| PRAISONAI_TEST_PROVIDERS: 'ollama' | |
| PRAISONAI_TEST_OLLAMA: '1' | |
| PRAISONAI_OLLAMA_TEST_MODEL: ${{ env.OLLAMA_TEST_MODEL }} | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| cd src/praisonai | |
| # No "|| echo" here: no secret is involved, so a failure is a real | |
| # regression rather than a missing credential. | |
| python -m pytest tests/integration/test_ollama_tool_calling_live.py::TestOllamaCIMinimum \ | |
| -v --tb=short \ | |
| --timeout=120 | |
| - name: Run remaining Ollama live tests (informational) | |
| if: always() | |
| env: | |
| PRAISONAI_TEST_TIER: extended | |
| PRAISONAI_ALLOW_NETWORK: '1' | |
| PRAISONAI_LIVE_TESTS: '1' | |
| PRAISONAI_TEST_PROVIDERS: 'ollama' | |
| PRAISONAI_TEST_OLLAMA: '1' | |
| PRAISONAI_OLLAMA_TEST_MODEL: ${{ env.OLLAMA_TEST_MODEL }} | |
| run: | | |
| cd src/praisonai | |
| python -m pytest tests/integration/test_ollama_tool_calling_live.py \ | |
| --deselect tests/integration/test_ollama_tool_calling_live.py::TestOllamaCIMinimum \ | |
| -v --tb=short \ | |
| --timeout=180 \ | |
| || echo "Ollama exploratory tests completed (small models are non-deterministic)" | |
| - name: Ollama server log | |
| if: failure() | |
| run: tail -100 /tmp/ollama-serve.log | |
| # ============================================ | |
| # LEGACY UNIT TESTS - Post-C6 re-enabled suite | |
| # Nightly / extended tier only (~401 tests) | |
| # ============================================ | |
| legacy-unit: | |
| if: github.event_name == 'schedule' || github.event.inputs.tier == 'extended' || github.event.inputs.tier == 'nightly' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install pytest pytest-asyncio pytest-timeout | |
| - uses: ./.github/actions/install-monorepo-packages | |
| with: | |
| agents-extras: ".[all]" | |
| wrapper-extras: ".[chat,code,crewai,autogen]" | |
| - name: Run post-C6 legacy unit tests | |
| env: | |
| PRAISONAI_TEST_TIER: extended | |
| PRAISONAI_ALLOW_NETWORK: '0' | |
| PRAISONAI_ALLOW_JOB_WORKFLOWS: 'true' | |
| PRAISONAI_TOOL_SAFETY: 'off' | |
| OPENAI_API_KEY: 'test-key' | |
| run: | | |
| set -euo pipefail | |
| cd src/praisonai | |
| LEGACY_FILES=( | |
| tests/unit/test_warm_runtime.py | |
| tests/unit/test_yaml_cli_backend.py | |
| tests/unit/test_tool_resolver.py | |
| tests/unit/test_serve_unified.py | |
| tests/unit/test_security_fixes_core.py | |
| tests/unit/test_query_rewriter_cli.py | |
| tests/unit/test_kanban_sqlite_store.py | |
| tests/unit/test_job_workflow_agent_steps.py | |
| tests/unit/test_hybrid_workflow.py | |
| ../praisonai-bot/tests/unit/test_gateway_session.py | |
| ../praisonai-bot/tests/unit/test_gateway_gaps.py | |
| tests/unit/test_discord_approval.py | |
| tests/unit/test_decorator_simple.py | |
| tests/unit/test_db_adapter_tracing_init.py | |
| ../praisonai-bot/tests/unit/test_bot_wiring.py | |
| ../praisonai-bot/tests/unit/bots/test_enable_shell.py | |
| ../praisonai-bot/tests/unit/gateway/test_preflight.py | |
| ../praisonai-bot/tests/unit/gateway/test_gateway_doctor.py | |
| ../praisonai-bot/tests/unit/test_bot_history.py | |
| ../praisonai-bot/tests/unit/test_bot_fixes.py | |
| tests/unit/test_auto_lazy_loading.py | |
| tests/unit/test_async_daemon_deployment.py | |
| tests/unit/test_approval_interactive.py | |
| tests/unit/test_approval_agent_integration.py | |
| tests/unit/test_agents_yaml_validation.py | |
| tests/unit/test_agent_os_wrapper.py | |
| tests/unit/test_lazy_imports.py | |
| tests/unit/test_profiler.py | |
| tests/unit/test_profiler_advanced.py | |
| tests/unit/test_serverless_postgres.py | |
| ) | |
| python -m pytest "${LEGACY_FILES[@]}" \ | |
| -q --tb=line --timeout=120 --maxfail=10 | |
| # ============================================ | |
| # TEST SUMMARY | |
| # ============================================ | |
| test-summary: | |
| needs: [smoke, main, extended-ollama] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Test Summary | |
| run: | | |
| echo "# 📊 Test Suite Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Job | Status |" >> $GITHUB_STEP_SUMMARY | |
| echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| Smoke | ${{ needs.smoke.result }} |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Main | ${{ needs.main.result }} |" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "## Test Tiers" >> $GITHUB_STEP_SUMMARY | |
| echo "- **smoke**: Fast unit tests, no network (≤2 min)" >> $GITHUB_STEP_SUMMARY | |
| echo "- **main**: Unit + integration, no network (≤5 min)" >> $GITHUB_STEP_SUMMARY | |
| echo "- **openai-live**: OpenAI API tests (when key available)" >> $GITHUB_STEP_SUMMARY | |
| echo "- **extended-***: Provider-specific tests (nightly)" >> $GITHUB_STEP_SUMMARY |