diff --git a/.github/actions/pypto-serving-tests/action.yml b/.github/actions/pypto-serving-tests/action.yml index 068db03e..f632acca 100644 --- a/.github/actions/pypto-serving-tests/action.yml +++ b/.github/actions/pypto-serving-tests/action.yml @@ -1,7 +1,7 @@ -name: Run pypto-serving Qwen tests +name: Run pypto-serving model tests description: >- Checkout pypto-serving, replace its pypto-lib submodule with the current - pypto-lib checkout under test, and run the Qwen serving tests via + pypto-lib checkout under test, and run the selected serving tests via task-submit. inputs: @@ -11,12 +11,11 @@ inputs: pypto-serving-checkout-path: description: Workspace-relative path where pypto-serving should be checked out. required: true - pypto-serving-ref: - description: pypto-serving ref to test against. - required: false - default: main + model-name: + description: Serving model guard to run (qwen3-14b or deepseek-v4). + required: true model-dir: - description: Qwen3 model directory used by pypto-serving tests. + description: Model directory used by the selected pypto-serving test. required: true pto2-ring-dep-pool: description: PTO2_RING_DEP_POOL value for pypto-serving tests. @@ -38,7 +37,7 @@ runs: uses: actions/checkout@v4 with: repository: hw-native-sys/pypto-serving - ref: ${{ inputs.pypto-serving-ref }} + ref: main path: ${{ inputs.pypto-serving-checkout-path }} submodules: recursive persist-credentials: false @@ -61,9 +60,10 @@ runs: working-directory: ${{ inputs.pypto-lib-checkout-path }} run: | source activate.sh - pip install transformers safetensors pytest + pip install transformers safetensors pytest fastapi uvicorn - name: Run pypto-serving Qwen tests + if: ${{ inputs.model-name == 'qwen3-14b' }} shell: bash env: PYPTO_QWEN3_MODEL_DIR: ${{ inputs.model-dir }} @@ -74,8 +74,8 @@ runs: PYPTO_SERVING_CHECKOUT: ${{ inputs.pypto-serving-checkout-path }} run: | source "$GITHUB_WORKSPACE/$PYPTO_LIB_CHECKOUT/activate.sh" - run_cmd="cd $GITHUB_WORKSPACE/$PYPTO_SERVING_CHECKOUT" - run_cmd="$run_cmd && source $GITHUB_WORKSPACE/$PYPTO_LIB_CHECKOUT/activate.sh" + run_cmd="source $GITHUB_WORKSPACE/$PYPTO_LIB_CHECKOUT/activate.sh" + run_cmd="$run_cmd && cd $GITHUB_WORKSPACE/$PYPTO_SERVING_CHECKOUT" run_cmd="$run_cmd && DEVICE_ID=\$TASK_DEVICE" run_cmd="$run_cmd PYTHONPATH=$GITHUB_WORKSPACE/$PYPTO_SERVING_CHECKOUT" run_cmd="$run_cmd PYPTO_QWEN3_MODEL_DIR=$PYPTO_QWEN3_MODEL_DIR" @@ -83,5 +83,40 @@ runs: run_cmd="$run_cmd PTO2_RING_TASK_WINDOW=$PTO2_RING_TASK_WINDOW" run_cmd="$run_cmd PTO2_RING_HEAP=$PTO2_RING_HEAP" run_cmd="$run_cmd python -m pytest tests/test_qwen3_accuracy.py -q -s" - task-submit --device "$DEVICE_ID" --timeout 0 --max-time 0 \ + task-submit --device "$DEVICE_ID" --timeout 1200 --max-time 1800 \ + --run "$run_cmd" + + - name: Run pypto-serving DeepSeek V4 accuracy test + if: ${{ inputs.model-name == 'deepseek-v4' }} + shell: bash + env: + PYPTO_DSV4_MODEL_DIR: ${{ inputs.model-dir }} + PYPTO_RUNTIME_LOG: error + PTO2_RING_DEP_POOL: ${{ inputs.pto2-ring-dep-pool }} + PTO2_RING_TASK_WINDOW: ${{ inputs.pto2-ring-task-window }} + PTO2_RING_HEAP: ${{ inputs.pto2-ring-heap }} + PTO2_OP_EXECUTE_TIMEOUT_US: 400000000 + PTO2_STREAM_SYNC_TIMEOUT_MS: 440000 + PTO2_SCHEDULER_TIMEOUT_MS: 320000 + SERVING_WORKER_STEP_TIMEOUT: 1800 + PYPTO_LIB_CHECKOUT: ${{ inputs.pypto-lib-checkout-path }} + PYPTO_SERVING_CHECKOUT: ${{ inputs.pypto-serving-checkout-path }} + run: | + source "$GITHUB_WORKSPACE/$PYPTO_LIB_CHECKOUT/activate.sh" + run_cmd="source $GITHUB_WORKSPACE/$PYPTO_LIB_CHECKOUT/activate.sh" + run_cmd="$run_cmd && cd $GITHUB_WORKSPACE/$PYPTO_SERVING_CHECKOUT" + run_cmd="$run_cmd && TASK_DEVICE=\$TASK_DEVICE" + run_cmd="$run_cmd PYTHONPATH=$GITHUB_WORKSPACE/$PYPTO_SERVING_CHECKOUT" + run_cmd="$run_cmd PYPTO_DSV4_MODEL_DIR=$PYPTO_DSV4_MODEL_DIR" + run_cmd="$run_cmd PYPTO_RUNTIME_LOG=$PYPTO_RUNTIME_LOG" + run_cmd="$run_cmd PTO2_RING_DEP_POOL=$PTO2_RING_DEP_POOL" + run_cmd="$run_cmd PTO2_RING_TASK_WINDOW=$PTO2_RING_TASK_WINDOW" + run_cmd="$run_cmd PTO2_RING_HEAP=$PTO2_RING_HEAP" + run_cmd="$run_cmd PTO2_OP_EXECUTE_TIMEOUT_US=$PTO2_OP_EXECUTE_TIMEOUT_US" + run_cmd="$run_cmd PTO2_STREAM_SYNC_TIMEOUT_MS=$PTO2_STREAM_SYNC_TIMEOUT_MS" + run_cmd="$run_cmd PTO2_SCHEDULER_TIMEOUT_MS=$PTO2_SCHEDULER_TIMEOUT_MS" + run_cmd="$run_cmd SERVING_WORKER_STEP_TIMEOUT=$SERVING_WORKER_STEP_TIMEOUT" + run_cmd="$run_cmd python -m pytest tests/test_deepseek_v4_accuracy.py -q -s" + task-submit --device auto --device-num 8 --ignore-whitelist \ + --timeout 1200 --max-time 1800 \ --run "$run_cmd" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 853af82a..db565472 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,8 @@ jobs: outputs: has_changes: ${{ steps.check.outputs.has_changes }} files: ${{ steps.check.outputs.files }} - run_serving_tests: ${{ steps.check.outputs.run_serving_tests }} + run_qwen_serving_tests: ${{ steps.check.outputs.run_qwen_serving_tests }} + run_deepseek_serving_tests: ${{ steps.check.outputs.run_deepseek_serving_tests }} steps: - uses: actions/checkout@v4 with: @@ -69,8 +70,9 @@ jobs: if [ "${{ github.event_name }}" != "pull_request" ]; then echo "has_changes=true" >> "$GITHUB_OUTPUT" echo "files=" >> "$GITHUB_OUTPUT" - # serving is a PR-only job (see its `if`), so never arm it here. - echo "run_serving_tests=false" >> "$GITHUB_OUTPUT" + # Serving is PR-only (see its `if`), so never arm it here. + echo "run_qwen_serving_tests=false" >> "$GITHUB_OUTPUT" + echo "run_deepseek_serving_tests=false" >> "$GITHUB_OUTPUT" exit 0 fi CHANGED_FILES=$(git diff --name-only \ @@ -88,13 +90,17 @@ jobs: echo "has_changes=false" >> "$GITHUB_OUTPUT" fi echo "files=$FILES" >> "$GITHUB_OUTPUT" - # pypto-serving e2e only exercises Qwen3-14B (tests/test_qwen3_accuracy.py - # against the Qwen3-14B checkpoint), so only a qwen3-14b source change can - # affect it. CI-infra edits (.github/**) must not trigger this expensive job. + # Serving e2e jobs are model-specific and expensive. CI-infra edits + # (.github/**) must not trigger them. if printf '%s\n' "$CHANGED_FILES" | grep -qE '^models/qwen3/14b/'; then - echo "run_serving_tests=true" >> "$GITHUB_OUTPUT" + echo "run_qwen_serving_tests=true" >> "$GITHUB_OUTPUT" else - echo "run_serving_tests=false" >> "$GITHUB_OUTPUT" + echo "run_qwen_serving_tests=false" >> "$GITHUB_OUTPUT" + fi + if printf '%s\n' "$CHANGED_FILES" | grep -qE '^models/deepseek/v4/'; then + echo "run_deepseek_serving_tests=true" >> "$GITHUB_OUTPUT" + else + echo "run_deepseek_serving_tests=false" >> "$GITHUB_OUTPUT" fi echo "Selected files: $FILES" @@ -301,12 +307,15 @@ jobs: working-directory: dist-checkout run: rm -rf build_output - serving-qwen3-14b: + serving: needs: detect-changes - # PR-only: push/dispatch never run the (expensive) qwen3-14b serving e2e. - if: github.event_name == 'pull_request' && needs.detect-changes.outputs.run_serving_tests == 'true' + # PR-only: run one shared setup, then the guards selected by model changes. + if: >- + github.event_name == 'pull_request' && + (needs.detect-changes.outputs.run_qwen_serving_tests == 'true' || + needs.detect-changes.outputs.run_deepseek_serving_tests == 'true') runs-on: [self-hosted, linux, arm64, npu] - timeout-minutes: 60 + timeout-minutes: 90 env: ASCEND_HOME_PATH: /usr/local/Ascend/cann-9.0.0 PTOAS_ROOT: ${{ github.workspace }}/dist-checkout/ptoas-bin @@ -333,13 +342,27 @@ jobs: checkout-path: dist-checkout build-namespace: serving - - name: Run pypto-serving e2e model tests + - name: Run pypto-serving Qwen3-14B tests + if: needs.detect-changes.outputs.run_qwen_serving_tests == 'true' uses: ./.github/actions/pypto-serving-tests with: pypto-lib-checkout-path: dist-checkout pypto-serving-checkout-path: serving-checkout + model-name: qwen3-14b model-dir: /data/l00955553/model/Qwen3-14B + - name: Run pypto-serving DeepSeek V4 tests + if: needs.detect-changes.outputs.run_deepseek_serving_tests == 'true' + uses: ./.github/actions/pypto-serving-tests + with: + pypto-lib-checkout-path: dist-checkout + pypto-serving-checkout-path: serving-checkout + model-name: deepseek-v4 + model-dir: /data/l00955553/model/dsv4-flash-w8a8 + pto2-ring-dep-pool: "131072" + pto2-ring-task-window: "131072" + pto2-ring-heap: "2147483648" + - name: Kill orphaned task-submit tasks if: always() uses: ./.github/actions/kill-orphaned-tasks @@ -348,4 +371,4 @@ jobs: - name: Clean up build artifacts if: always() - run: rm -rf dist-checkout/build_output serving-checkout/build_output serving-checkout/pypto-lib/build_output + run: rm -rf serving-checkout/build_output