pytest-windows.ini

# NOTE: This file is a copy of pytest.ini, but with additional tests disabled for Windows. This
# is intended to be a short-term solution to allow for incrementally enabling tests on Windows.
# This file is intended to be deleted once the enablement is complete.

[pytest]
addopts =
    # show summary of all tests that did not pass
    -rEfX
    # Make tracebacks shorter
    --tb=native
    # capture only Python print and C++ py::print, but not C output (low-level Python errors)
    --capture=sys
    # don't suppress warnings, but don't shove them all to the end either
    -p no:warnings
    # Ignore backends/arm tests you need to run examples/arm/setup.sh to install some tool to make them work
    # For GitHub testing this is setup/executed in the unittest-arm job see .github/workflows/pull.yml for more info.
    --ignore-glob=backends/arm/**/*
    # explicitly list out tests that are running successfully in oss
    .ci/scripts/tests
    examples/models/test
    devtools/
    --ignore=devtools/visualization/visualization_utils_test.py
    # examples
    # examples/models/llava/test TODO: enable this
    # exir
    exir/_serialize/test
    exir/backend/test
    exir/dialects/backend/test
    exir/dialects/edge/test
    exir/dialects/test
    exir/emit/test
    exir/program/test
    exir/tests/
    # executorch/export
    export/tests
    --ignore=export/tests/test_export_stages.py
    # kernels/
    kernels/prim_ops/test
    kernels/quantized
    # Because this test depends on test only cpp ops lib
    # Will add test only cmake targets to re-enable this test
    # but maybe it is a bit of anti-pattern
    --ignore=kernels/quantized/test/test_quant_dequant_per_token.py
    kernels/test/test_case_gen.py
    # backends/test
    # This effort is WIP and will be enabled in CI once testing infra
    # is stable and signal to noise ratio is good (no irrelevant failures).
    # See https://github.com/pytorch/executorch/discussions/11140
    --ignore=backends/test
    backends/test/harness/tests
    backends/test/suite/tests
    # backends/xnnpack
    backends/xnnpack/test/ops
    --ignore=backends/xnnpack/test/ops/test_bmm.py
    --ignore=backends/xnnpack/test/ops/test_conv2d.py
    --ignore=backends/xnnpack/test/ops/test_linear.py
    --ignore=backends/xnnpack/test/ops/test_sdpa.py
    backends/xnnpack/test/passes
    backends/xnnpack/test/recipes
    backends/xnnpack/test/serialization
    # backends/apple/coreml
    backends/apple/coreml/test
    # extension/
    extension/llm/custom_ops/test_sdpa_with_kv_cache.py
    extension/llm/custom_ops/test_update_cache.py
    extension/llm/custom_ops/test_quantized_sdpa.py
    extension/pybindings/test
    extension/training/pybindings/test
    # Runtime
    runtime
    # Tools
    codegen/test
    tools/cmake
    # test TODO: fix these tests
    # test/end2end/test_end2end.py
    --ignore=backends/xnnpack/test/ops/linear.py
    --ignore=backends/xnnpack/test/models/llama2_et_example.py
    # T200992559: Add torchao to ET as core dependency
    --ignore=examples/models/llama/tests/test_pre_quantization_transforms.py
    --ignore=exir/backend/test/demos
    --ignore=exir/backend/test/test_backends.py
    --ignore=exir/backend/test/test_backends_lifted.py
    --ignore=exir/backend/test/test_partitioner.py
    --ignore=exir/tests/test_common.py
    --ignore=exir/tests/test_memory_format_ops_pass_aten.py
    --ignore=exir/tests/test_memory_planning.py
    --ignore=exir/tests/test_op_convert.py
    --ignore=exir/tests/test_passes.py
    --ignore=exir/tests/test_quant_fusion_pass.py
    --ignore=exir/tests/test_quantization.py
    --ignore=exir/tests/test_verification.py
    # Tests that are (temporarily) disabled for Windows
    # TODO(gjcomer) Re-enable the LLM tests when tokenizers library is available on Windows.
    #examples/models/llama3_2_vision/preprocess
    #examples/models/llama3_2_vision/vision_encoder/test
    #examples/models/llama3_2_vision/text_decoder/test
    #examples/models/llama/tests
    #examples/models/llama/config
    #extension/llm/modules/test
    #extension/llm/export
    --deselect=extension/pybindings/test/test_pybindings.py::PybindingsTest::test_method_quantized_ops
    --deselect=extension/pybindings/test/test_pybindings.py::PybindingsTest::test_quantized_ops
    --deselect=extension/pybindings/test/test_pybindings.py::PybindingsTest::test_program_data_separation
    --deselect=runtime/test/test_runtime.py::RuntimeTest::test_load_program_with_path
    --deselect=exir/backend/test/test_compatibility.py::TestCompatibility::test_compatibility_in_runtime
    --deselect=exir/backend/test/test_compatibility.py::TestCompatibility::test_compatibility_in_runtime_edge_program_manager
    --deselect=exir/backend/test/test_lowered_backend_module.py::TestBackendAPI::test_emit_lowered_backend_module_end_to_end
    --deselect=exir/backend/test/test_to_backend_multi_method.py::TestToBackendMultiMethod::test_multi_method_end_to_end
    --deselect=extension/llm/custom_ops/test_sdpa_with_kv_cache.py::SDPATestForSpeculativeDecode::test_sdpa_with_cache_seq_len_130
    --deselect=devtools/inspector/tests/inspector_test.py::TestInspector::test_etrecord_populates_correct_edge_dialect_aot_intermediate_outputs
    --deselect=devtools/inspector/tests/inspector_test.py::TestInspector::test_etrecord_populates_correct_export_program_aot_intermediate_outputs
    --deselect=runtime/test/test_runtime_etdump_gen.py::RuntimeETDumpGenTest::test_etdump_generation
    --deselect=runtime/test/test_runtime_etdump_gen.py::RuntimeETDumpGenTest::test_etdump_params_with_bytes_and_buffer

# run the same tests multiple times to determine their
# flakiness status. Default to 50 re-runs
flake-finder = true
flake-runs = 50