Testing improvements

.editorconfig

@@ -0,0 +1,15 @@
+# http://editorconfig.org
+
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+indent_style = space
+indent_size = 4
+trim_trailing_whitespace = true
+
+[*.py]
+lines_after_imports = 2
+max_line_length = 120

Dockerfile-tox

@@ -0,0 +1,47 @@
+# Build
+# docker build -f Dockerfile-tox -t echo-tox:latest .
+# Run
+# docker run --volume .:/opt/tox/echo echo-tox:latest
+# pull official base image
+FROM ubuntu:22.04
+# Prevent having to answer question about geographic area
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update --quiet --yes && \
+    apt-get install --quiet --yes build-essential ca-certificates git python3 python3-dev python3-venv software-properties-common
+
+# Install cbc (solver used by)
+RUN apt-get install --quiet --yes coinor-cbc coinor-libcbc-dev
+
+# Install deadsnakes (required for tox testing)
+RUN add-apt-repository --yes --update ppa:deadsnakes/ppa && \
+    apt-get install --quiet --yes python3.8 python3.8-dev python3.8-venv python3.9 python3.9-dev python3.9-venv python3.10 python3.10-dev python3.10-venv python3.11 python3.11-dev python3.11-venv python3.12 python3.12-dev python3.12-venv && \
+    apt-get autoremove -yqq --purge && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Create user
+ARG uid=${UID:-1000}
+ARG gid=${GID:-1000}
+RUN groupadd group --gid=$gid
+RUN useradd user --uid=$uid --gid=$gid --home-dir=/tmp --no-create-home
+USER $uid:$gid
+
+# Create working directory
+WORKDIR /opt/tox
+
+# Install required python packages
+RUN python3 -m venv ./tox-venv
+RUN ./tox-venv/bin/python3 -m pip install --upgrade pip
+RUN ./tox-venv/bin/python3 -m pip install tox
+
+# Configure environment
+ENV LANG=C.UTF-8
+ENV LC_ALL=C.UTF-8
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+WORKDIR /opt/tox/echo
+
+# Defaults
+ENTRYPOINT [ "../tox-venv/bin/tox" ]

README.md

@@ -36,13 +36,74 @@ If you plan to use echo_scenario to solve power flows then you will need to inst
 The following solvers can be used
 
 Free for academic use only:
-- CPLEX (recommended): This has been tested the most. It requires a license but is free for academic users. To install, follow instructions [here](https://www.ibm.com/products/ilog-cplex-optimization-studio). After installing CPLEX you will need to add the binaries to your system path. 
+- CPLEX (recommended): This has been tested the most. It requires a license but is free for academic users. To install, follow instructions [here](https://www.ibm.com/products/ilog-cplex-optimization-studio). After installing CPLEX you will need to add the binaries to your system path.
 - GUROBI: Minimal testing: It requires a license but is free for academic users. Check their website for instructions on installing https://www.gurobi.com/documentation/9.5/remoteservices/linux_installation.html
 
 Open source:
 - CBC: This solver can be used provided you only include linear costs (no quadratic costs or regularisation). Information on the solver is available here https://github.com/coin-or/Cbc . For installing on ubuntu run `sudo apt-get install -y coinor-cbc1
 
 
+## Testing
+
+### Running Tests
+
+The test suite can be run with,
+
+```sh
+$ pytest
+```
+
+This will run the tests using the default optimiser engine (currently set to `cbc`). We default to `cbc` since this is the only solver available to the Github action.
+
+Some of the tests require a solver capable of performing non-linear optimisations. The `cbc` solver is not able to do this. To run the complete set of tests, you will need to have a solver capable of non-linear optimizations (for example `cplex`) installed on your system. You can run pytest and override the default solver using the environment variable `TESTING_OPTIMISER_ENGINE`. For example:
+
+```sh
+$ TESTING_OPTIMISER_ENGINE=cplex pytest
+```
+
+See the section below 'Writing tests', for instructions on how to mark a test as requiring a non-linear solver.
+
+### Writing tests
+
+Most tests will require an `engine_settings` object. A fixture `engine_settings` has been provided, which should be used by any tests that require engine settings. It is strongly recommended **not** to use `scenario.engine_settings_from_environment()` in order to obtain an engine settings object, since this can result in the your newly-written tests running on a different solver to the rest of the test suite.
+
+A small number of tests might need to be run with a solver capable of non-linear optimizations. These tests should be marked with a special pytest mark decorator:
+
+```py
+import pytest
+
+@pytest.mark.nonlinear
+def test_that_requires_nonlinear_solver():
+    ...
+```
+
+### Tox
+
+Tox makes it easy to test echo on multiple versions of python. The configuration file for tox is called `tox.ini`, which
+is located in the project root. Looking in `tox.ini`, the `env_list` parameter lists all the environments we will test against.
+
+To prevent echo developers having to install all the different versions of python on their system, tox has been setup
+to run in a docker container.
+
+To test with tox, the first step is to build the docker image,
+
+```sh
+docker build -f Dockerfile-tox -t echo-tox:latest .
+```
+
+This should only need doing once. Re-run the docker build command if you modify the Dockerfile, `Dockerfile-tox`.
+
+The test suite can be run across the different environments (python versions) with,
+
+```sh
+docker run -t --volume .:/opt/tox/echo echo-tox:latest -- -W ignore::DeprecationWarning
+```
+
+- The command that gets run is `tox`, which you can see by looking at the `ENTRYPOINT` in the Dockerfile, `Dockerfile-tox`.
+- The `-t` gives us colored output in the terminal.
+- The `--volume` switch mounts echo project inside the container
+- Arguments/options before the `--` are passed to `tox`. Anything after `--` is passed to the command `tox` runs i.e. `pytest`. In this case we are using `-W ignore::DeprecationWarning` to suppress an error raised because of deprecated datetime code in pandas (or one of its dependencies)
+
 ## Documentation
 
 ### Building the documentation

pyproject.toml

@@ -53,6 +53,7 @@ dependencies = [
     "scikit-learn>=1.1",
     "orjson>=3.8",
     "shortuuid>=1.0.11,<2",
+    "rich>=13.5.2,<14"
 ]
 
 [project.optional-dependencies]

src/echo/models/scenario.py

@@ -50,14 +50,22 @@ class EchoConcreteModel(ConcreteModel):
 def engine_settings_from_environment(optimiser_engine: Optional[str] = None) -> EngineSettings:
     """Configure the optimiser through setting appropriate environmental variables."""
 
+    DEFAULT_OPTIMISER_ENGINE = "cplex"
+
+    # optimiser_engine is None or "" (empty string)
     if not optimiser_engine:
-        optimiser_engine = os.environ.get(
-            "OPTIMISER_ENGINE", "cplex"
-        )  # Default to cplex, as we seem to want quadratic costs
+        optimiser_engine = os.environ.get("OPTIMISER_ENGINE", DEFAULT_OPTIMISER_ENGINE)
+    optimiser_engine = optimiser_engine.lower()
 
     return EngineSettings(
         engine=optimiser_engine,
         engine_executable=os.environ.get("OPTIMISER_ENGINE_EXECUTABLE", ""),
         bigM=5000000,  # This value has been arbitrarily chosen
         smallM=0.0001,  # This value has been arbitrarily chosen
     )
+
+
+def can_optimiser_do_non_linear_optimisation(optimiser: str) -> bool:
+    """Returns True if optimiser can *only* do linear optimisations"""
+    linear_optimisers = ["cbc"]  # TODO Make this list complete
+    return optimiser not in linear_optimisers

tests/conftest.py

@@ -1,15 +1,77 @@
 import os
 
 import pytest
+from rich.console import Console
+
+from echo.models import scenario
+
+
+def default_testing_optimiser_engine():
+    """The default test optimiser engine
+
+    If this is changed, make sure that the replacement optimiser engine
+    is installed in the github action.
+
+    This is deliberately not a fixture so it can be accessed in the `pytest_terminal_summary` hook
+    """
+    return "cbc"
+
+
+def testing_optimiser_engine():
+    """Gets the optimiser engine for testing.
+
+    Defaults to default_testing_optimiser_engine() but can be overriden by the
+    `TESTING_OPTIMISER_ENGINE` environment variable.
+
+    This is deliberately not a fixture so it can be accessed in the `pytest_terminal_summary` hook
+    """
+    return os.environ.get("TESTING_OPTIMISER_ENGINE", default_testing_optimiser_engine())
 
 
 @pytest.fixture
-def can_optimiser_do_non_linear_optimisation() -> bool:
-    # This test contains non-linear optimisation, so cbc won't be able to run it. Check the environment and skip this
-    # test if cbc is the optimiser engine
-    optimiser_engine = os.environ.get("OPTIMISER_ENGINE")
+def engine_settings() -> scenario.EngineSettings:
+    """Fixture returning engine settings for tests
+
+    This fixture should be used by all tests that required engine settings.
+    It is strongly recommended not to use `scenario.engine_settings_from_environment()` which is
+    intended for running echo and not testing echo.
+    """
+    return scenario.engine_settings_from_environment(optimiser_engine=testing_optimiser_engine())
+
+
+# Based off this stackoverflow answer: https://stackoverflow.com/a/28198398
+@pytest.fixture(autouse=True)
+def skip_by_optimiser_engine_not_do_non_linear(request, engine_settings):
+    optimiser = engine_settings.engine
+
+    # If test marked as nonlinear...
+    if request.node.get_closest_marker("nonlinear"):
+        # and optimiser can't do non-linear optimisations...
+        if not scenario.can_optimiser_do_non_linear_optimisation(optimiser=optimiser):
+            # ...then skip
+            pytest.skip(f"Optimiser '{optimiser}' cannot do non-linear optimisation")
+
+
+def pytest_terminal_summary(terminalreporter, exitstatus, config):
+
+    DEFAULT_SOLVER = default_testing_optimiser_engine()
 
-    if optimiser_engine == "cbc":
-        return False
+    if testing_optimiser_engine() != DEFAULT_SOLVER:
+        return
 
-    return True
+    with Console() as console:
+        console.print(
+            "\n==============================  Echo Solver Info  ==============================", style="magenta"
+        )
+        console.print(
+            f"This pytest run used the default solver '{DEFAULT_SOLVER}' (like the Github Action).", end="\n\n"
+        )
+        console.print(
+            "It is important to be aware that certain solvers (e.g. 'cbc') cannot perform all tests.", end="\n\n"
+        )
+        console.print("To run pytest with a different optimiser, set the 'TESTING_OPTIMISER_ENGINE'")
+        console.print("environment variable. For example, to use cplex:", end="\n\n")
+        console.print("$ TESTING_OPTIMISER_ENGINE=cplex pytest")
+        console.print(
+            "================================================================================", style="magenta"
+        )

tests/test_asset_types.py

@@ -6,7 +6,7 @@
 from echo.models.carbon import CarbonAggregation, CarbonSink, CarbonSource
 from echo.models.electrical import ElectricalDemand, ElectricalPort, ElectricalStorage
 from echo.models.gas import FlexGasPort, GasBoilerFixedCOP
-from echo.models.scenario import ScenarioSettings, engine_settings_from_environment
+from echo.models.scenario import ScenarioSettings
 from echo.models.thermal import FixedThermalPort, HeatSink, SimpleChiller
 from echo.objectives.base import ObjectiveSet
 from echo.objectives.tariff import ImportTariff
@@ -15,7 +15,7 @@
 N_INTERVALS = 48
 
 
-def test_gas_boiler_fixed_cop():
+def test_gas_boiler_fixed_cop(engine_settings):
     expansion_periods = 1
     time_periods = 48
     interval_duration = 30
@@ -42,7 +42,7 @@ def test_gas_boiler_fixed_cop():
             number_of_intervals=time_periods,
             number_of_expansion_intervals=expansion_periods,
         ),
-        engine_settings=engine_settings_from_environment(),
+        engine_settings=engine_settings,
         graph=system,
     )
 
@@ -58,7 +58,7 @@ def test_gas_boiler_fixed_cop():
         assert gas_mains[i] * boiler.cop == hl_p[(0, i)] * -1
 
 
-def test_modulating_gas_boiler():
+def test_modulating_gas_boiler(engine_settings):
     expansion_periods = 1
     time_periods = 48
     interval_duration = 30
@@ -85,7 +85,7 @@ def test_modulating_gas_boiler():
             number_of_intervals=time_periods,
             number_of_expansion_intervals=expansion_periods,
         ),
-        engine_settings=engine_settings_from_environment(),
+        engine_settings=engine_settings,
         graph=system,
     )
 
@@ -96,7 +96,7 @@ def test_modulating_gas_boiler():
         assert boiler_input[i] * boiler.cop == -1 * boiler_output[i]
 
 
-def test_chiller_operation():
+def test_chiller_operation(engine_settings):
     expansion_periods = 1
     time_periods = 48
     interval_duration = 30
@@ -127,7 +127,7 @@ def test_chiller_operation():
             number_of_intervals=time_periods,
             number_of_expansion_intervals=expansion_periods,
         ),
-        engine_settings=engine_settings_from_environment(),
+        engine_settings=engine_settings,
         graph=system,
     )
 
@@ -153,7 +153,7 @@ def test_chiller_operation():
         assert chiller_input[i] == 3
 
 
-def test_carbon_aggregation():
+def test_carbon_aggregation(engine_settings):
     expansion_periods = 1
     time_periods = 48
     interval_duration = 30
@@ -210,7 +210,7 @@ def test_carbon_aggregation():
             number_of_intervals=time_periods,
             number_of_expansion_intervals=expansion_periods,
         ),
-        engine_settings=engine_settings_from_environment(),
+        engine_settings=engine_settings,
         graph=system,
         objective_set=ObjectiveSet(objective_list=[import_tariff]),
     )