From 0cdc32a1bef3b8ac15b0894ca0fa3f7e6d5abc8d Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 30 Jun 2026 14:38:35 +0200 Subject: [PATCH 1/3] use all serializer celery parameters --- scripts/celeryconfig_redis.py | 10 +++++++++- scripts/celeryconfig_redis.yaml | 12 ++++++++---- scripts/celeryconfig_sql.py | 10 +++++++++- src/ewoksjob/tests/conftest.py | 20 ++++++++++++++++++-- src/ewoksjob/tests/test_config.py | 30 +++++++++++++++++++++++++----- 5 files changed, 69 insertions(+), 13 deletions(-) diff --git a/scripts/celeryconfig_redis.py b/scripts/celeryconfig_redis.py index e9509d3..1f7f0a0 100644 --- a/scripts/celeryconfig_redis.py +++ b/scripts/celeryconfig_redis.py @@ -1,8 +1,16 @@ CELERY = { "broker_url": "redis://localhost:6379/3", "result_backend": "redis://localhost:6379/4", + "task_serializer": "pickle", "result_serializer": "pickle", - "accept_content": ["application/json", "application/x-python-serialize"], + "accept_content": [ + "application/json", + "application/x-python-serialize", + ], + "result_accept_content": [ + "application/json", + "application/x-python-serialize", + ], "result_expires": 600, "task_remote_tracebacks": True, "broker_connection_retry_on_startup": True, diff --git a/scripts/celeryconfig_redis.yaml b/scripts/celeryconfig_redis.yaml index 4ad6309..d689f22 100644 --- a/scripts/celeryconfig_redis.yaml +++ b/scripts/celeryconfig_redis.yaml @@ -1,12 +1,16 @@ bliss_ignored: true celery: - accept_content: - - application/json - - application/x-python-serialize broker_url: redis://localhost:6379/3 result_backend: redis://localhost:6379/4 - result_expires: 600 + task_serializer: pickle result_serializer: pickle + accept_content: + - application/json + - application/x-python-serialize + result_accept_content: + - application/json + - application/x-python-serialize + result_expires: 600 task_remote_tracebacks: true broker_connection_retry_on_startup: true enable_utc: false diff --git a/scripts/celeryconfig_sql.py b/scripts/celeryconfig_sql.py index 8ca0534..70c0266 100644 --- a/scripts/celeryconfig_sql.py +++ b/scripts/celeryconfig_sql.py @@ -8,8 +8,16 @@ CELERY = { "broker_url": f"sqla+sqlite:///{os.path.join(_DATA_DIR, 'celery.db')}", "result_backend": f"db+sqlite:///{os.path.join(_DATA_DIR, 'celery_results.db')}", + "task_serializer": "pickle", "result_serializer": "pickle", - "accept_content": ["application/json", "application/x-python-serialize"], + "accept_content": [ + "application/json", + "application/x-python-serialize", + ], + "result_accept_content": [ + "application/json", + "application/x-python-serialize", + ], "result_expires": 600, "task_remote_tracebacks": True, "broker_connection_retry_on_startup": True, diff --git a/src/ewoksjob/tests/conftest.py b/src/ewoksjob/tests/conftest.py index 1218e4c..e6681cd 100644 --- a/src/ewoksjob/tests/conftest.py +++ b/src/ewoksjob/tests/conftest.py @@ -20,8 +20,16 @@ def celery_config(redis_proc): return { "broker_url": f"{url}/0", "result_backend": f"{url}/1", + "task_serializer": "pickle", "result_serializer": "pickle", - "accept_content": ["application/json", "application/x-python-serialize"], + "accept_content": [ + "application/json", + "application/x-python-serialize", + ], + "result_accept_content": [ + "application/json", + "application/x-python-serialize", + ], "task_remote_tracebacks": True, "enable_utc": False, } @@ -35,8 +43,16 @@ def celery_config(tmpdir_factory): "broker_url": "memory://", # "broker_url": f"sqla+sqlite:///{tmpdir / 'celery.db'}", "result_backend": f"db+sqlite:///{tmpdir / 'celery_results.db'}", + "task_serializer": "pickle", "result_serializer": "pickle", - "accept_content": ["application/json", "application/x-python-serialize"], + "accept_content": [ + "application/json", + "application/x-python-serialize", + ], + "result_accept_content": [ + "application/json", + "application/x-python-serialize", + ], "task_remote_tracebacks": True, "enable_utc": False, } diff --git a/src/ewoksjob/tests/test_config.py b/src/ewoksjob/tests/test_config.py index 4ada19c..415cacd 100644 --- a/src/ewoksjob/tests/test_config.py +++ b/src/ewoksjob/tests/test_config.py @@ -8,8 +8,16 @@ EXPECTED = { "broker_url": "redis://localhost:6379/3", "result_backend": "redis://localhost:6379/4", + "task_serializer": "pickle", "result_serializer": "pickle", - "accept_content": ["application/json", "application/x-python-serialize"], + "accept_content": [ + "application/json", + "application/x-python-serialize", + ], + "result_accept_content": [ + "application/json", + "application/x-python-serialize", + ], "result_expires": 600, "task_remote_tracebacks": True, "broker_connection_retry_on_startup": True, @@ -35,8 +43,16 @@ CELERY = { "broker_url": "redis://localhost:6379/3", "result_backend": "redis://localhost:6379/4", + "task_serializer": "pickle", "result_serializer": "pickle", - "accept_content": ["application/json", "application/x-python-serialize"], + "accept_content": [ + "application/json", + "application/x-python-serialize", + ], + "result_accept_content": [ + "application/json", + "application/x-python-serialize", + ], "result_expires": 600, "task_remote_tracebacks": True, "broker_connection_retry_on_startup": True, @@ -63,13 +79,17 @@ _YAML_CONTENT = """ celery: + broker_url: redis://localhost:6379/3 + result_backend: redis://localhost:6379/4 + task_serializer: pickle + result_serializer: pickle accept_content: - application/json - application/x-python-serialize - broker_url: redis://localhost:6379/3 - result_backend: redis://localhost:6379/4 + result_accept_content: + - application/json + - application/x-python-serialize result_expires: 600 - result_serializer: pickle task_remote_tracebacks: true broker_connection_retry_on_startup: true enable_utc: false From 3726c2143031de4edad2e1c9a288d72cfe4018b2 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 30 Jun 2026 14:40:00 +0200 Subject: [PATCH 2/3] docs: refactor configuration page --- doc/tutorials/configuration.rst | 185 +++++++++++++++++++++----------- 1 file changed, 123 insertions(+), 62 deletions(-) diff --git a/doc/tutorials/configuration.rst b/doc/tutorials/configuration.rst index 08d471f..80b674f 100644 --- a/doc/tutorials/configuration.rst +++ b/doc/tutorials/configuration.rst @@ -1,51 +1,152 @@ Configuration ============= -Celery scheduler ----------------- +Location +-------- -Both the client and the worker(s) need to know where the *celery* messages -and results are stored +The configuration can be declared in a + +- Python module: + - myproject.config +- Python file: + - /tmp/ewoks/config.py +- Yaml file: + - /tmp/ewoks/config.yml +- Beacon yaml file: + - beacon:///ewoks/config.yml + - beacon://id22:25000/ewoks/config.yml + +The configuration URI can be provided as an environment variable + +.. code-block:: bash + + export EWOKS_CONFIG_URI=myproject.config + +In case of a Beacon URL that has `/ewoks/config.yml` + +.. code-block:: bash + + export EWOKS_CONFIG_URI=beacon://hostname:25000/ewoks/config.yml + +it is enough to specify the `BEACON_HOST` environment variable + +.. code-block:: bash + + export BEACON_HOST=hostname:25000 + +On the worker side, the configuration URI can also be provided as a CLI argument + +.. code-block:: bash + + ewoksjob --config=myproject.config worker + +.. note:: + + The ``ewoksjob`` command line interface is a wrapper of the `Celery command line interface `_. + + For more information, see the :ref:`Ewoksjob commands reference ` + +Sections +-------- + +Celery +++++++ + +Both the client and the worker(s) must be configured to use the same +message broker and result backend. + +The **broker** transports task messages from the client to the worker, +while the **result backend** stores task states and return values for +retrieval by the client. .. code-block:: python - # SQLite backend + # SQLite CELERY = { - "broker_url": "sqla+sqlite:///path/to/celery.db" - "result_backend": "db+sqlite:///path/to/celery_results.db" + "broker_url": "sqla+sqlite:///path/to/celery.db", + "result_backend": "db+sqlite:///path/to/celery_results.db", } - # Redis backend + # Redis CELERY = { - "broker_url": "redis://localhost:10003/3 - "result_backend": "redis://localhost:10003/4" + "broker_url": "redis://localhost:10003/3", + "result_backend": "redis://localhost:10003/4", } - # RabbitMQ backend + # RabbitMQ CELERY = { - "broker_url": "pyamqp://guest@localhost//" - "result_backend": "rpc://" - "result_persistent": True + "broker_url": "pyamqp://guest@localhost//", + "result_backend": "rpc://", + "result_persistent": True, } -Other backends for results are supported (mongo, memcached). Other configuration parameters -are available, like for data serialization (json by default) +Celery supports many other broker and result backend implementations, +including MongoDB and Memcached. + +The following example shows a recommended configuration when Python +objects such as tuples must be preserved. In this case, tasks and +results are serialized using ``pickle`` while remaining compatible +with clients or workers that still use JSON. .. code-block:: python CELERY = { "broker_url": "...", "result_backend": "...", + "task_serializer": "pickle", "result_serializer": "pickle", - "accept_content: ["application/json", "application/x-python-serialize"], + "accept_content": [ + "application/json", + "application/x-python-serialize", + ], + "result_accept_content": [ + "application/json", + "application/x-python-serialize", + ], "result_expires": 600, "task_remote_tracebacks": True, "broker_connection_retry_on_startup": True, "enable_utc": False, } -The `Celery documentation `_ -describes the different parameters available. +The following Celery settings are commonly used: + +- ``broker_url`` *(Client ↔ Broker ↔ Worker)* + Connection URL of the message broker used to transport task messages. + +- ``result_backend`` *(Worker ↔ Result Backend ↔ Client)* + Connection URL of the backend used to store task states and return values. + +- ``task_serializer`` *(Client → Broker)* + Serializes outgoing task messages. ``"pickle"`` preserves Python types such as tuples. + +- ``accept_content`` *(Broker → Worker)* + List of MIME types that workers are allowed to deserialize and execute. The example accepts both JSON and pickle messages. + +- ``result_serializer`` *(Worker → Result Backend)* + Serializes task return values before storing them in the result backend. ``"pickle"`` preserves Python types. + +- ``result_accept_content`` *(Result Backend → Client)* + List of MIME types that clients are allowed to deserialize when retrieving task results. + +- ``result_expires`` *(Result Backend)* + Time-to-live (TTL), in seconds, for stored task results before they are automatically removed. + +- ``task_remote_tracebacks`` *(Worker → Client)* + Includes the worker's Python traceback in task failures, making debugging easier. + +- ``broker_connection_retry_on_startup`` *(Client / Worker)* + Retries connecting to the message broker during startup if it is temporarily unavailable. + +- ``enable_utc`` *(Global)* + Whether Celery uses UTC internally. Set to ``False`` to use the local timezone instead. + +See the `Celery configuration documentation +`_ +for a complete list of configuration options. + +Ewoks execution ++++++++++++++++ Workflow execution parameters can be defined through the `EWOKS_EXECUTION` variable. @@ -63,6 +164,9 @@ Workflow execution parameters can be defined through the `EWOKS_EXECUTION` varia } } +Job scheduler ++++++++++++++ + Ewoksjob options can be defined through the `EWOKSJOB_OPTIONS` variable. .. code-block:: python @@ -71,46 +175,3 @@ Ewoksjob options can be defined through the `EWOKSJOB_OPTIONS` variable. "log_memory_usage": True, "detect_memory_leaks": True, } - - -The configuration can be declared in a - -- Python module: - - myproject.config -- Python file: - - /tmp/ewoks/config.py -- Yaml file: - - /tmp/ewoks/config.yml -- Beacon yaml file: - - beacon:///ewoks/config.yml - - beacon://id22:25000/ewoks/config.yml - -The configuration URI can be provided as an environment variable - -.. code-block:: bash - - export EWOKS_CONFIG_URI=myproject.config - -In case of a Beacon URL that has `/ewoks/config.yml` - -.. code-block:: bash - - export EWOKS_CONFIG_URI=beacon://hostname:25000/ewoks/config.yml - -it is enough to specify the `BEACON_HOST` environment variable - -.. code-block:: bash - - export BEACON_HOST=hostname:25000 - -On the worker side, the configuration URI can also be provided as a CLI argument - -.. code-block:: bash - - ewoksjob --config=myproject.config worker - -.. note:: - - The ``ewoksjob`` command line interface is a wrapper of the `Celery command line interface `_. - - For more information, see the :ref:`Ewoksjob commands reference ` From c6ef77adaac19ba73c5ffbe48d83f0f422de66de Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 30 Jun 2026 15:41:27 +0200 Subject: [PATCH 3/3] doc fix --- doc/tutorials/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorials/configuration.rst b/doc/tutorials/configuration.rst index 80b674f..5d239b8 100644 --- a/doc/tutorials/configuration.rst +++ b/doc/tutorials/configuration.rst @@ -52,7 +52,7 @@ Sections Celery ++++++ -Both the client and the worker(s) must be configured to use the same +Both the client(s) and the worker(s) must be configured to use the same message broker and result backend. The **broker** transports task messages from the client to the worker,