Logs for debugging flaky test & rerun of failing container in specific order

.circleci/config.yml

@@ -251,7 +251,7 @@ jobs:
           POSTGRES_DB: integreat
           POSTGRES_PASSWORD: password
     resource_class: large
-    parallelism: 16
+    parallelism: 1
     steps:
       - checkout
       - attach_workspace:
@@ -266,8 +266,12 @@ jobs:
           name: Migrate database
           command: integreat-cms-cli migrate --settings=integreat_cms.core.circleci_settings
       - run:
-          name: Run tests
-          command: pytest --circleci-parallelize --disable-warnings --cov=integreat_cms --cov-report=xml:coverage.xml --junitxml=test-results/junit.xml  --ds=integreat_cms.core.circleci_settings
+          name: Run specific tests in strict order
+          command: |
+            pytest tests/cms/views/status_code/test_view_status_code_11.py tests/pdf/test_pdf_export.py tests/cms/test_media_library.py tests/cms/test_page_filters.py tests/cms/views/utils/test_hix.py tests/cms/views/poi_categories/test_poi_category_form_view.py tests/cms/views/test_public_view_status_code.py tests/core/management/commands/test_summ_ai_bulk.py --disable-warnings --ds=integreat_cms.core.circleci_settings
+      #- run:
+      #    name: Run tests
+      #    command: pytest --circleci-parallelize --disable-warnings --cov=integreat_cms --cov-report=xml:coverage.xml --junitxml=test-results/junit.xml  --ds=integreat_cms.core.circleci_settings
       - store_test_results:
           path: test-results
       - store_artifacts:

integreat_cms/cms/views/poi_categories/poi_category_form_view.py

@@ -243,9 +243,22 @@ def form_valid(self, form: POICategoryForm) -> HttpResponseRedirect:
         if TYPE_CHECKING:
             assert self.object
             assert self.formset
+        formset_has_changed = self.formset.has_changed()
+        form_has_changed = form.has_changed()
+        logger.debug(
+            "formset changed: %s; form changed: %s",
+            formset_has_changed,
+            form_has_changed,
+        )
         if not self.formset.has_changed() and not form.has_changed():
+            logger.debug("form has not changed")
             messages.info(self.request, _("No changes made"))
         else:
+            logger.debug("form has changed")
+            logger.debug("changed data in form is %s", form.changed_data)
+            logger.debug(
+                "changed data in formset is %s", [f.changed_data for f in self.formset]
+            )
             self.formset.save()
             messages.success(
                 self.request,

integreat_cms/core/settings.py

@@ -752,6 +752,10 @@
             "handlers": ["console-colored", "logfile"],
             "level": LOG_LEVEL,
         },
+        "tests": {
+            "handlers": ["console-colored", "logfile"],
+            "level": "DEBUG",
+        },
         "integreat_cms.core.management.commands": {
             "handlers": [
                 "management-command-stdout",

test_selection.txt

@@ -0,0 +1,8 @@
+tests/cms/views/status_code/test_view_status_code_11.py
+tests/pdf/test_pdf_export.py
+tests/cms/test_media_library.py
+tests/cms/test_page_filters.py
+tests/cms/views/utils/test_hix.py
+tests/cms/views/poi_categories/test_poi_category_form_view.py
+tests/cms/views/test_public_view_status_code.py
+tests/core/management/commands/test_summ_ai_bulk.py

tests/cms/views/poi_categories/test_poi_category_form_view.py

@@ -1,12 +1,20 @@
+import logging
+import re
+
 import pytest
 from django.conf import settings
 from django.test.client import Client
 from django.urls import resolve, reverse
+from django.utils.html import strip_tags
 
+from integreat_cms.cms.models.languages.language import Language
 from integreat_cms.cms.models.poi_categories.poi_category import POICategory
 from integreat_cms.cms.models.pois.poi import POI
 from tests.conftest import ANONYMOUS, CMS_TEAM, ROOT, SERVICE_TEAM, STAFF_ROLES
 
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
 DEFAULT_POST_DATA = {
     "icon": "daily_routine",
     "color": "#1DC6C6",
@@ -236,6 +244,9 @@ def test_no_changes_were_made_message(
 
     new_poicategory_url = reverse("new_poicategory")
 
+    for language in Language.objects.all():
+        logger.debug("Language: %s - language-id: %s - slug:%s",language, language.id, language.slug)
+
     response = client.post(
         new_poicategory_url,
         data=DEFAULT_POST_DATA
@@ -250,6 +261,12 @@ def test_no_changes_were_made_message(
     poicategory = POICategory.objects.get(id=id_of_poicategory)
     translation = poicategory.translations.get(language__slug="de")
 
+    translations = poicategory.translations.all()
+
+    logger.debug("Translations after first save")
+    for translation in translations:
+        logger.debug("Translation: %s - language: %s", translation, translation.language)
+
     response = client.post(
         edit_url,
         data={
@@ -275,10 +292,40 @@ def test_no_changes_were_made_message(
         },
     )
 
+    edit_url = response.headers.get("location")
+
+    id_of_poicategory = resolve(edit_url).kwargs["pk"]
+
+    poicategory = POICategory.objects.get(id=id_of_poicategory)
+    translation = poicategory.translations.get(language__slug="de")
+
+    translations = poicategory.translations.all()
+
+    logger.debug("Translations after second save")
+    for translation in translations:
+        logger.debug("Translation: %s - language: %s", translation, translation.language)
+
+
+
     assert response.status_code == 302
     response = client.get(edit_url)
     assert response.status_code == 200
 
+    content = response.content.decode("utf-8")
+
+    # clean up response content for logs:
+    # add newlines where HTML block elements usually separate content
+    clean_content = re.sub(
+        r"</?(div|p|br|li|ul|ol|tr|td|section|article)[^>]*>",
+        "\n",
+        content,
+        flags=re.IGNORECASE,
+    )
+    clean_content = strip_tags(clean_content)
+    clean_content = re.sub(r"\s*\n\s*", "\n", clean_content)  # clean up newlines
+    clean_content = re.sub(r"\n{2,}", "\n", clean_content)  # collapse double newlines
+    logger.debug("response is: %s", clean_content)
+
     assert "Keine Änderungen vorgenommen" in response.content.decode("utf-8")
 
 

tests/conftest.py

@@ -161,3 +161,30 @@ def configure_celery_for_tests(settings: SettingsWrapper) -> None:
     # so we set celery to run synchronously and propagate errors to the test runner
     settings.CELERY_TASK_ALWAYS_EAGER = True
     settings.CELERY_TASK_EAGER_PROPAGATES = True
+
+def pytest_collection_modifyitems(items):
+    DESIRED_ORDER = [
+        "tests.cms.views.status_code.test_view_status_code_11",
+        "tests.pdf.test_pdf_export",
+        "tests.cms.test_media_library",
+        "tests.cms.test_page_filters",
+        "test_hix",
+        "test_poi_category_form_view",
+        "tests.cms.views.test_public_view_status_code",
+        "tests.core.management.commands.test_summ_ai_bulk",
+    ]
+
+    def module_name(item):
+        return item.module.__name__
+
+    sorted_items = []
+    remaining = items.copy()
+
+    for module in DESIRED_ORDER:
+        matching = [it for it in remaining if module_name(it) == module]
+        remaining = [it for it in remaining if module_name(it) != module]
+        sorted_items.extend(matching)
+
+    print(sorted_items)
+
+    items[:] = sorted_items