From 03cac5f3976ced5cf023eb34e1cdda1adc69cbce Mon Sep 17 00:00:00 2001 From: Helena Thompson Date: Wed, 20 May 2026 21:22:48 +0100 Subject: [PATCH 01/36] WIP --- Makefile | 3 +- backend/consultations/admin.py | 4 +- backend/consultations/dummy_data.py | 267 ++++++++---- .../commands/generate_dummy_data.py | 40 +- .../commands/prepare_environment.py | 28 ++ .../management/commands/prepare_s3.py | 392 ++++++++++++++++++ backend/start.sh | 3 +- backend/tests/commands/test_dummy_data.py | 4 +- .../commands/test_prepare_environment.py | 72 ++++ backend/tests/commands/test_prepare_s3.py | 143 +++++++ backend/tests/examples/sample_questions.json | 74 ++++ backend/tests/examples/sample_questions.yml | 117 ------ .../tests/unit/test_generate_dummy_data.py | 104 ++++- 13 files changed, 1002 insertions(+), 249 deletions(-) create mode 100644 backend/consultations/management/commands/prepare_environment.py create mode 100644 backend/consultations/management/commands/prepare_s3.py create mode 100644 backend/tests/commands/test_prepare_environment.py create mode 100644 backend/tests/commands/test_prepare_s3.py create mode 100644 backend/tests/examples/sample_questions.json delete mode 100644 backend/tests/examples/sample_questions.yml diff --git a/Makefile b/Makefile index e781fb9c9..fcefaf0c4 100644 --- a/Makefile +++ b/Makefile @@ -178,7 +178,8 @@ dummy_data: ## Generate dummy consultations. Only works in dev cd backend && PYTHONPATH=.. uv run python manage.py generate_dummy_data .PHONY: dev_environment -dev_environment: reset_db migrate dummy_data ## set up the database with dummy data +dev_environment: setup_db ## set up the database with dummy data + cd backend && PYTHONPATH=.. uv run python manage.py prepare_environment # Docker AWS_REGION=eu-west-2 diff --git a/backend/consultations/admin.py b/backend/consultations/admin.py index 90190396c..98e08f1b1 100644 --- a/backend/consultations/admin.py +++ b/backend/consultations/admin.py @@ -6,7 +6,7 @@ from django.urls import path, reverse from simple_history.admin import SimpleHistoryAdmin -from consultations.dummy_data import create_dummy_consultation_from_yaml_job +from consultations.dummy_data import create_dummy_consultation_job from consultations.models import ( CandidateTheme, Consultation, @@ -55,7 +55,7 @@ def create_dummy_consultation(modeladmin, request, queryset, size=10): ) return - create_dummy_consultation_from_yaml_job.delay( + create_dummy_consultation_job.delay( number_respondents=size, consultation=consultation ) diff --git a/backend/consultations/dummy_data.py b/backend/consultations/dummy_data.py index e9799e0b1..224b568b7 100644 --- a/backend/consultations/dummy_data.py +++ b/backend/consultations/dummy_data.py @@ -1,14 +1,18 @@ +import json import random -from typing import Literal +from typing import Optional -import yaml from django.conf import settings from consultations.models import ( + CandidateTheme, + CandidateThemeResponse, Consultation, MultiChoiceAnswer, Question, + Response, ResponseAnnotation, + SelectedTheme, ) from factories import ( CandidateThemeFactory, @@ -20,44 +24,77 @@ SelectedThemeFactory, ) from hosting_environment import HostingEnvironment -from rq_context import job logger = settings.LOGGER -DATA_BY_STAGE = { - Consultation.Stage.ANALYSIS: { - "CONSULTATION_NAME": "Dummy Consultation at Analysis Stage", - "QUESTION_THEME_STATUS": Question.ThemeStatus.CONFIRMED, - "CONSULTATION_CODE": "dummy-consultation-analysis", +DUMMY_CONSULTATIONS = [ + { + "CONSULTATION_NAME": "Dummy Consultation - Data Setup", + "CONSULTATION_CODE": "dummy-setup", + "CONSULTATION_STAGE": Consultation.Stage.SETUP, + "QUESTION_THEME_STATUS": Question.ThemeStatus.DRAFT, }, - Consultation.Stage.FINALISING_THEMES: { - "CONSULTATION_NAME": "Dummy Consultation at Finalising Themes Stage", + { + "CONSULTATION_NAME": "Dummy Consultation - Starting finalising themes", + "CONSULTATION_CODE": "dummy-start-finalising-themes", + "CONSULTATION_STAGE": Consultation.Stage.THEME_SIGN_OFF, "QUESTION_THEME_STATUS": Question.ThemeStatus.DRAFT, - "CONSULTATION_CODE": "dummy-consultation-finalising", }, -} + { + "CONSULTATION_NAME": "Dummy Consultation - Finished finalising themes", + "CONSULTATION_CODE": "dummy-finished-finalising-themes", + "CONSULTATION_STAGE": Consultation.Stage.THEME_SIGN_OFF, + "QUESTION_THEME_STATUS": Question.ThemeStatus.CONFIRMED, + }, + { + "CONSULTATION_NAME": "Dummy Consultation - Analysis", + "CONSULTATION_CODE": "dummy-analysis", + "CONSULTATION_STAGE": Consultation.Stage.ANALYSIS, + "QUESTION_THEME_STATUS": Question.ThemeStatus.CONFIRMED, + }, +] + +NUMBER_RESPONDENTS = 100 +REGIONS = ["North East", "North West", "South East", "South West", "Midlands", "London"] +AGE_GROUPS = ["Under 18", "18-35", "36-50", "51-65", "66+"] +RESPONDENT_TYPES = ["Individual", "Organisation"] +SAMPLE_QUESTIONS_PATH = "./tests/examples/sample_questions.json" + + +def create_consultation(config): + """Create and return a Consultation from a config dict.""" + return ConsultationFactory( + title=config["CONSULTATION_NAME"], + code=config["CONSULTATION_CODE"], + stage=config["CONSULTATION_STAGE"], + ) -def create_consultation(stage): - """Create and return a Consultation.""" - name = DATA_BY_STAGE[stage]["CONSULTATION_NAME"] - code = DATA_BY_STAGE[stage]["CONSULTATION_CODE"] - return ConsultationFactory(title=name, stage=stage, code=code) +def _demographics_for(themefinder_id): + """Return deterministic demographics based on themefinder_id.""" + return { + "region": REGIONS[themefinder_id % len(REGIONS)], + "age_group": AGE_GROUPS[themefinder_id % len(AGE_GROUPS)], + "respondent_type": RESPONDENT_TYPES[themefinder_id % len(RESPONDENT_TYPES)], + } def create_respondents(consultation, number_respondents): """Create and return a list of Respondents.""" return [ - RespondentFactory(consultation=consultation, themefinder_id=i) + RespondentFactory( + consultation=consultation, + themefinder_id=i, + demographics=_demographics_for(i), + ) for i in range(1, number_respondents + 1) ] -def create_question(consultation, question_data): +def create_question(consultation, question_data, theme_status): """Create and return a Question.""" has_free_text = question_data["has_free_text"] has_multiple_choice = question_data["has_multiple_choice"] - theme_status = DATA_BY_STAGE[consultation.stage]["QUESTION_THEME_STATUS"] return QuestionFactory( text=question_data["question_text"], @@ -75,38 +112,54 @@ def create_multi_choice_answers(question, choices): MultiChoiceAnswer.objects.bulk_create(multi_choice_objects) -def create_candidate_theme_recursive( - question, number_respondents, candidate_theme_data, parent=None -): - """Create a CandidateTheme (and SelectedTheme if selected) recursively.""" - name = candidate_theme_data.get("name") - description = candidate_theme_data.get("description", "") - key = candidate_theme_data.get("key") - approximate_frequency = round( - candidate_theme_data.get("approximate_frequency_pct", 0) * number_respondents - ) +def create_candidate_themes(question, candidate_themes_data): + """Create CandidateThemes (and SelectedThemes if selected) from flat theme list.""" + key_to_candidate_theme = {} - candidate_theme = CandidateThemeFactory( + for theme_data in candidate_themes_data: + candidate_theme = CandidateThemeFactory( + question=question, + name=theme_data["name"], + description=theme_data.get("description", ""), + parent=None, + approximate_frequency=theme_data.get("approximate_frequency", 0), + ) + key_to_candidate_theme[theme_data["key"]] = candidate_theme + + if question.theme_status == Question.ThemeStatus.CONFIRMED and theme_data.get("selected"): + selected_theme = SelectedThemeFactory( + question=question, + name=theme_data["name"], + description=theme_data.get("description", ""), + key=theme_data["key"], + ) + candidate_theme.selectedtheme = selected_theme + candidate_theme.save() + + for theme_data in candidate_themes_data: + parent_key = theme_data.get("parent_key") + if parent_key and parent_key in key_to_candidate_theme: + candidate_theme = key_to_candidate_theme[theme_data["key"]] + candidate_theme.parent = key_to_candidate_theme[parent_key] + candidate_theme.save() + + +def create_default_selected_themes(question): + """Create the 'Other' and 'No Reason Given' themes added at start of assign-themes.""" + SelectedTheme.objects.get_or_create( question=question, - description=description, - name=name, - parent=parent, - approximate_frequency=approximate_frequency, + name="Other", + defaults={ + "description": "The response discusses an issue not covered by the listed themes" + }, + ) + SelectedTheme.objects.get_or_create( + question=question, + name="No Reason Given", + defaults={ + "description": "The response does not provide a substantive answer to the question" + }, ) - - if question.theme_status == Question.ThemeStatus.CONFIRMED and candidate_theme_data.get( - "selected" - ): - selected_theme = SelectedThemeFactory( - question=question, name=name, description=description, key=key - ) - candidate_theme.selectedtheme = selected_theme - candidate_theme.save() - - for child in candidate_theme_data.get("children", []): - create_candidate_theme_recursive( - question, number_respondents, child, parent=candidate_theme - ) def create_response(respondent, question, free_text_answers): @@ -142,79 +195,137 @@ def create_response_chosen_options(response, multiple_choice_options): response.chosen_options.add(*answers) -def create_dummy_consultation_from_yaml( - file_path: str = "./tests/examples/sample_questions.yml", +def candidate_theme_keys_for_respondent(themefinder_id, theme_keys): + """Deterministically assign 1-2 theme keys to a respondent based on their ID.""" + idx = themefinder_id % len(theme_keys) + if themefinder_id % 3 == 0 and len(theme_keys) > 1: + idx2 = (themefinder_id + 1) % len(theme_keys) + return [theme_keys[idx], theme_keys[idx2]] + return [theme_keys[idx]] + + +def create_candidate_theme_responses(question): + """Assign responses to candidate themes at all levels for a question deterministically.""" + all_themes = list(CandidateTheme.objects.filter(question=question)) + responses = list( + Response.objects.filter(question=question, free_text__isnull=False) + .exclude(free_text="") + .select_related("respondent") + ) + if not all_themes or not responses: + return + + themes_by_parent = {} + for theme in all_themes: + themes_by_parent.setdefault(theme.parent_id, []).append(theme) + + records = [] + for sibling_themes in themes_by_parent.values(): + for response in responses: + assigned = candidate_theme_keys_for_respondent( + response.respondent.themefinder_id, + sibling_themes, + ) + for theme in assigned: + records.append(CandidateThemeResponse(candidate_theme=theme, response=response)) + + CandidateThemeResponse.objects.bulk_create(records, ignore_conflicts=True) + + +def create_dummy_consultation( + file_path: str = SAMPLE_QUESTIONS_PATH, number_respondents: int = 10, - consultation: Consultation | None = None, - consultation_stage: Literal["finalising_themes", "analysis"] | None = None, -) -> ConsultationFactory: + consultation: Optional[Consultation] = None, + config: Optional[dict] = None, +) -> Consultation: """ - Create consultation with questions, responses and themes from yaml file. - Creates relevant objects: Consultation, Question, CandidateTheme, SelectedTheme, - Response, ResponseAnnotation, Respondent. + Create consultation with questions, responses and themes from JSON file. + Creates relevant objects depending on stage and theme status: + - SETUP: Consultation, Questions, Respondents, Responses (ready for finding themes) + - THEME_SIGN_OFF (DRAFT): + CandidateThemes + CandidateThemeResponses (finalising) + - THEME_SIGN_OFF (CONFIRMED): + CandidateThemes + SelectedThemes (ready for assignment) + - ANALYSIS: + SelectedThemes + ResponseAnnotations """ if HostingEnvironment.is_production(): raise RuntimeError("Dummy data generation should not be run in production") - consultation_stage = consultation_stage or "analysis" + if config is None: + config = DUMMY_CONSULTATIONS[-1] + + consultation_stage = config["CONSULTATION_STAGE"] + theme_status = config["QUESTION_THEME_STATUS"] + if consultation is None: logger.info("Creating consultation at stage: {stage}", stage=consultation_stage) - consultation = create_consultation(consultation_stage) + consultation = create_consultation(config) logger.info("Creating {number_respondents} respondents", number_respondents=number_respondents) respondents = create_respondents(consultation, number_respondents) with open(file_path, "r") as file: - questions_data = yaml.safe_load(file) + questions_data = json.load(file) + + has_candidate_themes = consultation_stage in [ + Consultation.Stage.THEME_SIGN_OFF, + Consultation.Stage.ANALYSIS, + ] + has_candidate_theme_responses = consultation_stage in [ + Consultation.Stage.THEME_SIGN_OFF, + Consultation.Stage.ANALYSIS, + ] + has_default_selected_themes = consultation_stage == Consultation.Stage.ANALYSIS + has_response_annotations = consultation_stage == Consultation.Stage.ANALYSIS for question_data in questions_data: logger.info("Creating a new question...") - question = create_question(consultation, question_data) + question = create_question(consultation, question_data, theme_status) multiple_choice_options = question_data.get("multiple_choice_options", []) free_text_answers = question_data.get("free_text_answers", []) if question.has_multiple_choice: - logger.info("Multiple choice question - create multi choice answers") create_multi_choice_answers(question, multiple_choice_options) - if question.has_free_text: - logger.info("Free text question - create candidate themes") + if question.has_free_text and has_candidate_themes: + create_candidate_themes(question, question_data["candidate_themes"]) - for candidate_theme_data in question_data["candidate_themes"]: - create_candidate_theme_recursive(question, len(respondents), candidate_theme_data) + if question.has_free_text and has_default_selected_themes: + create_default_selected_themes(question) for respondent in respondents: - logger.info("Creating a new response...") response = create_response(respondent, question, free_text_answers) - if question.has_free_text and consultation.stage == Consultation.Stage.ANALYSIS: - logger.info("Free text question - create response annotation") + if question.has_free_text and has_response_annotations: create_response_annotation(response, question) if question.has_multiple_choice: - logger.info("Multiple choice question - create response chosen options") create_response_chosen_options(response, multiple_choice_options) + if question.has_free_text and has_candidate_theme_responses: + create_candidate_theme_responses(question) + logger.info( "Finished adding question and responses for question {question_number}", question_number=question.number, ) - logger.info( - "Finished adding dummy data for consultation {consultation_code}", - consultation_code=consultation.code, - ) + + logger.info( + "Finished adding dummy data for consultation {consultation_code}", + consultation_code=consultation.code, + ) return consultation # Will only be run occasionally to create dummy data - not in prod @job("default", timeout=2400) -def create_dummy_consultation_from_yaml_job( - file_path: str = "./tests/examples/sample_questions.yml", +def create_dummy_consultation_job( + file_path: str = SAMPLE_QUESTIONS_PATH, number_respondents: int = 10, - consultation: Consultation | None = None, + consultation: Optional[Consultation] = None, + config: Optional[dict] = None, ): - create_dummy_consultation_from_yaml( + create_dummy_consultation( file_path=file_path, number_respondents=number_respondents, consultation=consultation, + config=config, ) diff --git a/backend/consultations/management/commands/generate_dummy_data.py b/backend/consultations/management/commands/generate_dummy_data.py index 92e2e959e..32c48d3e7 100644 --- a/backend/consultations/management/commands/generate_dummy_data.py +++ b/backend/consultations/management/commands/generate_dummy_data.py @@ -1,40 +1,18 @@ from django.core.management.base import BaseCommand -from authentication.models import User -from consultations.dummy_data import create_dummy_consultation_from_yaml -from consultations.models import Consultation - -ADMIN_USER_EMAIL = "admin@example.com" -POLICY_USER_EMAIL = "policy@example.com" +from consultations.dummy_data import ( + DUMMY_CONSULTATIONS, + NUMBER_RESPONDENTS, + create_dummy_consultation, +) class Command(BaseCommand): - help = ( - "Generate two dummy consultations, one at finalising themes stage and one at analysis stage. " - "Generate two dummy users, one with admin (staff) privileges and one without." - ) + help = "Generate dummy consultations at each pipeline stage." def handle(self, *args, **options): self.stdout.write("Generating dummy data...") - create_dummy_consultation_from_yaml( - number_respondents=100, consultation_stage=Consultation.Stage.FINALISING_THEMES - ) - analysis_consultation = create_dummy_consultation_from_yaml( - number_respondents=100, consultation_stage=Consultation.Stage.ANALYSIS - ) - - # Admin user is not added to any consultation as they can act on all consultations - _, created = User.objects.update_or_create( - email=ADMIN_USER_EMAIL, defaults={"is_staff": True} - ) - self.stdout.write(f"{'Created' if created else 'Updated'} admin user ({ADMIN_USER_EMAIL})") - - policy_user, created = User.objects.update_or_create( - email=POLICY_USER_EMAIL, defaults={"is_staff": False} - ) - analysis_consultation.users.add(policy_user) - self.stdout.write( - f"{'Created' if created else 'Updated'} policy user ({POLICY_USER_EMAIL})" - ) - + for config in DUMMY_CONSULTATIONS: + self.stdout.write(f" Creating consultation: {config['CONSULTATION_NAME']}...") + create_dummy_consultation(number_respondents=NUMBER_RESPONDENTS, config=config) self.stdout.write("Done.") diff --git a/backend/consultations/management/commands/prepare_environment.py b/backend/consultations/management/commands/prepare_environment.py new file mode 100644 index 000000000..627d35514 --- /dev/null +++ b/backend/consultations/management/commands/prepare_environment.py @@ -0,0 +1,28 @@ +from django.conf import settings +from django.core.management import call_command +from django.core.management.base import BaseCommand +from django.db import connections + + +class Command(BaseCommand): + help = "Prepare the environment: runs migrations on prod; resets and seeds the database and S3 on dev and preprod." + + def handle(self, *args, **options): + environment = getattr(settings, "ENVIRONMENT", "").lower() + + if environment == "prod": + self.stdout.write("Running migrate on prod.") + call_command("migrate", verbosity=1) + return + + self.stdout.write(f"Resetting database on {environment}...") + connection = connections["default"] + with connection.cursor() as cursor: + cursor.execute("DROP SCHEMA public CASCADE;") + cursor.execute("CREATE SCHEMA public;") + + call_command("migrate", verbosity=1) + call_command("createadminusers", verbosity=1) + call_command("generate_dummy_data", verbosity=1) + call_command("prepare_s3", verbosity=1) + self.stdout.write("Environment prepared.") diff --git a/backend/consultations/management/commands/prepare_s3.py b/backend/consultations/management/commands/prepare_s3.py new file mode 100644 index 000000000..ec5638d26 --- /dev/null +++ b/backend/consultations/management/commands/prepare_s3.py @@ -0,0 +1,392 @@ +import json +from datetime import date + +import boto3 +from django.conf import settings +from django.core.management.base import BaseCommand + +from consultations.dummy_data import ( + AGE_GROUPS, + NUMBER_RESPONDENTS, + REGIONS, + RESPONDENT_TYPES, + SAMPLE_QUESTIONS_PATH, + candidate_theme_keys_for_respondent, +) +from hosting_environment import HostingEnvironment + +TIMESTAMP = date.today().isoformat() + + +def _to_jsonl(records): + return "\n".join(json.dumps(r) for r in records) + + +def _load_questions(): + with open(SAMPLE_QUESTIONS_PATH, "r") as f: + return json.load(f) + + +def _build_respondents(): + return [ + { + "themefinder_id": i, + "demographic_data": { + "region": [REGIONS[i % len(REGIONS)]], + "age_group": [AGE_GROUPS[i % len(AGE_GROUPS)]], + "respondent_type": [RESPONDENT_TYPES[i % len(RESPONDENT_TYPES)]], + }, + } + for i in range(1, NUMBER_RESPONDENTS + 1) + ] + + +def _build_responses(question_data): + free_text_answers = question_data.get("free_text_answers", []) + if not free_text_answers: + return [] + non_empty = [a for a in free_text_answers if a] + return [ + {"themefinder_id": i, "text": non_empty[i % len(non_empty)]} + for i in range(1, NUMBER_RESPONDENTS + 1) + ] + + +def _build_multi_choice(question_data): + options = question_data.get("multiple_choice_options", []) + if not options: + return [] + return [ + {"themefinder_id": i, "options": [options[i % len(options)]]} + for i in range(1, NUMBER_RESPONDENTS + 1) + ] + + +def _build_question_json(question_data): + return { + "question_text": question_data["question_text"], + "question_number": question_data["number"], + "has_free_text": question_data["has_free_text"], + "multi_choice_options": question_data.get("multiple_choice_options", []), + } + + +def _build_clustered_themes(question_data): + """Build clustered_themes.json from the flat candidate_themes list.""" + candidate_themes = question_data.get("candidate_themes", []) + key_to_topic_id = {} + theme_nodes = [] + + for i, theme in enumerate(candidate_themes, start=1): + topic_id = str(i) + key_to_topic_id[theme["key"]] = topic_id + parent_key = theme.get("parent_key") + parent_id = key_to_topic_id.get(parent_key, "0") if parent_key else "0" + theme_nodes.append( + { + "topic_id": topic_id, + "parent_id": parent_id, + "topic_label": theme["name"], + "topic_description": theme.get("description", ""), + "source_topic_count": max(1, theme.get("approximate_frequency", 10)), + } + ) + + return {"theme_nodes": theme_nodes} + + +def _themes_grouped_by_parent(candidate_themes): + """Group flat themes by parent_key (None for top-level).""" + groups = {} + for theme in candidate_themes: + groups.setdefault(theme.get("parent_key"), []).append(theme) + return groups + + +def _build_candidate_themes_json(question_data): + """Build themes.json from all candidate_themes (for THEME_SIGN_OFF mapping).""" + candidate_themes = question_data.get("candidate_themes", []) + return [ + { + "theme_key": theme["key"], + "theme_name": theme["name"], + "theme_description": theme.get("description", ""), + } + for theme in candidate_themes + ] + + +DEFAULT_THEMES = [ + { + "theme_key": "OTHER", + "theme_name": "Other", + "theme_description": "The response discusses an issue not covered by the listed themes", + }, + { + "theme_key": "NO_REASON", + "theme_name": "No Reason Given", + "theme_description": "The response does not provide a substantive answer to the question", + }, +] + + +def _build_themes_json(question_data): + """Build themes.json (selected themes) from candidate_themes marked as selected, plus defaults.""" + candidate_themes = question_data.get("candidate_themes", []) + themes = [ + { + "theme_key": theme["key"], + "theme_name": theme["name"], + "theme_description": theme.get("description", ""), + } + for theme in candidate_themes + if theme.get("selected") + ] + themes.extend(DEFAULT_THEMES) + return themes + + + + +def _build_candidate_theme_mappings(question_data): + """Build mapping.jsonl for candidate themes using deterministic assignment per sibling group.""" + candidate_themes = question_data.get("candidate_themes", []) + if not candidate_themes: + return [] + + groups = _themes_grouped_by_parent(candidate_themes) + respondent_keys = {i: [] for i in range(1, NUMBER_RESPONDENTS + 1)} + for sibling_themes in groups.values(): + group_keys = [t["key"] for t in sibling_themes] + for i in range(1, NUMBER_RESPONDENTS + 1): + respondent_keys[i].extend(candidate_theme_keys_for_respondent(i, group_keys)) + + return [ + {"themefinder_id": i, "theme_keys": respondent_keys[i]} + for i in range(1, NUMBER_RESPONDENTS + 1) + ] + + +def _build_theme_mappings(question_data): + """Build mapping.jsonl for selected themes (analysis stage) using deterministic assignment.""" + themes = _build_themes_json(question_data) + if not themes: + return [] + theme_keys = [t["theme_key"] for t in themes] + return [ + {"themefinder_id": i, "theme_keys": candidate_theme_keys_for_respondent(i, theme_keys)} + for i in range(1, NUMBER_RESPONDENTS + 1) + ] + + +def _build_sentiments(): + choices = ["AGREEMENT", "DISAGREEMENT", "UNCLEAR"] + return [ + {"themefinder_id": i, "sentiment": choices[i % len(choices)]} + for i in range(1, NUMBER_RESPONDENTS + 1) + ] + + +def _build_evidence_rich(): + choices = ["YES", "NO"] + return [ + {"themefinder_id": i, "evidence_rich": choices[i % len(choices)]} + for i in range(1, NUMBER_RESPONDENTS + 1) + ] + + +def _build_themes_csv(question_data): + """Build themes.csv content (selected themes) for the assign-themes batch job.""" + themes = _build_themes_json(question_data) + lines = ["Theme Name,Theme Description"] + for theme in themes: + name = theme["theme_name"] + description = theme["theme_description"] + lines.append(f"{name},{description}") + return "\n".join(lines) + + +class Command(BaseCommand): + help = "Reset and seed S3 with dummy consultation data matching the DB. Only runs on deployed non-prod environments." + + def handle(self, *args, **options): + if HostingEnvironment.is_production(): + self.stdout.write("Skipping S3 seed on production.") + return + + if not HostingEnvironment.is_deployed(): + self.stdout.write("Skipping S3 seed on local environment (no real S3 bucket).") + return + + s3_client = boto3.client("s3") + bucket = settings.AWS_BUCKET_NAME + + self._delete_existing_data(s3_client, bucket) + questions_data = _load_questions() + + # S3-only consultation (no DB record) + self._seed_consultation(s3_client, bucket, "dummy-s3-only", questions_data) + + # SETUP — inputs only + self._seed_consultation(s3_client, bucket, "dummy-setup", questions_data) + + # Starting finalising themes — has clustered themes + candidate theme mappings + self._seed_consultation( + s3_client, + bucket, + "dummy-start-finalising-themes", + questions_data, + include_clustered_themes=True, + include_candidate_theme_mappings=True, + ) + + # Finished finalising themes — has clustered themes + themes.csv (ready for assignment) + self._seed_consultation( + s3_client, + bucket, + "dummy-finished-finalising-themes", + questions_data, + include_clustered_themes=True, + include_candidate_theme_mappings=True, + include_themes_csv=True, + ) + + # ANALYSIS — has everything from earlier stages + mapping outputs + self._seed_consultation( + s3_client, + bucket, + "dummy-analysis", + questions_data, + include_clustered_themes=True, + include_candidate_theme_mappings=True, + include_mapping_outputs=True, + include_themes_csv=True, + ) + + self.stdout.write("S3 seed complete.") + + def _delete_existing_data(self, s3_client, bucket): + prefix = "app_data/consultations/" + paginator = s3_client.get_paginator("list_objects_v2") + objects_to_delete = [] + + for page in paginator.paginate(Bucket=bucket, Prefix=prefix): + for obj in page.get("Contents", []): + objects_to_delete.append({"Key": obj["Key"]}) + + if objects_to_delete: + # delete_objects supports max 1000 keys per call + for i in range(0, len(objects_to_delete), 1000): + s3_client.delete_objects( + Bucket=bucket, Delete={"Objects": objects_to_delete[i : i + 1000]} + ) + self.stdout.write(f" Deleted {len(objects_to_delete)} existing objects.") + + def _seed_consultation( + self, + s3_client, + bucket, + code, + questions_data, + include_clustered_themes=False, + include_candidate_theme_mappings=False, + include_mapping_outputs=False, + include_themes_csv=False, + ): + self.stdout.write(f" Seeding S3 data for: {code}") + prefix = f"app_data/consultations/{code}" + + # Respondents + s3_client.put_object( + Bucket=bucket, + Key=f"{prefix}/inputs/respondents.jsonl", + Body=_to_jsonl(_build_respondents()), + ) + + for question_data in questions_data: + q_num = question_data["number"] + q_prefix = f"{prefix}/inputs/question_part_{q_num}" + + # Question definition + s3_client.put_object( + Bucket=bucket, + Key=f"{q_prefix}/question.json", + Body=json.dumps(_build_question_json(question_data)), + ) + + # Free text responses + if question_data["has_free_text"]: + responses = _build_responses(question_data) + if responses: + s3_client.put_object( + Bucket=bucket, + Key=f"{q_prefix}/responses.jsonl", + Body=_to_jsonl(responses), + ) + + # Multi choice + if question_data.get("multiple_choice_options"): + multi_choice = _build_multi_choice(question_data) + if multi_choice: + s3_client.put_object( + Bucket=bucket, + Key=f"{q_prefix}/multi_choice.jsonl", + Body=_to_jsonl(multi_choice), + ) + + # Stage-specific outputs (only for free text questions) + if not question_data["has_free_text"]: + continue + + if include_clustered_themes: + key = f"{prefix}/outputs/sign_off/{TIMESTAMP}/question_part_{q_num}/clustered_themes.json" + s3_client.put_object( + Bucket=bucket, + Key=key, + Body=json.dumps(_build_clustered_themes(question_data)), + ) + + if include_candidate_theme_mappings: + out_prefix = f"{prefix}/outputs/mapping/{TIMESTAMP}/question_part_{q_num}" + themes = _build_candidate_themes_json(question_data) + s3_client.put_object( + Bucket=bucket, + Key=f"{out_prefix}/themes.json", + Body=json.dumps(themes), + ) + s3_client.put_object( + Bucket=bucket, + Key=f"{out_prefix}/mapping.jsonl", + Body=_to_jsonl(_build_candidate_theme_mappings(question_data)), + ) + + if include_themes_csv: + s3_client.put_object( + Bucket=bucket, + Key=f"{q_prefix}/themes.csv", + Body=_build_themes_csv(question_data), + ) + + if include_mapping_outputs: + out_prefix = f"{prefix}/outputs/mapping/{TIMESTAMP}/question_part_{q_num}" + themes = _build_themes_json(question_data) + s3_client.put_object( + Bucket=bucket, + Key=f"{out_prefix}/themes.json", + Body=json.dumps(themes), + ) + s3_client.put_object( + Bucket=bucket, + Key=f"{out_prefix}/mapping.jsonl", + Body=_to_jsonl(_build_theme_mappings(question_data)), + ) + s3_client.put_object( + Bucket=bucket, + Key=f"{out_prefix}/sentiment.jsonl", + Body=_to_jsonl(_build_sentiments()), + ) + s3_client.put_object( + Bucket=bucket, + Key=f"{out_prefix}/detail_detection.jsonl", + Body=_to_jsonl(_build_evidence_rich()), + ) diff --git a/backend/start.sh b/backend/start.sh index bb8a28e13..660fad34b 100755 --- a/backend/start.sh +++ b/backend/start.sh @@ -3,8 +3,7 @@ # Collect static files venv/bin/django-admin collectstatic --noinput -venv/bin/django-admin migrate -venv/bin/django-admin createadminusers +venv/bin/django-admin prepare_environment venv/bin/django-admin populate_history --auto --batchsize 1000 exec venv/bin/gunicorn -c ./gunicorn_config.py backend.wsgi diff --git a/backend/tests/commands/test_dummy_data.py b/backend/tests/commands/test_dummy_data.py index cbc7dcc0d..e82a4ed6a 100644 --- a/backend/tests/commands/test_dummy_data.py +++ b/backend/tests/commands/test_dummy_data.py @@ -16,8 +16,8 @@ def test_name_parameter_sets_consultation_name(mock_is_local): stdout=StringIO(), # we'll ignore this ) - assert models.Consultation.objects.count() == 2 - assert models.Question.objects.count() == 8 + assert models.Consultation.objects.count() == 4 + assert models.Question.objects.count() == 16 @pytest.mark.django_db diff --git a/backend/tests/commands/test_prepare_environment.py b/backend/tests/commands/test_prepare_environment.py new file mode 100644 index 000000000..b04500eff --- /dev/null +++ b/backend/tests/commands/test_prepare_environment.py @@ -0,0 +1,72 @@ +from unittest.mock import patch + +import boto3 +import pytest +from django.core.management import call_command +from moto import mock_aws + +from authentication.models import User +from consultations.models import Consultation + + +class TestPrepareEnvironment: + @pytest.mark.django_db + def test_does_not_reset_on_prod(self, settings): + settings.ENVIRONMENT = "prod" + + Consultation.objects.create(title="Should survive", code="KEEP_ME") + call_command("prepare_environment") + + assert Consultation.objects.filter(code="KEEP_ME").exists() + + @pytest.mark.django_db + @pytest.mark.parametrize("environment", ["dev", "preprod"]) + def test_resets_and_seeds_db(self, settings, environment): + settings.ENVIRONMENT = environment + + Consultation.objects.create(title="Should be deleted", code="DELETE_ME") + call_command("prepare_environment") + + # Old data is gone + assert not Consultation.objects.filter(code="DELETE_ME").exists() + + # Consultations created at each stage + assert Consultation.objects.filter(stage=Consultation.Stage.SETUP).exists() + assert Consultation.objects.filter(stage=Consultation.Stage.THEME_SIGN_OFF).count() == 2 + assert Consultation.objects.filter(stage=Consultation.Stage.ANALYSIS).exists() + + # Admin user was created + assert User.objects.filter(email="email@example.com", is_staff=True).exists() + + @pytest.mark.django_db + @pytest.mark.parametrize("environment", ["dev", "preprod"]) + @mock_aws + @patch("factories.embed_text", return_value=[0.0] * 3072) + @patch( + "consultations.management.commands.prepare_s3.HostingEnvironment.is_production", + return_value=False, + ) + @patch( + "consultations.management.commands.prepare_s3.HostingEnvironment.is_deployed", + return_value=True, + ) + def test_seeds_s3(self, _mock_deployed, _mock_prod, _mock_embed, settings, environment): + settings.ENVIRONMENT = environment + settings.AWS_BUCKET_NAME = "test-bucket" + + conn = boto3.resource("s3", region_name="eu-west-2") + conn.create_bucket( + Bucket="test-bucket", + CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, + ) + + call_command("prepare_environment") + + bucket = conn.Bucket("test-bucket") + keys = [obj.key for obj in bucket.objects.all()] + + assert any("dummy-s3-only/" in k for k in keys) + assert any("dummy-setup/" in k for k in keys) + assert any("dummy-start-finalising-themes/" in k for k in keys) + assert any("dummy-finished-finalising-themes/" in k for k in keys) + assert any("dummy-analysis/" in k for k in keys) diff --git a/backend/tests/commands/test_prepare_s3.py b/backend/tests/commands/test_prepare_s3.py new file mode 100644 index 000000000..ee3f48309 --- /dev/null +++ b/backend/tests/commands/test_prepare_s3.py @@ -0,0 +1,143 @@ +from unittest.mock import patch + +import boto3 +import pytest +from django.core.management import call_command +from moto import mock_aws + + +class TestPrepareS3: + @pytest.mark.django_db + @mock_aws + @patch( + "consultations.management.commands.prepare_s3.HostingEnvironment.is_production", + return_value=False, + ) + @patch( + "consultations.management.commands.prepare_s3.HostingEnvironment.is_deployed", + return_value=True, + ) + def test_deletes_existing_data_before_seeding(self, _mock_deployed, _mock_prod, settings): + settings.AWS_BUCKET_NAME = "test-bucket" + + conn = boto3.resource("s3", region_name="eu-west-2") + conn.create_bucket( + Bucket="test-bucket", + CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, + ) + + # Pre-seed a stale object that should be cleaned up + s3_client = boto3.client("s3", region_name="eu-west-2") + s3_client.put_object( + Bucket="test-bucket", + Key="app_data/consultations/old-consultation/inputs/respondents.jsonl", + Body="stale data", + ) + + call_command("prepare_s3") + + bucket = conn.Bucket("test-bucket") + keys = [obj.key for obj in bucket.objects.all()] + + assert not any("old-consultation" in k for k in keys) + assert any("dummy-s3-only/" in k for k in keys) + + @pytest.mark.django_db + @mock_aws + @patch( + "consultations.management.commands.prepare_s3.HostingEnvironment.is_production", + return_value=False, + ) + @patch( + "consultations.management.commands.prepare_s3.HostingEnvironment.is_deployed", + return_value=True, + ) + def test_seeds_s3_with_data_for_each_stage(self, _mock_deployed, _mock_prod, settings): + settings.AWS_BUCKET_NAME = "test-bucket" + + conn = boto3.resource("s3", region_name="eu-west-2") + conn.create_bucket( + Bucket="test-bucket", + CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, + ) + + call_command("prepare_s3") + + bucket = conn.Bucket("test-bucket") + keys = [obj.key for obj in bucket.objects.all()] + + # S3-only consultation has input data + assert any("dummy-s3-only/inputs/respondents.jsonl" in k for k in keys) + assert any("dummy-s3-only/inputs/question_part_1/question.json" in k for k in keys) + + # Setup consultation has input data + assert any("dummy-setup/inputs/respondents.jsonl" in k for k in keys) + assert any("dummy-setup/inputs/question_part_1/question.json" in k for k in keys) + + # Starting finalising themes has input data, clustered themes and candidate theme mappings + assert any("dummy-start-finalising-themes/inputs/respondents.jsonl" in k for k in keys) + assert any( + "dummy-start-finalising-themes/inputs/question_part_1/question.json" in k for k in keys + ) + assert any( + "dummy-start-finalising-themes/outputs/sign_off/" in k and "clustered_themes.json" in k + for k in keys + ) + assert any( + "dummy-start-finalising-themes/outputs/mapping/" in k and "mapping.jsonl" in k + for k in keys + ) + + # Finished finalising themes has input data, clustered themes and candidate theme mappings + assert any("dummy-finished-finalising-themes/inputs/respondents.jsonl" in k for k in keys) + assert any( + "dummy-finished-finalising-themes/inputs/question_part_1/question.json" in k + for k in keys + ) + assert any( + "dummy-finished-finalising-themes/outputs/sign_off/" in k + and "clustered_themes.json" in k + for k in keys + ) + assert any( + "dummy-finished-finalising-themes/outputs/mapping/" in k and "mapping.jsonl" in k + for k in keys + ) + + # Analysis has all the above and mapping outputs + assert any("dummy-analysis/inputs/respondents.jsonl" in k for k in keys) + assert any("dummy-analysis/inputs/question_part_1/question.json" in k for k in keys) + assert any( + "dummy-analysis/outputs/sign_off/" in k and "clustered_themes.json" in k for k in keys + ) + assert any("dummy-analysis/outputs/mapping/" in k and "mapping.jsonl" in k for k in keys) + assert any("dummy-analysis/outputs/mapping/" in k and "themes.json" in k for k in keys) + assert any("dummy-analysis/outputs/mapping/" in k and "sentiment.jsonl" in k for k in keys) + assert any( + "dummy-analysis/outputs/mapping/" in k and "detail_detection.jsonl" in k for k in keys + ) + + @pytest.mark.django_db + @mock_aws + @patch( + "consultations.management.commands.prepare_s3.HostingEnvironment.is_production", + return_value=False, + ) + @patch( + "consultations.management.commands.prepare_s3.HostingEnvironment.is_deployed", + return_value=False, + ) + def test_skips_on_local(self, _mock_deployed, _mock_prod, settings): + settings.AWS_BUCKET_NAME = "test-bucket" + + conn = boto3.resource("s3", region_name="eu-west-2") + conn.create_bucket( + Bucket="test-bucket", + CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, + ) + + call_command("prepare_s3") + + bucket = conn.Bucket("test-bucket") + keys = [obj.key for obj in bucket.objects.all()] + assert len(keys) == 0 diff --git a/backend/tests/examples/sample_questions.json b/backend/tests/examples/sample_questions.json new file mode 100644 index 000000000..83f02fcca --- /dev/null +++ b/backend/tests/examples/sample_questions.json @@ -0,0 +1,74 @@ +[ + { + "question_text": "Do you agree with the proposal to align the flavour categories of chocolate bars as outlined in the draft guidelines of the Chocolate Bar Regulation for the United Kingdom?", + "number": 1, + "has_free_text": true, + "has_multiple_choice": true, + "multiple_choice_options": ["Yes", "No", "Don't know", "No answer"], + "free_text_answers": [ + "", + "Yes, I agree with the proposal as it will create a standardized framework that benefits both consumers and manufacturers.", + "No, I do not agree because I feel the proposed categories are too restrictive and may stifle innovation in flavour development.", + "I agree with the proposal, but I think there should be room for additional categories to accommodate future trends.", + "I disagree with the proposal as it does not sufficiently account for regional flavour preferences across the UK.", + "Yes, aligning the flavour categories will help consumers easily compare products and make informed choices.", + "I neither agree nor disagree at this moment; I would like more information on how the categories were determined.", + "I support the proposal because it will likely improve transparency and consistency in labelling for consumers.", + "No, I believe the existing categories are sufficient and that changes could confuse consumers.", + "I agree in principle, but I think the guidelines should include a provision for periodic review to adapt to market changes." + ], + "candidate_themes": [ + {"name": "Standardized framework", "description": "A standardized framework that benefits both consumers and manufacturers.", "key": "A", "selected": true, "approximate_frequency": 60, "parent_key": null}, + {"name": "Innovation", "description": "Innovation in flavour development.", "key": "B", "selected": true, "approximate_frequency": 30, "parent_key": null} + ] + }, + { + "question_text": "Do you agree with the proposal to align the flavour categories of chocolate bars as outlined in the draft guidelines of the Chocolate Bar Regulation for the United Kingdom?", + "number": 2, + "has_free_text": false, + "has_multiple_choice": true, + "multiple_choice_options": ["Yes", "No", "Don't know", "No answer"], + "free_text_answers": [], + "candidate_themes": [] + }, + { + "question_text": "Which of the following factors do you believe are important when considering the packaging of chocolate bars? Please select all that apply: a) Sustainability, b) Design, c) Cost-effectiveness, d) Durability, e) Brand recognition.", + "number": 3, + "has_free_text": false, + "has_multiple_choice": true, + "multiple_choice_options": ["Sustainability", "Design", "Cost-effectiveness", "Durability", "Brand recognition"], + "free_text_answers": [], + "candidate_themes": [] + }, + { + "question_text": "What are your thoughts on how the current chocolate bar regulations could be improved to better address consumer needs and industry standards?", + "number": 4, + "has_free_text": true, + "has_multiple_choice": false, + "multiple_choice_options": [], + "free_text_answers": [ + "", + "I believe the current regulations should include clearer guidelines on the sourcing of ingredients to ensure ethical practices and sustainability.", + "The regulations could be improved by setting specific limits on sugar content to promote healthier options for consumers.", + "Consideration should be given to standardizing portion sizes to help consumers make informed choices and manage their calorie intake.", + "It would be beneficial to include mandatory allergen warnings on all packaging to enhance consumer safety.", + "The regulations should encourage more innovation in packaging materials to reduce environmental impact.", + "Introducing incentives for using locally sourced ingredients could support local economies and reduce carbon footprints.", + "The current regulations could be improved by including a requirement for transparent labeling about nutritional information.", + "I think the regulations should mandate clearer expiration dates to prevent food waste and ensure product freshness.", + "The industry would benefit from guidelines that promote fair trade practices, giving consumers more ethical choices." + ], + "candidate_themes": [ + {"name": "More innovative", "description": "Innovative ideas to improve chocolate bar regulations.", "key": "A", "selected": true, "approximate_frequency": 40, "parent_key": null}, + {"name": "Innovative packaging", "description": "Ideas for innovative packaging solutions.", "key": "B", "selected": true, "approximate_frequency": 20, "parent_key": "A"}, + {"name": "New flavor combinations", "description": "Exploring new flavor combinations.", "key": "C", "selected": false, "approximate_frequency": 20, "parent_key": "A"}, + {"name": "Exotic flavors", "description": "Incorporating exotic flavors.", "key": "D", "selected": true, "approximate_frequency": 10, "parent_key": "C"}, + {"name": "Fusion flavors", "description": "Creating fusion flavors.", "key": "E", "selected": true, "approximate_frequency": 10, "parent_key": "C"}, + {"name": "Healthier options", "description": "It should encourage healthier options.", "key": "F", "selected": true, "approximate_frequency": 20, "parent_key": null}, + {"name": "Clearer guidelines", "description": "It should include clearer guidelines.", "key": "G", "selected": false, "approximate_frequency": 10, "parent_key": null}, + {"name": "Fair trade practices", "description": "It should promote fair trade practices.", "key": "H", "selected": false, "approximate_frequency": 10, "parent_key": null}, + {"name": "Sustainability", "description": "It should focus on sustainability.", "key": "I", "selected": false, "approximate_frequency": 10, "parent_key": null}, + {"name": "Transparent labeling", "description": "It should have transparent labeling.", "key": "J", "selected": false, "approximate_frequency": 10, "parent_key": null} + ] + } +] diff --git a/backend/tests/examples/sample_questions.yml b/backend/tests/examples/sample_questions.yml deleted file mode 100644 index 109962078..000000000 --- a/backend/tests/examples/sample_questions.yml +++ /dev/null @@ -1,117 +0,0 @@ -- question_text: "Do you agree with the proposal to align the flavour categories of chocolate bars as outlined in the draft guidelines of the Chocolate Bar Regulation for the United Kingdom?" - number: 1 - has_free_text: True - has_multiple_choice: True - multiple_choice_options: - - "Yes" - - "No" - - "Don't know" - - "No answer" - free_text_answers: - - "" - - "Yes, I agree with the proposal as it will create a standardized framework that benefits both consumers and manufacturers." - - "No, I do not agree because I feel the proposed categories are too restrictive and may stifle innovation in flavour development." - - "I agree with the proposal, but I think there should be room for additional categories to accommodate future trends." - - "I disagree with the proposal as it does not sufficiently account for regional flavour preferences across the UK." - - "Yes, aligning the flavour categories will help consumers easily compare products and make informed choices." - - "I neither agree nor disagree at this moment; I would like more information on how the categories were determined." - - "I support the proposal because it will likely improve transparency and consistency in labelling for consumers." - - "No, I believe the existing categories are sufficient and that changes could confuse consumers." - - "I agree in principle, but I think the guidelines should include a provision for periodic review to adapt to market changes." - candidate_themes: - - name: "Standardized framework" - description: "A standardized framework that benefits both consumers and manufacturers." - key: "A" - selected: true - approximate_frequency_pct: 0.6 - - name: "Innovation" - description: "Innovation in flavour development." - key: "B" - selected: true - approximate_frequency_pct: 0.3 - -- question_text: "Do you agree with the proposal to align the flavour categories of chocolate bars as outlined in the draft guidelines of the Chocolate Bar Regulation for the United Kingdom?" - number: 2 - has_free_text: False - has_multiple_choice: True - multiple_choice_options: - - "Yes" - - "No" - - "Don't know" - - "No answer" - -- question_text: "Which of the following factors do you believe are important when considering the packaging of chocolate bars? Please select all that apply: a) Sustainability, b) Design, c) Cost-effectiveness, d) Durability, e) Brand recognition." - number: 3 - has_free_text: False - has_multiple_choice: True - multiple_choice_options: - - "Sustainability" - - "Design" - - "Cost-effectiveness" - - "Durability" - - "Brand recognition" - -- question_text: "What are your thoughts on how the current chocolate bar regulations could be improved to better address consumer needs and industry standards?" - number: 4 - has_free_text: True - has_multiple_choice: False - free_text_answers: - - "" - - "I believe the current regulations should include clearer guidelines on the sourcing of ingredients to ensure ethical practices and sustainability." - - "The regulations could be improved by setting specific limits on sugar content to promote healthier options for consumers." - - "Consideration should be given to standardizing portion sizes to help consumers make informed choices and manage their calorie intake." - - "It would be beneficial to include mandatory allergen warnings on all packaging to enhance consumer safety." - - "The regulations should encourage more innovation in packaging materials to reduce environmental impact." - - "Introducing incentives for using locally sourced ingredients could support local economies and reduce carbon footprints." - - "The current regulations could be improved by including a requirement for transparent labeling about nutritional information." - - "I think the regulations should mandate clearer expiration dates to prevent food waste and ensure product freshness." - - "The industry would benefit from guidelines that promote fair trade practices, giving consumers more ethical choices." - candidate_themes: - - name: "More innovative" - description: "Innovative ideas to improve chocolate bar regulations." - key: "A" - selected: true - approximate_frequency_pct: 0.4 - children: - - name: "Innovative packaging" - description: "Ideas for innovative packaging solutions." - key: "B" - selected: true - approximate_frequency_pct: 0.2 - - name: "New flavor combinations" - description: "Exploring new flavor combinations." - key: "C" - approximate_frequency_pct: 0.2 - children: - - name: "Exotic flavors" - description: "Incorporating exotic flavors." - key: "D" - selected: true - approximate_frequency_pct: 0.1 - - name: "Fusion flavors" - description: "Creating fusion flavors." - key: "E" - selected: true - approximate_frequency_pct: 0.1 - - name: "Healthier options" - description: "It should encourage healthier options." - key: "F" - selected: true - approximate_frequency_pct: 0.2 - - name: "Clearer guidelines" - description: "It should include clearer guidelines." - key: "G" - approximate_frequency_pct: 0.1 - - name: "Fair trade practices" - description: "It should promote fair trade practices." - key: "H" - approximate_frequency_pct: 0.1 - - name: "Sustainability" - description: "It should focus on sustainability." - key: "I" - approximate_frequency_pct: 0.1 - - name: "Transparent labeling" - description: "It should have transparent labeling." - key: "J" - approximate_frequency_pct: 0.1 - key: "J" diff --git a/backend/tests/unit/test_generate_dummy_data.py b/backend/tests/unit/test_generate_dummy_data.py index b096071ee..c6a992666 100644 --- a/backend/tests/unit/test_generate_dummy_data.py +++ b/backend/tests/unit/test_generate_dummy_data.py @@ -4,14 +4,14 @@ import pytest from consultations import models -from consultations.dummy_data import create_dummy_consultation_from_yaml +from consultations.dummy_data import DUMMY_CONSULTATIONS, create_dummy_consultation @pytest.mark.django_db @patch("hosting_environment.HostingEnvironment.is_local", return_value=True) def test_a_consultation_is_generated(settings): assert models.Consultation.objects.count() == 0 - create_dummy_consultation_from_yaml() + create_dummy_consultation() assert models.Consultation.objects.count() == 1 assert models.Question.objects.count() == 4 @@ -19,27 +19,99 @@ def test_a_consultation_is_generated(settings): @pytest.mark.django_db @pytest.mark.parametrize("environment", ["prod"]) def test_the_tool_will_only_run_in_dev(environment): - with patch.dict(os.environ, {"ENVIRONMENT": environment}), pytest.raises( - Exception, match=r"Dummy data generation should not be run in production" - ): - create_dummy_consultation_from_yaml() + with patch.dict(os.environ, {"ENVIRONMENT": environment}): + with pytest.raises( + Exception, match=r"Dummy data generation should not be run in production" + ): + create_dummy_consultation() @pytest.mark.django_db -def test_create_dummy_consultation_from_yaml(): - consultation = create_dummy_consultation_from_yaml(number_respondents=10) +def test_setup_stage_has_consultation_data_but_no_themes(): + config = DUMMY_CONSULTATIONS[0] + consultation = create_dummy_consultation(number_respondents=10, config=config) questions = models.Question.objects.filter(consultation=consultation) + + assert consultation.stage == models.Consultation.Stage.SETUP assert questions.count() == 4 + assert models.Response.objects.filter(question__consultation=consultation).count() > 0 + assert models.CandidateTheme.objects.filter(question__consultation=consultation).count() == 0 + assert models.SelectedTheme.objects.filter(question__consultation=consultation).count() == 0 + # Multiple choice questions have answers q1 = questions.get(number=1) - assert q1.has_free_text assert q1.has_multiple_choice + assert models.MultiChoiceAnswer.objects.filter(question=q1).count() == 4 + + # Respondents have demographic data + respondent = models.Respondent.objects.filter(consultation=consultation).first() + assert respondent.demographics.count() > 0 + field_names = set(respondent.demographics.values_list("field_name", flat=True)) + assert "region" in field_names + assert "age_group" in field_names + assert "respondent_type" in field_names + + +@pytest.mark.django_db +def test_theme_sign_off_draft_has_candidate_themes_and_responses(): + config = DUMMY_CONSULTATIONS[1] + consultation = create_dummy_consultation(number_respondents=10, config=config) + q4 = models.Question.objects.get(consultation=consultation, number=4) + + assert consultation.stage == models.Consultation.Stage.THEME_SIGN_OFF + assert q4.theme_status == models.Question.ThemeStatus.DRAFT + + # Has candidate themes at multiple levels + all_themes = models.CandidateTheme.objects.filter(question=q4) + top_level = all_themes.filter(parent=None) + children = all_themes.exclude(parent=None) + assert top_level.count() > 0 + assert children.count() > 0 + + # Has CandidateThemeResponses for child themes too + child_theme = children.first() + assert models.CandidateThemeResponse.objects.filter(candidate_theme=child_theme).exists() + + # No SelectedThemes (still draft) + assert models.SelectedTheme.objects.filter(question=q4).count() == 0 + + +@pytest.mark.django_db +def test_theme_sign_off_confirmed_has_selected_themes(): + config = DUMMY_CONSULTATIONS[2] + consultation = create_dummy_consultation(number_respondents=10, config=config) + q4 = models.Question.objects.get(consultation=consultation, number=4) + + assert consultation.stage == models.Consultation.Stage.THEME_SIGN_OFF + assert q4.theme_status == models.Question.ThemeStatus.CONFIRMED + + # Has SelectedThemes + selected_themes = models.SelectedTheme.objects.filter(question=q4) + assert selected_themes.filter(key="A").exists() + assert not selected_themes.filter(name="Other").exists() + assert not selected_themes.filter(name="No Reason Given").exists() + + +@pytest.mark.django_db +def test_analysis_has_response_annotations(): + config = DUMMY_CONSULTATIONS[3] + consultation = create_dummy_consultation(number_respondents=10, config=config) + q1 = models.Question.objects.get(consultation=consultation, number=1) + + assert consultation.stage == models.Consultation.Stage.ANALYSIS + assert q1.theme_status == models.Question.ThemeStatus.CONFIRMED + + # Has SelectedThemes including defaults + selected_themes = models.SelectedTheme.objects.filter(question=q1) + assert selected_themes.count() == 4 + assert selected_themes.get(key="A").name == "Standardized framework" + assert selected_themes.filter(name="Other").exists() + assert selected_themes.filter(name="No Reason Given").exists() - q1_themes = models.SelectedTheme.objects.filter(question=q1) - assert len(q1_themes) == 2 - assert q1_themes.get(key="A").name == "Standardized framework" + # Has ResponseAnnotations for responses + annotations = models.ResponseAnnotation.objects.filter(response__question=q1) + assert annotations.count() > 0 - q3 = questions.get(number=3) - assert not q3.has_free_text - assert q3.has_multiple_choice - assert q3.multiple_choice_options + # Each annotation has themes assigned + for annotation in annotations: + assert annotation.themes.count() > 0 From 5abef94a71297dc6e33df0596a8758c35776e868 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Wed, 19 Aug 2026 13:10:42 +0100 Subject: [PATCH 02/36] Fix changes in rebase and ruff formatting --- backend/consultations/admin.py | 4 +- backend/consultations/api/serializers.py | 13 +- .../consultations/api/views/consultation.py | 4 +- backend/consultations/api/views/question.py | 4 +- backend/consultations/api/views/response.py | 6 +- backend/consultations/dummy_data.py | 10 +- .../management/commands/prepare_s3.py | 7 +- .../0099_alter_consultation_stage_default.py | 21 +++- .../0100_remove_legacy_stage_choices.py | 1 - ...2_responsereadby_alter_response_read_by.py | 1 - .../0103_add_response_question_id_index.py | 7 +- .../test_support/load_test_fixtures.py | 21 ++-- backend/consultations/utils/s3.py | 1 + backend/data_pipeline/s3.py | 12 +- .../data_pipeline/sync/candidate_themes.py | 4 +- .../data_pipeline/sync/consultation_setup.py | 12 +- .../sync/response_annotations.py | 27 ++--- backend/factories.py | 4 +- backend/rq_context.py | 4 +- backend/settings/local.py | 4 +- backend/tests/commands/test_dummy_data.py | 5 +- .../sync/test_consultation_setup.py | 22 ++-- .../tests/unit/data_pipeline/test_batch.py | 30 ++++- backend/tests/unit/data_pipeline/test_s3.py | 27 +---- .../tests/unit/test_generate_dummy_data.py | 10 +- backend/tests/unit/test_healthcheck_worker.py | 7 +- backend/tests/unit/test_middleware.py | 8 +- backend/tests/unit/test_rq_context.py | 4 +- uv.lock | 112 +++++++++--------- 29 files changed, 193 insertions(+), 199 deletions(-) diff --git a/backend/consultations/admin.py b/backend/consultations/admin.py index 98e08f1b1..95aa695fd 100644 --- a/backend/consultations/admin.py +++ b/backend/consultations/admin.py @@ -55,9 +55,7 @@ def create_dummy_consultation(modeladmin, request, queryset, size=10): ) return - create_dummy_consultation_job.delay( - number_respondents=size, consultation=consultation - ) + create_dummy_consultation_job.delay(number_respondents=size, consultation=consultation) @admin.action(description="create small dummy consultation") diff --git a/backend/consultations/api/serializers.py b/backend/consultations/api/serializers.py index 963658478..e5b62fed7 100644 --- a/backend/consultations/api/serializers.py +++ b/backend/consultations/api/serializers.py @@ -38,9 +38,7 @@ def validate_is_staff(self, value): # Check if this is an update operation and user is updating themselves if self.instance and request and request.user == self.instance and value is False: - raise serializers.ValidationError( - "You cannot remove admin privileges from yourself" - ) + raise serializers.ValidationError("You cannot remove admin privileges from yourself") return value @@ -150,7 +148,14 @@ class SelectedThemeSerializer(serializers.ModelSerializer): class Meta: model = SelectedTheme - fields: ClassVar[list] = ["id", "name", "description", "version", "modified_at", "last_modified_by"] + fields: ClassVar[list] = [ + "id", + "name", + "description", + "version", + "modified_at", + "last_modified_by", + ] read_only_fields: ClassVar[list] = ["id", "version", "modified_at", "last_modified_by"] def get_last_modified_by(self, obj): diff --git a/backend/consultations/api/views/consultation.py b/backend/consultations/api/views/consultation.py index 66e05f2f2..a311a8757 100644 --- a/backend/consultations/api/views/consultation.py +++ b/backend/consultations/api/views/consultation.py @@ -935,9 +935,7 @@ def evaluation(self, request, pk=None): SelectedTheme.objects.filter( question__consultation=consultation, ) - .filter( - Q(name__iexact=NO_REASON_GIVEN_THEME_NAME) | Q(name__iexact=OTHER_THEME_NAME) - ) + .filter(Q(name__iexact=NO_REASON_GIVEN_THEME_NAME) | Q(name__iexact=OTHER_THEME_NAME)) .values_list("id", flat=True) ) diff --git a/backend/consultations/api/views/question.py b/backend/consultations/api/views/question.py index e314a477f..d1dffa57f 100644 --- a/backend/consultations/api/views/question.py +++ b/backend/consultations/api/views/question.py @@ -142,9 +142,7 @@ def themes(self, request, pk=None, consultation_pk=None): ) ) else: - themes = themes.annotate( - count=Count("responseannotation", distinct=True) - ) + themes = themes.annotate(count=Count("responseannotation", distinct=True)) serializer = QuestionThemeSerializer(themes, many=True) return Response({"themes": serializer.data}) diff --git a/backend/consultations/api/views/response.py b/backend/consultations/api/views/response.py index 484f0ed31..c4a037ba0 100644 --- a/backend/consultations/api/views/response.py +++ b/backend/consultations/api/views/response.py @@ -253,11 +253,7 @@ def mark_read_bulk(self, request, consultation_pk=None, **kwargs): if len(requested_response_ids) > MAX_BULK_MARK_READ: return Response( - { - "message": ( - f"Too many response IDs provided. Maximum is {MAX_BULK_MARK_READ}." - ) - }, + {"message": (f"Too many response IDs provided. Maximum is {MAX_BULK_MARK_READ}.")}, status=status.HTTP_400_BAD_REQUEST, ) diff --git a/backend/consultations/dummy_data.py b/backend/consultations/dummy_data.py index 224b568b7..10f2b4861 100644 --- a/backend/consultations/dummy_data.py +++ b/backend/consultations/dummy_data.py @@ -1,6 +1,5 @@ import json import random -from typing import Optional from django.conf import settings @@ -24,6 +23,7 @@ SelectedThemeFactory, ) from hosting_environment import HostingEnvironment +from rq_context import job logger = settings.LOGGER @@ -235,8 +235,8 @@ def create_candidate_theme_responses(question): def create_dummy_consultation( file_path: str = SAMPLE_QUESTIONS_PATH, number_respondents: int = 10, - consultation: Optional[Consultation] = None, - config: Optional[dict] = None, + consultation: Consultation | None = None, + config: dict | None = None, ) -> Consultation: """ Create consultation with questions, responses and themes from JSON file. @@ -320,8 +320,8 @@ def create_dummy_consultation( def create_dummy_consultation_job( file_path: str = SAMPLE_QUESTIONS_PATH, number_respondents: int = 10, - consultation: Optional[Consultation] = None, - config: Optional[dict] = None, + consultation: Consultation | None = None, + config: dict | None = None, ): create_dummy_consultation( file_path=file_path, diff --git a/backend/consultations/management/commands/prepare_s3.py b/backend/consultations/management/commands/prepare_s3.py index ec5638d26..362ff08fa 100644 --- a/backend/consultations/management/commands/prepare_s3.py +++ b/backend/consultations/management/commands/prepare_s3.py @@ -1,5 +1,6 @@ +import datetime import json -from datetime import date +from zoneinfo import ZoneInfo import boto3 from django.conf import settings @@ -15,7 +16,7 @@ ) from hosting_environment import HostingEnvironment -TIMESTAMP = date.today().isoformat() +TIMESTAMP = datetime.datetime.now(tz=ZoneInfo("Europe/London")).date() def _to_jsonl(records): @@ -146,8 +147,6 @@ def _build_themes_json(question_data): return themes - - def _build_candidate_theme_mappings(question_data): """Build mapping.jsonl for candidate themes using deterministic assignment per sibling group.""" candidate_themes = question_data.get("candidate_themes", []) diff --git a/backend/consultations/migrations/0099_alter_consultation_stage_default.py b/backend/consultations/migrations/0099_alter_consultation_stage_default.py index d1a946354..380c5a05e 100644 --- a/backend/consultations/migrations/0099_alter_consultation_stage_default.py +++ b/backend/consultations/migrations/0099_alter_consultation_stage_default.py @@ -6,15 +6,26 @@ class Migration(migrations.Migration): - dependencies: ClassVar[list] = [ - ('consultations', '0098_migrate_legacy_stages'), + ("consultations", "0098_migrate_legacy_stages"), ] operations: ClassVar[list] = [ migrations.AlterField( - model_name='consultation', - name='stage', - field=models.CharField(choices=[('setup', 'Data Setup'), ('finding_themes', 'Finding Themes'), ('finalising_themes', 'Finalising Themes'), ('assigning_themes', 'Assigning Themes'), ('analysis', 'Analysis'), ('theme_sign_off', 'Theme Sign Off'), ('theme_mapping', 'Theme Mapping')], default='finalising_themes', max_length=32), + model_name="consultation", + name="stage", + field=models.CharField( + choices=[ + ("setup", "Data Setup"), + ("finding_themes", "Finding Themes"), + ("finalising_themes", "Finalising Themes"), + ("assigning_themes", "Assigning Themes"), + ("analysis", "Analysis"), + ("theme_sign_off", "Theme Sign Off"), + ("theme_mapping", "Theme Mapping"), + ], + default="finalising_themes", + max_length=32, + ), ), ] diff --git a/backend/consultations/migrations/0100_remove_legacy_stage_choices.py b/backend/consultations/migrations/0100_remove_legacy_stage_choices.py index fee9cd96b..1e205a9d1 100644 --- a/backend/consultations/migrations/0100_remove_legacy_stage_choices.py +++ b/backend/consultations/migrations/0100_remove_legacy_stage_choices.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies: ClassVar[list] = [ ("consultations", "0099_alter_consultation_stage_default"), ] diff --git a/backend/consultations/migrations/0102_responsereadby_alter_response_read_by.py b/backend/consultations/migrations/0102_responsereadby_alter_response_read_by.py index c69d03a63..7c0dfca2b 100644 --- a/backend/consultations/migrations/0102_responsereadby_alter_response_read_by.py +++ b/backend/consultations/migrations/0102_responsereadby_alter_response_read_by.py @@ -6,7 +6,6 @@ class Migration(migrations.Migration): - dependencies: ClassVar[list] = [ ("consultations", "0101_clean_empty_responses_and_denormalise_counts"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), diff --git a/backend/consultations/migrations/0103_add_response_question_id_index.py b/backend/consultations/migrations/0103_add_response_question_id_index.py index c547f465e..e1def4028 100644 --- a/backend/consultations/migrations/0103_add_response_question_id_index.py +++ b/backend/consultations/migrations/0103_add_response_question_id_index.py @@ -7,15 +7,14 @@ class Migration(migrations.Migration): - dependencies: ClassVar[list] = [ - ('consultations', '0102_responsereadby_alter_response_read_by'), + ("consultations", "0102_responsereadby_alter_response_read_by"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations: ClassVar[list] = [ migrations.AddIndex( - model_name='response', - index=models.Index(fields=['question', 'id'], name='consultatio_questio_13c9db_idx'), + model_name="response", + index=models.Index(fields=["question", "id"], name="consultatio_questio_13c9db_idx"), ), ] diff --git a/backend/consultations/test_support/load_test_fixtures.py b/backend/consultations/test_support/load_test_fixtures.py index 95c15981e..7ed9c9c3c 100644 --- a/backend/consultations/test_support/load_test_fixtures.py +++ b/backend/consultations/test_support/load_test_fixtures.py @@ -34,8 +34,7 @@ def create_response_from_fixtures(respondents, index, question_object, response_ if "themes" in response_data: annotation = ResponseAnnotation.objects.create( - response=response, - evidence_rich=response_data.get("evidence_rich", False) + response=response, evidence_rich=response_data.get("evidence_rich", False) ) annotation.add_original_ai_themes( SelectedTheme.objects.filter(question=question_object, key__in=response_data["themes"]) @@ -44,10 +43,10 @@ def create_response_from_fixtures(respondents, index, question_object, response_ if "demographics" in response_data: for key, value in response_data["demographics"].items(): option, _ = DemographicOption.objects.get_or_create( - consultation=question_object.consultation, - field_name=key, - field_value=value, - ) + consultation=question_object.consultation, + field_name=key, + field_value=value, + ) respondents[index].demographics.add(option) @@ -96,9 +95,7 @@ def create_question_from_fixtures(consultation_object, respondents, question_dat CandidateTheme.objects.bulk_create( [ CandidateTheme( - question=question_object, - name=t["name"], - description=t["description"] + question=question_object, name=t["name"], description=t["description"] ) for t in question_data["candidate_themes"] ] @@ -110,7 +107,7 @@ def create_question_from_fixtures(consultation_object, respondents, question_dat question_object.update_response_counts() MultiChoiceAnswer.update_response_counts(question_object) - + return question_object @@ -157,7 +154,9 @@ def create_data_from_fixtures(fixtures): respondents = create_respondents_from_fixtures(consultation_data, consultation_object) for question_data in consultation_data.get("questions", []): - question_object = create_question_from_fixtures(consultation_object, respondents, question_data) + question_object = create_question_from_fixtures( + consultation_object, respondents, question_data + ) questions.append(question_object.id) # Make sure that demographic response counts are updated after all responses have been created diff --git a/backend/consultations/utils/s3.py b/backend/consultations/utils/s3.py index c05e3d19c..3d3358dfc 100644 --- a/backend/consultations/utils/s3.py +++ b/backend/consultations/utils/s3.py @@ -4,6 +4,7 @@ logger = settings.LOGGER + def get_s3_client(config: Config | None = None): config = config or Config() if settings.ENVIRONMENT.upper() in ["LOCAL", "TEST"]: diff --git a/backend/data_pipeline/s3.py b/backend/data_pipeline/s3.py index da0a6df4d..1053068e3 100644 --- a/backend/data_pipeline/s3.py +++ b/backend/data_pipeline/s3.py @@ -9,9 +9,7 @@ account_id = settings.AWS_ACCOUNT_ID -def read_jsonl( - bucket_name: str, key: str, raise_if_missing: bool = True -) -> list[dict]: +def read_jsonl(bucket_name: str, key: str, raise_if_missing: bool = True) -> list[dict]: """ Read a JSONL file from S3 and return list of parsed objects. Args: @@ -46,9 +44,7 @@ def read_jsonl( return objects -def read_json( - bucket_name: str, key: str, raise_if_missing: bool = True -) -> dict | None: +def read_json(bucket_name: str, key: str, raise_if_missing: bool = True) -> dict | None: """ Read a JSON file from S3 and return parsed object. Args: @@ -101,7 +97,7 @@ def get_question_folders(inputs_path: str, bucket_name: str) -> list[str]: "Bucket": bucket_name, "Prefix": inputs_path, "Delimiter": "/", # Group by directory to get only subdirectories - "MaxKeys": 1000, # Max allowed per page by AWS + "MaxKeys": 1000, # Max allowed per page by AWS } if settings.ENVIRONMENT.upper() not in ["LOCAL", "TEST"]: @@ -193,7 +189,7 @@ def get_consultation_folders() -> list[str]: "Bucket": settings.AWS_BUCKET_NAME, "Prefix": "app_data/consultations/", "Delimiter": "/", # Group by directory to get only top-level consultation folders - "MaxKeys": 1000, # Max allowed per page by AWS + "MaxKeys": 1000, # Max allowed per page by AWS } if settings.ENVIRONMENT.upper() not in ["LOCAL", "TEST"]: diff --git a/backend/data_pipeline/sync/candidate_themes.py b/backend/data_pipeline/sync/candidate_themes.py index 25352026d..0ca91677f 100644 --- a/backend/data_pipeline/sync/candidate_themes.py +++ b/backend/data_pipeline/sync/candidate_themes.py @@ -54,9 +54,7 @@ def load_candidate_themes_from_s3( logger.info("Loading candidate themes from {key}", key=key) # Read and parse JSON file - theme_data = s3.read_json( - bucket_name=bucket_name_str, key=key, raise_if_missing=False - ) + theme_data = s3.read_json(bucket_name=bucket_name_str, key=key, raise_if_missing=False) if theme_data is None: logger.info("No candidate themes file found at {key}", key=key) diff --git a/backend/data_pipeline/sync/consultation_setup.py b/backend/data_pipeline/sync/consultation_setup.py index 46490db43..6861ee4b8 100644 --- a/backend/data_pipeline/sync/consultation_setup.py +++ b/backend/data_pipeline/sync/consultation_setup.py @@ -35,9 +35,7 @@ # ============================================================================= -def load_respondents_from_s3( - consultation_code: str, bucket_name: str -) -> list[RespondentInput]: +def load_respondents_from_s3(consultation_code: str, bucket_name: str) -> list[RespondentInput]: """ Load and validate respondents from S3. @@ -269,16 +267,12 @@ def load_consultation_data_batch( for question_number in question_numbers: # Load free text responses - responses = load_responses_from_s3( - consultation_code, question_number, bucket_name - ) + responses = load_responses_from_s3(consultation_code, question_number, bucket_name) if responses: responses_by_question[question_number] = responses # Load multi-choice data - multi_choices = load_multi_choice_from_s3( - consultation_code, question_number, bucket_name - ) + multi_choices = load_multi_choice_from_s3(consultation_code, question_number, bucket_name) if multi_choices: multi_choice_by_question[question_number] = multi_choices diff --git a/backend/data_pipeline/sync/response_annotations.py b/backend/data_pipeline/sync/response_annotations.py index b43adb963..83acd8604 100644 --- a/backend/data_pipeline/sync/response_annotations.py +++ b/backend/data_pipeline/sync/response_annotations.py @@ -1,4 +1,3 @@ - from botocore.exceptions import BotoCoreError, ClientError from django.conf import settings from django.db import transaction @@ -56,15 +55,13 @@ def load_selected_themes_from_s3( try: # Read and parse JSON file - theme_data = s3.read_json( - bucket_name=bucket_name_str, key=key, raise_if_missing=True - ) + theme_data = s3.read_json(bucket_name=bucket_name_str, key=key, raise_if_missing=True) except (ClientError, BotoCoreError) as e: logger.exception( "Failed to load selected themes from S3 for consultation '{consultation_code}'," " question {question_number}", consultation_code=consultation_code, - question_number=question_number + question_number=question_number, ) if isinstance(e, ClientError) and e.response["Error"]["Code"] == "NoSuchKey": raise ValueError( @@ -115,15 +112,13 @@ def load_sentiments_from_s3( try: # Read JSONL file (raise_if_missing=False because sentiment is optional) - sentiment_data = s3.read_jsonl( - bucket_name=bucket_name_str, key=key, raise_if_missing=False - ) + sentiment_data = s3.read_jsonl(bucket_name=bucket_name_str, key=key, raise_if_missing=False) except (ClientError, BotoCoreError): logger.exception( "Failed to load sentiments from S3 for consultation '{consultation_code}'," " question {question_number}", consultation_code=consultation_code, - question_number=question_number + question_number=question_number, ) raise @@ -172,15 +167,13 @@ def load_detail_detections_from_s3( try: # Read JSONL file - detail_data = s3.read_jsonl( - bucket_name=bucket_name_str, key=key, raise_if_missing=True - ) + detail_data = s3.read_jsonl(bucket_name=bucket_name_str, key=key, raise_if_missing=True) except (ClientError, BotoCoreError) as e: logger.exception( "Failed to load detail detections from S3 for consultation '{consultation_code}'," " question {question_number}", consultation_code=consultation_code, - question_number=question_number + question_number=question_number, ) if isinstance(e, ClientError) and e.response["Error"]["Code"] == "NoSuchKey": raise ValueError( @@ -230,15 +223,13 @@ def load_theme_mappings_from_s3( try: # Read JSONL file - mapping_data = s3.read_jsonl( - bucket_name=bucket_name_str, key=key, raise_if_missing=True - ) + mapping_data = s3.read_jsonl(bucket_name=bucket_name_str, key=key, raise_if_missing=True) except (ClientError, BotoCoreError) as e: logger.exception( "Failed to load theme mappings from S3 for consultation '{consultation_code}'," " question {question_number}", consultation_code=consultation_code, - question_number=question_number + question_number=question_number, ) if isinstance(e, ClientError) and e.response["Error"]["Code"] == "NoSuchKey": raise ValueError( @@ -301,7 +292,7 @@ def load_annotation_batch( logger.exception( "Consultation with code '{consultation_code}' does not exist. " "Base consultation data must be imported before annotations.", - consultation_code=consultation_code + consultation_code=consultation_code, ) raise ValueError( f"Consultation with code '{consultation_code}' does not exist. " diff --git a/backend/factories.py b/backend/factories.py index f2f8b904d..c8b6856ca 100644 --- a/backend/factories.py +++ b/backend/factories.py @@ -112,9 +112,7 @@ def encode(obj): field_value=encode(v), ) self.demographics.add(o) - DemographicOption.objects.filter(pk=o.pk).update( - response_count=F("response_count") + 1 - ) + DemographicOption.objects.filter(pk=o.pk).update(response_count=F("response_count") + 1) self.save() diff --git a/backend/rq_context.py b/backend/rq_context.py index bf323e5c3..6369d90c4 100644 --- a/backend/rq_context.py +++ b/backend/rq_context.py @@ -24,7 +24,9 @@ def context_aware(*args, context_id: str | None = None, **kwargs): @functools.wraps(func) def enqueue_with_context(*args, context_id: str | None = None, **kwargs): - return enqueue_call(*args, context_id=context_id or get_or_create_context_id(), **kwargs) + return enqueue_call( + *args, context_id=context_id or get_or_create_context_id(), **kwargs + ) decorated.delay = enqueue_with_context decorated.enqueue = enqueue_with_context diff --git a/backend/settings/local.py b/backend/settings/local.py index 3c08617b7..7504c5b8e 100644 --- a/backend/settings/local.py +++ b/backend/settings/local.py @@ -7,9 +7,7 @@ STORAGES["default"] = { "BACKEND": "django.core.files.storage.FileSystemStorage", - "OPTIONS": { - "location": BASE_DIR / "tmp" - }, + "OPTIONS": {"location": BASE_DIR / "tmp"}, } REST_FRAMEWORK["DEFAULT_SCHEMA_CLASS"] = "drf_spectacular.openapi.AutoSchema" diff --git a/backend/tests/commands/test_dummy_data.py b/backend/tests/commands/test_dummy_data.py index e82a4ed6a..9315476d4 100644 --- a/backend/tests/commands/test_dummy_data.py +++ b/backend/tests/commands/test_dummy_data.py @@ -23,8 +23,9 @@ def test_name_parameter_sets_consultation_name(mock_is_local): @pytest.mark.django_db @pytest.mark.parametrize("environment", ["prod"]) def test_the_tool_will_only_run_in_dev(environment): - with patch.dict(os.environ, {"ENVIRONMENT": environment}), pytest.raises( - Exception, match=r"Dummy data generation should not be run in production" + with ( + patch.dict(os.environ, {"ENVIRONMENT": environment}), + pytest.raises(Exception, match=r"Dummy data generation should not be run in production"), ): call_command( "generate_dummy_data", diff --git a/backend/tests/unit/data_pipeline/sync/test_consultation_setup.py b/backend/tests/unit/data_pipeline/sync/test_consultation_setup.py index 4d767ea71..ec865d10b 100644 --- a/backend/tests/unit/data_pipeline/sync/test_consultation_setup.py +++ b/backend/tests/unit/data_pipeline/sync/test_consultation_setup.py @@ -124,7 +124,9 @@ def test_import_consultation_from_s3(self, mock_enqueue, minio_test_bucket, mini # Create question 1 files (free text only) key = "app_data/consultations/test-code/inputs/question_part_1/question.json" - minio_client.put_object(Bucket=minio_test_bucket, Key=key, Body=json.dumps(question_1).encode()) + minio_client.put_object( + Bucket=minio_test_bucket, Key=key, Body=json.dumps(question_1).encode() + ) created_keys.append(key) key = "app_data/consultations/test-code/inputs/question_part_1/responses.jsonl" @@ -138,7 +140,9 @@ def test_import_consultation_from_s3(self, mock_enqueue, minio_test_bucket, mini # Create question 2 files (multi choice only) key = "app_data/consultations/test-code/inputs/question_part_2/question.json" - minio_client.put_object(Bucket=minio_test_bucket, Key=key, Body=json.dumps(question_2).encode()) + minio_client.put_object( + Bucket=minio_test_bucket, Key=key, Body=json.dumps(question_2).encode() + ) created_keys.append(key) key = "app_data/consultations/test-code/inputs/question_part_2/responses.jsonl" @@ -152,7 +156,9 @@ def test_import_consultation_from_s3(self, mock_enqueue, minio_test_bucket, mini # Create question 3 files (hybrid) key = "app_data/consultations/test-code/inputs/question_part_3/question.json" - minio_client.put_object(Bucket=minio_test_bucket, Key=key, Body=json.dumps(question_3).encode()) + minio_client.put_object( + Bucket=minio_test_bucket, Key=key, Body=json.dumps(question_3).encode() + ) created_keys.append(key) key = "app_data/consultations/test-code/inputs/question_part_3/responses.jsonl" @@ -229,7 +235,10 @@ def test_import_consultation_from_s3(self, mock_enqueue, minio_test_bucket, mini assert [opt.text for opt in q2_response_1.chosen_options.all()] == ["Option A"] q2_response_2 = Response.objects.get(question=question_2_db, respondent=respondent_2) - assert {opt.text for opt in q2_response_2.chosen_options.all()} == {"Option B", "Option C"} + assert {opt.text for opt in q2_response_2.chosen_options.all()} == { + "Option B", + "Option C", + } # Verify responses for hybrid question q3_response_1 = Response.objects.get(question=question_3_db, respondent=respondent_1) @@ -247,10 +256,7 @@ def test_import_consultation_from_s3(self, mock_enqueue, minio_test_bucket, mini # Cleanup: Delete all objects we created for key in created_keys: try: - minio_client.delete_object( - Bucket=minio_test_bucket, - Key=key - ) + minio_client.delete_object(Bucket=minio_test_bucket, Key=key) except Exception as e: # noqa: BLE001 logger.warning("Failed to cleanup object {key}: {e}", key=key, e=e) diff --git a/backend/tests/unit/data_pipeline/test_batch.py b/backend/tests/unit/data_pipeline/test_batch.py index ad4a5118d..e2f495621 100644 --- a/backend/tests/unit/data_pipeline/test_batch.py +++ b/backend/tests/unit/data_pipeline/test_batch.py @@ -11,9 +11,19 @@ class TestSubmitBatchJob: @staticmethod def _configure_batch_job_settings(mock_settings, job_type: str) -> None: mock_settings.SUBMIT_BATCH_JOBS = True - setattr(mock_settings, f"{job_type}_BATCH_JOB_NAME", f"{job_type.lower()}-job".replace("_", "-")) - setattr(mock_settings, f"{job_type}_BATCH_JOB_QUEUE", f"{job_type.lower()}-queue".replace("_", "-")) - setattr(mock_settings, f"{job_type}_BATCH_JOB_DEFINITION", f"{job_type.lower()}-def".replace("_", "-")) + setattr( + mock_settings, f"{job_type}_BATCH_JOB_NAME", f"{job_type.lower()}-job".replace("_", "-") + ) + setattr( + mock_settings, + f"{job_type}_BATCH_JOB_QUEUE", + f"{job_type.lower()}-queue".replace("_", "-"), + ) + setattr( + mock_settings, + f"{job_type}_BATCH_JOB_DEFINITION", + f"{job_type.lower()}-def".replace("_", "-"), + ) @staticmethod def _context_id_from_command(command: list[str]) -> str | None: @@ -149,7 +159,10 @@ def test_submit_job_propagates_context_id(self, mock_settings, mock_boto3): finally: structlog.contextvars.unbind_contextvars("context_id") - assert self._context_id_from_command(call_args["containerOverrides"]["command"]) == "request-context-abc" + assert ( + self._context_id_from_command(call_args["containerOverrides"]["command"]) + == "request-context-abc" + ) assert call_args["parameters"]["context_id"] == "request-context-abc" @patch("data_pipeline.batch.boto3") @@ -158,11 +171,16 @@ def test_submit_job_explicit_context_id_overrides_ambient(self, mock_settings, m """An explicit context_id wins over whatever's ambiently bound, in both the command and parameters""" structlog.contextvars.bind_contextvars(context_id="ambient-id") try: - call_args = self._submit_find_themes_job(mock_settings, mock_boto3, context_id="explicit-id") + call_args = self._submit_find_themes_job( + mock_settings, mock_boto3, context_id="explicit-id" + ) finally: structlog.contextvars.unbind_contextvars("context_id") - assert self._context_id_from_command(call_args["containerOverrides"]["command"]) == "explicit-id" + assert ( + self._context_id_from_command(call_args["containerOverrides"]["command"]) + == "explicit-id" + ) assert call_args["parameters"]["context_id"] == "explicit-id" @patch("data_pipeline.batch.boto3") diff --git a/backend/tests/unit/data_pipeline/test_s3.py b/backend/tests/unit/data_pipeline/test_s3.py index b4b336467..75e6e7605 100644 --- a/backend/tests/unit/data_pipeline/test_s3.py +++ b/backend/tests/unit/data_pipeline/test_s3.py @@ -29,18 +29,11 @@ def test_get_question_folders(self, minio_test_bucket, minio_client): ] for key in test_objects: - minio_client.put_object( - Bucket=minio_test_bucket, - Key=key, - Body=b"test content" - ) + minio_client.put_object(Bucket=minio_test_bucket, Key=key, Body=b"test content") created_keys.append(key) # Execute: Call the function under test - result = get_question_folders( - "app_data/consultations/test/inputs/", - minio_test_bucket - ) + result = get_question_folders("app_data/consultations/test/inputs/", minio_test_bucket) # Verify: Check results expected = [ @@ -53,10 +46,7 @@ def test_get_question_folders(self, minio_test_bucket, minio_client): # Cleanup: Delete all objects we created for key in created_keys: try: - minio_client.delete_object( - Bucket=minio_test_bucket, - Key=key - ) + minio_client.delete_object(Bucket=minio_test_bucket, Key=key) except Exception as e: # noqa: BLE001 logger.warning("Failed to cleanup object {key}: {e}", key=key, e=e) @@ -85,11 +75,7 @@ def test_get_consultation_folders(self, minio_test_bucket, minio_client): ] for key in test_objects: - minio_client.put_object( - Bucket=minio_test_bucket, - Key=key, - Body=b"test content" - ) + minio_client.put_object(Bucket=minio_test_bucket, Key=key, Body=b"test content") created_keys.append(key) # Execute: Call the function under test @@ -103,9 +89,6 @@ def test_get_consultation_folders(self, minio_test_bucket, minio_client): # Cleanup: Delete all objects we created for key in created_keys: try: - minio_client.delete_object( - Bucket=minio_test_bucket, - Key=key - ) + minio_client.delete_object(Bucket=minio_test_bucket, Key=key) except Exception as e: # noqa: BLE001 logger.warning("Failed to cleanup object {key}: {e}", key=key, e=e) diff --git a/backend/tests/unit/test_generate_dummy_data.py b/backend/tests/unit/test_generate_dummy_data.py index c6a992666..9bf1e15f2 100644 --- a/backend/tests/unit/test_generate_dummy_data.py +++ b/backend/tests/unit/test_generate_dummy_data.py @@ -19,11 +19,11 @@ def test_a_consultation_is_generated(settings): @pytest.mark.django_db @pytest.mark.parametrize("environment", ["prod"]) def test_the_tool_will_only_run_in_dev(environment): - with patch.dict(os.environ, {"ENVIRONMENT": environment}): - with pytest.raises( - Exception, match=r"Dummy data generation should not be run in production" - ): - create_dummy_consultation() + with ( + patch.dict(os.environ, {"ENVIRONMENT": environment}), + pytest.raises(Exception, match=r"Dummy data generation should not be run in production"), + ): + create_dummy_consultation() @pytest.mark.django_db diff --git a/backend/tests/unit/test_healthcheck_worker.py b/backend/tests/unit/test_healthcheck_worker.py index bff9ffc75..0d15666e2 100644 --- a/backend/tests/unit/test_healthcheck_worker.py +++ b/backend/tests/unit/test_healthcheck_worker.py @@ -60,5 +60,8 @@ def test_fails_when_the_worker_heartbeat_is_stale(self, rq_worker): call_command("healthcheck_worker") def test_fails_with_a_clear_message_when_redis_is_unreachable(self): - with patch("rq.worker.Worker.all", side_effect=redis.exceptions.ConnectionError("boom")), pytest.raises(CommandError, match="Could not reach Redis"): - call_command("healthcheck_worker") + with ( + patch("rq.worker.Worker.all", side_effect=redis.exceptions.ConnectionError("boom")), + pytest.raises(CommandError, match="Could not reach Redis"), + ): + call_command("healthcheck_worker") diff --git a/backend/tests/unit/test_middleware.py b/backend/tests/unit/test_middleware.py index cbb84808d..8a8e0d03d 100644 --- a/backend/tests/unit/test_middleware.py +++ b/backend/tests/unit/test_middleware.py @@ -17,7 +17,9 @@ def view(request): if captured is not None: captured["context_id"] = structlog.contextvars.get_contextvars().get("context_id") captured["path"] = structlog.contextvars.get_contextvars().get("path") - captured["execution_context"] = structlog.contextvars.get_contextvars().get("execution_context") + captured["execution_context"] = structlog.contextvars.get_contextvars().get( + "execution_context" + ) return HttpResponse(status=200) return RequestCorrelationMiddleware(view) @@ -38,7 +40,9 @@ def test_reuses_inbound_context_id(request_factory): middleware = _make_middleware(captured) inbound = "inbound-correlation-id" - response = middleware(request_factory.get("/api/consultations/", headers={"x-context-id": inbound})) + response = middleware( + request_factory.get("/api/consultations/", headers={"x-context-id": inbound}) + ) assert captured["context_id"] == inbound assert response[CONTEXT_ID_HEADER] == inbound diff --git a/backend/tests/unit/test_rq_context.py b/backend/tests/unit/test_rq_context.py index 6d1bc3f8e..7571e623a 100644 --- a/backend/tests/unit/test_rq_context.py +++ b/backend/tests/unit/test_rq_context.py @@ -66,11 +66,11 @@ def test_rebinds_execution_context_after_refresh(self, settings): class TestRQContextJob: """rq_context.job auto-fills context_id at enqueue time and rebinds it at execution time; - ASYNC=False in tests runs .delay()/.enqueue() through the real path.""" + ASYNC=False in tests runs .delay()/.enqueue() through the real path.""" def test_ambient_context_id_survives_a_real_delay_round_trip(self): """Proves context_id survives the full enqueue -> execution round trip; - the job.kwargs check rules out a same-process leak masquerading as propagation.""" + the job.kwargs check rules out a same-process leak masquerading as propagation.""" rebind_context("ambient-real-id") job = probe_job.delay(42) diff --git a/uv.lock b/uv.lock index ad03e58c2..0d5a6dc95 100644 --- a/uv.lock +++ b/uv.lock @@ -149,30 +149,30 @@ wheels = [ [[package]] name = "boto3" -version = "1.43.56" +version = "1.43.64" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, { name = "jmespath" }, { name = "s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/53/05/23e1aa8c9e4b0399a61e7fd65c4f9cc0625121f24760e37471f776404abb/boto3-1.43.56.tar.gz", hash = "sha256:57c90df9fb026f2e6ae22530861198130203733c5c9ec4e5cca3a4037f5a8db4", size = 112673, upload-time = "2026-07-24T19:31:48.606Z" } +sdist = { url = "https://files.pythonhosted.org/packages/51/da/d5ca15f34a567f2d027df23395315983bd7ee35011e93c1cca12b7f89823/boto3-1.43.64.tar.gz", hash = "sha256:fc7522c3ed97d38176e0b1366406bca0fc8c888ae8d416bf953477b3f015a7b2", size = 112655, upload-time = "2026-08-04T20:12:42.45Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b8/57/3a960c9f581c00f2a591901b46e035ff79ab3956d16607f12306b3b8d483/boto3-1.43.56-py3-none-any.whl", hash = "sha256:feb699d4ab241ef5c1b80bb58277be2aaad365cd4b672d7817e0bc59ee45131b", size = 140026, upload-time = "2026-07-24T19:31:47.155Z" }, + { url = "https://files.pythonhosted.org/packages/15/94/14cf3ec553edf78e5196289087ac52862c87ac179fa1cfdf1022ec366b79/boto3-1.43.64-py3-none-any.whl", hash = "sha256:2b555e63ece57cffb1ab1666fa6c23a0957e61f9d19ea8424d6c884124c3fd98", size = 140024, upload-time = "2026-08-04T20:12:41.147Z" }, ] [[package]] name = "botocore" -version = "1.43.56" +version = "1.43.64" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b9/cc/7f84a5d3071fe878380e9f610ab36ca87b8cbbc4aa81ba2727f90e1f3ea3/botocore-1.43.56.tar.gz", hash = "sha256:6c01f85f0ff9863076f4c761e74ee3aa96c5ccc1ad09fc1efd62ef8f2d22bf57", size = 15733117, upload-time = "2026-07-24T19:31:38.125Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/67/0e39203c5c67f750874478cf89124486044580ec5cb391eec8eb13cf25b8/botocore-1.43.64.tar.gz", hash = "sha256:a2bb131f48111094fae0a2c896b42593797e935d9c22abb2ebae4290c6dbb5ea", size = 15842274, upload-time = "2026-08-04T20:12:31.598Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/cd/86fe9e659e9699f62f8dd5ecd8c6725474334b23cab8aa71d82b5f56f1a4/botocore-1.43.56-py3-none-any.whl", hash = "sha256:aafc741f1b10f6fd63253eaf6ea029680c1ff436d87e1b8969d62aefa0c76976", size = 15418773, upload-time = "2026-07-24T19:31:34.758Z" }, + { url = "https://files.pythonhosted.org/packages/3d/d1/f27d1a4fb571e102828a4672cb3d19e0409275eb92c701b67a85879c0b04/botocore-1.43.64-py3-none-any.whl", hash = "sha256:f0a01c47d631ab95589c244566bca971294724b21862de1c12c7c3b0de236272", size = 15525459, upload-time = "2026-08-04T20:12:28.58Z" }, ] [[package]] @@ -312,7 +312,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "boto3", specifier = ">=1.43.56" }, + { name = "boto3", specifier = ">=1.43.62" }, { name = "django", specifier = ">=6.0.7" }, { name = "django-environ", specifier = ">=0.14.0" }, { name = "django-filter", specifier = ">=26.1" }, @@ -322,18 +322,18 @@ requires-dist = [ { name = "django-storages", specifier = ">=1.14.6" }, { name = "djangorestframework", specifier = ">=3.17.1" }, { name = "djangorestframework-simplejwt", extras = ["crypto"], specifier = ">=5.5.1" }, - { name = "drf-nested-routers", specifier = ">=0.95.0" }, + { name = "drf-nested-routers", specifier = ">=0.95.3" }, { name = "drf-orjson-renderer", specifier = ">=1.8.0" }, { name = "drf-spectacular", specifier = "==0.30.0" }, { name = "factory-boy", specifier = ">=3.3.3" }, { name = "gunicorn", specifier = ">=26.0.0" }, { name = "i-dot-ai-utilities", extras = ["auth"], specifier = ">=0.6.0" }, - { name = "openai", specifier = ">=2.48.0" }, + { name = "openai", specifier = ">=2.52.0" }, { name = "pgvector", specifier = ">=0.5.0" }, { name = "psycopg", specifier = ">=3.3.4" }, { name = "pydantic", specifier = ">=2.13.4" }, { name = "pytest-random-order", specifier = ">=1.2.0" }, - { name = "sentry-sdk", extras = ["django", "redis", "rq"], specifier = ">=2.66.0" }, + { name = "sentry-sdk", extras = ["django", "redis", "rq"], specifier = ">=2.66.1" }, { name = "themefinder", editable = "themefinder" }, { name = "werkzeug", specifier = "==3.1.8" }, ] @@ -352,7 +352,7 @@ dev = [ { name = "pydot", specifier = ">=4.0.1" }, { name = "pytest-django", specifier = ">=4.12.0" }, { name = "pytest-lazy-fixtures", specifier = ">=1.4.0" }, - { name = "ruff", specifier = ">=0.16.0" }, + { name = "ruff", specifier = ">=0.16.1" }, ] [[package]] @@ -433,7 +433,7 @@ name = "cuda-bindings" version = "13.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder" }, + { name = "cuda-pathfinder", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/ce/67/5e7dba1ba576dd73da5dee894ca076ca5e959450dfff66d6d510a255d1f7/cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49", size = 6025351, upload-time = "2026-05-29T23:11:49.685Z" }, @@ -458,34 +458,34 @@ wheels = [ [package.optional-dependencies] cudart = [ - { name = "nvidia-cuda-runtime" }, + { name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux'" }, ] cufft = [ - { name = "nvidia-cufft" }, + { name = "nvidia-cufft", marker = "sys_platform == 'linux'" }, ] cufile = [ - { name = "nvidia-cufile" }, + { name = "nvidia-cufile", marker = "sys_platform == 'linux'" }, ] cupti = [ - { name = "nvidia-cuda-cupti" }, + { name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux'" }, ] curand = [ - { name = "nvidia-curand" }, + { name = "nvidia-curand", marker = "sys_platform == 'linux'" }, ] cusolver = [ - { name = "nvidia-cusolver" }, + { name = "nvidia-cusolver", marker = "sys_platform == 'linux'" }, ] cusparse = [ - { name = "nvidia-cusparse" }, + { name = "nvidia-cusparse", marker = "sys_platform == 'linux'" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux'" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc" }, + { name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux'" }, ] nvtx = [ - { name = "nvidia-nvtx" }, + { name = "nvidia-nvtx", marker = "sys_platform == 'linux'" }, ] [[package]] @@ -674,15 +674,15 @@ crypto = [ [[package]] name = "drf-nested-routers" -version = "0.95.0" +version = "0.95.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "django" }, { name = "djangorestframework" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ff/b2/070908ccd41bd6f025f5507d7f6e50009abe8be48393cc697d159b539f0b/drf_nested_routers-0.95.0.tar.gz", hash = "sha256:815978f802e578fd7035c74040c104909cbe97615de89a275d77e928f4029891", size = 23318, upload-time = "2025-09-09T02:01:58.201Z" } +sdist = { url = "https://files.pythonhosted.org/packages/86/fa/82d61dd87bfbc94f9d33657e7c82f8fe3e029103d4be4b7536e4877ac544/drf_nested_routers-0.95.3.tar.gz", hash = "sha256:3d5ffad87b110c9d58ee0c688cf540a7fa4ccbf1080b2d318a5e2cf634322d96", size = 22398, upload-time = "2026-07-31T15:19:02.529Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3e/61/69d951a6e8031389f5feade38531432325019e1047b23d5c6036a86c5e8f/drf_nested_routers-0.95.0-py2.py3-none-any.whl", hash = "sha256:dd489c33d667aaa81383ffaa8c74781d2b353d8f0795716ae37fc59ee297b7c4", size = 36407, upload-time = "2025-09-09T02:01:56.999Z" }, + { url = "https://files.pythonhosted.org/packages/12/33/c814bfd41d343045f4690c8f078601104fc75782c36913b2b7c9a7cec84c/drf_nested_routers-0.95.3-py3-none-any.whl", hash = "sha256:bb02f4fea712f7f0fc649fc1399718e458a06387fdb2fb161cc9aeaad314f4ef", size = 20990, upload-time = "2026-07-31T15:19:01.119Z" }, ] [[package]] @@ -1386,7 +1386,7 @@ name = "nvidia-cublas" version = "13.1.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cuda-nvrtc" }, + { name = "nvidia-cuda-nvrtc", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" }, @@ -1425,7 +1425,7 @@ name = "nvidia-cudnn-cu13" version = "9.20.0.48" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas" }, + { name = "nvidia-cublas", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" }, @@ -1437,7 +1437,7 @@ name = "nvidia-cufft" version = "12.0.0.61" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, @@ -1467,9 +1467,9 @@ name = "nvidia-cusolver" version = "12.0.4.66" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas" }, - { name = "nvidia-cusparse" }, - { name = "nvidia-nvjitlink" }, + { name = "nvidia-cublas", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-cusparse", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-nvjitlink", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, @@ -1481,7 +1481,7 @@ name = "nvidia-cusparse" version = "12.6.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, @@ -1535,7 +1535,7 @@ wheels = [ [[package]] name = "openai" -version = "2.48.0" +version = "2.53.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -1547,9 +1547,9 @@ dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2a/ae/d4d1835488c0350424009dac5095b9a3e173bee12fd2e421ee27e2142c42/openai-2.48.0.tar.gz", hash = "sha256:231b1e7661dda14574986c2f71451e9d584b7fe69e0ee6480e12ed090b48fc16", size = 1093427, upload-time = "2026-07-23T20:15:50.402Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ef/cf/36e3e7235fdf6d125c052acc0970924611b17a20a4fe580596faf4566a65/openai-2.53.0.tar.gz", hash = "sha256:baf5802ad08980e1d9d561e1b996e800c8bcd14af5847c6d0e7a5cc59e4d4116", size = 1099435, upload-time = "2026-08-03T21:42:01.664Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/2a/dcb891114e303c4379d4a498f10222e33eee540bcef4e1e493bd0af2b242/openai-2.48.0-py3-none-any.whl", hash = "sha256:c98df30aaaf93c51979f64d3e7c5b76464f8be0173368266229eb8fe6bd30f2c", size = 1648520, upload-time = "2026-07-23T20:15:48.052Z" }, + { url = "https://files.pythonhosted.org/packages/78/0f/cc6afea3542a5142c5d8fc8211c5e059a8375105d004a41dfa2c7948dbb0/openai-2.53.0-py3-none-any.whl", hash = "sha256:c694ffc747a3c4d1663ef2b07b811315a476164ee5efa3a993967349ebca7618", size = 1659829, upload-time = "2026-08-03T21:41:59.581Z" }, ] [[package]] @@ -2246,27 +2246,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4d/94/1e5e4967626faf12fa56999cd6222dff6992ceb086ad7945756baf70c7a7/ruff-0.16.0.tar.gz", hash = "sha256:e460aafd5495ec89efaa6ced2e4a9a581116451e1c88b9d37ef497e0f8e93982", size = 4790557, upload-time = "2026-07-23T19:11:30.981Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4b/81/1c8818fee7ce1a04cd7d1b3172e0a8f8e4f1dc4feb7fc390e16daa8af323/ruff-0.16.0-py3-none-linux_armv6l.whl", hash = "sha256:e5115729eb08c585e5121978ba5d5b60caeae394ce21b9fb5e6cd33a1c6c9b1e", size = 10754633, upload-time = "2026-07-23T19:10:46.415Z" }, - { url = "https://files.pythonhosted.org/packages/23/df/beaf59c09d68db84304d555f188b276a77132a5d5b0b67a5c762aa143628/ruff-0.16.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:3c954b1d580bfa035b41654f7858cc7e71d5fc3ac5b723dd62bd9133830ed522", size = 10969164, upload-time = "2026-07-23T19:10:50.271Z" }, - { url = "https://files.pythonhosted.org/packages/42/ce/741cd197496a1abbf51352710fd15ed995d2a2be87189c1da26a450d6e83/ruff-0.16.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e01c21d10eb1b29f47b7454e1f4056db9a3f0260c646aa88457c610291db9f81", size = 10488846, upload-time = "2026-07-23T19:10:52.639Z" }, - { url = "https://files.pythonhosted.org/packages/52/2a/a2db8e88cade358f5cdcb05674a917751074109315d014eb6352d9a893f7/ruff-0.16.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e364e5ed22ed8dc05082fd78e35308618260907ac2d3c1d637b2e682415b6c9", size = 10889729, upload-time = "2026-07-23T19:10:54.89Z" }, - { url = "https://files.pythonhosted.org/packages/42/65/62a771694ebd63029dc953e27dbad40e1588bd4860ff9fe881018fddaa49/ruff-0.16.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d327b8fc113a1d4421a04f3839d3752057c8dd1ee320223a6f3f52d04ada462a", size = 10568275, upload-time = "2026-07-23T19:10:56.993Z" }, - { url = "https://files.pythonhosted.org/packages/3f/e2/ced249fe8af5f086c5c58cc21cc3356d50f32f7401c5df87050c999620a7/ruff-0.16.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9b50c55e263103586b3dcf5f73d479eb8cb5fdb6098fec59a62891dab653717", size = 11385112, upload-time = "2026-07-23T19:10:59.615Z" }, - { url = "https://files.pythonhosted.org/packages/87/0b/05154977a8fd69eeb6c103271f55403bfd8711f5c0f8ed07489d95a504e7/ruff-0.16.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ff4a79ce3ec0172f3241943835de1c4cb4e2dcd07f0f8c2d02603dbbbee4b17", size = 12207008, upload-time = "2026-07-23T19:11:02.154Z" }, - { url = "https://files.pythonhosted.org/packages/fb/29/98225831a3a1eab0e02f4acc6ca6559a98611dcc68b6965ff4b7234627c1/ruff-0.16.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e95c448fca1fb2a18372a9440926c5a6ee789639bb975c72e7ae6d0b04218ab4", size = 11650842, upload-time = "2026-07-23T19:11:04.557Z" }, - { url = "https://files.pythonhosted.org/packages/91/66/6bd3cf90500653d55dc0ffc8507aa8300bd49d0214b2e8cb4d3fef2943ba/ruff-0.16.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f11a8d11010301d0a398a2fdef67691feca7294da6aef55e2150e8fa2cd520b", size = 11400718, upload-time = "2026-07-23T19:11:09.233Z" }, - { url = "https://files.pythonhosted.org/packages/8e/a2/a54eb4eae05d66364050a5d3b8a9c5ef88196531b3cbe7109d873f87f819/ruff-0.16.0-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:48044c678e9cb8698246c99b14aaccfa6601dea7379eb48a6f8f73f7a6d86cd0", size = 11426177, upload-time = "2026-07-23T19:11:11.994Z" }, - { url = "https://files.pythonhosted.org/packages/1a/be/16e3eea4b2a478a496919f5e36f17c4559e54620bd3bbac5d6affa068006/ruff-0.16.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:7aa0959bad8eb8bef50340154fc9b58678dae31fa4293afa38b44b6e552c0213", size = 10856126, upload-time = "2026-07-23T19:11:14.221Z" }, - { url = "https://files.pythonhosted.org/packages/a2/84/252eb8b868a16eec7257c14f504f77537e734b2d69c762e639e588e304a3/ruff-0.16.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:28ea2b7df8ebf7f9da6b7d47b230ab48f387c0a29be3b474c4d0740e197bb9af", size = 10571208, upload-time = "2026-07-23T19:11:16.378Z" }, - { url = "https://files.pythonhosted.org/packages/21/09/817a482f542f7570cbb4554b26e896610c7114f539b1d9e2d2145bf6bef6/ruff-0.16.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:33a3dfac8c35f81498dea9181bccc2f4c4bc8f1521a1dd9406e77643e0f0fb09", size = 11063329, upload-time = "2026-07-23T19:11:19.173Z" }, - { url = "https://files.pythonhosted.org/packages/2e/23/9403c180ca1cb9b1f7335f5c3e5305c09d49ea5b345196682a36028bde4a/ruff-0.16.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a5237a0bda500d30d81b8e07a6973a5cbc772864cbf746ae2f4e8a2e01c9f4ed", size = 11489751, upload-time = "2026-07-23T19:11:21.74Z" }, - { url = "https://files.pythonhosted.org/packages/b2/1d/1b2ef7bcde851c78d7f17f1cca13fd6dc695fc4b3d6197941e72cae5b132/ruff-0.16.0-py3-none-win32.whl", hash = "sha256:7fab76fa065c873f41ff744347c6e77bcc3dfec4bcc754dc26b63d23c0f7f5fb", size = 10785885, upload-time = "2026-07-23T19:11:23.947Z" }, - { url = "https://files.pythonhosted.org/packages/b2/a3/d5e4ef7a56be3f928ffb90b94c25ba7d3cb9c7fe0736aeaaedf361770712/ruff-0.16.0-py3-none-win_amd64.whl", hash = "sha256:429c117f022bf481fabd9d551e7a3952b24c65e6ef44337ea09d90bebef14472", size = 11923141, upload-time = "2026-07-23T19:11:26.409Z" }, - { url = "https://files.pythonhosted.org/packages/cb/9a/8415f2657cbe200f41a4531ccededf135505a92d4a012229121f885b26f9/ruff-0.16.0-py3-none-win_arm64.whl", hash = "sha256:14296fedcd2705c77ab8235439278bbb38f285cf7da5528b00b3e330c3d4872d", size = 11273407, upload-time = "2026-07-23T19:11:28.705Z" }, +version = "0.16.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/70/25/7113f6d5498888c5fb7db34081cba7d5971c4cb1bfb26819966eee68f003/ruff-0.16.1.tar.gz", hash = "sha256:fedad7c801dabd3fb9741d76aca39246e6ddd9ca446a015875207bf19f1e6bc7", size = 4877500, upload-time = "2026-07-30T19:37:01.379Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/bd/694da69368e0973de65df2ddc73ab18d43c469d5963d9b150911de6bc513/ruff-0.16.1-py3-none-linux_armv6l.whl", hash = "sha256:58edb313b88f0c5460a26adf5f39a37a3be789494a15e3e411e35fa78b89f9a0", size = 10839126, upload-time = "2026-07-30T19:36:13.697Z" }, + { url = "https://files.pythonhosted.org/packages/3f/f0/b626e5d5bd0dd9576263658ef12885e2288afd1029a48e26ffed65ec1ac1/ruff-0.16.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fde5a99e2f97479af66edd6622c6d5a2a7592c77cf4153d9e4428f5eeb55b60c", size = 11070253, upload-time = "2026-07-30T19:36:17.14Z" }, + { url = "https://files.pythonhosted.org/packages/83/63/f40acfb6b35b88623e71684942b552c3edd96035f5d98f313815f7b277de/ruff-0.16.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e0d4c20532fca4f7fa609369161d968dd28f65d83dabbd61d8e9c7edbf7001f6", size = 10561425, upload-time = "2026-07-30T19:36:20.04Z" }, + { url = "https://files.pythonhosted.org/packages/aa/dd/14ec0e9c2b4d315547dd38765004b4863e354e1b52cb308272215d9f6f6d/ruff-0.16.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30affbcedf59ad5703d9c91f82266e02b47739f797e1a7b6e158e5526a6dae38", size = 10948879, upload-time = "2026-07-30T19:36:22.476Z" }, + { url = "https://files.pythonhosted.org/packages/33/e9/9d870cbae575030fdef595f04b4b97573c525b5497cce4f4498cf2f85446/ruff-0.16.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:24e9c631573cbca9d20f1283f8f479b2afa4a8503504822bd71a293889f16743", size = 10643691, upload-time = "2026-07-30T19:36:24.914Z" }, + { url = "https://files.pythonhosted.org/packages/c4/09/12743d544e2173f53ecd27217c65f90d2bc0f8424a66a60339e56bbc0457/ruff-0.16.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b41bdd48fb420987a9b5212e4957c26ad4abce401fa9ea9d4d85843727945f4f", size = 11435354, upload-time = "2026-07-30T19:36:28.447Z" }, + { url = "https://files.pythonhosted.org/packages/7f/89/a1652b2daee52083c9554a6333b678a8b01d0400f976827bb87857f9449a/ruff-0.16.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b0d1e1393b7648079e13669de1c1f4fde06d4583e84d8fd5c1551e0a77a2aa75", size = 12259033, upload-time = "2026-07-30T19:36:31.326Z" }, + { url = "https://files.pythonhosted.org/packages/16/96/ecdcb8c54ee7b123b487f807eb014e6e019155a0b81dfb669acd52f28ce3/ruff-0.16.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:07bf434b1c95f4e093be4532068ef4fcf00924eb2ade8796075980902d6fd54a", size = 11667981, upload-time = "2026-07-30T19:36:34.394Z" }, + { url = "https://files.pythonhosted.org/packages/cd/90/c52e12e0d862e9572f2a33aa227409143520abe53111e9a6babbac7b4af8/ruff-0.16.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39897739f112253ee4fdd2e8aa9a4f9ded99fb2be367d5f31dfa4ded6025584c", size = 11468183, upload-time = "2026-07-30T19:36:37.339Z" }, + { url = "https://files.pythonhosted.org/packages/2c/6b/4ffb7ad1d83eb16cf8cbb3c8815d3f11c88460fd162d4b372a2059be1c2a/ruff-0.16.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:82ae3c0c0d74daf17b968a10b7b3bb3ef297ab7de0c1f749646b25e690ccb150", size = 11470071, upload-time = "2026-07-30T19:36:39.91Z" }, + { url = "https://files.pythonhosted.org/packages/9c/72/32ae7db4c0b5e32ab611787caa19d1546800676d79f7483b7100a3561bf4/ruff-0.16.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4d5f2ed10f8242d83fc08d521301089364e3375375705356f20c0e31606ef3ef", size = 10919503, upload-time = "2026-07-30T19:36:42.65Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ca/3d901ba6ad6fc38da39c3448fc6c59ac945679293a17c3ceb6d6c1cba13e/ruff-0.16.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a4665b309891f83f3e3c25447935f1213e9abbd4b5640af7a1f2def9f8d413c1", size = 10649861, upload-time = "2026-07-30T19:36:45.18Z" }, + { url = "https://files.pythonhosted.org/packages/92/79/894ef1ced26552d5f8c9cf6d85b0687840e1128c55aeab7b9c2d54a0d880/ruff-0.16.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:26e9ca5c9bc3971f20d3cf18a957f52ffd6a5f6564ff15c4912a144dcac22494", size = 11148137, upload-time = "2026-07-30T19:36:47.936Z" }, + { url = "https://files.pythonhosted.org/packages/2d/69/3609a09fa1cb46cc28b762363e440a354204e5dff01bd0c8d7437874d6b9/ruff-0.16.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:67e1e1e3fa4f0c82f0e36d4cd61e661f6e7a6196cb1aa92fe0828fa7b8f257cd", size = 11559211, upload-time = "2026-07-30T19:36:50.448Z" }, + { url = "https://files.pythonhosted.org/packages/fc/8a/fb22af2fd78a736e241fabf67e30ce1799a64244026377a49e133af90762/ruff-0.16.1-py3-none-win32.whl", hash = "sha256:d31765e131295b8445caf301e3e8a85b34d1b9b211b4109b7ba457888b051806", size = 10838258, upload-time = "2026-07-30T19:36:53.298Z" }, + { url = "https://files.pythonhosted.org/packages/d4/35/e57fd9fb5d423961df087a00b12d42c0a830288dc2f3b45ecca299158b4f/ruff-0.16.1-py3-none-win_amd64.whl", hash = "sha256:09b05e8b90c2cb06ad63464350e7a45e8e44a2dfe52072ebfba6666ca8d3f596", size = 11961111, upload-time = "2026-07-30T19:36:56.107Z" }, + { url = "https://files.pythonhosted.org/packages/cb/46/240ea004bf6dc4feb40e9832f2205a476a47dd5b8a3f8211a5fc5f95e20e/ruff-0.16.1-py3-none-win_arm64.whl", hash = "sha256:dbaadaac38c70239f056d306b7476f246b0bf000fa6b3876402acbf5b227eaf8", size = 11309414, upload-time = "2026-07-30T19:36:58.79Z" }, ] [[package]] From 6057762bdfaa811c3f8893e9cfd60c2b37f4e8dc Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Wed, 19 Aug 2026 13:14:21 +0100 Subject: [PATCH 03/36] Update script to only run on dev --- .../consultations/management/commands/prepare_environment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/consultations/management/commands/prepare_environment.py b/backend/consultations/management/commands/prepare_environment.py index 627d35514..0e95273a3 100644 --- a/backend/consultations/management/commands/prepare_environment.py +++ b/backend/consultations/management/commands/prepare_environment.py @@ -5,12 +5,12 @@ class Command(BaseCommand): - help = "Prepare the environment: runs migrations on prod; resets and seeds the database and S3 on dev and preprod." + help = "Prepare the environment: runs migrations on prod; resets and seeds the database and S3 on dev only." def handle(self, *args, **options): environment = getattr(settings, "ENVIRONMENT", "").lower() - if environment == "prod": + if environment in ["prod", "preprod"]: self.stdout.write("Running migrate on prod.") call_command("migrate", verbosity=1) return From e770eb95f5b8ce7d74e995cf773ee5334ee75641 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Wed, 19 Aug 2026 13:16:50 +0100 Subject: [PATCH 04/36] Update comment in prepare_environment.py to reflect code changes --- .../consultations/management/commands/prepare_environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/consultations/management/commands/prepare_environment.py b/backend/consultations/management/commands/prepare_environment.py index 0e95273a3..c608d320a 100644 --- a/backend/consultations/management/commands/prepare_environment.py +++ b/backend/consultations/management/commands/prepare_environment.py @@ -5,7 +5,7 @@ class Command(BaseCommand): - help = "Prepare the environment: runs migrations on prod; resets and seeds the database and S3 on dev only." + help = "Prepare the environment: runs migrations on prod and preprod; resets and seeds the database and S3 on dev only." def handle(self, *args, **options): environment = getattr(settings, "ENVIRONMENT", "").lower() From 883394f4ca3d83fd10dabf7b4eb77073cdca8b60 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Thu, 20 Aug 2026 09:23:40 +0100 Subject: [PATCH 05/36] Update test --- backend/consultations/dummy_data.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/backend/consultations/dummy_data.py b/backend/consultations/dummy_data.py index 10f2b4861..72b23ce3d 100644 --- a/backend/consultations/dummy_data.py +++ b/backend/consultations/dummy_data.py @@ -37,13 +37,13 @@ { "CONSULTATION_NAME": "Dummy Consultation - Starting finalising themes", "CONSULTATION_CODE": "dummy-start-finalising-themes", - "CONSULTATION_STAGE": Consultation.Stage.THEME_SIGN_OFF, + "CONSULTATION_STAGE": Consultation.Stage.FINALISING_THEMES, "QUESTION_THEME_STATUS": Question.ThemeStatus.DRAFT, }, { "CONSULTATION_NAME": "Dummy Consultation - Finished finalising themes", "CONSULTATION_CODE": "dummy-finished-finalising-themes", - "CONSULTATION_STAGE": Consultation.Stage.THEME_SIGN_OFF, + "CONSULTATION_STAGE": Consultation.Stage.ASSIGNING_THEMES, "QUESTION_THEME_STATUS": Question.ThemeStatus.CONFIRMED, }, { @@ -242,9 +242,9 @@ def create_dummy_consultation( Create consultation with questions, responses and themes from JSON file. Creates relevant objects depending on stage and theme status: - SETUP: Consultation, Questions, Respondents, Responses (ready for finding themes) - - THEME_SIGN_OFF (DRAFT): + CandidateThemes + CandidateThemeResponses (finalising) - - THEME_SIGN_OFF (CONFIRMED): + CandidateThemes + SelectedThemes (ready for assignment) - - ANALYSIS: + SelectedThemes + ResponseAnnotations + - FINALISING_THEMES (DRAFT): + CandidateThemes + CandidateThemeResponses (finalising) + - ASSIGNING_THEMES (CONFIRMED): + CandidateThemes + SelectedThemes (ready for assignment) + - ANALYSIS: + CandidateThemes + SelectedThemes + ResponseAnnotations """ if HostingEnvironment.is_production(): raise RuntimeError("Dummy data generation should not be run in production") @@ -266,11 +266,13 @@ def create_dummy_consultation( questions_data = json.load(file) has_candidate_themes = consultation_stage in [ - Consultation.Stage.THEME_SIGN_OFF, + Consultation.Stage.FINALISING_THEMES, + Consultation.Stage.ASSIGNING_THEMES, Consultation.Stage.ANALYSIS, ] has_candidate_theme_responses = consultation_stage in [ - Consultation.Stage.THEME_SIGN_OFF, + Consultation.Stage.FINALISING_THEMES, + Consultation.Stage.ASSIGNING_THEMES, Consultation.Stage.ANALYSIS, ] has_default_selected_themes = consultation_stage == Consultation.Stage.ANALYSIS From a49eab1521e0a4aa1fbe63253bc0ef159e840f73 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Thu, 20 Aug 2026 09:45:32 +0100 Subject: [PATCH 06/36] Update test_generate_dummy_data.py and test_prepare_environment.py --- .../tests/commands/test_prepare_environment.py | 18 +++++++++--------- backend/tests/unit/test_generate_dummy_data.py | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/backend/tests/commands/test_prepare_environment.py b/backend/tests/commands/test_prepare_environment.py index b04500eff..bc57d5750 100644 --- a/backend/tests/commands/test_prepare_environment.py +++ b/backend/tests/commands/test_prepare_environment.py @@ -11,8 +11,9 @@ class TestPrepareEnvironment: @pytest.mark.django_db - def test_does_not_reset_on_prod(self, settings): - settings.ENVIRONMENT = "prod" + @pytest.mark.parametrize("environment", ["prod", "preprod"]) + def test_does_not_reset_on_prod_or_preprod(self, settings, environment): + settings.ENVIRONMENT = environment Consultation.objects.create(title="Should survive", code="KEEP_ME") call_command("prepare_environment") @@ -20,9 +21,8 @@ def test_does_not_reset_on_prod(self, settings): assert Consultation.objects.filter(code="KEEP_ME").exists() @pytest.mark.django_db - @pytest.mark.parametrize("environment", ["dev", "preprod"]) - def test_resets_and_seeds_db(self, settings, environment): - settings.ENVIRONMENT = environment + def test_resets_and_seeds_db(self, settings): + settings.ENVIRONMENT = "dev" Consultation.objects.create(title="Should be deleted", code="DELETE_ME") call_command("prepare_environment") @@ -32,14 +32,14 @@ def test_resets_and_seeds_db(self, settings, environment): # Consultations created at each stage assert Consultation.objects.filter(stage=Consultation.Stage.SETUP).exists() - assert Consultation.objects.filter(stage=Consultation.Stage.THEME_SIGN_OFF).count() == 2 + assert Consultation.objects.filter(stage=Consultation.Stage.FINALISING_THEMES).exists() + assert Consultation.objects.filter(stage=Consultation.Stage.ASSIGNING_THEMES).exists() assert Consultation.objects.filter(stage=Consultation.Stage.ANALYSIS).exists() # Admin user was created assert User.objects.filter(email="email@example.com", is_staff=True).exists() @pytest.mark.django_db - @pytest.mark.parametrize("environment", ["dev", "preprod"]) @mock_aws @patch("factories.embed_text", return_value=[0.0] * 3072) @patch( @@ -50,8 +50,8 @@ def test_resets_and_seeds_db(self, settings, environment): "consultations.management.commands.prepare_s3.HostingEnvironment.is_deployed", return_value=True, ) - def test_seeds_s3(self, _mock_deployed, _mock_prod, _mock_embed, settings, environment): - settings.ENVIRONMENT = environment + def test_seeds_s3(self, _mock_deployed, _mock_prod, _mock_embed, settings): + settings.ENVIRONMENT = "dev" settings.AWS_BUCKET_NAME = "test-bucket" conn = boto3.resource("s3", region_name="eu-west-2") diff --git a/backend/tests/unit/test_generate_dummy_data.py b/backend/tests/unit/test_generate_dummy_data.py index 9bf1e15f2..baed21ab5 100644 --- a/backend/tests/unit/test_generate_dummy_data.py +++ b/backend/tests/unit/test_generate_dummy_data.py @@ -58,7 +58,7 @@ def test_theme_sign_off_draft_has_candidate_themes_and_responses(): consultation = create_dummy_consultation(number_respondents=10, config=config) q4 = models.Question.objects.get(consultation=consultation, number=4) - assert consultation.stage == models.Consultation.Stage.THEME_SIGN_OFF + assert consultation.stage == models.Consultation.Stage.FINALISING_THEMES assert q4.theme_status == models.Question.ThemeStatus.DRAFT # Has candidate themes at multiple levels @@ -82,7 +82,7 @@ def test_theme_sign_off_confirmed_has_selected_themes(): consultation = create_dummy_consultation(number_respondents=10, config=config) q4 = models.Question.objects.get(consultation=consultation, number=4) - assert consultation.stage == models.Consultation.Stage.THEME_SIGN_OFF + assert consultation.stage == models.Consultation.Stage.ASSIGNING_THEMES assert q4.theme_status == models.Question.ThemeStatus.CONFIRMED # Has SelectedThemes From 496724a5d9f7c56aa1cdb744d8bbde7b2ff31852 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 21 Aug 2026 09:56:18 +0100 Subject: [PATCH 07/36] Correct test db setup using bypass logic and updated env vars --- .env.test | 2 +- Makefile | 10 +++------- .../management/commands/prepare_environment.py | 4 ++-- backend/start.sh | 6 ++++-- backend/tests/commands/test_prepare_environment.py | 2 +- docker-compose.yml | 2 +- 6 files changed, 12 insertions(+), 14 deletions(-) diff --git a/.env.test b/.env.test index a8abbf452..0f899508f 100644 --- a/.env.test +++ b/.env.test @@ -1,7 +1,7 @@ ENVIRONMENT=TEST DEBUG=True DJANGO_SECRET_KEY=dummy-key -DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres +DATABASE_URL=postgresql://postgres:postgres@postgres:5432/consult_e2e_test BATCH_JOB_QUEUE=dummy-queue BATCH_JOB_DEFINITION=dummy-definition AWS_REGION=eu-west-2 diff --git a/Makefile b/Makefile index fcefaf0c4..6c8adb40b 100644 --- a/Makefile +++ b/Makefile @@ -77,8 +77,7 @@ test-end-to-end: ## Run end-to-end tests with Playwright # Run the tests, then ALWAYS clean up, then re-raise the tests' exit status so CI # still fails on failure. Without this wrapper a failure mid-run (e.g. a service # health-check timeout or a failing test) would skip cleanup and leave - # docker-compose.override.yml behind, silently repointing every later - # `docker compose` command at the E2E database. + # the e2e test database behind or a stale Astro dev.json lock file. @$(MAKE) _run-e2e-tests; status=$$?; $(MAKE) _clean-e2e; exit $$status .PHONY: _run-e2e-tests @@ -95,11 +94,7 @@ _run-e2e-tests: user = User.objects.get(email='admin@example.com'); \ [c.users.add(user) for c in Consultation.objects.all()]" @echo "Starting services..." - @echo "services:" > docker-compose.override.yml - @echo " backend:" >> docker-compose.override.yml - @echo " environment:" >> docker-compose.override.yml - @echo " - DATABASE_URL=$(E2E_DB_URL)" >> docker-compose.override.yml - @docker compose down backend 2>/dev/null || true + @rm -f frontend/.astro/dev.json @docker compose up -d backend frontend @echo "Waiting for services to be ready..." @timeout 120 sh -c 'until curl -s http://localhost:3000 > /dev/null; do sleep 2; done' || \ @@ -116,6 +111,7 @@ _clean-e2e: ## Internal: always-run cleanup for test-end-to-end (drop test DB, r @echo "Cleaning up..." @docker exec -i $$(docker compose ps -q postgres) psql -U postgres -c "DROP DATABASE IF EXISTS consult_e2e_test;" 2>/dev/null || true @rm -f docker-compose.override.yml + @rm -f frontend/.astro/dev.json .PHONY: build-consultation-template diff --git a/backend/consultations/management/commands/prepare_environment.py b/backend/consultations/management/commands/prepare_environment.py index c608d320a..2ec32d452 100644 --- a/backend/consultations/management/commands/prepare_environment.py +++ b/backend/consultations/management/commands/prepare_environment.py @@ -10,8 +10,8 @@ class Command(BaseCommand): def handle(self, *args, **options): environment = getattr(settings, "ENVIRONMENT", "").lower() - if environment in ["prod", "preprod"]: - self.stdout.write("Running migrate on prod.") + if environment in ["prod", "preprod", "test"]: + self.stdout.write(f"Running migrate on {environment}.") call_command("migrate", verbosity=1) return diff --git a/backend/start.sh b/backend/start.sh index 660fad34b..fde00ca0b 100755 --- a/backend/start.sh +++ b/backend/start.sh @@ -1,7 +1,9 @@ #!/bin/sh -# Collect static files -venv/bin/django-admin collectstatic --noinput +# Collect static files (skip in test environment - no static serving needed) +if [ "$(echo "$ENVIRONMENT" | tr '[:upper:]' '[:lower:]')" != "test" ]; then + venv/bin/django-admin collectstatic --noinput +fi venv/bin/django-admin prepare_environment venv/bin/django-admin populate_history --auto --batchsize 1000 diff --git a/backend/tests/commands/test_prepare_environment.py b/backend/tests/commands/test_prepare_environment.py index bc57d5750..705e45a7d 100644 --- a/backend/tests/commands/test_prepare_environment.py +++ b/backend/tests/commands/test_prepare_environment.py @@ -11,7 +11,7 @@ class TestPrepareEnvironment: @pytest.mark.django_db - @pytest.mark.parametrize("environment", ["prod", "preprod"]) + @pytest.mark.parametrize("environment", ["prod", "preprod", "test"]) def test_does_not_reset_on_prod_or_preprod(self, settings, environment): settings.ENVIRONMENT = environment diff --git a/docker-compose.yml b/docker-compose.yml index e65b528a0..5ab3d26c5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,7 +71,7 @@ services: env_file: - .env environment: - - DATABASE_URL=postgresql://postgres:postgres@postgres:5432/postgres + - DATABASE_URL=${DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/postgres} - DJANGO_SETTINGS_MODULE=backend.settings.local - DOMAIN_NAME=localhost:3000 - MINIO_ENDPOINT=http://minio:9100 From a607cc8b725606ec84bc16df31acd463c28b53b2 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 21 Aug 2026 12:20:44 +0100 Subject: [PATCH 08/36] Change difference between backend and e2e tests --- .env.test | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.env.test b/.env.test index 0f899508f..a8abbf452 100644 --- a/.env.test +++ b/.env.test @@ -1,7 +1,7 @@ ENVIRONMENT=TEST DEBUG=True DJANGO_SECRET_KEY=dummy-key -DATABASE_URL=postgresql://postgres:postgres@postgres:5432/consult_e2e_test +DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres BATCH_JOB_QUEUE=dummy-queue BATCH_JOB_DEFINITION=dummy-definition AWS_REGION=eu-west-2 diff --git a/Makefile b/Makefile index 6c8adb40b..018ac66c4 100644 --- a/Makefile +++ b/Makefile @@ -95,7 +95,7 @@ _run-e2e-tests: [c.users.add(user) for c in Consultation.objects.all()]" @echo "Starting services..." @rm -f frontend/.astro/dev.json - @docker compose up -d backend frontend + @DATABASE_URL=$(E2E_DB_URL) docker compose up -d backend frontend @echo "Waiting for services to be ready..." @timeout 120 sh -c 'until curl -s http://localhost:3000 > /dev/null; do sleep 2; done' || \ (echo "Frontend failed to start" && docker compose logs frontend && exit 1) From a926b391187abbe4a4c5421fd7dd3f64b41fc420 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 21 Aug 2026 12:31:57 +0100 Subject: [PATCH 09/36] Fix test users that were created --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 018ac66c4..1a70d8c3c 100644 --- a/Makefile +++ b/Makefile @@ -88,10 +88,10 @@ _run-e2e-tests: @docker exec -i $$(docker compose ps -q postgres) psql -U postgres -c "CREATE DATABASE consult_e2e_test;" @echo "Initializing test data..." @docker compose run --rm -e DATABASE_URL=$(E2E_DB_URL) backend venv/bin/python manage.py migrate - @docker compose run --rm -e DATABASE_URL=$(E2E_DB_URL) -e ADMIN_USERS=admin@example.com backend venv/bin/python manage.py createadminusers + @docker compose run --rm -e DATABASE_URL=$(E2E_DB_URL) backend venv/bin/python manage.py createadminusers @docker compose run --rm -e DATABASE_URL=$(E2E_DB_URL) backend venv/bin/python manage.py shell -c \ "from authentication.models import User; from consultations.models import Consultation; \ - user = User.objects.get(email='admin@example.com'); \ + user = User.objects.get(email='email@example.com'); \ [c.users.add(user) for c in Consultation.objects.all()]" @echo "Starting services..." @rm -f frontend/.astro/dev.json From c6c89d793b8f3404d6f721779705e252d06147d3 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 21 Aug 2026 16:02:03 +0100 Subject: [PATCH 10/36] Fix final test that relied on a bucket var being populated, and tried to return the test user back to normalcy --- .env.test | 2 +- Makefile | 2 +- docker-compose.yml | 1 + e2e_tests/constants.ts | 4 ++-- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.env.test b/.env.test index a8abbf452..de3e888fe 100644 --- a/.env.test +++ b/.env.test @@ -19,4 +19,4 @@ FIND_THEMES_BATCH_JOB_QUEUE=i-dot-ai-dev-consult-sign-off-FARGATE-batch-job-queu FIND_THEMES_BATCH_JOB_DEFINITION=i-dot-ai-dev-consult-sign-off-FARGATE-batch-job-definition LITELLM_CONSULT_OPENAI_API_KEY=insert-litellm-api-key-here LLM_GATEWAY_URL=https://llm-gateway.i.ai.gov.uk -ADMIN_USERS=email@example.com +ADMIN_USERS=admin@example.com,policy@example.com diff --git a/Makefile b/Makefile index 1a70d8c3c..e3c631fce 100644 --- a/Makefile +++ b/Makefile @@ -91,7 +91,7 @@ _run-e2e-tests: @docker compose run --rm -e DATABASE_URL=$(E2E_DB_URL) backend venv/bin/python manage.py createadminusers @docker compose run --rm -e DATABASE_URL=$(E2E_DB_URL) backend venv/bin/python manage.py shell -c \ "from authentication.models import User; from consultations.models import Consultation; \ - user = User.objects.get(email='email@example.com'); \ + user = User.objects.get(email='admin@example.com'); \ [c.users.add(user) for c in Consultation.objects.all()]" @echo "Starting services..." @rm -f frontend/.astro/dev.json diff --git a/docker-compose.yml b/docker-compose.yml index 5ab3d26c5..d414beb64 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -75,6 +75,7 @@ services: - DJANGO_SETTINGS_MODULE=backend.settings.local - DOMAIN_NAME=localhost:3000 - MINIO_ENDPOINT=http://minio:9100 + - REDIS_HOST=redis depends_on: postgres: condition: service_healthy diff --git a/e2e_tests/constants.ts b/e2e_tests/constants.ts index f5199d8ca..e773fba01 100644 --- a/e2e_tests/constants.ts +++ b/e2e_tests/constants.ts @@ -1,5 +1,5 @@ export const testAccessToken = - "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImVtYWlsQGV4YW1wbGUuY29tIn0.k27nav4gbG-2lIArYInTqP1GUz2LRuzb3lWandMKRoY"; // pragma: allowlist secret + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImFkbWluQGV4YW1wbGUuY29tIn0.ignored"; // pragma: allowlist secret // Minio (local S3) config, sourced from the environment (loaded from .env) so // it stays in sync with what the backend uses. Playwright runs on the host, so @@ -8,4 +8,4 @@ export const testAccessToken = export const MINIO_ENDPOINT = process.env.MINIO_ENDPOINT ?? "http://localhost:9100"; export const MINIO_ACCESS_KEY = process.env.MINIO_ACCESS_KEY ?? "minioadmin"; export const MINIO_SECRET_KEY = process.env.MINIO_SECRET_KEY ?? "minioadmin"; -export const S3_BUCKET = process.env.AWS_BUCKET_NAME ?? "i-dot-ai-dev-consult-data"; +export const S3_BUCKET = process.env.AWS_BUCKET_NAME ?? "i-dot-ai-test-consult-data"; From c46dd53b73792979ea2954cc6b0f269913bc0d64 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 21 Aug 2026 16:10:13 +0100 Subject: [PATCH 11/36] Change email assertion in test_prepare_environment.py --- backend/tests/commands/test_prepare_environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/tests/commands/test_prepare_environment.py b/backend/tests/commands/test_prepare_environment.py index 705e45a7d..8160a64db 100644 --- a/backend/tests/commands/test_prepare_environment.py +++ b/backend/tests/commands/test_prepare_environment.py @@ -37,7 +37,7 @@ def test_resets_and_seeds_db(self, settings): assert Consultation.objects.filter(stage=Consultation.Stage.ANALYSIS).exists() # Admin user was created - assert User.objects.filter(email="email@example.com", is_staff=True).exists() + assert User.objects.filter(email="admin@example.com", is_staff=True).exists() @pytest.mark.django_db @mock_aws From 066d5708f3a662f5a46201b20eab0d49a75d0d69 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Wed, 26 Aug 2026 14:43:05 +0100 Subject: [PATCH 12/36] Change empty free text in dummy_data.py to None to match existing ingestion pipeline --- backend/consultations/dummy_data.py | 3 +- uv.lock | 66 ++++++++++++++--------------- 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/backend/consultations/dummy_data.py b/backend/consultations/dummy_data.py index 72b23ce3d..96a67ad37 100644 --- a/backend/consultations/dummy_data.py +++ b/backend/consultations/dummy_data.py @@ -164,7 +164,8 @@ def create_default_selected_themes(question): def create_response(respondent, question, free_text_answers): """Create and return a Response.""" - free_text = random.choice(free_text_answers) if question.has_free_text else None + raw = random.choice(free_text_answers) if question.has_free_text else None + free_text = raw if raw not in ("", "Not Provided", "-") else None return ResponseFactory(question=question, free_text=free_text, respondent=respondent) diff --git a/uv.lock b/uv.lock index 0d5a6dc95..3e9d043ee 100644 --- a/uv.lock +++ b/uv.lock @@ -341,7 +341,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "django-extensions", specifier = ">=4.1" }, - { name = "django-test-migrations", specifier = ">=1.5.0" }, + { name = "django-test-migrations", specifier = ">=1.6.0" }, { name = "django-types", specifier = ">=0.24.0" }, { name = "django-webtest", specifier = ">=1.9.14" }, { name = "freezegun", specifier = ">=1.5.5" }, @@ -350,9 +350,9 @@ dev = [ { name = "mypy", specifier = ">=2.3.0" }, { name = "pre-commit", specifier = ">=4.6.1" }, { name = "pydot", specifier = ">=4.0.1" }, - { name = "pytest-django", specifier = ">=4.12.0" }, + { name = "pytest-django", specifier = ">=4.14.0" }, { name = "pytest-lazy-fixtures", specifier = ">=1.4.0" }, - { name = "ruff", specifier = ">=0.16.1" }, + { name = "ruff", specifier = ">=0.16.2" }, ] [[package]] @@ -607,14 +607,14 @@ wheels = [ [[package]] name = "django-test-migrations" -version = "1.5.0" +version = "1.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/be/ed/7fc6f8e89d83565fc4acb93ae0a2387d885ac83cda445cb6c570f302bf55/django_test_migrations-1.5.0.tar.gz", hash = "sha256:1cbff04b1e82c5564a6f635284907b381cc11a2ff883adff46776d9126824f07", size = 20143, upload-time = "2025-04-18T10:15:38.547Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ff/9f/d8197fa49c4e6632e9e14867f2cf7be7d29cd93de611559610d63b5c3e30/django_test_migrations-1.6.0.tar.gz", hash = "sha256:a36e13d3f6cb139707896a8e5acefa0fbb24df0ee19e4f986e37faa8ef858819", size = 20798, upload-time = "2026-08-05T08:07:25.976Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c0/fe/38789c69f71adff9156bda7542d8fd05fcde1a109cf67bd7a1a139f8199f/django_test_migrations-1.5.0-py3-none-any.whl", hash = "sha256:96a08f085fc8bfaa53d44618341d82a2d22fd194c821cd81b147b66f0bec0da8", size = 25099, upload-time = "2025-04-18T10:15:37.16Z" }, + { url = "https://files.pythonhosted.org/packages/e5/ee/5c03667edfc91fddd06f457814581d25d7380eeca0da5ab03fc7c0955e9e/django_test_migrations-1.6.0-py3-none-any.whl", hash = "sha256:ecff93a5aa8bbcba9862a057c658c4ca35d92848253ca5cb47bad372c81d528f", size = 25695, upload-time = "2026-08-05T08:07:27.223Z" }, ] [[package]] @@ -2020,14 +2020,14 @@ wheels = [ [[package]] name = "pytest-django" -version = "4.12.0" +version = "4.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pytest" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/2b/db9a193df89e5660137f5428063bcc2ced7ad790003b26974adf5c5ceb3b/pytest_django-4.12.0.tar.gz", hash = "sha256:df94ec819a83c8979c8f6de13d9cdfbe76e8c21d39473cfe2b40c9fc9be3c758", size = 91156, upload-time = "2026-02-14T18:40:49.235Z" } +sdist = { url = "https://files.pythonhosted.org/packages/44/f6/3851312120c2bf2f19cafff931e75059aad1ba670703cd751e2fde9bc942/pytest_django-4.14.0.tar.gz", hash = "sha256:26787dd3f422cfbab8f55b80a776e2edea7a11092cb74e960bef1312515708ef", size = 94700, upload-time = "2026-08-10T14:13:08.319Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/83/a5/41d091f697c09609e7ef1d5d61925494e0454ebf51de7de05f0f0a728f1d/pytest_django-4.12.0-py3-none-any.whl", hash = "sha256:3ff300c49f8350ba2953b90297d23bf5f589db69545f56f1ec5f8cff5da83e85", size = 26123, upload-time = "2026-02-14T18:40:47.381Z" }, + { url = "https://files.pythonhosted.org/packages/9c/03/850bffad2b581c440ca51c039d74504d5a422c94bda0bdb8a8ba5068d48b/pytest_django-4.14.0-py3-none-any.whl", hash = "sha256:c533b08d89cc675efcd5398eea270b34547e35f9a3608e2c9748dd88428ea187", size = 27067, upload-time = "2026-08-10T14:13:06.998Z" }, ] [[package]] @@ -2246,27 +2246,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.16.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/70/25/7113f6d5498888c5fb7db34081cba7d5971c4cb1bfb26819966eee68f003/ruff-0.16.1.tar.gz", hash = "sha256:fedad7c801dabd3fb9741d76aca39246e6ddd9ca446a015875207bf19f1e6bc7", size = 4877500, upload-time = "2026-07-30T19:37:01.379Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/bd/694da69368e0973de65df2ddc73ab18d43c469d5963d9b150911de6bc513/ruff-0.16.1-py3-none-linux_armv6l.whl", hash = "sha256:58edb313b88f0c5460a26adf5f39a37a3be789494a15e3e411e35fa78b89f9a0", size = 10839126, upload-time = "2026-07-30T19:36:13.697Z" }, - { url = "https://files.pythonhosted.org/packages/3f/f0/b626e5d5bd0dd9576263658ef12885e2288afd1029a48e26ffed65ec1ac1/ruff-0.16.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fde5a99e2f97479af66edd6622c6d5a2a7592c77cf4153d9e4428f5eeb55b60c", size = 11070253, upload-time = "2026-07-30T19:36:17.14Z" }, - { url = "https://files.pythonhosted.org/packages/83/63/f40acfb6b35b88623e71684942b552c3edd96035f5d98f313815f7b277de/ruff-0.16.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e0d4c20532fca4f7fa609369161d968dd28f65d83dabbd61d8e9c7edbf7001f6", size = 10561425, upload-time = "2026-07-30T19:36:20.04Z" }, - { url = "https://files.pythonhosted.org/packages/aa/dd/14ec0e9c2b4d315547dd38765004b4863e354e1b52cb308272215d9f6f6d/ruff-0.16.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30affbcedf59ad5703d9c91f82266e02b47739f797e1a7b6e158e5526a6dae38", size = 10948879, upload-time = "2026-07-30T19:36:22.476Z" }, - { url = "https://files.pythonhosted.org/packages/33/e9/9d870cbae575030fdef595f04b4b97573c525b5497cce4f4498cf2f85446/ruff-0.16.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:24e9c631573cbca9d20f1283f8f479b2afa4a8503504822bd71a293889f16743", size = 10643691, upload-time = "2026-07-30T19:36:24.914Z" }, - { url = "https://files.pythonhosted.org/packages/c4/09/12743d544e2173f53ecd27217c65f90d2bc0f8424a66a60339e56bbc0457/ruff-0.16.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b41bdd48fb420987a9b5212e4957c26ad4abce401fa9ea9d4d85843727945f4f", size = 11435354, upload-time = "2026-07-30T19:36:28.447Z" }, - { url = "https://files.pythonhosted.org/packages/7f/89/a1652b2daee52083c9554a6333b678a8b01d0400f976827bb87857f9449a/ruff-0.16.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b0d1e1393b7648079e13669de1c1f4fde06d4583e84d8fd5c1551e0a77a2aa75", size = 12259033, upload-time = "2026-07-30T19:36:31.326Z" }, - { url = "https://files.pythonhosted.org/packages/16/96/ecdcb8c54ee7b123b487f807eb014e6e019155a0b81dfb669acd52f28ce3/ruff-0.16.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:07bf434b1c95f4e093be4532068ef4fcf00924eb2ade8796075980902d6fd54a", size = 11667981, upload-time = "2026-07-30T19:36:34.394Z" }, - { url = "https://files.pythonhosted.org/packages/cd/90/c52e12e0d862e9572f2a33aa227409143520abe53111e9a6babbac7b4af8/ruff-0.16.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39897739f112253ee4fdd2e8aa9a4f9ded99fb2be367d5f31dfa4ded6025584c", size = 11468183, upload-time = "2026-07-30T19:36:37.339Z" }, - { url = "https://files.pythonhosted.org/packages/2c/6b/4ffb7ad1d83eb16cf8cbb3c8815d3f11c88460fd162d4b372a2059be1c2a/ruff-0.16.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:82ae3c0c0d74daf17b968a10b7b3bb3ef297ab7de0c1f749646b25e690ccb150", size = 11470071, upload-time = "2026-07-30T19:36:39.91Z" }, - { url = "https://files.pythonhosted.org/packages/9c/72/32ae7db4c0b5e32ab611787caa19d1546800676d79f7483b7100a3561bf4/ruff-0.16.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4d5f2ed10f8242d83fc08d521301089364e3375375705356f20c0e31606ef3ef", size = 10919503, upload-time = "2026-07-30T19:36:42.65Z" }, - { url = "https://files.pythonhosted.org/packages/f7/ca/3d901ba6ad6fc38da39c3448fc6c59ac945679293a17c3ceb6d6c1cba13e/ruff-0.16.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a4665b309891f83f3e3c25447935f1213e9abbd4b5640af7a1f2def9f8d413c1", size = 10649861, upload-time = "2026-07-30T19:36:45.18Z" }, - { url = "https://files.pythonhosted.org/packages/92/79/894ef1ced26552d5f8c9cf6d85b0687840e1128c55aeab7b9c2d54a0d880/ruff-0.16.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:26e9ca5c9bc3971f20d3cf18a957f52ffd6a5f6564ff15c4912a144dcac22494", size = 11148137, upload-time = "2026-07-30T19:36:47.936Z" }, - { url = "https://files.pythonhosted.org/packages/2d/69/3609a09fa1cb46cc28b762363e440a354204e5dff01bd0c8d7437874d6b9/ruff-0.16.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:67e1e1e3fa4f0c82f0e36d4cd61e661f6e7a6196cb1aa92fe0828fa7b8f257cd", size = 11559211, upload-time = "2026-07-30T19:36:50.448Z" }, - { url = "https://files.pythonhosted.org/packages/fc/8a/fb22af2fd78a736e241fabf67e30ce1799a64244026377a49e133af90762/ruff-0.16.1-py3-none-win32.whl", hash = "sha256:d31765e131295b8445caf301e3e8a85b34d1b9b211b4109b7ba457888b051806", size = 10838258, upload-time = "2026-07-30T19:36:53.298Z" }, - { url = "https://files.pythonhosted.org/packages/d4/35/e57fd9fb5d423961df087a00b12d42c0a830288dc2f3b45ecca299158b4f/ruff-0.16.1-py3-none-win_amd64.whl", hash = "sha256:09b05e8b90c2cb06ad63464350e7a45e8e44a2dfe52072ebfba6666ca8d3f596", size = 11961111, upload-time = "2026-07-30T19:36:56.107Z" }, - { url = "https://files.pythonhosted.org/packages/cb/46/240ea004bf6dc4feb40e9832f2205a476a47dd5b8a3f8211a5fc5f95e20e/ruff-0.16.1-py3-none-win_arm64.whl", hash = "sha256:dbaadaac38c70239f056d306b7476f246b0bf000fa6b3876402acbf5b227eaf8", size = 11309414, upload-time = "2026-07-30T19:36:58.79Z" }, +version = "0.16.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/73/e1/4508a569211b35599016e84ba65c1a992b7a4004b4b6c4bea02a851cba1b/ruff-0.16.2.tar.gz", hash = "sha256:c3d7828d12e8927a6fc65fe38e2c2541b9e762d360a1786d752cb1b8883b3c9c", size = 4885811, upload-time = "2026-08-07T13:31:01.432Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/57/db19951540f98859c956b50bdb4d31089b4d91e9f15e2968e7d5193806d5/ruff-0.16.2-py3-none-linux_armv6l.whl", hash = "sha256:3c8de4cf2181f01d57946d87d777aa52916976fc09942aed89938fab5e013318", size = 10847925, upload-time = "2026-08-07T13:30:14.468Z" }, + { url = "https://files.pythonhosted.org/packages/13/5a/995fe85a8470d3e391ac0f7fa8054bb454eaf33ee138196d6172ed1079c0/ruff-0.16.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9a48cc05c6fbc811ca81b5d7ba95375affea6582d1b8024e455e41afbbf55344", size = 11072662, upload-time = "2026-08-07T13:30:18.143Z" }, + { url = "https://files.pythonhosted.org/packages/32/53/370d767c61c71a971a4ace36703a7ecd8c393956349a7325d7fab2b56827/ruff-0.16.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a2c0d14fcbb26c91f0f867a6dc9bd71bbc30b1b6151829c884f23faeab2e5700", size = 10566771, upload-time = "2026-08-07T13:30:20.899Z" }, + { url = "https://files.pythonhosted.org/packages/85/d6/9d96948caf5a632be62d62202d5ec914d6856f204fd79eb036e5915e79ea/ruff-0.16.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:335c621622c4650330be50842561c6586ac6971bb8ab5407fe34dcc9efb16bbe", size = 10975825, upload-time = "2026-08-07T13:30:23.517Z" }, + { url = "https://files.pythonhosted.org/packages/3b/92/ea87129b3414acb0b5770563779c51804d37ac67675c7ba35447ddb14773/ruff-0.16.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:20e66910f2c37cc753f9ef6580c914a621b80c4fa3549d3e3521e29d0f5bfc3f", size = 10649437, upload-time = "2026-08-07T13:30:26.097Z" }, + { url = "https://files.pythonhosted.org/packages/ac/43/f8f291dcd4af5bb7872b74fdfa41a7cd7c856ca1d4069670971cf1b9f5cb/ruff-0.16.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7e36fbfba65510548156902bcf1350a979a958ce0347ce0f90d73894036b39f", size = 11446761, upload-time = "2026-08-07T13:30:28.752Z" }, + { url = "https://files.pythonhosted.org/packages/71/4a/ef991fb2fcf516ab71f0808adcdd8da5e18c8cde447f4ceaf5f47a5132a5/ruff-0.16.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f0eab35f80df8f134aae5d1630e751901321d317cc8e50dc39e36fa3ed34cd12", size = 12336364, upload-time = "2026-08-07T13:30:31.468Z" }, + { url = "https://files.pythonhosted.org/packages/f3/24/f615e74f307e6ca0e56a482872477b856c70d530aa356abfb6dfe5ca8a80/ruff-0.16.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40ea8c0594feb894e89c8c61ab9c103d38b0ea72dfde6c594107147ca31b1140", size = 11630720, upload-time = "2026-08-07T13:30:34.426Z" }, + { url = "https://files.pythonhosted.org/packages/c5/d3/8ef50149e8412a77f7ab409efdef0e2b23803707a3863da4fc64cb23d459/ruff-0.16.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab3d62dde0b19facdd632008cc4827fc28ada7736c6bd35ab6f1050f0bfed53f", size = 11466130, upload-time = "2026-08-07T13:30:36.958Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a7/a19334985c4dea8c381981fa252cd854c7ee52dc4b1686dc16f4a911c702/ruff-0.16.2-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:e43e1f5b8388da9eca1b9e88328d47a5cec794633ccf6f7484ac2dd15eee92c0", size = 11523634, upload-time = "2026-08-07T13:30:39.822Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6c/96d192b0e742412ceda08c0a50f9669b253dde9fd6a60ea1a10c9fa79a63/ruff-0.16.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c24788a980581e1d7ea3a0cbe4344c4fbeb0a6a9b1f4713aa46bb104f8294690", size = 10949807, upload-time = "2026-08-07T13:30:42.745Z" }, + { url = "https://files.pythonhosted.org/packages/fa/51/e26599ceca11e79ee255c7df515995561edf87e9ca1893284e44d98f5a86/ruff-0.16.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:81806b08329130005dd4a8a8394a0c9da8c6f4cafb16ba438d2a2ee6a18bedf1", size = 10646891, upload-time = "2026-08-07T13:30:45.522Z" }, + { url = "https://files.pythonhosted.org/packages/68/01/800c4b1f97bc8d7c6029e06b1f20473a3cf1e13c4933d8f3342add83fc55/ruff-0.16.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:4ce4e02bad779bef557f541a1b31f20d6abeae1cc05ed1b1ac019d4ffd1044c8", size = 11162063, upload-time = "2026-08-07T13:30:48.131Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d0/1477ea50fc5a0d4b0b71d1d63d50770bdd794d90b43e37a7618e63ec9894/ruff-0.16.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e0422abdf70070255fc4073ce9dfc814cc03db577013761ddd09bc1e4a9a4fbd", size = 11556038, upload-time = "2026-08-07T13:30:50.686Z" }, + { url = "https://files.pythonhosted.org/packages/b8/76/a7776f32048d991e16d4fa8ff91790b877342d3596cc3ed04acdbf1aaedc/ruff-0.16.2-py3-none-win32.whl", hash = "sha256:bf3a63d78fb39f4bf5ac8ae52051c5520505301abe19ba4e204c453b3f09bb0b", size = 10872850, upload-time = "2026-08-07T13:30:53.471Z" }, + { url = "https://files.pythonhosted.org/packages/00/0d/929c800d920e61397d82a01b60bffc68da3052c17d31de59efaad2e4ed75/ruff-0.16.2-py3-none-win_amd64.whl", hash = "sha256:bcabe2f6d0fc7819f1431793005af4e4de7371927d037345bf941252b195b9fa", size = 12023338, upload-time = "2026-08-07T13:30:56.193Z" }, + { url = "https://files.pythonhosted.org/packages/5b/6c/93e26c22c5f78ff87363e07da49c84955affbeb1098bd1936bf3b3f293bf/ruff-0.16.2-py3-none-win_arm64.whl", hash = "sha256:d614e95cedf38a2053fd351c55b103ba30d017d61688fdbfd40ee0412852a99f", size = 11374065, upload-time = "2026-08-07T13:30:58.775Z" }, ] [[package]] @@ -2697,11 +2697,11 @@ wheels = [ [[package]] name = "typing-extensions" -version = "4.15.0" +version = "4.16.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, ] [[package]] From de9a730b30cf1c3ae36c13b5c64018de604f4ad2 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Thu, 27 Aug 2026 13:14:26 +0100 Subject: [PATCH 13/36] Correct comment to remove preprod from prepare_environment.py --- .../consultations/management/commands/prepare_environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/consultations/management/commands/prepare_environment.py b/backend/consultations/management/commands/prepare_environment.py index 2ec32d452..aa5a4e8a7 100644 --- a/backend/consultations/management/commands/prepare_environment.py +++ b/backend/consultations/management/commands/prepare_environment.py @@ -5,7 +5,7 @@ class Command(BaseCommand): - help = "Prepare the environment: runs migrations on prod and preprod; resets and seeds the database and S3 on dev only." + help = "Prepare the environment: runs migrations on prod; resets and seeds the database and S3 on dev only." def handle(self, *args, **options): environment = getattr(settings, "ENVIRONMENT", "").lower() From 939e34fc5b7295ce0a24a538844eac1f1ab58321 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Thu, 27 Aug 2026 13:15:33 +0100 Subject: [PATCH 14/36] Correct timestamp and csv import in prepare_s3.py --- .../management/commands/prepare_s3.py | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/backend/consultations/management/commands/prepare_s3.py b/backend/consultations/management/commands/prepare_s3.py index 362ff08fa..ac87a2e6a 100644 --- a/backend/consultations/management/commands/prepare_s3.py +++ b/backend/consultations/management/commands/prepare_s3.py @@ -1,4 +1,6 @@ +import csv import datetime +import io import json from zoneinfo import ZoneInfo @@ -16,9 +18,6 @@ ) from hosting_environment import HostingEnvironment -TIMESTAMP = datetime.datetime.now(tz=ZoneInfo("Europe/London")).date() - - def _to_jsonl(records): return "\n".join(json.dumps(r) for r in records) @@ -197,12 +196,12 @@ def _build_evidence_rich(): def _build_themes_csv(question_data): """Build themes.csv content (selected themes) for the assign-themes batch job.""" themes = _build_themes_json(question_data) - lines = ["Theme Name,Theme Description"] + buf = io.StringIO() + writer = csv.writer(buf) + writer.writerow(["Theme Name", "Theme Description"]) for theme in themes: - name = theme["theme_name"] - description = theme["theme_description"] - lines.append(f"{name},{description}") - return "\n".join(lines) + writer.writerow([theme["theme_name"], theme["theme_description"]]) + return buf.getvalue() class Command(BaseCommand): @@ -219,15 +218,16 @@ def handle(self, *args, **options): s3_client = boto3.client("s3") bucket = settings.AWS_BUCKET_NAME + timestamp = datetime.datetime.now(tz=ZoneInfo("Europe/London")).date() self._delete_existing_data(s3_client, bucket) questions_data = _load_questions() # S3-only consultation (no DB record) - self._seed_consultation(s3_client, bucket, "dummy-s3-only", questions_data) + self._seed_consultation(s3_client, bucket, "dummy-s3-only", questions_data, timestamp) # SETUP — inputs only - self._seed_consultation(s3_client, bucket, "dummy-setup", questions_data) + self._seed_consultation(s3_client, bucket, "dummy-setup", questions_data, timestamp) # Starting finalising themes — has clustered themes + candidate theme mappings self._seed_consultation( @@ -235,6 +235,7 @@ def handle(self, *args, **options): bucket, "dummy-start-finalising-themes", questions_data, + timestamp, include_clustered_themes=True, include_candidate_theme_mappings=True, ) @@ -245,6 +246,7 @@ def handle(self, *args, **options): bucket, "dummy-finished-finalising-themes", questions_data, + timestamp, include_clustered_themes=True, include_candidate_theme_mappings=True, include_themes_csv=True, @@ -256,6 +258,7 @@ def handle(self, *args, **options): bucket, "dummy-analysis", questions_data, + timestamp, include_clustered_themes=True, include_candidate_theme_mappings=True, include_mapping_outputs=True, @@ -287,6 +290,7 @@ def _seed_consultation( bucket, code, questions_data, + timestamp, include_clustered_themes=False, include_candidate_theme_mappings=False, include_mapping_outputs=False, @@ -338,7 +342,7 @@ def _seed_consultation( continue if include_clustered_themes: - key = f"{prefix}/outputs/sign_off/{TIMESTAMP}/question_part_{q_num}/clustered_themes.json" + key = f"{prefix}/outputs/sign_off/{timestamp}/question_part_{q_num}/clustered_themes.json" s3_client.put_object( Bucket=bucket, Key=key, @@ -346,7 +350,7 @@ def _seed_consultation( ) if include_candidate_theme_mappings: - out_prefix = f"{prefix}/outputs/mapping/{TIMESTAMP}/question_part_{q_num}" + out_prefix = f"{prefix}/outputs/mapping/{timestamp}/question_part_{q_num}" themes = _build_candidate_themes_json(question_data) s3_client.put_object( Bucket=bucket, @@ -367,7 +371,7 @@ def _seed_consultation( ) if include_mapping_outputs: - out_prefix = f"{prefix}/outputs/mapping/{TIMESTAMP}/question_part_{q_num}" + out_prefix = f"{prefix}/outputs/mapping/{timestamp}/question_part_{q_num}" themes = _build_themes_json(question_data) s3_client.put_object( Bucket=bucket, From ff1e7384e7d17b3b0fe5c7699e464d560157de54 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Thu, 27 Aug 2026 13:19:48 +0100 Subject: [PATCH 15/36] Organise imports in prepare_s3.py --- backend/consultations/management/commands/prepare_s3.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/consultations/management/commands/prepare_s3.py b/backend/consultations/management/commands/prepare_s3.py index ac87a2e6a..3b6d36033 100644 --- a/backend/consultations/management/commands/prepare_s3.py +++ b/backend/consultations/management/commands/prepare_s3.py @@ -18,6 +18,7 @@ ) from hosting_environment import HostingEnvironment + def _to_jsonl(records): return "\n".join(json.dumps(r) for r in records) From e20fb80c30c624533ca41e3ec1ca9f01f250079d Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Thu, 27 Aug 2026 13:24:22 +0100 Subject: [PATCH 16/36] Added preprod gate for s3 wipe in prepare_s3.py and preprod environment check in hosting_environment.py --- backend/consultations/management/commands/prepare_s3.py | 7 ++++--- backend/hosting_environment.py | 5 +++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/backend/consultations/management/commands/prepare_s3.py b/backend/consultations/management/commands/prepare_s3.py index 3b6d36033..89f2b997b 100644 --- a/backend/consultations/management/commands/prepare_s3.py +++ b/backend/consultations/management/commands/prepare_s3.py @@ -207,14 +207,15 @@ def _build_themes_csv(question_data): class Command(BaseCommand): help = "Reset and seed S3 with dummy consultation data matching the DB. Only runs on deployed non-prod environments." + environment = getattr(settings, "ENVIRONMENT", "").lower() def handle(self, *args, **options): - if HostingEnvironment.is_production(): - self.stdout.write("Skipping S3 seed on production.") + if HostingEnvironment.is_production() or HostingEnvironment.is_preprod_environment(): + self.stdout.write(f"Skipping S3 seed on {self.environment} environment.") return if not HostingEnvironment.is_deployed(): - self.stdout.write("Skipping S3 seed on local environment (no real S3 bucket).") + self.stdout.write(f"Skipping S3 seed on {self.environment} environment (no real S3 bucket).") return s3_client = boto3.client("s3") diff --git a/backend/hosting_environment.py b/backend/hosting_environment.py index 83d6375c3..f0f05cb84 100644 --- a/backend/hosting_environment.py +++ b/backend/hosting_environment.py @@ -22,6 +22,11 @@ def is_deployed() -> bool: def is_production() -> bool: return env.str("ENVIRONMENT", "").upper() == "PROD" + @staticmethod + def is_preprod_environment() -> bool: + environment = env.str("ENVIRONMENT", "").upper() + return environment == "PREPROD" + @staticmethod def is_development_environment() -> bool: environment = env.str("ENVIRONMENT", "").upper() From b831521a4bfb3b6209fdc4d24e3b80272ff1e2ab Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Thu, 27 Aug 2026 13:32:40 +0100 Subject: [PATCH 17/36] Flip gated check so only dev gets wiped to protect against mistyped envs being wiped --- .../consultations/management/commands/prepare_environment.py | 2 +- backend/tests/commands/test_prepare_environment.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/consultations/management/commands/prepare_environment.py b/backend/consultations/management/commands/prepare_environment.py index aa5a4e8a7..bdee6c043 100644 --- a/backend/consultations/management/commands/prepare_environment.py +++ b/backend/consultations/management/commands/prepare_environment.py @@ -10,7 +10,7 @@ class Command(BaseCommand): def handle(self, *args, **options): environment = getattr(settings, "ENVIRONMENT", "").lower() - if environment in ["prod", "preprod", "test"]: + if environment != "dev": self.stdout.write(f"Running migrate on {environment}.") call_command("migrate", verbosity=1) return diff --git a/backend/tests/commands/test_prepare_environment.py b/backend/tests/commands/test_prepare_environment.py index 8160a64db..13fd5095f 100644 --- a/backend/tests/commands/test_prepare_environment.py +++ b/backend/tests/commands/test_prepare_environment.py @@ -11,8 +11,8 @@ class TestPrepareEnvironment: @pytest.mark.django_db - @pytest.mark.parametrize("environment", ["prod", "preprod", "test"]) - def test_does_not_reset_on_prod_or_preprod(self, settings, environment): + @pytest.mark.parametrize("environment", ["prod", "preprod", "test", "", "unknown", "staging"]) + def test_does_not_reset_on_non_dev(self, settings, environment): settings.ENVIRONMENT = environment Consultation.objects.create(title="Should survive", code="KEEP_ME") From e4236981826770a692dc773c04af94a451dffc56 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Thu, 27 Aug 2026 14:10:15 +0100 Subject: [PATCH 18/36] More gate check improvements --- .../commands/prepare_environment.py | 4 ++- .../management/commands/prepare_s3.py | 7 +++-- .../commands/test_prepare_environment.py | 29 +++++++++++++++++-- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/backend/consultations/management/commands/prepare_environment.py b/backend/consultations/management/commands/prepare_environment.py index bdee6c043..b5441bec5 100644 --- a/backend/consultations/management/commands/prepare_environment.py +++ b/backend/consultations/management/commands/prepare_environment.py @@ -3,6 +3,8 @@ from django.core.management.base import BaseCommand from django.db import connections +from hosting_environment import HostingEnvironment + class Command(BaseCommand): help = "Prepare the environment: runs migrations on prod; resets and seeds the database and S3 on dev only." @@ -10,7 +12,7 @@ class Command(BaseCommand): def handle(self, *args, **options): environment = getattr(settings, "ENVIRONMENT", "").lower() - if environment != "dev": + if environment != "dev" or not HostingEnvironment.is_deployed(): self.stdout.write(f"Running migrate on {environment}.") call_command("migrate", verbosity=1) return diff --git a/backend/consultations/management/commands/prepare_s3.py b/backend/consultations/management/commands/prepare_s3.py index 89f2b997b..0f08ee3a2 100644 --- a/backend/consultations/management/commands/prepare_s3.py +++ b/backend/consultations/management/commands/prepare_s3.py @@ -207,15 +207,16 @@ def _build_themes_csv(question_data): class Command(BaseCommand): help = "Reset and seed S3 with dummy consultation data matching the DB. Only runs on deployed non-prod environments." - environment = getattr(settings, "ENVIRONMENT", "").lower() def handle(self, *args, **options): + environment = getattr(settings, "ENVIRONMENT", "").lower() + if HostingEnvironment.is_production() or HostingEnvironment.is_preprod_environment(): - self.stdout.write(f"Skipping S3 seed on {self.environment} environment.") + self.stdout.write(f"Skipping S3 seed on {environment} environment.") return if not HostingEnvironment.is_deployed(): - self.stdout.write(f"Skipping S3 seed on {self.environment} environment (no real S3 bucket).") + self.stdout.write(f"Skipping S3 seed on {environment} environment (no real S3 bucket).") return s3_client = boto3.client("s3") diff --git a/backend/tests/commands/test_prepare_environment.py b/backend/tests/commands/test_prepare_environment.py index 13fd5095f..ce14b5716 100644 --- a/backend/tests/commands/test_prepare_environment.py +++ b/backend/tests/commands/test_prepare_environment.py @@ -21,7 +21,24 @@ def test_does_not_reset_on_non_dev(self, settings, environment): assert Consultation.objects.filter(code="KEEP_ME").exists() @pytest.mark.django_db - def test_resets_and_seeds_db(self, settings): + @patch( + "consultations.management.commands.prepare_environment.HostingEnvironment.is_deployed", + return_value=False, + ) + def test_does_not_reset_when_dev_but_not_deployed(self, _mock_deployed, settings): + settings.ENVIRONMENT = "dev" + + Consultation.objects.create(title="Should survive", code="KEEP_ME") + call_command("prepare_environment") + + assert Consultation.objects.filter(code="KEEP_ME").exists() + + @pytest.mark.django_db + @patch( + "consultations.management.commands.prepare_environment.HostingEnvironment.is_deployed", + return_value=True, + ) + def test_resets_and_seeds_db(self, _mock_deployed, settings): settings.ENVIRONMENT = "dev" Consultation.objects.create(title="Should be deleted", code="DELETE_ME") @@ -50,7 +67,15 @@ def test_resets_and_seeds_db(self, settings): "consultations.management.commands.prepare_s3.HostingEnvironment.is_deployed", return_value=True, ) - def test_seeds_s3(self, _mock_deployed, _mock_prod, _mock_embed, settings): + @patch( + "consultations.management.commands.prepare_environment.HostingEnvironment.is_deployed", + return_value=True, + ) + def test_seeds_s3(self, _mock_env_deployed, _mock_s3_deployed, _mock_prod, _mock_embed, settings): + # prepare_environment calls prepare_s3 from the dev branch; prepare_s3 itself guards + # on is_deployed(). We patch both here (and mock S3 via moto) so the test runs + # without a real S3 bucket, while still exercising the full prepare_environment → prepare_s3 + # call chain on a dev environment. settings.ENVIRONMENT = "dev" settings.AWS_BUCKET_NAME = "test-bucket" From 5a72343192aef96e126dcdc7d50497e7429e0da6 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 28 Aug 2026 09:08:16 +0100 Subject: [PATCH 19/36] Add mock for text embedding in test_prepare_environment.py --- backend/tests/commands/test_prepare_environment.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/tests/commands/test_prepare_environment.py b/backend/tests/commands/test_prepare_environment.py index ce14b5716..344d854b1 100644 --- a/backend/tests/commands/test_prepare_environment.py +++ b/backend/tests/commands/test_prepare_environment.py @@ -34,11 +34,12 @@ def test_does_not_reset_when_dev_but_not_deployed(self, _mock_deployed, settings assert Consultation.objects.filter(code="KEEP_ME").exists() @pytest.mark.django_db + @patch("factories.embed_text", return_value=[0.0] * 3072) @patch( "consultations.management.commands.prepare_environment.HostingEnvironment.is_deployed", return_value=True, ) - def test_resets_and_seeds_db(self, _mock_deployed, settings): + def test_resets_and_seeds_db(self, _mock_deployed, _mock_embed, settings): settings.ENVIRONMENT = "dev" Consultation.objects.create(title="Should be deleted", code="DELETE_ME") From f8a9a1727126b68573553243d5e8416c359ee822 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 28 Aug 2026 09:22:47 +0100 Subject: [PATCH 20/36] Add mock aws in test_prepare_environment.py --- .../commands/test_prepare_environment.py | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/backend/tests/commands/test_prepare_environment.py b/backend/tests/commands/test_prepare_environment.py index 344d854b1..1d845a1ed 100644 --- a/backend/tests/commands/test_prepare_environment.py +++ b/backend/tests/commands/test_prepare_environment.py @@ -34,13 +34,30 @@ def test_does_not_reset_when_dev_but_not_deployed(self, _mock_deployed, settings assert Consultation.objects.filter(code="KEEP_ME").exists() @pytest.mark.django_db + @mock_aws @patch("factories.embed_text", return_value=[0.0] * 3072) + @patch( + "consultations.management.commands.prepare_s3.HostingEnvironment.is_production", + return_value=False, + ) + @patch( + "consultations.management.commands.prepare_s3.HostingEnvironment.is_deployed", + return_value=True, + ) @patch( "consultations.management.commands.prepare_environment.HostingEnvironment.is_deployed", return_value=True, ) - def test_resets_and_seeds_db(self, _mock_deployed, _mock_embed, settings): + def test_resets_and_seeds_db( + self, _mock_env_deployed, _mock_s3_deployed, _mock_s3_prod, _mock_embed, settings + ): settings.ENVIRONMENT = "dev" + settings.AWS_BUCKET_NAME = "test-bucket" + + boto3.resource("s3", region_name="eu-west-2").create_bucket( + Bucket="test-bucket", + CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, + ) Consultation.objects.create(title="Should be deleted", code="DELETE_ME") call_command("prepare_environment") @@ -72,7 +89,9 @@ def test_resets_and_seeds_db(self, _mock_deployed, _mock_embed, settings): "consultations.management.commands.prepare_environment.HostingEnvironment.is_deployed", return_value=True, ) - def test_seeds_s3(self, _mock_env_deployed, _mock_s3_deployed, _mock_prod, _mock_embed, settings): + def test_seeds_s3( + self, _mock_env_deployed, _mock_s3_deployed, _mock_prod, _mock_embed, settings + ): # prepare_environment calls prepare_s3 from the dev branch; prepare_s3 itself guards # on is_deployed(). We patch both here (and mock S3 via moto) so the test runs # without a real S3 bucket, while still exercising the full prepare_environment → prepare_s3 From 510e384e445dc370f4703daed52f5535354f2a72 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 28 Aug 2026 10:00:54 +0100 Subject: [PATCH 21/36] Fix double-write in prepare_s3 and add preprod skip coverage --- .../management/commands/prepare_s3.py | 2 +- .../commands/test_prepare_environment.py | 6 +++- backend/tests/commands/test_prepare_s3.py | 29 +++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/backend/consultations/management/commands/prepare_s3.py b/backend/consultations/management/commands/prepare_s3.py index 0f08ee3a2..55129a248 100644 --- a/backend/consultations/management/commands/prepare_s3.py +++ b/backend/consultations/management/commands/prepare_s3.py @@ -352,7 +352,7 @@ def _seed_consultation( Body=json.dumps(_build_clustered_themes(question_data)), ) - if include_candidate_theme_mappings: + if include_candidate_theme_mappings and not include_mapping_outputs: out_prefix = f"{prefix}/outputs/mapping/{timestamp}/question_part_{q_num}" themes = _build_candidate_themes_json(question_data) s3_client.put_object( diff --git a/backend/tests/commands/test_prepare_environment.py b/backend/tests/commands/test_prepare_environment.py index 1d845a1ed..1fae445d4 100644 --- a/backend/tests/commands/test_prepare_environment.py +++ b/backend/tests/commands/test_prepare_environment.py @@ -77,6 +77,10 @@ def test_resets_and_seeds_db( @pytest.mark.django_db @mock_aws @patch("factories.embed_text", return_value=[0.0] * 3072) + @patch( + "consultations.management.commands.prepare_s3.HostingEnvironment.is_preprod_environment", + return_value=False, + ) @patch( "consultations.management.commands.prepare_s3.HostingEnvironment.is_production", return_value=False, @@ -90,7 +94,7 @@ def test_resets_and_seeds_db( return_value=True, ) def test_seeds_s3( - self, _mock_env_deployed, _mock_s3_deployed, _mock_prod, _mock_embed, settings + self, _mock_env_deployed, _mock_s3_deployed, _mock_prod, _mock_preprod, _mock_embed, settings ): # prepare_environment calls prepare_s3 from the dev branch; prepare_s3 itself guards # on is_deployed(). We patch both here (and mock S3 via moto) so the test runs diff --git a/backend/tests/commands/test_prepare_s3.py b/backend/tests/commands/test_prepare_s3.py index ee3f48309..ca344181a 100644 --- a/backend/tests/commands/test_prepare_s3.py +++ b/backend/tests/commands/test_prepare_s3.py @@ -117,6 +117,35 @@ def test_seeds_s3_with_data_for_each_stage(self, _mock_deployed, _mock_prod, set "dummy-analysis/outputs/mapping/" in k and "detail_detection.jsonl" in k for k in keys ) + @pytest.mark.django_db + @mock_aws + @patch( + "consultations.management.commands.prepare_s3.HostingEnvironment.is_production", + return_value=True, + ) + @patch( + "consultations.management.commands.prepare_s3.HostingEnvironment.is_preprod_environment", + return_value=False, + ) + @patch( + "consultations.management.commands.prepare_s3.HostingEnvironment.is_deployed", + return_value=True, + ) + def test_skips_on_preprod(self, _mock_deployed, _mock_preprod, _mock_prod, settings): + settings.AWS_BUCKET_NAME = "test-bucket" + + conn = boto3.resource("s3", region_name="eu-west-2") + conn.create_bucket( + Bucket="test-bucket", + CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, + ) + + call_command("prepare_s3") + + bucket = conn.Bucket("test-bucket") + keys = [obj.key for obj in bucket.objects.all()] + assert len(keys) == 0 + @pytest.mark.django_db @mock_aws @patch( From 9a12bb85908ee731f34f404e529ec4e50312aac2 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 28 Aug 2026 10:10:32 +0100 Subject: [PATCH 22/36] Fix test_skips_on_preprod to exercise preprod guard; patch is_preprod_environment in seeding tests --- backend/tests/commands/test_prepare_s3.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/backend/tests/commands/test_prepare_s3.py b/backend/tests/commands/test_prepare_s3.py index ca344181a..ede78c900 100644 --- a/backend/tests/commands/test_prepare_s3.py +++ b/backend/tests/commands/test_prepare_s3.py @@ -5,10 +5,16 @@ from django.core.management import call_command from moto import mock_aws +PREPROD_PATCH = patch( + "consultations.management.commands.prepare_s3.HostingEnvironment.is_preprod_environment", + return_value=False, +) + class TestPrepareS3: @pytest.mark.django_db @mock_aws + @PREPROD_PATCH @patch( "consultations.management.commands.prepare_s3.HostingEnvironment.is_production", return_value=False, @@ -17,7 +23,7 @@ class TestPrepareS3: "consultations.management.commands.prepare_s3.HostingEnvironment.is_deployed", return_value=True, ) - def test_deletes_existing_data_before_seeding(self, _mock_deployed, _mock_prod, settings): + def test_deletes_existing_data_before_seeding(self, _mock_deployed, _mock_prod, _mock_preprod, settings): settings.AWS_BUCKET_NAME = "test-bucket" conn = boto3.resource("s3", region_name="eu-west-2") @@ -44,6 +50,7 @@ def test_deletes_existing_data_before_seeding(self, _mock_deployed, _mock_prod, @pytest.mark.django_db @mock_aws + @PREPROD_PATCH @patch( "consultations.management.commands.prepare_s3.HostingEnvironment.is_production", return_value=False, @@ -52,7 +59,7 @@ def test_deletes_existing_data_before_seeding(self, _mock_deployed, _mock_prod, "consultations.management.commands.prepare_s3.HostingEnvironment.is_deployed", return_value=True, ) - def test_seeds_s3_with_data_for_each_stage(self, _mock_deployed, _mock_prod, settings): + def test_seeds_s3_with_data_for_each_stage(self, _mock_deployed, _mock_prod, _mock_preprod, settings): settings.AWS_BUCKET_NAME = "test-bucket" conn = boto3.resource("s3", region_name="eu-west-2") @@ -121,11 +128,11 @@ def test_seeds_s3_with_data_for_each_stage(self, _mock_deployed, _mock_prod, set @mock_aws @patch( "consultations.management.commands.prepare_s3.HostingEnvironment.is_production", - return_value=True, + return_value=False, ) @patch( "consultations.management.commands.prepare_s3.HostingEnvironment.is_preprod_environment", - return_value=False, + return_value=True, ) @patch( "consultations.management.commands.prepare_s3.HostingEnvironment.is_deployed", From 92d79f0e10f668a7bf3c20569a0e890f196eb494 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 28 Aug 2026 13:50:31 +0100 Subject: [PATCH 23/36] Split prepare_environment guard into two explicit early returns for clarity --- .../management/commands/prepare_environment.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/consultations/management/commands/prepare_environment.py b/backend/consultations/management/commands/prepare_environment.py index b5441bec5..1d78ef1e8 100644 --- a/backend/consultations/management/commands/prepare_environment.py +++ b/backend/consultations/management/commands/prepare_environment.py @@ -12,11 +12,16 @@ class Command(BaseCommand): def handle(self, *args, **options): environment = getattr(settings, "ENVIRONMENT", "").lower() - if environment != "dev" or not HostingEnvironment.is_deployed(): + if environment != "dev": self.stdout.write(f"Running migrate on {environment}.") call_command("migrate", verbosity=1) return + if not HostingEnvironment.is_deployed(): + self.stdout.write("Running migrate on dev (local — skipping DB reset and seed).") + call_command("migrate", verbosity=1) + return + self.stdout.write(f"Resetting database on {environment}...") connection = connections["default"] with connection.cursor() as cursor: From e4f2517c6275a1f9d307a5008a417e3ef0cb0197 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 28 Aug 2026 13:53:51 +0100 Subject: [PATCH 24/36] Updated comment and reverted changes so a migration-only exit is done on every env except dev --- .../management/commands/prepare_environment.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/backend/consultations/management/commands/prepare_environment.py b/backend/consultations/management/commands/prepare_environment.py index 1d78ef1e8..12a0565c0 100644 --- a/backend/consultations/management/commands/prepare_environment.py +++ b/backend/consultations/management/commands/prepare_environment.py @@ -3,11 +3,9 @@ from django.core.management.base import BaseCommand from django.db import connections -from hosting_environment import HostingEnvironment - class Command(BaseCommand): - help = "Prepare the environment: runs migrations on prod; resets and seeds the database and S3 on dev only." + help = "Prepare the environment: runs migrations on prod/preprod/test/local; resets and seeds the database and S3 on deployed dev only." def handle(self, *args, **options): environment = getattr(settings, "ENVIRONMENT", "").lower() @@ -17,11 +15,6 @@ def handle(self, *args, **options): call_command("migrate", verbosity=1) return - if not HostingEnvironment.is_deployed(): - self.stdout.write("Running migrate on dev (local — skipping DB reset and seed).") - call_command("migrate", verbosity=1) - return - self.stdout.write(f"Resetting database on {environment}...") connection = connections["default"] with connection.cursor() as cursor: From f71a7fd91d1d6f4f88f4715c433364b5b350f203 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 28 Aug 2026 13:54:24 +0100 Subject: [PATCH 25/36] Updated tests in test_prepare_environment.py to match --- .../commands/test_prepare_environment.py | 35 +++---------------- 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/backend/tests/commands/test_prepare_environment.py b/backend/tests/commands/test_prepare_environment.py index 1fae445d4..3f4522735 100644 --- a/backend/tests/commands/test_prepare_environment.py +++ b/backend/tests/commands/test_prepare_environment.py @@ -11,7 +11,7 @@ class TestPrepareEnvironment: @pytest.mark.django_db - @pytest.mark.parametrize("environment", ["prod", "preprod", "test", "", "unknown", "staging"]) + @pytest.mark.parametrize("environment", ["prod", "preprod", "test", "", "unknown", "staging", "local"]) def test_does_not_reset_on_non_dev(self, settings, environment): settings.ENVIRONMENT = environment @@ -20,19 +20,6 @@ def test_does_not_reset_on_non_dev(self, settings, environment): assert Consultation.objects.filter(code="KEEP_ME").exists() - @pytest.mark.django_db - @patch( - "consultations.management.commands.prepare_environment.HostingEnvironment.is_deployed", - return_value=False, - ) - def test_does_not_reset_when_dev_but_not_deployed(self, _mock_deployed, settings): - settings.ENVIRONMENT = "dev" - - Consultation.objects.create(title="Should survive", code="KEEP_ME") - call_command("prepare_environment") - - assert Consultation.objects.filter(code="KEEP_ME").exists() - @pytest.mark.django_db @mock_aws @patch("factories.embed_text", return_value=[0.0] * 3072) @@ -44,13 +31,7 @@ def test_does_not_reset_when_dev_but_not_deployed(self, _mock_deployed, settings "consultations.management.commands.prepare_s3.HostingEnvironment.is_deployed", return_value=True, ) - @patch( - "consultations.management.commands.prepare_environment.HostingEnvironment.is_deployed", - return_value=True, - ) - def test_resets_and_seeds_db( - self, _mock_env_deployed, _mock_s3_deployed, _mock_s3_prod, _mock_embed, settings - ): + def test_resets_and_seeds_db(self, _mock_s3_deployed, _mock_s3_prod, _mock_embed, settings): settings.ENVIRONMENT = "dev" settings.AWS_BUCKET_NAME = "test-bucket" @@ -89,17 +70,11 @@ def test_resets_and_seeds_db( "consultations.management.commands.prepare_s3.HostingEnvironment.is_deployed", return_value=True, ) - @patch( - "consultations.management.commands.prepare_environment.HostingEnvironment.is_deployed", - return_value=True, - ) - def test_seeds_s3( - self, _mock_env_deployed, _mock_s3_deployed, _mock_prod, _mock_preprod, _mock_embed, settings - ): + def test_seeds_s3(self, _mock_s3_deployed, _mock_prod, _mock_preprod, _mock_embed, settings): # prepare_environment calls prepare_s3 from the dev branch; prepare_s3 itself guards - # on is_deployed(). We patch both here (and mock S3 via moto) so the test runs + # on is_deployed(). We patch it here (and mock S3 via moto) so the test runs # without a real S3 bucket, while still exercising the full prepare_environment → prepare_s3 - # call chain on a dev environment. + # call chain. settings.ENVIRONMENT = "dev" settings.AWS_BUCKET_NAME = "test-bucket" From 77c192a75afc12e41bb17d72ced5e15f6d3f959c Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 28 Aug 2026 13:58:37 +0100 Subject: [PATCH 26/36] Updated comment in prepare_s3.py to specify this only runs on dev --- backend/consultations/management/commands/prepare_s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/consultations/management/commands/prepare_s3.py b/backend/consultations/management/commands/prepare_s3.py index 55129a248..e9100195e 100644 --- a/backend/consultations/management/commands/prepare_s3.py +++ b/backend/consultations/management/commands/prepare_s3.py @@ -206,7 +206,7 @@ def _build_themes_csv(question_data): class Command(BaseCommand): - help = "Reset and seed S3 with dummy consultation data matching the DB. Only runs on deployed non-prod environments." + help = "Reset and seed S3 with dummy consultation data matching the DB. Only runs on deployed dev environments." def handle(self, *args, **options): environment = getattr(settings, "ENVIRONMENT", "").lower() From 795567a23fe9f496dd6be40bcdcef7fce81d6b42 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 28 Aug 2026 14:06:08 +0100 Subject: [PATCH 27/36] Update guard in dummy_data.py to not run against preprod either, and update test in test_dummy_data.py to match --- backend/consultations/dummy_data.py | 2 +- backend/tests/commands/test_dummy_data.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/consultations/dummy_data.py b/backend/consultations/dummy_data.py index 96a67ad37..12337f411 100644 --- a/backend/consultations/dummy_data.py +++ b/backend/consultations/dummy_data.py @@ -247,7 +247,7 @@ def create_dummy_consultation( - ASSIGNING_THEMES (CONFIRMED): + CandidateThemes + SelectedThemes (ready for assignment) - ANALYSIS: + CandidateThemes + SelectedThemes + ResponseAnnotations """ - if HostingEnvironment.is_production(): + if HostingEnvironment.is_production() or HostingEnvironment.is_preprod_environment(): raise RuntimeError("Dummy data generation should not be run in production") if config is None: diff --git a/backend/tests/commands/test_dummy_data.py b/backend/tests/commands/test_dummy_data.py index 9315476d4..43578c62f 100644 --- a/backend/tests/commands/test_dummy_data.py +++ b/backend/tests/commands/test_dummy_data.py @@ -21,7 +21,7 @@ def test_name_parameter_sets_consultation_name(mock_is_local): @pytest.mark.django_db -@pytest.mark.parametrize("environment", ["prod"]) +@pytest.mark.parametrize("environment", ["prod", "preprod"]) def test_the_tool_will_only_run_in_dev(environment): with ( patch.dict(os.environ, {"ENVIRONMENT": environment}), From ea4d211a24d727598cd351c02b6c4e5299445e15 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 28 Aug 2026 14:13:55 +0100 Subject: [PATCH 28/36] Change the way tests mock and patch in test_prepare_environment.py to make it more readable --- .../commands/test_prepare_environment.py | 38 ++++++------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/backend/tests/commands/test_prepare_environment.py b/backend/tests/commands/test_prepare_environment.py index 3f4522735..bcd22530c 100644 --- a/backend/tests/commands/test_prepare_environment.py +++ b/backend/tests/commands/test_prepare_environment.py @@ -22,16 +22,13 @@ def test_does_not_reset_on_non_dev(self, settings, environment): @pytest.mark.django_db @mock_aws + @patch("consultations.management.commands.prepare_s3.HostingEnvironment") @patch("factories.embed_text", return_value=[0.0] * 3072) - @patch( - "consultations.management.commands.prepare_s3.HostingEnvironment.is_production", - return_value=False, - ) - @patch( - "consultations.management.commands.prepare_s3.HostingEnvironment.is_deployed", - return_value=True, - ) - def test_resets_and_seeds_db(self, _mock_s3_deployed, _mock_s3_prod, _mock_embed, settings): + def test_resets_and_seeds_db(self, _mock_embed, mock_hosting_env, settings): + mock_hosting_env.is_production.return_value = False + mock_hosting_env.is_preprod_environment.return_value = False + mock_hosting_env.is_deployed.return_value = True + settings.ENVIRONMENT = "dev" settings.AWS_BUCKET_NAME = "test-bucket" @@ -57,24 +54,13 @@ def test_resets_and_seeds_db(self, _mock_s3_deployed, _mock_s3_prod, _mock_embed @pytest.mark.django_db @mock_aws + @patch("consultations.management.commands.prepare_s3.HostingEnvironment") @patch("factories.embed_text", return_value=[0.0] * 3072) - @patch( - "consultations.management.commands.prepare_s3.HostingEnvironment.is_preprod_environment", - return_value=False, - ) - @patch( - "consultations.management.commands.prepare_s3.HostingEnvironment.is_production", - return_value=False, - ) - @patch( - "consultations.management.commands.prepare_s3.HostingEnvironment.is_deployed", - return_value=True, - ) - def test_seeds_s3(self, _mock_s3_deployed, _mock_prod, _mock_preprod, _mock_embed, settings): - # prepare_environment calls prepare_s3 from the dev branch; prepare_s3 itself guards - # on is_deployed(). We patch it here (and mock S3 via moto) so the test runs - # without a real S3 bucket, while still exercising the full prepare_environment → prepare_s3 - # call chain. + def test_seeds_s3(self, _mock_embed, mock_hosting_env, settings): + mock_hosting_env.is_production.return_value = False + mock_hosting_env.is_preprod_environment.return_value = False + mock_hosting_env.is_deployed.return_value = True + settings.ENVIRONMENT = "dev" settings.AWS_BUCKET_NAME = "test-bucket" From 5428ce8b738c783710af8f2bdefcd8a9f64dbd2f Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 28 Aug 2026 14:23:05 +0100 Subject: [PATCH 29/36] Rename candidate theme generation function in prepare_s3.py to match what it does closer --- .../consultations/management/commands/prepare_s3.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/backend/consultations/management/commands/prepare_s3.py b/backend/consultations/management/commands/prepare_s3.py index e9100195e..d9a1492f0 100644 --- a/backend/consultations/management/commands/prepare_s3.py +++ b/backend/consultations/management/commands/prepare_s3.py @@ -147,8 +147,13 @@ def _build_themes_json(question_data): return themes -def _build_candidate_theme_mappings(question_data): - """Build mapping.jsonl for candidate themes using deterministic assignment per sibling group.""" +def _build_hierarchical_candidate_theme_mappings(question_data): + """Build mapping.jsonl for candidate themes, respecting the parent-child hierarchy. + + Themes are grouped by parent so that sibling themes compete against each other — + each respondent is assigned 1–2 keys from each sibling group independently. + This mirrors how the real find-themes pipeline produces clustered outputs. + """ candidate_themes = question_data.get("candidate_themes", []) if not candidate_themes: return [] @@ -363,7 +368,7 @@ def _seed_consultation( s3_client.put_object( Bucket=bucket, Key=f"{out_prefix}/mapping.jsonl", - Body=_to_jsonl(_build_candidate_theme_mappings(question_data)), + Body=_to_jsonl(_build_hierarchical_candidate_theme_mappings(question_data)), ) if include_themes_csv: From 2f5e8b50292229be05de3e27d845d24e7e1e16a2 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 28 Aug 2026 14:25:18 +0100 Subject: [PATCH 30/36] Change guard in prepare_s3.py to skip if it's any env except dev --- backend/consultations/management/commands/prepare_s3.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/backend/consultations/management/commands/prepare_s3.py b/backend/consultations/management/commands/prepare_s3.py index d9a1492f0..d18e69868 100644 --- a/backend/consultations/management/commands/prepare_s3.py +++ b/backend/consultations/management/commands/prepare_s3.py @@ -216,14 +216,10 @@ class Command(BaseCommand): def handle(self, *args, **options): environment = getattr(settings, "ENVIRONMENT", "").lower() - if HostingEnvironment.is_production() or HostingEnvironment.is_preprod_environment(): + if not HostingEnvironment.is_development_environment(): self.stdout.write(f"Skipping S3 seed on {environment} environment.") return - if not HostingEnvironment.is_deployed(): - self.stdout.write(f"Skipping S3 seed on {environment} environment (no real S3 bucket).") - return - s3_client = boto3.client("s3") bucket = settings.AWS_BUCKET_NAME timestamp = datetime.datetime.now(tz=ZoneInfo("Europe/London")).date() From bb065d916760f38c5e8699e8839ef99bcef38e01 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 28 Aug 2026 14:34:54 +0100 Subject: [PATCH 31/36] Change guard in prepare_s3.py to skip if it's any env except dev (corrected) --- backend/consultations/management/commands/prepare_s3.py | 2 +- backend/hosting_environment.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/consultations/management/commands/prepare_s3.py b/backend/consultations/management/commands/prepare_s3.py index d18e69868..30098c32b 100644 --- a/backend/consultations/management/commands/prepare_s3.py +++ b/backend/consultations/management/commands/prepare_s3.py @@ -216,7 +216,7 @@ class Command(BaseCommand): def handle(self, *args, **options): environment = getattr(settings, "ENVIRONMENT", "").lower() - if not HostingEnvironment.is_development_environment(): + if not HostingEnvironment.is_dev(): self.stdout.write(f"Skipping S3 seed on {environment} environment.") return diff --git a/backend/hosting_environment.py b/backend/hosting_environment.py index f0f05cb84..9f24f2ec8 100644 --- a/backend/hosting_environment.py +++ b/backend/hosting_environment.py @@ -32,3 +32,7 @@ def is_development_environment() -> bool: environment = env.str("ENVIRONMENT", "").upper() development_environments = ["LOCAL", "TEST", "DEV", "DEVELOPMENT"] return environment in development_environments + + @staticmethod + def is_dev() -> bool: + return env.str("ENVIRONMENT", "").upper() == "DEV" From 63f83ab4c45277575cd80e11f516ea5037a6e7fe Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 28 Aug 2026 14:46:19 +0100 Subject: [PATCH 32/36] Move mock and patch settings in test_prepare_s3.py to a separate function --- backend/tests/commands/test_prepare_s3.py | 70 ++++++++--------------- 1 file changed, 24 insertions(+), 46 deletions(-) diff --git a/backend/tests/commands/test_prepare_s3.py b/backend/tests/commands/test_prepare_s3.py index ede78c900..72d7e2102 100644 --- a/backend/tests/commands/test_prepare_s3.py +++ b/backend/tests/commands/test_prepare_s3.py @@ -5,25 +5,20 @@ from django.core.management import call_command from moto import mock_aws -PREPROD_PATCH = patch( - "consultations.management.commands.prepare_s3.HostingEnvironment.is_preprod_environment", - return_value=False, -) + +def _mock_deployed_dev(mock_env): + """Configure HostingEnvironment mock for a deployed dev environment.""" + mock_env.is_production.return_value = False + mock_env.is_preprod_environment.return_value = False + mock_env.is_deployed.return_value = True class TestPrepareS3: @pytest.mark.django_db @mock_aws - @PREPROD_PATCH - @patch( - "consultations.management.commands.prepare_s3.HostingEnvironment.is_production", - return_value=False, - ) - @patch( - "consultations.management.commands.prepare_s3.HostingEnvironment.is_deployed", - return_value=True, - ) - def test_deletes_existing_data_before_seeding(self, _mock_deployed, _mock_prod, _mock_preprod, settings): + @patch("consultations.management.commands.prepare_s3.HostingEnvironment") + def test_deletes_existing_data_before_seeding(self, mock_hosting_env, settings): + _mock_deployed_dev(mock_hosting_env) settings.AWS_BUCKET_NAME = "test-bucket" conn = boto3.resource("s3", region_name="eu-west-2") @@ -50,16 +45,9 @@ def test_deletes_existing_data_before_seeding(self, _mock_deployed, _mock_prod, @pytest.mark.django_db @mock_aws - @PREPROD_PATCH - @patch( - "consultations.management.commands.prepare_s3.HostingEnvironment.is_production", - return_value=False, - ) - @patch( - "consultations.management.commands.prepare_s3.HostingEnvironment.is_deployed", - return_value=True, - ) - def test_seeds_s3_with_data_for_each_stage(self, _mock_deployed, _mock_prod, _mock_preprod, settings): + @patch("consultations.management.commands.prepare_s3.HostingEnvironment") + def test_seeds_s3_with_data_for_each_stage(self, mock_hosting_env, settings): + _mock_deployed_dev(mock_hosting_env) settings.AWS_BUCKET_NAME = "test-bucket" conn = boto3.resource("s3", region_name="eu-west-2") @@ -76,6 +64,8 @@ def test_seeds_s3_with_data_for_each_stage(self, _mock_deployed, _mock_prod, _mo # S3-only consultation has input data assert any("dummy-s3-only/inputs/respondents.jsonl" in k for k in keys) assert any("dummy-s3-only/inputs/question_part_1/question.json" in k for k in keys) + # Multi-choice-only questions also get responses.jsonl (respondent manifest) + assert any("dummy-s3-only/inputs/question_part_2/responses.jsonl" in k for k in keys) # Setup consultation has input data assert any("dummy-setup/inputs/respondents.jsonl" in k for k in keys) @@ -126,19 +116,11 @@ def test_seeds_s3_with_data_for_each_stage(self, _mock_deployed, _mock_prod, _mo @pytest.mark.django_db @mock_aws - @patch( - "consultations.management.commands.prepare_s3.HostingEnvironment.is_production", - return_value=False, - ) - @patch( - "consultations.management.commands.prepare_s3.HostingEnvironment.is_preprod_environment", - return_value=True, - ) - @patch( - "consultations.management.commands.prepare_s3.HostingEnvironment.is_deployed", - return_value=True, - ) - def test_skips_on_preprod(self, _mock_deployed, _mock_preprod, _mock_prod, settings): + @patch("consultations.management.commands.prepare_s3.HostingEnvironment") + def test_skips_on_preprod(self, mock_hosting_env, settings): + mock_hosting_env.is_production.return_value = False + mock_hosting_env.is_preprod_environment.return_value = True + mock_hosting_env.is_deployed.return_value = True settings.AWS_BUCKET_NAME = "test-bucket" conn = boto3.resource("s3", region_name="eu-west-2") @@ -155,15 +137,11 @@ def test_skips_on_preprod(self, _mock_deployed, _mock_preprod, _mock_prod, setti @pytest.mark.django_db @mock_aws - @patch( - "consultations.management.commands.prepare_s3.HostingEnvironment.is_production", - return_value=False, - ) - @patch( - "consultations.management.commands.prepare_s3.HostingEnvironment.is_deployed", - return_value=False, - ) - def test_skips_on_local(self, _mock_deployed, _mock_prod, settings): + @patch("consultations.management.commands.prepare_s3.HostingEnvironment") + def test_skips_on_local(self, mock_hosting_env, settings): + mock_hosting_env.is_production.return_value = False + mock_hosting_env.is_preprod_environment.return_value = False + mock_hosting_env.is_deployed.return_value = False settings.AWS_BUCKET_NAME = "test-bucket" conn = boto3.resource("s3", region_name="eu-west-2") From caa5bda56706ed23caf06b8cc0859c4159a79780 Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 28 Aug 2026 14:46:45 +0100 Subject: [PATCH 33/36] Change conditional gate in prepare_environment.py to use HostingEnvironment to align with what prepare_s3.py does --- .../consultations/management/commands/prepare_environment.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/consultations/management/commands/prepare_environment.py b/backend/consultations/management/commands/prepare_environment.py index 12a0565c0..e3fd8f81b 100644 --- a/backend/consultations/management/commands/prepare_environment.py +++ b/backend/consultations/management/commands/prepare_environment.py @@ -3,6 +3,8 @@ from django.core.management.base import BaseCommand from django.db import connections +from hosting_environment import HostingEnvironment + class Command(BaseCommand): help = "Prepare the environment: runs migrations on prod/preprod/test/local; resets and seeds the database and S3 on deployed dev only." @@ -10,7 +12,7 @@ class Command(BaseCommand): def handle(self, *args, **options): environment = getattr(settings, "ENVIRONMENT", "").lower() - if environment != "dev": + if not HostingEnvironment.is_dev(): self.stdout.write(f"Running migrate on {environment}.") call_command("migrate", verbosity=1) return From 6998d1321a26efb90fa3339108a60556ac18922a Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 28 Aug 2026 14:57:57 +0100 Subject: [PATCH 34/36] Replace non-deterministic random calls in dummy_data with themefinder_id-based indexing --- backend/consultations/dummy_data.py | 43 ++++++++++++++++++----------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/backend/consultations/dummy_data.py b/backend/consultations/dummy_data.py index 12337f411..6e36a1d9b 100644 --- a/backend/consultations/dummy_data.py +++ b/backend/consultations/dummy_data.py @@ -1,5 +1,4 @@ import json -import random from django.conf import settings @@ -164,34 +163,46 @@ def create_default_selected_themes(question): def create_response(respondent, question, free_text_answers): """Create and return a Response.""" - raw = random.choice(free_text_answers) if question.has_free_text else None - free_text = raw if raw not in ("", "Not Provided", "-") else None - return ResponseFactory(question=question, free_text=free_text, respondent=respondent) + if question.has_free_text: + non_empty = [a for a in free_text_answers if a not in ("", "Not Provided", "-")] + raw = non_empty[respondent.themefinder_id % len(non_empty)] if non_empty else None + else: + raw = None + return ResponseFactory(question=question, free_text=raw, respondent=respondent) def create_response_annotation(response, question): """Create a ResponseAnnotation and ResponseAnnotationThemes for a free text response.""" selected_themes = list(question.selectedtheme_set.all()) - themes_for_response = random.sample( - selected_themes, - k=random.randint(1, len(selected_themes)), - ) - random_sentiment = random.choice([s[0] for s in ResponseAnnotation.Sentiment.choices]) - random_evidence_rich = random.choice([True, False]) + tf_id = response.respondent.themefinder_id + + # Assign 1 or 2 themes deterministically based on themefinder_id + num_themes = 1 + (tf_id % 2) + themes_for_response = [selected_themes[tf_id % len(selected_themes)]] + if num_themes > 1 and len(selected_themes) > 1: + themes_for_response.append(selected_themes[(tf_id + 1) % len(selected_themes)]) + + sentiment_choices = [s[0] for s in ResponseAnnotation.Sentiment.choices] + sentiment = sentiment_choices[tf_id % len(sentiment_choices)] + evidence_rich = tf_id % 3 == 0 + response_annotation = ResponseAnnotationFactoryNoThemes( response=response, - sentiment=random_sentiment, - evidence_rich=random_evidence_rich, + sentiment=sentiment, + evidence_rich=evidence_rich, ) response_annotation.add_original_ai_themes(themes_for_response) def create_response_chosen_options(response, multiple_choice_options): """Add chosen options to a multiple choice response.""" - chosen_options = random.sample( - multiple_choice_options, - k=random.randint(1, len(multiple_choice_options)), - ) + tf_id = response.respondent.themefinder_id + # Pick 1 or 2 options deterministically based on themefinder_id + num_options = 1 + (tf_id % 2) + chosen_options = [ + multiple_choice_options[(tf_id + j) % len(multiple_choice_options)] + for j in range(num_options) + ] answers = MultiChoiceAnswer.objects.filter(question=response.question, text__in=chosen_options) response.chosen_options.add(*answers) From dd7461e4a72bbd125b8669e8f9c5db0c8e05e22f Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 28 Aug 2026 15:05:20 +0100 Subject: [PATCH 35/36] Simplify test mocking to match simplified conditional gates in both test_prepare_environment.py and test_prepare_s3.py --- .../commands/test_prepare_environment.py | 16 +++++++------- backend/tests/commands/test_prepare_s3.py | 21 ++++--------------- 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/backend/tests/commands/test_prepare_environment.py b/backend/tests/commands/test_prepare_environment.py index bcd22530c..14f96548a 100644 --- a/backend/tests/commands/test_prepare_environment.py +++ b/backend/tests/commands/test_prepare_environment.py @@ -23,11 +23,11 @@ def test_does_not_reset_on_non_dev(self, settings, environment): @pytest.mark.django_db @mock_aws @patch("consultations.management.commands.prepare_s3.HostingEnvironment") + @patch("consultations.management.commands.prepare_environment.HostingEnvironment") @patch("factories.embed_text", return_value=[0.0] * 3072) - def test_resets_and_seeds_db(self, _mock_embed, mock_hosting_env, settings): - mock_hosting_env.is_production.return_value = False - mock_hosting_env.is_preprod_environment.return_value = False - mock_hosting_env.is_deployed.return_value = True + def test_resets_and_seeds_db(self, _mock_embed, mock_env, mock_s3_env, settings): + mock_env.is_dev.return_value = True + mock_s3_env.is_dev.return_value = True settings.ENVIRONMENT = "dev" settings.AWS_BUCKET_NAME = "test-bucket" @@ -55,11 +55,11 @@ def test_resets_and_seeds_db(self, _mock_embed, mock_hosting_env, settings): @pytest.mark.django_db @mock_aws @patch("consultations.management.commands.prepare_s3.HostingEnvironment") + @patch("consultations.management.commands.prepare_environment.HostingEnvironment") @patch("factories.embed_text", return_value=[0.0] * 3072) - def test_seeds_s3(self, _mock_embed, mock_hosting_env, settings): - mock_hosting_env.is_production.return_value = False - mock_hosting_env.is_preprod_environment.return_value = False - mock_hosting_env.is_deployed.return_value = True + def test_seeds_s3(self, _mock_embed, mock_env, mock_s3_env, settings): + mock_env.is_dev.return_value = True + mock_s3_env.is_dev.return_value = True settings.ENVIRONMENT = "dev" settings.AWS_BUCKET_NAME = "test-bucket" diff --git a/backend/tests/commands/test_prepare_s3.py b/backend/tests/commands/test_prepare_s3.py index 72d7e2102..b187df55b 100644 --- a/backend/tests/commands/test_prepare_s3.py +++ b/backend/tests/commands/test_prepare_s3.py @@ -6,19 +6,12 @@ from moto import mock_aws -def _mock_deployed_dev(mock_env): - """Configure HostingEnvironment mock for a deployed dev environment.""" - mock_env.is_production.return_value = False - mock_env.is_preprod_environment.return_value = False - mock_env.is_deployed.return_value = True - - class TestPrepareS3: @pytest.mark.django_db @mock_aws @patch("consultations.management.commands.prepare_s3.HostingEnvironment") def test_deletes_existing_data_before_seeding(self, mock_hosting_env, settings): - _mock_deployed_dev(mock_hosting_env) + mock_hosting_env.is_dev.return_value = True settings.AWS_BUCKET_NAME = "test-bucket" conn = boto3.resource("s3", region_name="eu-west-2") @@ -47,7 +40,7 @@ def test_deletes_existing_data_before_seeding(self, mock_hosting_env, settings): @mock_aws @patch("consultations.management.commands.prepare_s3.HostingEnvironment") def test_seeds_s3_with_data_for_each_stage(self, mock_hosting_env, settings): - _mock_deployed_dev(mock_hosting_env) + mock_hosting_env.is_dev.return_value = True settings.AWS_BUCKET_NAME = "test-bucket" conn = boto3.resource("s3", region_name="eu-west-2") @@ -64,8 +57,6 @@ def test_seeds_s3_with_data_for_each_stage(self, mock_hosting_env, settings): # S3-only consultation has input data assert any("dummy-s3-only/inputs/respondents.jsonl" in k for k in keys) assert any("dummy-s3-only/inputs/question_part_1/question.json" in k for k in keys) - # Multi-choice-only questions also get responses.jsonl (respondent manifest) - assert any("dummy-s3-only/inputs/question_part_2/responses.jsonl" in k for k in keys) # Setup consultation has input data assert any("dummy-setup/inputs/respondents.jsonl" in k for k in keys) @@ -118,9 +109,7 @@ def test_seeds_s3_with_data_for_each_stage(self, mock_hosting_env, settings): @mock_aws @patch("consultations.management.commands.prepare_s3.HostingEnvironment") def test_skips_on_preprod(self, mock_hosting_env, settings): - mock_hosting_env.is_production.return_value = False - mock_hosting_env.is_preprod_environment.return_value = True - mock_hosting_env.is_deployed.return_value = True + mock_hosting_env.is_dev.return_value = False settings.AWS_BUCKET_NAME = "test-bucket" conn = boto3.resource("s3", region_name="eu-west-2") @@ -139,9 +128,7 @@ def test_skips_on_preprod(self, mock_hosting_env, settings): @mock_aws @patch("consultations.management.commands.prepare_s3.HostingEnvironment") def test_skips_on_local(self, mock_hosting_env, settings): - mock_hosting_env.is_production.return_value = False - mock_hosting_env.is_preprod_environment.return_value = False - mock_hosting_env.is_deployed.return_value = False + mock_hosting_env.is_dev.return_value = False settings.AWS_BUCKET_NAME = "test-bucket" conn = boto3.resource("s3", region_name="eu-west-2") From 8cfb36068d808462905dc112d2fba262ee3d52af Mon Sep 17 00:00:00 2001 From: Elliot Moore Date: Fri, 28 Aug 2026 15:10:51 +0100 Subject: [PATCH 36/36] Fix test failures: update patches to is_dev, merge duplicate skip tests, remove stale responses.jsonl assertion --- backend/tests/commands/test_prepare_s3.py | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/backend/tests/commands/test_prepare_s3.py b/backend/tests/commands/test_prepare_s3.py index b187df55b..5f9762ef3 100644 --- a/backend/tests/commands/test_prepare_s3.py +++ b/backend/tests/commands/test_prepare_s3.py @@ -107,27 +107,9 @@ def test_seeds_s3_with_data_for_each_stage(self, mock_hosting_env, settings): @pytest.mark.django_db @mock_aws + @pytest.mark.parametrize("environment", ["local", "preprod", "prod", "test"]) @patch("consultations.management.commands.prepare_s3.HostingEnvironment") - def test_skips_on_preprod(self, mock_hosting_env, settings): - mock_hosting_env.is_dev.return_value = False - settings.AWS_BUCKET_NAME = "test-bucket" - - conn = boto3.resource("s3", region_name="eu-west-2") - conn.create_bucket( - Bucket="test-bucket", - CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, - ) - - call_command("prepare_s3") - - bucket = conn.Bucket("test-bucket") - keys = [obj.key for obj in bucket.objects.all()] - assert len(keys) == 0 - - @pytest.mark.django_db - @mock_aws - @patch("consultations.management.commands.prepare_s3.HostingEnvironment") - def test_skips_on_local(self, mock_hosting_env, settings): + def test_skips_on_non_dev(self, mock_hosting_env, environment, settings): mock_hosting_env.is_dev.return_value = False settings.AWS_BUCKET_NAME = "test-bucket"