diff --git a/.env.test b/.env.test index a8abbf452..de3e888fe 100644 --- a/.env.test +++ b/.env.test @@ -19,4 +19,4 @@ FIND_THEMES_BATCH_JOB_QUEUE=i-dot-ai-dev-consult-sign-off-FARGATE-batch-job-queu FIND_THEMES_BATCH_JOB_DEFINITION=i-dot-ai-dev-consult-sign-off-FARGATE-batch-job-definition LITELLM_CONSULT_OPENAI_API_KEY=insert-litellm-api-key-here LLM_GATEWAY_URL=https://llm-gateway.i.ai.gov.uk -ADMIN_USERS=email@example.com +ADMIN_USERS=admin@example.com,policy@example.com diff --git a/Makefile b/Makefile index e781fb9c9..e3c631fce 100644 --- a/Makefile +++ b/Makefile @@ -77,8 +77,7 @@ test-end-to-end: ## Run end-to-end tests with Playwright # Run the tests, then ALWAYS clean up, then re-raise the tests' exit status so CI # still fails on failure. Without this wrapper a failure mid-run (e.g. a service # health-check timeout or a failing test) would skip cleanup and leave - # docker-compose.override.yml behind, silently repointing every later - # `docker compose` command at the E2E database. + # the e2e test database behind or a stale Astro dev.json lock file. @$(MAKE) _run-e2e-tests; status=$$?; $(MAKE) _clean-e2e; exit $$status .PHONY: _run-e2e-tests @@ -89,18 +88,14 @@ _run-e2e-tests: @docker exec -i $$(docker compose ps -q postgres) psql -U postgres -c "CREATE DATABASE consult_e2e_test;" @echo "Initializing test data..." @docker compose run --rm -e DATABASE_URL=$(E2E_DB_URL) backend venv/bin/python manage.py migrate - @docker compose run --rm -e DATABASE_URL=$(E2E_DB_URL) -e ADMIN_USERS=admin@example.com backend venv/bin/python manage.py createadminusers + @docker compose run --rm -e DATABASE_URL=$(E2E_DB_URL) backend venv/bin/python manage.py createadminusers @docker compose run --rm -e DATABASE_URL=$(E2E_DB_URL) backend venv/bin/python manage.py shell -c \ "from authentication.models import User; from consultations.models import Consultation; \ user = User.objects.get(email='admin@example.com'); \ [c.users.add(user) for c in Consultation.objects.all()]" @echo "Starting services..." - @echo "services:" > docker-compose.override.yml - @echo " backend:" >> docker-compose.override.yml - @echo " environment:" >> docker-compose.override.yml - @echo " - DATABASE_URL=$(E2E_DB_URL)" >> docker-compose.override.yml - @docker compose down backend 2>/dev/null || true - @docker compose up -d backend frontend + @rm -f frontend/.astro/dev.json + @DATABASE_URL=$(E2E_DB_URL) docker compose up -d backend frontend @echo "Waiting for services to be ready..." @timeout 120 sh -c 'until curl -s http://localhost:3000 > /dev/null; do sleep 2; done' || \ (echo "Frontend failed to start" && docker compose logs frontend && exit 1) @@ -116,6 +111,7 @@ _clean-e2e: ## Internal: always-run cleanup for test-end-to-end (drop test DB, r @echo "Cleaning up..." @docker exec -i $$(docker compose ps -q postgres) psql -U postgres -c "DROP DATABASE IF EXISTS consult_e2e_test;" 2>/dev/null || true @rm -f docker-compose.override.yml + @rm -f frontend/.astro/dev.json .PHONY: build-consultation-template @@ -178,7 +174,8 @@ dummy_data: ## Generate dummy consultations. Only works in dev cd backend && PYTHONPATH=.. uv run python manage.py generate_dummy_data .PHONY: dev_environment -dev_environment: reset_db migrate dummy_data ## set up the database with dummy data +dev_environment: setup_db ## set up the database with dummy data + cd backend && PYTHONPATH=.. uv run python manage.py prepare_environment # Docker AWS_REGION=eu-west-2 diff --git a/backend/consultations/admin.py b/backend/consultations/admin.py index 90190396c..95aa695fd 100644 --- a/backend/consultations/admin.py +++ b/backend/consultations/admin.py @@ -6,7 +6,7 @@ from django.urls import path, reverse from simple_history.admin import SimpleHistoryAdmin -from consultations.dummy_data import create_dummy_consultation_from_yaml_job +from consultations.dummy_data import create_dummy_consultation_job from consultations.models import ( CandidateTheme, Consultation, @@ -55,9 +55,7 @@ def create_dummy_consultation(modeladmin, request, queryset, size=10): ) return - create_dummy_consultation_from_yaml_job.delay( - number_respondents=size, consultation=consultation - ) + create_dummy_consultation_job.delay(number_respondents=size, consultation=consultation) @admin.action(description="create small dummy consultation") diff --git a/backend/consultations/api/serializers.py b/backend/consultations/api/serializers.py index 963658478..e5b62fed7 100644 --- a/backend/consultations/api/serializers.py +++ b/backend/consultations/api/serializers.py @@ -38,9 +38,7 @@ def validate_is_staff(self, value): # Check if this is an update operation and user is updating themselves if self.instance and request and request.user == self.instance and value is False: - raise serializers.ValidationError( - "You cannot remove admin privileges from yourself" - ) + raise serializers.ValidationError("You cannot remove admin privileges from yourself") return value @@ -150,7 +148,14 @@ class SelectedThemeSerializer(serializers.ModelSerializer): class Meta: model = SelectedTheme - fields: ClassVar[list] = ["id", "name", "description", "version", "modified_at", "last_modified_by"] + fields: ClassVar[list] = [ + "id", + "name", + "description", + "version", + "modified_at", + "last_modified_by", + ] read_only_fields: ClassVar[list] = ["id", "version", "modified_at", "last_modified_by"] def get_last_modified_by(self, obj): diff --git a/backend/consultations/api/views/consultation.py b/backend/consultations/api/views/consultation.py index 1fa8e5b0f..4f0f0cec0 100644 --- a/backend/consultations/api/views/consultation.py +++ b/backend/consultations/api/views/consultation.py @@ -946,9 +946,7 @@ def evaluation(self, request, pk=None): SelectedTheme.objects.filter( question__consultation=consultation, ) - .filter( - Q(name__iexact=NO_REASON_GIVEN_THEME_NAME) | Q(name__iexact=OTHER_THEME_NAME) - ) + .filter(Q(name__iexact=NO_REASON_GIVEN_THEME_NAME) | Q(name__iexact=OTHER_THEME_NAME)) .values_list("id", flat=True) ) diff --git a/backend/consultations/api/views/question.py b/backend/consultations/api/views/question.py index e314a477f..d1dffa57f 100644 --- a/backend/consultations/api/views/question.py +++ b/backend/consultations/api/views/question.py @@ -142,9 +142,7 @@ def themes(self, request, pk=None, consultation_pk=None): ) ) else: - themes = themes.annotate( - count=Count("responseannotation", distinct=True) - ) + themes = themes.annotate(count=Count("responseannotation", distinct=True)) serializer = QuestionThemeSerializer(themes, many=True) return Response({"themes": serializer.data}) diff --git a/backend/consultations/api/views/response.py b/backend/consultations/api/views/response.py index 484f0ed31..c4a037ba0 100644 --- a/backend/consultations/api/views/response.py +++ b/backend/consultations/api/views/response.py @@ -253,11 +253,7 @@ def mark_read_bulk(self, request, consultation_pk=None, **kwargs): if len(requested_response_ids) > MAX_BULK_MARK_READ: return Response( - { - "message": ( - f"Too many response IDs provided. Maximum is {MAX_BULK_MARK_READ}." - ) - }, + {"message": (f"Too many response IDs provided. Maximum is {MAX_BULK_MARK_READ}.")}, status=status.HTTP_400_BAD_REQUEST, ) diff --git a/backend/consultations/dummy_data.py b/backend/consultations/dummy_data.py index e9799e0b1..6e36a1d9b 100644 --- a/backend/consultations/dummy_data.py +++ b/backend/consultations/dummy_data.py @@ -1,14 +1,16 @@ -import random -from typing import Literal +import json -import yaml from django.conf import settings from consultations.models import ( + CandidateTheme, + CandidateThemeResponse, Consultation, MultiChoiceAnswer, Question, + Response, ResponseAnnotation, + SelectedTheme, ) from factories import ( CandidateThemeFactory, @@ -24,40 +26,74 @@ logger = settings.LOGGER -DATA_BY_STAGE = { - Consultation.Stage.ANALYSIS: { - "CONSULTATION_NAME": "Dummy Consultation at Analysis Stage", - "QUESTION_THEME_STATUS": Question.ThemeStatus.CONFIRMED, - "CONSULTATION_CODE": "dummy-consultation-analysis", +DUMMY_CONSULTATIONS = [ + { + "CONSULTATION_NAME": "Dummy Consultation - Data Setup", + "CONSULTATION_CODE": "dummy-setup", + "CONSULTATION_STAGE": Consultation.Stage.SETUP, + "QUESTION_THEME_STATUS": Question.ThemeStatus.DRAFT, }, - Consultation.Stage.FINALISING_THEMES: { - "CONSULTATION_NAME": "Dummy Consultation at Finalising Themes Stage", + { + "CONSULTATION_NAME": "Dummy Consultation - Starting finalising themes", + "CONSULTATION_CODE": "dummy-start-finalising-themes", + "CONSULTATION_STAGE": Consultation.Stage.FINALISING_THEMES, "QUESTION_THEME_STATUS": Question.ThemeStatus.DRAFT, - "CONSULTATION_CODE": "dummy-consultation-finalising", }, -} + { + "CONSULTATION_NAME": "Dummy Consultation - Finished finalising themes", + "CONSULTATION_CODE": "dummy-finished-finalising-themes", + "CONSULTATION_STAGE": Consultation.Stage.ASSIGNING_THEMES, + "QUESTION_THEME_STATUS": Question.ThemeStatus.CONFIRMED, + }, + { + "CONSULTATION_NAME": "Dummy Consultation - Analysis", + "CONSULTATION_CODE": "dummy-analysis", + "CONSULTATION_STAGE": Consultation.Stage.ANALYSIS, + "QUESTION_THEME_STATUS": Question.ThemeStatus.CONFIRMED, + }, +] +NUMBER_RESPONDENTS = 100 +REGIONS = ["North East", "North West", "South East", "South West", "Midlands", "London"] +AGE_GROUPS = ["Under 18", "18-35", "36-50", "51-65", "66+"] +RESPONDENT_TYPES = ["Individual", "Organisation"] +SAMPLE_QUESTIONS_PATH = "./tests/examples/sample_questions.json" -def create_consultation(stage): - """Create and return a Consultation.""" - name = DATA_BY_STAGE[stage]["CONSULTATION_NAME"] - code = DATA_BY_STAGE[stage]["CONSULTATION_CODE"] - return ConsultationFactory(title=name, stage=stage, code=code) + +def create_consultation(config): + """Create and return a Consultation from a config dict.""" + return ConsultationFactory( + title=config["CONSULTATION_NAME"], + code=config["CONSULTATION_CODE"], + stage=config["CONSULTATION_STAGE"], + ) + + +def _demographics_for(themefinder_id): + """Return deterministic demographics based on themefinder_id.""" + return { + "region": REGIONS[themefinder_id % len(REGIONS)], + "age_group": AGE_GROUPS[themefinder_id % len(AGE_GROUPS)], + "respondent_type": RESPONDENT_TYPES[themefinder_id % len(RESPONDENT_TYPES)], + } def create_respondents(consultation, number_respondents): """Create and return a list of Respondents.""" return [ - RespondentFactory(consultation=consultation, themefinder_id=i) + RespondentFactory( + consultation=consultation, + themefinder_id=i, + demographics=_demographics_for(i), + ) for i in range(1, number_respondents + 1) ] -def create_question(consultation, question_data): +def create_question(consultation, question_data, theme_status): """Create and return a Question.""" has_free_text = question_data["has_free_text"] has_multiple_choice = question_data["has_multiple_choice"] - theme_status = DATA_BY_STAGE[consultation.stage]["QUESTION_THEME_STATUS"] return QuestionFactory( text=question_data["question_text"], @@ -75,146 +111,235 @@ def create_multi_choice_answers(question, choices): MultiChoiceAnswer.objects.bulk_create(multi_choice_objects) -def create_candidate_theme_recursive( - question, number_respondents, candidate_theme_data, parent=None -): - """Create a CandidateTheme (and SelectedTheme if selected) recursively.""" - name = candidate_theme_data.get("name") - description = candidate_theme_data.get("description", "") - key = candidate_theme_data.get("key") - approximate_frequency = round( - candidate_theme_data.get("approximate_frequency_pct", 0) * number_respondents - ) +def create_candidate_themes(question, candidate_themes_data): + """Create CandidateThemes (and SelectedThemes if selected) from flat theme list.""" + key_to_candidate_theme = {} - candidate_theme = CandidateThemeFactory( + for theme_data in candidate_themes_data: + candidate_theme = CandidateThemeFactory( + question=question, + name=theme_data["name"], + description=theme_data.get("description", ""), + parent=None, + approximate_frequency=theme_data.get("approximate_frequency", 0), + ) + key_to_candidate_theme[theme_data["key"]] = candidate_theme + + if question.theme_status == Question.ThemeStatus.CONFIRMED and theme_data.get("selected"): + selected_theme = SelectedThemeFactory( + question=question, + name=theme_data["name"], + description=theme_data.get("description", ""), + key=theme_data["key"], + ) + candidate_theme.selectedtheme = selected_theme + candidate_theme.save() + + for theme_data in candidate_themes_data: + parent_key = theme_data.get("parent_key") + if parent_key and parent_key in key_to_candidate_theme: + candidate_theme = key_to_candidate_theme[theme_data["key"]] + candidate_theme.parent = key_to_candidate_theme[parent_key] + candidate_theme.save() + + +def create_default_selected_themes(question): + """Create the 'Other' and 'No Reason Given' themes added at start of assign-themes.""" + SelectedTheme.objects.get_or_create( question=question, - description=description, - name=name, - parent=parent, - approximate_frequency=approximate_frequency, + name="Other", + defaults={ + "description": "The response discusses an issue not covered by the listed themes" + }, + ) + SelectedTheme.objects.get_or_create( + question=question, + name="No Reason Given", + defaults={ + "description": "The response does not provide a substantive answer to the question" + }, ) - - if question.theme_status == Question.ThemeStatus.CONFIRMED and candidate_theme_data.get( - "selected" - ): - selected_theme = SelectedThemeFactory( - question=question, name=name, description=description, key=key - ) - candidate_theme.selectedtheme = selected_theme - candidate_theme.save() - - for child in candidate_theme_data.get("children", []): - create_candidate_theme_recursive( - question, number_respondents, child, parent=candidate_theme - ) def create_response(respondent, question, free_text_answers): """Create and return a Response.""" - free_text = random.choice(free_text_answers) if question.has_free_text else None - return ResponseFactory(question=question, free_text=free_text, respondent=respondent) + if question.has_free_text: + non_empty = [a for a in free_text_answers if a not in ("", "Not Provided", "-")] + raw = non_empty[respondent.themefinder_id % len(non_empty)] if non_empty else None + else: + raw = None + return ResponseFactory(question=question, free_text=raw, respondent=respondent) def create_response_annotation(response, question): """Create a ResponseAnnotation and ResponseAnnotationThemes for a free text response.""" selected_themes = list(question.selectedtheme_set.all()) - themes_for_response = random.sample( - selected_themes, - k=random.randint(1, len(selected_themes)), - ) - random_sentiment = random.choice([s[0] for s in ResponseAnnotation.Sentiment.choices]) - random_evidence_rich = random.choice([True, False]) + tf_id = response.respondent.themefinder_id + + # Assign 1 or 2 themes deterministically based on themefinder_id + num_themes = 1 + (tf_id % 2) + themes_for_response = [selected_themes[tf_id % len(selected_themes)]] + if num_themes > 1 and len(selected_themes) > 1: + themes_for_response.append(selected_themes[(tf_id + 1) % len(selected_themes)]) + + sentiment_choices = [s[0] for s in ResponseAnnotation.Sentiment.choices] + sentiment = sentiment_choices[tf_id % len(sentiment_choices)] + evidence_rich = tf_id % 3 == 0 + response_annotation = ResponseAnnotationFactoryNoThemes( response=response, - sentiment=random_sentiment, - evidence_rich=random_evidence_rich, + sentiment=sentiment, + evidence_rich=evidence_rich, ) response_annotation.add_original_ai_themes(themes_for_response) def create_response_chosen_options(response, multiple_choice_options): """Add chosen options to a multiple choice response.""" - chosen_options = random.sample( - multiple_choice_options, - k=random.randint(1, len(multiple_choice_options)), - ) + tf_id = response.respondent.themefinder_id + # Pick 1 or 2 options deterministically based on themefinder_id + num_options = 1 + (tf_id % 2) + chosen_options = [ + multiple_choice_options[(tf_id + j) % len(multiple_choice_options)] + for j in range(num_options) + ] answers = MultiChoiceAnswer.objects.filter(question=response.question, text__in=chosen_options) response.chosen_options.add(*answers) -def create_dummy_consultation_from_yaml( - file_path: str = "./tests/examples/sample_questions.yml", +def candidate_theme_keys_for_respondent(themefinder_id, theme_keys): + """Deterministically assign 1-2 theme keys to a respondent based on their ID.""" + idx = themefinder_id % len(theme_keys) + if themefinder_id % 3 == 0 and len(theme_keys) > 1: + idx2 = (themefinder_id + 1) % len(theme_keys) + return [theme_keys[idx], theme_keys[idx2]] + return [theme_keys[idx]] + + +def create_candidate_theme_responses(question): + """Assign responses to candidate themes at all levels for a question deterministically.""" + all_themes = list(CandidateTheme.objects.filter(question=question)) + responses = list( + Response.objects.filter(question=question, free_text__isnull=False) + .exclude(free_text="") + .select_related("respondent") + ) + if not all_themes or not responses: + return + + themes_by_parent = {} + for theme in all_themes: + themes_by_parent.setdefault(theme.parent_id, []).append(theme) + + records = [] + for sibling_themes in themes_by_parent.values(): + for response in responses: + assigned = candidate_theme_keys_for_respondent( + response.respondent.themefinder_id, + sibling_themes, + ) + for theme in assigned: + records.append(CandidateThemeResponse(candidate_theme=theme, response=response)) + + CandidateThemeResponse.objects.bulk_create(records, ignore_conflicts=True) + + +def create_dummy_consultation( + file_path: str = SAMPLE_QUESTIONS_PATH, number_respondents: int = 10, consultation: Consultation | None = None, - consultation_stage: Literal["finalising_themes", "analysis"] | None = None, -) -> ConsultationFactory: + config: dict | None = None, +) -> Consultation: """ - Create consultation with questions, responses and themes from yaml file. - Creates relevant objects: Consultation, Question, CandidateTheme, SelectedTheme, - Response, ResponseAnnotation, Respondent. + Create consultation with questions, responses and themes from JSON file. + Creates relevant objects depending on stage and theme status: + - SETUP: Consultation, Questions, Respondents, Responses (ready for finding themes) + - FINALISING_THEMES (DRAFT): + CandidateThemes + CandidateThemeResponses (finalising) + - ASSIGNING_THEMES (CONFIRMED): + CandidateThemes + SelectedThemes (ready for assignment) + - ANALYSIS: + CandidateThemes + SelectedThemes + ResponseAnnotations """ - if HostingEnvironment.is_production(): + if HostingEnvironment.is_production() or HostingEnvironment.is_preprod_environment(): raise RuntimeError("Dummy data generation should not be run in production") - consultation_stage = consultation_stage or "analysis" + if config is None: + config = DUMMY_CONSULTATIONS[-1] + + consultation_stage = config["CONSULTATION_STAGE"] + theme_status = config["QUESTION_THEME_STATUS"] + if consultation is None: logger.info("Creating consultation at stage: {stage}", stage=consultation_stage) - consultation = create_consultation(consultation_stage) + consultation = create_consultation(config) logger.info("Creating {number_respondents} respondents", number_respondents=number_respondents) respondents = create_respondents(consultation, number_respondents) with open(file_path, "r") as file: - questions_data = yaml.safe_load(file) + questions_data = json.load(file) + + has_candidate_themes = consultation_stage in [ + Consultation.Stage.FINALISING_THEMES, + Consultation.Stage.ASSIGNING_THEMES, + Consultation.Stage.ANALYSIS, + ] + has_candidate_theme_responses = consultation_stage in [ + Consultation.Stage.FINALISING_THEMES, + Consultation.Stage.ASSIGNING_THEMES, + Consultation.Stage.ANALYSIS, + ] + has_default_selected_themes = consultation_stage == Consultation.Stage.ANALYSIS + has_response_annotations = consultation_stage == Consultation.Stage.ANALYSIS for question_data in questions_data: logger.info("Creating a new question...") - question = create_question(consultation, question_data) + question = create_question(consultation, question_data, theme_status) multiple_choice_options = question_data.get("multiple_choice_options", []) free_text_answers = question_data.get("free_text_answers", []) if question.has_multiple_choice: - logger.info("Multiple choice question - create multi choice answers") create_multi_choice_answers(question, multiple_choice_options) - if question.has_free_text: - logger.info("Free text question - create candidate themes") + if question.has_free_text and has_candidate_themes: + create_candidate_themes(question, question_data["candidate_themes"]) - for candidate_theme_data in question_data["candidate_themes"]: - create_candidate_theme_recursive(question, len(respondents), candidate_theme_data) + if question.has_free_text and has_default_selected_themes: + create_default_selected_themes(question) for respondent in respondents: - logger.info("Creating a new response...") response = create_response(respondent, question, free_text_answers) - if question.has_free_text and consultation.stage == Consultation.Stage.ANALYSIS: - logger.info("Free text question - create response annotation") + if question.has_free_text and has_response_annotations: create_response_annotation(response, question) if question.has_multiple_choice: - logger.info("Multiple choice question - create response chosen options") create_response_chosen_options(response, multiple_choice_options) + if question.has_free_text and has_candidate_theme_responses: + create_candidate_theme_responses(question) + logger.info( "Finished adding question and responses for question {question_number}", question_number=question.number, ) - logger.info( - "Finished adding dummy data for consultation {consultation_code}", - consultation_code=consultation.code, - ) + + logger.info( + "Finished adding dummy data for consultation {consultation_code}", + consultation_code=consultation.code, + ) return consultation # Will only be run occasionally to create dummy data - not in prod @job("default", timeout=2400) -def create_dummy_consultation_from_yaml_job( - file_path: str = "./tests/examples/sample_questions.yml", +def create_dummy_consultation_job( + file_path: str = SAMPLE_QUESTIONS_PATH, number_respondents: int = 10, consultation: Consultation | None = None, + config: dict | None = None, ): - create_dummy_consultation_from_yaml( + create_dummy_consultation( file_path=file_path, number_respondents=number_respondents, consultation=consultation, + config=config, ) diff --git a/backend/consultations/management/commands/generate_dummy_data.py b/backend/consultations/management/commands/generate_dummy_data.py index 92e2e959e..32c48d3e7 100644 --- a/backend/consultations/management/commands/generate_dummy_data.py +++ b/backend/consultations/management/commands/generate_dummy_data.py @@ -1,40 +1,18 @@ from django.core.management.base import BaseCommand -from authentication.models import User -from consultations.dummy_data import create_dummy_consultation_from_yaml -from consultations.models import Consultation - -ADMIN_USER_EMAIL = "admin@example.com" -POLICY_USER_EMAIL = "policy@example.com" +from consultations.dummy_data import ( + DUMMY_CONSULTATIONS, + NUMBER_RESPONDENTS, + create_dummy_consultation, +) class Command(BaseCommand): - help = ( - "Generate two dummy consultations, one at finalising themes stage and one at analysis stage. " - "Generate two dummy users, one with admin (staff) privileges and one without." - ) + help = "Generate dummy consultations at each pipeline stage." def handle(self, *args, **options): self.stdout.write("Generating dummy data...") - create_dummy_consultation_from_yaml( - number_respondents=100, consultation_stage=Consultation.Stage.FINALISING_THEMES - ) - analysis_consultation = create_dummy_consultation_from_yaml( - number_respondents=100, consultation_stage=Consultation.Stage.ANALYSIS - ) - - # Admin user is not added to any consultation as they can act on all consultations - _, created = User.objects.update_or_create( - email=ADMIN_USER_EMAIL, defaults={"is_staff": True} - ) - self.stdout.write(f"{'Created' if created else 'Updated'} admin user ({ADMIN_USER_EMAIL})") - - policy_user, created = User.objects.update_or_create( - email=POLICY_USER_EMAIL, defaults={"is_staff": False} - ) - analysis_consultation.users.add(policy_user) - self.stdout.write( - f"{'Created' if created else 'Updated'} policy user ({POLICY_USER_EMAIL})" - ) - + for config in DUMMY_CONSULTATIONS: + self.stdout.write(f" Creating consultation: {config['CONSULTATION_NAME']}...") + create_dummy_consultation(number_respondents=NUMBER_RESPONDENTS, config=config) self.stdout.write("Done.") diff --git a/backend/consultations/management/commands/prepare_environment.py b/backend/consultations/management/commands/prepare_environment.py new file mode 100644 index 000000000..e3fd8f81b --- /dev/null +++ b/backend/consultations/management/commands/prepare_environment.py @@ -0,0 +1,30 @@ +from django.conf import settings +from django.core.management import call_command +from django.core.management.base import BaseCommand +from django.db import connections + +from hosting_environment import HostingEnvironment + + +class Command(BaseCommand): + help = "Prepare the environment: runs migrations on prod/preprod/test/local; resets and seeds the database and S3 on deployed dev only." + + def handle(self, *args, **options): + environment = getattr(settings, "ENVIRONMENT", "").lower() + + if not HostingEnvironment.is_dev(): + self.stdout.write(f"Running migrate on {environment}.") + call_command("migrate", verbosity=1) + return + + self.stdout.write(f"Resetting database on {environment}...") + connection = connections["default"] + with connection.cursor() as cursor: + cursor.execute("DROP SCHEMA public CASCADE;") + cursor.execute("CREATE SCHEMA public;") + + call_command("migrate", verbosity=1) + call_command("createadminusers", verbosity=1) + call_command("generate_dummy_data", verbosity=1) + call_command("prepare_s3", verbosity=1) + self.stdout.write("Environment prepared.") diff --git a/backend/consultations/management/commands/prepare_s3.py b/backend/consultations/management/commands/prepare_s3.py new file mode 100644 index 000000000..30098c32b --- /dev/null +++ b/backend/consultations/management/commands/prepare_s3.py @@ -0,0 +1,399 @@ +import csv +import datetime +import io +import json +from zoneinfo import ZoneInfo + +import boto3 +from django.conf import settings +from django.core.management.base import BaseCommand + +from consultations.dummy_data import ( + AGE_GROUPS, + NUMBER_RESPONDENTS, + REGIONS, + RESPONDENT_TYPES, + SAMPLE_QUESTIONS_PATH, + candidate_theme_keys_for_respondent, +) +from hosting_environment import HostingEnvironment + + +def _to_jsonl(records): + return "\n".join(json.dumps(r) for r in records) + + +def _load_questions(): + with open(SAMPLE_QUESTIONS_PATH, "r") as f: + return json.load(f) + + +def _build_respondents(): + return [ + { + "themefinder_id": i, + "demographic_data": { + "region": [REGIONS[i % len(REGIONS)]], + "age_group": [AGE_GROUPS[i % len(AGE_GROUPS)]], + "respondent_type": [RESPONDENT_TYPES[i % len(RESPONDENT_TYPES)]], + }, + } + for i in range(1, NUMBER_RESPONDENTS + 1) + ] + + +def _build_responses(question_data): + free_text_answers = question_data.get("free_text_answers", []) + if not free_text_answers: + return [] + non_empty = [a for a in free_text_answers if a] + return [ + {"themefinder_id": i, "text": non_empty[i % len(non_empty)]} + for i in range(1, NUMBER_RESPONDENTS + 1) + ] + + +def _build_multi_choice(question_data): + options = question_data.get("multiple_choice_options", []) + if not options: + return [] + return [ + {"themefinder_id": i, "options": [options[i % len(options)]]} + for i in range(1, NUMBER_RESPONDENTS + 1) + ] + + +def _build_question_json(question_data): + return { + "question_text": question_data["question_text"], + "question_number": question_data["number"], + "has_free_text": question_data["has_free_text"], + "multi_choice_options": question_data.get("multiple_choice_options", []), + } + + +def _build_clustered_themes(question_data): + """Build clustered_themes.json from the flat candidate_themes list.""" + candidate_themes = question_data.get("candidate_themes", []) + key_to_topic_id = {} + theme_nodes = [] + + for i, theme in enumerate(candidate_themes, start=1): + topic_id = str(i) + key_to_topic_id[theme["key"]] = topic_id + parent_key = theme.get("parent_key") + parent_id = key_to_topic_id.get(parent_key, "0") if parent_key else "0" + theme_nodes.append( + { + "topic_id": topic_id, + "parent_id": parent_id, + "topic_label": theme["name"], + "topic_description": theme.get("description", ""), + "source_topic_count": max(1, theme.get("approximate_frequency", 10)), + } + ) + + return {"theme_nodes": theme_nodes} + + +def _themes_grouped_by_parent(candidate_themes): + """Group flat themes by parent_key (None for top-level).""" + groups = {} + for theme in candidate_themes: + groups.setdefault(theme.get("parent_key"), []).append(theme) + return groups + + +def _build_candidate_themes_json(question_data): + """Build themes.json from all candidate_themes (for THEME_SIGN_OFF mapping).""" + candidate_themes = question_data.get("candidate_themes", []) + return [ + { + "theme_key": theme["key"], + "theme_name": theme["name"], + "theme_description": theme.get("description", ""), + } + for theme in candidate_themes + ] + + +DEFAULT_THEMES = [ + { + "theme_key": "OTHER", + "theme_name": "Other", + "theme_description": "The response discusses an issue not covered by the listed themes", + }, + { + "theme_key": "NO_REASON", + "theme_name": "No Reason Given", + "theme_description": "The response does not provide a substantive answer to the question", + }, +] + + +def _build_themes_json(question_data): + """Build themes.json (selected themes) from candidate_themes marked as selected, plus defaults.""" + candidate_themes = question_data.get("candidate_themes", []) + themes = [ + { + "theme_key": theme["key"], + "theme_name": theme["name"], + "theme_description": theme.get("description", ""), + } + for theme in candidate_themes + if theme.get("selected") + ] + themes.extend(DEFAULT_THEMES) + return themes + + +def _build_hierarchical_candidate_theme_mappings(question_data): + """Build mapping.jsonl for candidate themes, respecting the parent-child hierarchy. + + Themes are grouped by parent so that sibling themes compete against each other — + each respondent is assigned 1–2 keys from each sibling group independently. + This mirrors how the real find-themes pipeline produces clustered outputs. + """ + candidate_themes = question_data.get("candidate_themes", []) + if not candidate_themes: + return [] + + groups = _themes_grouped_by_parent(candidate_themes) + respondent_keys = {i: [] for i in range(1, NUMBER_RESPONDENTS + 1)} + for sibling_themes in groups.values(): + group_keys = [t["key"] for t in sibling_themes] + for i in range(1, NUMBER_RESPONDENTS + 1): + respondent_keys[i].extend(candidate_theme_keys_for_respondent(i, group_keys)) + + return [ + {"themefinder_id": i, "theme_keys": respondent_keys[i]} + for i in range(1, NUMBER_RESPONDENTS + 1) + ] + + +def _build_theme_mappings(question_data): + """Build mapping.jsonl for selected themes (analysis stage) using deterministic assignment.""" + themes = _build_themes_json(question_data) + if not themes: + return [] + theme_keys = [t["theme_key"] for t in themes] + return [ + {"themefinder_id": i, "theme_keys": candidate_theme_keys_for_respondent(i, theme_keys)} + for i in range(1, NUMBER_RESPONDENTS + 1) + ] + + +def _build_sentiments(): + choices = ["AGREEMENT", "DISAGREEMENT", "UNCLEAR"] + return [ + {"themefinder_id": i, "sentiment": choices[i % len(choices)]} + for i in range(1, NUMBER_RESPONDENTS + 1) + ] + + +def _build_evidence_rich(): + choices = ["YES", "NO"] + return [ + {"themefinder_id": i, "evidence_rich": choices[i % len(choices)]} + for i in range(1, NUMBER_RESPONDENTS + 1) + ] + + +def _build_themes_csv(question_data): + """Build themes.csv content (selected themes) for the assign-themes batch job.""" + themes = _build_themes_json(question_data) + buf = io.StringIO() + writer = csv.writer(buf) + writer.writerow(["Theme Name", "Theme Description"]) + for theme in themes: + writer.writerow([theme["theme_name"], theme["theme_description"]]) + return buf.getvalue() + + +class Command(BaseCommand): + help = "Reset and seed S3 with dummy consultation data matching the DB. Only runs on deployed dev environments." + + def handle(self, *args, **options): + environment = getattr(settings, "ENVIRONMENT", "").lower() + + if not HostingEnvironment.is_dev(): + self.stdout.write(f"Skipping S3 seed on {environment} environment.") + return + + s3_client = boto3.client("s3") + bucket = settings.AWS_BUCKET_NAME + timestamp = datetime.datetime.now(tz=ZoneInfo("Europe/London")).date() + + self._delete_existing_data(s3_client, bucket) + questions_data = _load_questions() + + # S3-only consultation (no DB record) + self._seed_consultation(s3_client, bucket, "dummy-s3-only", questions_data, timestamp) + + # SETUP — inputs only + self._seed_consultation(s3_client, bucket, "dummy-setup", questions_data, timestamp) + + # Starting finalising themes — has clustered themes + candidate theme mappings + self._seed_consultation( + s3_client, + bucket, + "dummy-start-finalising-themes", + questions_data, + timestamp, + include_clustered_themes=True, + include_candidate_theme_mappings=True, + ) + + # Finished finalising themes — has clustered themes + themes.csv (ready for assignment) + self._seed_consultation( + s3_client, + bucket, + "dummy-finished-finalising-themes", + questions_data, + timestamp, + include_clustered_themes=True, + include_candidate_theme_mappings=True, + include_themes_csv=True, + ) + + # ANALYSIS — has everything from earlier stages + mapping outputs + self._seed_consultation( + s3_client, + bucket, + "dummy-analysis", + questions_data, + timestamp, + include_clustered_themes=True, + include_candidate_theme_mappings=True, + include_mapping_outputs=True, + include_themes_csv=True, + ) + + self.stdout.write("S3 seed complete.") + + def _delete_existing_data(self, s3_client, bucket): + prefix = "app_data/consultations/" + paginator = s3_client.get_paginator("list_objects_v2") + objects_to_delete = [] + + for page in paginator.paginate(Bucket=bucket, Prefix=prefix): + for obj in page.get("Contents", []): + objects_to_delete.append({"Key": obj["Key"]}) + + if objects_to_delete: + # delete_objects supports max 1000 keys per call + for i in range(0, len(objects_to_delete), 1000): + s3_client.delete_objects( + Bucket=bucket, Delete={"Objects": objects_to_delete[i : i + 1000]} + ) + self.stdout.write(f" Deleted {len(objects_to_delete)} existing objects.") + + def _seed_consultation( + self, + s3_client, + bucket, + code, + questions_data, + timestamp, + include_clustered_themes=False, + include_candidate_theme_mappings=False, + include_mapping_outputs=False, + include_themes_csv=False, + ): + self.stdout.write(f" Seeding S3 data for: {code}") + prefix = f"app_data/consultations/{code}" + + # Respondents + s3_client.put_object( + Bucket=bucket, + Key=f"{prefix}/inputs/respondents.jsonl", + Body=_to_jsonl(_build_respondents()), + ) + + for question_data in questions_data: + q_num = question_data["number"] + q_prefix = f"{prefix}/inputs/question_part_{q_num}" + + # Question definition + s3_client.put_object( + Bucket=bucket, + Key=f"{q_prefix}/question.json", + Body=json.dumps(_build_question_json(question_data)), + ) + + # Free text responses + if question_data["has_free_text"]: + responses = _build_responses(question_data) + if responses: + s3_client.put_object( + Bucket=bucket, + Key=f"{q_prefix}/responses.jsonl", + Body=_to_jsonl(responses), + ) + + # Multi choice + if question_data.get("multiple_choice_options"): + multi_choice = _build_multi_choice(question_data) + if multi_choice: + s3_client.put_object( + Bucket=bucket, + Key=f"{q_prefix}/multi_choice.jsonl", + Body=_to_jsonl(multi_choice), + ) + + # Stage-specific outputs (only for free text questions) + if not question_data["has_free_text"]: + continue + + if include_clustered_themes: + key = f"{prefix}/outputs/sign_off/{timestamp}/question_part_{q_num}/clustered_themes.json" + s3_client.put_object( + Bucket=bucket, + Key=key, + Body=json.dumps(_build_clustered_themes(question_data)), + ) + + if include_candidate_theme_mappings and not include_mapping_outputs: + out_prefix = f"{prefix}/outputs/mapping/{timestamp}/question_part_{q_num}" + themes = _build_candidate_themes_json(question_data) + s3_client.put_object( + Bucket=bucket, + Key=f"{out_prefix}/themes.json", + Body=json.dumps(themes), + ) + s3_client.put_object( + Bucket=bucket, + Key=f"{out_prefix}/mapping.jsonl", + Body=_to_jsonl(_build_hierarchical_candidate_theme_mappings(question_data)), + ) + + if include_themes_csv: + s3_client.put_object( + Bucket=bucket, + Key=f"{q_prefix}/themes.csv", + Body=_build_themes_csv(question_data), + ) + + if include_mapping_outputs: + out_prefix = f"{prefix}/outputs/mapping/{timestamp}/question_part_{q_num}" + themes = _build_themes_json(question_data) + s3_client.put_object( + Bucket=bucket, + Key=f"{out_prefix}/themes.json", + Body=json.dumps(themes), + ) + s3_client.put_object( + Bucket=bucket, + Key=f"{out_prefix}/mapping.jsonl", + Body=_to_jsonl(_build_theme_mappings(question_data)), + ) + s3_client.put_object( + Bucket=bucket, + Key=f"{out_prefix}/sentiment.jsonl", + Body=_to_jsonl(_build_sentiments()), + ) + s3_client.put_object( + Bucket=bucket, + Key=f"{out_prefix}/detail_detection.jsonl", + Body=_to_jsonl(_build_evidence_rich()), + ) diff --git a/backend/consultations/migrations/0099_alter_consultation_stage_default.py b/backend/consultations/migrations/0099_alter_consultation_stage_default.py index d1a946354..380c5a05e 100644 --- a/backend/consultations/migrations/0099_alter_consultation_stage_default.py +++ b/backend/consultations/migrations/0099_alter_consultation_stage_default.py @@ -6,15 +6,26 @@ class Migration(migrations.Migration): - dependencies: ClassVar[list] = [ - ('consultations', '0098_migrate_legacy_stages'), + ("consultations", "0098_migrate_legacy_stages"), ] operations: ClassVar[list] = [ migrations.AlterField( - model_name='consultation', - name='stage', - field=models.CharField(choices=[('setup', 'Data Setup'), ('finding_themes', 'Finding Themes'), ('finalising_themes', 'Finalising Themes'), ('assigning_themes', 'Assigning Themes'), ('analysis', 'Analysis'), ('theme_sign_off', 'Theme Sign Off'), ('theme_mapping', 'Theme Mapping')], default='finalising_themes', max_length=32), + model_name="consultation", + name="stage", + field=models.CharField( + choices=[ + ("setup", "Data Setup"), + ("finding_themes", "Finding Themes"), + ("finalising_themes", "Finalising Themes"), + ("assigning_themes", "Assigning Themes"), + ("analysis", "Analysis"), + ("theme_sign_off", "Theme Sign Off"), + ("theme_mapping", "Theme Mapping"), + ], + default="finalising_themes", + max_length=32, + ), ), ] diff --git a/backend/consultations/migrations/0100_remove_legacy_stage_choices.py b/backend/consultations/migrations/0100_remove_legacy_stage_choices.py index fee9cd96b..1e205a9d1 100644 --- a/backend/consultations/migrations/0100_remove_legacy_stage_choices.py +++ b/backend/consultations/migrations/0100_remove_legacy_stage_choices.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies: ClassVar[list] = [ ("consultations", "0099_alter_consultation_stage_default"), ] diff --git a/backend/consultations/migrations/0102_responsereadby_alter_response_read_by.py b/backend/consultations/migrations/0102_responsereadby_alter_response_read_by.py index c69d03a63..7c0dfca2b 100644 --- a/backend/consultations/migrations/0102_responsereadby_alter_response_read_by.py +++ b/backend/consultations/migrations/0102_responsereadby_alter_response_read_by.py @@ -6,7 +6,6 @@ class Migration(migrations.Migration): - dependencies: ClassVar[list] = [ ("consultations", "0101_clean_empty_responses_and_denormalise_counts"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), diff --git a/backend/consultations/migrations/0103_add_response_question_id_index.py b/backend/consultations/migrations/0103_add_response_question_id_index.py index c547f465e..e1def4028 100644 --- a/backend/consultations/migrations/0103_add_response_question_id_index.py +++ b/backend/consultations/migrations/0103_add_response_question_id_index.py @@ -7,15 +7,14 @@ class Migration(migrations.Migration): - dependencies: ClassVar[list] = [ - ('consultations', '0102_responsereadby_alter_response_read_by'), + ("consultations", "0102_responsereadby_alter_response_read_by"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations: ClassVar[list] = [ migrations.AddIndex( - model_name='response', - index=models.Index(fields=['question', 'id'], name='consultatio_questio_13c9db_idx'), + model_name="response", + index=models.Index(fields=["question", "id"], name="consultatio_questio_13c9db_idx"), ), ] diff --git a/backend/consultations/test_support/load_test_fixtures.py b/backend/consultations/test_support/load_test_fixtures.py index 95c15981e..7ed9c9c3c 100644 --- a/backend/consultations/test_support/load_test_fixtures.py +++ b/backend/consultations/test_support/load_test_fixtures.py @@ -34,8 +34,7 @@ def create_response_from_fixtures(respondents, index, question_object, response_ if "themes" in response_data: annotation = ResponseAnnotation.objects.create( - response=response, - evidence_rich=response_data.get("evidence_rich", False) + response=response, evidence_rich=response_data.get("evidence_rich", False) ) annotation.add_original_ai_themes( SelectedTheme.objects.filter(question=question_object, key__in=response_data["themes"]) @@ -44,10 +43,10 @@ def create_response_from_fixtures(respondents, index, question_object, response_ if "demographics" in response_data: for key, value in response_data["demographics"].items(): option, _ = DemographicOption.objects.get_or_create( - consultation=question_object.consultation, - field_name=key, - field_value=value, - ) + consultation=question_object.consultation, + field_name=key, + field_value=value, + ) respondents[index].demographics.add(option) @@ -96,9 +95,7 @@ def create_question_from_fixtures(consultation_object, respondents, question_dat CandidateTheme.objects.bulk_create( [ CandidateTheme( - question=question_object, - name=t["name"], - description=t["description"] + question=question_object, name=t["name"], description=t["description"] ) for t in question_data["candidate_themes"] ] @@ -110,7 +107,7 @@ def create_question_from_fixtures(consultation_object, respondents, question_dat question_object.update_response_counts() MultiChoiceAnswer.update_response_counts(question_object) - + return question_object @@ -157,7 +154,9 @@ def create_data_from_fixtures(fixtures): respondents = create_respondents_from_fixtures(consultation_data, consultation_object) for question_data in consultation_data.get("questions", []): - question_object = create_question_from_fixtures(consultation_object, respondents, question_data) + question_object = create_question_from_fixtures( + consultation_object, respondents, question_data + ) questions.append(question_object.id) # Make sure that demographic response counts are updated after all responses have been created diff --git a/backend/consultations/utils/s3.py b/backend/consultations/utils/s3.py index c05e3d19c..3d3358dfc 100644 --- a/backend/consultations/utils/s3.py +++ b/backend/consultations/utils/s3.py @@ -4,6 +4,7 @@ logger = settings.LOGGER + def get_s3_client(config: Config | None = None): config = config or Config() if settings.ENVIRONMENT.upper() in ["LOCAL", "TEST"]: diff --git a/backend/data_pipeline/s3.py b/backend/data_pipeline/s3.py index da0a6df4d..1053068e3 100644 --- a/backend/data_pipeline/s3.py +++ b/backend/data_pipeline/s3.py @@ -9,9 +9,7 @@ account_id = settings.AWS_ACCOUNT_ID -def read_jsonl( - bucket_name: str, key: str, raise_if_missing: bool = True -) -> list[dict]: +def read_jsonl(bucket_name: str, key: str, raise_if_missing: bool = True) -> list[dict]: """ Read a JSONL file from S3 and return list of parsed objects. Args: @@ -46,9 +44,7 @@ def read_jsonl( return objects -def read_json( - bucket_name: str, key: str, raise_if_missing: bool = True -) -> dict | None: +def read_json(bucket_name: str, key: str, raise_if_missing: bool = True) -> dict | None: """ Read a JSON file from S3 and return parsed object. Args: @@ -101,7 +97,7 @@ def get_question_folders(inputs_path: str, bucket_name: str) -> list[str]: "Bucket": bucket_name, "Prefix": inputs_path, "Delimiter": "/", # Group by directory to get only subdirectories - "MaxKeys": 1000, # Max allowed per page by AWS + "MaxKeys": 1000, # Max allowed per page by AWS } if settings.ENVIRONMENT.upper() not in ["LOCAL", "TEST"]: @@ -193,7 +189,7 @@ def get_consultation_folders() -> list[str]: "Bucket": settings.AWS_BUCKET_NAME, "Prefix": "app_data/consultations/", "Delimiter": "/", # Group by directory to get only top-level consultation folders - "MaxKeys": 1000, # Max allowed per page by AWS + "MaxKeys": 1000, # Max allowed per page by AWS } if settings.ENVIRONMENT.upper() not in ["LOCAL", "TEST"]: diff --git a/backend/data_pipeline/sync/candidate_themes.py b/backend/data_pipeline/sync/candidate_themes.py index 25352026d..0ca91677f 100644 --- a/backend/data_pipeline/sync/candidate_themes.py +++ b/backend/data_pipeline/sync/candidate_themes.py @@ -54,9 +54,7 @@ def load_candidate_themes_from_s3( logger.info("Loading candidate themes from {key}", key=key) # Read and parse JSON file - theme_data = s3.read_json( - bucket_name=bucket_name_str, key=key, raise_if_missing=False - ) + theme_data = s3.read_json(bucket_name=bucket_name_str, key=key, raise_if_missing=False) if theme_data is None: logger.info("No candidate themes file found at {key}", key=key) diff --git a/backend/data_pipeline/sync/consultation_setup.py b/backend/data_pipeline/sync/consultation_setup.py index 46490db43..6861ee4b8 100644 --- a/backend/data_pipeline/sync/consultation_setup.py +++ b/backend/data_pipeline/sync/consultation_setup.py @@ -35,9 +35,7 @@ # ============================================================================= -def load_respondents_from_s3( - consultation_code: str, bucket_name: str -) -> list[RespondentInput]: +def load_respondents_from_s3(consultation_code: str, bucket_name: str) -> list[RespondentInput]: """ Load and validate respondents from S3. @@ -269,16 +267,12 @@ def load_consultation_data_batch( for question_number in question_numbers: # Load free text responses - responses = load_responses_from_s3( - consultation_code, question_number, bucket_name - ) + responses = load_responses_from_s3(consultation_code, question_number, bucket_name) if responses: responses_by_question[question_number] = responses # Load multi-choice data - multi_choices = load_multi_choice_from_s3( - consultation_code, question_number, bucket_name - ) + multi_choices = load_multi_choice_from_s3(consultation_code, question_number, bucket_name) if multi_choices: multi_choice_by_question[question_number] = multi_choices diff --git a/backend/data_pipeline/sync/response_annotations.py b/backend/data_pipeline/sync/response_annotations.py index b43adb963..83acd8604 100644 --- a/backend/data_pipeline/sync/response_annotations.py +++ b/backend/data_pipeline/sync/response_annotations.py @@ -1,4 +1,3 @@ - from botocore.exceptions import BotoCoreError, ClientError from django.conf import settings from django.db import transaction @@ -56,15 +55,13 @@ def load_selected_themes_from_s3( try: # Read and parse JSON file - theme_data = s3.read_json( - bucket_name=bucket_name_str, key=key, raise_if_missing=True - ) + theme_data = s3.read_json(bucket_name=bucket_name_str, key=key, raise_if_missing=True) except (ClientError, BotoCoreError) as e: logger.exception( "Failed to load selected themes from S3 for consultation '{consultation_code}'," " question {question_number}", consultation_code=consultation_code, - question_number=question_number + question_number=question_number, ) if isinstance(e, ClientError) and e.response["Error"]["Code"] == "NoSuchKey": raise ValueError( @@ -115,15 +112,13 @@ def load_sentiments_from_s3( try: # Read JSONL file (raise_if_missing=False because sentiment is optional) - sentiment_data = s3.read_jsonl( - bucket_name=bucket_name_str, key=key, raise_if_missing=False - ) + sentiment_data = s3.read_jsonl(bucket_name=bucket_name_str, key=key, raise_if_missing=False) except (ClientError, BotoCoreError): logger.exception( "Failed to load sentiments from S3 for consultation '{consultation_code}'," " question {question_number}", consultation_code=consultation_code, - question_number=question_number + question_number=question_number, ) raise @@ -172,15 +167,13 @@ def load_detail_detections_from_s3( try: # Read JSONL file - detail_data = s3.read_jsonl( - bucket_name=bucket_name_str, key=key, raise_if_missing=True - ) + detail_data = s3.read_jsonl(bucket_name=bucket_name_str, key=key, raise_if_missing=True) except (ClientError, BotoCoreError) as e: logger.exception( "Failed to load detail detections from S3 for consultation '{consultation_code}'," " question {question_number}", consultation_code=consultation_code, - question_number=question_number + question_number=question_number, ) if isinstance(e, ClientError) and e.response["Error"]["Code"] == "NoSuchKey": raise ValueError( @@ -230,15 +223,13 @@ def load_theme_mappings_from_s3( try: # Read JSONL file - mapping_data = s3.read_jsonl( - bucket_name=bucket_name_str, key=key, raise_if_missing=True - ) + mapping_data = s3.read_jsonl(bucket_name=bucket_name_str, key=key, raise_if_missing=True) except (ClientError, BotoCoreError) as e: logger.exception( "Failed to load theme mappings from S3 for consultation '{consultation_code}'," " question {question_number}", consultation_code=consultation_code, - question_number=question_number + question_number=question_number, ) if isinstance(e, ClientError) and e.response["Error"]["Code"] == "NoSuchKey": raise ValueError( @@ -301,7 +292,7 @@ def load_annotation_batch( logger.exception( "Consultation with code '{consultation_code}' does not exist. " "Base consultation data must be imported before annotations.", - consultation_code=consultation_code + consultation_code=consultation_code, ) raise ValueError( f"Consultation with code '{consultation_code}' does not exist. " diff --git a/backend/factories.py b/backend/factories.py index f2f8b904d..c8b6856ca 100644 --- a/backend/factories.py +++ b/backend/factories.py @@ -112,9 +112,7 @@ def encode(obj): field_value=encode(v), ) self.demographics.add(o) - DemographicOption.objects.filter(pk=o.pk).update( - response_count=F("response_count") + 1 - ) + DemographicOption.objects.filter(pk=o.pk).update(response_count=F("response_count") + 1) self.save() diff --git a/backend/hosting_environment.py b/backend/hosting_environment.py index 83d6375c3..9f24f2ec8 100644 --- a/backend/hosting_environment.py +++ b/backend/hosting_environment.py @@ -22,8 +22,17 @@ def is_deployed() -> bool: def is_production() -> bool: return env.str("ENVIRONMENT", "").upper() == "PROD" + @staticmethod + def is_preprod_environment() -> bool: + environment = env.str("ENVIRONMENT", "").upper() + return environment == "PREPROD" + @staticmethod def is_development_environment() -> bool: environment = env.str("ENVIRONMENT", "").upper() development_environments = ["LOCAL", "TEST", "DEV", "DEVELOPMENT"] return environment in development_environments + + @staticmethod + def is_dev() -> bool: + return env.str("ENVIRONMENT", "").upper() == "DEV" diff --git a/backend/rq_context.py b/backend/rq_context.py index 544163d2b..be58399fa 100644 --- a/backend/rq_context.py +++ b/backend/rq_context.py @@ -37,7 +37,9 @@ def context_aware(*args, context_id: str | None = None, **kwargs): @functools.wraps(func) def enqueue_with_context(*args, context_id: str | None = None, **kwargs): - return enqueue_call(*args, context_id=context_id or get_or_create_context_id(), **kwargs) + return enqueue_call( + *args, context_id=context_id or get_or_create_context_id(), **kwargs + ) decorated.delay = enqueue_with_context decorated.enqueue = enqueue_with_context diff --git a/backend/settings/local.py b/backend/settings/local.py index 3c08617b7..7504c5b8e 100644 --- a/backend/settings/local.py +++ b/backend/settings/local.py @@ -7,9 +7,7 @@ STORAGES["default"] = { "BACKEND": "django.core.files.storage.FileSystemStorage", - "OPTIONS": { - "location": BASE_DIR / "tmp" - }, + "OPTIONS": {"location": BASE_DIR / "tmp"}, } REST_FRAMEWORK["DEFAULT_SCHEMA_CLASS"] = "drf_spectacular.openapi.AutoSchema" diff --git a/backend/start.sh b/backend/start.sh index bb8a28e13..fde00ca0b 100755 --- a/backend/start.sh +++ b/backend/start.sh @@ -1,10 +1,11 @@ #!/bin/sh -# Collect static files -venv/bin/django-admin collectstatic --noinput +# Collect static files (skip in test environment - no static serving needed) +if [ "$(echo "$ENVIRONMENT" | tr '[:upper:]' '[:lower:]')" != "test" ]; then + venv/bin/django-admin collectstatic --noinput +fi -venv/bin/django-admin migrate -venv/bin/django-admin createadminusers +venv/bin/django-admin prepare_environment venv/bin/django-admin populate_history --auto --batchsize 1000 exec venv/bin/gunicorn -c ./gunicorn_config.py backend.wsgi diff --git a/backend/tests/commands/test_dummy_data.py b/backend/tests/commands/test_dummy_data.py index cbc7dcc0d..43578c62f 100644 --- a/backend/tests/commands/test_dummy_data.py +++ b/backend/tests/commands/test_dummy_data.py @@ -16,15 +16,16 @@ def test_name_parameter_sets_consultation_name(mock_is_local): stdout=StringIO(), # we'll ignore this ) - assert models.Consultation.objects.count() == 2 - assert models.Question.objects.count() == 8 + assert models.Consultation.objects.count() == 4 + assert models.Question.objects.count() == 16 @pytest.mark.django_db -@pytest.mark.parametrize("environment", ["prod"]) +@pytest.mark.parametrize("environment", ["prod", "preprod"]) def test_the_tool_will_only_run_in_dev(environment): - with patch.dict(os.environ, {"ENVIRONMENT": environment}), pytest.raises( - Exception, match=r"Dummy data generation should not be run in production" + with ( + patch.dict(os.environ, {"ENVIRONMENT": environment}), + pytest.raises(Exception, match=r"Dummy data generation should not be run in production"), ): call_command( "generate_dummy_data", diff --git a/backend/tests/commands/test_prepare_environment.py b/backend/tests/commands/test_prepare_environment.py new file mode 100644 index 000000000..14f96548a --- /dev/null +++ b/backend/tests/commands/test_prepare_environment.py @@ -0,0 +1,82 @@ +from unittest.mock import patch + +import boto3 +import pytest +from django.core.management import call_command +from moto import mock_aws + +from authentication.models import User +from consultations.models import Consultation + + +class TestPrepareEnvironment: + @pytest.mark.django_db + @pytest.mark.parametrize("environment", ["prod", "preprod", "test", "", "unknown", "staging", "local"]) + def test_does_not_reset_on_non_dev(self, settings, environment): + settings.ENVIRONMENT = environment + + Consultation.objects.create(title="Should survive", code="KEEP_ME") + call_command("prepare_environment") + + assert Consultation.objects.filter(code="KEEP_ME").exists() + + @pytest.mark.django_db + @mock_aws + @patch("consultations.management.commands.prepare_s3.HostingEnvironment") + @patch("consultations.management.commands.prepare_environment.HostingEnvironment") + @patch("factories.embed_text", return_value=[0.0] * 3072) + def test_resets_and_seeds_db(self, _mock_embed, mock_env, mock_s3_env, settings): + mock_env.is_dev.return_value = True + mock_s3_env.is_dev.return_value = True + + settings.ENVIRONMENT = "dev" + settings.AWS_BUCKET_NAME = "test-bucket" + + boto3.resource("s3", region_name="eu-west-2").create_bucket( + Bucket="test-bucket", + CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, + ) + + Consultation.objects.create(title="Should be deleted", code="DELETE_ME") + call_command("prepare_environment") + + # Old data is gone + assert not Consultation.objects.filter(code="DELETE_ME").exists() + + # Consultations created at each stage + assert Consultation.objects.filter(stage=Consultation.Stage.SETUP).exists() + assert Consultation.objects.filter(stage=Consultation.Stage.FINALISING_THEMES).exists() + assert Consultation.objects.filter(stage=Consultation.Stage.ASSIGNING_THEMES).exists() + assert Consultation.objects.filter(stage=Consultation.Stage.ANALYSIS).exists() + + # Admin user was created + assert User.objects.filter(email="admin@example.com", is_staff=True).exists() + + @pytest.mark.django_db + @mock_aws + @patch("consultations.management.commands.prepare_s3.HostingEnvironment") + @patch("consultations.management.commands.prepare_environment.HostingEnvironment") + @patch("factories.embed_text", return_value=[0.0] * 3072) + def test_seeds_s3(self, _mock_embed, mock_env, mock_s3_env, settings): + mock_env.is_dev.return_value = True + mock_s3_env.is_dev.return_value = True + + settings.ENVIRONMENT = "dev" + settings.AWS_BUCKET_NAME = "test-bucket" + + conn = boto3.resource("s3", region_name="eu-west-2") + conn.create_bucket( + Bucket="test-bucket", + CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, + ) + + call_command("prepare_environment") + + bucket = conn.Bucket("test-bucket") + keys = [obj.key for obj in bucket.objects.all()] + + assert any("dummy-s3-only/" in k for k in keys) + assert any("dummy-setup/" in k for k in keys) + assert any("dummy-start-finalising-themes/" in k for k in keys) + assert any("dummy-finished-finalising-themes/" in k for k in keys) + assert any("dummy-analysis/" in k for k in keys) diff --git a/backend/tests/commands/test_prepare_s3.py b/backend/tests/commands/test_prepare_s3.py new file mode 100644 index 000000000..5f9762ef3 --- /dev/null +++ b/backend/tests/commands/test_prepare_s3.py @@ -0,0 +1,126 @@ +from unittest.mock import patch + +import boto3 +import pytest +from django.core.management import call_command +from moto import mock_aws + + +class TestPrepareS3: + @pytest.mark.django_db + @mock_aws + @patch("consultations.management.commands.prepare_s3.HostingEnvironment") + def test_deletes_existing_data_before_seeding(self, mock_hosting_env, settings): + mock_hosting_env.is_dev.return_value = True + settings.AWS_BUCKET_NAME = "test-bucket" + + conn = boto3.resource("s3", region_name="eu-west-2") + conn.create_bucket( + Bucket="test-bucket", + CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, + ) + + # Pre-seed a stale object that should be cleaned up + s3_client = boto3.client("s3", region_name="eu-west-2") + s3_client.put_object( + Bucket="test-bucket", + Key="app_data/consultations/old-consultation/inputs/respondents.jsonl", + Body="stale data", + ) + + call_command("prepare_s3") + + bucket = conn.Bucket("test-bucket") + keys = [obj.key for obj in bucket.objects.all()] + + assert not any("old-consultation" in k for k in keys) + assert any("dummy-s3-only/" in k for k in keys) + + @pytest.mark.django_db + @mock_aws + @patch("consultations.management.commands.prepare_s3.HostingEnvironment") + def test_seeds_s3_with_data_for_each_stage(self, mock_hosting_env, settings): + mock_hosting_env.is_dev.return_value = True + settings.AWS_BUCKET_NAME = "test-bucket" + + conn = boto3.resource("s3", region_name="eu-west-2") + conn.create_bucket( + Bucket="test-bucket", + CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, + ) + + call_command("prepare_s3") + + bucket = conn.Bucket("test-bucket") + keys = [obj.key for obj in bucket.objects.all()] + + # S3-only consultation has input data + assert any("dummy-s3-only/inputs/respondents.jsonl" in k for k in keys) + assert any("dummy-s3-only/inputs/question_part_1/question.json" in k for k in keys) + + # Setup consultation has input data + assert any("dummy-setup/inputs/respondents.jsonl" in k for k in keys) + assert any("dummy-setup/inputs/question_part_1/question.json" in k for k in keys) + + # Starting finalising themes has input data, clustered themes and candidate theme mappings + assert any("dummy-start-finalising-themes/inputs/respondents.jsonl" in k for k in keys) + assert any( + "dummy-start-finalising-themes/inputs/question_part_1/question.json" in k for k in keys + ) + assert any( + "dummy-start-finalising-themes/outputs/sign_off/" in k and "clustered_themes.json" in k + for k in keys + ) + assert any( + "dummy-start-finalising-themes/outputs/mapping/" in k and "mapping.jsonl" in k + for k in keys + ) + + # Finished finalising themes has input data, clustered themes and candidate theme mappings + assert any("dummy-finished-finalising-themes/inputs/respondents.jsonl" in k for k in keys) + assert any( + "dummy-finished-finalising-themes/inputs/question_part_1/question.json" in k + for k in keys + ) + assert any( + "dummy-finished-finalising-themes/outputs/sign_off/" in k + and "clustered_themes.json" in k + for k in keys + ) + assert any( + "dummy-finished-finalising-themes/outputs/mapping/" in k and "mapping.jsonl" in k + for k in keys + ) + + # Analysis has all the above and mapping outputs + assert any("dummy-analysis/inputs/respondents.jsonl" in k for k in keys) + assert any("dummy-analysis/inputs/question_part_1/question.json" in k for k in keys) + assert any( + "dummy-analysis/outputs/sign_off/" in k and "clustered_themes.json" in k for k in keys + ) + assert any("dummy-analysis/outputs/mapping/" in k and "mapping.jsonl" in k for k in keys) + assert any("dummy-analysis/outputs/mapping/" in k and "themes.json" in k for k in keys) + assert any("dummy-analysis/outputs/mapping/" in k and "sentiment.jsonl" in k for k in keys) + assert any( + "dummy-analysis/outputs/mapping/" in k and "detail_detection.jsonl" in k for k in keys + ) + + @pytest.mark.django_db + @mock_aws + @pytest.mark.parametrize("environment", ["local", "preprod", "prod", "test"]) + @patch("consultations.management.commands.prepare_s3.HostingEnvironment") + def test_skips_on_non_dev(self, mock_hosting_env, environment, settings): + mock_hosting_env.is_dev.return_value = False + settings.AWS_BUCKET_NAME = "test-bucket" + + conn = boto3.resource("s3", region_name="eu-west-2") + conn.create_bucket( + Bucket="test-bucket", + CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, + ) + + call_command("prepare_s3") + + bucket = conn.Bucket("test-bucket") + keys = [obj.key for obj in bucket.objects.all()] + assert len(keys) == 0 diff --git a/backend/tests/examples/sample_questions.json b/backend/tests/examples/sample_questions.json new file mode 100644 index 000000000..83f02fcca --- /dev/null +++ b/backend/tests/examples/sample_questions.json @@ -0,0 +1,74 @@ +[ + { + "question_text": "Do you agree with the proposal to align the flavour categories of chocolate bars as outlined in the draft guidelines of the Chocolate Bar Regulation for the United Kingdom?", + "number": 1, + "has_free_text": true, + "has_multiple_choice": true, + "multiple_choice_options": ["Yes", "No", "Don't know", "No answer"], + "free_text_answers": [ + "", + "Yes, I agree with the proposal as it will create a standardized framework that benefits both consumers and manufacturers.", + "No, I do not agree because I feel the proposed categories are too restrictive and may stifle innovation in flavour development.", + "I agree with the proposal, but I think there should be room for additional categories to accommodate future trends.", + "I disagree with the proposal as it does not sufficiently account for regional flavour preferences across the UK.", + "Yes, aligning the flavour categories will help consumers easily compare products and make informed choices.", + "I neither agree nor disagree at this moment; I would like more information on how the categories were determined.", + "I support the proposal because it will likely improve transparency and consistency in labelling for consumers.", + "No, I believe the existing categories are sufficient and that changes could confuse consumers.", + "I agree in principle, but I think the guidelines should include a provision for periodic review to adapt to market changes." + ], + "candidate_themes": [ + {"name": "Standardized framework", "description": "A standardized framework that benefits both consumers and manufacturers.", "key": "A", "selected": true, "approximate_frequency": 60, "parent_key": null}, + {"name": "Innovation", "description": "Innovation in flavour development.", "key": "B", "selected": true, "approximate_frequency": 30, "parent_key": null} + ] + }, + { + "question_text": "Do you agree with the proposal to align the flavour categories of chocolate bars as outlined in the draft guidelines of the Chocolate Bar Regulation for the United Kingdom?", + "number": 2, + "has_free_text": false, + "has_multiple_choice": true, + "multiple_choice_options": ["Yes", "No", "Don't know", "No answer"], + "free_text_answers": [], + "candidate_themes": [] + }, + { + "question_text": "Which of the following factors do you believe are important when considering the packaging of chocolate bars? Please select all that apply: a) Sustainability, b) Design, c) Cost-effectiveness, d) Durability, e) Brand recognition.", + "number": 3, + "has_free_text": false, + "has_multiple_choice": true, + "multiple_choice_options": ["Sustainability", "Design", "Cost-effectiveness", "Durability", "Brand recognition"], + "free_text_answers": [], + "candidate_themes": [] + }, + { + "question_text": "What are your thoughts on how the current chocolate bar regulations could be improved to better address consumer needs and industry standards?", + "number": 4, + "has_free_text": true, + "has_multiple_choice": false, + "multiple_choice_options": [], + "free_text_answers": [ + "", + "I believe the current regulations should include clearer guidelines on the sourcing of ingredients to ensure ethical practices and sustainability.", + "The regulations could be improved by setting specific limits on sugar content to promote healthier options for consumers.", + "Consideration should be given to standardizing portion sizes to help consumers make informed choices and manage their calorie intake.", + "It would be beneficial to include mandatory allergen warnings on all packaging to enhance consumer safety.", + "The regulations should encourage more innovation in packaging materials to reduce environmental impact.", + "Introducing incentives for using locally sourced ingredients could support local economies and reduce carbon footprints.", + "The current regulations could be improved by including a requirement for transparent labeling about nutritional information.", + "I think the regulations should mandate clearer expiration dates to prevent food waste and ensure product freshness.", + "The industry would benefit from guidelines that promote fair trade practices, giving consumers more ethical choices." + ], + "candidate_themes": [ + {"name": "More innovative", "description": "Innovative ideas to improve chocolate bar regulations.", "key": "A", "selected": true, "approximate_frequency": 40, "parent_key": null}, + {"name": "Innovative packaging", "description": "Ideas for innovative packaging solutions.", "key": "B", "selected": true, "approximate_frequency": 20, "parent_key": "A"}, + {"name": "New flavor combinations", "description": "Exploring new flavor combinations.", "key": "C", "selected": false, "approximate_frequency": 20, "parent_key": "A"}, + {"name": "Exotic flavors", "description": "Incorporating exotic flavors.", "key": "D", "selected": true, "approximate_frequency": 10, "parent_key": "C"}, + {"name": "Fusion flavors", "description": "Creating fusion flavors.", "key": "E", "selected": true, "approximate_frequency": 10, "parent_key": "C"}, + {"name": "Healthier options", "description": "It should encourage healthier options.", "key": "F", "selected": true, "approximate_frequency": 20, "parent_key": null}, + {"name": "Clearer guidelines", "description": "It should include clearer guidelines.", "key": "G", "selected": false, "approximate_frequency": 10, "parent_key": null}, + {"name": "Fair trade practices", "description": "It should promote fair trade practices.", "key": "H", "selected": false, "approximate_frequency": 10, "parent_key": null}, + {"name": "Sustainability", "description": "It should focus on sustainability.", "key": "I", "selected": false, "approximate_frequency": 10, "parent_key": null}, + {"name": "Transparent labeling", "description": "It should have transparent labeling.", "key": "J", "selected": false, "approximate_frequency": 10, "parent_key": null} + ] + } +] diff --git a/backend/tests/examples/sample_questions.yml b/backend/tests/examples/sample_questions.yml deleted file mode 100644 index 109962078..000000000 --- a/backend/tests/examples/sample_questions.yml +++ /dev/null @@ -1,117 +0,0 @@ -- question_text: "Do you agree with the proposal to align the flavour categories of chocolate bars as outlined in the draft guidelines of the Chocolate Bar Regulation for the United Kingdom?" - number: 1 - has_free_text: True - has_multiple_choice: True - multiple_choice_options: - - "Yes" - - "No" - - "Don't know" - - "No answer" - free_text_answers: - - "" - - "Yes, I agree with the proposal as it will create a standardized framework that benefits both consumers and manufacturers." - - "No, I do not agree because I feel the proposed categories are too restrictive and may stifle innovation in flavour development." - - "I agree with the proposal, but I think there should be room for additional categories to accommodate future trends." - - "I disagree with the proposal as it does not sufficiently account for regional flavour preferences across the UK." - - "Yes, aligning the flavour categories will help consumers easily compare products and make informed choices." - - "I neither agree nor disagree at this moment; I would like more information on how the categories were determined." - - "I support the proposal because it will likely improve transparency and consistency in labelling for consumers." - - "No, I believe the existing categories are sufficient and that changes could confuse consumers." - - "I agree in principle, but I think the guidelines should include a provision for periodic review to adapt to market changes." - candidate_themes: - - name: "Standardized framework" - description: "A standardized framework that benefits both consumers and manufacturers." - key: "A" - selected: true - approximate_frequency_pct: 0.6 - - name: "Innovation" - description: "Innovation in flavour development." - key: "B" - selected: true - approximate_frequency_pct: 0.3 - -- question_text: "Do you agree with the proposal to align the flavour categories of chocolate bars as outlined in the draft guidelines of the Chocolate Bar Regulation for the United Kingdom?" - number: 2 - has_free_text: False - has_multiple_choice: True - multiple_choice_options: - - "Yes" - - "No" - - "Don't know" - - "No answer" - -- question_text: "Which of the following factors do you believe are important when considering the packaging of chocolate bars? Please select all that apply: a) Sustainability, b) Design, c) Cost-effectiveness, d) Durability, e) Brand recognition." - number: 3 - has_free_text: False - has_multiple_choice: True - multiple_choice_options: - - "Sustainability" - - "Design" - - "Cost-effectiveness" - - "Durability" - - "Brand recognition" - -- question_text: "What are your thoughts on how the current chocolate bar regulations could be improved to better address consumer needs and industry standards?" - number: 4 - has_free_text: True - has_multiple_choice: False - free_text_answers: - - "" - - "I believe the current regulations should include clearer guidelines on the sourcing of ingredients to ensure ethical practices and sustainability." - - "The regulations could be improved by setting specific limits on sugar content to promote healthier options for consumers." - - "Consideration should be given to standardizing portion sizes to help consumers make informed choices and manage their calorie intake." - - "It would be beneficial to include mandatory allergen warnings on all packaging to enhance consumer safety." - - "The regulations should encourage more innovation in packaging materials to reduce environmental impact." - - "Introducing incentives for using locally sourced ingredients could support local economies and reduce carbon footprints." - - "The current regulations could be improved by including a requirement for transparent labeling about nutritional information." - - "I think the regulations should mandate clearer expiration dates to prevent food waste and ensure product freshness." - - "The industry would benefit from guidelines that promote fair trade practices, giving consumers more ethical choices." - candidate_themes: - - name: "More innovative" - description: "Innovative ideas to improve chocolate bar regulations." - key: "A" - selected: true - approximate_frequency_pct: 0.4 - children: - - name: "Innovative packaging" - description: "Ideas for innovative packaging solutions." - key: "B" - selected: true - approximate_frequency_pct: 0.2 - - name: "New flavor combinations" - description: "Exploring new flavor combinations." - key: "C" - approximate_frequency_pct: 0.2 - children: - - name: "Exotic flavors" - description: "Incorporating exotic flavors." - key: "D" - selected: true - approximate_frequency_pct: 0.1 - - name: "Fusion flavors" - description: "Creating fusion flavors." - key: "E" - selected: true - approximate_frequency_pct: 0.1 - - name: "Healthier options" - description: "It should encourage healthier options." - key: "F" - selected: true - approximate_frequency_pct: 0.2 - - name: "Clearer guidelines" - description: "It should include clearer guidelines." - key: "G" - approximate_frequency_pct: 0.1 - - name: "Fair trade practices" - description: "It should promote fair trade practices." - key: "H" - approximate_frequency_pct: 0.1 - - name: "Sustainability" - description: "It should focus on sustainability." - key: "I" - approximate_frequency_pct: 0.1 - - name: "Transparent labeling" - description: "It should have transparent labeling." - key: "J" - approximate_frequency_pct: 0.1 - key: "J" diff --git a/backend/tests/unit/data_pipeline/sync/test_consultation_setup.py b/backend/tests/unit/data_pipeline/sync/test_consultation_setup.py index 4d767ea71..ec865d10b 100644 --- a/backend/tests/unit/data_pipeline/sync/test_consultation_setup.py +++ b/backend/tests/unit/data_pipeline/sync/test_consultation_setup.py @@ -124,7 +124,9 @@ def test_import_consultation_from_s3(self, mock_enqueue, minio_test_bucket, mini # Create question 1 files (free text only) key = "app_data/consultations/test-code/inputs/question_part_1/question.json" - minio_client.put_object(Bucket=minio_test_bucket, Key=key, Body=json.dumps(question_1).encode()) + minio_client.put_object( + Bucket=minio_test_bucket, Key=key, Body=json.dumps(question_1).encode() + ) created_keys.append(key) key = "app_data/consultations/test-code/inputs/question_part_1/responses.jsonl" @@ -138,7 +140,9 @@ def test_import_consultation_from_s3(self, mock_enqueue, minio_test_bucket, mini # Create question 2 files (multi choice only) key = "app_data/consultations/test-code/inputs/question_part_2/question.json" - minio_client.put_object(Bucket=minio_test_bucket, Key=key, Body=json.dumps(question_2).encode()) + minio_client.put_object( + Bucket=minio_test_bucket, Key=key, Body=json.dumps(question_2).encode() + ) created_keys.append(key) key = "app_data/consultations/test-code/inputs/question_part_2/responses.jsonl" @@ -152,7 +156,9 @@ def test_import_consultation_from_s3(self, mock_enqueue, minio_test_bucket, mini # Create question 3 files (hybrid) key = "app_data/consultations/test-code/inputs/question_part_3/question.json" - minio_client.put_object(Bucket=minio_test_bucket, Key=key, Body=json.dumps(question_3).encode()) + minio_client.put_object( + Bucket=minio_test_bucket, Key=key, Body=json.dumps(question_3).encode() + ) created_keys.append(key) key = "app_data/consultations/test-code/inputs/question_part_3/responses.jsonl" @@ -229,7 +235,10 @@ def test_import_consultation_from_s3(self, mock_enqueue, minio_test_bucket, mini assert [opt.text for opt in q2_response_1.chosen_options.all()] == ["Option A"] q2_response_2 = Response.objects.get(question=question_2_db, respondent=respondent_2) - assert {opt.text for opt in q2_response_2.chosen_options.all()} == {"Option B", "Option C"} + assert {opt.text for opt in q2_response_2.chosen_options.all()} == { + "Option B", + "Option C", + } # Verify responses for hybrid question q3_response_1 = Response.objects.get(question=question_3_db, respondent=respondent_1) @@ -247,10 +256,7 @@ def test_import_consultation_from_s3(self, mock_enqueue, minio_test_bucket, mini # Cleanup: Delete all objects we created for key in created_keys: try: - minio_client.delete_object( - Bucket=minio_test_bucket, - Key=key - ) + minio_client.delete_object(Bucket=minio_test_bucket, Key=key) except Exception as e: # noqa: BLE001 logger.warning("Failed to cleanup object {key}: {e}", key=key, e=e) diff --git a/backend/tests/unit/data_pipeline/test_batch.py b/backend/tests/unit/data_pipeline/test_batch.py index ad4a5118d..e2f495621 100644 --- a/backend/tests/unit/data_pipeline/test_batch.py +++ b/backend/tests/unit/data_pipeline/test_batch.py @@ -11,9 +11,19 @@ class TestSubmitBatchJob: @staticmethod def _configure_batch_job_settings(mock_settings, job_type: str) -> None: mock_settings.SUBMIT_BATCH_JOBS = True - setattr(mock_settings, f"{job_type}_BATCH_JOB_NAME", f"{job_type.lower()}-job".replace("_", "-")) - setattr(mock_settings, f"{job_type}_BATCH_JOB_QUEUE", f"{job_type.lower()}-queue".replace("_", "-")) - setattr(mock_settings, f"{job_type}_BATCH_JOB_DEFINITION", f"{job_type.lower()}-def".replace("_", "-")) + setattr( + mock_settings, f"{job_type}_BATCH_JOB_NAME", f"{job_type.lower()}-job".replace("_", "-") + ) + setattr( + mock_settings, + f"{job_type}_BATCH_JOB_QUEUE", + f"{job_type.lower()}-queue".replace("_", "-"), + ) + setattr( + mock_settings, + f"{job_type}_BATCH_JOB_DEFINITION", + f"{job_type.lower()}-def".replace("_", "-"), + ) @staticmethod def _context_id_from_command(command: list[str]) -> str | None: @@ -149,7 +159,10 @@ def test_submit_job_propagates_context_id(self, mock_settings, mock_boto3): finally: structlog.contextvars.unbind_contextvars("context_id") - assert self._context_id_from_command(call_args["containerOverrides"]["command"]) == "request-context-abc" + assert ( + self._context_id_from_command(call_args["containerOverrides"]["command"]) + == "request-context-abc" + ) assert call_args["parameters"]["context_id"] == "request-context-abc" @patch("data_pipeline.batch.boto3") @@ -158,11 +171,16 @@ def test_submit_job_explicit_context_id_overrides_ambient(self, mock_settings, m """An explicit context_id wins over whatever's ambiently bound, in both the command and parameters""" structlog.contextvars.bind_contextvars(context_id="ambient-id") try: - call_args = self._submit_find_themes_job(mock_settings, mock_boto3, context_id="explicit-id") + call_args = self._submit_find_themes_job( + mock_settings, mock_boto3, context_id="explicit-id" + ) finally: structlog.contextvars.unbind_contextvars("context_id") - assert self._context_id_from_command(call_args["containerOverrides"]["command"]) == "explicit-id" + assert ( + self._context_id_from_command(call_args["containerOverrides"]["command"]) + == "explicit-id" + ) assert call_args["parameters"]["context_id"] == "explicit-id" @patch("data_pipeline.batch.boto3") diff --git a/backend/tests/unit/data_pipeline/test_s3.py b/backend/tests/unit/data_pipeline/test_s3.py index b4b336467..75e6e7605 100644 --- a/backend/tests/unit/data_pipeline/test_s3.py +++ b/backend/tests/unit/data_pipeline/test_s3.py @@ -29,18 +29,11 @@ def test_get_question_folders(self, minio_test_bucket, minio_client): ] for key in test_objects: - minio_client.put_object( - Bucket=minio_test_bucket, - Key=key, - Body=b"test content" - ) + minio_client.put_object(Bucket=minio_test_bucket, Key=key, Body=b"test content") created_keys.append(key) # Execute: Call the function under test - result = get_question_folders( - "app_data/consultations/test/inputs/", - minio_test_bucket - ) + result = get_question_folders("app_data/consultations/test/inputs/", minio_test_bucket) # Verify: Check results expected = [ @@ -53,10 +46,7 @@ def test_get_question_folders(self, minio_test_bucket, minio_client): # Cleanup: Delete all objects we created for key in created_keys: try: - minio_client.delete_object( - Bucket=minio_test_bucket, - Key=key - ) + minio_client.delete_object(Bucket=minio_test_bucket, Key=key) except Exception as e: # noqa: BLE001 logger.warning("Failed to cleanup object {key}: {e}", key=key, e=e) @@ -85,11 +75,7 @@ def test_get_consultation_folders(self, minio_test_bucket, minio_client): ] for key in test_objects: - minio_client.put_object( - Bucket=minio_test_bucket, - Key=key, - Body=b"test content" - ) + minio_client.put_object(Bucket=minio_test_bucket, Key=key, Body=b"test content") created_keys.append(key) # Execute: Call the function under test @@ -103,9 +89,6 @@ def test_get_consultation_folders(self, minio_test_bucket, minio_client): # Cleanup: Delete all objects we created for key in created_keys: try: - minio_client.delete_object( - Bucket=minio_test_bucket, - Key=key - ) + minio_client.delete_object(Bucket=minio_test_bucket, Key=key) except Exception as e: # noqa: BLE001 logger.warning("Failed to cleanup object {key}: {e}", key=key, e=e) diff --git a/backend/tests/unit/test_generate_dummy_data.py b/backend/tests/unit/test_generate_dummy_data.py index b096071ee..baed21ab5 100644 --- a/backend/tests/unit/test_generate_dummy_data.py +++ b/backend/tests/unit/test_generate_dummy_data.py @@ -4,14 +4,14 @@ import pytest from consultations import models -from consultations.dummy_data import create_dummy_consultation_from_yaml +from consultations.dummy_data import DUMMY_CONSULTATIONS, create_dummy_consultation @pytest.mark.django_db @patch("hosting_environment.HostingEnvironment.is_local", return_value=True) def test_a_consultation_is_generated(settings): assert models.Consultation.objects.count() == 0 - create_dummy_consultation_from_yaml() + create_dummy_consultation() assert models.Consultation.objects.count() == 1 assert models.Question.objects.count() == 4 @@ -19,27 +19,99 @@ def test_a_consultation_is_generated(settings): @pytest.mark.django_db @pytest.mark.parametrize("environment", ["prod"]) def test_the_tool_will_only_run_in_dev(environment): - with patch.dict(os.environ, {"ENVIRONMENT": environment}), pytest.raises( - Exception, match=r"Dummy data generation should not be run in production" + with ( + patch.dict(os.environ, {"ENVIRONMENT": environment}), + pytest.raises(Exception, match=r"Dummy data generation should not be run in production"), ): - create_dummy_consultation_from_yaml() + create_dummy_consultation() @pytest.mark.django_db -def test_create_dummy_consultation_from_yaml(): - consultation = create_dummy_consultation_from_yaml(number_respondents=10) +def test_setup_stage_has_consultation_data_but_no_themes(): + config = DUMMY_CONSULTATIONS[0] + consultation = create_dummy_consultation(number_respondents=10, config=config) questions = models.Question.objects.filter(consultation=consultation) + + assert consultation.stage == models.Consultation.Stage.SETUP assert questions.count() == 4 + assert models.Response.objects.filter(question__consultation=consultation).count() > 0 + assert models.CandidateTheme.objects.filter(question__consultation=consultation).count() == 0 + assert models.SelectedTheme.objects.filter(question__consultation=consultation).count() == 0 + # Multiple choice questions have answers q1 = questions.get(number=1) - assert q1.has_free_text assert q1.has_multiple_choice + assert models.MultiChoiceAnswer.objects.filter(question=q1).count() == 4 + + # Respondents have demographic data + respondent = models.Respondent.objects.filter(consultation=consultation).first() + assert respondent.demographics.count() > 0 + field_names = set(respondent.demographics.values_list("field_name", flat=True)) + assert "region" in field_names + assert "age_group" in field_names + assert "respondent_type" in field_names + + +@pytest.mark.django_db +def test_theme_sign_off_draft_has_candidate_themes_and_responses(): + config = DUMMY_CONSULTATIONS[1] + consultation = create_dummy_consultation(number_respondents=10, config=config) + q4 = models.Question.objects.get(consultation=consultation, number=4) + + assert consultation.stage == models.Consultation.Stage.FINALISING_THEMES + assert q4.theme_status == models.Question.ThemeStatus.DRAFT + + # Has candidate themes at multiple levels + all_themes = models.CandidateTheme.objects.filter(question=q4) + top_level = all_themes.filter(parent=None) + children = all_themes.exclude(parent=None) + assert top_level.count() > 0 + assert children.count() > 0 + + # Has CandidateThemeResponses for child themes too + child_theme = children.first() + assert models.CandidateThemeResponse.objects.filter(candidate_theme=child_theme).exists() + + # No SelectedThemes (still draft) + assert models.SelectedTheme.objects.filter(question=q4).count() == 0 + + +@pytest.mark.django_db +def test_theme_sign_off_confirmed_has_selected_themes(): + config = DUMMY_CONSULTATIONS[2] + consultation = create_dummy_consultation(number_respondents=10, config=config) + q4 = models.Question.objects.get(consultation=consultation, number=4) + + assert consultation.stage == models.Consultation.Stage.ASSIGNING_THEMES + assert q4.theme_status == models.Question.ThemeStatus.CONFIRMED + + # Has SelectedThemes + selected_themes = models.SelectedTheme.objects.filter(question=q4) + assert selected_themes.filter(key="A").exists() + assert not selected_themes.filter(name="Other").exists() + assert not selected_themes.filter(name="No Reason Given").exists() + + +@pytest.mark.django_db +def test_analysis_has_response_annotations(): + config = DUMMY_CONSULTATIONS[3] + consultation = create_dummy_consultation(number_respondents=10, config=config) + q1 = models.Question.objects.get(consultation=consultation, number=1) + + assert consultation.stage == models.Consultation.Stage.ANALYSIS + assert q1.theme_status == models.Question.ThemeStatus.CONFIRMED + + # Has SelectedThemes including defaults + selected_themes = models.SelectedTheme.objects.filter(question=q1) + assert selected_themes.count() == 4 + assert selected_themes.get(key="A").name == "Standardized framework" + assert selected_themes.filter(name="Other").exists() + assert selected_themes.filter(name="No Reason Given").exists() - q1_themes = models.SelectedTheme.objects.filter(question=q1) - assert len(q1_themes) == 2 - assert q1_themes.get(key="A").name == "Standardized framework" + # Has ResponseAnnotations for responses + annotations = models.ResponseAnnotation.objects.filter(response__question=q1) + assert annotations.count() > 0 - q3 = questions.get(number=3) - assert not q3.has_free_text - assert q3.has_multiple_choice - assert q3.multiple_choice_options + # Each annotation has themes assigned + for annotation in annotations: + assert annotation.themes.count() > 0 diff --git a/backend/tests/unit/test_healthcheck_worker.py b/backend/tests/unit/test_healthcheck_worker.py index bff9ffc75..0d15666e2 100644 --- a/backend/tests/unit/test_healthcheck_worker.py +++ b/backend/tests/unit/test_healthcheck_worker.py @@ -60,5 +60,8 @@ def test_fails_when_the_worker_heartbeat_is_stale(self, rq_worker): call_command("healthcheck_worker") def test_fails_with_a_clear_message_when_redis_is_unreachable(self): - with patch("rq.worker.Worker.all", side_effect=redis.exceptions.ConnectionError("boom")), pytest.raises(CommandError, match="Could not reach Redis"): - call_command("healthcheck_worker") + with ( + patch("rq.worker.Worker.all", side_effect=redis.exceptions.ConnectionError("boom")), + pytest.raises(CommandError, match="Could not reach Redis"), + ): + call_command("healthcheck_worker") diff --git a/backend/tests/unit/test_middleware.py b/backend/tests/unit/test_middleware.py index cbb84808d..8a8e0d03d 100644 --- a/backend/tests/unit/test_middleware.py +++ b/backend/tests/unit/test_middleware.py @@ -17,7 +17,9 @@ def view(request): if captured is not None: captured["context_id"] = structlog.contextvars.get_contextvars().get("context_id") captured["path"] = structlog.contextvars.get_contextvars().get("path") - captured["execution_context"] = structlog.contextvars.get_contextvars().get("execution_context") + captured["execution_context"] = structlog.contextvars.get_contextvars().get( + "execution_context" + ) return HttpResponse(status=200) return RequestCorrelationMiddleware(view) @@ -38,7 +40,9 @@ def test_reuses_inbound_context_id(request_factory): middleware = _make_middleware(captured) inbound = "inbound-correlation-id" - response = middleware(request_factory.get("/api/consultations/", headers={"x-context-id": inbound})) + response = middleware( + request_factory.get("/api/consultations/", headers={"x-context-id": inbound}) + ) assert captured["context_id"] == inbound assert response[CONTEXT_ID_HEADER] == inbound diff --git a/backend/tests/unit/test_rq_context.py b/backend/tests/unit/test_rq_context.py index 6d1bc3f8e..7571e623a 100644 --- a/backend/tests/unit/test_rq_context.py +++ b/backend/tests/unit/test_rq_context.py @@ -66,11 +66,11 @@ def test_rebinds_execution_context_after_refresh(self, settings): class TestRQContextJob: """rq_context.job auto-fills context_id at enqueue time and rebinds it at execution time; - ASYNC=False in tests runs .delay()/.enqueue() through the real path.""" + ASYNC=False in tests runs .delay()/.enqueue() through the real path.""" def test_ambient_context_id_survives_a_real_delay_round_trip(self): """Proves context_id survives the full enqueue -> execution round trip; - the job.kwargs check rules out a same-process leak masquerading as propagation.""" + the job.kwargs check rules out a same-process leak masquerading as propagation.""" rebind_context("ambient-real-id") job = probe_job.delay(42) diff --git a/docker-compose.yml b/docker-compose.yml index e65b528a0..d414beb64 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,10 +71,11 @@ services: env_file: - .env environment: - - DATABASE_URL=postgresql://postgres:postgres@postgres:5432/postgres + - DATABASE_URL=${DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/postgres} - DJANGO_SETTINGS_MODULE=backend.settings.local - DOMAIN_NAME=localhost:3000 - MINIO_ENDPOINT=http://minio:9100 + - REDIS_HOST=redis depends_on: postgres: condition: service_healthy diff --git a/e2e_tests/constants.ts b/e2e_tests/constants.ts index f5199d8ca..e773fba01 100644 --- a/e2e_tests/constants.ts +++ b/e2e_tests/constants.ts @@ -1,5 +1,5 @@ export const testAccessToken = - "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImVtYWlsQGV4YW1wbGUuY29tIn0.k27nav4gbG-2lIArYInTqP1GUz2LRuzb3lWandMKRoY"; // pragma: allowlist secret + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImFkbWluQGV4YW1wbGUuY29tIn0.ignored"; // pragma: allowlist secret // Minio (local S3) config, sourced from the environment (loaded from .env) so // it stays in sync with what the backend uses. Playwright runs on the host, so @@ -8,4 +8,4 @@ export const testAccessToken = export const MINIO_ENDPOINT = process.env.MINIO_ENDPOINT ?? "http://localhost:9100"; export const MINIO_ACCESS_KEY = process.env.MINIO_ACCESS_KEY ?? "minioadmin"; export const MINIO_SECRET_KEY = process.env.MINIO_SECRET_KEY ?? "minioadmin"; -export const S3_BUCKET = process.env.AWS_BUCKET_NAME ?? "i-dot-ai-dev-consult-data"; +export const S3_BUCKET = process.env.AWS_BUCKET_NAME ?? "i-dot-ai-test-consult-data";