From 698f503c3ed13f4b79518985aca4d165c5bfd52d Mon Sep 17 00:00:00 2001 From: ramonavic Date: Mon, 29 Jun 2026 15:54:16 +0200 Subject: [PATCH 1/2] create a copy of ml.csv export but with more specific filtering and slugs --- .../reporting/csv/datawarehouse/__init__.py | 2 + .../reporting/csv/datawarehouse/signals.py | 29 ++++++++++ .../apps/reporting/csv/datawarehouse/tasks.py | 2 + .../reporting/management/commands/dumpcsv.py | 2 + .../reporting/tests/csv/test_datawarehouse.py | 55 +++++++++++++++++++ 5 files changed, 90 insertions(+) diff --git a/app/signals/apps/reporting/csv/datawarehouse/__init__.py b/app/signals/apps/reporting/csv/datawarehouse/__init__.py index 765ef405f..066396824 100644 --- a/app/signals/apps/reporting/csv/datawarehouse/__init__.py +++ b/app/signals/apps/reporting/csv/datawarehouse/__init__.py @@ -11,6 +11,7 @@ from signals.apps.reporting.csv.datawarehouse.locations import create_locations_csv from signals.apps.reporting.csv.datawarehouse.reporters import create_reporters_csv from signals.apps.reporting.csv.datawarehouse.signals import ( + create_ml_amsterdam_csv, create_ml_csv, create_signals_assigned_user_csv, create_signals_csv, @@ -31,6 +32,7 @@ 'create_kto_feedback_csv', 'create_locations_csv', 'create_ml_csv', + 'create_ml_amsterdam_csv', 'create_reporters_csv', 'create_statuses_csv', 'create_signals_assigned_user_csv', diff --git a/app/signals/apps/reporting/csv/datawarehouse/signals.py b/app/signals/apps/reporting/csv/datawarehouse/signals.py index b73f89415..366e51f8b 100644 --- a/app/signals/apps/reporting/csv/datawarehouse/signals.py +++ b/app/signals/apps/reporting/csv/datawarehouse/signals.py @@ -96,6 +96,35 @@ def create_ml_csv(location: str) -> str: return csv_file.name +def create_ml_amsterdam_csv(location: str) -> str: + """ + Create the CSV file with all categorized `Signal` objects for ML purposes. + + :param location: Directory for saving the CSV file + :returns: Path to CSV file + """ + queryset = Signal.objects.filter( + category_assignment__isnull=False, + category_assignment__category__is_active=True, + category_assignment__category__isnull=False, + status__state__in=[AFGEHANDELD, GEANNULEERD], + ).exclude( + Q(category_assignment__category__parent__name__contains='Overig') | + Q(category_assignment__category__name__contains='Overig') + ).values( + Text=F('text'), + Main=F('category_assignment__category__parent__slug'), + Sub=F('category_assignment__category__slug'), + ).order_by('created_at') + + csv_file = queryset_to_csv_file(queryset, os.path.join(location, 'ml_amsterdam.csv')) + + ordered_field_names = ['Text', 'Main', 'Sub'] + reorder_csv(csv_file.name, ordered_field_names) + + return csv_file.name + + def create_signals_assigned_user_csv(location: str) -> str: """ Create the CSV file with all `Signal - assigned user relation` objects. diff --git a/app/signals/apps/reporting/csv/datawarehouse/tasks.py b/app/signals/apps/reporting/csv/datawarehouse/tasks.py index bd323c013..8bb4e6d4c 100644 --- a/app/signals/apps/reporting/csv/datawarehouse/tasks.py +++ b/app/signals/apps/reporting/csv/datawarehouse/tasks.py @@ -14,6 +14,7 @@ from signals.apps.reporting.csv.datawarehouse.locations import create_locations_csv from signals.apps.reporting.csv.datawarehouse.reporters import create_reporters_csv from signals.apps.reporting.csv.datawarehouse.signals import ( + create_ml_amsterdam_csv, create_ml_csv, create_signals_assigned_user_csv, create_signals_csv, @@ -55,6 +56,7 @@ def save_csv_files_datawarehouse(using: str = 'datawarehouse') -> list[str]: csv_files = list() csv_files.extend(save_csv_file_datawarehouse(create_signals_csv, using=using)) csv_files.extend(save_csv_file_datawarehouse(create_ml_csv, using=using)) + csv_files.extend(save_csv_file_datawarehouse(create_ml_amsterdam_csv, using=using)) csv_files.extend(save_csv_file_datawarehouse(create_signals_assigned_user_csv, using=using)) csv_files.extend(save_csv_file_datawarehouse(create_locations_csv, using=using)) csv_files.extend(save_csv_file_datawarehouse(create_reporters_csv, using=using)) diff --git a/app/signals/apps/reporting/management/commands/dumpcsv.py b/app/signals/apps/reporting/management/commands/dumpcsv.py index 1f90bc86a..0f9b1cec2 100644 --- a/app/signals/apps/reporting/management/commands/dumpcsv.py +++ b/app/signals/apps/reporting/management/commands/dumpcsv.py @@ -17,6 +17,7 @@ from signals.apps.reporting.csv.datawarehouse.locations import create_locations_csv from signals.apps.reporting.csv.datawarehouse.reporters import create_reporters_csv from signals.apps.reporting.csv.datawarehouse.signals import ( + create_ml_amsterdam_csv, create_ml_csv, create_signals_assigned_user_csv, create_signals_csv, @@ -33,6 +34,7 @@ # Option, Func 'signals': create_signals_csv, 'ml': create_ml_csv, + 'ml_amsterdam': create_ml_amsterdam_csv, 'signals_assigned_user': create_signals_assigned_user_csv, 'locations': create_locations_csv, 'reporters': create_reporters_csv, diff --git a/app/signals/apps/reporting/tests/csv/test_datawarehouse.py b/app/signals/apps/reporting/tests/csv/test_datawarehouse.py index 3d1545c54..be3059f1e 100644 --- a/app/signals/apps/reporting/tests/csv/test_datawarehouse.py +++ b/app/signals/apps/reporting/tests/csv/test_datawarehouse.py @@ -258,6 +258,61 @@ def test_create_ml_csv(self): self.assertNotIn('Overig sub text', texts) self.assertNotIn('Open text', texts) + def test_create_ml_amsterdam_csv(self): + category = CategoryFactory.create(name='Sub', parent__name='Main') + signal = SignalFactory.create(text='ML text', category_assignment__category=category, status__state=AFGEHANDELD) + SignalFactory.create( + text='Cancelled text', + category_assignment__category=category, + status__state=GEANNULEERD, + ) + SignalFactory.create( + text='Overig main text', + category_assignment__category=CategoryFactory.create(name='Sub', parent__name='Overig main'), + status__state=AFGEHANDELD, + ) + SignalFactory.create( + text='Overig sub text', + category_assignment__category=CategoryFactory.create(name='Overig sub', parent__name='Main'), + status__state=AFGEHANDELD, + ) + + # Invalid state, should not be included + SignalFactory.create( + text='Open text', + category_assignment__category=category, + status__state=GEMELD, + ) + uncategorized_signal = SignalFactory.create() + uncategorized_signal.category_assignment = None + uncategorized_signal.save(update_fields=['category_assignment']) + + # Inactive category should not be included + SignalFactory.create( + text='Inactive category text', + category_assignment__category=CategoryFactory.create(name='Sub', parent__name='Main', is_active=False), + status__state=AFGEHANDELD, + ) + + csv_file = datawarehouse.create_ml_csv(self.csv_tmp_dir) + + self.assertEqual(path.join(self.csv_tmp_dir, 'ml.csv'), csv_file) + + with open(csv_file) as opened_csv_file: + rows = list(csv.DictReader(opened_csv_file)) + texts = [row['Text'] for row in rows] + + self.assertEqual(len(rows), 2) + self.assertEqual(rows[0]['Text'], signal.text) + self.assertEqual(rows[0]['Main'], signal.category_assignment.category.parent.slug) + self.assertEqual(rows[0]['Sub'], signal.category_assignment.category.slug) + self.assertEqual(rows[1]['Text'], 'Cancelled text') + self.assertEqual(rows[1]['Main'], 'Main') + self.assertEqual(rows[1]['Sub'], 'Sub') + self.assertNotIn('Overig main text', texts) + self.assertNotIn('Overig sub text', texts) + self.assertNotIn('Open text', texts) + def test_create_locations_csv(self): signal = SignalFactory.create(location__area_code='AREACODE', location__area_name='AREA_NAME') location = signal.location From bcbd7ad8c95a858c8f020a34747992b34cd68867 Mon Sep 17 00:00:00 2001 From: ramonavic Date: Mon, 29 Jun 2026 17:03:37 +0200 Subject: [PATCH 2/2] fix tests --- .../reporting/tests/csv/test_datawarehouse.py | 16 ++++++++++------ .../tests/management/commands/test_command.py | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/app/signals/apps/reporting/tests/csv/test_datawarehouse.py b/app/signals/apps/reporting/tests/csv/test_datawarehouse.py index be3059f1e..eec6d3af6 100644 --- a/app/signals/apps/reporting/tests/csv/test_datawarehouse.py +++ b/app/signals/apps/reporting/tests/csv/test_datawarehouse.py @@ -288,15 +288,17 @@ def test_create_ml_amsterdam_csv(self): uncategorized_signal.save(update_fields=['category_assignment']) # Inactive category should not be included - SignalFactory.create( + inactive = SignalFactory.create( text='Inactive category text', - category_assignment__category=CategoryFactory.create(name='Sub', parent__name='Main', is_active=False), + category_assignment__category=CategoryFactory.create(name='Inactive', parent__name='Main'), status__state=AFGEHANDELD, ) + inactive.category_assignment.category.is_active = False + inactive.category_assignment.category.save(update_fields=['is_active']) - csv_file = datawarehouse.create_ml_csv(self.csv_tmp_dir) + csv_file = datawarehouse.create_ml_amsterdam_csv(self.csv_tmp_dir) - self.assertEqual(path.join(self.csv_tmp_dir, 'ml.csv'), csv_file) + self.assertEqual(path.join(self.csv_tmp_dir, 'ml_amsterdam.csv'), csv_file) with open(csv_file) as opened_csv_file: rows = list(csv.DictReader(opened_csv_file)) @@ -307,11 +309,13 @@ def test_create_ml_amsterdam_csv(self): self.assertEqual(rows[0]['Main'], signal.category_assignment.category.parent.slug) self.assertEqual(rows[0]['Sub'], signal.category_assignment.category.slug) self.assertEqual(rows[1]['Text'], 'Cancelled text') - self.assertEqual(rows[1]['Main'], 'Main') - self.assertEqual(rows[1]['Sub'], 'Sub') + self.assertEqual(rows[1]['Main'], 'main') + self.assertEqual(rows[1]['Sub'], 'sub') + self.assertNotIn('Overig main text', texts) self.assertNotIn('Overig sub text', texts) self.assertNotIn('Open text', texts) + self.assertNotIn('Inactive category text', texts) def test_create_locations_csv(self): signal = SignalFactory.create(location__area_code='AREACODE', location__area_name='AREA_NAME') diff --git a/app/signals/apps/reporting/tests/management/commands/test_command.py b/app/signals/apps/reporting/tests/management/commands/test_command.py index bc90ae4f9..48a38108f 100644 --- a/app/signals/apps/reporting/tests/management/commands/test_command.py +++ b/app/signals/apps/reporting/tests/management/commands/test_command.py @@ -18,7 +18,7 @@ def test_command(self, patched_save_csv_files_datawarehouse): self.assertNotEqual(out.getvalue(), '') self.assertEqual(err.getvalue(), '') - self.assertEqual(patched_save_csv_files_datawarehouse.call_count, 12) + self.assertEqual(patched_save_csv_files_datawarehouse.call_count, 13) class TestCSVHorecaCommand(TransactionTestCase):