diff --git a/app/signals/apps/reporting/csv/datawarehouse/__init__.py b/app/signals/apps/reporting/csv/datawarehouse/__init__.py index 765ef405f..066396824 100644 --- a/app/signals/apps/reporting/csv/datawarehouse/__init__.py +++ b/app/signals/apps/reporting/csv/datawarehouse/__init__.py @@ -11,6 +11,7 @@ from signals.apps.reporting.csv.datawarehouse.locations import create_locations_csv from signals.apps.reporting.csv.datawarehouse.reporters import create_reporters_csv from signals.apps.reporting.csv.datawarehouse.signals import ( + create_ml_amsterdam_csv, create_ml_csv, create_signals_assigned_user_csv, create_signals_csv, @@ -31,6 +32,7 @@ 'create_kto_feedback_csv', 'create_locations_csv', 'create_ml_csv', + 'create_ml_amsterdam_csv', 'create_reporters_csv', 'create_statuses_csv', 'create_signals_assigned_user_csv', diff --git a/app/signals/apps/reporting/csv/datawarehouse/signals.py b/app/signals/apps/reporting/csv/datawarehouse/signals.py index b73f89415..366e51f8b 100644 --- a/app/signals/apps/reporting/csv/datawarehouse/signals.py +++ b/app/signals/apps/reporting/csv/datawarehouse/signals.py @@ -96,6 +96,35 @@ def create_ml_csv(location: str) -> str: return csv_file.name +def create_ml_amsterdam_csv(location: str) -> str: + """ + Create the CSV file with all categorized `Signal` objects for ML purposes. + + :param location: Directory for saving the CSV file + :returns: Path to CSV file + """ + queryset = Signal.objects.filter( + category_assignment__isnull=False, + category_assignment__category__is_active=True, + category_assignment__category__isnull=False, + status__state__in=[AFGEHANDELD, GEANNULEERD], + ).exclude( + Q(category_assignment__category__parent__name__contains='Overig') | + Q(category_assignment__category__name__contains='Overig') + ).values( + Text=F('text'), + Main=F('category_assignment__category__parent__slug'), + Sub=F('category_assignment__category__slug'), + ).order_by('created_at') + + csv_file = queryset_to_csv_file(queryset, os.path.join(location, 'ml_amsterdam.csv')) + + ordered_field_names = ['Text', 'Main', 'Sub'] + reorder_csv(csv_file.name, ordered_field_names) + + return csv_file.name + + def create_signals_assigned_user_csv(location: str) -> str: """ Create the CSV file with all `Signal - assigned user relation` objects. diff --git a/app/signals/apps/reporting/csv/datawarehouse/tasks.py b/app/signals/apps/reporting/csv/datawarehouse/tasks.py index bd323c013..8bb4e6d4c 100644 --- a/app/signals/apps/reporting/csv/datawarehouse/tasks.py +++ b/app/signals/apps/reporting/csv/datawarehouse/tasks.py @@ -14,6 +14,7 @@ from signals.apps.reporting.csv.datawarehouse.locations import create_locations_csv from signals.apps.reporting.csv.datawarehouse.reporters import create_reporters_csv from signals.apps.reporting.csv.datawarehouse.signals import ( + create_ml_amsterdam_csv, create_ml_csv, create_signals_assigned_user_csv, create_signals_csv, @@ -55,6 +56,7 @@ def save_csv_files_datawarehouse(using: str = 'datawarehouse') -> list[str]: csv_files = list() csv_files.extend(save_csv_file_datawarehouse(create_signals_csv, using=using)) csv_files.extend(save_csv_file_datawarehouse(create_ml_csv, using=using)) + csv_files.extend(save_csv_file_datawarehouse(create_ml_amsterdam_csv, using=using)) csv_files.extend(save_csv_file_datawarehouse(create_signals_assigned_user_csv, using=using)) csv_files.extend(save_csv_file_datawarehouse(create_locations_csv, using=using)) csv_files.extend(save_csv_file_datawarehouse(create_reporters_csv, using=using)) diff --git a/app/signals/apps/reporting/management/commands/dumpcsv.py b/app/signals/apps/reporting/management/commands/dumpcsv.py index 1f90bc86a..0f9b1cec2 100644 --- a/app/signals/apps/reporting/management/commands/dumpcsv.py +++ b/app/signals/apps/reporting/management/commands/dumpcsv.py @@ -17,6 +17,7 @@ from signals.apps.reporting.csv.datawarehouse.locations import create_locations_csv from signals.apps.reporting.csv.datawarehouse.reporters import create_reporters_csv from signals.apps.reporting.csv.datawarehouse.signals import ( + create_ml_amsterdam_csv, create_ml_csv, create_signals_assigned_user_csv, create_signals_csv, @@ -33,6 +34,7 @@ # Option, Func 'signals': create_signals_csv, 'ml': create_ml_csv, + 'ml_amsterdam': create_ml_amsterdam_csv, 'signals_assigned_user': create_signals_assigned_user_csv, 'locations': create_locations_csv, 'reporters': create_reporters_csv, diff --git a/app/signals/apps/reporting/tests/csv/test_datawarehouse.py b/app/signals/apps/reporting/tests/csv/test_datawarehouse.py index 3d1545c54..eec6d3af6 100644 --- a/app/signals/apps/reporting/tests/csv/test_datawarehouse.py +++ b/app/signals/apps/reporting/tests/csv/test_datawarehouse.py @@ -258,6 +258,65 @@ def test_create_ml_csv(self): self.assertNotIn('Overig sub text', texts) self.assertNotIn('Open text', texts) + def test_create_ml_amsterdam_csv(self): + category = CategoryFactory.create(name='Sub', parent__name='Main') + signal = SignalFactory.create(text='ML text', category_assignment__category=category, status__state=AFGEHANDELD) + SignalFactory.create( + text='Cancelled text', + category_assignment__category=category, + status__state=GEANNULEERD, + ) + SignalFactory.create( + text='Overig main text', + category_assignment__category=CategoryFactory.create(name='Sub', parent__name='Overig main'), + status__state=AFGEHANDELD, + ) + SignalFactory.create( + text='Overig sub text', + category_assignment__category=CategoryFactory.create(name='Overig sub', parent__name='Main'), + status__state=AFGEHANDELD, + ) + + # Invalid state, should not be included + SignalFactory.create( + text='Open text', + category_assignment__category=category, + status__state=GEMELD, + ) + uncategorized_signal = SignalFactory.create() + uncategorized_signal.category_assignment = None + uncategorized_signal.save(update_fields=['category_assignment']) + + # Inactive category should not be included + inactive = SignalFactory.create( + text='Inactive category text', + category_assignment__category=CategoryFactory.create(name='Inactive', parent__name='Main'), + status__state=AFGEHANDELD, + ) + inactive.category_assignment.category.is_active = False + inactive.category_assignment.category.save(update_fields=['is_active']) + + csv_file = datawarehouse.create_ml_amsterdam_csv(self.csv_tmp_dir) + + self.assertEqual(path.join(self.csv_tmp_dir, 'ml_amsterdam.csv'), csv_file) + + with open(csv_file) as opened_csv_file: + rows = list(csv.DictReader(opened_csv_file)) + texts = [row['Text'] for row in rows] + + self.assertEqual(len(rows), 2) + self.assertEqual(rows[0]['Text'], signal.text) + self.assertEqual(rows[0]['Main'], signal.category_assignment.category.parent.slug) + self.assertEqual(rows[0]['Sub'], signal.category_assignment.category.slug) + self.assertEqual(rows[1]['Text'], 'Cancelled text') + self.assertEqual(rows[1]['Main'], 'main') + self.assertEqual(rows[1]['Sub'], 'sub') + + self.assertNotIn('Overig main text', texts) + self.assertNotIn('Overig sub text', texts) + self.assertNotIn('Open text', texts) + self.assertNotIn('Inactive category text', texts) + def test_create_locations_csv(self): signal = SignalFactory.create(location__area_code='AREACODE', location__area_name='AREA_NAME') location = signal.location diff --git a/app/signals/apps/reporting/tests/management/commands/test_command.py b/app/signals/apps/reporting/tests/management/commands/test_command.py index bc90ae4f9..48a38108f 100644 --- a/app/signals/apps/reporting/tests/management/commands/test_command.py +++ b/app/signals/apps/reporting/tests/management/commands/test_command.py @@ -18,7 +18,7 @@ def test_command(self, patched_save_csv_files_datawarehouse): self.assertNotEqual(out.getvalue(), '') self.assertEqual(err.getvalue(), '') - self.assertEqual(patched_save_csv_files_datawarehouse.call_count, 12) + self.assertEqual(patched_save_csv_files_datawarehouse.call_count, 13) class TestCSVHorecaCommand(TransactionTestCase):