ML Export for Amsterdam

app/signals/apps/reporting/csv/datawarehouse/__init__.py

@@ -11,6 +11,7 @@
 from signals.apps.reporting.csv.datawarehouse.locations import create_locations_csv
 from signals.apps.reporting.csv.datawarehouse.reporters import create_reporters_csv
 from signals.apps.reporting.csv.datawarehouse.signals import (
+    create_ml_amsterdam_csv,
     create_ml_csv,
     create_signals_assigned_user_csv,
     create_signals_csv,
@@ -31,6 +32,7 @@
     'create_kto_feedback_csv',
     'create_locations_csv',
     'create_ml_csv',
+    'create_ml_amsterdam_csv',
     'create_reporters_csv',
     'create_statuses_csv',
     'create_signals_assigned_user_csv',

app/signals/apps/reporting/csv/datawarehouse/signals.py

@@ -96,6 +96,35 @@ def create_ml_csv(location: str) -> str:
     return csv_file.name
 
 
+def create_ml_amsterdam_csv(location: str) -> str:
+    """
+    Create the CSV file with all categorized `Signal` objects for ML purposes.
+
+    :param location: Directory for saving the CSV file
+    :returns: Path to CSV file
+    """
+    queryset = Signal.objects.filter(
+        category_assignment__isnull=False,
+        category_assignment__category__is_active=True,
+        category_assignment__category__isnull=False,
+        status__state__in=[AFGEHANDELD, GEANNULEERD],
+    ).exclude(
+        Q(category_assignment__category__parent__name__contains='Overig') |
+        Q(category_assignment__category__name__contains='Overig')
+    ).values(
+        Text=F('text'),
+        Main=F('category_assignment__category__parent__slug'),
+        Sub=F('category_assignment__category__slug'),
+    ).order_by('created_at')
+
+    csv_file = queryset_to_csv_file(queryset, os.path.join(location, 'ml_amsterdam.csv'))
+
+    ordered_field_names = ['Text', 'Main', 'Sub']
+    reorder_csv(csv_file.name, ordered_field_names)
+
+    return csv_file.name
+
+
 def create_signals_assigned_user_csv(location: str) -> str:
     """
     Create the CSV file with all `Signal - assigned user relation` objects.

app/signals/apps/reporting/csv/datawarehouse/tasks.py

@@ -14,6 +14,7 @@
 from signals.apps.reporting.csv.datawarehouse.locations import create_locations_csv
 from signals.apps.reporting.csv.datawarehouse.reporters import create_reporters_csv
 from signals.apps.reporting.csv.datawarehouse.signals import (
+    create_ml_amsterdam_csv,
     create_ml_csv,
     create_signals_assigned_user_csv,
     create_signals_csv,
@@ -55,6 +56,7 @@ def save_csv_files_datawarehouse(using: str = 'datawarehouse') -> list[str]:
     csv_files = list()
     csv_files.extend(save_csv_file_datawarehouse(create_signals_csv, using=using))
     csv_files.extend(save_csv_file_datawarehouse(create_ml_csv, using=using))
+    csv_files.extend(save_csv_file_datawarehouse(create_ml_amsterdam_csv, using=using))
     csv_files.extend(save_csv_file_datawarehouse(create_signals_assigned_user_csv, using=using))
     csv_files.extend(save_csv_file_datawarehouse(create_locations_csv, using=using))
     csv_files.extend(save_csv_file_datawarehouse(create_reporters_csv, using=using))

app/signals/apps/reporting/management/commands/dumpcsv.py

@@ -17,6 +17,7 @@
 from signals.apps.reporting.csv.datawarehouse.locations import create_locations_csv
 from signals.apps.reporting.csv.datawarehouse.reporters import create_reporters_csv
 from signals.apps.reporting.csv.datawarehouse.signals import (
+    create_ml_amsterdam_csv,
     create_ml_csv,
     create_signals_assigned_user_csv,
     create_signals_csv,
@@ -33,6 +34,7 @@
     # Option, Func
     'signals': create_signals_csv,
     'ml': create_ml_csv,
+    'ml_amsterdam': create_ml_amsterdam_csv,
     'signals_assigned_user': create_signals_assigned_user_csv,
     'locations': create_locations_csv,
     'reporters': create_reporters_csv,

app/signals/apps/reporting/tests/csv/test_datawarehouse.py

@@ -258,6 +258,65 @@ def test_create_ml_csv(self):
         self.assertNotIn('Overig sub text', texts)
         self.assertNotIn('Open text', texts)
 
+    def test_create_ml_amsterdam_csv(self):
+        category = CategoryFactory.create(name='Sub', parent__name='Main')
+        signal = SignalFactory.create(text='ML text', category_assignment__category=category, status__state=AFGEHANDELD)
+        SignalFactory.create(
+            text='Cancelled text',
+            category_assignment__category=category,
+            status__state=GEANNULEERD,
+        )
+        SignalFactory.create(
+            text='Overig main text',
+            category_assignment__category=CategoryFactory.create(name='Sub', parent__name='Overig main'),
+            status__state=AFGEHANDELD,
+        )
+        SignalFactory.create(
+            text='Overig sub text',
+            category_assignment__category=CategoryFactory.create(name='Overig sub', parent__name='Main'),
+            status__state=AFGEHANDELD,
+        )
+
+        # Invalid state, should not be included
+        SignalFactory.create(
+            text='Open text',
+            category_assignment__category=category,
+            status__state=GEMELD,
+        )
+        uncategorized_signal = SignalFactory.create()
+        uncategorized_signal.category_assignment = None
+        uncategorized_signal.save(update_fields=['category_assignment'])
+
+        # Inactive category should not be included
+        inactive = SignalFactory.create(
+            text='Inactive category text',
+            category_assignment__category=CategoryFactory.create(name='Inactive', parent__name='Main'),
+            status__state=AFGEHANDELD,
+        )
+        inactive.category_assignment.category.is_active = False
+        inactive.category_assignment.category.save(update_fields=['is_active'])
+
+        csv_file = datawarehouse.create_ml_amsterdam_csv(self.csv_tmp_dir)
+
+        self.assertEqual(path.join(self.csv_tmp_dir, 'ml_amsterdam.csv'), csv_file)
+
+        with open(csv_file) as opened_csv_file:
+            rows = list(csv.DictReader(opened_csv_file))
+            texts = [row['Text'] for row in rows]
+
+        self.assertEqual(len(rows), 2)
+        self.assertEqual(rows[0]['Text'], signal.text)
+        self.assertEqual(rows[0]['Main'], signal.category_assignment.category.parent.slug)
+        self.assertEqual(rows[0]['Sub'], signal.category_assignment.category.slug)
+        self.assertEqual(rows[1]['Text'], 'Cancelled text')
+        self.assertEqual(rows[1]['Main'], 'main')
+        self.assertEqual(rows[1]['Sub'], 'sub')
+
+        self.assertNotIn('Overig main text', texts)
+        self.assertNotIn('Overig sub text', texts)
+        self.assertNotIn('Open text', texts)
+        self.assertNotIn('Inactive category text', texts)
+
     def test_create_locations_csv(self):
         signal = SignalFactory.create(location__area_code='AREACODE', location__area_name='AREA_NAME')
         location = signal.location

app/signals/apps/reporting/tests/management/commands/test_command.py

@@ -18,7 +18,7 @@ def test_command(self, patched_save_csv_files_datawarehouse):
         self.assertNotEqual(out.getvalue(), '')
         self.assertEqual(err.getvalue(), '')
 
-        self.assertEqual(patched_save_csv_files_datawarehouse.call_count, 12)
+        self.assertEqual(patched_save_csv_files_datawarehouse.call_count, 13)
 
 
 class TestCSVHorecaCommand(TransactionTestCase):