From c4f9b2914e8b168aac4656a7fb01e257edc49bfd Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Thu, 14 Aug 2025 13:00:33 -0300 Subject: [PATCH 01/10] Retrieve code references for a feature --- api/projects/code_references/permissions.py | 10 +- api/projects/code_references/serializers.py | 41 +++- api/projects/code_references/types.py | 19 ++ api/projects/code_references/urls.py | 5 + api/projects/code_references/views.py | 106 +++++++++- ...est_unit_projects_code_references_views.py | 181 ++++++++++++++++++ 6 files changed, 349 insertions(+), 13 deletions(-) create mode 100644 api/projects/code_references/types.py diff --git a/api/projects/code_references/permissions.py b/api/projects/code_references/permissions.py index c9c08b9b8ef0..409dff1008db 100644 --- a/api/projects/code_references/permissions.py +++ b/api/projects/code_references/permissions.py @@ -7,7 +7,7 @@ from users.models import FFAdminUser -class SubmitFeatureFlagCodeReferences(IsAuthenticated): +class _BaseCodeReferencePermission(IsAuthenticated): def has_permission(self, request: Request, view: APIView) -> bool: if not super().has_permission(request, view): return False @@ -17,3 +17,11 @@ def has_permission(self, request: Request, view: APIView) -> bool: project = Project.objects.get(id=view.kwargs["project_pk"]) return request.user.has_project_permission(VIEW_PROJECT, project) + + +class SubmitFeatureFlagCodeReferences(_BaseCodeReferencePermission): + pass + + +class ViewFeatureFlagCodeReferences(_BaseCodeReferencePermission): + pass diff --git a/api/projects/code_references/serializers.py b/api/projects/code_references/serializers.py index f4be377c52e1..6d2d4cf16559 100644 --- a/api/projects/code_references/serializers.py +++ b/api/projects/code_references/serializers.py @@ -1,26 +1,31 @@ -from typing import TypedDict - from rest_framework import serializers from projects.code_references.models import FeatureFlagCodeReferencesScan +from projects.code_references.types import ( + CodeReference, + FeatureFlagCodeReferences, +) -class _CodeReference(TypedDict): - feature_name: str - file_path: str - line_number: int +class _BaseCodeReferenceSerializer(serializers.Serializer[CodeReference]): + file_path = serializers.CharField(max_length=200) + line_number = serializers.IntegerField(min_value=1) -class _CodeReferenceSerializer(serializers.Serializer[_CodeReference]): +class _CodeReferenceSubmitSerializer(_BaseCodeReferenceSerializer): feature_name = serializers.CharField(max_length=100) - file_path = serializers.CharField(max_length=200) - line_number = serializers.IntegerField(min_value=1) + + +class _CodeReferenceDetailSerializer(_BaseCodeReferenceSerializer): + permalink = serializers.URLField() + scanned_at = serializers.DateTimeField() + revision = serializers.CharField() class FeatureFlagCodeReferencesScanSerializer( serializers.ModelSerializer[FeatureFlagCodeReferencesScan], ): - code_references = _CodeReferenceSerializer( + code_references = _CodeReferenceSubmitSerializer( many=True, required=True, allow_empty=False ) @@ -37,3 +42,19 @@ class Meta: "created_at", "project", ] + + +class FeatureFlagCodeReferencesSerializer( + serializers.Serializer[FeatureFlagCodeReferences], +): + first_scanned_at = serializers.DateTimeField() + last_scanned_at = serializers.DateTimeField() + + code_references = _CodeReferenceDetailSerializer(many=True) + + class Meta: + fields = read_only_fields = [ + "first_scanned_at", + "last_scanned_at", + "code_references", + ] diff --git a/api/projects/code_references/types.py b/api/projects/code_references/types.py new file mode 100644 index 000000000000..45922b3d3a22 --- /dev/null +++ b/api/projects/code_references/types.py @@ -0,0 +1,19 @@ +from dataclasses import dataclass +from datetime import datetime + + +@dataclass +class CodeReference: + scanned_at: datetime + revision: str + feature_name: str + file_path: str + line_number: int + permalink: str + + +@dataclass +class FeatureFlagCodeReferences: + first_scanned_at: datetime | None + last_scanned_at: datetime | None + code_references: list[CodeReference] diff --git a/api/projects/code_references/urls.py b/api/projects/code_references/urls.py index 85a56fb4a8cd..e7482f3fef3b 100644 --- a/api/projects/code_references/urls.py +++ b/api/projects/code_references/urls.py @@ -10,4 +10,9 @@ views.FeatureFlagCodeReferencesScanCreateAPIView.as_view(), name="code_reference_create", ), + path( + "projects//features//code-references/", + views.FeatureFlagCodeReferenceDetailAPIView.as_view(), + name="feature_code_reference_detail", + ), ] diff --git a/api/projects/code_references/views.py b/api/projects/code_references/views.py index 2bbdf75f7fde..031546f35ec8 100644 --- a/api/projects/code_references/views.py +++ b/api/projects/code_references/views.py @@ -1,8 +1,25 @@ +from datetime import datetime +from functools import cache +from urllib.parse import urljoin + +from django.db.models import QuerySet +from django.shortcuts import get_object_or_404 from rest_framework import generics +from features.models import Feature from projects.code_references.models import FeatureFlagCodeReferencesScan -from projects.code_references.permissions import SubmitFeatureFlagCodeReferences -from projects.code_references.serializers import FeatureFlagCodeReferencesScanSerializer +from projects.code_references.permissions import ( + SubmitFeatureFlagCodeReferences, + ViewFeatureFlagCodeReferences, +) +from projects.code_references.serializers import ( + FeatureFlagCodeReferencesScanSerializer, + FeatureFlagCodeReferencesSerializer, +) +from projects.code_references.types import ( + CodeReference, + FeatureFlagCodeReferences, +) class FeatureFlagCodeReferencesScanCreateAPIView( @@ -19,3 +36,88 @@ def perform_create( # type: ignore[override] self, serializer: FeatureFlagCodeReferencesScanSerializer ) -> None: serializer.save(project_id=self.kwargs["project_pk"]) + + +class FeatureFlagCodeReferenceDetailAPIView( + generics.RetrieveAPIView[FeatureFlagCodeReferences], # type: ignore[type-var] +): + """ + API view to retrieve code references for a specific feature in a project + """ + + serializer_class = FeatureFlagCodeReferencesSerializer + permission_classes = [ViewFeatureFlagCodeReferences] + + def get_object(self) -> FeatureFlagCodeReferences: + return FeatureFlagCodeReferences( + first_scanned_at=self._get_first_scanned_at(), + last_scanned_at=self._get_last_scanned_at(), + code_references=self._get_code_references(), + ) + + @cache + def _get_feature(self) -> Feature: + return get_object_or_404( + Feature, + project_id=self.kwargs["project_pk"], + pk=self.kwargs["feature_pk"], + ) + + def _get_related_references(self) -> QuerySet[FeatureFlagCodeReferencesScan]: + feature = self._get_feature() + return FeatureFlagCodeReferencesScan.objects.filter( + project_id=self.kwargs["project_pk"], + code_references__contains=[{"feature_name": feature.name}], + ) + + def _get_first_scanned_at(self) -> datetime | None: + related = self._get_related_references() + first_match = related.only("created_at").order_by("created_at").first() + return first_match.created_at if first_match else None + + def _get_last_scanned_at(self) -> datetime | None: + related = self._get_related_references() + last_match = related.only("created_at").order_by("-created_at").first() + return last_match.created_at if last_match else None + + def _get_code_references(self) -> list[CodeReference]: + feature = self._get_feature() + last_scans_of_each_repository = ( + FeatureFlagCodeReferencesScan.objects.filter( + project_id=self.kwargs["project_pk"], + ) + .order_by("repository_url", "-created_at") + .distinct("repository_url") + ) + + return [ + CodeReference( + feature_name=feature.name, + file_path=reference["file_path"], + line_number=reference["line_number"], + permalink=self._get_permalink( + repository_url=scan.repository_url, + revision=scan.revision, + file_path=reference["file_path"], + line_number=reference["line_number"], + ), + scanned_at=scan.created_at, + revision=scan.revision, + ) + for scan in last_scans_of_each_repository + for reference in scan.code_references + if reference["feature_name"] == feature.name + ] + + def _get_permalink( + self, + repository_url: str, + revision: str, + file_path: str, + line_number: int, + ) -> str: + """Generate a permalink for the code reference. + + NOTE: Only GitHub is supported right now. + """ + return urljoin(repository_url, f"blob/{revision}/{file_path}#L{line_number}") diff --git a/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py b/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py index a9b1c617ea14..b267f2c2362e 100644 --- a/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py +++ b/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py @@ -2,6 +2,7 @@ from common.projects.permissions import VIEW_PROJECT from rest_framework.test import APIClient +from features.models import Feature from projects.code_references.models import FeatureFlagCodeReferencesScan from projects.models import Project from tests.types import WithProjectPermissionsCallable @@ -126,3 +127,183 @@ def test_CodeReferenceCreateAPIView__responds_400_when_invalid_data( "code_references": [{"line_number": ["This field is required."]}] } assert not FeatureFlagCodeReferencesScan.objects.exists() + + +def test_FeatureCodeReferencesDetailAPIView__responds_200_with_code_references_for_given_feature( + feature: Feature, + project: Project, + staff_client: APIClient, + with_project_permissions: WithProjectPermissionsCallable, +) -> None: + # Given + with_project_permissions([VIEW_PROJECT]) # type: ignore[call-arg] + with freezegun.freeze_time("2099-01-01T10:00:00Z"): + FeatureFlagCodeReferencesScan.objects.create( + project=project, + repository_url="https://github.flagsmith.com/backend/", + revision="backend-1", + code_references=[ + { + "feature_name": feature.name, + "file_path": "backend/file1.py", + "line_number": 20, + }, + ], + ) + FeatureFlagCodeReferencesScan.objects.create( + project=project, + repository_url="https://github.flagsmith.com/frontend/", + revision="frontend-1", + code_references=[ + { + "feature_name": feature.name, + "file_path": "frontend/file1.js", + "line_number": 10, + }, + ], + ) + with freezegun.freeze_time("2099-01-02T11:00:00Z"): + FeatureFlagCodeReferencesScan.objects.create( + project=project, + repository_url="https://github.flagsmith.com/frontend/", + revision="frontend-2", + code_references=[ + { + "feature_name": feature.name, + "file_path": "frontend/file1.js", + "line_number": 12, + }, + { + "feature_name": feature.name, + "file_path": "frontend/file2.js", + "line_number": 5, + }, + ], + ) + + # When + response = staff_client.get( + f"/api/v1/projects/{project.pk}/features/{feature.pk}/code-references/", + ) + + # Then + assert response.status_code == 200 + assert response.data["first_scanned_at"] == "2099-01-01T10:00:00Z" + assert response.data["last_scanned_at"] == "2099-01-02T11:00:00Z" + assert len(references := response.data["code_references"]) == 3 + assert references[0]["scanned_at"] == "2099-01-01T10:00:00Z" + assert references[0]["revision"] == "backend-1" + assert references[1]["scanned_at"] == "2099-01-02T11:00:00Z" + assert references[0]["file_path"] == "backend/file1.py" + assert references[0]["line_number"] == 20 + assert references[0]["permalink"] == ( + "https://github.flagsmith.com/backend/blob/backend-1/backend/file1.py#L20" + ) + assert references[1]["scanned_at"] == "2099-01-02T11:00:00Z" + assert references[1]["revision"] == "frontend-2" + assert references[1]["file_path"] == "frontend/file1.js" + assert references[1]["line_number"] == 12 + assert references[1]["permalink"] == ( + "https://github.flagsmith.com/frontend/blob/frontend-2/frontend/file1.js#L12" + ) + assert references[2]["scanned_at"] == "2099-01-02T11:00:00Z" + assert references[2]["revision"] == "frontend-2" + assert references[2]["file_path"] == "frontend/file2.js" + assert references[2]["line_number"] == 5 + assert references[2]["permalink"] == ( + "https://github.flagsmith.com/frontend/blob/frontend-2/frontend/file2.js#L5" + ) + + +def test_FeatureCodeReferencesDetailAPIView__responds_200_with_feature_flag_removed( + feature: Feature, + project: Project, + staff_client: APIClient, + with_project_permissions: WithProjectPermissionsCallable, +) -> None: + # Given + with_project_permissions([VIEW_PROJECT]) # type: ignore[call-arg] + with freezegun.freeze_time("2099-01-01T10:00:00Z"): + FeatureFlagCodeReferencesScan.objects.create( + project=project, + repository_url="https://github.flagsmith.com/", + revision="revision-hash-1", + code_references=[ + { + "feature_name": feature.name, + "file_path": "path/to/file1.py", + "line_number": 10, + }, + ], + ) + with freezegun.freeze_time("2099-01-02T11:00:00Z"): + FeatureFlagCodeReferencesScan.objects.create( + project=project, + repository_url="https://github.flagsmith.com/", + revision="revision-hash-2", + code_references=[], # Feature flag removed + ) + + # When + response = staff_client.get( + f"/api/v1/projects/{project.pk}/features/{feature.pk}/code-references/", + ) + + # Then + assert response.status_code == 200 + assert response.data["first_scanned_at"] == "2099-01-01T10:00:00Z" + assert response.data["last_scanned_at"] == "2099-01-01T10:00:00Z" + assert len(response.data["code_references"]) == 0 + + +def test_FeatureCodeReferencesDetailAPIView__responds_200_even_without_code_references( + feature: Feature, + project: Project, + staff_client: APIClient, + with_project_permissions: WithProjectPermissionsCallable, +) -> None: + # Given + with_project_permissions([VIEW_PROJECT]) # type: ignore[call-arg] + + # When + response = staff_client.get( + f"/api/v1/projects/{project.pk}/features/{feature.pk}/code-references/", + ) + + # Then + assert response.status_code == 200 + assert response.data["first_scanned_at"] is None + assert response.data["last_scanned_at"] is None + assert response.data["code_references"] == [] + + +def test_FeatureCodeReferencesDetailAPIView__responds_401_when_not_authenticated( + feature: Feature, + project: Project, + client: APIClient, +) -> None: + # When + response = client.get( + f"/api/v1/projects/{project.pk}/features/{feature.pk}/code-references/", + ) + + # Then + assert response.status_code == 401 + + +def test_FeatureCodeReferencesDetailAPIView__responds_404_when_feature_not_found( + project: Project, + staff_client: APIClient, + with_project_permissions: WithProjectPermissionsCallable, +) -> None: + # Given + with_project_permissions([VIEW_PROJECT]) # type: ignore[call-arg] + + # When + response = staff_client.get( + f"/api/v1/projects/{project.pk}/features/9999/code-references/", + ) + + # Then + assert response.status_code == 404 + assert response.data["detail"] == "No Feature matches the given query." From f5015b9925607228aaba285af210a90e5d47e493 Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Thu, 14 Aug 2025 12:58:33 -0300 Subject: [PATCH 02/10] Improve JSON typing --- api/projects/code_references/models.py | 4 +++- api/projects/code_references/types.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/api/projects/code_references/models.py b/api/projects/code_references/models.py index b8abb7c58b50..319fc64caf46 100644 --- a/api/projects/code_references/models.py +++ b/api/projects/code_references/models.py @@ -1,5 +1,7 @@ from django.db import models +from projects.code_references.types import JSONCodeReference + class FeatureFlagCodeReferencesScan(models.Model): """ @@ -24,7 +26,7 @@ class Providers(models.TextChoices): default=Providers.GITHUB, # TODO: Remove when adding other providers ) revision = models.CharField(max_length=100) - code_references = models.JSONField(default=list) + code_references = models.JSONField[list[JSONCodeReference]](default=list) created_at = models.DateTimeField(auto_now_add=True, db_index=True) diff --git a/api/projects/code_references/types.py b/api/projects/code_references/types.py index 45922b3d3a22..56eebce8b82a 100644 --- a/api/projects/code_references/types.py +++ b/api/projects/code_references/types.py @@ -1,5 +1,12 @@ from dataclasses import dataclass from datetime import datetime +from typing import TypedDict + + +class JSONCodeReference(TypedDict): + feature_name: str + file_path: str + line_number: int @dataclass From dc72ef4f3485f16b9107451512150461220498a2 Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Thu, 14 Aug 2025 12:57:29 -0300 Subject: [PATCH 03/10] Add GIN index to help filtering code references --- .../migrations/0002_gotta_go_fast.py | 25 +++++++++++++++++++ api/projects/code_references/models.py | 12 +++++++++ 2 files changed, 37 insertions(+) create mode 100644 api/projects/code_references/migrations/0002_gotta_go_fast.py diff --git a/api/projects/code_references/migrations/0002_gotta_go_fast.py b/api/projects/code_references/migrations/0002_gotta_go_fast.py new file mode 100644 index 000000000000..5ab2998040aa --- /dev/null +++ b/api/projects/code_references/migrations/0002_gotta_go_fast.py @@ -0,0 +1,25 @@ +# Generated by Django 4.2.22 on 2025-08-14 16:51 + +import django.contrib.postgres.indexes +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("code_references", "0001_code_references"), + ] + + operations = [ + migrations.AddIndex( + model_name="featureflagcodereferencesscan", + index=django.contrib.postgres.indexes.GinIndex( + models.Func( + models.F("code_references"), + models.Value("$[*].feature_name"), + function="jsonb_path_query_array", + ), + name="code_references_feature_name", + ), + ), + ] diff --git a/api/projects/code_references/models.py b/api/projects/code_references/models.py index 319fc64caf46..f3c4a652c3f5 100644 --- a/api/projects/code_references/models.py +++ b/api/projects/code_references/models.py @@ -1,4 +1,6 @@ +from django.contrib.postgres.indexes import GinIndex from django.db import models +from django.db.models.expressions import Func from projects.code_references.types import JSONCodeReference @@ -32,3 +34,13 @@ class Providers(models.TextChoices): class Meta: ordering = ["-created_at"] + indexes = [ + GinIndex( # Helps filtering code references by feature name + Func( + models.F("code_references"), + models.Value("$[*].feature_name"), + function="jsonb_path_query_array", + ), + name="code_references_feature_name", + ), + ] From 9823242dd0e21b1cd4ceea657e3e5c1fb10903eb Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Thu, 14 Aug 2025 14:03:29 -0300 Subject: [PATCH 04/10] Add frontend-required info to code references spec --- api/projects/code_references/models.py | 9 +++------ api/projects/code_references/serializers.py | 5 ++++- api/projects/code_references/types.py | 8 ++++++++ api/projects/code_references/views.py | 9 +++++++-- .../test_unit_projects_code_references_views.py | 6 ++++++ 5 files changed, 28 insertions(+), 9 deletions(-) diff --git a/api/projects/code_references/models.py b/api/projects/code_references/models.py index f3c4a652c3f5..6c7e97590916 100644 --- a/api/projects/code_references/models.py +++ b/api/projects/code_references/models.py @@ -2,7 +2,7 @@ from django.db import models from django.db.models.expressions import Func -from projects.code_references.types import JSONCodeReference +from projects.code_references.types import JSONCodeReference, VCSProvider class FeatureFlagCodeReferencesScan(models.Model): @@ -10,9 +10,6 @@ class FeatureFlagCodeReferencesScan(models.Model): A scan of feature flag code references in a repository """ - class Providers(models.TextChoices): - GITHUB = "github", "GitHub" - project = models.ForeignKey( "projects.Project", on_delete=models.CASCADE, @@ -24,8 +21,8 @@ class Providers(models.TextChoices): vcs_provider = models.CharField( max_length=50, - choices=Providers.choices, - default=Providers.GITHUB, # TODO: Remove when adding other providers + choices=VCSProvider.choices, + default=VCSProvider.GITHUB, # TODO: Remove when adding other providers ) revision = models.CharField(max_length=100) code_references = models.JSONField[list[JSONCodeReference]](default=list) diff --git a/api/projects/code_references/serializers.py b/api/projects/code_references/serializers.py index 6d2d4cf16559..c50b8d4a074a 100644 --- a/api/projects/code_references/serializers.py +++ b/api/projects/code_references/serializers.py @@ -4,6 +4,7 @@ from projects.code_references.types import ( CodeReference, FeatureFlagCodeReferences, + VCSProvider, ) @@ -17,9 +18,11 @@ class _CodeReferenceSubmitSerializer(_BaseCodeReferenceSerializer): class _CodeReferenceDetailSerializer(_BaseCodeReferenceSerializer): - permalink = serializers.URLField() scanned_at = serializers.DateTimeField() + vcs_provider = serializers.ChoiceField(choices=VCSProvider.choices) + repository_url = serializers.URLField() revision = serializers.CharField() + permalink = serializers.URLField() class FeatureFlagCodeReferencesScanSerializer( diff --git a/api/projects/code_references/types.py b/api/projects/code_references/types.py index 56eebce8b82a..b89dbe6e9ee6 100644 --- a/api/projects/code_references/types.py +++ b/api/projects/code_references/types.py @@ -2,6 +2,12 @@ from datetime import datetime from typing import TypedDict +from django.db.models import TextChoices + + +class VCSProvider(TextChoices): + GITHUB = "github", "GitHub" + class JSONCodeReference(TypedDict): feature_name: str @@ -12,6 +18,8 @@ class JSONCodeReference(TypedDict): @dataclass class CodeReference: scanned_at: datetime + vcs_provider: VCSProvider + repository_url: str revision: str feature_name: str file_path: str diff --git a/api/projects/code_references/views.py b/api/projects/code_references/views.py index 031546f35ec8..0d163eb4b6be 100644 --- a/api/projects/code_references/views.py +++ b/api/projects/code_references/views.py @@ -19,6 +19,7 @@ from projects.code_references.types import ( CodeReference, FeatureFlagCodeReferences, + VCSProvider, ) @@ -92,17 +93,20 @@ def _get_code_references(self) -> list[CodeReference]: return [ CodeReference( + scanned_at=scan.created_at, + vcs_provider=scan.vcs_provider, + repository_url=scan.repository_url, + revision=scan.revision, feature_name=feature.name, file_path=reference["file_path"], line_number=reference["line_number"], permalink=self._get_permalink( + provider=scan.vcs_provider, repository_url=scan.repository_url, revision=scan.revision, file_path=reference["file_path"], line_number=reference["line_number"], ), - scanned_at=scan.created_at, - revision=scan.revision, ) for scan in last_scans_of_each_repository for reference in scan.code_references @@ -111,6 +115,7 @@ def _get_code_references(self) -> list[CodeReference]: def _get_permalink( self, + provider: VCSProvider, repository_url: str, revision: str, file_path: str, diff --git a/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py b/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py index b267f2c2362e..701184b77c91 100644 --- a/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py +++ b/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py @@ -192,6 +192,8 @@ def test_FeatureCodeReferencesDetailAPIView__responds_200_with_code_references_f assert response.data["last_scanned_at"] == "2099-01-02T11:00:00Z" assert len(references := response.data["code_references"]) == 3 assert references[0]["scanned_at"] == "2099-01-01T10:00:00Z" + assert references[0]["vcs_provider"] == "github" + assert references[0]["repository_url"] == "https://github.flagsmith.com/backend/" assert references[0]["revision"] == "backend-1" assert references[1]["scanned_at"] == "2099-01-02T11:00:00Z" assert references[0]["file_path"] == "backend/file1.py" @@ -200,6 +202,8 @@ def test_FeatureCodeReferencesDetailAPIView__responds_200_with_code_references_f "https://github.flagsmith.com/backend/blob/backend-1/backend/file1.py#L20" ) assert references[1]["scanned_at"] == "2099-01-02T11:00:00Z" + assert references[1]["vcs_provider"] == "github" + assert references[1]["repository_url"] == "https://github.flagsmith.com/frontend/" assert references[1]["revision"] == "frontend-2" assert references[1]["file_path"] == "frontend/file1.js" assert references[1]["line_number"] == 12 @@ -207,6 +211,8 @@ def test_FeatureCodeReferencesDetailAPIView__responds_200_with_code_references_f "https://github.flagsmith.com/frontend/blob/frontend-2/frontend/file1.js#L12" ) assert references[2]["scanned_at"] == "2099-01-02T11:00:00Z" + assert references[2]["vcs_provider"] == "github" + assert references[2]["repository_url"] == "https://github.flagsmith.com/frontend/" assert references[2]["revision"] == "frontend-2" assert references[2]["file_path"] == "frontend/file2.js" assert references[2]["line_number"] == 5 From 0b2a17feb5cb34c0278f7071cab8db50d466f48d Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Thu, 14 Aug 2025 14:29:12 -0300 Subject: [PATCH 05/10] Fix typing --- api/projects/code_references/views.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/api/projects/code_references/views.py b/api/projects/code_references/views.py index 0d163eb4b6be..b4150ce277f7 100644 --- a/api/projects/code_references/views.py +++ b/api/projects/code_references/views.py @@ -94,14 +94,14 @@ def _get_code_references(self) -> list[CodeReference]: return [ CodeReference( scanned_at=scan.created_at, - vcs_provider=scan.vcs_provider, + vcs_provider=VCSProvider(scan.vcs_provider), repository_url=scan.repository_url, revision=scan.revision, feature_name=feature.name, file_path=reference["file_path"], line_number=reference["line_number"], permalink=self._get_permalink( - provider=scan.vcs_provider, + provider=VCSProvider(scan.vcs_provider), repository_url=scan.repository_url, revision=scan.revision, file_path=reference["file_path"], @@ -121,8 +121,12 @@ def _get_permalink( file_path: str, line_number: int, ) -> str: - """Generate a permalink for the code reference. - - NOTE: Only GitHub is supported right now. - """ - return urljoin(repository_url, f"blob/{revision}/{file_path}#L{line_number}") + """Generate a permalink for the code reference.""" + match provider: + case VCSProvider.GITHUB: + return urljoin( + repository_url, f"blob/{revision}/{file_path}#L{line_number}" + ) + raise NotImplementedError( # pragma: no cover + f"Permalink generation for {provider} is not implemented." + ) From 2ca2632c62a8dd18240d5d86713841fe01c07901 Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Thu, 14 Aug 2025 20:50:43 -0300 Subject: [PATCH 06/10] Add code references counts to feature list --- api/features/serializers.py | 11 ++- api/features/views.py | 5 ++ api/projects/code_references/serializers.py | 8 +++ api/projects/code_references/services.py | 52 ++++++++++++++ api/projects/code_references/types.py | 6 ++ .../unit/features/test_unit_features_views.py | 69 +++++++++++++++++++ 6 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 api/projects/code_references/services.py diff --git a/api/features/serializers.py b/api/features/serializers.py index 7e9bd68f1304..1050f9331ce6 100644 --- a/api/features/serializers.py +++ b/api/features/serializers.py @@ -24,6 +24,7 @@ from integrations.github.constants import GitHubEventType from integrations.github.github import call_github_task from metadata.serializers import MetadataSerializer, MetadataSerializerMixin +from projects.code_references.serializers import CodeReferencesRepositoryCountSerializer from projects.models import Project from users.serializers import ( UserIdsSerializer, @@ -327,8 +328,16 @@ def get_last_modified_in_current_environment( class FeatureSerializerWithMetadata(MetadataSerializerMixin, CreateFeatureSerializer): metadata = MetadataSerializer(required=False, many=True) + code_references_counts = CodeReferencesRepositoryCountSerializer( + many=True, + read_only=True, + ) + class Meta(CreateFeatureSerializer.Meta): - fields = CreateFeatureSerializer.Meta.fields + ("metadata",) # type: ignore[assignment] + fields = CreateFeatureSerializer.Meta.fields + ( # type: ignore[assignment] + "metadata", + "code_references_counts", + ) def validate(self, attrs: dict[str, Any]) -> dict[str, Any]: attrs = super().validate(attrs) diff --git a/api/features/views.py b/api/features/views.py index ef55ffe82978..5cf5e04d1539 100644 --- a/api/features/views.py +++ b/api/features/views.py @@ -39,6 +39,9 @@ NestedEnvironmentPermissions, ) from features.value_types import BOOLEAN, INTEGER, STRING +from projects.code_references.services import ( + annotate_feature_queryset_with_code_references_summary, +) from projects.models import Project from users.models import FFAdminUser, UserPermissionGroup from webhooks.webhooks import WebhookEventType @@ -143,6 +146,8 @@ def get_queryset(self): # type: ignore[no-untyped-def] query_serializer.is_valid(raise_exception=True) query_data = query_serializer.validated_data + queryset = annotate_feature_queryset_with_code_references_summary(queryset) + queryset = self._filter_queryset(queryset) if environment_id := query_data.get("environment"): diff --git a/api/projects/code_references/serializers.py b/api/projects/code_references/serializers.py index c50b8d4a074a..4e7084fde157 100644 --- a/api/projects/code_references/serializers.py +++ b/api/projects/code_references/serializers.py @@ -3,6 +3,7 @@ from projects.code_references.models import FeatureFlagCodeReferencesScan from projects.code_references.types import ( CodeReference, + CodeReferencesRepositoryCount, FeatureFlagCodeReferences, VCSProvider, ) @@ -61,3 +62,10 @@ class Meta: "last_scanned_at", "code_references", ] + + +class CodeReferencesRepositoryCountSerializer( + serializers.Serializer[CodeReferencesRepositoryCount], +): + repository_url = serializers.URLField() + count = serializers.IntegerField() diff --git a/api/projects/code_references/services.py b/api/projects/code_references/services.py new file mode 100644 index 000000000000..4cd8859427e4 --- /dev/null +++ b/api/projects/code_references/services.py @@ -0,0 +1,52 @@ +from datetime import timedelta + +from django.contrib.postgres.expressions import ArraySubquery +from django.db.models import F, Func, OuterRef, QuerySet, Value +from django.db.models.functions import JSONObject +from django.utils import timezone + +from features.models import Feature +from projects.code_references.models import FeatureFlagCodeReferencesScan + + +def annotate_feature_queryset_with_code_references_summary( + queryset: QuerySet[Feature], +) -> QuerySet[Feature]: + """Extend feature objects with a `code_references_counts` + + NOTE: This adds compatibility with `CodeReferenceRepositoryCountSerializer` + while preventing N+1 queries from the serializer. + """ + counts_by_repository = ( + FeatureFlagCodeReferencesScan.objects + # Count code references from JSON matching the feature name + .annotate( + feature_name=OuterRef("name"), + count=Func( + Func( + F("code_references"), + Value("$[*] ? (@.feature_name == $feature_name)"), + JSONObject(feature_name=F("feature_name")), + function="jsonb_path_query_array", + ), + function="jsonb_array_length", + ), + ) + # Only from the latest scans of each repository, within a 30-day sanity window + .filter( + created_at__gte=timezone.now() - timedelta(days=30), + project_id=OuterRef("project_id"), + ) + .order_by("repository_url", "-created_at") + .distinct("repository_url") + .values( + json=JSONObject( + repository_url=F("repository_url"), + count=F("count"), + ), + ) + ) + + return queryset.annotate( + code_references_counts=ArraySubquery(counts_by_repository), + ) diff --git a/api/projects/code_references/types.py b/api/projects/code_references/types.py index b89dbe6e9ee6..84924c69a20f 100644 --- a/api/projects/code_references/types.py +++ b/api/projects/code_references/types.py @@ -32,3 +32,9 @@ class FeatureFlagCodeReferences: first_scanned_at: datetime | None last_scanned_at: datetime | None code_references: list[CodeReference] + + +@dataclass +class CodeReferencesRepositoryCount: + repository_url: str + count: int diff --git a/api/tests/unit/features/test_unit_features_views.py b/api/tests/unit/features/test_unit_features_views.py index d80431b9c1dd..2499748374dc 100644 --- a/api/tests/unit/features/test_unit_features_views.py +++ b/api/tests/unit/features/test_unit_features_views.py @@ -51,6 +51,7 @@ from metadata.models import MetadataModelField from organisations.models import Organisation, OrganisationRole from permissions.models import PermissionModel +from projects.code_references.models import FeatureFlagCodeReferencesScan from projects.models import Project, UserProjectPermission from projects.tags.models import Tag from segments.models import Segment @@ -3343,6 +3344,74 @@ def test_list_features_with_filter_by_search_value_boolean( assert response.data["results"][0]["name"] == feature2.name +def test_FeatureViewSet_list__includes_code_references_counts( + staff_client: APIClient, + project: Project, + feature: Feature, + with_project_permissions: WithProjectPermissionsCallable, + environment: Environment, +) -> None: + # Given + with_project_permissions([VIEW_PROJECT]) # type: ignore[call-arg] + FeatureFlagCodeReferencesScan.objects.create( + project=project, + repository_url="https://github.flagsmith.com/backend/", + revision="backend-1", + code_references=[ + { + "feature_name": feature.name, + "file_path": "path/to/file.py", + "line_number": 42, + }, + ], + ) + FeatureFlagCodeReferencesScan.objects.create( + project=project, + repository_url="https://gitlab.flagsmith.com/frontend/", + revision="frontend-1", + code_references=[ + { + "feature_name": feature.name, + "file_path": "path/to/file.js", + "line_number": 23, + }, + ], + ) + FeatureFlagCodeReferencesScan.objects.create( + project=project, + repository_url="https://gitlab.flagsmith.com/frontend/", + revision="frontend-2", + code_references=[ + { + "feature_name": feature.name, + "file_path": "path/to/file.js", + "line_number": 24, + }, + { + "feature_name": feature.name, + "file_path": "path/to/another/file.js", + "line_number": 10, + }, + { + "feature_name": f"Another {feature.name}", + "file_path": "path/to/another/file.js", + "line_number": 11, + }, + ], + ) + + # When + response = staff_client.get(f"/api/v1/projects/{project.pk}/features/") + + # Then + assert response.status_code == status.HTTP_200_OK + assert (counts := response.data["results"][0]["code_references_counts"]) + assert counts[0]["repository_url"] == "https://github.flagsmith.com/backend/" + assert counts[0]["count"] == 1 + assert counts[1]["repository_url"] == "https://gitlab.flagsmith.com/frontend/" + assert counts[1]["count"] == 2 + + def test_simple_feature_state_returns_only_latest_versions( staff_client: APIClient, staff_user: FFAdminUser, From 60a111258411ce957d10acb4843849da1b2c1aec Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Fri, 15 Aug 2025 14:03:04 -0300 Subject: [PATCH 07/10] Rely on data, not guesses --- api/projects/code_references/serializers.py | 2 +- ...est_unit_projects_code_references_views.py | 39 ++++++++++++++++++- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/api/projects/code_references/serializers.py b/api/projects/code_references/serializers.py index 4e7084fde157..2499d779a5bf 100644 --- a/api/projects/code_references/serializers.py +++ b/api/projects/code_references/serializers.py @@ -10,7 +10,7 @@ class _BaseCodeReferenceSerializer(serializers.Serializer[CodeReference]): - file_path = serializers.CharField(max_length=200) + file_path = serializers.CharField(max_length=260) # Windows' MAX_PATH line_number = serializers.IntegerField(min_value=1) diff --git a/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py b/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py index 701184b77c91..a30664dda56c 100644 --- a/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py +++ b/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py @@ -96,7 +96,7 @@ def test_CodeReferenceCreateAPIView__responds_401_when_not_authenticated( assert not FeatureFlagCodeReferencesScan.objects.exists() -def test_CodeReferenceCreateAPIView__responds_400_when_invalid_data( +def test_CodeReferenceCreateAPIView__responds_400_when_missing_field( project: Project, staff_client: APIClient, with_project_permissions: WithProjectPermissionsCallable, @@ -124,7 +124,42 @@ def test_CodeReferenceCreateAPIView__responds_400_when_invalid_data( # Then assert response.status_code == 400 assert response.data == { - "code_references": [{"line_number": ["This field is required."]}] + "code_references": [{"line_number": ["This field is required."]}], + } + assert not FeatureFlagCodeReferencesScan.objects.exists() + + +def test_CodeReferenceCreateAPIView__responds_400_when_file_path_too_long( + project: Project, + staff_client: APIClient, + with_project_permissions: WithProjectPermissionsCallable, +) -> None: + # Given + with_project_permissions([VIEW_PROJECT]) # type: ignore[call-arg] + + # When + response = staff_client.post( + f"/api/v1/projects/{project.pk}/code-references/", + data={ + "repository_url": "https://svn.flagsmith.com/", + "revision": "revision-hash", + "code_references": [ + { + "feature_name": "feature-1", + "file_path": "windows/limit/" * 100 + "file.py", + "line_number": 10, + }, + ], + }, + format="json", + ) + + # Then + assert response.status_code == 400 + assert response.data == { + "code_references": [ + {"file_path": ["Ensure this field has no more than 260 characters."]} + ], } assert not FeatureFlagCodeReferencesScan.objects.exists() From 7bc6166b96ebf760b9c9fc6ffa224ca0bdcd2b83 Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Fri, 15 Aug 2025 15:31:32 -0300 Subject: [PATCH 08/10] Revert "Add GIN index to help filtering code references" This reverts commit dc72ef4f3485f16b9107451512150461220498a2. --- .../migrations/0002_gotta_go_fast.py | 25 ------------------- api/projects/code_references/models.py | 12 --------- 2 files changed, 37 deletions(-) delete mode 100644 api/projects/code_references/migrations/0002_gotta_go_fast.py diff --git a/api/projects/code_references/migrations/0002_gotta_go_fast.py b/api/projects/code_references/migrations/0002_gotta_go_fast.py deleted file mode 100644 index 5ab2998040aa..000000000000 --- a/api/projects/code_references/migrations/0002_gotta_go_fast.py +++ /dev/null @@ -1,25 +0,0 @@ -# Generated by Django 4.2.22 on 2025-08-14 16:51 - -import django.contrib.postgres.indexes -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("code_references", "0001_code_references"), - ] - - operations = [ - migrations.AddIndex( - model_name="featureflagcodereferencesscan", - index=django.contrib.postgres.indexes.GinIndex( - models.Func( - models.F("code_references"), - models.Value("$[*].feature_name"), - function="jsonb_path_query_array", - ), - name="code_references_feature_name", - ), - ), - ] diff --git a/api/projects/code_references/models.py b/api/projects/code_references/models.py index 6c7e97590916..5c1d7b3f0e48 100644 --- a/api/projects/code_references/models.py +++ b/api/projects/code_references/models.py @@ -1,6 +1,4 @@ -from django.contrib.postgres.indexes import GinIndex from django.db import models -from django.db.models.expressions import Func from projects.code_references.types import JSONCodeReference, VCSProvider @@ -31,13 +29,3 @@ class FeatureFlagCodeReferencesScan(models.Model): class Meta: ordering = ["-created_at"] - indexes = [ - GinIndex( # Helps filtering code references by feature name - Func( - models.F("code_references"), - models.Value("$[*].feature_name"), - function="jsonb_path_query_array", - ), - name="code_references_feature_name", - ), - ] From 6244ebff5fe948cfc187eb43d71aa398afc9f69c Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Fri, 15 Aug 2025 17:21:31 -0300 Subject: [PATCH 09/10] Refactor GitHub PoC retrieval endpoints --- api/features/serializers.py | 6 +- api/projects/code_references/constants.py | 2 + .../migrations/0001_code_references.py | 2 +- api/projects/code_references/serializers.py | 25 ++-- api/projects/code_references/services.py | 112 +++++++++++++++- api/projects/code_references/types.py | 15 ++- api/projects/code_references/views.py | 98 ++------------ .../unit/features/test_unit_features_views.py | 120 ++++++++++-------- ...est_unit_projects_code_references_views.py | 100 +++++++++------ 9 files changed, 273 insertions(+), 207 deletions(-) create mode 100644 api/projects/code_references/constants.py diff --git a/api/features/serializers.py b/api/features/serializers.py index 1050f9331ce6..af69f1c4d38c 100644 --- a/api/features/serializers.py +++ b/api/features/serializers.py @@ -24,7 +24,9 @@ from integrations.github.constants import GitHubEventType from integrations.github.github import call_github_task from metadata.serializers import MetadataSerializer, MetadataSerializerMixin -from projects.code_references.serializers import CodeReferencesRepositoryCountSerializer +from projects.code_references.serializers import ( + FeatureFlagCodeReferencesRepositoryCountSerializer, +) from projects.models import Project from users.serializers import ( UserIdsSerializer, @@ -328,7 +330,7 @@ def get_last_modified_in_current_environment( class FeatureSerializerWithMetadata(MetadataSerializerMixin, CreateFeatureSerializer): metadata = MetadataSerializer(required=False, many=True) - code_references_counts = CodeReferencesRepositoryCountSerializer( + code_references_counts = FeatureFlagCodeReferencesRepositoryCountSerializer( many=True, read_only=True, ) diff --git a/api/projects/code_references/constants.py b/api/projects/code_references/constants.py new file mode 100644 index 000000000000..605b199aee7e --- /dev/null +++ b/api/projects/code_references/constants.py @@ -0,0 +1,2 @@ +# TODO: Implement history cleanup? +FEATURE_FLAG_CODE_REFERENCES_RETENTION_DAYS = 30 diff --git a/api/projects/code_references/migrations/0001_code_references.py b/api/projects/code_references/migrations/0001_code_references.py index 478eff2f1aa6..2e21758cb0ab 100644 --- a/api/projects/code_references/migrations/0001_code_references.py +++ b/api/projects/code_references/migrations/0001_code_references.py @@ -1,7 +1,7 @@ # Generated by Django 4.2.22 on 2025-08-14 15:12 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/api/projects/code_references/serializers.py b/api/projects/code_references/serializers.py index 2499d779a5bf..f92b7a159767 100644 --- a/api/projects/code_references/serializers.py +++ b/api/projects/code_references/serializers.py @@ -4,7 +4,7 @@ from projects.code_references.types import ( CodeReference, CodeReferencesRepositoryCount, - FeatureFlagCodeReferences, + FeatureFlagCodeReferencesRepositorySummary, VCSProvider, ) @@ -48,24 +48,21 @@ class Meta: ] -class FeatureFlagCodeReferencesSerializer( - serializers.Serializer[FeatureFlagCodeReferences], +class FeatureFlagCodeReferencesRepositorySummarySerializer( + serializers.Serializer[FeatureFlagCodeReferencesRepositorySummary], ): - first_scanned_at = serializers.DateTimeField() - last_scanned_at = serializers.DateTimeField() - + repository_url = serializers.URLField() + vcs_provider = serializers.ChoiceField(choices=VCSProvider.choices) + revision = serializers.CharField() + last_successful_repository_scanned_at = serializers.DateTimeField() + last_feature_found_at = serializers.DateTimeField(allow_null=True) code_references = _CodeReferenceDetailSerializer(many=True) - class Meta: - fields = read_only_fields = [ - "first_scanned_at", - "last_scanned_at", - "code_references", - ] - -class CodeReferencesRepositoryCountSerializer( +class FeatureFlagCodeReferencesRepositoryCountSerializer( serializers.Serializer[CodeReferencesRepositoryCount], ): repository_url = serializers.URLField() count = serializers.IntegerField() + last_successful_repository_scanned_at = serializers.DateTimeField() + last_feature_found_at = serializers.DateTimeField(allow_null=True) diff --git a/api/projects/code_references/services.py b/api/projects/code_references/services.py index 4cd8859427e4..f8d95739d57e 100644 --- a/api/projects/code_references/services.py +++ b/api/projects/code_references/services.py @@ -1,12 +1,21 @@ from datetime import timedelta +from urllib.parse import urljoin from django.contrib.postgres.expressions import ArraySubquery -from django.db.models import F, Func, OuterRef, QuerySet, Value +from django.db.models import BooleanField, F, Func, OuterRef, QuerySet, Subquery, Value from django.db.models.functions import JSONObject from django.utils import timezone from features.models import Feature +from projects.code_references.constants import ( + FEATURE_FLAG_CODE_REFERENCES_RETENTION_DAYS, +) from projects.code_references.models import FeatureFlagCodeReferencesScan +from projects.code_references.types import ( + CodeReference, + FeatureFlagCodeReferencesRepositorySummary, + VCSProvider, +) def annotate_feature_queryset_with_code_references_summary( @@ -17,11 +26,33 @@ def annotate_feature_queryset_with_code_references_summary( NOTE: This adds compatibility with `CodeReferenceRepositoryCountSerializer` while preventing N+1 queries from the serializer. """ + history_delta = timedelta(days=FEATURE_FLAG_CODE_REFERENCES_RETENTION_DAYS) + last_feature_found_at = ( + FeatureFlagCodeReferencesScan.objects.annotate( + feature_name=OuterRef("feature_name"), + contains_feature_name=Func( + F("code_references"), + Value("$[*] ? (@.feature_name == $feature_name)"), + JSONObject(feature_name=F("feature_name")), + function="jsonb_path_exists", + output_field=BooleanField(), + ), + ) + .filter( + project=OuterRef("project_id"), + created_at__gte=timezone.now() - history_delta, + repository_url=OuterRef("repository_url"), + contains_feature_name=True, + ) + .values("created_at") + .order_by("-created_at")[:1] + ) counts_by_repository = ( FeatureFlagCodeReferencesScan.objects # Count code references from JSON matching the feature name .annotate( feature_name=OuterRef("name"), + last_feature_found_at=Subquery(last_feature_found_at), count=Func( Func( F("code_references"), @@ -32,9 +63,9 @@ def annotate_feature_queryset_with_code_references_summary( function="jsonb_array_length", ), ) - # Only from the latest scans of each repository, within a 30-day sanity window + # Only from the latest scans of each repository .filter( - created_at__gte=timezone.now() - timedelta(days=30), + created_at__gte=timezone.now() - history_delta, project_id=OuterRef("project_id"), ) .order_by("repository_url", "-created_at") @@ -43,6 +74,8 @@ def annotate_feature_queryset_with_code_references_summary( json=JSONObject( repository_url=F("repository_url"), count=F("count"), + last_successful_repository_scanned_at=F("created_at"), + last_feature_found_at=F("last_feature_found_at"), ), ) ) @@ -50,3 +83,76 @@ def annotate_feature_queryset_with_code_references_summary( return queryset.annotate( code_references_counts=ArraySubquery(counts_by_repository), ) + + +def get_code_references_for_feature_flag( + feature: Feature, +) -> list[FeatureFlagCodeReferencesRepositorySummary]: + """Obtain a summary of latest code references for a feature + + Only query from the latest scans of each repository_url. This is used to + populate `FeatureFlagCodeReferencesSerializer`. + """ + history_delta = timedelta(days=FEATURE_FLAG_CODE_REFERENCES_RETENTION_DAYS) + last_feature_found_at = ( + FeatureFlagCodeReferencesScan.objects.filter( + project=feature.project, + created_at__gte=timezone.now() - history_delta, + repository_url=OuterRef("repository_url"), + code_references__contains=[{"feature_name": feature.name}], + ) + .values("created_at") + .order_by("-created_at")[:1] + ) + + last_scans_of_each_repository = ( + FeatureFlagCodeReferencesScan.objects.filter(project=feature.project) + .annotate(last_feature_found_at=Subquery(last_feature_found_at)) + .order_by("repository_url", "-created_at") + .distinct("repository_url") + ) + + return [ + FeatureFlagCodeReferencesRepositorySummary( + repository_url=scan.repository_url, + vcs_provider=VCSProvider(scan.vcs_provider), + revision=scan.revision, + last_successful_repository_scanned_at=scan.created_at, + last_feature_found_at=scan.last_feature_found_at, + code_references=[ + CodeReference( + feature_name=feature.name, + file_path=reference["file_path"], + line_number=reference["line_number"], + permalink=_get_permalink( + provider=VCSProvider(scan.vcs_provider), + repository_url=scan.repository_url, + revision=scan.revision, + file_path=reference["file_path"], + line_number=reference["line_number"], + ), + ) + for reference in scan.code_references + if reference["feature_name"] == feature.name + ], + ) + for scan in last_scans_of_each_repository + ] + + +def _get_permalink( + provider: VCSProvider, + repository_url: str, + revision: str, + file_path: str, + line_number: int, +) -> str: + """Generate a permalink for the code reference.""" + match provider: + case VCSProvider.GITHUB: + return urljoin( + repository_url, f"blob/{revision}/{file_path}#L{line_number}" + ) + raise NotImplementedError( # pragma: no cover + f"Permalink generation for {provider} is not implemented." + ) diff --git a/api/projects/code_references/types.py b/api/projects/code_references/types.py index 84924c69a20f..346dde597742 100644 --- a/api/projects/code_references/types.py +++ b/api/projects/code_references/types.py @@ -17,10 +17,6 @@ class JSONCodeReference(TypedDict): @dataclass class CodeReference: - scanned_at: datetime - vcs_provider: VCSProvider - repository_url: str - revision: str feature_name: str file_path: str line_number: int @@ -28,9 +24,12 @@ class CodeReference: @dataclass -class FeatureFlagCodeReferences: - first_scanned_at: datetime | None - last_scanned_at: datetime | None +class FeatureFlagCodeReferencesRepositorySummary: + repository_url: str + vcs_provider: VCSProvider + revision: str + last_successful_repository_scanned_at: datetime + last_feature_found_at: datetime | None code_references: list[CodeReference] @@ -38,3 +37,5 @@ class FeatureFlagCodeReferences: class CodeReferencesRepositoryCount: repository_url: str count: int + last_successful_repository_scanned_at: datetime + last_feature_found_at: datetime | None diff --git a/api/projects/code_references/views.py b/api/projects/code_references/views.py index b4150ce277f7..a9c1b7a478e2 100644 --- a/api/projects/code_references/views.py +++ b/api/projects/code_references/views.py @@ -1,10 +1,7 @@ -from datetime import datetime -from functools import cache -from urllib.parse import urljoin +from typing import Any -from django.db.models import QuerySet from django.shortcuts import get_object_or_404 -from rest_framework import generics +from rest_framework import generics, response from features.models import Feature from projects.code_references.models import FeatureFlagCodeReferencesScan @@ -13,13 +10,12 @@ ViewFeatureFlagCodeReferences, ) from projects.code_references.serializers import ( + FeatureFlagCodeReferencesRepositorySummarySerializer, FeatureFlagCodeReferencesScanSerializer, - FeatureFlagCodeReferencesSerializer, ) +from projects.code_references.services import get_code_references_for_feature_flag from projects.code_references.types import ( - CodeReference, - FeatureFlagCodeReferences, - VCSProvider, + FeatureFlagCodeReferencesRepositorySummary, ) @@ -40,93 +36,19 @@ def perform_create( # type: ignore[override] class FeatureFlagCodeReferenceDetailAPIView( - generics.RetrieveAPIView[FeatureFlagCodeReferences], # type: ignore[type-var] + generics.RetrieveAPIView[FeatureFlagCodeReferencesRepositorySummary], # type: ignore[type-var] ): """ API view to retrieve code references for a specific feature in a project """ - serializer_class = FeatureFlagCodeReferencesSerializer + serializer_class = FeatureFlagCodeReferencesRepositorySummarySerializer permission_classes = [ViewFeatureFlagCodeReferences] - def get_object(self) -> FeatureFlagCodeReferences: - return FeatureFlagCodeReferences( - first_scanned_at=self._get_first_scanned_at(), - last_scanned_at=self._get_last_scanned_at(), - code_references=self._get_code_references(), - ) - - @cache - def _get_feature(self) -> Feature: - return get_object_or_404( + def get(self, *args: Any, **kwargs: Any) -> response.Response: + feature = get_object_or_404( Feature, - project_id=self.kwargs["project_pk"], pk=self.kwargs["feature_pk"], - ) - - def _get_related_references(self) -> QuerySet[FeatureFlagCodeReferencesScan]: - feature = self._get_feature() - return FeatureFlagCodeReferencesScan.objects.filter( project_id=self.kwargs["project_pk"], - code_references__contains=[{"feature_name": feature.name}], - ) - - def _get_first_scanned_at(self) -> datetime | None: - related = self._get_related_references() - first_match = related.only("created_at").order_by("created_at").first() - return first_match.created_at if first_match else None - - def _get_last_scanned_at(self) -> datetime | None: - related = self._get_related_references() - last_match = related.only("created_at").order_by("-created_at").first() - return last_match.created_at if last_match else None - - def _get_code_references(self) -> list[CodeReference]: - feature = self._get_feature() - last_scans_of_each_repository = ( - FeatureFlagCodeReferencesScan.objects.filter( - project_id=self.kwargs["project_pk"], - ) - .order_by("repository_url", "-created_at") - .distinct("repository_url") - ) - - return [ - CodeReference( - scanned_at=scan.created_at, - vcs_provider=VCSProvider(scan.vcs_provider), - repository_url=scan.repository_url, - revision=scan.revision, - feature_name=feature.name, - file_path=reference["file_path"], - line_number=reference["line_number"], - permalink=self._get_permalink( - provider=VCSProvider(scan.vcs_provider), - repository_url=scan.repository_url, - revision=scan.revision, - file_path=reference["file_path"], - line_number=reference["line_number"], - ), - ) - for scan in last_scans_of_each_repository - for reference in scan.code_references - if reference["feature_name"] == feature.name - ] - - def _get_permalink( - self, - provider: VCSProvider, - repository_url: str, - revision: str, - file_path: str, - line_number: int, - ) -> str: - """Generate a permalink for the code reference.""" - match provider: - case VCSProvider.GITHUB: - return urljoin( - repository_url, f"blob/{revision}/{file_path}#L{line_number}" - ) - raise NotImplementedError( # pragma: no cover - f"Permalink generation for {provider} is not implemented." ) + return response.Response(get_code_references_for_feature_flag(feature)) diff --git a/api/tests/unit/features/test_unit_features_views.py b/api/tests/unit/features/test_unit_features_views.py index 2499748374dc..53a33b6fee5d 100644 --- a/api/tests/unit/features/test_unit_features_views.py +++ b/api/tests/unit/features/test_unit_features_views.py @@ -3353,63 +3353,81 @@ def test_FeatureViewSet_list__includes_code_references_counts( ) -> None: # Given with_project_permissions([VIEW_PROJECT]) # type: ignore[call-arg] - FeatureFlagCodeReferencesScan.objects.create( - project=project, - repository_url="https://github.flagsmith.com/backend/", - revision="backend-1", - code_references=[ - { - "feature_name": feature.name, - "file_path": "path/to/file.py", - "line_number": 42, - }, - ], - ) - FeatureFlagCodeReferencesScan.objects.create( - project=project, - repository_url="https://gitlab.flagsmith.com/frontend/", - revision="frontend-1", - code_references=[ - { - "feature_name": feature.name, - "file_path": "path/to/file.js", - "line_number": 23, - }, - ], - ) - FeatureFlagCodeReferencesScan.objects.create( - project=project, - repository_url="https://gitlab.flagsmith.com/frontend/", - revision="frontend-2", - code_references=[ - { - "feature_name": feature.name, - "file_path": "path/to/file.js", - "line_number": 24, - }, - { - "feature_name": feature.name, - "file_path": "path/to/another/file.js", - "line_number": 10, - }, - { - "feature_name": f"Another {feature.name}", - "file_path": "path/to/another/file.js", - "line_number": 11, - }, - ], - ) + with freeze_time("2099-01-01T10:00:00-0300"): + FeatureFlagCodeReferencesScan.objects.create( + project=project, + repository_url="https://github.flagsmith.com/backend/", + revision="backend-1", + code_references=[ + { + "feature_name": feature.name, + "file_path": "path/to/file.py", + "line_number": 42, + }, + ], + ) + FeatureFlagCodeReferencesScan.objects.create( + project=project, + repository_url="https://gitlab.flagsmith.com/frontend/", + revision="frontend-1", + code_references=[ + { + "feature_name": feature.name, + "file_path": "path/to/file.js", + "line_number": 23, + }, + ], + ) + with freeze_time("2099-01-02T11:00:00-0300"): + FeatureFlagCodeReferencesScan.objects.create( + project=project, + repository_url="https://github.flagsmith.com/backend/", + revision="backend-2", + code_references=[ + { + "feature_name": f"Another {feature.name}", + "file_path": "path/to/another/file.py", + "line_number": 11, + }, + ], + ) + FeatureFlagCodeReferencesScan.objects.create( + project=project, + repository_url="https://gitlab.flagsmith.com/frontend/", + revision="frontend-2", + code_references=[ + { + "feature_name": feature.name, + "file_path": "path/to/file.js", + "line_number": 23, + }, + { + "feature_name": feature.name, + "file_path": "path/to/another/file.js", + "line_number": 50, + }, + ], + ) # When response = staff_client.get(f"/api/v1/projects/{project.pk}/features/") # Then assert response.status_code == status.HTTP_200_OK - assert (counts := response.data["results"][0]["code_references_counts"]) - assert counts[0]["repository_url"] == "https://github.flagsmith.com/backend/" - assert counts[0]["count"] == 1 - assert counts[1]["repository_url"] == "https://gitlab.flagsmith.com/frontend/" - assert counts[1]["count"] == 2 + assert response.json()["results"][0]["code_references_counts"] == [ + { + "repository_url": "https://github.flagsmith.com/backend/", + "count": 0, + "last_successful_repository_scanned_at": "2099-01-02T14:00:00+00:00", + "last_feature_found_at": "2099-01-01T13:00:00+00:00", + }, + { + "repository_url": "https://gitlab.flagsmith.com/frontend/", + "count": 2, + "last_successful_repository_scanned_at": "2099-01-02T14:00:00+00:00", + "last_feature_found_at": "2099-01-02T14:00:00+00:00", + }, + ] def test_simple_feature_state_returns_only_latest_versions( diff --git a/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py b/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py index a30664dda56c..591201478a27 100644 --- a/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py +++ b/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py @@ -172,7 +172,7 @@ def test_FeatureCodeReferencesDetailAPIView__responds_200_with_code_references_f ) -> None: # Given with_project_permissions([VIEW_PROJECT]) # type: ignore[call-arg] - with freezegun.freeze_time("2099-01-01T10:00:00Z"): + with freezegun.freeze_time("2099-01-01T10:00:00-0300"): FeatureFlagCodeReferencesScan.objects.create( project=project, repository_url="https://github.flagsmith.com/backend/", @@ -197,7 +197,7 @@ def test_FeatureCodeReferencesDetailAPIView__responds_200_with_code_references_f }, ], ) - with freezegun.freeze_time("2099-01-02T11:00:00Z"): + with freezegun.freeze_time("2099-01-02T11:00:00-0300"): FeatureFlagCodeReferencesScan.objects.create( project=project, repository_url="https://github.flagsmith.com/frontend/", @@ -223,37 +223,50 @@ def test_FeatureCodeReferencesDetailAPIView__responds_200_with_code_references_f # Then assert response.status_code == 200 - assert response.data["first_scanned_at"] == "2099-01-01T10:00:00Z" - assert response.data["last_scanned_at"] == "2099-01-02T11:00:00Z" - assert len(references := response.data["code_references"]) == 3 - assert references[0]["scanned_at"] == "2099-01-01T10:00:00Z" - assert references[0]["vcs_provider"] == "github" - assert references[0]["repository_url"] == "https://github.flagsmith.com/backend/" - assert references[0]["revision"] == "backend-1" - assert references[1]["scanned_at"] == "2099-01-02T11:00:00Z" - assert references[0]["file_path"] == "backend/file1.py" - assert references[0]["line_number"] == 20 - assert references[0]["permalink"] == ( - "https://github.flagsmith.com/backend/blob/backend-1/backend/file1.py#L20" - ) - assert references[1]["scanned_at"] == "2099-01-02T11:00:00Z" - assert references[1]["vcs_provider"] == "github" - assert references[1]["repository_url"] == "https://github.flagsmith.com/frontend/" - assert references[1]["revision"] == "frontend-2" - assert references[1]["file_path"] == "frontend/file1.js" - assert references[1]["line_number"] == 12 - assert references[1]["permalink"] == ( - "https://github.flagsmith.com/frontend/blob/frontend-2/frontend/file1.js#L12" - ) - assert references[2]["scanned_at"] == "2099-01-02T11:00:00Z" - assert references[2]["vcs_provider"] == "github" - assert references[2]["repository_url"] == "https://github.flagsmith.com/frontend/" - assert references[2]["revision"] == "frontend-2" - assert references[2]["file_path"] == "frontend/file2.js" - assert references[2]["line_number"] == 5 - assert references[2]["permalink"] == ( - "https://github.flagsmith.com/frontend/blob/frontend-2/frontend/file2.js#L5" - ) + assert response.json() == [ + { + "repository_url": "https://github.flagsmith.com/backend/", + "vcs_provider": "github", + "revision": "backend-1", + "last_successful_repository_scanned_at": "2099-01-01T13:00:00+00:00", + "last_feature_found_at": "2099-01-01T13:00:00+00:00", + "code_references": [ + { + "feature_name": feature.name, + "file_path": "backend/file1.py", + "line_number": 20, + "permalink": ( + "https://github.flagsmith.com/backend/blob/backend-1/backend/file1.py#L20" + ), + }, + ], + }, + { + "repository_url": "https://github.flagsmith.com/frontend/", + "vcs_provider": "github", + "revision": "frontend-2", + "last_successful_repository_scanned_at": "2099-01-02T14:00:00+00:00", + "last_feature_found_at": "2099-01-02T14:00:00+00:00", + "code_references": [ + { + "feature_name": feature.name, + "file_path": "frontend/file1.js", + "line_number": 12, + "permalink": ( + "https://github.flagsmith.com/frontend/blob/frontend-2/frontend/file1.js#L12" + ), + }, + { + "feature_name": feature.name, + "file_path": "frontend/file2.js", + "line_number": 5, + "permalink": ( + "https://github.flagsmith.com/frontend/blob/frontend-2/frontend/file2.js#L5" + ), + }, + ], + }, + ] def test_FeatureCodeReferencesDetailAPIView__responds_200_with_feature_flag_removed( @@ -264,7 +277,7 @@ def test_FeatureCodeReferencesDetailAPIView__responds_200_with_feature_flag_remo ) -> None: # Given with_project_permissions([VIEW_PROJECT]) # type: ignore[call-arg] - with freezegun.freeze_time("2099-01-01T10:00:00Z"): + with freezegun.freeze_time("2099-01-01T10:00:00-0300"): FeatureFlagCodeReferencesScan.objects.create( project=project, repository_url="https://github.flagsmith.com/", @@ -277,7 +290,7 @@ def test_FeatureCodeReferencesDetailAPIView__responds_200_with_feature_flag_remo }, ], ) - with freezegun.freeze_time("2099-01-02T11:00:00Z"): + with freezegun.freeze_time("2099-01-02T11:00:00-0300"): FeatureFlagCodeReferencesScan.objects.create( project=project, repository_url="https://github.flagsmith.com/", @@ -292,9 +305,16 @@ def test_FeatureCodeReferencesDetailAPIView__responds_200_with_feature_flag_remo # Then assert response.status_code == 200 - assert response.data["first_scanned_at"] == "2099-01-01T10:00:00Z" - assert response.data["last_scanned_at"] == "2099-01-01T10:00:00Z" - assert len(response.data["code_references"]) == 0 + assert response.json() == [ + { + "repository_url": "https://github.flagsmith.com/", + "vcs_provider": "github", + "revision": "revision-hash-2", + "last_successful_repository_scanned_at": "2099-01-02T14:00:00+00:00", + "last_feature_found_at": "2099-01-01T13:00:00+00:00", + "code_references": [], + }, + ] def test_FeatureCodeReferencesDetailAPIView__responds_200_even_without_code_references( @@ -313,9 +333,7 @@ def test_FeatureCodeReferencesDetailAPIView__responds_200_even_without_code_refe # Then assert response.status_code == 200 - assert response.data["first_scanned_at"] is None - assert response.data["last_scanned_at"] is None - assert response.data["code_references"] == [] + assert response.json() == [] def test_FeatureCodeReferencesDetailAPIView__responds_401_when_not_authenticated( From 38466ffdd8a3224fde68d357bb9e0b613a840af4 Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Fri, 15 Aug 2025 17:28:44 -0300 Subject: [PATCH 10/10] Improve max file path setting --- api/projects/code_references/constants.py | 3 +++ api/projects/code_references/serializers.py | 3 ++- .../test_unit_projects_code_references_views.py | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/api/projects/code_references/constants.py b/api/projects/code_references/constants.py index 605b199aee7e..8d7377ca8d68 100644 --- a/api/projects/code_references/constants.py +++ b/api/projects/code_references/constants.py @@ -1,2 +1,5 @@ # TODO: Implement history cleanup? FEATURE_FLAG_CODE_REFERENCES_RETENTION_DAYS = 30 + +# Linux maximum file path length, as per limits.h/PATH_MAX +MAX_FILE_PATH_LENGTH = 4096 diff --git a/api/projects/code_references/serializers.py b/api/projects/code_references/serializers.py index f92b7a159767..78dfd413f7e8 100644 --- a/api/projects/code_references/serializers.py +++ b/api/projects/code_references/serializers.py @@ -1,5 +1,6 @@ from rest_framework import serializers +from projects.code_references.constants import MAX_FILE_PATH_LENGTH from projects.code_references.models import FeatureFlagCodeReferencesScan from projects.code_references.types import ( CodeReference, @@ -10,7 +11,7 @@ class _BaseCodeReferenceSerializer(serializers.Serializer[CodeReference]): - file_path = serializers.CharField(max_length=260) # Windows' MAX_PATH + file_path = serializers.CharField(max_length=MAX_FILE_PATH_LENGTH) line_number = serializers.IntegerField(min_value=1) diff --git a/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py b/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py index 591201478a27..d7fa09b78dff 100644 --- a/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py +++ b/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py @@ -146,7 +146,7 @@ def test_CodeReferenceCreateAPIView__responds_400_when_file_path_too_long( "code_references": [ { "feature_name": "feature-1", - "file_path": "windows/limit/" * 100 + "file.py", + "file_path": "would/you/even/" * 1000 + "file.py", "line_number": 10, }, ], @@ -158,7 +158,7 @@ def test_CodeReferenceCreateAPIView__responds_400_when_file_path_too_long( assert response.status_code == 400 assert response.data == { "code_references": [ - {"file_path": ["Ensure this field has no more than 260 characters."]} + {"file_path": ["Ensure this field has no more than 4096 characters."]} ], } assert not FeatureFlagCodeReferencesScan.objects.exists()