Refactor with AdvisoryHistoryDiff model and pipeline

Samk1710 · Samk1710 · commit 48e311bcb643 · 2026-07-30T00:22:44.000+05:30
Signed-off-by: Sampurna Pyne &lt;sampurnapyne1710@gmail.com&gt;
diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py
@@ -21,6 +21,7 @@
 from vulnerabilities.pipelines.v2_improvers import (
     group_advisories_for_packages as group_advisories_for_packages_v2,
 )
+from vulnerabilities.pipelines.v2_improvers import history_diff_backfill
 from vulnerabilities.pipelines.v2_improvers import mark_unfurl_version_range
 from vulnerabilities.pipelines.v2_improvers import reference_collect_commits
 from vulnerabilities.pipelines.v2_improvers import relate_severities
@@ -42,6 +43,7 @@
         reference_collect_commits.CollectReferencesFixCommitsPipeline,
         enhance_with_github_poc.GithubPocsImproverPipeline,
         mark_unfurl_version_range.MarkUnfurlVersionRangePipeline,
+        history_diff_backfill.HistoryDiffImproverPipeline,
         group_advisories_for_packages_v2.GroupAdvisoriesForPackages,
     ]
 )
diff --git a/vulnerabilities/migrations/0138_advisoryhistorydiff.py b/vulnerabilities/migrations/0138_advisoryhistorydiff.py
@@ -0,0 +1,38 @@
+# Generated by Django 5.2.11 on 2026-07-25 16:54
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('vulnerabilities', '0137_alter_pipelineschedule_run_interval'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='AdvisoryHistoryDiff',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('summary_added', models.TextField(blank=True, null=True)),
+                ('summary_removed', models.TextField(blank=True, null=True)),
+                ('url_added', models.URLField(blank=True, null=True)),
+                ('url_removed', models.URLField(blank=True, null=True)),
+                ('added_aliases', models.ManyToManyField(blank=True, related_name='added_in_history', to='vulnerabilities.advisoryalias')),
+                ('added_impacted_packages', models.ManyToManyField(blank=True, related_name='added_in_history', to='vulnerabilities.impactedpackage')),
+                ('added_patches', models.ManyToManyField(blank=True, related_name='added_in_history', to='vulnerabilities.patch')),
+                ('added_references', models.ManyToManyField(blank=True, related_name='added_in_history', to='vulnerabilities.advisoryreference')),
+                ('added_severities', models.ManyToManyField(blank=True, related_name='added_in_history', to='vulnerabilities.advisoryseverity')),
+                ('added_weaknesses', models.ManyToManyField(blank=True, related_name='added_in_history', to='vulnerabilities.advisoryweakness')),
+                ('advisory_after', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='history_diff', to='vulnerabilities.advisoryv2')),
+                ('advisory_before', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='history_diff_before', to='vulnerabilities.advisoryv2')),
+                ('removed_aliases', models.ManyToManyField(blank=True, related_name='removed_in_history', to='vulnerabilities.advisoryalias')),
+                ('removed_impacted_packages', models.ManyToManyField(blank=True, related_name='removed_in_history', to='vulnerabilities.impactedpackage')),
+                ('removed_patches', models.ManyToManyField(blank=True, related_name='removed_in_history', to='vulnerabilities.patch')),
+                ('removed_references', models.ManyToManyField(blank=True, related_name='removed_in_history', to='vulnerabilities.advisoryreference')),
+                ('removed_severities', models.ManyToManyField(blank=True, related_name='removed_in_history', to='vulnerabilities.advisoryseverity')),
+                ('removed_weaknesses', models.ManyToManyField(blank=True, related_name='removed_in_history', to='vulnerabilities.advisoryweakness')),
+            ],
+        ),
+    ]
diff --git a/vulnerabilities/migrations/0142_merge_20260726_0000.py b/vulnerabilities/migrations/0142_merge_20260726_0000.py
@@ -0,0 +1,14 @@
+# Generated by Django 5.2.11 on 2026-07-26 00:00
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('vulnerabilities', '0138_advisoryhistorydiff'),
+        ('vulnerabilities', '0141_advisorymitigations'),
+    ]
+
+    operations = [
+    ]
diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py
@@ -3367,6 +3367,72 @@ def get_aliases(self):
     alias = get_aliases
 
 
+class AdvisoryHistoryDiff(models.Model):
+    """
+    Stores the pre-computed diff between an AdvisoryV2 snapshot (advisory_after)
+    and its immediate predecessor (advisory_before).
+    Created for every snapshot.
+    """
+
+    # Points to the older snapshot in the comparison
+    advisory_before = models.OneToOneField(
+        AdvisoryV2,
+        on_delete=models.PROTECT,
+        related_name="history_diff_before",
+        null=True,  # The very first snapshot won't have a predecessor
+        blank=True,
+    )
+
+    # Points to the newer snapshot in the comparison
+    advisory_after = models.OneToOneField(
+        AdvisoryV2, on_delete=models.PROTECT, related_name="history_diff", null=True, blank=True
+    )
+
+    summary_added = models.TextField(blank=True, null=True)
+    summary_removed = models.TextField(blank=True, null=True)
+
+    url_added = models.URLField(blank=True, null=True)
+    url_removed = models.URLField(blank=True, null=True)
+
+    added_severities = models.ManyToManyField(
+        "AdvisorySeverity", related_name="added_in_history", blank=True
+    )
+    removed_severities = models.ManyToManyField(
+        "AdvisorySeverity", related_name="removed_in_history", blank=True
+    )
+
+    added_impacted_packages = models.ManyToManyField(
+        "ImpactedPackage", related_name="added_in_history", blank=True
+    )
+    removed_impacted_packages = models.ManyToManyField(
+        "ImpactedPackage", related_name="removed_in_history", blank=True
+    )
+
+    added_references = models.ManyToManyField(
+        "AdvisoryReference", related_name="added_in_history", blank=True
+    )
+    removed_references = models.ManyToManyField(
+        "AdvisoryReference", related_name="removed_in_history", blank=True
+    )
+
+    added_aliases = models.ManyToManyField(
+        "AdvisoryAlias", related_name="added_in_history", blank=True
+    )
+    removed_aliases = models.ManyToManyField(
+        "AdvisoryAlias", related_name="removed_in_history", blank=True
+    )
+
+    added_weaknesses = models.ManyToManyField(
+        "AdvisoryWeakness", related_name="added_in_history", blank=True
+    )
+    removed_weaknesses = models.ManyToManyField(
+        "AdvisoryWeakness", related_name="removed_in_history", blank=True
+    )
+
+    added_patches = models.ManyToManyField("Patch", related_name="added_in_history", blank=True)
+    removed_patches = models.ManyToManyField("Patch", related_name="removed_in_history", blank=True)
+
+
 class ImpactedPackage(models.Model):
     """
     Represents a single impact for an advisory, including affected range and fixed version and
diff --git a/vulnerabilities/pipelines/__init__.py b/vulnerabilities/pipelines/__init__.py
@@ -284,6 +284,10 @@ class VulnerableCodeBaseImporterPipelineV2(VulnerableCodePipeline):
     # pull requests, commit messages, EPSS, exploits, etc.
     exclude_from_package_todo = False
 
+    # Set this to True if computing history diff is not fruitful for this source.
+    # An example of such advisory would be pipeline dedicated to EPSS
+    exclude_from_history_diff = False
+
     # Control how often progress log is shown (range: 1–100, higher value = less frequent log)
     progress_step = 10
 
@@ -344,6 +348,7 @@ def collect_and_store_advisories(self):
                     datasource_id=self.datasource_id,
                     logger=self.log,
                     precedence=self.precedence,
+                    calculate_history_diff=not self.exclude_from_history_diff,
                 ):
                     collected_advisory_count += 1
             except Exception as e:
diff --git a/vulnerabilities/pipelines/v2_improvers/history_diff_backfill.py b/vulnerabilities/pipelines/v2_improvers/history_diff_backfill.py
@@ -0,0 +1,81 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from aboutcode.pipeline import LoopProgress
+
+from vulnerabilities.importers import IMPORTERS_REGISTRY
+from vulnerabilities.models import AdvisoryHistoryDiff
+from vulnerabilities.models import AdvisoryV2
+from vulnerabilities.pipelines import VulnerableCodePipeline
+from vulnerabilities.utils import compute_advisory_v2_diff
+
+
+class HistoryDiffImproverPipeline(VulnerableCodePipeline):
+    """
+    Pipeline to compute and store relational history snapshot diffs.
+    """
+
+    pipeline_id = "history_diff_improver_v2"
+
+    @classmethod
+    def steps(cls):
+        return (cls.calculate_history_diffs,)
+
+    def calculate_history_diffs(self):
+        # Skip excluded importers
+        excluded_ids = [
+            cls.pipeline_id
+            for cls in IMPORTERS_REGISTRY.values()
+            if getattr(cls, "exclude_from_history_diff", False)
+            and getattr(cls, "pipeline_id", None)
+        ]
+        qs = (
+            AdvisoryV2.objects.exclude(pipeline_id__in=excluded_ids)
+            if excluded_ids
+            else AdvisoryV2.objects.all()
+        )
+
+        avids_qs = qs.filter(history_diff__isnull=True).values_list("avid", flat=True).distinct()
+        avids_count = avids_qs.count()
+        self.log(f"Computing history diffs for {avids_count} advisories")
+
+        for avid in LoopProgress(
+            total_iterations=avids_count, logger=self.log, progress_step=10
+        ).iter(avids_qs.iterator()):
+            snapshots = list(
+                qs.filter(avid=avid)
+                .order_by("date_collected", "unique_content_id")
+                .prefetch_related(
+                    "aliases",
+                    "references",
+                    "weaknesses",
+                    "severities",
+                    "patches",
+                    "impacted_packages",
+                )
+            )
+
+            previous_snapshot = None
+
+            # TODO:
+            # Bulk diff creation
+
+            for snapshot in snapshots:
+                if not hasattr(snapshot, "history_diff"):  # Skip already processed advisories
+                    if previous_snapshot is not None:
+                        compute_advisory_v2_diff(previous_snapshot, snapshot)
+                    else:
+                        # For the very first snapshot, we create an empty diff to mark it as processed
+                        AdvisoryHistoryDiff.objects.create(
+                            advisory_after=snapshot, advisory_before=None
+                        )
+
+                previous_snapshot = snapshot
+
+        self.log("Successfully finished history diff backfill.")
diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py
@@ -33,6 +33,7 @@
 from vulnerabilities.improver import MAX_CONFIDENCE
 from vulnerabilities.models import Advisory
 from vulnerabilities.models import AdvisoryAlias
+from vulnerabilities.models import AdvisoryHistoryDiff
 from vulnerabilities.models import AdvisoryReference
 from vulnerabilities.models import AdvisorySeverity
 from vulnerabilities.models import AdvisoryV2
@@ -49,6 +50,7 @@
 from vulnerabilities.models import Weakness
 from vulnerabilities.pipes.risk_score import compute_advisory_risk_score
 from vulnerabilities.pipes.univers_utils import get_exact_purls_v2
+from vulnerabilities.utils import compute_advisory_v2_diff
 
 
 def get_or_create_aliases(aliases: List) -> QuerySet:
@@ -295,6 +297,7 @@ def insert_advisory_v2(
     logger: Callable,
     datasource_id: str,
     precedence: int = 0,
+    calculate_history_diff: bool = True,
 ):
     from vulnerabilities.models import ImpactedPackage
     from vulnerabilities.models import PackageV2
@@ -338,12 +341,16 @@ def insert_advisory_v2(
     if not created:
         return advisory_obj
 
-    AdvisoryV2.objects.filter(
+    latest_qs = AdvisoryV2.objects.filter(
         avid=f"{datasource_id}/{advisory.advisory_id}",
         is_latest=True,
-    ).update(is_latest=False)
+    )
+
+    previous_latest_advisory = latest_qs.first()
+    latest_qs.update(is_latest=False)
+
     advisory_obj.is_latest = True
-    advisory_obj.save()
+    advisory_obj.save(update_fields=["is_latest"])
 
     aliases = get_or_create_advisory_aliases(aliases=advisory.aliases)
     references = get_or_create_advisory_references(references=advisory.references)
@@ -416,6 +423,31 @@ def insert_advisory_v2(
         )
         impact.introduced_by_package_commit_patches.add(*introduced_commit_v2)
         impact.fixed_by_package_commit_patches.add(*fixed_commit_v2)
+
+    if calculate_history_diff:
+        if previous_latest_advisory:
+            previous = AdvisoryV2.objects.prefetch_related(
+                "aliases",
+                "references",
+                "weaknesses",
+                "severities",
+                "patches",
+                "impacted_packages",
+            ).get(id=previous_latest_advisory.id)
+
+            current = AdvisoryV2.objects.prefetch_related(
+                "aliases",
+                "references",
+                "weaknesses",
+                "severities",
+                "patches",
+                "impacted_packages",
+            ).get(id=advisory_obj.id)
+
+            compute_advisory_v2_diff(previous, current)
+        else:
+            AdvisoryHistoryDiff.objects.create(advisory_after=advisory_obj, advisory_before=None)
+
     return advisory_obj
 
 
diff --git a/vulnerabilities/templates/advisory_detail.html b/vulnerabilities/templates/advisory_detail.html
@@ -776,6 +776,9 @@
                                 <span class="tag is-info is-light">Oldest version collected</span>
                                 {% elif entry.diff %}
                                 {% with formatted_diff=entry.diff|format_diff_for_ui %}
+                                {% if formatted_diff %}
+                                <a class="diff-toggle-btn is-block has-text-right has-text-link is-size-7 has-text-weight-semibold mb-1 is-clickable"
+                                    onclick="toggleDiff(this)">Show more ▾</a>
                                 <div class="history-diff-collapsible">
                                     {% for field, change in formatted_diff.items %}
                                     <div class="mb-2">
@@ -794,15 +797,13 @@
                                         <div class="mt-1">
                                             {% for item in change.added %}
                                             <div class="mt-1">
-                                                <span class="diff-badge diff-badge-green diff-wrap">&plus; Added {% if item.attributes %}{{ item.header }} with
-{% for key, value in item.attributes %}    {{ key }} <b>{{ value }}</b>{% if not forloop.last %}
+                                                <span class="diff-badge diff-badge-green diff-wrap">{% if item.attributes %}{% for key, value in item.attributes %}{{ key }} <b>{{ value }}</b>{% if not forloop.last %}
 {% endif %}{% endfor %}{% else %}{{ item }}{% endif %}</span>
                                             </div>
                                             {% endfor %}
                                             {% for item in change.removed %}
                                             <div class="mt-1">
-                                                <span class="diff-badge diff-badge-red diff-wrap">&minus; Removed {% if item.attributes %}{{ item.header }} with
-{% for key, value in item.attributes %}    {{ key }} <b>{{ value }}</b>{% if not forloop.last %}
+                                                <span class="diff-badge diff-badge-red diff-wrap">{% if item.attributes %}{% for key, value in item.attributes %}{{ key }} <b>{{ value }}</b>{% if not forloop.last %}
 {% endif %}{% endfor %}{% else %}{{ item }}{% endif %}</span>
                                             </div>
                                             {% endfor %}
@@ -811,8 +812,9 @@
                                     </div>
                                     {% endfor %}
                                 </div>
-                                <a class="diff-toggle-btn is-block has-text-right has-text-link is-size-7 has-text-weight-semibold mt-1 is-clickable"
-                                    onclick="toggleDiff(this)">Show more ▾</a>
+                                {% else %}
+                                <span class="has-text-grey is-italic is-size-7">No changes detected in this snapshot.</span>
+                                {% endif %}
                                 {% endwith %}
                                 {% endif %}
                             </td>
@@ -873,7 +875,7 @@
 
     function checkDiffHeights() {
         document.querySelectorAll('.history-diff-collapsible').forEach(function (diffContainer) {
-            const toggleButton = diffContainer.nextElementSibling;
+            const toggleButton = diffContainer.previousElementSibling;
             if (!toggleButton || !toggleButton.classList.contains('diff-toggle-btn')) return;
 
             // Measure true height 
@@ -893,7 +895,7 @@
     }
 
     function toggleDiff(toggleButton) {
-        const diffContainer = toggleButton.previousElementSibling;
+        const diffContainer = toggleButton.nextElementSibling;
         diffContainer.classList.toggle('is-expanded');
         toggleButton.textContent = diffContainer.classList.contains('is-expanded')
             ? 'Show less \u25b4'
diff --git a/vulnerabilities/templatetags/diff_advisory_history.py b/vulnerabilities/templatetags/diff_advisory_history.py
diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py
diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py
diff --git a/vulnerablecode/urls.py b/vulnerablecode/urls.py

Original file line number	Diff line number	Diff line change
`@@ -21,6 +21,7 @@`
`21`	`21`	`from vulnerabilities.pipelines.v2_improvers import (`
`22`	`22`	`group_advisories_for_packages as group_advisories_for_packages_v2,`
`23`	`23`	`)`
	`24`	`+from vulnerabilities.pipelines.v2_improvers import history_diff_backfill`
`24`	`25`	`from vulnerabilities.pipelines.v2_improvers import mark_unfurl_version_range`
`25`	`26`	`from vulnerabilities.pipelines.v2_improvers import reference_collect_commits`
`26`	`27`	`from vulnerabilities.pipelines.v2_improvers import relate_severities`
`@@ -42,6 +43,7 @@`
`42`	`43`	`reference_collect_commits.CollectReferencesFixCommitsPipeline,`
`43`	`44`	`enhance_with_github_poc.GithubPocsImproverPipeline,`
`44`	`45`	`mark_unfurl_version_range.MarkUnfurlVersionRangePipeline,`
	`46`	`+ history_diff_backfill.HistoryDiffImproverPipeline,`
`45`	`47`	`group_advisories_for_packages_v2.GroupAdvisoriesForPackages,`
`46`	`48`	`]`
`47`	`49`	`)`