Refactor to get_epss_history and update tests

Signed-off-by: Sampurna Pyne <sampurnapyne1710@gmail.com>

Author: Samk1710

vulnerabilities/templates/advisory_detail.html

@@ -25,7 +25,7 @@
 
         <div class="tabs is-boxed" id="tabs">
             <ul>
-                <li class="{% if not is_epss_tab_active %}is-active{% endif %}" data-tab="essentials">
+                <li class="{% if not 'epss_page' in request.GET %}is-active{% endif %}" data-tab="essentials">
                     <a>
                         <span>Essentials</span>
                     </a>
@@ -55,7 +55,7 @@
                     </li>
                 {% endif %}
 
-                <li class="{% if is_epss_tab_active %}is-active{% endif %}" data-tab="epss">
+                <li class="{% if 'epss_page' in request.GET %}is-active{% endif %}" data-tab="epss">
                     <a>
                         <span>
                             EPSS
@@ -80,7 +80,7 @@
                         </a>
                     </li>
                 {% endif %}
-            
+
                 <li data-tab="patch-url">
                     <a>
                         <span>
@@ -90,17 +90,18 @@
                         </span>
                     </a>
                 </li>
-                <!-- <li data-tab="history">
+
+                {% comment %} <li data-tab="history">
                     <a>
                         <span>
                             History ({{ history|length }})
                         </span>
                     </a>
-                </li> -->
+                </li> {% endcomment %}
             </ul>
         </div>
         <div id="tab-content">
-            <div class="tab-div {% if not is_epss_tab_active %}is-active{% endif %}" data-content="essentials">
+            <div class="tab-div {% if not 'epss_page' in request.GET %}is-active{% endif %}" data-content="essentials">
                 <div class="tab-nested-div">
                     <table class="table vcio-table width-100-pct mt-2">
                         <tbody>
@@ -458,7 +459,7 @@
                 {% endfor %}
             </div>
 
-            <div class="tab-div content {% if is_epss_tab_active %}is-active{% endif %}" data-content="epss">
+            <div class="tab-div content {% if 'epss_page' in request.GET %}is-active{% endif %}" data-content="epss">
                 {% if epss_data %}
                     <div class="has-text-weight-bold tab-nested-div ml-1 mb-1 mt-1">
                         Exploit Prediction Scoring System (EPSS)
@@ -533,14 +534,14 @@
                         </button>
                     </div>
 
-                    <div id="epss-chart-wrap" style="display:none;">
+                    <div id="epss-chart-container" style="display:none;">
                         <div class="has-text-weight-bold tab-nested-div ml-1 mb-3">
                             30-Day Trend
                         </div>
                         <div id="epss-chart" style="width:100%; height:260px;"></div>
                     </div>
 
-                    <div id="epss-history-table-wrap" class="mt-5" {% if not is_epss_tab_active %}style="display:none;"{% endif %}>
+                    <div id="epss-history-table-container" class="mt-5" {% if not 'epss_page' in request.GET %}style="display:none;"{% endif %}>
                         <div class="has-text-weight-bold tab-nested-div ml-1 mb-3">
                             EPSS History Table
                         </div>
@@ -563,16 +564,16 @@
                             </tbody>
                         </table>
 
-                        {% if epss_page_obj and epss_page_obj.has_other_pages %}
+                        {% if epss_pagination_obj and epss_pagination_obj.has_other_pages %}
                         <nav class="pagination is-small is-centered mt-4" role="navigation" aria-label="pagination">
-                            {% if epss_page_obj.has_previous %}
-                                <a class="pagination-previous" href="?epss_page={{ epss_page_obj.previous_page_number }}">Newer</a>
+                            {% if epss_pagination_obj.has_previous %}
+                                <a class="pagination-previous" href="?epss_page={{ epss_pagination_obj.previous_page_number }}">Newer</a>
                             {% else %}
                                 <a class="pagination-previous" disabled>Newer</a>
                             {% endif %}
 
-                            {% if epss_page_obj.has_next %}
-                                <a class="pagination-next" href="?epss_page={{ epss_page_obj.next_page_number }}">Older</a>
+                            {% if epss_pagination_obj.has_next %}
+                                <a class="pagination-next" href="?epss_page={{ epss_pagination_obj.next_page_number }}">Older</a>
                             {% else %}
                                 <a class="pagination-next" disabled>Older</a>
                             {% endif %}
@@ -746,7 +747,7 @@
                 {% endif %}
               </div>
 
-        <!-- <div class="tab-div content" data-content="history">
+        {% comment %} <div class="tab-div content" data-content="history">
             <table class="table is-bordered is-striped is-narrow is-hoverable is-fullwidth">
                 <thead>
                     <tr>
@@ -790,16 +791,19 @@
                 </tr>
                 {% endfor %}
             </table>
-        </div> -->
+        </div> {% endcomment %}
         </div>
     </div>
 </section>
 {% endif %}
 
-<script src="{% static 'js/main.js' %}" crossorigin="anonymous"></script>
+<script src="{% static 'js/main.js' %}"></script>
 
-<link rel="stylesheet" href="{% static 'css/billboard-3.18.0.min.css' %}">
-<script src="{% static 'js/billboard-3.18.0.pkgd.min.js' %}"></script>
+{% if 'epss_page' in request.GET or severities %}
+    <link rel="stylesheet" href="{% static 'css/billboard.css' %}">
+    <script src="{% static 'js/d3.js' %}"></script>
+    <script src="{% static 'js/billboard.js' %}"></script>
+{% endif %}
 
 {{ epss_history_data|json_script:"epss-history-data" }}
 

vulnerabilities/tests/test_epss_history.py

@@ -1,13 +1,24 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
 import json
+import time
+import pytest
 from pathlib import Path
 
 from django.test import TestCase
 from django.urls import reverse
 
-from vulnerabilities.models import AdvisoryAlias
+from vulnerabilities.models import EPSS
 from vulnerabilities.models import AdvisoryReference
-from vulnerabilities.models import AdvisorySeverity
 from vulnerabilities.models import AdvisoryV2
+from vulnerabilities.pipelines.v2_importers.epss_importer_v2 import EPSSImporterPipeline
 from vulnerabilities.tests.util_tests import check_results_against_json
 
 
@@ -27,25 +38,30 @@ def setUp(self):
                 advisory_id="CVE-2022-25204",
                 url="https://epss.cyentia.com/epss_scores-current.csv.gz",
                 is_latest=is_latest,
-                datasource_id="test_epss",
-                pipeline_id="test_epss",
+                datasource_id=EPSS.identifier,
+                pipeline_id=EPSSImporterPipeline.pipeline_id,
                 unique_content_id=f"hash_{i}",
             )
 
             advisory.references.add(ref_obj)
 
-            severity = data["severities"][0]
-            advisory.severities.create(
-                scoring_system=severity["system"],
-                value=severity["value"],
-                scoring_elements=severity["scoring_elements"],
-                published_at=severity["published_at"],
-            )
+            for severity in data["severities"]:
+                advisory.severities.create(
+                    scoring_system=severity["system"],
+                    value=severity["value"],
+                    scoring_elements=severity["scoring_elements"],
+                    published_at=severity["published_at"],
+                )
 
             if i == 0:
                 self.advisory = advisory
 
+    @pytest.mark.ignore_template_errors()
     def test_epss_history_context(self):
+        session = self.client.session
+        session["altcha_verified_at"] = time.time()
+        session.save()
+
         response = self.client.get(reverse("advisory_details", kwargs={"avid": self.advisory.avid}))
         self.assertEqual(response.status_code, 200)
 
@@ -56,3 +72,18 @@ def test_epss_history_context(self):
         ]
 
         check_results_against_json(history_data_for_json, expected_file)
+
+    def test_get_epss_history(self):
+        from vulnerabilities.views import get_epss_history
+
+        epss_history_data, epss_pagination_obj = get_epss_history(self.advisory, 1)
+
+        expected_file = Path(__file__).parent / "test_data" / "epss" / "epss_history_expected.json"
+        history_data_for_json = [
+            {**record, "published_at": record["published_at"].isoformat()}
+            for record in epss_history_data
+        ]
+
+        check_results_against_json(history_data_for_json, expected_file)
+        self.assertEqual(epss_pagination_obj.number, 1)
+        self.assertTrue(epss_pagination_obj.has_next())

vulnerabilities/views.py

@@ -7,6 +7,7 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import datetime
 import json
 import logging
 import time
@@ -31,16 +32,13 @@
 from django.db.models import Q
 from django.db.models.functions import Cast
 from django.db.models.functions import TruncDate
-from django.http import HttpRequest
 from django.http import HttpResponse
-from django.http import JsonResponse
 from django.http.response import Http404
 from django.shortcuts import get_object_or_404
 from django.shortcuts import render
 from django.urls import reverse_lazy
 from django.views import View
 from django.views import generic
-from django.views.decorators.http import require_safe
 from django.views.generic.detail import DetailView
 from django.views.generic.edit import FormMixin
 from django.views.generic.edit import FormView
@@ -589,6 +587,61 @@ def get_context_data(self, **kwargs):
         return context
 
 
+def get_epss_history(advisory, epss_page_param):
+    """
+    Fetch and return EPSS history data for an advisory.
+    """
+
+    epss_history_data = []
+    epss_pagination_obj = None
+
+    epss_scores_queryset = (
+        models.AdvisorySeverity.objects.filter(
+            advisories__advisory_id=advisory.advisory_id,
+            advisories__datasource_id=EPSS.identifier,
+            scoring_system=EPSS.identifier,
+            published_at__isnull=False,
+        )
+        .annotate(pub_date=TruncDate("published_at"))
+        .values("pub_date")
+        .annotate(
+            max_score=Max(Cast("value", FloatField())),
+            max_percentile=Max(Cast("scoring_elements", FloatField())),
+        )
+        .order_by("-pub_date")
+    )
+
+    paginator = Paginator(epss_scores_queryset, 30)
+    epss_page = epss_page_param or 1
+    epss_pagination_obj = paginator.get_page(epss_page)
+
+    if epss_pagination_obj.object_list:
+        records = list(epss_pagination_obj.object_list)
+        newest_date = records[0]["pub_date"]
+        oldest_date = records[-1]["pub_date"]
+
+        records_by_date = {
+            record["pub_date"]: {
+                "score": record["max_score"],
+                "percentile": record["max_percentile"],
+                "published_at": record["pub_date"],
+            }
+            for record in records
+        }
+
+        total_days = (newest_date - oldest_date).days
+        all_dates = [oldest_date + datetime.timedelta(days=i) for i in range(total_days + 1)]
+
+        # Handle missing dates
+        for date in all_dates:
+            if date in records_by_date:
+                epss_history_data.append(records_by_date[date])
+            else:
+                epss_history_data.append({"published_at": date, "score": None, "percentile": None})
+
+    return epss_history_data, epss_pagination_obj
+
+
 class AdvisoryDetails(VulnerableCodeDetailView):
     model = models.AdvisoryV2
     template_name = "advisory_detail.html"
@@ -749,47 +802,9 @@ def add_ssvc(ssvc):
 
         context["ssvcs"] = ssvc_entries
 
-        # EPSS history
-        cves = {
-            alias_obj.alias
-            for alias_obj in advisory.aliases.all()
-            if alias_obj.alias.startswith("CVE-")
-        }
-        if advisory.advisory_id and advisory.advisory_id.startswith("CVE-"):
-            cves.add(advisory.advisory_id)
-
-        if cves:
-            epss_scores_queryset = (
-                models.AdvisorySeverity.objects.filter(
-                    advisories__advisory_id__in=cves,
-                    scoring_system=EPSS.identifier,
-                    published_at__isnull=False,
-                )
-                .annotate(pub_date=TruncDate("published_at"))
-                .values("pub_date")
-                .annotate(
-                    max_score=Max(Cast("value", FloatField())),
-                    max_percentile=Max(Cast("scoring_elements", FloatField())),
-                )
-                .order_by("-pub_date")
-            )
-
-            paginator = Paginator(epss_scores_queryset, 30)
-            epss_page = self.request.GET.get("epss_page", 1)
-            epss_page_obj = paginator.get_page(epss_page)
-
-            epss_history_data = [
-                {
-                    "score": record["max_score"],
-                    "percentile": record["max_percentile"],
-                    "published_at": record["pub_date"],
-                }
-                for record in epss_page_obj.object_list
-            ]
-            epss_history_data.reverse()
-        else:
-            epss_history_data = []
-            epss_page_obj = None
+        epss_history_data, epss_pagination_obj = get_epss_history(
+            advisory, self.request.GET.get("epss_page")
+        )
 
         context.update(
             {
@@ -803,8 +818,7 @@ def add_ssvc(ssvc):
                 "status": advisory.get_status_label,
                 "epss_data": epss_data,
                 "epss_history_data": epss_history_data,
-                "epss_page_obj": epss_page_obj,
-                "is_epss_tab_active": "epss_page" in self.request.GET,
+                "epss_pagination_obj": epss_pagination_obj,
             }
         )
         return context