diff --git a/insights/__init__.py b/insights/__init__.py new file mode 100644 index 000000000..20854f2ad --- /dev/null +++ b/insights/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# diff --git a/insights/apps.py b/insights/apps.py new file mode 100644 index 000000000..7200f28cd --- /dev/null +++ b/insights/apps.py @@ -0,0 +1,14 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from django.apps import AppConfig + + +class InsightsConfig(AppConfig): + name = "insights" diff --git a/insights/charts/__init__.py b/insights/charts/__init__.py new file mode 100644 index 000000000..c989f493a --- /dev/null +++ b/insights/charts/__init__.py @@ -0,0 +1,155 @@ +from functools import partial + +from insights.models import ChartDefinition + +from .data_quality_panel import collect_data_quality_todos_resolutions +from .data_quality_panel import collect_open_issues_to_datasource +from .data_quality_panel import format_issue_contribution_donut +from .data_quality_panel import format_issue_type_bar +from .data_quality_panel import format_todos_resolution_timeline +from .importer_panel import _get_snapshot_data +from .importer_panel import build_importer_exploit_columns +from .importer_panel import build_importer_package_columns +from .importer_panel import collect_importers +from .overview_panel import collect_kpi_card +from .overview_panel import collect_yearly_distribution +from .overview_panel import format_coverage_comparison +from .overview_panel import format_growth_trend +from .overview_panel import format_kpi_card +from .overview_panel import format_yearly_distribution +from .package_panel import collect_cwes +from .package_panel import collect_ecosystem_distribution +from .package_panel import collect_packages +from .package_panel import format_ecosystem_distribution +from .package_panel import format_top_cwes +from .package_panel import format_top_packages +from .severity_panel import collect_severities +from .severity_panel import get_severity_snapshot_data + +CHARTS = [ + ChartDefinition( + id="overview-stats", + title="Overview Stats", + panel="overview_panel", + chart_type="custom", + formatter_fn=format_kpi_card, + collect_fn=collect_kpi_card, + ), + ChartDefinition( + id="overview-cross-validation", + title="Coverage Comparison", + panel="overview_panel", + chart_type="stacked_bar", + formatter_fn=format_coverage_comparison, + collect_fn=collect_importers, + ), + ChartDefinition( + id="overview-growth-trend", + title="Advisories Imported in the last 30 days", + panel="overview_panel", + chart_type="colored_bar", + formatter_fn=format_growth_trend, + collect_fn=collect_kpi_card, + ), + ChartDefinition( + id="overview-historical", + title="Advisories Published in the Last 10 years", + panel="overview_panel", + chart_type="colored_bar", + formatter_fn=format_yearly_distribution, + collect_fn=collect_yearly_distribution, + ), + ChartDefinition( + id="pkg-dist-donut", + title="Ecosystem Distribution", + panel="package_panel", + chart_type="donut", + formatter_fn=format_ecosystem_distribution, + collect_fn=collect_ecosystem_distribution, + ), + ChartDefinition( + id="pkg-name-bar", + title="Top 10 Packages", + panel="package_panel", + chart_type="colored_bar", + formatter_fn=format_top_packages, + collect_fn=collect_packages, + is_per_package=True, + ), + ChartDefinition( + id="pkg-cwe-bar", + title="Top 10 CWE Distribution", + panel="package_panel", + chart_type="colored_bar", + formatter_fn=format_top_cwes, + collect_fn=collect_cwes, + is_per_package=False, + has_search=True, + ), + ChartDefinition( + id="severity-scatter-plot", + title="Severity Distribution across Packages", + panel="severity_panel", + chart_type="scatter", + formatter_fn=get_severity_snapshot_data, + collect_fn=collect_severities, + has_search=True, + ), + ChartDefinition( + id="importer-empty-pkg-bar", + title="PURL Coverage across Importers", + panel="importer_panel", + chart_type="importer_bar", + formatter_fn=partial(_get_snapshot_data, build_columns_fn=build_importer_package_columns), + collect_fn=collect_importers, + ), + ChartDefinition( + id="importer-exploit-bar", + title="Exploit Coverage across Importers", + panel="importer_panel", + chart_type="importer_bar", + formatter_fn=partial(_get_snapshot_data, build_columns_fn=build_importer_exploit_columns), + collect_fn=collect_importers, + ), + ChartDefinition( + id="dq-issue-type-bar", + title="Open Issues by Type per Datasource", + panel="data_quality_panel", + chart_type="colored_bar", + formatter_fn=format_issue_type_bar, + collect_fn=collect_open_issues_to_datasource, + ), + ChartDefinition( + id="dq-importer-contribution-donut", + title="Datasource Contribution per Open Issue", + panel="data_quality_panel", + chart_type="donut", + formatter_fn=format_issue_contribution_donut, + ), + ChartDefinition( + id="dq-resolution-timeline", + title="Issue Opened and Resolved Monthly", + panel="data_quality_panel", + chart_type="line", + formatter_fn=format_todos_resolution_timeline, + collect_fn=collect_data_quality_todos_resolutions, + ), +] + +PANELS = [ + "overview_panel", + "package_panel", + "severity_panel", + "importer_panel", + "data_quality_panel", +] +PANEL_LABELS = { + "overview_panel": "Overview", + "package_panel": "Package Analytics", + "severity_panel": "Severity Analytics", + "importer_panel": "Importer Analytics", + "data_quality_panel": "Data Quality Analytics", +} +PANEL_LAYOUTS = { + "package_panel": "split_top", +} diff --git a/insights/charts/data_quality_panel.py b/insights/charts/data_quality_panel.py new file mode 100644 index 000000000..43ad03134 --- /dev/null +++ b/insights/charts/data_quality_panel.py @@ -0,0 +1,238 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from collections import defaultdict +from datetime import timedelta +from itertools import accumulate +from typing import Any +from typing import Dict + +from django.db.models import Count +from django.db.models.functions import TruncMonth +from django.utils import timezone + +from insights.models import DataQualityIssueByDatasourceInsight +from insights.models import DataQualityToDosResolutionInsight +from insights.utils import format_issue_type_label +from vulnerabilities.models import AdvisoryToDoV2 + +# Ignore Phantom Importers that don't collect Affected Packages +IGNORED_IMPORTERS = { + "epss_importer_v2", + "epss", + "vulnrichment_importer_v2", + "vulnrichment", + "suse_importer_v2", + "suse_score", +} + + +# Open Issues by Type and Datasource Contribution per Open Issue +def open_issues_to_datasource_queryset(): + """Return a query set of open issue counts by type and datasource.""" + return ( + AdvisoryToDoV2.objects.filter(is_resolved=False, advisories__datasource_id__isnull=False) + .exclude(advisories__datasource_id__in=IGNORED_IMPORTERS) + .values("issue_type", "advisories__datasource_id") + .annotate(count=Count("todo_id", distinct=True)) + .iterator() + ) + + +def iter_open_issues_to_datasource_insights(): + """Yield DataQualityIssueByDatasourceInsight objects.""" + for row in open_issues_to_datasource_queryset(): + yield DataQualityIssueByDatasourceInsight( + issue_type=row["issue_type"], + datasource_id=row["advisories__datasource_id"], + count=row["count"], + ) + + +def collect_open_issues_to_datasource(pipeline: Any) -> None: + """Collect open issue counts by type and datasource.""" + pipeline.data_quality_issue_types = list(iter_open_issues_to_datasource_insights()) + + +def build_issue_type_columns(issue_counts: dict) -> Dict[str, Any]: + """Helper to build column data for the issue type bar chart as expected by Billboard""" + issue_types = list(issue_counts.keys()) + open_issue_counts = list(issue_counts.values()) + + return { + "columns": [ + ["x"] + issue_types, + ["To-Dos"] + open_issue_counts, + ], + "x_label": "Type of Issue", + "y_label": "To-Dos Open Issue Count", + "color": "var(--bulma-danger)", + } + + +def format_issue_type_bar(snapshot: Any) -> Dict[str, Any]: + """Format open issues by type per datasource for the colored bar chart.""" + open_issue_counts_by_datasource = defaultdict(dict) + total_issues_by_type = defaultdict(int) + + for insight in snapshot.data_quality_issue_types.all(): + label = format_issue_type_label(insight.issue_type) + open_issue_counts_by_datasource[insight.datasource_id][label] = insight.count + total_issues_by_type[label] += insight.count + + data = {} + for datasource_id, counts in open_issue_counts_by_datasource.items(): + data[datasource_id] = build_issue_type_columns(counts) + + if total_issues_by_type: + data["global"] = build_issue_type_columns(total_issues_by_type) + + return data + + +def format_issue_contribution_donut(snapshot: Any) -> Dict[str, Any]: + """Format datasource contribution per open issue for the donut chart as expected by Billboard""" + datasource_counts_by_issue = defaultdict(dict) + total_issues_by_datasource = defaultdict(int) + + for insight in snapshot.data_quality_issue_types.all(): + label = format_issue_type_label(insight.issue_type) + datasource_counts_by_issue[label][insight.datasource_id] = insight.count + total_issues_by_datasource[insight.datasource_id] += insight.count + + data = {} + for issue_type, counts in datasource_counts_by_issue.items(): + columns = [[datasource_id, count] for datasource_id, count in counts.items()] + data[issue_type] = {"columns": columns} + + if total_issues_by_datasource: + global_columns = [ + [datasource_id, count] for datasource_id, count in total_issues_by_datasource.items() + ] + data["global"] = {"columns": global_columns} + + return data + + +# To-Dos Issue Resolution Timeline +def data_quality_todos_resolutions_queryset(): + """Return resolution rates by month for open and resolved to-dos.""" + start_date = timezone.now() - timedelta(days=365) # Collect last 12 months only + + open_todos = ( + AdvisoryToDoV2.objects.filter( + created_at__isnull=False, + created_at__gte=start_date, + advisories__datasource_id__isnull=False, + ) + .exclude(advisories__datasource_id__in=IGNORED_IMPORTERS) + .annotate(month=TruncMonth("created_at")) + .values("month", "advisories__datasource_id") + .annotate(count=Count("todo_id", distinct=True)) + .iterator() + ) + + resolved_todos = ( + AdvisoryToDoV2.objects.filter( + is_resolved=True, + resolved_at__isnull=False, + resolved_at__gte=start_date, + advisories__datasource_id__isnull=False, + ) + .exclude(advisories__datasource_id__in=IGNORED_IMPORTERS) + .annotate(month=TruncMonth("resolved_at")) + .values("month", "advisories__datasource_id") + .annotate(count=Count("todo_id", distinct=True)) + .iterator() + ) + return open_todos, resolved_todos + + +def iter_data_quality_todos_resolutions_insights(): + """Yield DataQualityToDosResolutionInsight objects.""" + open_todos, resolved_todos = data_quality_todos_resolutions_queryset() + + open_counts = defaultdict(int) + for open_record in open_todos: + datasource = open_record["advisories__datasource_id"] + month = open_record["month"].date() + open_counts[datasource, month] += open_record["count"] + + resolved_counts = defaultdict(int) + for resolved_record in resolved_todos: + datasource = resolved_record["advisories__datasource_id"] + month = resolved_record["month"].date() + resolved_counts[datasource, month] += resolved_record["count"] + + all_keys = set(open_counts.keys()) | set( + resolved_counts.keys() + ) # All unique (datasource, month) pairs across opened and resolved to-dos + + for datasource_id, month_date in sorted(all_keys): + yield DataQualityToDosResolutionInsight( + datasource_id=datasource_id, + month=month_date, + open_count=open_counts[(datasource_id, month_date)], + resolved_count=resolved_counts[(datasource_id, month_date)], + ) + + +def collect_data_quality_todos_resolutions(pipeline: Any) -> None: + """Collect historical resolution rates by month.""" + pipeline.data_quality_todos_resolutions = list(iter_data_quality_todos_resolutions_insights()) + + +def build_todos_resolution_columns(month_counts: Dict[str, Dict[str, int]]) -> Dict[str, Any]: + """Helper to build issue resolution line chart as expected by Billboard""" + sorted_months = sorted(month_counts.keys()) + new_open_counts = [month_counts[month]["open"] for month in sorted_months] + new_resolved_counts = [month_counts[month]["resolved"] for month in sorted_months] + + return { + "columns": [ + ["x"] + sorted_months, + ["Open"] + list(accumulate(new_open_counts)), + ["Resolved"] + list(accumulate(new_resolved_counts)), + ], + "new_open_counts": new_open_counts, + "new_resolved_counts": new_resolved_counts, + "y_label": "Cumulative Count", + } + + +def format_todos_resolution_timeline(snapshot: Any) -> Dict[str, Any]: + """Format cumulative to-dos resolution line chart""" + resolution_insights = snapshot.data_quality_todos_resolutions.all().order_by("month") + + datasource_month_counts = defaultdict(dict) + global_month_counts = {} + + for insight in resolution_insights: + formatted_month = insight.month.strftime("%Y-%m-%d") + datasource_id = insight.datasource_id + + # Initialize global month count if not exists + if formatted_month not in global_month_counts: + global_month_counts[formatted_month] = {"open": 0, "resolved": 0} + + global_month_counts[formatted_month]["open"] += insight.open_count + global_month_counts[formatted_month]["resolved"] += insight.resolved_count + + datasource_month_counts[datasource_id][formatted_month] = { + "open": insight.open_count, + "resolved": insight.resolved_count, + } + + data = {} + if global_month_counts: + data["global"] = build_todos_resolution_columns(global_month_counts) + for datasource_id, month_counts in datasource_month_counts.items(): + data[datasource_id] = build_todos_resolution_columns(month_counts) + + return data diff --git a/insights/charts/importer_panel.py b/insights/charts/importer_panel.py new file mode 100644 index 000000000..84c948963 --- /dev/null +++ b/insights/charts/importer_panel.py @@ -0,0 +1,214 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from typing import Any +from typing import Dict + +from django.db.models import Count +from django.db.models import Exists +from django.db.models import OuterRef +from django.db.models import Q + +from insights.models import ImporterInsight +from insights.utils import format_importer_name +from vulnerabilities.models import AdvisoryExploit +from vulnerabilities.models import AdvisoryV2 + +# Ignore Phantom Importers that don't collect Affected Packages +IGNORED_IMPORTERS = { + "epss_importer_v2", + "epss", + "vulnrichment_importer_v2", + "suse_importer_v2", + "suse_score", +} + + +def packages_queryset(): + """Return a query set of package statistics by importer.""" + return ( + AdvisoryV2.objects.filter(is_latest=True) + .exclude(datasource_id__in=IGNORED_IMPORTERS) + .values("datasource_id") + .annotate( + total_advisories=Count("avid", distinct=True), + advisories_with_packages=Count( + "avid", filter=Q(impacted_packages__affecting_packages__isnull=False), distinct=True + ), + advisories_with_ghost_packages=Count( + "avid", + filter=Q(impacted_packages__affecting_packages__is_ghost=True), + distinct=True, + ), + ) + .order_by("-total_advisories") + .iterator() + ) + + +def exploits_queryset(): + """Return a query set of exploit statistics by importer.""" + kev_exploits = AdvisoryExploit.objects.filter(advisory=OuterRef("pk"), data_source="KEV") + metasploit_exploits = AdvisoryExploit.objects.filter( + advisory=OuterRef("pk"), data_source="Metasploit" + ) + exploitdb_exploits = AdvisoryExploit.objects.filter( + advisory=OuterRef("pk"), data_source="Exploit-DB" + ) + + return ( + AdvisoryV2.objects.filter(is_latest=True) + .exclude(datasource_id__in=IGNORED_IMPORTERS) + .annotate( + has_kev=Exists(kev_exploits), + has_metasploit=Exists(metasploit_exploits), + has_exploitdb=Exists(exploitdb_exploits), + ) + .values("datasource_id") + .annotate( + advisories_with_kev=Count("avid", filter=Q(has_kev=True), distinct=True), + advisories_with_metasploit=Count( + "avid", filter=Q(has_kev=False, has_metasploit=True), distinct=True + ), + advisories_with_exploitdb=Count( + "avid", + filter=Q(has_kev=False, has_metasploit=False, has_exploitdb=True), + distinct=True, + ), + ) + .iterator() + ) + + +def iter_importer_insights(): + """Yield ImporterInsight objects by merging package and exploit statistics.""" + exploit_stats_by_importer = {} + for record in exploits_queryset(): + exploit_stats_by_importer[record["datasource_id"]] = record + + for pkg_record in packages_queryset(): + datasource_id = pkg_record["datasource_id"] + exploit_record = exploit_stats_by_importer.get(datasource_id, {}) + + yield ImporterInsight( + importer=datasource_id, + total_advisories=pkg_record["total_advisories"], + advisories_with_packages=pkg_record["advisories_with_packages"], + advisories_with_ghost_packages=pkg_record["advisories_with_ghost_packages"], + advisories_with_kev=exploit_record.get("advisories_with_kev", 0), + advisories_with_metasploit=exploit_record.get("advisories_with_metasploit", 0), + advisories_with_exploitdb=exploit_record.get("advisories_with_exploitdb", 0), + ) + + +def collect_importers(pipeline): + """Calculates and stores importer insights.""" + + # Two charts use this function, so we avoid rerunning the query twice + if pipeline.importer_insights: + return + + for insight in iter_importer_insights(): + pipeline.importer_insights.append(insight) + + +def _get_snapshot_data(snapshot: Any, build_columns_fn) -> Dict[str, Any]: + """Helper to format importer statistics using a provided column builder.""" + all_importers = list(snapshot.importer_insights.order_by("-total_advisories")) + data = {} + + if all_importers: + # Top 5 by default for global view + data["global"] = build_columns_fn(all_importers[:5]) + + # Individual data for each importer + for importer_insight in all_importers: + formatted_name = format_importer_name(importer_insight.importer) + data[formatted_name] = build_columns_fn([importer_insight]) + + return data + + +def build_importer_package_columns(importers) -> Dict[str, Any]: + """Helper to build mappings for Package Coverage chart as expected by Billboard.JS""" + importer_names = [] + total_advisories = [] + advisories_with_packages = [] + advisories_without_packages = [] + advisories_with_ghost_packages = [] + advisories_without_ghost_packages = [] + + for importer_insight in importers: + importer_names.append(format_importer_name(importer_insight.importer)) + total_advisories.append(importer_insight.total_advisories) + advisories_with_packages.append(importer_insight.advisories_with_packages) + advisories_without_packages.append( + importer_insight.total_advisories - importer_insight.advisories_with_packages + ) + advisories_with_ghost_packages.append(importer_insight.advisories_with_ghost_packages) + advisories_without_ghost_packages.append( + importer_insight.advisories_with_packages + - importer_insight.advisories_with_ghost_packages + ) + + return { + "x_categories": importer_names, + "columns": [ + ["total_advisories"] + total_advisories, + ["advisories_with_packages"] + advisories_with_packages, + ["advisories_without_packages"] + advisories_without_packages, + ["advisories_without_ghost_packages"] + advisories_without_ghost_packages, + ["advisories_with_ghost_packages"] + advisories_with_ghost_packages, + ], + "groups": [ + ["advisories_with_packages", "advisories_without_packages"], + ["advisories_without_ghost_packages", "advisories_with_ghost_packages"], + ], + } + + +def build_importer_exploit_columns(importers) -> Dict[str, Any]: + """Helper to build mappings for Exploit Coverage chart as expected by Billboard.JS""" + importer_names = [] + total_advisories = [] + advisories_with_exploits = [] + advisories_without_exploits = [] + advisories_with_kev = [] + advisories_with_metasploit = [] + advisories_with_exploitdb = [] + + for importer_insight in importers: + importer_names.append(format_importer_name(importer_insight.importer)) + total_advisories.append(importer_insight.total_advisories) + advisories_with_kev.append(importer_insight.advisories_with_kev) + advisories_with_metasploit.append(importer_insight.advisories_with_metasploit) + advisories_with_exploitdb.append(importer_insight.advisories_with_exploitdb) + + total_with_exploits = ( + importer_insight.advisories_with_kev + + importer_insight.advisories_with_metasploit + + importer_insight.advisories_with_exploitdb + ) + advisories_with_exploits.append(total_with_exploits) + advisories_without_exploits.append(importer_insight.total_advisories - total_with_exploits) + + return { + "x_categories": importer_names, + "columns": [ + ["total_advisories"] + total_advisories, + ["advisories_with_exploits"] + advisories_with_exploits, + ["advisories_without_exploits"] + advisories_without_exploits, + ["advisories_with_exploitdb"] + advisories_with_exploitdb, + ["advisories_with_metasploit"] + advisories_with_metasploit, + ["advisories_with_kev"] + advisories_with_kev, + ], + "groups": [ + ["advisories_with_exploits", "advisories_without_exploits"], + ["advisories_with_exploitdb", "advisories_with_metasploit", "advisories_with_kev"], + ], + } diff --git a/insights/charts/overview_panel.py b/insights/charts/overview_panel.py new file mode 100644 index 000000000..fab450e41 --- /dev/null +++ b/insights/charts/overview_panel.py @@ -0,0 +1,186 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from datetime import timedelta +from typing import Any +from typing import Dict + +from django.db.models import Count +from django.db.models.functions import ExtractYear +from django.db.models.functions import TruncDate +from django.utils import timezone + +from insights.models import DailySnapshot +from insights.models import OverviewInsight +from insights.models import OverviewYearlyInsight +from insights.utils import format_importer_name +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import PackageV2 + + +# Overview KPIs +def kpi_card_queryset() -> Dict[str, int]: + """Return querysets for KPI cards.""" + return { + "total_advisories": AdvisoryV2.objects.filter(is_latest=True).aggregate( + count=Count("avid", distinct=True) + )["count"], + "total_packages": PackageV2.objects.values("plain_package_url").distinct().count(), + "total_data_sources": AdvisoryV2.objects.values("datasource_id").distinct().count(), + } + + +def daily_growth_counts(reference_time) -> list[int]: + """Return daily advisory ingestion counts for the last 30 days as a list.""" + start_date = reference_time.date() - timedelta(days=30) + daily_counts = { + entry["day"]: entry["count"] + for entry in ( + AdvisoryV2.objects.filter(is_latest=True, date_collected__gte=start_date) + .annotate(day=TruncDate("date_collected")) + .values("day") + .annotate(count=Count("avid", distinct=True)) + ) + if entry["day"] + } + return [daily_counts.get(start_date + timedelta(days=i), 0) for i in range(31)] + + +def collect_kpi_card(pipeline: Any) -> None: + """Collect overall totals and 30-day ingestion counts for OverviewInsight.""" + reference_time = timezone.now() + pipeline.overview_insight = OverviewInsight( + **kpi_card_queryset(), + last_30days=daily_growth_counts(reference_time), + ) + + +def build_kpi_card( + overview: OverviewInsight, previous_snapshot: DailySnapshot = None +) -> Dict[str, Any]: + """Build the KPI card data with deltas compared to the previous snapshot.""" + delta_advisories = 0 + delta_packages = 0 + delta_sources = 0 + + if previous_snapshot: + previous_overview = previous_snapshot.overview + delta_advisories = overview.total_advisories - previous_overview.total_advisories + delta_packages = overview.total_packages - previous_overview.total_packages + delta_sources = overview.total_data_sources - previous_overview.total_data_sources + + return { + "total_advisories": overview.total_advisories, + "delta_advisories": delta_advisories, + "total_packages": overview.total_packages, + "delta_packages": delta_packages, + "total_data_sources": overview.total_data_sources, + "delta_data_sources": delta_sources, + } + + +def format_kpi_card(snapshot: Any) -> Dict[str, Any]: + """Format the Overview KPI cards and calculate deltas compared to the previous snapshot.""" + previous_snapshot = ( + DailySnapshot.objects.filter(created_at__lt=snapshot.created_at) + .order_by("-created_at") + .first() + ) + return {"kpis": build_kpi_card(snapshot.overview, previous_snapshot)} + + +# Year wise distribution of advisories +def yearly_distribution_queryset(): + """Return a queryset of advisories grouped by year for the last 10 years.""" + current_year = timezone.now().year + start_year = current_year - 9 + + return ( + AdvisoryV2.objects.filter( + date_published__isnull=False, + date_published__year__gte=start_year, + date_published__year__lte=current_year, + ) + .annotate(year=ExtractYear("date_published")) + .values("year") + .annotate(count=Count("avid", distinct=True)) + .order_by("-year") + .iterator() + ) + + +def iter_yearly_distribution_insights(): + """Yield OverviewYearlyInsight objects.""" + for yearly_record in yearly_distribution_queryset(): + yield OverviewYearlyInsight(**yearly_record) + + +def collect_yearly_distribution(pipeline: Any) -> None: + """Collect advisories by published year for the last 10 years.""" + pipeline.overview_yearly_insights = list(iter_yearly_distribution_insights()) + + +def format_yearly_distribution(snapshot: Any) -> Dict[str, Any]: + """Format historical advisories published over the last 10 years as expected by Billboard""" + yearly_insights = snapshot.overview.yearly_insights.order_by("year") + + return { + "columns": [ + ["x"] + [str(yi.year) for yi in yearly_insights], + ["Advisories Published"] + [yi.count for yi in yearly_insights], + ], + "x_label": "Year", + "y_label": "Advisories Published", + "color": "var(--bulma-primary)", + "rotated": False, + "x_tick_culling": 15, + } + + +# Advisory Ingestion Trend +def format_growth_trend(snapshot: Any) -> Dict[str, Any]: + """Format daily advisory ingestion trend as expected by Billboard.""" + reference_time = getattr(snapshot, "created_at", None) or timezone.now() + start_date = reference_time.date() - timedelta(days=30) + + dates = [(start_date + timedelta(days=i)).strftime("%b %d") for i in range(31)] + counts = getattr(snapshot.overview, "last_30days", []) or [0] * len(dates) + + return { + "columns": [ + ["x"] + dates, + ["Advisories Imported"] + counts, + ], + "x_label": "Date", + "y_label": "Advisories Imported", + "color": "var(--bulma-primary-dark)", + "rotated": False, + "x_tick_culling": 7, + } + + +# Coverage Comparison +def format_coverage_comparison(snapshot: Any) -> Dict[str, Any]: + """Format the coverage comparison graph as expected by Billboard.""" + importers = list(snapshot.importer_insights.all().order_by("-total_advisories")) + importer_names = [format_importer_name(importer.importer) for importer in importers] + total_vulnerablecode_advisories = sum(importer.total_advisories for importer in importers) + + columns = [ + ["x", "VulnerableCode", "Sources"], + ["VulnerableCode", total_vulnerablecode_advisories, 0], + ] + for importer in importers: + columns.append([format_importer_name(importer.importer), 0, importer.total_advisories]) + + return { + "columns": columns, + "groups": [importer_names], + "y_label": "Advisories", + } diff --git a/insights/charts/package_panel.py b/insights/charts/package_panel.py new file mode 100644 index 000000000..78482b4c0 --- /dev/null +++ b/insights/charts/package_panel.py @@ -0,0 +1,194 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from collections import Counter +from collections import defaultdict +from typing import Any +from typing import Dict + +from django.db.models import Count +from django.db.models import F +from django.db.models import Q + +from insights.models import PackageCWEInsight +from insights.models import PackageInsight +from insights.models import PackageNameInsight +from insights.utils import get_cwe_label +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import PackageV2 + +# Generic OWASP category CWEs used by NVD, not actual weakness IDs. +IGNORED_CWE_IDS = [937, 1035] + + +# Package Distribution +def ecosystem_distribution_queryset(package_types): + """Return a query set of total package counts per ecosystem.""" + return ( + PackageV2.objects.filter(type__in=package_types) + .values("type") + .annotate(total_package_count=Count("id")) + .iterator() + ) + + +def iter_ecosystem_distribution_insights(package_types): + """Yield total package type counts per ecosystem.""" + for package_stat in ecosystem_distribution_queryset(package_types): + yield package_stat["type"], package_stat["total_package_count"] + + +def collect_ecosystem_distribution(pipeline: Any) -> None: + """Collect total package type counts per ecosystem.""" + for package_type, total_packages in iter_ecosystem_distribution_insights(pipeline.packages): + if package_type in pipeline.package_insights: + pipeline.package_insights[package_type].total_packages = total_packages + + +def format_ecosystem_distribution(snapshot: Any) -> Dict[str, Any]: + """Format the Ecosystem Distribution Chart as expected by Billboard.JS""" + stats = list(snapshot.package_insights.exclude(package="global").order_by("-total_packages")) + + columns = [[stat.package, stat.total_packages] for stat in stats[:10]] + + others_count = sum(stat.total_packages for stat in stats[10:]) + others_list = [[stat.package, stat.total_packages] for stat in stats[10:20]] + if others_count > 0: + columns.append(["Others", others_count]) + + return {"global": {"columns": columns, "others_list": others_list}} + + +# Top 10 Packages +def top_packages_queryset(package_type: str): + """Return a query set of the top 10 packages by number of distinct advisories.""" + return ( + PackageV2.objects.filter(type=package_type) + .values("name") + .annotate( + count=Count( + "affected_in_impacts__advisory", + filter=Q(affected_in_impacts__advisory__is_latest=True), + ) + ) + .filter(count__gt=0) + .order_by("-count")[:10] + .iterator() + ) + + +def iter_packages_insights(package_type: str, insight_obj: Any): + """Yield PackageNameInsight objects for the top 10 packages of a given type.""" + for stat in top_packages_queryset(package_type): + yield PackageNameInsight( + package_insight=insight_obj, name=stat["name"], count=stat["count"] + ) + + +def collect_packages(pipeline: Any, package_type: str) -> None: + """Collect the top 10 packages for a package type""" + insight_obj = pipeline.package_insights[package_type] + for insight in iter_packages_insights(package_type, insight_obj): + pipeline.package_names.append(insight) + + +def build_name_chart_columns(name_counts: dict) -> Dict[str, Any]: + """Helper to build package name bar charts.""" + names = list(name_counts.keys()) + counts = list(name_counts.values()) + + return { + "columns": [ + ["x"] + names, + ["Advisories"] + counts, + ], + "x_label": "Package Name", + "y_label": "Advisories", + "color": "var(--bulma-orange)", + } + + +def format_top_packages(snapshot: Any) -> Dict[str, Any]: + """ + Format name data for the frontend donut chart as expected by Billboard.JS + Returns Top 10 package names for each package type and global. + """ + data = {} + global_counts = defaultdict(int) + + for package_insight in ( + snapshot.package_insights.exclude(package="global").prefetch_related("names").all() + ): + name_counts = {ns.name: ns.count for ns in package_insight.names.all()} + data[package_insight.package] = build_name_chart_columns(name_counts) + for name, count in name_counts.items(): + global_counts[name] += count + + top_global = dict(Counter(global_counts).most_common(10)) + data["global"] = build_name_chart_columns(top_global) + return data + + +# Top CWE Distribution +def compute_top_cwes(packages=None) -> dict: + """Computes the top 10 CWEs for the given queryset of packages. If None, computes globally.""" + qs = AdvisoryV2.objects.filter(weaknesses__isnull=False) + if packages is not None: + qs = qs.filter(impacted_packages__affecting_packages__in=packages) + + top_10 = ( + qs.values(cwe=F("weaknesses__cwe_id")) + .exclude(cwe__in=IGNORED_CWE_IDS) + .annotate(count=Count("avid", distinct=True)) + .order_by("-count")[:10] + ) + return {stat["cwe"]: stat["count"] for stat in top_10 if stat["cwe"]} + + +def iter_cwes_insights(insight_obj: Any): + """Yield PackageCWEInsight objects for the global top 10 CWE distribution.""" + for cwe_id, count in compute_top_cwes().items(): + yield PackageCWEInsight(package_insight=insight_obj, cwe_id=cwe_id, count=count) + + +def collect_cwes(pipeline: Any) -> None: + """Collect the global top 10 CWE distribution.""" + if "global" not in pipeline.package_insights: + pipeline.package_insights["global"] = PackageInsight(package="global") + + # Collect top 10 CWEs globally + insight_obj = pipeline.package_insights["global"] + for insight in iter_cwes_insights(insight_obj): + pipeline.package_cwes.append(insight) + + +def build_cwe_chart_columns(cwe_counts: dict) -> Dict[str, Any]: + """Helper to CWE chart as expected by Billboard.JS""" + # Sort CWEs by count in descending order + sorted_cwes = sorted(cwe_counts.items(), key=lambda item: item[1], reverse=True) + return { + "columns": [ + ["x"] + [f"CWE-{cwe_id}" for cwe_id, count in sorted_cwes], + ["Advisories"] + [count for cwe_id, count in sorted_cwes], + ], + "full_labels": [get_cwe_label(cwe_id) for cwe_id, count in sorted_cwes], + } + + +def format_top_cwes(snapshot: Any) -> Dict[str, Any]: + """Format CWE distribution data.""" + data = {} + try: + global_insight = snapshot.package_insights.get(package="global") + cwe_counts = {cwe.cwe_id: cwe.count for cwe in global_insight.cwes.all()} + if cwe_counts: + data["global"] = build_cwe_chart_columns(cwe_counts) + except PackageInsight.DoesNotExist: + pass + + return data diff --git a/insights/charts/severity_panel.py b/insights/charts/severity_panel.py new file mode 100644 index 000000000..51430bb30 --- /dev/null +++ b/insights/charts/severity_panel.py @@ -0,0 +1,74 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from typing import Any +from typing import Dict + +from django.db.models import Count + +from insights.models import SeverityInsight +from vulnerabilities.models import AdvisoryV2 + +CVSS_SCORE_LABELS = ["0-1", "1-2", "2-3", "3-4", "4-5", "5-6", "6-7", "7-8", "8-9", "9-10"] + + +def aggregate_severity_buckets(queryset: Any, count_field: str) -> list[int]: + """Helper to aggregate a queryset of severity values into exactly 10 CVSS buckets.""" + buckets = [0] * 10 + + for severity in queryset: + try: + score = float(severity.get("severities__value")) + if 0 <= score <= 10: + buckets[min(9, int(score))] += severity[count_field] + except (ValueError, TypeError): + continue + + return buckets + + +def iter_severity_insights(): + """Yield SeverityInsight objects.""" + advisory_severities = ( + AdvisoryV2.objects.filter( + severities__isnull=False, severities__scoring_system__icontains="cvss" + ) + .values("severities__value") + .annotate(advisory_count=Count("avid", distinct=True)) + .iterator() + ) + + severity_buckets = aggregate_severity_buckets(advisory_severities, "advisory_count") + yield SeverityInsight(buckets=severity_buckets) + + +def collect_severities(pipeline: Any) -> None: + """Pre-compute the global severity distribution.""" + for insight in iter_severity_insights(): + pipeline.severity_insight = insight + + +def format_severity_chart_data(buckets: list[int]) -> Dict[str, Any]: + """Helper to build mappings for Severity Distribution chart as expected by Billboard.JS""" + return { + "columns": [ + ["x"] + CVSS_SCORE_LABELS, + ["Advisories"] + buckets, + ] + } + + +def get_severity_snapshot_data(snapshot: Any) -> Dict[str, Any]: + """Return the global severity distribution for the frontend scatter plot.""" + if hasattr(snapshot, "severity_insight"): + insight = snapshot.severity_insight + global_buckets = insight.buckets + else: + global_buckets = [0] * 10 + + return {"global": format_severity_chart_data(global_buckets)} diff --git a/insights/insights_snapshot_pipeline.py b/insights/insights_snapshot_pipeline.py new file mode 100644 index 000000000..5e4644a63 --- /dev/null +++ b/insights/insights_snapshot_pipeline.py @@ -0,0 +1,118 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from aboutcode.pipeline import LoopProgress +from django.db import transaction + +from insights.charts import CHARTS +from insights.models import DailySnapshot +from insights.models import DataQualityIssueByDatasourceInsight +from insights.models import DataQualityToDosResolutionInsight +from insights.models import ImporterInsight +from insights.models import OverviewCoverageInsight +from insights.models import OverviewYearlyInsight +from insights.models import PackageCWEInsight +from insights.models import PackageInsight +from insights.models import PackageNameInsight +from vulnerabilities.models import PackageV2 +from vulnerabilities.pipelines import VulnerableCodePipeline + + +class InsightsSnapshotPipeline(VulnerableCodePipeline): + """Pipeline to compute aggregated statistics for the Insights Dashboard.""" + + pipeline_id = "insights_snapshot" + + @classmethod + def steps(cls): + return ( + cls.compute_chart_analytics, + cls.save_snapshot, + ) + + def compute_chart_analytics(self): + """Run chart collect_fns to compute analytics.""" + + # List all package types + self.packages = list(PackageV2.objects.order_by().values_list("type", flat=True).distinct()) + + self.package_insights = {pkg: PackageInsight(package=pkg) for pkg in self.packages} + self.package_names = [] + self.package_cwes = [] + self.importer_insights = [] + self.severity_insight = None + self.overview_insight = None + self.overview_yearly_insights = [] + self.overview_coverage_insights = [] + self.data_quality_issue_types = [] + self.data_quality_todos_resolutions = [] + active_charts = [chart_def for chart_def in CHARTS if chart_def.collect_fn] + + # Count steps for progress bar + total_steps = sum( + len(self.packages) if chart_def.is_per_package else 1 for chart_def in active_charts + ) + + progress = LoopProgress(total_iterations=total_steps, logger=self.log, progress_step=1) + progress_iter = iter(progress.iter(range(total_steps))) + + for chart_def in active_charts: + self.log(f"Running collect_fn for {chart_def.id}") + + if chart_def.is_per_package: + for pkg in self.packages: + next(progress_iter, None) + chart_def.collect_fn(self, pkg) + else: + next(progress_iter, None) + chart_def.collect_fn(self) + + @transaction.atomic + def save_snapshot(self): + self.log("Saving snapshot") + snapshot = DailySnapshot.objects.create() + + for insight in self.package_insights.values(): + insight.snapshot_id = snapshot.id + + for insight in self.importer_insights: + insight.snapshot_id = snapshot.id + + if self.severity_insight: + self.severity_insight.snapshot_id = snapshot.id + self.severity_insight.save() + + if self.overview_insight: + self.overview_insight.snapshot_id = snapshot.id + self.overview_insight.save() + + for yearly in self.overview_yearly_insights: + yearly.overview_id = self.overview_insight.id + for coverage in self.overview_coverage_insights: + coverage.overview_id = self.overview_insight.id + + for insight in self.data_quality_issue_types: + insight.snapshot_id = snapshot.id + for insight in self.data_quality_todos_resolutions: + insight.snapshot_id = snapshot.id + + PackageInsight.objects.bulk_create(self.package_insights.values(), batch_size=5000) + PackageNameInsight.objects.bulk_create(self.package_names, batch_size=5000) + PackageCWEInsight.objects.bulk_create(self.package_cwes, batch_size=5000) + ImporterInsight.objects.bulk_create(self.importer_insights, batch_size=5000) + OverviewYearlyInsight.objects.bulk_create(self.overview_yearly_insights, batch_size=5000) + OverviewCoverageInsight.objects.bulk_create( + self.overview_coverage_insights, batch_size=5000 + ) + DataQualityIssueByDatasourceInsight.objects.bulk_create( + self.data_quality_issue_types, batch_size=5000 + ) + DataQualityToDosResolutionInsight.objects.bulk_create( + self.data_quality_todos_resolutions, batch_size=5000 + ) + self.log("Snapshot saved successfully.") diff --git a/insights/migrations/0001_initial.py b/insights/migrations/0001_initial.py new file mode 100644 index 000000000..26b349636 --- /dev/null +++ b/insights/migrations/0001_initial.py @@ -0,0 +1,174 @@ +# Generated by Django 5.2.11 on 2026-07-23 21:56 + +import django.contrib.postgres.fields +import django.db.models.deletion +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="DailySnapshot", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("created_at", models.DateTimeField(auto_now_add=True)), + ], + options={ + "ordering": ["-created_at"], + "get_latest_by": "created_at", + }, + ), + migrations.CreateModel( + name="PackageInsight", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("package", models.CharField(max_length=50)), + ("total_packages", models.IntegerField(default=0)), + ( + "snapshot", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="package_insights", + to="insights.dailysnapshot", + ), + ), + ], + ), + migrations.CreateModel( + name="PackageCWEInsight", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("cwe_id", models.CharField(max_length=50)), + ("count", models.IntegerField()), + ( + "package_insight", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="cwes", + to="insights.packageinsight", + ), + ), + ], + ), + migrations.CreateModel( + name="PackageNameInsight", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("name", models.CharField(max_length=255)), + ("count", models.IntegerField()), + ( + "package_insight", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="names", + to="insights.packageinsight", + ), + ), + ], + ), + migrations.CreateModel( + name="SeverityInsight", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "buckets", + django.contrib.postgres.fields.ArrayField( + base_field=models.IntegerField(), + default=list, + help_text="Scores mapped to buckets 0-10 (e.g. 0.0-0.9 -> index 0)", + size=10, + ), + ), + ( + "snapshot", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="severity_insight", + to="insights.dailysnapshot", + ), + ), + ], + ), + migrations.CreateModel( + name="ImporterInsight", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("importer", models.CharField(max_length=100)), + ("total_advisories", models.IntegerField(default=0)), + ("advisories_with_packages", models.IntegerField(default=0)), + ("advisories_with_ghost_packages", models.IntegerField(default=0)), + ("advisories_with_kev", models.IntegerField(default=0)), + ("advisories_with_metasploit", models.IntegerField(default=0)), + ("advisories_with_exploitdb", models.IntegerField(default=0)), + ( + "snapshot", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="importer_insights", + to="insights.dailysnapshot", + ), + ), + ], + options={ + "constraints": [ + models.UniqueConstraint( + fields=("snapshot", "importer"), name="unique_snapshot_importer" + ) + ], + }, + ), + migrations.AddConstraint( + model_name="packageinsight", + constraint=models.UniqueConstraint( + fields=("snapshot", "package"), name="unique_snapshot_package" + ), + ), + migrations.AddConstraint( + model_name="packagecweinsight", + constraint=models.UniqueConstraint( + fields=("package_insight", "cwe_id"), name="unique_package_cwe" + ), + ), + migrations.AddConstraint( + model_name="packagenameinsight", + constraint=models.UniqueConstraint( + fields=("package_insight", "name"), name="unique_package_name" + ), + ), + ] diff --git a/insights/migrations/0002_overviewinsight_overviewcoverageinsight_and_more.py b/insights/migrations/0002_overviewinsight_overviewcoverageinsight_and_more.py new file mode 100644 index 000000000..be15737cb --- /dev/null +++ b/insights/migrations/0002_overviewinsight_overviewcoverageinsight_and_more.py @@ -0,0 +1,150 @@ +# Generated by Django 5.2.11 on 2026-08-08 22:59 + +import django.contrib.postgres.fields +import django.db.models.deletion +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("insights", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="OverviewInsight", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("total_advisories", models.IntegerField(default=0)), + ("total_packages", models.IntegerField(default=0)), + ("total_data_sources", models.IntegerField(default=0)), + ( + "last_30days", + django.contrib.postgres.fields.ArrayField( + base_field=models.IntegerField(), default=list, size=None + ), + ), + ( + "snapshot", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="overview", + to="insights.dailysnapshot", + ), + ), + ], + ), + migrations.CreateModel( + name="OverviewCoverageInsight", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("source_name", models.CharField(max_length=100)), + ("count", models.IntegerField()), + ( + "overview", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="coverage_insights", + to="insights.overviewinsight", + ), + ), + ], + ), + migrations.CreateModel( + name="OverviewYearlyInsight", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("year", models.IntegerField()), + ("count", models.IntegerField()), + ( + "overview", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="yearly_insights", + to="insights.overviewinsight", + ), + ), + ], + ), + migrations.CreateModel( + name="DataQualityIssueByDatasourceInsight", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("issue_type", models.CharField(max_length=50)), + ("datasource_id", models.CharField(max_length=100)), + ("count", models.IntegerField(default=0)), + ( + "snapshot", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="data_quality_issue_types", + to="insights.dailysnapshot", + ), + ), + ], + options={ + "constraints": [ + models.UniqueConstraint( + fields=("snapshot", "issue_type", "datasource_id"), + name="unique_snapshot_issue_type_datasource", + ) + ], + }, + ), + migrations.CreateModel( + name="DataQualityToDosResolutionInsight", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("datasource_id", models.CharField(max_length=100)), + ( + "month", + models.DateField(help_text="The first day of the month for this data point."), + ), + ("open_count", models.IntegerField(default=0)), + ("resolved_count", models.IntegerField(default=0)), + ( + "snapshot", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="data_quality_todos_resolutions", + to="insights.dailysnapshot", + ), + ), + ], + options={ + "constraints": [ + models.UniqueConstraint( + fields=("snapshot", "datasource_id", "month"), + name="unique_snapshot_todos_resolution_datasource_month", + ) + ], + }, + ), + ] diff --git a/insights/migrations/__init__.py b/insights/migrations/__init__.py new file mode 100644 index 000000000..20854f2ad --- /dev/null +++ b/insights/migrations/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# diff --git a/insights/models.py b/insights/models.py new file mode 100644 index 000000000..89f9843c9 --- /dev/null +++ b/insights/models.py @@ -0,0 +1,173 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from dataclasses import dataclass +from typing import Any +from typing import Callable +from typing import Dict +from typing import Optional + +from django.contrib.postgres.fields import ArrayField +from django.db import models + + +class DailySnapshot(models.Model): + created_at = models.DateTimeField(auto_now_add=True) + + class Meta: + get_latest_by = "created_at" + ordering = ["-created_at"] + + +class OverviewInsight(models.Model): + snapshot = models.OneToOneField( + "DailySnapshot", related_name="overview", on_delete=models.CASCADE + ) + + total_advisories = models.IntegerField(default=0) + total_packages = models.IntegerField(default=0) + total_data_sources = models.IntegerField(default=0) + last_30days = ArrayField( + models.IntegerField(), + default=list, + ) + + +class OverviewYearlyInsight(models.Model): + overview = models.ForeignKey( + OverviewInsight, related_name="yearly_insights", on_delete=models.CASCADE + ) + year = models.IntegerField() + count = models.IntegerField() + + +class OverviewCoverageInsight(models.Model): + overview = models.ForeignKey( + OverviewInsight, related_name="coverage_insights", on_delete=models.CASCADE + ) + source_name = models.CharField(max_length=100) + count = models.IntegerField() + + +class PackageInsight(models.Model): + snapshot = models.ForeignKey( + DailySnapshot, related_name="package_insights", on_delete=models.CASCADE + ) + package = models.CharField(max_length=50) + total_packages = models.IntegerField(default=0) + + class Meta: + constraints = [ + models.UniqueConstraint(fields=["snapshot", "package"], name="unique_snapshot_package") + ] + + +class PackageNameInsight(models.Model): + package_insight = models.ForeignKey( + PackageInsight, related_name="names", on_delete=models.CASCADE + ) + name = models.CharField(max_length=255) + count = models.IntegerField() + + class Meta: + constraints = [ + models.UniqueConstraint(fields=["package_insight", "name"], name="unique_package_name") + ] + + +class PackageCWEInsight(models.Model): + package_insight = models.ForeignKey( + PackageInsight, related_name="cwes", on_delete=models.CASCADE + ) + cwe_id = models.CharField(max_length=50) + count = models.IntegerField() + + class Meta: + constraints = [ + models.UniqueConstraint(fields=["package_insight", "cwe_id"], name="unique_package_cwe") + ] + + +class SeverityInsight(models.Model): + snapshot = models.OneToOneField( + DailySnapshot, related_name="severity_insight", on_delete=models.CASCADE + ) + buckets = ArrayField( + models.IntegerField(), + size=10, + default=list, + help_text="Scores mapped to buckets 0-10 (e.g. 0.0-0.9 -> index 0)", + ) + + +class ImporterInsight(models.Model): + snapshot = models.ForeignKey( + DailySnapshot, related_name="importer_insights", on_delete=models.CASCADE + ) + importer = models.CharField(max_length=100) + total_advisories = models.IntegerField(default=0) + advisories_with_packages = models.IntegerField(default=0) + advisories_with_ghost_packages = models.IntegerField(default=0) + advisories_with_kev = models.IntegerField(default=0) + advisories_with_metasploit = models.IntegerField(default=0) + advisories_with_exploitdb = models.IntegerField(default=0) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["snapshot", "importer"], name="unique_snapshot_importer" + ) + ] + + +class DataQualityIssueByDatasourceInsight(models.Model): + snapshot = models.ForeignKey( + DailySnapshot, related_name="data_quality_issue_types", on_delete=models.CASCADE + ) + issue_type = models.CharField(max_length=50) + datasource_id = models.CharField(max_length=100) + count = models.IntegerField(default=0) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["snapshot", "issue_type", "datasource_id"], + name="unique_snapshot_issue_type_datasource", + ) + ] + + +class DataQualityToDosResolutionInsight(models.Model): + snapshot = models.ForeignKey( + DailySnapshot, related_name="data_quality_todos_resolutions", on_delete=models.CASCADE + ) + datasource_id = models.CharField(max_length=100) + month = models.DateField(help_text="The first day of the month for this data point.") + open_count = models.IntegerField(default=0) + resolved_count = models.IntegerField(default=0) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["snapshot", "datasource_id", "month"], + name="unique_snapshot_todos_resolution_datasource_month", + ) + ] + + +@dataclass(frozen=True) +class ChartDefinition: + + id: str + title: str + panel: str + chart_type: str + formatter_fn: Callable[[Dict[str, Any]], Dict[str, Any]] + collect_fn: Optional[Callable] = None + is_per_package: bool = False + has_search: bool = False diff --git a/insights/static/insights/css/insights.css b/insights/static/insights/css/insights.css new file mode 100644 index 000000000..2f3676e17 --- /dev/null +++ b/insights/static/insights/css/insights.css @@ -0,0 +1,330 @@ +/* Palette and CSS variables */ +:root { + --bulma-primary: #00D1B2; + --bulma-link: #3273DC; + --bulma-info: #209CEE; + --bulma-success: #48C774; + --bulma-warning: #FFDD57; + --bulma-danger: #FF3860; + --bulma-orange: #FF470F; + --bulma-purple: #B86BFF; + --bulma-grey: #7A7A7A; + --bulma-primary-dark: #00947e; + --bulma-black: #0A0A0A; + --bulma-white: #FFFFFF; +} + +/* Dashboard layout */ +.insights-layout { + display: flex; + align-items: flex-start; + gap: 1.5rem; + padding: 1.5rem 1rem; +} + +/* Override opacity of scatter plot bubbles */ +.bb-circles circle { + opacity: 0.7 !important; +} + +.insights-sidenav { + width: 220px; + flex-shrink: 0; + position: sticky; + top: 5rem; + height: calc(100vh - 7rem); + display: flex; + flex-direction: column; +} + +.insights-sidenav nav.menu { + flex: 1; + overflow-y: auto; + min-height: 0; + margin-bottom: 1rem; +} + +.insights-sidenav .menu-label { + color: var(--bulma-black); + font-weight: 600; +} + +.insights-sidenav .menu-list { + list-style: none !important; + padding-left: 0; + margin-left: 0; +} + +.insights-sidenav .menu-list li { + margin-bottom: 0.25rem; +} + +.insights-sidenav .menu-list a { + color: var(--bulma-black); + padding-left: 1rem; + border-radius: 0 4px 4px 0; +} + +.insights-sidenav .menu-list a:hover { + background-color: #f6f8fa; + color: var(--bulma-black); +} + +.insights-sidenav .menu-list a.is-active { + background-color: #f0f5fa; + color: var(--bulma-link); + font-weight: 600; + border-left: 3px solid var(--bulma-link); + padding-left: calc(1rem - 3px); +} + +.insights-main { + flex: 1; + min-width: 0; +} + +.chart-inner { + max-width: 860px; + margin-left: auto; + margin-right: auto; +} + +.chart-row { + margin-bottom: 1.25rem; + padding: 1.25rem 1.5rem; + background: var(--bulma-white); +} + +.chart-row h3 { + margin-bottom: 0.75rem; + font-size: 1rem; + font-weight: 600; + color: #2c3e50; +} + + + +/* Chart containers and loading state */ +.bb svg { + width: 100% !important; +} + +.chart-container { + width: 100%; + min-height: 320px; +} + +.chart-container.is-tall { + min-height: 420px; +} + +.chart-loading { + display: flex; + align-items: center; + justify-content: center; + min-height: 200px; + color: var(--bulma-grey); + font-size: 0.9rem; +} + + + +/* Billboard.js tooltip overrides */ +.bb-tooltip-container table.bb-tooltip { + min-width: 260px; + white-space: nowrap; +} + +.bb-tooltip-container table.bb-tooltip th { + padding: 0.4rem 0.8rem !important; +} + +.bb-tooltip-container table.bb-tooltip td { + padding: 0.3rem 0.8rem !important; +} + +.bb-tooltip-container table.bb-tooltip td.value, +.bb-tooltip-container table.bb-tooltip td:last-child { + text-align: right; + font-variant-numeric: tabular-nums; + font-weight: 600; + padding-right: 1.2rem !important; +} + +.sidenav-footer { + margin-top: auto; + padding: 1rem 0; + border-top: 1px solid #e8e8e8; +} + +.snapshot-text { + font-size: 0.75rem; + color: var(--bulma-black); + line-height: 1.4; +} + +.snapshot-text strong { + color: var(--bulma-black); +} + +/* Classes replacing inline styles */ +.panel-section { + background-color: var(--bulma-white); + overflow: hidden; +} + +.flat-panel { + margin-bottom: 0; + box-shadow: none; +} + +.flat-panel-heading { + border-radius: 0; + display: flex; + justify-content: space-between; + align-items: center; +} + +.w-full { + max-width: 100%; +} + +.card-chart-row { + border: 1px solid #e8e8e8; + border-radius: 6px; + box-shadow: 0 1px 2px rgba(0, 0, 0, 0.02); +} + +.card-chart-title { + border-bottom: 1px solid #e8e8e8; + padding-bottom: 0.5rem; + margin-bottom: 0.75rem; +} + +/* Package Panel Specific Styles */ +.chart-dropdown-wrapper { + display: flex; + justify-content: flex-end; + margin-bottom: 8px; +} + +.severity-chart-layout { + display: flex; + align-items: flex-start; + gap: 0; + width: 100%; + font-family: inherit; +} + +.severity-table-wrapper { + flex: 0 0 380px; + min-width: 260px; +} + +.severity-table { + width: 100%; + border-collapse: collapse; + font-size: 13px; +} + +.sev-th-left { + text-align: left; + padding: 4px 8px; + color: #555; + font-weight: 600; + border-bottom: 1px solid #ddd; +} + +.sev-th-right { + text-align: right; + padding: 4px 8px; + color: #555; + font-weight: 600; + border-bottom: 1px solid #ddd; +} + +.sev-tfoot-tr { + border-top: 1px solid #ddd; +} + +.sev-td-total-label { + padding: 5px 8px; + font-weight: 600; + color: #333; +} + +.sev-td-total-value { + padding: 5px 8px; + text-align: right; + font-weight: 600; + color: #2563eb; +} + +.severity-bb-container { + flex: 1; + min-height: 280px; + margin-top: 52px; +} + +.sev-td-bucket { + padding: 4px 8px; + vertical-align: middle; +} + +.sev-bucket-label { + display: inline-block; + width: 2.5em; + color: #444; +} + +.sev-bucket-bar { + display: inline-block; + height: 7px; + border-radius: 2px; + vertical-align: middle; + margin-left: 4px; +} + +.sev-td-count { + padding: 4px 8px; + text-align: right; + color: #2563eb; +} + +.sev-empty { + color: #aaa; +} + +.card-chart-title-with-search { + display: flex; + justify-content: space-between; + align-items: center; +} + +.severity-search-wrapper { + display: flex; + align-items: center; + margin-left: auto; +} + +/* Overview Panel Specific Styles */ +.kpi-card { + border-top: 4px solid var(--bulma-link); + height: 100%; + margin-bottom: 0 !important; +} + +.kpi-card:hover { + transform: translateY(-2px); + transition: transform 0.2s ease-in-out; +} + +.delta-positive { + color: var(--bulma-success); + font-weight: bold; +} + +.delta-negative { + color: var(--bulma-danger); + font-weight: bold; +} \ No newline at end of file diff --git a/insights/static/insights/js/core.js b/insights/static/insights/js/core.js new file mode 100644 index 000000000..3b046d6e9 --- /dev/null +++ b/insights/static/insights/js/core.js @@ -0,0 +1,57 @@ +// +// Copyright (c) nexB Inc. and others. All rights reserved. +// VulnerableCode is a trademark of nexB Inc. +// SPDX-License-Identifier: Apache-2.0 +// See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +// See https://github.com/aboutcode-org/vulnerablecode for support or download. +// See https://aboutcode.org for more information about nexB OSS projects. +// + +import { renderers } from './renderers.js'; + +export function renderChartWithData(chartId, key, chartDataMap = {}) { + // Handle dropdown charts with key + const chartData = key && chartDataMap[key] ? chartDataMap[key] : chartDataMap; + const chartContainer = document.getElementById(`chart-${chartId}`); + if (!chartContainer) return; + + if (!chartData?.columns?.length) { + chartContainer.innerHTML = "

No data available.

"; + return; + } + + const type = chartContainer.dataset.chartType; + if (renderers[type]) renderers[type](chartId, chartData); +} + +export function initDropdownChart(chartId, chartData, defaultLabel) { + const chartContainer = document.getElementById(`chart-${chartId}`); + if (!chartData || !chartContainer || chartContainer.previousElementSibling?.classList.contains("chart-dropdown-wrapper")) return; + + const options = Object.keys(chartData).filter(k => k !== "global"); + if (options.length) { + chartContainer.insertAdjacentHTML("beforebegin", ` +
+
+ +
+
+ `); + chartContainer.previousElementSibling.querySelector("select").addEventListener("change", ev => + renderChartWithData(chartId, ev.target.value, chartData) + ); + } + renderChartWithData(chartId, "global", chartData); +} + +export function initDashboard() { + const layout = document.querySelector(".insights-layout"); + if (!layout) return; + const data = JSON.parse(document.getElementById("insights-snapshot-data")?.textContent || "{}"); + document.dispatchEvent(new CustomEvent('insightsDataLoaded', { detail: { data, snapshotData: data.snapshot_data || {}, panelId: layout.dataset.panel } })); +} + +setTimeout(initDashboard, 0); diff --git a/insights/static/insights/js/data_quality_panel.js b/insights/static/insights/js/data_quality_panel.js new file mode 100644 index 000000000..0ab1e2522 --- /dev/null +++ b/insights/static/insights/js/data_quality_panel.js @@ -0,0 +1,21 @@ +// +// Copyright (c) nexB Inc. and others. All rights reserved. +// VulnerableCode is a trademark of nexB Inc. +// SPDX-License-Identifier: Apache-2.0 +// See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +// See https://github.com/aboutcode-org/vulnerablecode for support or download. +// See https://aboutcode.org for more information about nexB OSS projects. +// + +import { initDropdownChart } from './core.js'; + +document.addEventListener('insightsDataLoaded', (e) => { + if (e.detail.panelId !== "data_quality_panel") return; + + const { snapshotData } = e.detail; + if (!snapshotData) return; + + initDropdownChart("dq-issue-type-bar", snapshotData["dq-issue-type-bar"], "All Datasources"); + initDropdownChart("dq-importer-contribution-donut", snapshotData["dq-importer-contribution-donut"], "All Issue Types"); + initDropdownChart("dq-resolution-timeline", snapshotData["dq-resolution-timeline"], "All Importers"); +}); diff --git a/insights/static/insights/js/importer_panel.js b/insights/static/insights/js/importer_panel.js new file mode 100644 index 000000000..99353f02a --- /dev/null +++ b/insights/static/insights/js/importer_panel.js @@ -0,0 +1,85 @@ +// +// Copyright (c) nexB Inc. and others. All rights reserved. +// VulnerableCode is a trademark of nexB Inc. +// SPDX-License-Identifier: Apache-2.0 +// See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +// See https://github.com/aboutcode-org/vulnerablecode for support or download. +// See https://aboutcode.org for more information about nexB OSS projects. +// + +import { initDropdownChart } from './core.js'; +import { formatWholeNumbersOnly, getCssVar, renderers } from './renderers.js'; + +const names = { + advisories_with_packages: "with a PURL", + advisories_without_packages: "without a PURL", + advisories_without_ghost_packages: "with an existing PURL", + advisories_with_ghost_packages: "with a non-existing PURL", + advisories_with_exploits: "with an Exploit", + advisories_without_exploits: "without an Exploit", + advisories_with_exploitdb: "with a Exploit-DB exploit", + advisories_with_metasploit: "with a Metasploit exploit", + advisories_with_kev: "with a KEV", +}; + +function renderImporterBar(id, config) { + // Get the total count of the importer from config.columns + const totals = config.columns.find((c) => c[0] === "total_advisories"); + + bb.generate({ + bindto: `#chart-${id}`, + data: { + x: "x", + columns: [["x", ...config.x_categories], ...config.columns], + type: "bar", + order: null, + hide: ["total_advisories"], + colors: { + advisories_with_packages: getCssVar("--bulma-primary"), + advisories_without_packages: getCssVar("--bulma-danger"), + advisories_with_exploits: getCssVar("--bulma-primary"), + advisories_without_exploits: getCssVar("--bulma-danger"), + advisories_with_kev: getCssVar("--bulma-warning"), + advisories_with_metasploit: getCssVar("--bulma-purple"), + advisories_with_exploitdb: getCssVar("--bulma-link"), + advisories_with_ghost_packages: getCssVar("--bulma-orange"), + advisories_without_ghost_packages: getCssVar("--bulma-primary-dark"), + }, + names, + groups: config.groups, + }, + axis: { + rotated: true, + x: { type: "category" }, + y: { tick: { format: formatWholeNumbersOnly } }, + }, + bar: { width: { ratio: 0.8 } }, + tooltip: { + grouped: true, + order: (a, b) => { + //Keep a stable order of rows + const cols = config.columns.map(c => c[0]); + return cols.indexOf(a.id) - cols.indexOf(b.id); + }, + format: { + title: (x) => `Advisories from ${config.x_categories[x]} (Total: ${totals?.[x + 1]?.toLocaleString()})`, + value: (val) => val.toLocaleString(), + }, + }, + legend: { + show: true, + hide: ["total_advisories"], + }, + }); +} + +renderers.importer_bar = renderImporterBar; + +document.addEventListener('insightsDataLoaded', (e) => { + if (e.detail.panelId !== "importer_panel") return; + const { snapshotData } = e.detail; + + ["importer-empty-pkg-bar", "importer-exploit-bar"].forEach(id => { + initDropdownChart(id, snapshotData[id], "Top 5 Importers"); + }); +}); diff --git a/insights/static/insights/js/overview_panel.js b/insights/static/insights/js/overview_panel.js new file mode 100644 index 000000000..9e17dc736 --- /dev/null +++ b/insights/static/insights/js/overview_panel.js @@ -0,0 +1,41 @@ +// +// Copyright (c) nexB Inc. and others. All rights reserved. +// VulnerableCode is a trademark of nexB Inc. +// SPDX-License-Identifier: Apache-2.0 +// See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +// See https://github.com/aboutcode-org/vulnerablecode for support or download. +// See https://aboutcode.org for more information about nexB OSS projects. +// + +import { renderChartWithData } from './core.js'; + +function formatDelta(delta) { + if (delta > 0) return `+${delta.toLocaleString()} from last snapshot`; + if (delta < 0) return `${delta.toLocaleString()} from last snapshot`; + return "No change from last snapshot"; +} + +document.addEventListener('insightsDataLoaded', (e) => { + if (e.detail.panelId !== "overview_panel") return; + + const snapshotData = e.detail.snapshotData; + if (!snapshotData || !snapshotData["overview-stats"]) return; + + const overviewData = snapshotData["overview-stats"]; + + const kpis = overviewData.kpis; + if (kpis) { + document.getElementById("kpi-total-advisories").textContent = kpis.total_advisories.toLocaleString(); + document.getElementById("kpi-delta-advisories").innerHTML = formatDelta(kpis.delta_advisories); + + document.getElementById("kpi-total-packages").textContent = kpis.total_packages.toLocaleString(); + document.getElementById("kpi-delta-packages").innerHTML = formatDelta(kpis.delta_packages); + + document.getElementById("kpi-total-sources").textContent = kpis.total_data_sources.toLocaleString(); + document.getElementById("kpi-delta-sources").innerHTML = formatDelta(kpis.delta_data_sources); + } + + renderChartWithData("overview-historical", "global", snapshotData["overview-historical"]); + renderChartWithData("overview-cross-validation", "global", snapshotData["overview-cross-validation"]); + renderChartWithData("overview-growth-trend", "global", snapshotData["overview-growth-trend"]); +}); diff --git a/insights/static/insights/js/package_panel.js b/insights/static/insights/js/package_panel.js new file mode 100644 index 000000000..5c132aa3b --- /dev/null +++ b/insights/static/insights/js/package_panel.js @@ -0,0 +1,28 @@ +// +// Copyright (c) nexB Inc. and others. All rights reserved. +// VulnerableCode is a trademark of nexB Inc. +// SPDX-License-Identifier: Apache-2.0 +// See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +// See https://github.com/aboutcode-org/vulnerablecode for support or download. +// See https://aboutcode.org for more information about nexB OSS projects. +// + +import { renderChartWithData, initDropdownChart } from './core.js'; + +document.addEventListener('insightsDataLoaded', (e) => { + if (e.detail.panelId !== "package_panel") return; + + const { snapshotData } = e.detail; + const searchData = document.getElementById("chart-search-data")?.textContent; + const parsedSearch = searchData ? JSON.parse(searchData) : null; + + initDropdownChart("pkg-name-bar", snapshotData["pkg-name-bar"], "All Packages"); + + renderChartWithData( + "pkg-cwe-bar", + parsedSearch ? null : "global", + parsedSearch?.["pkg-cwe-bar"] || snapshotData["pkg-cwe-bar"] + ); + + renderChartWithData("pkg-dist-donut", "global", snapshotData["pkg-dist-donut"]); +}); diff --git a/insights/static/insights/js/renderers.js b/insights/static/insights/js/renderers.js new file mode 100644 index 000000000..0c2471de8 --- /dev/null +++ b/insights/static/insights/js/renderers.js @@ -0,0 +1,244 @@ +// +// Copyright (c) nexB Inc. and others. All rights reserved. +// VulnerableCode is a trademark of nexB Inc. +// SPDX-License-Identifier: Apache-2.0 +// See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +// See https://github.com/aboutcode-org/vulnerablecode for support or download. +// See https://aboutcode.org for more information about nexB OSS projects. +// + +export const getCssVar = (name) => getComputedStyle(document.documentElement).getPropertyValue(name).trim(); + +const paletteVars = [ + "--bulma-primary", + "--bulma-link", + "--bulma-info", + "--bulma-success", + "--bulma-warning", + "--bulma-danger", + "--bulma-orange", + "--bulma-purple", + "--bulma-grey", + "--bulma-primary-dark", +]; + +export const getPalette = () => paletteVars.map(getCssVar).filter(Boolean); + +const getBucketColors = () => [ + getCssVar("--bulma-success"), getCssVar("--bulma-success"), getCssVar("--bulma-success"), getCssVar("--bulma-success"), // 0-3: Success + getCssVar("--bulma-warning"), getCssVar("--bulma-warning"), getCssVar("--bulma-warning"), // 4-6: Warning + getCssVar("--bulma-orange"), // 7: Orange + getCssVar("--bulma-danger"), getCssVar("--bulma-danger") // 8-9: Danger +]; + +export const formatWholeNumbersOnly = (x) => (Number.isInteger(x) ? x : ""); + +export const renderers = { + donut(id, config) { + const bbConfig = { + bindto: `#chart-${id}`, + data: { columns: config.columns, type: "donut", colors: { "Others": "#1b1b1b3a" } }, + color: { pattern: getPalette() }, + legend: { show: true } + }; + + if (config.others_list?.length) { + bbConfig.tooltip = { + contents(d, defaultTitle, defaultVal, color) { + if (d[0].id !== "Others") return this.internal.getTooltipContent(d, defaultTitle, defaultVal, color); + + //Customize tooltip to show table for Others + const total = config.columns.reduce((sum, col) => sum + col[1], 0); + let html = ""; + config.others_list.forEach(([name, val]) => { + html += ``; + }); + return html + "
Others
${name}${val.toLocaleString()} (${((val / total) * 100).toFixed(1)}%)
"; + } + }; + } + bb.generate(bbConfig); + }, + + colored_bar(id, config) { + const monoColor = config.color || getCssVar("--bulma-link"); + const isRotated = config.rotated !== undefined ? config.rotated : true; + bb.generate({ + bindto: `#chart-${id}`, + padding: { right: 30 }, + data: { x: "x", columns: config.columns, type: "bar", color: () => monoColor }, + axis: { + rotated: isRotated, + x: { + type: "category", + label: { text: config.x_label || "CWE", position: isRotated ? "outer-middle" : "outer-center" }, + tick: { + culling: isRotated ? false : { max: config.x_tick_culling !== undefined ? config.x_tick_culling : 8 }, + multiline: false + } + }, + y: { + label: { text: config.y_label || "Advisories", position: isRotated ? "outer-center" : "outer-middle" }, + tick: { format: formatWholeNumbersOnly } + } + }, + tooltip: { format: { title: x => config.full_labels?.[x] || config.columns[0][x + 1], value: val => val.toLocaleString() } }, + legend: { show: false } + }); + }, + + stacked_bar(id, config) { + bb.generate({ + bindto: `#chart-${id}`, + data: { + x: "x", + columns: config.columns, + type: "bar", + groups: config.groups, + colors: { + "VulnerableCode": getCssVar("--bulma-link"), + }, + }, + color: { pattern: getPalette() }, + axis: { + rotated: true, + x: { type: "category" }, + y: { + label: { text: config.y_label || "Advisories", position: "outer-center" }, + tick: { count: 6, format: formatWholeNumbersOnly }, + }, + }, + tooltip: { + contents(dataPoints, defaultTitle, defaultVal, color) { + const nonZeroPoints = dataPoints.filter((point) => point.value > 0); + if (!nonZeroPoints.length) return ""; + return this.internal.getTooltipContent(nonZeroPoints, defaultTitle, defaultVal, color); + }, + format: { value: (val) => val.toLocaleString() }, + }, + }); + }, + + + line(id, config) { + const isMultiSeries = config.columns.length > 2; + const color = config.color || getCssVar("--bulma-primary"); + + bb.generate({ + bindto: `#chart-${id}`, + data: { + x: "x", + columns: config.columns, + type: "line", + colors: { + "Open": getCssVar("--bulma-danger"), + "Resolved": getCssVar("--bulma-primary"), + "Total Advisories": color, + "Advisories Ingested": color, + }, + }, + axis: { + x: { + type: "timeseries", + tick: { + format: isMultiSeries ? "%Y-%m" : "%b %d", + fit: true, + count: 6, + }, + }, + y: { + min: 0, + padding: { bottom: 0 }, + label: { text: config.y_label || "Count", position: "outer-middle" }, + tick: { format: formatWholeNumbersOnly }, + }, + }, + point: { r: 3, focus: { expand: { r: 5 } } }, + legend: { show: isMultiSeries }, + tooltip: { + format: { + value(val, ratio, id, index) { + const added = + id === "Open" + ? config.new_open_counts?.[index] + : config.new_resolved_counts?.[index]; + return added !== undefined + ? `${val.toLocaleString()} (+${added.toLocaleString()} new)` + : val.toLocaleString(); + }, + }, + }, + }); + }, + + scatter(id, config) { + const [, ...buckets] = config.columns[0]; + const [, ...counts] = config.columns[1]; + const BUCKET_COLORS = getBucketColors(); + + const rows = document.getElementById(`chart-${id}-rows`); + const total = document.getElementById(`chart-${id}-total`); + const rowTemplate = document.getElementById(`chart-${id}-row-template`); + + if (!rows || !total || !rowTemplate) return; + + const maxCount = Math.max(1, ...counts); + const frag = document.createDocumentFragment(); + + // Build the table rows for each CVSS bucket + buckets.forEach((bucket, i) => { + const count = counts[i]; + const clone = rowTemplate.content.cloneNode(true); + + // Set bucket label (e.g. "9-10") + const labelNode = clone.querySelector('.sev-bucket-label'); + labelNode.textContent = bucket; + + // Calculate size and paint the colored bubble + const barNode = clone.querySelector('.sev-bucket-bar'); + const barWidth = Math.max(2, (count / maxCount) * 140); + barNode.style.width = `${barWidth}px`; + barNode.style.background = BUCKET_COLORS[i]; + + // Display the vulnerability count on tooltip + const countNode = clone.querySelector('.sev-td-count'); + countNode.title = `CVSS ${bucket}: ${count.toLocaleString()}`; + countNode.innerHTML = count.toLocaleString(); + + frag.appendChild(clone); + }); + + // Render the completed table to the DOM + rows.innerHTML = ""; + rows.appendChild(frag); + const sumOfCounts = counts.reduce((sum, count) => sum + count, 0); + total.textContent = sumOfCounts.toLocaleString(); + + bb.generate({ + bindto: `#chart-${id}-bb`, + data: { + x: "x", columns: config.columns, type: "bubble", + color: (defaultColor, dataPoint) => dataPoint.x !== undefined ? getBucketColors()[dataPoint.x] || defaultColor : defaultColor, + labels: false + }, + bubble: { maxR: 40 }, + axis: { + x: { + type: "category", + tick: { multiline: false }, + label: { text: "CVSS Score Range", position: "outer-center" } + }, + y: { show: false, min: 0, max: maxCount * 1.2, padding: { top: 70, bottom: 0 } } + }, + grid: { x: { show: true } }, + legend: { show: false }, + tooltip: { + format: { + title: x => `CVSS ${buckets[x]}`, + name: () => "Advisories", + value: val => val.toLocaleString() + } + } + }); + } +}; diff --git a/insights/static/insights/js/severity_panel.js b/insights/static/insights/js/severity_panel.js new file mode 100644 index 000000000..d5321a68d --- /dev/null +++ b/insights/static/insights/js/severity_panel.js @@ -0,0 +1,23 @@ +// +// Copyright (c) nexB Inc. and others. All rights reserved. +// VulnerableCode is a trademark of nexB Inc. +// SPDX-License-Identifier: Apache-2.0 +// See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +// See https://github.com/aboutcode-org/vulnerablecode for support or download. +// See https://aboutcode.org for more information about nexB OSS projects. +// + +import { renderChartWithData } from './core.js'; + +document.addEventListener('insightsDataLoaded', (e) => { + if (e.detail.panelId !== "severity_panel") return; + + const chartId = "severity-scatter-plot"; + const searchData = document.getElementById("chart-search-data")?.textContent; + + if (searchData) { + renderChartWithData(chartId, null, JSON.parse(searchData)); + } else { + renderChartWithData(chartId, "global", e.detail.snapshotData[chartId]); + } +}); diff --git a/insights/templates/insights/components/chart_card.html b/insights/templates/insights/components/chart_card.html new file mode 100644 index 000000000..c57515023 --- /dev/null +++ b/insights/templates/insights/components/chart_card.html @@ -0,0 +1,45 @@ +
+

+ {{ chart.title }} + {% if chart.has_search %} +
+ {% if not search_query %} + Showing All Packages + {% else %} + Show All Packages + {% endif %} + +
+
+
+ +
+
+ +
+
+ {% if search_error %} +

{{ search_error }}

+ {% endif %} +
+
+ {% endif %} +

+ + + {% if chart.chart_type == 'line' %} +
+
+ {% endif %} +
diff --git a/insights/templates/insights/components/data_quality_panel_content.html b/insights/templates/insights/components/data_quality_panel_content.html new file mode 100644 index 000000000..0c1f56b32 --- /dev/null +++ b/insights/templates/insights/components/data_quality_panel_content.html @@ -0,0 +1,16 @@ +
+
+ {% with chart=panel.charts.0 %} + {% include 'insights/components/chart_card.html' %} + {% endwith %} + {% with chart=panel.charts.1 %} + {% include 'insights/components/chart_card.html' %} + {% endwith %} +
+ +
+ {% with chart=panel.charts.2 %} + {% include 'insights/components/chart_card.html' %} + {% endwith %} +
+
diff --git a/insights/templates/insights/components/overview_panel_content.html b/insights/templates/insights/components/overview_panel_content.html new file mode 100644 index 000000000..b2c3e7b88 --- /dev/null +++ b/insights/templates/insights/components/overview_panel_content.html @@ -0,0 +1,48 @@ +
+ +
+
+
+

Total Active Advisories

+

--

+

--

+
+
+
+
+

Total Packages

+

--

+

--

+
+
+
+
+

Data Sources

+

--

+

--

+
+
+
+ +
+
+
+
+
+

{{ panel.charts.3.title }}

+
+
+
+

{{ panel.charts.2.title }}

+
+
+
+
+
+
+ {% with chart=panel.charts.1 %} + {% include 'insights/components/chart_card.html' %} + {% endwith %} +
+
+
diff --git a/insights/templates/insights/components/package_panel_donuts.html b/insights/templates/insights/components/package_panel_donuts.html new file mode 100644 index 000000000..ccfae67fd --- /dev/null +++ b/insights/templates/insights/components/package_panel_donuts.html @@ -0,0 +1,12 @@ +
+
+
+

{{ panel.charts.0.title }}

+
+
+
+

{{ panel.charts.1.title }}

+
+
+
+
diff --git a/insights/templates/insights/components/severity_scatter_chart.html b/insights/templates/insights/components/severity_scatter_chart.html new file mode 100644 index 000000000..a2924f96e --- /dev/null +++ b/insights/templates/insights/components/severity_scatter_chart.html @@ -0,0 +1,30 @@ +
+
+ + + + + + + + + + + + + + + +
CVSS Score RangeAdvisories
Total
+ +
+
+
diff --git a/insights/templates/insights/dashboard.html b/insights/templates/insights/dashboard.html new file mode 100644 index 000000000..5c6d5e29d --- /dev/null +++ b/insights/templates/insights/dashboard.html @@ -0,0 +1,80 @@ +{% extends "base.html" %} +{% load static %} + +{% block title %}Insights{% endblock %} + +{% block extrahead %} + + +{% endblock %} + +{% block content %} +
+ + + +
+ {% for panel in panels %} + {% if panel.id == current_panel_id %} +
+
+
+ {{ panel.label }} +
+
+ +
+ {% if panel.id == 'overview_panel' %} + {% include 'insights/components/overview_panel_content.html' with panel=panel %} + {% elif panel.id == 'data_quality_panel' %} + {% include 'insights/components/data_quality_panel_content.html' with panel=panel %} + {% elif panel.layout == 'split_top' %} + {% include 'insights/components/package_panel_donuts.html' with panel=panel %} + {% for chart in panel.charts|slice:"2:" %} + {% include 'insights/components/chart_card.html' with chart=chart current_panel_id=current_panel_id search_query=search_query search_error=search_error %} + {% endfor %} + {% else %} + {% for chart in panel.charts %} + {% include 'insights/components/chart_card.html' with chart=chart current_panel_id=current_panel_id search_query=search_query search_error=search_error %} + {% endfor %} + {% endif %} +
+
+ {% endif %} + {% endfor %} +
+
+{% endblock %} + +{% block scripts %} + + +{{ snapshot_dict|json_script:"insights-snapshot-data" }} +{% if search_results_dict %} +{{ search_results_dict|json_script:"chart-search-data" }} +{% endif %} + + + + + + +{% endblock %} \ No newline at end of file diff --git a/insights/tests/test_data_quality_panel.py b/insights/tests/test_data_quality_panel.py new file mode 100644 index 000000000..9c8854403 --- /dev/null +++ b/insights/tests/test_data_quality_panel.py @@ -0,0 +1,92 @@ +import datetime + +from django.test import TestCase +from django.utils import timezone + +from insights.charts.data_quality_panel import data_quality_todos_resolutions_queryset +from insights.charts.data_quality_panel import open_issues_to_datasource_queryset +from vulnerabilities.models import AdvisoryToDoV2 +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import ToDoRelatedAdvisoryV2 + + +class TestDataQualityPanelQuerysets(TestCase): + def setUp(self): + self.adv1 = AdvisoryV2.objects.create( + avid="github_osv/GHSA-1", + datasource_id="github_osv", + unique_content_id="1", + is_latest=True, + pipeline_id="github_osv_pipeline", + advisory_id="GHSA-1", + url="https://example.com/GHSA-1", + ) + self.adv2 = AdvisoryV2.objects.create( + avid="nvd/CVE-2023-1234", + datasource_id="nvd", + unique_content_id="2", + is_latest=True, + pipeline_id="nvd_pipeline", + advisory_id="CVE-2023-1234", + url="https://nvd.nist.gov/vuln/detail/CVE-2023-1234", + ) + + now = timezone.now() + + self.todo1 = AdvisoryToDoV2.objects.create( + alias="GHSA-1-TODO", + related_advisories_id="hash1", + issue_type="MISSING_AFFECTED_PACKAGE", + is_resolved=False, + created_at=now - datetime.timedelta(days=60), + ) + ToDoRelatedAdvisoryV2.objects.create(todo=self.todo1, advisory=self.adv1) + + self.todo2 = AdvisoryToDoV2.objects.create( + alias="CVE-2023-1234-TODO", + related_advisories_id="hash2", + issue_type="CONFLICTING_SEVERITY_SCORES", + is_resolved=False, + created_at=now - datetime.timedelta(days=45), + ) + ToDoRelatedAdvisoryV2.objects.create(todo=self.todo2, advisory=self.adv2) + + self.todo3 = AdvisoryToDoV2.objects.create( + alias="GHSA-1-TODO-2", + related_advisories_id="hash3", + issue_type="CONFLICTING_SEVERITY_SCORES", + is_resolved=True, + created_at=now - datetime.timedelta(days=60), + resolved_at=now - datetime.timedelta(days=15), + ) + ToDoRelatedAdvisoryV2.objects.create(todo=self.todo3, advisory=self.adv1) + + def test_open_issues_to_datasource_queryset(self): + """Test open issue query correctly aggregates unresolved to-dos by type and source.""" + qs = list(open_issues_to_datasource_queryset()) + + self.assertEqual(len(qs), 2) + + missing_pkg_todo = next(t for t in qs if t["issue_type"] == "MISSING_AFFECTED_PACKAGE") + self.assertEqual(missing_pkg_todo["advisories__datasource_id"], "github_osv") + self.assertEqual(missing_pkg_todo["count"], 1) + + conflicting_sev_todo = next( + t for t in qs if t["issue_type"] == "CONFLICTING_SEVERITY_SCORES" + ) + self.assertEqual(conflicting_sev_todo["advisories__datasource_id"], "nvd") + self.assertEqual(conflicting_sev_todo["count"], 1) + + def test_data_quality_todos_resolutions_queryset(self): + """Test timeline query separately aggregates opened and resolved to-dos by month.""" + open_todos_qs, resolved_todos_qs = data_quality_todos_resolutions_queryset() + + open_todos = list(open_todos_qs) + resolved_todos = list(resolved_todos_qs) + + self.assertEqual(sum(t["count"] for t in open_todos), 3) + self.assertEqual(sum(t["count"] for t in resolved_todos), 1) + + resolved = resolved_todos[0] + self.assertEqual(resolved["advisories__datasource_id"], "github_osv") + self.assertEqual(resolved["count"], 1) diff --git a/insights/tests/test_importer_panel.py b/insights/tests/test_importer_panel.py new file mode 100644 index 000000000..23a063404 --- /dev/null +++ b/insights/tests/test_importer_panel.py @@ -0,0 +1,68 @@ +from django.test import TestCase + +from insights.charts.importer_panel import exploits_queryset +from insights.charts.importer_panel import packages_queryset +from vulnerabilities.models import AdvisoryExploit +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import ImpactedPackage +from vulnerabilities.models import ImpactedPackageAffecting +from vulnerabilities.models import PackageV2 + + +def create_adv(avid, unique_id): + return AdvisoryV2.objects.create( + avid=f"github_osv/{avid}", + datasource_id="github_osv", + unique_content_id=unique_id, + is_latest=True, + pipeline_id="github_osv_pipeline", + advisory_id=avid, + url="https://example.com/" + avid, + ) + + +class TestImporterPanelQuerysets(TestCase): + def test_packages_queryset(self): + advisory_with_package = create_adv("GHSA-1", "1") + pkg1 = PackageV2.objects.create(type="npm", name="test1", version="1.0.0") + impact1 = ImpactedPackage.objects.create(advisory=advisory_with_package) + ImpactedPackageAffecting.objects.create(impacted_package=impact1, package=pkg1) + + advisory_with_ghost_package = create_adv("GHSA-2", "2") + pkg2 = PackageV2.objects.create(type="npm", name="test2", version="2.0.0", is_ghost=True) + impact2 = ImpactedPackage.objects.create(advisory=advisory_with_ghost_package) + ImpactedPackageAffecting.objects.create(impacted_package=impact2, package=pkg2) + + advisory_without_package = create_adv("GHSA-3", "3") + + qs = list(packages_queryset()) + + self.assertEqual(len(qs), 1) + stats = qs[0] + self.assertEqual(stats["datasource_id"], "github_osv") + self.assertEqual(stats["total_advisories"], 3) + self.assertEqual(stats["advisories_with_packages"], 2) # First two have packages + self.assertEqual(stats["advisories_with_ghost_packages"], 1) + + def test_exploits_queryset(self): + advisory_with_kev = create_adv("GHSA-1", "1") + AdvisoryExploit.objects.create(advisory=advisory_with_kev, data_source="KEV") + + advisory_with_metasploit = create_adv("GHSA-2", "2") + AdvisoryExploit.objects.create(advisory=advisory_with_metasploit, data_source="Metasploit") + + advisory_with_exploitdb = create_adv("GHSA-3", "3") + AdvisoryExploit.objects.create(advisory=advisory_with_exploitdb, data_source="Exploit-DB") + + advisory_without_exploits = create_adv("GHSA-4", "4") + + qs = list(exploits_queryset()) + + self.assertEqual(len(qs), 1) + stats = qs[0] + self.assertEqual(stats["datasource_id"], "github_osv") + + # GHSA-4 shouldn't count in these + self.assertEqual(stats["advisories_with_kev"], 1) + self.assertEqual(stats["advisories_with_metasploit"], 1) + self.assertEqual(stats["advisories_with_exploitdb"], 1) diff --git a/insights/tests/test_insights_snapshot_pipeline.py b/insights/tests/test_insights_snapshot_pipeline.py new file mode 100644 index 000000000..0efda828d --- /dev/null +++ b/insights/tests/test_insights_snapshot_pipeline.py @@ -0,0 +1,60 @@ +from django.test import TestCase +from django.utils import timezone + +from insights.insights_snapshot_pipeline import InsightsSnapshotPipeline +from insights.models import DailySnapshot +from insights.tests.test_importer_panel import create_adv +from vulnerabilities.models import AdvisorySeverity +from vulnerabilities.models import AdvisoryToDoV2 +from vulnerabilities.models import PackageV2 +from vulnerabilities.models import ToDoRelatedAdvisoryV2 + + +class TestInsightsSnapshotPipeline(TestCase): + def test_pipeline_execution_with_data(self): + """Test pipeline populates insights based on existing data.""" + + PackageV2.objects.create(type="pypi", name="django", version="1.0.0") + PackageV2.objects.create(type="npm", name="lodash", version="1.0.0") + + # Create Advisory for Importer and Severity charts + advisory_1 = create_adv("GHSA-1234", "1") + advisory_1.date_published = timezone.now() + advisory_1.save() + severity_1 = AdvisorySeverity.objects.create(scoring_system="cvssv3.1", value="9.8") + advisory_1.severities.add(severity_1) + + todo = AdvisoryToDoV2.objects.create( + alias="GHSA-1234-TODO", + related_advisories_id="hash1", + issue_type="MISSING_AFFECTED_PACKAGE", + is_resolved=False, + created_at=timezone.now(), + ) + ToDoRelatedAdvisoryV2.objects.create(todo=todo, advisory=advisory_1) + + pipeline = InsightsSnapshotPipeline() + pipeline.execute() + + self.assertEqual(DailySnapshot.objects.count(), 1) + snapshot = DailySnapshot.objects.first() + + # Verify captured package types (pypi, npm) and 'global' + self.assertEqual(snapshot.package_insights.count(), 3) + + # Verify ImporterInsight was created for 'github_osv' + self.assertEqual(snapshot.importer_insights.count(), 1) + self.assertTrue(snapshot.importer_insights.filter(importer="github_osv").exists()) + + # Verify SeverityInsight was generated + self.assertTrue(hasattr(snapshot, "severity_insight")) + self.assertEqual(snapshot.severity_insight.buckets[9], 1) + + self.assertTrue(hasattr(snapshot, "overview")) + self.assertEqual(snapshot.overview.total_advisories, 1) + self.assertEqual(len(snapshot.overview.last_30days), 31) + self.assertEqual(snapshot.overview.yearly_insights.count(), 1) + + self.assertEqual(snapshot.data_quality_issue_types.count(), 1) + self.assertEqual(snapshot.data_quality_issue_types.first().datasource_id, "github_osv") + self.assertEqual(snapshot.data_quality_todos_resolutions.count(), 1) diff --git a/insights/tests/test_overview_panel.py b/insights/tests/test_overview_panel.py new file mode 100644 index 000000000..b054a65d1 --- /dev/null +++ b/insights/tests/test_overview_panel.py @@ -0,0 +1,71 @@ +import datetime + +from django.test import TestCase +from django.utils import timezone + +from insights.charts.overview_panel import daily_growth_counts +from insights.charts.overview_panel import kpi_card_queryset +from insights.charts.overview_panel import yearly_distribution_queryset +from insights.tests.test_importer_panel import create_adv +from vulnerabilities.models import PackageV2 + + +class TestOverviewPanelQuerysets(TestCase): + def setUp(self): + PackageV2.objects.create(type="pypi", name="django", version="1.0.0") + PackageV2.objects.create(type="npm", name="lodash", version="1.0.0") + + self.now = timezone.now() + + adv1 = create_adv("GHSA-1234", "1") + adv1.date_published = self.now - datetime.timedelta(days=365 * 2) + adv1.date_collected = self.now - datetime.timedelta(days=5) + adv1.datasource_id = "github_osv" + adv1.save() + + adv2 = create_adv("GHSA-5678", "2") + adv2.date_published = self.now + adv2.date_collected = self.now + adv2.datasource_id = "github_osv" + adv2.save() + + adv3 = create_adv("CVE-2023-7777", "3") + adv3.date_published = self.now - datetime.timedelta(days=10) + adv3.date_collected = self.now + adv3.datasource_id = "nvd" + adv3.save() + + adv4 = create_adv("CVE-2024-9999", "4") + adv4.date_published = self.now + adv4.date_collected = self.now + adv4.datasource_id = "nvd" + adv4.save() + + adv5 = create_adv("CVE-2024-8888", "5") + adv5.date_published = self.now + adv5.date_collected = self.now + adv5.datasource_id = "nvd" + adv5.save() + + def test_kpi_card_queryset(self): + """Test KPI queries""" + stats = kpi_card_queryset() + self.assertEqual(stats["total_advisories"], 5) + self.assertEqual(stats["total_packages"], 2) + self.assertEqual(stats["total_data_sources"], 2) + + def test_yearly_distribution_queryset(self): + """Test yearly distribution query aggregates advisories by publish year.""" + yearly_records = list(yearly_distribution_queryset()) + self.assertEqual(len(yearly_records), 2) + + this_year = next(y for y in yearly_records if y["year"] == self.now.year) + two_years_ago = next(y for y in yearly_records if y["year"] == self.now.year - 2) + + self.assertEqual(this_year["count"], 4) + self.assertEqual(two_years_ago["count"], 1) + + def test_daily_growth_counts(self): + """Test daily ingestion query""" + counts = daily_growth_counts(self.now) + self.assertEqual(sum(counts), 5) diff --git a/insights/tests/test_package_panel.py b/insights/tests/test_package_panel.py new file mode 100644 index 000000000..5bae820bb --- /dev/null +++ b/insights/tests/test_package_panel.py @@ -0,0 +1,56 @@ +from django.test import TestCase + +from insights.charts.package_panel import ecosystem_distribution_queryset +from insights.charts.package_panel import top_packages_queryset +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import ImpactedPackage +from vulnerabilities.models import ImpactedPackageAffecting +from vulnerabilities.models import PackageV2 + + +class TestPackagePanelQuerysets(TestCase): + def test_ecosystem_distribution_queryset(self): + PackageV2.objects.create(type="npm", name="test1", version="1.0.0") + PackageV2.objects.create(type="npm", name="test2", version="1.0.0") + PackageV2.objects.create(type="pypi", name="test3", version="1.0.0") + PackageV2.objects.create(type="gem", name="test4", version="1.0.0") + + tracked_types = ["npm", "pypi"] + + qs = list(ecosystem_distribution_queryset(tracked_types)) + stats = {item["type"]: item["total_package_count"] for item in qs} + + self.assertEqual(len(stats), 2) # gem should be excluded + self.assertEqual(stats["npm"], 2) + self.assertEqual(stats["pypi"], 1) + + def test_top_packages_queryset(self): + django_v1 = PackageV2.objects.create(type="pypi", name="django", version="1.0.0") + django_v2 = PackageV2.objects.create(type="pypi", name="django", version="2.0.0") + flask_v1 = PackageV2.objects.create(type="pypi", name="flask", version="1.0.0") + + def create_impact(advisory_id, package): + adv = AdvisoryV2.objects.create( + avid=f"github_osv/{advisory_id}", + datasource_id="github_osv", + unique_content_id=advisory_id, + is_latest=True, + pipeline_id="github_osv_pipeline", + advisory_id=advisory_id, + url=f"https://github.com/advisories/{advisory_id}", + ) + impact = ImpactedPackage.objects.create(advisory=adv) + ImpactedPackageAffecting.objects.create(impacted_package=impact, package=package) + + create_impact("GHSA-1111", django_v1) + create_impact("GHSA-2222", django_v2) + create_impact("GHSA-3333", django_v2) + create_impact("GHSA-4444", flask_v1) + + qs = list(top_packages_queryset("pypi")) + + self.assertEqual(len(qs), 2) + self.assertEqual(qs[0]["name"], "django") + self.assertEqual(qs[0]["count"], 3) + self.assertEqual(qs[1]["name"], "flask") + self.assertEqual(qs[1]["count"], 1) diff --git a/insights/tests/test_severity_panel.py b/insights/tests/test_severity_panel.py new file mode 100644 index 000000000..cbe8e8291 --- /dev/null +++ b/insights/tests/test_severity_panel.py @@ -0,0 +1,41 @@ +from django.test import TestCase + +from insights.charts.severity_panel import iter_severity_insights +from vulnerabilities.models import AdvisorySeverity +from vulnerabilities.models import AdvisoryV2 + + +class TestSeverityPanelQuerysets(TestCase): + def test_iter_severity_insights(self): + advisory1 = AdvisoryV2.objects.create( + avid="github_osv/GHSA-1", + datasource_id="github_osv", + unique_content_id="1", + is_latest=True, + pipeline_id="github_osv_pipeline", + advisory_id="GHSA-1", + url="https://example.com/GHSA-1", + ) + severity1 = AdvisorySeverity.objects.create(scoring_system="cvssv3.1", value="9.8") + advisory1.severities.add(severity1) + + advisory2 = AdvisoryV2.objects.create( + avid="github_osv/GHSA-2", + datasource_id="github_osv", + unique_content_id="2", + is_latest=True, + pipeline_id="github_osv_pipeline", + advisory_id="GHSA-2", + url="https://example.com/GHSA-2", + ) + severity2 = AdvisorySeverity.objects.create(scoring_system="cvssv3.1", value="4.5") + advisory2.severities.add(severity2) + + qs = list(iter_severity_insights()) + + self.assertEqual(len(qs), 1) + insight = qs[0] + + self.assertEqual(insight.buckets[9], 1) + self.assertEqual(insight.buckets[4], 1) + self.assertEqual(sum(insight.buckets), 2) diff --git a/insights/tests/test_views.py b/insights/tests/test_views.py new file mode 100644 index 000000000..85e6eedb6 --- /dev/null +++ b/insights/tests/test_views.py @@ -0,0 +1,45 @@ +from django.test import TestCase +from django.urls import reverse + +from insights.models import DailySnapshot +from vulnerabilities.models import PackageV2 + + +class TestInsightsViews(TestCase): + def test_dashboard_redirects_to_default_panel(self): + """Root insights URL should redirect to the first panel (overview_panel).""" + response = self.client.get("/insights/") + self.assertEqual(response.status_code, 302) + self.assertTrue(response.url.endswith("/insights/overview_panel/")) + + def test_dashboard_invalid_panel_redirects(self): + """Invalid panel ID should redirect to default.""" + response = self.client.get("/insights/non_existent_panel/") + self.assertEqual(response.status_code, 302) + self.assertTrue(response.url.endswith("/insights/overview_panel/")) + + def test_dashboard_valid_panel_loads(self): + """Valid panels should load with a 200 OK.""" + response = self.client.get("/insights/package_panel/") + self.assertEqual(response.status_code, 200) + self.assertContains(response, "package_panel") + + def test_dashboard_search_package_panel(self): + """Test CWE search on package panel.""" + PackageV2.objects.create(type="pypi", name="django", version="1.0.0") + response = self.client.get("/insights/package_panel/?q=pkg:pypi/django") + self.assertEqual(response.status_code, 200) + self.assertIn("search_results_dict", response.context) + + def test_dashboard_search_severity_panel(self): + """Test Severity search on severity panel.""" + PackageV2.objects.create(type="pypi", name="flask", version="1.0.0") + response = self.client.get("/insights/severity_panel/?q=pkg:pypi/flask") + self.assertEqual(response.status_code, 200) + self.assertIn("search_results_dict", response.context) + + def test_dashboard_search_no_results(self): + """Test search with no matching packages.""" + response = self.client.get("/insights/severity_panel/?q=pkg:pypi/does_not_exist") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.context["search_error"], "No data available for this query.") diff --git a/insights/urls.py b/insights/urls.py new file mode 100644 index 000000000..0b7a9c4e5 --- /dev/null +++ b/insights/urls.py @@ -0,0 +1,17 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from django.urls import path + +from insights import views + +urlpatterns = [ + path("", views.insights_dashboard, name="insights-dashboard"), + path("/", views.insights_dashboard, name="insights-panel"), +] diff --git a/insights/utils.py b/insights/utils.py new file mode 100644 index 000000000..30ea807c8 --- /dev/null +++ b/insights/utils.py @@ -0,0 +1,35 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from vulnerabilities.models import Weakness + + +def get_cwe_label(cwe_id: str) -> str: + num = str(cwe_id).replace("CWE-", "") + + try: + name = Weakness().get_cwe(num).name + return f"CWE-{num} · {name}" if name else f"CWE-{num}" + except KeyError: + return f"CWE-{num}" + + +def format_importer_name(name: str) -> str: + """ + Format importer internal names into human-readable labels. + e.g., 'pysec_importer_v2' -> 'Pysec Importer' + """ + parts = name.rsplit("_v", 1) + if len(parts) == 2 and parts[1].isdigit(): + name = parts[0] + return name.replace("_", " ") + + +def format_issue_type_label(issue_type: str) -> str: + """Format issue types labels for UI""" + return issue_type.replace("_", " ").lower() diff --git a/insights/views.py b/insights/views.py new file mode 100644 index 000000000..8d01c4fca --- /dev/null +++ b/insights/views.py @@ -0,0 +1,113 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from django.db.models import Count +from django.shortcuts import redirect +from django.shortcuts import render + +from insights.charts import CHARTS +from insights.charts import PANEL_LABELS +from insights.charts import PANEL_LAYOUTS +from insights.charts import PANELS +from insights.charts.package_panel import build_cwe_chart_columns +from insights.charts.package_panel import compute_top_cwes +from insights.charts.severity_panel import aggregate_severity_buckets +from insights.charts.severity_panel import format_severity_chart_data +from insights.models import DailySnapshot +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import PackageV2 + + +def _severity_search(search_query): + """Executes a severity search for a given PUrl""" + packages = PackageV2.objects.search(search_query) + error = None + + if packages.exists(): + severities = ( + AdvisoryV2.objects.filter( + impacted_packages__affecting_packages__in=packages, + severities__isnull=False, + severities__scoring_system__icontains="cvss", + ) + .values("severities__value") + .annotate(sev_count=Count("id", distinct=True)) + ) + buckets = aggregate_severity_buckets(severities, "sev_count") + else: + buckets = [0] * 10 + error = "No data available for this query." + + return format_severity_chart_data(buckets), error + + +def _cwe_search(search_query): + """Executes a CWE search for a given PUrl""" + packages = PackageV2.objects.search(search_query) + error = None + + if packages.exists(): + cwe_counts = compute_top_cwes(packages) + chart_data = build_cwe_chart_columns(cwe_counts) + else: + chart_data = {} + error = "No packages found for this query." + + return {"pkg-cwe-bar": chart_data}, error + + +def insights_dashboard(request, panel_id=None): + """Dashboard view to display insights.""" + + if not panel_id or panel_id not in PANELS: + return redirect("insights-panel", panel_id=PANELS[0]) + + panels = [] + for panel_key in PANELS: + panel_charts = [chart for chart in CHARTS if chart.panel == panel_key] + panels.append( + { + "id": panel_key, + "label": PANEL_LABELS[panel_key], + "layout": PANEL_LAYOUTS.get(panel_key, "standard"), + "charts": panel_charts, + } + ) + + try: + snapshot = DailySnapshot.objects.latest() + last_time = snapshot.created_at + + snapshot_data = {} + for chart_def in CHARTS: + if chart_def.formatter_fn: + snapshot_data[chart_def.id] = chart_def.formatter_fn(snapshot) + except DailySnapshot.DoesNotExist: + last_time = None + snapshot_data = {} + + search_query = request.GET.get("q", "").strip() + search_results_dict = None + search_error = None + + if search_query: + if panel_id == "severity_panel": + search_results_dict, search_error = _severity_search(search_query) + elif panel_id == "package_panel": + search_results_dict, search_error = _cwe_search(search_query) + + context = { + "panels": panels, + "current_panel_id": panel_id, + "last_snapshot_time": last_time, + "snapshot_dict": {"snapshot_data": snapshot_data}, + "search_query": search_query, + "search_results_dict": search_results_dict, + "search_error": search_error, + } + return render(request, "insights/dashboard.html", context) diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 51694bde9..aeaa57654 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +from insights.insights_snapshot_pipeline import InsightsSnapshotPipeline from vulnerabilities.pipelines.v2_improvers import archive_urls from vulnerabilities.pipelines.v2_improvers import collect_ssvc_trees from vulnerabilities.pipelines.v2_improvers import compute_advisory_todo as compute_advisory_todo_v2 @@ -43,5 +44,6 @@ enhance_with_github_poc.GithubPocsImproverPipeline, mark_unfurl_version_range.MarkUnfurlVersionRangePipeline, group_advisories_for_packages_v2.GroupAdvisoriesForPackages, + InsightsSnapshotPipeline, ] ) diff --git a/vulnerabilities/templates/navbar.html b/vulnerabilities/templates/navbar.html index 530c2521e..b50cc3fb2 100644 --- a/vulnerabilities/templates/navbar.html +++ b/vulnerabilities/templates/navbar.html @@ -1,7 +1,7 @@ {% load utils %} -{% if STAGING %} +