From 3298268153cfca7b79d6c4862d5eaf05fb4e9fbe Mon Sep 17 00:00:00 2001 From: Warren du Toit Date: Mon, 10 Aug 2026 18:44:46 +0200 Subject: [PATCH] fix: exclude non-Azure controls from confidence/sovereignty, split review UI Three related accuracy issues in the mapping review flow: 1. Average Confidence (and High/Low Confidence counts) averaged in every C_Process/D_MicrosoftAttestation control's confidence_score, which is a fixed 0.0 placeholder - Azure Policy mapping is never attempted for these controls, so 0.0 isn't a low score, it's "not applicable". Mixing them with real A/B confidence judgements dragged a batch that matched its enforceable controls well down to a misleading headline number (e.g. 30% average confidence on a batch where 22 of 27 policy-eligible controls were actually high-confidence). Added coverage.confidence_eligible() / frontend utils/coverage.py and scoped every average-confidence computation (ai_mapping_service.py, validator.py, and all three frontend pages) to A/B (+ legacy unclassified) controls only. 2. C/D controls were showing a full Sovereignty Mapping card (Level: L1, Target Archetype: sovereign_root) even when the AI's own reasoning said "no sovereignty requirement applies" - the prompt asks for a sovereignty verdict on every control, defaulting to L1/sovereign_root with an empty objectives list when none applies. Next to a control that constructs no initiative entry, that default read as a second, contradictory verdict. Added coverage.clear_moot_sovereignty(), which nulls the sovereignty field for C/D controls with no real sovereignty_objectives, while preserving the one legitimate exception (a procedural objective like SO-2 Customer Lockbox, which has no policy but genuine sovereignty relevance). 3. Split Review & Edit into two tabs: "Policy Mappings" (A_AzurePolicy / B_AzureConfig - confidence, sovereignty, Azure Policy candidates, exactly as before) and "Manual Register" (C_Process / D_MicrosoftAttestation - read-only, shows coverage_reason, responsibility, and for D controls the grounded attestation citation or an explicit attestation-gap warning). No Azure Policy initiative entry is constructed for Manual Register controls, so they no longer carry a confidence score or sovereignty verdict that never applied to them. Verified: full backend suite (639 passed) and frontend suite (846 passed, 14 pre-existing failures identical to baseline) both regression-clean. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 322e5e54-ab2e-43b0-956c-5ec01cd955e7 --- app/backend/app/pipeline/validator.py | 31 +- .../app/services/ai_mapping_service.py | 22 +- app/backend/app/services/coverage.py | 61 ++ app/frontend/pages/2_AI_Mapping.py | 38 +- app/frontend/pages/3_Review_Edit.py | 699 +++++++++++------- app/frontend/pages/4_Export_Policy.py | 17 +- app/frontend/utils/coverage.py | 56 ++ 7 files changed, 638 insertions(+), 286 deletions(-) create mode 100644 app/frontend/utils/coverage.py diff --git a/app/backend/app/pipeline/validator.py b/app/backend/app/pipeline/validator.py index bfa7456..ccc577e 100644 --- a/app/backend/app/pipeline/validator.py +++ b/app/backend/app/pipeline/validator.py @@ -7,6 +7,8 @@ import re from typing import Optional +from ..services import coverage + from .models import ( ControlPolicyMapping, ControlExtractionResult, @@ -111,19 +113,26 @@ def validate_mappings( automatable_count = 0 manual_count = 0 confidence_sum = 0.0 + confidence_eligible_count = 0 for mapping in mappings: control_id = mapping.control_id - # Check confidence score - confidence_sum += mapping.confidence_score - if mapping.confidence_score < min_confidence: - issues.append(ValidationIssue( - severity="warning", - control_id=control_id, - message=f"Low confidence mapping ({mapping.confidence_score:.2f} < {min_confidence})", - suggestion="Review this mapping manually before deployment", - )) + # Check confidence score. Scoped to controls where Azure Policy was + # actually attempted: C_Process/D_MicrosoftAttestation mappings carry + # a fixed 0.0 confidence_score placeholder (no match was ever tried), + # so including them here would both drag the average down misleadingly + # and raise a spurious "low confidence" warning on every one of them. + if coverage.confidence_eligible(getattr(mapping, "coverage_category", None)): + confidence_sum += mapping.confidence_score + confidence_eligible_count += 1 + if mapping.confidence_score < min_confidence: + issues.append(ValidationIssue( + severity="warning", + control_id=control_id, + message=f"Low confidence mapping ({mapping.confidence_score:.2f} < {min_confidence})", + suggestion="Review this mapping manually before deployment", + )) # Check Azure Policy IDs if mapping.is_automatable: @@ -179,7 +188,9 @@ def validate_mappings( suggestion="Add guidance on what manual steps or evidence are needed", )) - avg_confidence = confidence_sum / len(mappings) if mappings else 0.0 + avg_confidence = ( + confidence_sum / confidence_eligible_count if confidence_eligible_count else 0.0 + ) error_count = sum(1 for i in issues if i.severity == "error") is_valid = error_count == 0 diff --git a/app/backend/app/services/ai_mapping_service.py b/app/backend/app/services/ai_mapping_service.py index 3023f53..aade4c6 100644 --- a/app/backend/app/services/ai_mapping_service.py +++ b/app/backend/app/services/ai_mapping_service.py @@ -300,6 +300,14 @@ async def map_control( self._apply_procedural_sovereignty(mapping, external_control) + # A process/attestation control that carries no real sovereignty + # objective should not show the model's default L1/sovereign_root + # placeholder as if it were a genuine sovereignty verdict - see + # clear_moot_sovereignty for why. Must run after apply_coverage + # (needs coverage_category) and after the procedural sovereignty + # step above (which may have attached a real objective to keep). + coverage.clear_moot_sovereignty(mapping) + # policy_category is a resolvable fact about the real catalog # entries selected, not a judgement call, so it is always computed # here rather than asked of the model - same rationale as @@ -415,9 +423,19 @@ async def map_one(control: ExternalControl) -> tuple[str, Optional[ControlMappin mappings.append(mapping) mapped_count = len(mappings) + # Scope the average to controls where confidence is a real signal. + # C_Process/D_MicrosoftAttestation mappings never attempt an Azure + # Policy match, so their confidence_score is a fixed 0.0 placeholder - + # averaging it in with A/B controls' genuine scores drags a batch that + # matched its enforceable controls well down to a misleadingly low + # headline number. See coverage.confidence_eligible. + confidence_scoped = [ + m for m in mappings + if coverage.confidence_eligible(getattr(m, "coverage_category", None)) + ] avg_confidence = ( - sum(m.confidence_score for m in mappings) / mapped_count - if mapped_count > 0 else 0.0 + sum(m.confidence_score for m in confidence_scoped) / len(confidence_scoped) + if confidence_scoped else 0.0 ) summary = self._generate_summary(total_controls, mapped_count, avg_confidence) diff --git a/app/backend/app/services/coverage.py b/app/backend/app/services/coverage.py index 26fd3a2..bf22859 100644 --- a/app/backend/app/services/coverage.py +++ b/app/backend/app/services/coverage.py @@ -72,6 +72,33 @@ # so stripping its policies deletes real enforcement the customer is entitled to. POLICY_BEARING_CATEGORIES = frozenset({COVERAGE_A, COVERAGE_B}) +# Categories for which confidence_score is never a graded match-quality signal. +# C_Process and D_MicrosoftAttestation controls never attempt an Azure Policy +# match - confidence_score is forced to 0.0 as a placeholder (see +# ai_mapping_service.py's ControlMapping construction), not a real assessment +# of how well anything matched. Averaging that fixed 0.0 in with A/B controls' +# genuine confidence judgements drags a legitimate "every enforceable control +# matched well" result down to a misleading low headline number - exactly the +# defect this constant exists to prevent. ``None`` (legacy/unclassified +# mappings) is deliberately NOT included here, for the same backward- +# compatibility reason coverage_summary buckets it as "unclassified" rather +# than excluding it. +CONFIDENCE_EXCLUDED_CATEGORIES = frozenset({COVERAGE_C, COVERAGE_D}) + + +def confidence_eligible(category: Optional[str]) -> bool: + """True when a mapping's confidence_score is a meaningful match-quality signal. + + Use this to scope any "average confidence" / "high confidence" statistic + to the controls where confidence was actually judged - A_AzurePolicy and + B_AzureConfig (plus legacy ``None`` mappings, for backward compatibility). + C_Process and D_MicrosoftAttestation controls are excluded: Azure was + never attempted for them, so their fixed 0.0 confidence_score is a + placeholder, not a low score. + """ + return category not in CONFIDENCE_EXCLUDED_CATEGORIES + + # The analyst's display names, from the source workbook's Legend sheet. The # A_/B_/C_/D_ codes are internal identifiers; these are what a reader sees. COVERAGE_DISPLAY_NAMES = { @@ -374,6 +401,40 @@ def apply_coverage( return mapping +def clear_moot_sovereignty(mapping) -> None: + """Clear a C/D control's sovereignty card when it carries no real objective. + + The mapping prompt asks the model to assign a sovereignty dimension to + EVERY control, defaulting to L1/sovereign_root with an empty objectives + list when none applies. For a process or Microsoft-attestation control + that will never carry an Azure Policy or SLZ policy, that default renders + as a real assignment ("Level: L1 - Global", "Target Archetype: + sovereign_root") when the model's own sovereignty reasoning says the + opposite - no sovereignty requirement applies here at all. Shown next to + a mapping that constructs no initiative entry, it reads as a second, + contradictory verdict rather than a placeholder. + + The one legitimate exception is a real procedural sovereignty objective + (e.g. SO-2 Customer Lockbox), which has no Azure Policy but genuine + sovereignty relevance - callers apply that (see + ``AIMappingService._apply_procedural_sovereignty``) before this runs, so + it is preserved by checking for a non-empty ``sovereignty_objectives`` + rather than category alone. + + Must run after ``apply_coverage`` (needs the resolved ``coverage_category``) + and after any procedural sovereignty enrichment. + """ + category = getattr(mapping, "coverage_category", None) + if category not in (COVERAGE_C, COVERAGE_D): + return + sovereignty = getattr(mapping, "sovereignty", None) + if sovereignty is None: + return + if getattr(sovereignty, "sovereignty_objectives", None): + return # a real procedural objective is attached - keep it + mapping.sovereignty = None + + def apply_provenance(mapping, catalog=None): """Record what verified this mapping and when. diff --git a/app/frontend/pages/2_AI_Mapping.py b/app/frontend/pages/2_AI_Mapping.py index 2cfe478..fe467ab 100644 --- a/app/frontend/pages/2_AI_Mapping.py +++ b/app/frontend/pages/2_AI_Mapping.py @@ -1,5 +1,5 @@ """ -AI Mapping Page - Map controls to MCSB using AI. +AI Mapping Page - Map controls to Azure Policy using AI. """ import streamlit as st @@ -7,6 +7,7 @@ from utils.theme import inject_azure_theme, render_sidebar, render_footer from utils.components import render_page_header from utils.state_init import init_session_state, restore_workflow_state +from utils.coverage import confidence_eligible from utils.task_manager import ( cancel_task, register_task, @@ -227,6 +228,12 @@ def _session_mapping_from_result(mapping: dict, controls: list) -> dict: "enforcement_plane": mapping.get("enforcement_plane"), "policy_effects": mapping.get("policy_effects", []), "policy_type": mapping.get("policy_type"), + # For D_MicrosoftAttestation: the resolved citation an auditor could + # actually retrieve, and whether the claim is grounded at all. Without + # these the Manual Register can only assert "Microsoft attested" with + # nothing to back it, which is indistinguishable from an invented pass. + "attestation": mapping.get("attestation"), + "attestation_gap": mapping.get("attestation_gap", False), } @@ -466,19 +473,33 @@ def _render_active_mapping_job( if completed_notice := st.session_state.pop("mapping_completed_notice", None): st.success(f"✅ {completed_notice}") mappings = st.session_state.mappings + # Scope confidence stats to controls where Azure Policy mapping was + # actually attempted (A/B, plus legacy unclassified). C_Process and + # D_MicrosoftAttestation controls carry a fixed 0.0 confidence_score + # placeholder - averaging it in misrepresents how well the real + # mapping work went. See utils/coverage.py. + scored_mappings = [m for m in mappings if confidence_eligible(m)] col_sum1, col_sum2, col_sum3 = st.columns(3) with col_sum1: avg_confidence = ( - sum(mapping.get("confidence_score", 0) for mapping in mappings) - / len(mappings) - if mappings + sum(mapping.get("confidence_score", 0) for mapping in scored_mappings) + / len(scored_mappings) + if scored_mappings else 0 ) - st.metric("Average Confidence", f"{avg_confidence:.0%}") + st.metric( + "Average Confidence", + f"{avg_confidence:.0%}", + help=( + f"Across {len(scored_mappings)} Azure-mappable control(s); " + "excludes process/Microsoft-attested controls, which never " + "attempt an Azure Policy match." + ), + ) with col_sum2: high_confidence = sum( 1 - for mapping in mappings + for mapping in scored_mappings if mapping.get("confidence_score", 0) >= 0.8 ) st.metric("High Confidence (≥80%)", high_confidence) @@ -617,8 +638,9 @@ def _render_active_mapping_job( if st.session_state.mappings: st.success(f"✅ {len(st.session_state.mappings)} mappings created") - # Statistics - avg_conf = sum(m.get('confidence_score', 0) for m in st.session_state.mappings) / len(st.session_state.mappings) + # Statistics — scoped to Azure-mappable controls (see utils/coverage.py) + scored = [m for m in st.session_state.mappings if confidence_eligible(m)] + avg_conf = sum(m.get('confidence_score', 0) for m in scored) / len(scored) if scored else 0 st.metric("Avg Confidence", f"{avg_conf:.0%}") else: st.info("No mappings yet") diff --git a/app/frontend/pages/3_Review_Edit.py b/app/frontend/pages/3_Review_Edit.py index 5c456b7..552bdb7 100644 --- a/app/frontend/pages/3_Review_Edit.py +++ b/app/frontend/pages/3_Review_Edit.py @@ -1,5 +1,6 @@ """ -Review & Edit Page - Review and modify AI-generated mappings. +Review & Edit Page - Review AI-generated Azure Policy mappings and track +controls Azure cannot address. """ import streamlit as st @@ -8,6 +9,7 @@ from utils.theme import inject_azure_theme, render_sidebar, render_footer from utils.components import render_page_header from utils.state_init import init_session_state, restore_workflow_state +from utils.coverage import is_policy_mapping, is_manual_register, COVERAGE_D from components.log_viewer import render_log_viewer from components.backend_log_viewer import render_backend_log_viewer from components.task_status_bar import render_task_status_bar @@ -29,7 +31,7 @@ render_page_header( "Review & edit mappings", eyebrow="Map", - description="Review and refine the AI-generated mappings before exporting.", + description="Review the AI-generated Azure Policy mappings before exporting.", ) st.markdown("---") @@ -44,265 +46,436 @@ # Get API client api_client = get_api_client() -# Display summary -col1, col2, col3, col4, col5 = st.columns(5) - -with col1: - st.metric("Total Mappings", len(st.session_state.mappings)) - -with col2: - avg_confidence = sum(m.get('confidence_score', 0) for m in st.session_state.mappings) / len(st.session_state.mappings) - st.metric("Avg Confidence", f"{avg_confidence:.0%}") - -with col3: - high_conf_count = sum(1 for m in st.session_state.mappings if m.get('confidence_score', 0) >= 0.8) - st.metric("High Confidence (≥80%)", high_conf_count) - -with col4: - low_conf_count = sum(1 for m in st.session_state.mappings if m.get('confidence_score', 0) < 0.6) - st.metric("Low Confidence (<60%)", low_conf_count) - -with col5: - sov_count = sum(1 for m in st.session_state.mappings if m.get('sovereignty')) - st.metric("Sovereignty Mapped", sov_count) - -st.markdown("---") - -# Filter options -st.markdown("### 🔍 Filter Mappings") - -col_filter1, col_filter2, col_filter3, col_filter4 = st.columns(4) - -with col_filter1: - confidence_filter = st.selectbox( - "Confidence Level", - options=["All", "High (≥80%)", "Medium (60-80%)", "Low (<60%)"], - index=0 +# Split once, up front: Azure covers policy_mappings (A_AzurePolicy / +# B_AzureConfig, plus legacy unclassified) and constructs an initiative entry +# for them - that is the only group where "confidence", "Azure Policy +# candidates", and "sovereignty level" are meaningful review dimensions. +# manual_register (C_Process / D_MicrosoftAttestation) is Azure cannot +# address at all: no initiative entry is built, so reviewing it alongside a +# confidence score or a defaulted sovereignty archetype misrepresents both +# groups. See utils/coverage.py. +policy_mappings = [m for m in st.session_state.mappings if is_policy_mapping(m)] +manual_register = [m for m in st.session_state.mappings if is_manual_register(m)] + +# Top-level split summary +col_top1, col_top2, col_top3 = st.columns(3) +with col_top1: + st.metric("Total Controls", len(st.session_state.mappings)) +with col_top2: + st.metric( + "🎯 Policy Mappings", + len(policy_mappings), + help="Controls Azure covers - an Azure Policy initiative entry is constructed for these.", ) - -with col_filter2: - # Get unique policy categories (server-computed from the mapped Azure - # Policy definitions' own catalog category - see policy_category on - # ControlMapping - rather than a fixed MCSB domain list). - domains = sorted(set( - m.get('policy_category') for m in st.session_state.mappings - if m.get('policy_category') - )) - - domain_filter = st.selectbox( - "Policy Category", - options=["All"] + domains, - index=0 +with col_top3: + st.metric( + "📋 Manual Register", + len(manual_register), + help="Process, legal, contractual, or Microsoft-attested controls - Azure has no technical means to enforce these, so no initiative entry is built.", ) -with col_filter3: - mapping_types = sorted(set(m.get('mapping_type', 'direct') for m in st.session_state.mappings)) - - type_filter = st.selectbox( - "Mapping Type", - options=["All"] + mapping_types, - index=0 - ) +st.markdown("---") -with col_filter4: - sov_level_filter = st.selectbox( - "Sovereignty Level", - options=["All", "L1 — Global", "L2 — CMK", "L3 — Confidential", "None"], - index=0, - help="Filter by AI-recommended sovereignty level" +tab_policy, tab_manual = st.tabs([ + f"🎯 Policy Mappings ({len(policy_mappings)})", + f"📋 Manual Register ({len(manual_register)})", +]) + +# ── Tab 1: Policy Mappings (A_AzurePolicy / B_AzureConfig) ──────────────────── +with tab_policy: + if not policy_mappings: + st.info( + "No controls in this framework are Azure-enforceable — every " + "control landed in the Manual Register. See that tab." + ) + else: + # Display summary + col1, col2, col3, col4, col5 = st.columns(5) + + with col1: + st.metric("Policy Mappings", len(policy_mappings)) + + with col2: + avg_confidence = sum(m.get('confidence_score', 0) for m in policy_mappings) / len(policy_mappings) + st.metric("Avg Confidence", f"{avg_confidence:.0%}") + + with col3: + high_conf_count = sum(1 for m in policy_mappings if m.get('confidence_score', 0) >= 0.8) + st.metric("High Confidence (≥80%)", high_conf_count) + + with col4: + low_conf_count = sum(1 for m in policy_mappings if m.get('confidence_score', 0) < 0.6) + st.metric("Low Confidence (<60%)", low_conf_count) + + with col5: + sov_count = sum(1 for m in policy_mappings if m.get('sovereignty')) + st.metric("Sovereignty Mapped", sov_count) + + st.markdown("---") + + # Filter options + st.markdown("### 🔍 Filter Mappings") + + col_filter1, col_filter2, col_filter3, col_filter4 = st.columns(4) + + with col_filter1: + confidence_filter = st.selectbox( + "Confidence Level", + options=["All", "High (≥80%)", "Medium (60-80%)", "Low (<60%)"], + index=0 + ) + + with col_filter2: + # Get unique policy categories (server-computed from the mapped Azure + # Policy definitions' own catalog category - see policy_category on + # ControlMapping - rather than a fixed MCSB domain list). + domains = sorted(set( + m.get('policy_category') for m in policy_mappings + if m.get('policy_category') + )) + + domain_filter = st.selectbox( + "Policy Category", + options=["All"] + domains, + index=0 + ) + + with col_filter3: + mapping_types = sorted(set(m.get('mapping_type', 'direct') for m in policy_mappings)) + + type_filter = st.selectbox( + "Mapping Type", + options=["All"] + mapping_types, + index=0 + ) + + with col_filter4: + sov_level_filter = st.selectbox( + "Sovereignty Level", + options=["All", "L1 — Global", "L2 — CMK", "L3 — Confidential", "None"], + index=0, + help="Filter by AI-recommended sovereignty level" + ) + + # Apply filters + filtered_mappings = policy_mappings.copy() + + if confidence_filter == "High (≥80%)": + filtered_mappings = [m for m in filtered_mappings if m.get('confidence_score', 0) >= 0.8] + elif confidence_filter == "Medium (60-80%)": + filtered_mappings = [m for m in filtered_mappings if 0.6 <= m.get('confidence_score', 0) < 0.8] + elif confidence_filter == "Low (<60%)": + filtered_mappings = [m for m in filtered_mappings if m.get('confidence_score', 0) < 0.6] + + if domain_filter != "All": + filtered_mappings = [m for m in filtered_mappings + if m.get('policy_category') == domain_filter] + + if type_filter != "All": + filtered_mappings = [m for m in filtered_mappings if m.get('mapping_type') == type_filter] + + if sov_level_filter != "All": + if sov_level_filter == "None": + filtered_mappings = [m for m in filtered_mappings if not m.get('sovereignty')] + else: + target_level = sov_level_filter.split(" ")[0] # "L1", "L2", "L3" + filtered_mappings = [m for m in filtered_mappings + if m.get('sovereignty') and m['sovereignty'].get('sovereignty_level') == target_level] + + st.info(f"📋 Showing **{len(filtered_mappings)}** of **{len(policy_mappings)}** policy mappings") + + st.markdown("---") + + # Review each mapping + st.markdown("### 📝 Azure Policy Mappings") + + if not filtered_mappings: + st.warning("No mappings match the current filters.") + else: + for idx, mapping in enumerate(filtered_mappings): + with st.expander( + f"{'⚠️' if mapping.get('confidence_score', 0) < 0.6 else '✅'} " + f"{mapping.get('control_id', mapping.get('external_control_id', 'N/A'))} → " + f"{len(mapping.get('azure_policy_ids') or [])} Azure Polic{'y' if len(mapping.get('azure_policy_ids') or []) == 1 else 'ies'} " + f"({mapping.get('confidence_score', 0):.0%})", + expanded=mapping.get('confidence_score', 0) < 0.6 + ): + col_edit1, col_edit2 = st.columns([1, 1]) + + with col_edit1: + st.markdown("#### 📋 Source Control") + control_id = mapping.get('control_id', mapping.get('external_control_id', 'N/A')) + control_name = mapping.get('control_name', mapping.get('external_control_name', 'N/A')) + st.markdown(f"**ID:** {control_id}") + st.markdown(f"**Name:** {control_name}") + st.markdown(f"**Description:** {mapping.get('description', 'N/A')}") + if mapping.get('domain'): + st.markdown(f"**Domain:** {mapping['domain']}") + + with col_edit2: + st.markdown("#### 🎯 Azure Policy Mapping") + + # policy_category is server-computed from the catalog category + # of the mapped Azure Policy definitions (see policy_category + # on ControlMapping) - there is no fixed intermediate taxonomy + # to pick from anymore, so this is informational, not editable. + if mapping.get('policy_category'): + st.caption(f"**Category:** {mapping['policy_category']}") + if mapping.get('coverage_display'): + st.caption(f"**Coverage:** {mapping['coverage_display']}") + if mapping.get('outside_step'): + st.caption(f"**Also needs:** {mapping['outside_step']}") + + # Confidence score (read-only if not manually overridden) + st.metric("Confidence Score", f"{mapping.get('confidence_score', 0):.0%}") + + # Mapping type + st.caption(f"**Type:** {mapping.get('mapping_type', 'direct').replace('_', ' ').title()}") + + # AI Reasoning + st.markdown("#### 💡 AI Reasoning") + st.info(mapping.get('reasoning', 'No reasoning provided')) + + # Azure Policies + if mapping.get('azure_policy_ids'): + st.markdown("#### 🎯 Recommended Azure Policies") + render_policy_list(api_client, mapping['azure_policy_ids']) + + # Sovereignty mapping + sov = mapping.get('sovereignty') + if sov: + st.markdown("#### 🏛️ Sovereignty Mapping") + sov_level = sov.get('sovereignty_level', 'N/A') + _level_colors = {'L1': '🟢', 'L2': '🟡', 'L3': '🔴'} + _level_labels = { + 'L1': 'Global (Data Residency + Trusted Launch)', + 'L2': 'CMK (Customer-Managed Keys)', + 'L3': 'Confidential Computing', + } + col_sov1, col_sov2 = st.columns(2) + with col_sov1: + st.markdown( + f"**Level:** {_level_colors.get(sov_level, '⚪')} **{sov_level}** — " + f"{_level_labels.get(sov_level, sov_level)}" + ) + if sov.get('sovereignty_objectives'): + st.markdown("**Objectives:** " + ", ".join(sov['sovereignty_objectives'])) + if sov.get('target_archetype'): + st.markdown(f"**Target Archetype:** `{sov['target_archetype']}`") + with col_sov2: + if sov.get('slz_policy_names'): + st.markdown("**SLZ Policies:**") + for pname in sov['slz_policy_names'][:5]: + st.caption(f"• {pname}") + if len(sov['slz_policy_names']) > 5: + st.caption(f" ... and {len(sov['slz_policy_names']) - 5} more") + if sov.get('reasoning'): + st.info(sov['reasoning']) + + # Delete mapping option + col_delete1, col_delete2 = st.columns([3, 1]) + with col_delete2: + delete_id = mapping.get('control_id', mapping.get('external_control_id', f'unknown_{idx}')) + if st.button("🗑️ Delete", key=f"delete_policy_{idx}_{delete_id}"): + mapping_id = mapping.get('control_id', mapping.get('external_control_id')) + st.session_state.mappings = [m for m in st.session_state.mappings + if m.get('control_id', m.get('external_control_id')) != mapping_id] + st.success(f"Deleted mapping for {delete_id}") + st.rerun() + + # Confidence / category statistics + st.markdown("---") + st.markdown("### 📊 Mapping Statistics") + + df = pd.DataFrame(policy_mappings) + + col_stat1, col_stat2 = st.columns(2) + + with col_stat1: + st.markdown("#### Confidence Distribution") + confidence_bins = pd.cut(df['confidence_score'], bins=[0, 0.6, 0.8, 1.0], labels=['Low', 'Medium', 'High']) + confidence_dist = confidence_bins.value_counts().sort_index() + st.bar_chart(confidence_dist) + + with col_stat2: + st.markdown("#### Top Policy Categories") + if 'policy_category' in df.columns: + top_categories = df['policy_category'].dropna().value_counts().head(10) + st.bar_chart(top_categories) + else: + st.caption("No policy category data available.") + + # Sovereignty statistics + sov_mappings = [m for m in policy_mappings if m.get('sovereignty')] + if sov_mappings: + st.markdown("#### 🏛️ Sovereignty Level Distribution") + col_sov_stat1, col_sov_stat2, col_sov_stat3 = st.columns(3) + level_counts = {'L1': 0, 'L2': 0, 'L3': 0} + for m in sov_mappings: + lvl = m['sovereignty'].get('sovereignty_level', '') + if lvl in level_counts: + level_counts[lvl] += 1 + with col_sov_stat1: + st.metric("🟢 L1 — Global", level_counts['L1']) + with col_sov_stat2: + st.metric("🟡 L2 — CMK", level_counts['L2']) + with col_sov_stat3: + st.metric("🔴 L3 — Confidential", level_counts['L3']) + +# ── Tab 2: Manual Register (C_Process / D_MicrosoftAttestation) ────────────── +with tab_manual: + st.markdown( + "These controls are **not addressable by Azure Policy** — they are " + "process, legal, contractual, or Microsoft-operated requirements. " + "**No Azure Policy initiative entry is constructed for them**, so " + "there is no confidence score, Azure Policy candidate list, or " + "sovereignty level to review here. Where a control is matched to " + "Microsoft's own attestation (an ISO/SOC certification clause, audit " + "report, or published documentation), that citation is shown so an " + "auditor can retrieve it; otherwise track it for manual attestation." ) -# Apply filters -filtered_mappings = st.session_state.mappings.copy() - -if confidence_filter == "High (≥80%)": - filtered_mappings = [m for m in filtered_mappings if m.get('confidence_score', 0) >= 0.8] -elif confidence_filter == "Medium (60-80%)": - filtered_mappings = [m for m in filtered_mappings if 0.6 <= m.get('confidence_score', 0) < 0.8] -elif confidence_filter == "Low (<60%)": - filtered_mappings = [m for m in filtered_mappings if m.get('confidence_score', 0) < 0.6] - -if domain_filter != "All": - filtered_mappings = [m for m in filtered_mappings - if m.get('policy_category') == domain_filter] - -if type_filter != "All": - filtered_mappings = [m for m in filtered_mappings if m.get('mapping_type') == type_filter] - -if sov_level_filter != "All": - if sov_level_filter == "None": - filtered_mappings = [m for m in filtered_mappings if not m.get('sovereignty')] + if not manual_register: + st.success("✅ Every control in this framework is Azure-addressable — the manual register is empty.") else: - target_level = sov_level_filter.split(" ")[0] # "L1", "L2", "L3" - filtered_mappings = [m for m in filtered_mappings - if m.get('sovereignty') and m['sovereignty'].get('sovereignty_level') == target_level] - -st.info(f"📋 Showing **{len(filtered_mappings)}** of **{len(st.session_state.mappings)}** mappings") + col_m1, col_m2, col_m3 = st.columns(3) + with col_m1: + st.metric("Manual Register", len(manual_register)) + with col_m2: + process_count = sum(1 for m in manual_register if m.get('coverage_category') != COVERAGE_D) + st.metric("Process / Organisational", process_count) + with col_m3: + attested_count = sum(1 for m in manual_register if m.get('coverage_category') == COVERAGE_D) + grounded_count = sum( + 1 for m in manual_register + if m.get('coverage_category') == COVERAGE_D and m.get('attestation') and not m.get('attestation_gap') + ) + st.metric( + "Microsoft Attested", + attested_count, + help=f"{grounded_count} of {attested_count} have a grounded citation; the rest are attestation gaps.", + ) + + st.markdown("---") + + col_mfilter1, col_mfilter2 = st.columns(2) + with col_mfilter1: + category_filter = st.selectbox( + "Category", + options=["All", "Process / organisational", "Microsoft attested"], + index=0, + key="manual_register_category_filter", + ) + with col_mfilter2: + responsibility_options = sorted(set( + m.get('responsibility') for m in manual_register if m.get('responsibility') + )) + responsibility_filter = st.selectbox( + "Responsibility", + options=["All"] + responsibility_options, + index=0, + key="manual_register_responsibility_filter", + ) + + filtered_manual = manual_register.copy() + if category_filter == "Process / organisational": + filtered_manual = [m for m in filtered_manual if m.get('coverage_category') != COVERAGE_D] + elif category_filter == "Microsoft attested": + filtered_manual = [m for m in filtered_manual if m.get('coverage_category') == COVERAGE_D] + if responsibility_filter != "All": + filtered_manual = [m for m in filtered_manual if m.get('responsibility') == responsibility_filter] + + st.info(f"📋 Showing **{len(filtered_manual)}** of **{len(manual_register)}** manual-register controls") + + st.markdown("---") + + for idx, mapping in enumerate(filtered_manual): + control_id = mapping.get('control_id', mapping.get('external_control_id', 'N/A')) + control_name = mapping.get('control_name', mapping.get('external_control_name', 'N/A')) + is_attested = mapping.get('coverage_category') == COVERAGE_D + badge = "🏛️" if is_attested else "📄" + + with st.expander(f"{badge} {control_id} — {mapping.get('coverage_display') or mapping.get('coverage_category', 'N/A')}"): + col_a, col_b = st.columns([1, 1]) + + with col_a: + st.markdown("#### 📋 Source Control") + st.markdown(f"**ID:** {control_id}") + st.markdown(f"**Name:** {control_name}") + st.markdown(f"**Description:** {mapping.get('description', 'N/A')}") + if mapping.get('domain'): + st.markdown(f"**Domain:** {mapping['domain']}") + if mapping.get('control_type'): + st.markdown(f"**Control Type:** {mapping['control_type']}") + + with col_b: + st.markdown("#### 📋 Coverage Classification") + st.markdown(f"**Category:** {mapping.get('coverage_display') or mapping.get('coverage_category', 'N/A')}") + if mapping.get('responsibility'): + st.markdown(f"**Responsibility:** {mapping['responsibility']}") + st.caption("No Azure Policy initiative entry is constructed for this control.") + if mapping.get('coverage_reason'): + st.info(mapping['coverage_reason']) + + if is_attested: + st.markdown("#### 🏛️ Microsoft Attestation") + attestation = mapping.get('attestation') or {} + if mapping.get('attestation_gap') or not attestation: + st.warning( + "⚠️ **Attestation gap** — this control was classified as " + "Microsoft-attested, but no grounded certification clause, " + "audit-report criterion, or published documentation could be " + "found. Escalate commercially; do not report this as covered." + ) + else: + col_c, col_d = st.columns(2) + with col_c: + if attestation.get('scheme'): + st.markdown(f"**Scheme:** {attestation['scheme']}") + if attestation.get('citation'): + st.markdown(f"**Citation:** {attestation['citation']}") + if attestation.get('basis_kind'): + st.caption(f"Basis: {attestation['basis_kind']}") + with col_d: + if attestation.get('evidence_document'): + st.markdown(f"**Evidence document:** {attestation['evidence_document']}") + if attestation.get('evidence_location'): + st.markdown(f"**Where to retrieve it:** {attestation['evidence_location']}") + if attestation.get('access_condition'): + st.caption(f"Access: {attestation['access_condition']}") + elif mapping.get('evidence_source'): + st.markdown("#### 📄 Evidence") + st.info(mapping['evidence_source']) + + col_del1, col_del2 = st.columns([3, 1]) + with col_del2: + if st.button("🗑️ Delete", key=f"delete_manual_{idx}_{control_id}"): + st.session_state.mappings = [ + m for m in st.session_state.mappings + if m.get('control_id', m.get('external_control_id')) != control_id + ] + st.success(f"Deleted {control_id} from the manual register") + st.rerun() + + st.markdown("---") + manual_df = pd.DataFrame(filtered_manual) + if not manual_df.empty: + _manual_cols = [ + c for c in ( + 'control_id', 'control_name', 'control_type', 'coverage_display', + 'responsibility', 'coverage_reason', + ) if c in manual_df.columns + ] + st.download_button( + label="📥 Download Manual Register (CSV)", + data=manual_df[_manual_cols].to_csv(index=False), + file_name=f"{st.session_state.framework_name.replace(' ', '_')}_manual_register.csv", + mime="text/csv", + ) st.markdown("---") -# Review and edit each mapping -st.markdown("### 📝 Edit Mappings") - -if not filtered_mappings: - st.warning("No mappings match the current filters.") -else: - # Track if any changes were made - changes_made = False - - for idx, mapping in enumerate(filtered_mappings): - with st.expander( - f"{'⚠️' if mapping.get('confidence_score', 0) < 0.6 else '✅'} " - f"{mapping.get('control_id', mapping.get('external_control_id', 'N/A'))} → " - f"{len(mapping.get('azure_policy_ids') or [])} Azure Polic{'y' if len(mapping.get('azure_policy_ids') or []) == 1 else 'ies'} " - f"({mapping.get('confidence_score', 0):.0%})", - expanded=mapping.get('confidence_score', 0) < 0.6 - ): - col_edit1, col_edit2 = st.columns([1, 1]) - - with col_edit1: - st.markdown("#### 📋 Source Control") - control_id = mapping.get('control_id', mapping.get('external_control_id', 'N/A')) - control_name = mapping.get('control_name', mapping.get('external_control_name', 'N/A')) - st.markdown(f"**ID:** {control_id}") - st.markdown(f"**Name:** {control_name}") - st.markdown(f"**Description:** {mapping.get('description', 'N/A')}") - if mapping.get('domain'): - st.markdown(f"**Domain:** {mapping['domain']}") - - with col_edit2: - st.markdown("#### 🎯 Azure Policy Mapping") - - control_id_key = mapping.get('control_id', mapping.get('external_control_id', f'unknown_{idx}')) - - # policy_category is server-computed from the catalog category - # of the mapped Azure Policy definitions (see policy_category - # on ControlMapping) - there is no fixed intermediate taxonomy - # to pick from anymore, so this is informational, not editable. - if mapping.get('policy_category'): - st.caption(f"**Category:** {mapping['policy_category']}") - - # Confidence score (read-only if not manually overridden) - st.metric("Confidence Score", f"{mapping.get('confidence_score', 0):.0%}") - - # Mapping type - st.caption(f"**Type:** {mapping.get('mapping_type', 'direct').replace('_', ' ').title()}") - - # AI Reasoning - st.markdown("#### 💡 AI Reasoning") - st.info(mapping.get('reasoning', 'No reasoning provided')) - - # Azure Policies - if mapping.get('azure_policy_ids'): - st.markdown("#### 🎯 Recommended Azure Policies") - render_policy_list(api_client, mapping['azure_policy_ids']) - - # Sovereignty mapping - sov = mapping.get('sovereignty') - if sov: - st.markdown("#### 🏛️ Sovereignty Mapping") - sov_level = sov.get('sovereignty_level', 'N/A') - _level_colors = {'L1': '🟢', 'L2': '🟡', 'L3': '🔴'} - _level_labels = { - 'L1': 'Global (Data Residency + Trusted Launch)', - 'L2': 'CMK (Customer-Managed Keys)', - 'L3': 'Confidential Computing', - } - col_sov1, col_sov2 = st.columns(2) - with col_sov1: - st.markdown( - f"**Level:** {_level_colors.get(sov_level, '⚪')} **{sov_level}** — " - f"{_level_labels.get(sov_level, sov_level)}" - ) - if sov.get('sovereignty_objectives'): - st.markdown("**Objectives:** " + ", ".join(sov['sovereignty_objectives'])) - if sov.get('target_archetype'): - st.markdown(f"**Target Archetype:** `{sov['target_archetype']}`") - with col_sov2: - if sov.get('slz_policy_names'): - st.markdown("**SLZ Policies:**") - for pname in sov['slz_policy_names'][:5]: - st.caption(f"• {pname}") - if len(sov['slz_policy_names']) > 5: - st.caption(f" ... and {len(sov['slz_policy_names']) - 5} more") - if sov.get('reasoning'): - st.info(sov['reasoning']) - - # Delete mapping option - col_delete1, col_delete2 = st.columns([3, 1]) - with col_delete2: - delete_id = mapping.get('control_id', mapping.get('external_control_id', f'unknown_{idx}')) - if st.button("🗑️ Delete", key=f"delete_{idx}_{delete_id}"): - # Remove from session state - mapping_id = mapping.get('control_id', mapping.get('external_control_id')) - st.session_state.mappings = [m for m in st.session_state.mappings - if m.get('control_id', m.get('external_control_id')) != mapping_id] - st.success(f"Deleted mapping for {delete_id}") - st.rerun() - -# Show changes notification -if changes_made: - st.success("✅ Changes saved! Mappings have been updated.") - # Auto-save session after mapping edits - try: - api_client.save_session( - st.session_state["session_uuid"], - { - "controls": st.session_state.get("controls", []), - "mappings": st.session_state.mappings, - "framework_name": st.session_state.get("framework_name", ""), - "policy_decisions": st.session_state.get("policy_decisions", {}), - "selected_platform": st.session_state.get("selected_platform", "azure_defender"), - "platform_display_name": st.session_state.get("platform_display_name", ""), - }, - ) - except Exception: - pass # session save is best-effort - -# Export statistics -st.markdown("---") -st.markdown("### 📊 Mapping Statistics") - -if st.session_state.mappings: - # Create DataFrame for analysis - df = pd.DataFrame(st.session_state.mappings) - - col_stat1, col_stat2 = st.columns(2) - - with col_stat1: - st.markdown("#### Confidence Distribution") - confidence_bins = pd.cut(df['confidence_score'], bins=[0, 0.6, 0.8, 1.0], labels=['Low', 'Medium', 'High']) - confidence_dist = confidence_bins.value_counts().sort_index() - st.bar_chart(confidence_dist) - - with col_stat2: - st.markdown("#### Top Policy Categories") - if 'policy_category' in df.columns: - top_categories = df['policy_category'].dropna().value_counts().head(10) - st.bar_chart(top_categories) - else: - st.caption("No policy category data available.") - - # Sovereignty statistics - sov_mappings = [m for m in st.session_state.mappings if m.get('sovereignty')] - if sov_mappings: - st.markdown("#### 🏛️ Sovereignty Level Distribution") - col_sov_stat1, col_sov_stat2, col_sov_stat3 = st.columns(3) - level_counts = {'L1': 0, 'L2': 0, 'L3': 0} - for m in sov_mappings: - lvl = m['sovereignty'].get('sovereignty_level', '') - if lvl in level_counts: - level_counts[lvl] += 1 - with col_sov_stat1: - st.metric("🟢 L1 — Global", level_counts['L1']) - with col_sov_stat2: - st.metric("🟡 L2 — CMK", level_counts['L2']) - with col_sov_stat3: - st.metric("🔴 L3 — Confidential", level_counts['L3']) - # Action buttons -st.markdown("---") - col_action1, col_action2, col_action3 = st.columns(3) with col_action1: @@ -312,7 +485,7 @@ with col_action2: # Download current mappings import json - + if st.download_button( label="📥 Download Mappings (JSON)", data=json.dumps(st.session_state.mappings, indent=2), @@ -329,32 +502,30 @@ # Sidebar with st.sidebar: st.markdown("### 📊 Review Status") - - st.metric("Mappings", len(st.session_state.mappings)) - st.metric("Filtered View", len(filtered_mappings)) - - manual_overrides = sum(1 for m in st.session_state.mappings if m.get('manual_override', False)) - if manual_overrides > 0: - st.metric("Manual Edits", manual_overrides) - + + st.metric("Total Controls", len(st.session_state.mappings)) + st.metric("Policy Mappings", len(policy_mappings)) + st.metric("Manual Register", len(manual_register)) + st.markdown("---") - + st.markdown("### 💡 Tips") st.markdown(""" - - Review low confidence mappings first + - Review low confidence policy mappings first - Use filters to focus on specific areas - - Change MCSB control if needed + - Escalate ungrounded attestation gaps - Delete incorrect mappings - Download for backup """) - + st.markdown("---") - + st.markdown("### ⚠️ Confidence Guide") st.markdown(""" - **High (≥80%)**: Strong match - **Medium (60-80%)**: Good match - **Low (<60%)**: Review needed + - *(Manual Register controls have no confidence score — Azure was never attempted for them.)* """) render_footer() diff --git a/app/frontend/pages/4_Export_Policy.py b/app/frontend/pages/4_Export_Policy.py index 565c555..82149a0 100644 --- a/app/frontend/pages/4_Export_Policy.py +++ b/app/frontend/pages/4_Export_Policy.py @@ -22,6 +22,7 @@ from components.backend_log_viewer import render_backend_log_viewer from components.task_status_bar import render_task_status_bar from utils.policy_parameters import satisfied_parameter_values +from utils.coverage import confidence_eligible _VALID_MAPPING_TYPES = {"exact", "partial", "conceptual", "none"} @@ -379,8 +380,20 @@ def _regenerate_with_parameters( st.metric("Total Mappings", len(st.session_state.mappings)) with col3: - avg_confidence = sum(m.get('confidence_score', 0) for m in st.session_state.mappings) / len(st.session_state.mappings) - st.metric("Avg Confidence", f"{avg_confidence:.0%}") + scored_mappings = [m for m in st.session_state.mappings if confidence_eligible(m)] + avg_confidence = ( + sum(m.get('confidence_score', 0) for m in scored_mappings) / len(scored_mappings) + if scored_mappings else 0.0 + ) + st.metric( + "Avg Confidence", + f"{avg_confidence:.0%}", + help=( + f"Across {len(scored_mappings)} Azure-mappable control(s); excludes " + "process/Microsoft-attested controls in the Manual Register, which " + "never attempt an Azure Policy match." + ), + ) with col4: unique_categories = len(set(m.get('policy_category', '') for m in st.session_state.mappings if m.get('policy_category'))) diff --git a/app/frontend/utils/coverage.py b/app/frontend/utils/coverage.py new file mode 100644 index 0000000..64d48db --- /dev/null +++ b/app/frontend/utils/coverage.py @@ -0,0 +1,56 @@ +"""Frontend mirror of the backend's coverage taxonomy constants. + +Keeps ``2_AI_Mapping.py``, ``3_Review_Edit.py`` and ``4_Export_Policy.py`` +agreeing on which controls are "policy mappings" (Azure covers them, an +initiative entry is constructed) versus the "manual register" (Azure cannot +address them at all - process/legal/contractual or Microsoft-attested), and on +which controls' confidence_score is a real match-quality signal rather than a +fixed 0.0 placeholder. See ``app/backend/app/services/coverage.py`` for the +authoritative definitions this mirrors. +""" + +from typing import Any, Dict + +COVERAGE_A = "A_AzurePolicy" +COVERAGE_B = "B_AzureConfig" +COVERAGE_C = "C_Process" +COVERAGE_D = "D_MicrosoftAttestation" + +# Categories Azure actually covers - these carry azure_policy_ids and appear +# in the generated initiative. +POLICY_BEARING_CATEGORIES = frozenset({COVERAGE_A, COVERAGE_B}) + +# Categories Azure cannot address at all - no initiative entry is constructed +# for these, so they do not belong in a "review the Azure Policy mapping" +# workflow. They still need tracking (manual attestation, process evidence), +# just not alongside confidence scores and sovereignty verdicts that never +# applied to them. +NON_POLICY_CATEGORIES = frozenset({COVERAGE_C, COVERAGE_D}) + + +def is_policy_mapping(mapping: Dict[str, Any]) -> bool: + """True when Azure covers this control - it belongs in "Policy Mappings". + + ``coverage_category`` of ``None`` (legacy/unclassified mappings, or a + mapping the coverage stage never touched) is treated as a policy mapping + for backward compatibility, matching the backend's own convention (see + ``coverage_summary``'s "unclassified" bucket). + """ + category = mapping.get("coverage_category") + return category not in NON_POLICY_CATEGORIES + + +def is_manual_register(mapping: Dict[str, Any]) -> bool: + """True when this control belongs in the Manual Register (C/D only).""" + return mapping.get("coverage_category") in NON_POLICY_CATEGORIES + + +def confidence_eligible(mapping: Dict[str, Any]) -> bool: + """True when confidence_score is a real match-quality signal. + + C_Process/D_MicrosoftAttestation controls never attempt an Azure Policy + match, so their confidence_score is a fixed 0.0 placeholder rather than a + graded assessment - averaging it in with A/B controls' genuine scores + misrepresents how well the actual mapping work went. + """ + return is_policy_mapping(mapping)