diff --git a/.github/skills/complianceiq/reference/api.md b/.github/skills/complianceiq/reference/api.md index 5f0d3d6..3bd1b47 100644 --- a/.github/skills/complianceiq/reference/api.md +++ b/.github/skills/complianceiq/reference/api.md @@ -32,7 +32,7 @@ The deployed OpenAPI document reports **94 operations across 89 paths** ## Granular mapping (`/mapping`) — for advanced flows `POST /mapping/analyze` (job) · `POST /mapping/map-batch` · `POST /mapping/map-single` -· `GET /mapping/status/{job_id}` · `GET /mapping/mcsb/{controls,domains}`. +· `GET /mapping/status/{job_id}`. ## Policy generation (`/policy`) — **authenticated** `POST /policy/generate` (+ `/generate/{json,bicep,scripts,slz}`) · diff --git a/README.md b/README.md index 1887358..43f7ee9 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,6 @@ compliance-iq/ │ ├── Oman_Government_Azure_Mappings.csv │ └── CATALOG_SUMMARY.md │ -├── compliance-pipeline/ # Standalone CLI tool for batch processing ├── framework/ # Azure Policy initiative JSON & deployment scripts ├── reference_documents/ # Source compliance framework PDFs ├── templates/ # Simplified control templates for gap analysis diff --git a/app/README.md b/app/README.md index c44550d..926190c 100644 --- a/app/README.md +++ b/app/README.md @@ -183,8 +183,7 @@ app/ │ │ │ ├── policy.py # Policy initiative models │ │ │ └── sovereignty.py # SLZ / sovereignty models (NEW) │ │ ├── services/ # Business logic -│ │ │ ├── ai_mapping_service.py # AI mapping (MCSB + SLZ) -│ │ │ ├── mcsb_service.py # MCSB control loader +│ │ │ ├── ai_mapping_service.py # AI mapping (direct to Azure Policy + SLZ) │ │ │ ├── microsoft_learn_client.py # MS Learn policy search │ │ │ ├── policy_service.py # Policy gen (+ SLZ initiatives) │ │ │ └── sovereignty_service.py # SLZ data service (NEW) @@ -208,13 +207,12 @@ app/ │ │ ├── 1_📁_Upload_Controls.py │ │ ├── 2_🤖_AI_Mapping.py # Shows SLZ level badges │ │ ├── 3_✏️_Review_Edit.py # Sovereignty filter + panels -│ │ └── 4_📦_Export_Policy.py # MCSB + SLZ export tabs +│ │ └── 4_📦_Export_Policy.py # Azure Policy + SLZ export tabs │ ├── utils/ # API client (+ SLZ methods) │ ├── app.py # Main app (SLZ status in sidebar) │ └── requirements.txt │ ├── data/ # Reference data -│ ├── mcsb/ # MCSB controls │ └── examples/ # Sample files │ ├── tests/ # Test suite @@ -277,7 +275,6 @@ SAMA-AC-01,Strong Authentication,Enforce MFA and disable legacy protocols ```json { "external_control_id": "SAMA-AC-01", - "mcsb_control_id": "IM-6", "confidence_score": 0.92, "reasoning": "Both controls focus on enforcing MFA...", "azure_policy_ids": ["4e6c27d5-a6ee-49cf-b2b4-d8fe90fa2b8b"] diff --git a/app/backend/README.md b/app/backend/README.md index a1cf077..07a9eb0 100644 --- a/app/backend/README.md +++ b/app/backend/README.md @@ -91,8 +91,6 @@ Once the server is running: - `POST /api/v1/mapping/map-single` - Map single control - `POST /api/v1/mapping/analyze` - Batch mapping (async) - `GET /api/v1/mapping/status/{job_id}` - Job status -- `GET /api/v1/mapping/mcsb/controls` - Get MCSB controls -- `GET /api/v1/mapping/mcsb/domains` - Get MCSB domains ### Policy Generation @@ -114,9 +112,7 @@ Expected output: { "status": "healthy", "version": "1.0.0", - "azure_openai_connected": true, - "mcsb_controls_loaded": true, - "mcsb_control_count": 10 + "azure_openai_connected": true } ``` @@ -135,12 +131,6 @@ curl -X POST http://localhost:8000/api/v1/mapping/map-single \ }' | jq ``` -### Test MCSB Controls Retrieval - -```bash -curl http://localhost:8000/api/v1/mapping/mcsb/controls | jq -``` - ## 🚀 Azure Container Apps Deployment Notes - Container Apps pull images from ACR using the system-assigned managed identity (AcrPull granted in Bicep). No ACR admin credentials are needed at runtime. @@ -162,7 +152,6 @@ backend/ │ │ ├── mapping.py │ │ └── policy.py │ ├── services/ # Business logic -│ │ ├── mcsb_service.py │ │ ├── ai_mapping_service.py │ │ └── policy_service.py │ ├── api/routes/ # API endpoints @@ -232,14 +221,6 @@ from app.auth import test_azure_openai_connection test_azure_openai_connection() # Should return True ``` -### MCSB Controls Not Loading - -The service includes 10 default MCSB controls for demonstration. For full MCSB catalog: - -1. Download from GitHub SecurityBenchmarks -2. Place JSON at `../data/mcsb/mcsb_v1_controls.json` -3. Restart server - ## 📝 Development ### Run with Auto-reload diff --git a/app/backend/app/api/routes/health.py b/app/backend/app/api/routes/health.py index 0ee47ee..639ef73 100644 --- a/app/backend/app/api/routes/health.py +++ b/app/backend/app/api/routes/health.py @@ -8,7 +8,7 @@ from app import __version__ from app.auth import test_azure_openai_connection -from app.services import get_mcsb_service, get_sovereignty_service, get_policy_catalog_service +from app.services import get_sovereignty_service, get_policy_catalog_service logger = logging.getLogger(__name__) router = APIRouter(tags=["health"]) @@ -19,9 +19,6 @@ class HealthResponse(BaseModel): status: str version: str azure_openai_connected: bool - mcsb_controls_loaded: bool - mcsb_control_count: int - mcsb_is_demonstration_data: bool = False slz_policies_loaded: bool = False slz_policy_count: int = 0 policy_catalog_count: int = 0 @@ -46,19 +43,6 @@ async def health_check(): logger.error(f"Azure OpenAI health check failed: {e}") azure_openai_connected = False - # Check MCSB service - try: - mcsb_service = get_mcsb_service() - controls = mcsb_service.get_all_controls() - mcsb_controls_loaded = True - mcsb_control_count = len(controls) - mcsb_is_demonstration_data = mcsb_service.is_demonstration_data - except Exception as e: - logger.error(f"MCSB service health check failed: {e}") - mcsb_controls_loaded = False - mcsb_control_count = 0 - mcsb_is_demonstration_data = False - # Check SLZ sovereignty service try: slz_service = get_sovereignty_service() @@ -81,12 +65,9 @@ async def health_check(): policy_catalog_source = "error" return HealthResponse( - status="healthy" if (azure_openai_connected and mcsb_controls_loaded) else "degraded", + status="healthy" if azure_openai_connected else "degraded", version=__version__, azure_openai_connected=azure_openai_connected, - mcsb_controls_loaded=mcsb_controls_loaded, - mcsb_control_count=mcsb_control_count, - mcsb_is_demonstration_data=mcsb_is_demonstration_data, slz_policies_loaded=slz_policies_loaded, slz_policy_count=slz_policy_count, policy_catalog_count=policy_catalog_count, diff --git a/app/backend/app/api/routes/mapping.py b/app/backend/app/api/routes/mapping.py index 9ea5b5b..46f7695 100644 --- a/app/backend/app/api/routes/mapping.py +++ b/app/backend/app/api/routes/mapping.py @@ -18,7 +18,7 @@ MappingJob, ) from app.models.mapping import record_mapping_activity -from app.services import get_ai_mapping_service, get_mcsb_service +from app.services import get_ai_mapping_service from app.db import cosmos_client from app.config import get_settings @@ -268,58 +268,6 @@ async def get_mapping_status(job_id: str): return job -@router.get("/mcsb/controls") -async def get_mcsb_controls(domain: Optional[str] = None): - """ - Get MCSB controls, optionally filtered by domain. - - Args: - domain: Optional domain filter - - Returns: - List of MCSB controls. ``is_demonstration_data`` is True whenever no - real MCSB catalog file is loaded, in which case the returned - ``defender_recommendations`` are illustrative examples, not verified - against a live Microsoft Defender for Cloud subscription (see - docs/BACKLOG.md B4). - """ - try: - mcsb_service = get_mcsb_service() - - if domain: - controls = mcsb_service.get_controls_by_domain(domain) - else: - controls = mcsb_service.get_all_controls() - - return { - "controls": controls, - "count": len(controls), - "is_demonstration_data": mcsb_service.is_demonstration_data, - } - - except Exception as e: - logger.error(f"Failed to get MCSB controls: {e}") - raise HTTPException(status_code=500, detail=str(e)) - - -@router.get("/mcsb/domains") -async def get_mcsb_domains(): - """Get all MCSB security domains.""" - try: - mcsb_service = get_mcsb_service() - domains = mcsb_service.get_all_domains() - - return { - "domains": domains, - "count": len(domains), - "is_demonstration_data": mcsb_service.is_demonstration_data, - } - - except Exception as e: - logger.error(f"Failed to get MCSB domains: {e}") - raise HTTPException(status_code=500, detail=str(e)) - - async def process_mapping_job( job_id: str, controls: List[ExternalControl], diff --git a/app/backend/app/models/__init__.py b/app/backend/app/models/__init__.py index 2a8c03a..0e6f59c 100644 --- a/app/backend/app/models/__init__.py +++ b/app/backend/app/models/__init__.py @@ -2,7 +2,7 @@ Pydantic models for the AI Control Mapping Agent. """ -from app.models.control import ExternalControl, MCSBControl, FrameworkUpload +from app.models.control import ExternalControl, FrameworkUpload from app.models.mapping import ( ControlMapping, MappingBatch, @@ -60,7 +60,6 @@ __all__ = [ # Control models "ExternalControl", - "MCSBControl", "FrameworkUpload", # Mapping models "ControlMapping", diff --git a/app/backend/app/models/control.py b/app/backend/app/models/control.py index d048847..5b8d5cb 100644 --- a/app/backend/app/models/control.py +++ b/app/backend/app/models/control.py @@ -28,47 +28,6 @@ class ExternalControl(BaseModel): }) -class MCSBControl(BaseModel): - """Model for Microsoft Cloud Security Benchmark control.""" - - control_id: str = Field(..., description="MCSB control ID (e.g., IM-1)") - domain: str = Field(..., description="Security domain") - control_name: str = Field(..., description="Control title") - description: str = Field(..., description="Full control description") - azure_policy_ids: List[str] = Field( - default_factory=list, - description="Associated Azure Policy definition GUIDs" - ) - defender_recommendations: List[str] = Field( - default_factory=list, - description=( - "Illustrative Microsoft Defender for Cloud recommendation names, " - "not verified against a live Defender for Cloud subscription. " - "Populated only when the deployment serves demonstration data " - "(see MCSBService.is_demonstration_data / docs/BACKLOG.md B4)." - ), - ) - related_frameworks: Dict[str, List[str]] = Field( - default_factory=dict, - description="Mappings to other frameworks (CIS, NIST, etc.)" - ) - - model_config = ConfigDict(json_schema_extra={ - "example": { - "control_id": "IM-1", - "domain": "Identity Management", - "control_name": "Use centralized identity and authentication system", - "description": "Use a centralized identity and authentication system...", - "azure_policy_ids": ["4e6c27d5-a6ee-49cf-b2b4-d8fe90fa2b8b"], - "defender_recommendations": ["Enable MFA for all users"], - "related_frameworks": { - "CIS": ["CIS-5.1"], - "NIST": ["IA-2"] - } - } - }) - - class FrameworkUpload(BaseModel): """Model for uploaded framework control data.""" diff --git a/app/backend/app/models/mapping.py b/app/backend/app/models/mapping.py index 854789d..42d42db 100644 --- a/app/backend/app/models/mapping.py +++ b/app/backend/app/models/mapping.py @@ -83,15 +83,18 @@ class ControlMapping(BaseModel): external_control_id: str = Field(..., description="External framework control ID") external_control_name: str = Field(..., description="External control name") - mcsb_control_id: str = Field(..., description="Mapped MCSB control ID") - mcsb_control_name: str = Field(..., description="Mapped MCSB control name") - mcsb_domain: str = Field(..., description="MCSB security domain") - confidence_score: float = Field( ..., ge=0.0, le=1.0, - description="Confidence score for this mapping (0.0 to 1.0)" + description=( + "Confidence that the selected azure_policy_ids (or, for non-" + "enforceable controls, the coverage classification) genuinely " + "match this control's literal text - scored against the actual " + "Azure Policy definitions retrieved for it, not against any " + "intermediate control taxonomy. See SYSTEM_PROMPT for the " + "worked calibration examples this rubric is grounded in." + ), ) reasoning: str = Field(..., description="Explanation for why this mapping was chosen") @@ -106,6 +109,19 @@ class ControlMapping(BaseModel): description="Type of mapping relationship" ) + policy_category: Optional[str] = Field( + default=None, + description=( + "Grouping label derived from the catalog `category` of the " + "selected azure_policy_ids (e.g. 'Key Vault', 'Storage', " + "'Network'), or the external control's own domain when no policy " + "was selected. Always server-computed after the model responds - " + "never model-authored - because it is a resolvable fact about the " + "real catalog entries chosen, not a judgement call. Replaces the " + "old mcsb_domain fallback." + ), + ) + control_type: Optional[str] = Field( default=None, description=( @@ -312,13 +328,11 @@ class ControlMapping(BaseModel): "example": { "external_control_id": "SAMA-AC-01", "external_control_name": "Strong Authentication", - "mcsb_control_id": "IM-6", - "mcsb_control_name": "Use strong authentication controls", - "mcsb_domain": "Identity Management", "confidence_score": 0.92, "reasoning": "Both controls focus on enforcing MFA and strong authentication mechanisms", "azure_policy_ids": ["4e6c27d5-a6ee-49cf-b2b4-d8fe90fa2b8b"], "mapping_type": "exact", + "policy_category": "Identity", "control_type": "Technical", "coverage_category": "A_AzurePolicy", "azure_enforceable": True, diff --git a/app/backend/app/models/policy.py b/app/backend/app/models/policy.py index 96a9346..56a70b0 100644 --- a/app/backend/app/models/policy.py +++ b/app/backend/app/models/policy.py @@ -291,7 +291,14 @@ class ManualControlEntry(BaseModel): "'Microsoft attested'" ), ) - mcsb_control_id: str = Field("", description="Associated MCSB control ID, if any") + policy_category: str = Field( + "", + description=( + "Grouping label derived from the catalog category of any mapped " + "policies, or the control's own domain. Replaces the old " + "mcsb_control_id field." + ), + ) responsibility: str = Field( "", description=( diff --git a/app/backend/app/pipeline/initiative_builder.py b/app/backend/app/pipeline/initiative_builder.py index 25c5337..9911781 100644 --- a/app/backend/app/pipeline/initiative_builder.py +++ b/app/backend/app/pipeline/initiative_builder.py @@ -622,8 +622,7 @@ def _write_mappings_csv( "Coverage_Gap", "Outside_Step", "Enforcement_Plane", - "MCSB_Control_ID", - "MCSB_Control_Name", + "Policy_Category", "Confidence", "Azure_Policy_IDs", "Azure_Policy_Names", @@ -683,8 +682,7 @@ def _joined(values) -> str: "Coverage_Gap": str(getattr(m, "coverage_gap", False)) if m else "", "Outside_Step": (getattr(m, "outside_step", None) or "") if m else "", "Enforcement_Plane": (getattr(m, "enforcement_plane", None) or "") if m else "", - "MCSB_Control_ID": m.mcsb_control_id if m else "", - "MCSB_Control_Name": m.mcsb_control_name if m else "", + "Policy_Category": (getattr(m, "policy_category", None) or "") if m else "", "Confidence": f"{m.confidence_score:.2f}" if m else "", "Azure_Policy_IDs": policy_ids, "Azure_Policy_Names": policy_names, @@ -786,7 +784,7 @@ def _write_coverage_reports( "control_type", "coverage_category", "coverage_display", - "mcsb_control_id", + "policy_category", "responsibility", "evidence_source", "enforcement_plane", diff --git a/app/backend/app/pipeline/models.py b/app/backend/app/pipeline/models.py index e446be8..8812249 100644 --- a/app/backend/app/pipeline/models.py +++ b/app/backend/app/pipeline/models.py @@ -96,11 +96,15 @@ class ControlPolicyMapping(BaseModel): domain: str = Field( ..., description="Security domain" ) - mcsb_control_id: str = Field( - ..., description="Best matching MCSB control ID (e.g., 'IM-1', 'NS-1', 'DP-3')" - ) - mcsb_control_name: str = Field( - ..., description="Name of the matched MCSB control" + policy_category: Optional[str] = Field( + default=None, + description=( + "Grouping label derived from the catalog category of the mapped " + "Azure Policy definitions. Server-computed, never model-authored. " + "Replaces the old mcsb_control_id/mcsb_control_name/mcsb_domain " + "fields, which forced every control through a 10-control MCSB " + "demonstration set instead of the full ~2,467-definition catalog." + ), ) confidence_score: float = Field( ..., ge=0.0, le=1.0, diff --git a/app/backend/app/pipeline/policy_mapper.py b/app/backend/app/pipeline/policy_mapper.py index 95d92a8..1817ad7 100644 --- a/app/backend/app/pipeline/policy_mapper.py +++ b/app/backend/app/pipeline/policy_mapper.py @@ -123,8 +123,6 @@ def _placeholder(control: ExtractedControl, reason: str) -> ControlPolicyMapping control_id=control.control_id, control_title=control.control_title, domain=control.domain, - mcsb_control_id="", - mcsb_control_name="", confidence_score=0.0, mapping_rationale=reason, azure_policies=[], @@ -159,9 +157,10 @@ def _to_pipeline_mapping( return ControlPolicyMapping( control_id=mapping.external_control_id, control_title=mapping.external_control_name, - domain=(control.domain if control else None) or mapping.mcsb_domain or "", - mcsb_control_id=mapping.mcsb_control_id or "", - mcsb_control_name=mapping.mcsb_control_name or "", + domain=(control.domain if control else None) + or getattr(mapping, "policy_category", None) + or "", + policy_category=getattr(mapping, "policy_category", None), confidence_score=mapping.confidence_score, mapping_rationale=mapping.reasoning, azure_policies=policies, diff --git a/app/backend/app/pipeline/validator.py b/app/backend/app/pipeline/validator.py index 2a3d83d..bfa7456 100644 --- a/app/backend/app/pipeline/validator.py +++ b/app/backend/app/pipeline/validator.py @@ -35,10 +35,6 @@ def _identifier_exists(catalog, name: str) -> bool: return bool(catalog.exists(name)) -VALID_MCSB_PREFIXES = { "NS", "IM", "PA", "DP", "AM", "LT", "IR", "PV", "ES", "BR", "DS", "GS", -} - - def validate_mappings( extraction: ControlExtractionResult, mappings: list[ControlPolicyMapping], @@ -119,17 +115,6 @@ def validate_mappings( for mapping in mappings: control_id = mapping.control_id - # Check MCSB control ID format - if mapping.mcsb_control_id: - parts = mapping.mcsb_control_id.split("-") - if len(parts) < 2 or parts[0] not in VALID_MCSB_PREFIXES: - issues.append(ValidationIssue( - severity="warning", - control_id=control_id, - message=f"MCSB control ID '{mapping.mcsb_control_id}' has unexpected format", - suggestion=f"Expected format like 'NS-1', 'IM-6', 'DP-3'. Valid prefixes: {', '.join(sorted(VALID_MCSB_PREFIXES))}", - )) - # Check confidence score confidence_sum += mapping.confidence_score if mapping.confidence_score < min_confidence: diff --git a/app/backend/app/services/__init__.py b/app/backend/app/services/__init__.py index ce3e92d..2c5e41e 100644 --- a/app/backend/app/services/__init__.py +++ b/app/backend/app/services/__init__.py @@ -2,7 +2,6 @@ Services module for AI Control Mapping Agent. """ -from app.services.mcsb_service import MCSBService, get_mcsb_service from app.services.policy_catalog_service import PolicyCatalogService, get_policy_catalog_service from app.services.ai_mapping_service import AIMappingService, get_ai_mapping_service from app.services.policy_service import PolicyGenerationService, get_policy_service @@ -15,8 +14,6 @@ from app.services.graph_client import GraphClient, get_graph_client __all__ = [ - "MCSBService", - "get_mcsb_service", "PolicyCatalogService", "get_policy_catalog_service", "AIMappingService", diff --git a/app/backend/app/services/ai_mapping_service.py b/app/backend/app/services/ai_mapping_service.py index 433521c..3023f53 100644 --- a/app/backend/app/services/ai_mapping_service.py +++ b/app/backend/app/services/ai_mapping_service.py @@ -1,6 +1,6 @@ """ AI Mapping Service using Azure OpenAI with structured outputs. -Maps external framework controls to MCSB controls. +Maps external framework controls directly to Azure Policy definitions. Enhanced with Microsoft Learn MCP server for Azure Policy discovery. """ @@ -12,9 +12,8 @@ import openai from pydantic import ValidationError -from app.models import ExternalControl, MCSBControl, ControlMapping, MappingBatch +from app.models import ExternalControl, ControlMapping, MappingBatch from app.models.sovereignty import SovereigntyMapping -from app.services.mcsb_service import get_mcsb_service from app.services.microsoft_learn_client import get_microsoft_learn_client from app.services.policy_catalog_service import get_policy_catalog_service from app.services.control_intent_service import get_control_intent_service @@ -33,34 +32,92 @@ # System prompt for AI mapping -SYSTEM_PROMPT = """You are an expert cybersecurity compliance analyst specializing in mapping compliance framework controls to the Microsoft Cloud Security Benchmark (MCSB) and the Microsoft Sovereign Landing Zone (SLZ). +SYSTEM_PROMPT = """You are an expert cybersecurity compliance analyst specializing in mapping compliance framework controls directly to Azure Policy (built-in policy definitions and the initiatives that bundle them, including Microsoft Defender for Cloud's own initiatives) and to the Microsoft Sovereign Landing Zone (SLZ). Your task is to analyze external compliance framework controls and: -1. Map them to the most appropriate MCSB controls +1. Map them to the Azure Policy definition(s) that genuinely enforce or evidence them 2. Recommend the appropriate Sovereign Landing Zone (SLZ) sovereignty level and policies -## MCSB Mapping Guidelines - -For each external control, you should: -1. Understand the primary security objective and intent -2. Identify the security domain (Identity, Network, Data Protection, etc.) -3. Analyze technical requirements and implementation guidance -4. Match to the most appropriate MCSB control(s) -5. Provide a confidence score (0.0 to 1.0) based on alignment -6. Explain your reasoning clearly - -Confidence Score Guidelines: -- 0.9-1.0: Exact match - controls have identical or nearly identical objectives -- 0.7-0.8: Strong match - controls address the same security goal with similar requirements -- 0.5-0.6: Partial match - controls share some common objectives but differ in scope -- 0.3-0.4: Conceptual match - controls are related but address different aspects -- 0.0-0.2: Weak or no match - controls are fundamentally different +## Azure Policy Mapping Guidelines + +There is no intermediate control taxonomy standing between the external control and +Azure Policy. Map directly against the real candidates: + +1. Understand the primary security objective, intent, and literal wording of the + external control - what it actually requires, not a paraphrase of it. +2. Review the "Azure Policy Context" section below: real built-in Azure Policy + definitions retrieved for THIS control from the full built-in catalog (~2,467 + definitions), which also names the built-in initiatives that already bundle a + candidate (e.g. "Microsoft cloud security benchmark", "ASC Default", or another + Microsoft Defender for Cloud initiative) when one exists. There is no smaller + pre-filtered subset behind these candidates - if a real Azure Policy or Defender + for Cloud initiative enforces this control, it is reachable here. +3. Select every azure_policy_ids GUID whose own display name, description, and + effect genuinely enforce or evidence the control's literal requirement. Azure + Policy definitions that back Microsoft Defender for Cloud recommendations, + configurations, and initiatives are valid, in-scope candidates like any other + built-in. A candidate initiative name shown for context is not itself a + selectable ID - only its GUIDs go in azure_policy_ids. +4. Provide a confidence score and mapping type based on how closely the TEXT of + the control matches the TEXT of the Azure Policy definition(s) you selected - + never on fit to any external taxonomy. If nothing in the candidate list truly + enforces the control, say so honestly (empty azure_policy_ids, mapping_type + "none") instead of forcing a weak match to look like a successful one. +5. Explain your reasoning clearly, citing what in the control's own wording the + selected policy addresses. + +Confidence Score Guidelines — grounded in real expert-verified mappings, not the +abstract categories they name: + +- 0.9-1.0 ("exact"): the selected policy/policies enforce the control's literal + subject and mechanism directly. + Worked example: control text "...consumers shall implement it [HYOK] to retain + exclusive control over encryption keys and mitigate the risk of unauthorized + access to data" maps at ~0.95 to the built-in policies "OS and data disks should + be encrypted with a customer-managed key", "Storage accounts should use + customer-managed key for encryption", and "SQL servers should use + customer-managed keys to encrypt data at rest" - the control's subject + (customer-held encryption keys) and the policies' mechanism (CMK enforcement) + are the same requirement, just phrased in regulatory vs. Azure vocabulary. + +- 0.7-0.8 ("exact"/"partial"): the policy addresses the same security goal, but + only part of the control's scope, or several policies must combine to + approximate full coverage. + Worked example: a control requiring encryption "at rest ... in use ... and in + transmission" across "file servers, databases, and end-user devices" maps at + ~0.75 to a list of resource-specific CMK policies (SQL, storage, managed disks, + PostgreSQL, etc.) plus a transit-specific policy such as "Secure transfer to + storage accounts should be enabled" - each candidate covers one resource type or + one leg of the requirement, not the whole sentence, so no single policy is an + exact match even though the combination is a strong one. + +- 0.5-0.6 ("partial"/"conceptual"): the control's intent is achievable in Azure, + but primarily through configuration Azure Policy itself cannot enforce or audit + (Entra Conditional Access, Purview labelling, key management outside ARM). + Score in this band, set coverage_category to "B_AzureConfig", describe the + configuration step in outside_step, and only include azure_policy_ids for a + definition that genuinely audits some part of it. + Worked example: "Multi-factor authentication shall be implemented for accounts + with elevated privileges" is delivered through an Entra Conditional Access + policy, which Azure Policy (ARM) cannot itself configure or audit - score + around 0.5-0.6 and classify "B_AzureConfig", not "A_AzurePolicy". + +- 0.0-0.3 ("conceptual"/"none"): no candidate policy or initiative addresses the + control, or the control is process/organisational and Azure has no technical + means to enforce it at all. + Worked example: "senior leadership shall mandate the establishment of a cloud + security program with apparent oversight" is pure governance - score 0.0, + coverage_category "C_Process", empty azure_policy_ids. Do NOT reach for a + governance catch-all policy just to attach something. Mapping Type Guidelines: -- "exact": Controls have identical security objectives and requirements -- "partial": Controls share primary objectives but differ in implementation details -- "conceptual": Controls are related conceptually but address different scopes -- "none": No appropriate MCSB control exists for this requirement +- "exact": the selected policy/policies enforce the control's literal subject and + mechanism directly (confidence typically 0.8-1.0) +- "partial": the selected policy/policies address the same goal but cover only + part of the control's scope, or require several policies combined +- "conceptual": related in intent but achieved mainly through configuration Azure + Policy cannot itself enforce, or only loosely related +- "none": no Azure Policy definition or initiative can address this requirement ## Sovereign Landing Zone (SLZ) Mapping Guidelines @@ -92,7 +149,7 @@ Not every control can be enforced by Azure Policy. Many compliance frameworks contain process, legal, HR, contractual, and organisational-governance controls that Azure has no technical means to enforce. Attaching an Azure Policy to these -(especially a catch-all MCSB "Governance & Strategy" entry) is a FALSE POSITIVE. +just to appear complete is a FALSE POSITIVE. Classify every control into exactly one coverage_category: - "A_AzurePolicy": technically enforceable or auditable by an Azure Policy @@ -113,13 +170,19 @@ for Microsoft-operated items) and return an EMPTY azure_policy_ids list. Do NOT reach for a governance catch-all policy just to attach something. -## Defender for Cloud Recommendations +## Defender for Cloud -You have no access to a live Microsoft Defender for Cloud subscription and no -data about actual recommendation state. ALWAYS return an EMPTY -defender_recommendations list. Never invent or guess a Defender for Cloud -recommendation name - an invented one is indistinguishable from a real one to -the reader and is worse than reporting nothing. +Microsoft Defender for Cloud's underlying built-in policies, configurations, and +initiatives (e.g. "Microsoft cloud security benchmark", "ASC Default") are +legitimate Azure Policy candidates and should be mapped and considered like any +other built-in when the "Azure Policy Context" shows one enforces the control. + +This is distinct from Defender for Cloud RECOMMENDATIONS (free-text names such as +"Enable MFA for all users"): you have no access to a live Defender for Cloud +subscription and no data about actual recommendation state, so ALWAYS return an +EMPTY defender_recommendations list. Never invent or guess a recommendation name +- an invented one is indistinguishable from a real one to the reader and is worse +than reporting nothing. Always be conservative with confidence scores - it's better to flag uncertain mappings for human review.""" @@ -135,22 +198,18 @@ def __init__(self): self.control_intent = get_control_intent_service() self.control_classification = get_control_classification_service() self.policy_rerank = get_policy_rerank_service() - self.mcsb_service = get_mcsb_service() self.sovereignty_service = get_sovereignty_service() self.model = settings.azure_openai_deployment_name async def map_control( self, external_control: ExternalControl, - mcsb_controls: Optional[List[MCSBControl]] = None ) -> ControlMapping: """ - Map a single external control to MCSB using AI. + Map a single external control directly to Azure Policy using AI. Args: external_control: External framework control to map - mcsb_controls: Optional list of MCSB controls to consider - (if None, uses all controls) Returns: ControlMapping with AI-generated mapping @@ -160,13 +219,6 @@ async def map_control( """ logger.info(f"Mapping control: {external_control.control_id}") - # Get relevant MCSB controls - if mcsb_controls is None: - mcsb_controls = self.mcsb_service.get_controls_for_external_control( - external_control.description, - external_control.domain - ) - # Stage 1 — classify BLIND, before any policy candidates exist. Showing a # ranked candidate list first anchors the model into attaching a policy to # a control Azure cannot enforce; on the NCSP gold mapping 113 of 137 @@ -199,9 +251,8 @@ async def map_control( logger.debug(f"Sovereignty context ready, length: {len(sovereignty_context)} chars") # Create user prompt with policy context - logger.debug(f"Creating AI mapping prompt with {len(mcsb_controls)} MCSB controls") - user_prompt = self._create_mapping_prompt(external_control, mcsb_controls, policy_context, sovereignty_context) - logger.info(f"Generated prompt for AI ({len(user_prompt)} chars) with {len(mcsb_controls)} MCSB controls and policy context") + user_prompt = self._create_mapping_prompt(external_control, policy_context, sovereignty_context) + logger.info(f"Generated prompt for AI ({len(user_prompt)} chars) with policy context") logger.debug(f"Prompt preview: {user_prompt[:300]}...") try: @@ -249,8 +300,17 @@ async def map_control( self._apply_procedural_sovereignty(mapping, external_control) + # policy_category is a resolvable fact about the real catalog + # entries selected, not a judgement call, so it is always computed + # here rather than asked of the model - same rationale as + # _strip_ungrounded_defender_recommendations above. This replaces + # the old mcsb_domain fallback, which came from a 10-control demo + # taxonomy rather than the actual policies chosen. + self._set_policy_category(mapping, external_control) + logger.info( - f"Mapped {external_control.control_id} -> {mapping.mcsb_control_id} " + f"Mapped {external_control.control_id} -> {len(mapping.azure_policy_ids or [])} " + f"Azure Policy definition(s) " f"(confidence: {mapping.confidence_score:.2f}, " f"coverage: {mapping.coverage_category})" ) @@ -490,19 +550,39 @@ async def _search_azure_policies( desc = (c.description or "").strip().replace("\n", " ") if len(desc) > 220: desc = desc[:217] + "..." + # Surface built-in initiatives (e.g. "Microsoft cloud security + # benchmark", "ASC Default", other Defender for Cloud + # initiatives) that already bundle this candidate, as context + # only - initiatives are not directly selectable, only the + # definition GUIDs inside azure_policy_ids are. + initiative_note = "" + try: + initiatives = self.catalog.initiatives_containing(c.name) + except Exception: + initiatives = [] + if initiatives: + names = ", ".join( + i.get("display_name") or i.get("name", "") + for i in initiatives[:3] + ) + initiative_note = f"\n Bundled in built-in initiative(s): {names}" lines.append( f" - {c.display_name} [{c.category}]\n" f" ID: {c.name}\n" f" {desc}" + f"{initiative_note}" ) return ( "Candidate Azure Policy definitions (retrieved from the Azure " - "built-in policy catalog):\n" + "built-in policy catalog, including Microsoft Defender for " + "Cloud's own policies where relevant):\n" f"{len(candidates)} candidates ranked by relevance.\n" + "\n".join(lines) + "\n\nSelect azure_policy_ids ONLY from the ID (GUID) values " "listed above that genuinely enforce this control. You may " - "select several. Do NOT invent GUIDs or use policy names as IDs. " + "select several. Do NOT invent GUIDs or use policy names or " + "initiative names as IDs - only a definition GUID belongs in " + "azure_policy_ids, never an initiative. " "Prefer enforceable policies (Audit/Deny/DeployIfNotExists) over " "'Regulatory Compliance' entries, which are manual-attestation " "controls with no enforcement logic - only pick those if no " @@ -529,7 +609,6 @@ async def _search_azure_policies( def _create_mapping_prompt( self, external_control: ExternalControl, - mcsb_controls: List[MCSBControl], policy_context: str = "", sovereignty_context: str = "" ) -> str: @@ -538,24 +617,12 @@ def _create_mapping_prompt( Args: external_control: External control to map - mcsb_controls: Available MCSB controls policy_context: Azure Policy search results from Microsoft Learn sovereignty_context: SLZ sovereignty policy context Returns: Formatted prompt string """ - # Prepare MCSB controls context - mcsb_context = [] - for ctrl in mcsb_controls: - mcsb_context.append({ - "control_id": ctrl.control_id, - "domain": ctrl.domain, - "control_name": ctrl.control_name, - "description": ctrl.description[:200], # Truncate for token efficiency - "azure_policy_ids": ctrl.azure_policy_ids - }) - prompt = f""" External Control to Map: ----------------------- @@ -565,10 +632,6 @@ def _create_mapping_prompt( Domain: {external_control.domain or 'Not specified'} Control Type: {external_control.control_type or 'Not specified'} -Available MCSB Controls: ------------------------ -{json.dumps(mcsb_context, indent=2)} - Azure Policy Context: -------------------- {policy_context} @@ -577,16 +640,17 @@ def _create_mapping_prompt( Task: ----- -1. MCSB Mapping: Analyze the external control and identify the best matching MCSB control. - Provide a confidence score, mapping type, and detailed reasoning. - -2. Azure Policy selection: From the "Azure Policy Context" section below, select the - Azure Policy definition GUIDs that genuinely enforce this control and put them in - azure_policy_ids. Use ONLY the ID (GUID) values listed there. Select as many as truly - apply (there may be several). Never invent GUIDs, and never put a policy name or MCSB - control ID in azure_policy_ids. If none of the candidates fit, return an empty list. - -3. Sovereignty Mapping: Determine the appropriate SLZ sovereignty level (L1/L2/L3), +1. Azure Policy selection: From the "Azure Policy Context" section above, select the + Azure Policy definition GUIDs that genuinely enforce this control's literal + requirement and put them in azure_policy_ids. Use ONLY the ID (GUID) values listed + there. Select as many as truly apply (there may be several, and a genuine match + may span several resource-specific definitions). Never invent GUIDs, and never put + a policy name or initiative name in azure_policy_ids. If none of the candidates + fit, return an empty list. Score confidence_score and mapping_type against how + closely the selected definition(s) match the control's own wording (see the + worked calibration examples above), and explain that match in reasoning. + +2. Sovereignty Mapping: Determine the appropriate SLZ sovereignty level (L1/L2/L3), relevant sovereignty control objectives (SO-1 through SO-5), and matching SLZ policies. Provide the sovereignty mapping in the 'sovereignty' field with: - sovereignty_level: "L1", "L2", or "L3" @@ -617,6 +681,31 @@ def _strip_ungrounded_defender_recommendations(mapping) -> None: if getattr(mapping, "defender_recommendations", None): mapping.defender_recommendations = [] + def _set_policy_category(self, mapping, external_control: ExternalControl) -> None: + """Derive ``policy_category`` from the catalog, not from the model. + + Replaces the old ``mcsb_domain`` fallback. The category of a resolved + Azure Policy definition is a fact recorded in the catalog snapshot - + looking it up is strictly more accurate than asking the model to + restate a taxonomy label, and it stays truthful even when the model's + own domain guess would have been stale or invented. Falls back to the + external control's own extracted domain when no policy was selected + (process/organisational controls, or a coverage gap). + """ + categories: list[str] = [] + for policy_id in (mapping.azure_policy_ids or []): + entry = self.catalog.get(policy_id) if self.catalog else None + category = (entry or {}).get("category") + if category: + categories.append(category) + + if categories: + # Most common category among the selected definitions; ties break + # on first-seen order, which is retrieval-rank order. + mapping.policy_category = max(set(categories), key=categories.count) + else: + mapping.policy_category = external_control.domain or None + def _apply_procedural_sovereignty(self, mapping, external_control) -> None: """Name the Azure feature that meets a sovereignty objective with no policy. @@ -763,13 +852,11 @@ def _create_fallback_mapping( return ControlMapping( external_control_id=external_control.control_id, external_control_name=external_control.control_name, - mcsb_control_id="N/A", - mcsb_control_name="Mapping failed - manual review required", - mcsb_domain="Unknown", confidence_score=0.0, reasoning=f"Automated mapping failed: {error_msg}. This control requires manual review and mapping.", azure_policy_ids=[], mapping_type="none", + policy_category=external_control.domain or None, defender_recommendations=[], control_type=external_control.control_type, coverage_category=coverage.COVERAGE_C, diff --git a/app/backend/app/services/coverage.py b/app/backend/app/services/coverage.py index 321a5dd..26fd3a2 100644 --- a/app/backend/app/services/coverage.py +++ b/app/backend/app/services/coverage.py @@ -832,7 +832,7 @@ def manual_register_rows(mappings) -> List[dict]: "control_type": getattr(m, "control_type", None) or "", "coverage_category": category, "coverage_display": coverage_display_name(category), - "mcsb_control_id": m.mcsb_control_id, + "policy_category": getattr(m, "policy_category", None) or "", "responsibility": getattr(m, "responsibility", None) or "", "evidence_source": getattr(m, "evidence_source", None) or "", "enforcement_plane": ( @@ -1062,7 +1062,7 @@ def manual_controls_csv(mappings) -> str: "control_type", "coverage_category", "coverage_display", - "mcsb_control_id", + "policy_category", "responsibility", "evidence_source", "enforcement_plane", diff --git a/app/backend/app/services/mcsb_service.py b/app/backend/app/services/mcsb_service.py deleted file mode 100644 index ff4fd88..0000000 --- a/app/backend/app/services/mcsb_service.py +++ /dev/null @@ -1,360 +0,0 @@ -""" -MCSB (Microsoft Cloud Security Benchmark) data service. -Loads, indexes, and provides search functionality for MCSB controls. -""" - -import json -import logging -from pathlib import Path -from typing import List, Dict, Optional -from functools import lru_cache - -from app.models import MCSBControl -from app.config import get_settings - -logger = logging.getLogger(__name__) -settings = get_settings() - - -class MCSBService: - """Service for loading and searching MCSB controls.""" - - def __init__(self, data_path: Optional[str] = None): - """ - Initialize MCSB service. - - Args: - data_path: Path to MCSB controls JSON file - """ - self.data_path = data_path or settings.mcsb_data_path - self._controls: List[MCSBControl] = [] - self._controls_by_id: Dict[str, MCSBControl] = {} - self._controls_by_domain: Dict[str, List[MCSBControl]] = {} - self._loaded = False - # True whenever the loaded set is the small illustrative fallback - # below rather than a real MCSB catalog file - the shipped image has - # never actually included ``data_path`` (see docs/BACKLOG.md B4), so - # this has been True in every deployment to date. Surfaced via - # /api/v1/health and /api/v1/mcsb/controls so a caller isn't misled - # into thinking it has the full published benchmark. - self._is_demonstration_data = False - - @property - def is_demonstration_data(self) -> bool: - """True when the loaded controls are the illustrative fallback set, - not a real MCSB catalog file.""" - if not self._loaded: - self.load_controls() - return self._is_demonstration_data - - def load_controls(self) -> None: - """Load MCSB controls from JSON file.""" - if self._loaded: - logger.info("MCSB controls already loaded") - return - - try: - file_path = Path(self.data_path) - - if not file_path.exists(): - logger.warning(f"MCSB data file not found: {file_path}") - # Load from existing ComplianceIQ catalogs as fallback - self._load_from_existing_catalogs() - return - - logger.info(f"Loading MCSB controls from {file_path}") - - with open(file_path, 'r', encoding='utf-8') as f: - data = json.load(f) - - # Parse controls - if isinstance(data, list): - controls_data = data - elif isinstance(data, dict) and 'controls' in data: - controls_data = data['controls'] - else: - raise ValueError("Invalid MCSB data format") - - self._controls = [MCSBControl(**ctrl) for ctrl in controls_data] - - # Build indexes - self._build_indexes() - - self._loaded = True - self._is_demonstration_data = False - logger.info(f"Successfully loaded {len(self._controls)} MCSB controls") - - except Exception as e: - logger.error(f"Failed to load MCSB controls: {e}") - # Load from existing catalogs as fallback - self._load_from_existing_catalogs() - - def _load_from_existing_catalogs(self) -> None: - """ - Load MCSB-like structure from existing ComplianceIQ catalogs. - This provides a fallback if the MCSB JSON file doesn't exist. - """ - logger.info("Loading MCSB data from existing ComplianceIQ catalogs") - - try: - import pandas as pd - - # Load existing SAMA catalog as reference (bundled into the image - # under app/data/catalogues/; resolved robustly relative to this file). - catalog_path = ( - Path(__file__).resolve().parent.parent - / "data" / "catalogues" / "SAMA_Catalog_Azure_Mappings.csv" - ) - - if not catalog_path.exists(): - logger.warning("No existing catalogs found, creating default controls") - self._create_default_controls() - return - - df = pd.read_csv(catalog_path) - - # Extract unique MCSB-like controls from mappings - # This is simplified - in reality you'd parse the actual MCSB structure - self._create_default_controls() - - except Exception as e: - logger.error(f"Failed to load from existing catalogs: {e}") - self._create_default_controls() - - def _create_default_controls(self) -> None: - """Create default MCSB controls for demonstration.""" - logger.info("Creating default MCSB control set") - - default_controls = [ - { - "control_id": "IM-1", - "domain": "Identity Management", - "control_name": "Use centralized identity and authentication system", - "description": "Use a centralized identity and authentication system to manage organizational identities for people and services.", - "azure_policy_ids": [], - "defender_recommendations": ["Enable MFA for all users"], - "related_frameworks": {"CIS": ["CIS-5.1"], "NIST": ["IA-2"]} - }, - { - "control_id": "IM-6", - "domain": "Identity Management", - "control_name": "Use strong authentication controls", - "description": "Use strong authentication controls including MFA and passwordless authentication.", - "azure_policy_ids": ["4e6c27d5-a6ee-49cf-b2b4-d8fe90fa2b8b"], - "defender_recommendations": ["Enable MFA - Require multifactor authentication for all users"], - "related_frameworks": {"CIS": ["CIS-5.2"], "NIST": ["IA-2(1)"]} - }, - { - "control_id": "PA-7", - "domain": "Privileged Access", - "control_name": "Follow just enough administration (least privilege) principle", - "description": "Follow the principle of least privilege when granting access.", - "azure_policy_ids": [], - "defender_recommendations": ["Manage access and permissions"], - "related_frameworks": {"CIS": ["CIS-5.4"], "NIST": ["AC-6"]} - }, - { - "control_id": "NS-2", - "domain": "Network Security", - "control_name": "Secure cloud services with network controls", - "description": "Protect cloud services by implementing network security controls.", - "azure_policy_ids": ["ca610c1d-041c-4332-9d88-7ed3094967c7"], - "defender_recommendations": ["Restrict unauthorized network access"], - "related_frameworks": {"CIS": ["CIS-9.2"], "NIST": ["SC-7"]} - }, - { - "control_id": "DP-3", - "domain": "Data Protection", - "control_name": "Encrypt sensitive data in transit", - "description": "Encrypt sensitive data in transit using approved cryptographic protocols.", - "azure_policy_ids": [], - "defender_recommendations": ["Encrypt data in transit"], - "related_frameworks": {"CIS": ["CIS-3.7"], "NIST": ["SC-8"]} - }, - { - "control_id": "DP-5", - "domain": "Data Protection", - "control_name": "Use customer-managed key option in data at rest encryption", - "description": "Use customer-managed keys for encryption at rest when required.", - "azure_policy_ids": ["18adea5e-f416-4d0f-8aa8-d24321e3e274"], - "defender_recommendations": ["Enable encryption at rest"], - "related_frameworks": {"CIS": ["CIS-3.5"], "NIST": ["SC-28"]} - }, - { - "control_id": "LT-3", - "domain": "Logging and Threat Detection", - "control_name": "Enable logging for security investigation", - "description": "Enable logging capabilities for security investigation and monitoring.", - "azure_policy_ids": [], - "defender_recommendations": ["Enable auditing and logging"], - "related_frameworks": {"CIS": ["CIS-6.1"], "NIST": ["AU-2"]} - }, - { - "control_id": "PV-3", - "domain": "Posture and Vulnerability Management", - "control_name": "Establish and maintain a secure configuration process", - "description": "Establish and maintain security configuration standards.", - "azure_policy_ids": [], - "defender_recommendations": ["Apply system updates", "Remediate vulnerabilities"], - "related_frameworks": {"CIS": ["CIS-5.1"], "NIST": ["CM-6"]} - }, - { - "control_id": "BR-1", - "domain": "Backup and Recovery", - "control_name": "Ensure regular automated backups", - "description": "Implement automated backup solutions for critical data and systems.", - "azure_policy_ids": [], - "defender_recommendations": ["Enable backup for critical resources"], - "related_frameworks": {"CIS": ["CIS-10.1"], "NIST": ["CP-9"]} - }, - { - "control_id": "GS-1", - "domain": "Governance and Strategy", - "control_name": "Establish security governance and compliance program", - "description": "Establish a comprehensive security governance program.", - "azure_policy_ids": [], - "defender_recommendations": ["Implement security policies"], - "related_frameworks": {"CIS": ["CIS-1.1"], "NIST": ["PM-1"]} - } - ] - - self._controls = [MCSBControl(**ctrl) for ctrl in default_controls] - self._build_indexes() - self._loaded = True - self._is_demonstration_data = True - - logger.info(f"Created {len(self._controls)} default MCSB controls") - - def _build_indexes(self) -> None: - """Build lookup indexes for faster searching.""" - self._controls_by_id = {ctrl.control_id: ctrl for ctrl in self._controls} - - # Group by domain - self._controls_by_domain = {} - for ctrl in self._controls: - if ctrl.domain not in self._controls_by_domain: - self._controls_by_domain[ctrl.domain] = [] - self._controls_by_domain[ctrl.domain].append(ctrl) - - logger.debug(f"Built indexes: {len(self._controls_by_id)} controls, " - f"{len(self._controls_by_domain)} domains") - - def get_all_controls(self) -> List[MCSBControl]: - """ - Get all MCSB controls. - - Returns: - List of all MCSB controls - """ - if not self._loaded: - self.load_controls() - return self._controls - - def get_control_by_id(self, control_id: str) -> Optional[MCSBControl]: - """ - Get MCSB control by ID. - - Args: - control_id: MCSB control ID (e.g., "IM-1") - - Returns: - MCSB control or None if not found - """ - if not self._loaded: - self.load_controls() - return self._controls_by_id.get(control_id) - - def get_controls_by_domain(self, domain: str) -> List[MCSBControl]: - """ - Get all controls in a specific domain. - - Args: - domain: Security domain name - - Returns: - List of controls in the domain - """ - if not self._loaded: - self.load_controls() - return self._controls_by_domain.get(domain, []) - - def get_all_domains(self) -> List[str]: - """ - Get all unique domain names. - - Returns: - List of domain names - """ - if not self._loaded: - self.load_controls() - return list(self._controls_by_domain.keys()) - - def search_by_keywords(self, keywords: List[str]) -> List[MCSBControl]: - """ - Search controls by keywords in name and description. - - Args: - keywords: List of keywords to search for - - Returns: - List of matching controls - """ - if not self._loaded: - self.load_controls() - - results = [] - keywords_lower = [kw.lower() for kw in keywords] - - for ctrl in self._controls: - # Search in control name and description - search_text = f"{ctrl.control_name} {ctrl.description}".lower() - - # Check if any keyword matches - if any(kw in search_text for kw in keywords_lower): - results.append(ctrl) - - logger.debug(f"Keyword search for {keywords} returned {len(results)} results") - return results - - def get_controls_for_external_control( - self, - external_control_description: str, - external_control_domain: Optional[str] = None - ) -> List[MCSBControl]: - """ - Get relevant MCSB controls for an external control. - Used to provide context to the AI mapping service. - - Args: - external_control_description: Description of external control - external_control_domain: Optional domain hint - - Returns: - List of potentially relevant MCSB controls - """ - if not self._loaded: - self.load_controls() - - # If domain is provided, filter by domain first - if external_control_domain: - # Try exact match - domain_controls = self.get_controls_by_domain(external_control_domain) - if domain_controls: - return domain_controls - - # Try fuzzy match on domain names - for domain in self._controls_by_domain.keys(): - if external_control_domain.lower() in domain.lower(): - return self._controls_by_domain[domain] - - # Otherwise, return all controls for the AI to analyze - # In production, you might want to use semantic search here - return self._controls - - -@lru_cache -def get_mcsb_service() -> MCSBService: - """Get cached MCSB service instance.""" - service = MCSBService() - service.load_controls() - return service diff --git a/app/backend/app/services/policy_service.py b/app/backend/app/services/policy_service.py index ea684fe..e8a5cab 100644 --- a/app/backend/app/services/policy_service.py +++ b/app/backend/app/services/policy_service.py @@ -417,7 +417,7 @@ def _create_policy_definitions( group = PolicyDefinitionGroup( name=group_name, display_name=display_name, - category=mapping.mcsb_domain or None, + category=mapping.policy_category or None, description=mapping.reasoning or None, ) groups_by_name[group_name] = group diff --git a/app/frontend/app.py b/app/frontend/app.py index 6987249..1d775e5 100644 --- a/app/frontend/app.py +++ b/app/frontend/app.py @@ -119,7 +119,7 @@ if health.get("status") == "healthy": st.success("Backend connected") - st.caption(f"MCSB Controls: {health.get('mcsb_controls_loaded', 0)}") + st.caption(f"Azure Policy catalog: {health.get('policy_catalog_count', 0)}") slz_count = health.get("slz_policy_count", 0) if slz_count > 0: diff --git a/app/frontend/pages/0_Platform_Selection.py b/app/frontend/pages/0_Platform_Selection.py index c1c6cbb..4f75e52 100644 --- a/app/frontend/pages/0_Platform_Selection.py +++ b/app/frontend/pages/0_Platform_Selection.py @@ -50,7 +50,7 @@ **Capabilities:** - Azure Policy Initiatives - - MCSB Control Mapping + - Direct Azure Policy Control Mapping - Defender for Cloud Onboarding (Regulatory Compliance) - Sovereign Landing Zone (SLZ) Policies diff --git a/app/frontend/pages/1_Upload_Controls.py b/app/frontend/pages/1_Upload_Controls.py index ed6e547..07e7bf4 100644 --- a/app/frontend/pages/1_Upload_Controls.py +++ b/app/frontend/pages/1_Upload_Controls.py @@ -336,7 +336,7 @@ def _auto_detect_columns() -> None: # Show navigation after controls are loaded (persists across reruns) if st.session_state.get('controls_loaded') and st.session_state.controls: st.markdown("---") - st.info("👉 Go to **AI Mapping** to start mapping these controls to MCSB") + st.info("👉 Go to **AI Mapping** to start mapping these controls to Azure Policy") if st.button("Continue to AI Mapping →", type="primary"): st.switch_page("pages/2_AI_Mapping.py") diff --git a/app/frontend/pages/2_AI_Mapping.py b/app/frontend/pages/2_AI_Mapping.py index abde8f7..2cfe478 100644 --- a/app/frontend/pages/2_AI_Mapping.py +++ b/app/frontend/pages/2_AI_Mapping.py @@ -201,9 +201,7 @@ def _session_mapping_from_result(mapping: dict, controls: list) -> dict: ), None, ), - "mcsb_control_id": mapping.get("mcsb_control_id", "N/A"), - "mcsb_control_name": mapping.get("mcsb_control_name", "N/A"), - "mcsb_domain": mapping.get("mcsb_domain", "N/A"), + "policy_category": mapping.get("policy_category"), "confidence_score": mapping.get("confidence_score", 0.0), "reasoning": mapping.get("reasoning", ""), "azure_policy_ids": mapping.get("azure_policy_ids", []), @@ -275,11 +273,6 @@ def _complete_mapping_job(job_id: str, status: dict) -> None: "domain": m.get("domain"), "confidence_score": m.get("confidence_score", 0.0), "reasoning": m.get("reasoning", ""), - "mcsb_mappings": [{ - "mcsbControlId": m.get("mcsb_control_id"), - "mcsbControlName": m.get("mcsb_control_name"), - "mcsbDomain": m.get("mcsb_domain"), - }], "policy_recommendations": m.get("azure_policy_ids", []), } for m in mappings @@ -426,7 +419,7 @@ def _render_active_mapping_job( with col_result1: st.markdown("#### 📊 Mapping Result") st.metric("Confidence Score", f"{result['confidence_score']:.0%}") - st.metric("MCSB Control", result['mcsb_control_id']) + st.metric("Azure Policies Matched", len(result.get('azure_policy_ids') or [])) st.metric("Mapping Type", result['mapping_type'].replace('_', ' ').title()) with col_result2: @@ -490,13 +483,14 @@ def _render_active_mapping_job( ) st.metric("High Confidence (≥80%)", high_confidence) with col_sum3: - unique_mcsb = len( + unique_categories = len( { - mapping.get("mcsb_control_id", "") + mapping.get("policy_category", "") for mapping in mappings + if mapping.get("policy_category") } ) - st.metric("Unique MCSB Controls", unique_mcsb) + st.metric("Unique Policy Categories", unique_categories) st.info("👉 Go to **Review & Edit** to validate the mappings") st.page_link( "pages/3_Review_Edit.py", @@ -591,7 +585,7 @@ def _render_active_mapping_job( { 'Control ID': m.get('control_id', m.get('external_control_id', 'N/A')), 'Control Name': m.get('control_name', m.get('external_control_name', 'N/A')), - 'MCSB Control': m.get('mcsb_control_id', 'N/A'), + 'Policy Category': m.get('policy_category', 'N/A'), 'Confidence': f"{m.get('confidence_score', 0):.0%}", 'SLZ Level': (m.get('sovereignty') or {}).get('sovereignty_level', '—'), 'Type': m.get('mapping_type', 'unknown') diff --git a/app/frontend/pages/3_Review_Edit.py b/app/frontend/pages/3_Review_Edit.py index 86d9f8d..5c456b7 100644 --- a/app/frontend/pages/3_Review_Edit.py +++ b/app/frontend/pages/3_Review_Edit.py @@ -44,28 +44,6 @@ # Get API client api_client = get_api_client() -# Load MCSB controls for reference (cached) -# Fix stale cache: force refetch if data is not a list-of-dicts -_mcsb = st.session_state.mcsb_controls -if _mcsb is not None and (isinstance(_mcsb, dict) or (isinstance(_mcsb, list) and _mcsb and not isinstance(_mcsb[0], dict))): - st.session_state.mcsb_controls = None - -if st.session_state.mcsb_controls is None: - with st.spinner("Loading MCSB controls..."): - try: - data = api_client.get_mcsb_controls() - # Unwrap {"controls": [...]} if the module cache served old code - if isinstance(data, dict) and "controls" in data: - data = data["controls"] - st.session_state.mcsb_controls = data - except Exception as e: - st.error(f"❌ Error loading MCSB controls: {str(e)}") - st.session_state.mcsb_controls = [] - -# Create MCSB lookup dictionary -mcsb_lookup = {c['control_id']: c for c in st.session_state.mcsb_controls} if st.session_state.mcsb_controls else {} -mcsb_options = sorted([c['control_id'] for c in st.session_state.mcsb_controls]) if st.session_state.mcsb_controls else [] - # Display summary col1, col2, col3, col4, col5 = st.columns(5) @@ -103,13 +81,16 @@ ) with col_filter2: - # Get unique MCSB domains - domains = sorted(set(mcsb_lookup[m.get('mcsb_control_id', '')].get('domain', 'Unknown') - for m in st.session_state.mappings - if m.get('mcsb_control_id', '') in mcsb_lookup)) - + # Get unique policy categories (server-computed from the mapped Azure + # Policy definitions' own catalog category - see policy_category on + # ControlMapping - rather than a fixed MCSB domain list). + domains = sorted(set( + m.get('policy_category') for m in st.session_state.mappings + if m.get('policy_category') + )) + domain_filter = st.selectbox( - "MCSB Domain", + "Policy Category", options=["All"] + domains, index=0 ) @@ -141,10 +122,9 @@ elif confidence_filter == "Low (<60%)": filtered_mappings = [m for m in filtered_mappings if m.get('confidence_score', 0) < 0.6] -if domain_filter != "All" and mcsb_lookup: - filtered_mappings = [m for m in filtered_mappings - if m.get('mcsb_control_id', '') in mcsb_lookup - and mcsb_lookup[m.get('mcsb_control_id', '')].get('domain') == domain_filter] +if domain_filter != "All": + filtered_mappings = [m for m in filtered_mappings + if m.get('policy_category') == domain_filter] if type_filter != "All": filtered_mappings = [m for m in filtered_mappings if m.get('mapping_type') == type_filter] @@ -173,7 +153,8 @@ for idx, mapping in enumerate(filtered_mappings): with st.expander( f"{'⚠️' if mapping.get('confidence_score', 0) < 0.6 else '✅'} " - f"{mapping.get('control_id', mapping.get('external_control_id', 'N/A'))} → {mapping.get('mcsb_control_id', 'N/A')} " + f"{mapping.get('control_id', mapping.get('external_control_id', 'N/A'))} → " + f"{len(mapping.get('azure_policy_ids') or [])} Azure Polic{'y' if len(mapping.get('azure_policy_ids') or []) == 1 else 'ies'} " f"({mapping.get('confidence_score', 0):.0%})", expanded=mapping.get('confidence_score', 0) < 0.6 ): @@ -190,37 +171,17 @@ st.markdown(f"**Domain:** {mapping['domain']}") with col_edit2: - st.markdown("#### 🎯 MCSB Mapping") - - # Edit MCSB control - current_mcsb = mapping.get('mcsb_control_id', '') - current_idx = mcsb_options.index(current_mcsb) if current_mcsb in mcsb_options else 0 + st.markdown("#### 🎯 Azure Policy Mapping") + control_id_key = mapping.get('control_id', mapping.get('external_control_id', f'unknown_{idx}')) - - new_mcsb = st.selectbox( - "MCSB Control", - options=mcsb_options, - index=current_idx, - key=f"mcsb_{idx}_{control_id_key}" - ) - - if new_mcsb != current_mcsb: - # Find the original mapping in session state and update it - mapping_id = mapping.get('control_id', mapping.get('external_control_id')) - for i, m in enumerate(st.session_state.mappings): - m_id = m.get('control_id', m.get('external_control_id')) - if m_id == mapping_id: - st.session_state.mappings[i]['mcsb_control_id'] = new_mcsb - st.session_state.mappings[i]['manual_override'] = True - changes_made = True - break - - # Show MCSB control details - if new_mcsb in mcsb_lookup: - mcsb_control = mcsb_lookup[new_mcsb] - st.caption(f"**Title:** {mcsb_control.get('control_name', 'N/A')}") - st.caption(f"**Domain:** {mcsb_control.get('domain', 'N/A')}") - + + # policy_category is server-computed from the catalog category + # of the mapped Azure Policy definitions (see policy_category + # on ControlMapping) - there is no fixed intermediate taxonomy + # to pick from anymore, so this is informational, not editable. + if mapping.get('policy_category'): + st.caption(f"**Category:** {mapping['policy_category']}") + # Confidence score (read-only if not manually overridden) st.metric("Confidence Score", f"{mapping.get('confidence_score', 0):.0%}") @@ -315,9 +276,12 @@ st.bar_chart(confidence_dist) with col_stat2: - st.markdown("#### Top MCSB Controls") - top_mcsb = df['mcsb_control_id'].value_counts().head(10) - st.bar_chart(top_mcsb) + st.markdown("#### Top Policy Categories") + if 'policy_category' in df.columns: + top_categories = df['policy_category'].dropna().value_counts().head(10) + st.bar_chart(top_categories) + else: + st.caption("No policy category data available.") # Sovereignty statistics sov_mappings = [m for m in st.session_state.mappings if m.get('sovereignty')] diff --git a/app/frontend/pages/4_Export_Policy.py b/app/frontend/pages/4_Export_Policy.py index 11ebf5a..565c555 100644 --- a/app/frontend/pages/4_Export_Policy.py +++ b/app/frontend/pages/4_Export_Policy.py @@ -35,13 +35,11 @@ def _to_backend_mapping(m: Dict[str, Any]) -> Dict[str, Any]: return { "external_control_id": m.get("control_id") or m.get("external_control_id", ""), "external_control_name": m.get("control_name") or m.get("external_control_name", ""), - "mcsb_control_id": m.get("mcsb_control_id", ""), - "mcsb_control_name": m.get("mcsb_control_name", ""), - "mcsb_domain": m.get("mcsb_domain", ""), "confidence_score": m.get("confidence_score", 0.0), "reasoning": m.get("reasoning", ""), "azure_policy_ids": m.get("azure_policy_ids", []), "mapping_type": mt, + "policy_category": m.get("policy_category"), "defender_recommendations": m.get("defender_recommendations", []), "sovereignty": m.get("sovereignty"), # Carry the coverage taxonomy through so the backend can exclude @@ -182,7 +180,7 @@ def _render_manual_register(policy: Dict[str, Any]) -> None: "control_type": "Type", "coverage_display": "Coverage", "coverage_category": "Category code", - "mcsb_control_id": "MCSB", + "policy_category": "Policy Category", # Responsibility is an independent axis from coverage, not a # restatement of it: a process control is frequently Microsoft-owned. "responsibility": "Responsibility", @@ -385,8 +383,8 @@ def _regenerate_with_parameters( st.metric("Avg Confidence", f"{avg_confidence:.0%}") with col4: - unique_mcsb = len(set(m.get('mcsb_control_id', '') for m in st.session_state.mappings if m.get('mcsb_control_id'))) - st.metric("Unique MCSB Controls", unique_mcsb) + unique_categories = len(set(m.get('policy_category', '') for m in st.session_state.mappings if m.get('policy_category'))) + st.metric("Unique Policy Categories", unique_categories) with col5: st.metric("Sovereignty Mapped", len(sov_mappings)) @@ -813,7 +811,7 @@ def _regenerate_with_parameters( - Framework: {st.session_state.framework_name} - Total Mappings: {len(filtered_mappings)} - Average Confidence: {avg_confidence:.0%} -- Unique MCSB Controls: {unique_mcsb} +- Unique Policy Categories: {unique_categories} ## Quick Start 1. Review the mappings in `*_mappings.json` diff --git a/app/frontend/utils/api_client.py b/app/frontend/utils/api_client.py index 5eb6a02..5e052f3 100644 --- a/app/frontend/utils/api_client.py +++ b/app/frontend/utils/api_client.py @@ -181,42 +181,6 @@ def health_check(self) -> Dict[str, Any]: response.raise_for_status() return response.json() - @st.cache_data(ttl=3600, show_spinner=False) - def get_mcsb_controls(_self) -> List[Dict[str, Any]]: - """Get all MCSB controls. - - Cached: the MCSB catalog is static reference data, so this avoids a - backend round-trip on every rerun/navigation. Cache keyed only on the - endpoint (``_self`` is excluded from the cache key by the leading - underscore). - - Returns: - List of MCSB controls - """ - with _self._get_client() as client: - response = client.get(f"{_self.base_url}/api/v1/mapping/mcsb/controls") - response.raise_for_status() - data = response.json() - # Backend wraps the list in {"controls": [...]} - if isinstance(data, dict) and "controls" in data: - return data["controls"] - return data - - @st.cache_data(ttl=3600, show_spinner=False) - def get_mcsb_domains(_self) -> List[str]: - """Get all MCSB domains (cached static reference data). - - Returns: - List of MCSB domain names - """ - with _self._get_client() as client: - response = client.get(f"{_self.base_url}/api/v1/mapping/mcsb/domains") - response.raise_for_status() - data = response.json() - if isinstance(data, dict) and "domains" in data: - return data["domains"] - return data - def map_single_control( self, control_id: str, diff --git a/app/frontend/utils/state_init.py b/app/frontend/utils/state_init.py index 6863b36..8b2fe52 100644 --- a/app/frontend/utils/state_init.py +++ b/app/frontend/utils/state_init.py @@ -55,7 +55,7 @@ "session_uuid": None, # lazily set to uuid4 # MCSB cache - "mcsb_controls": None, + "mcsb_controls": None, # legacy cache key, kept for backward-compat state files # Policy decisions (approve / deny per mapping, keyed by control_id) "policy_decisions": {}, diff --git a/app/tests/test_acceptance_invariants.py b/app/tests/test_acceptance_invariants.py index 6cb5e1f..c220eb2 100644 --- a/app/tests/test_acceptance_invariants.py +++ b/app/tests/test_acceptance_invariants.py @@ -48,9 +48,7 @@ class _Mapping: def __init__(self, control_id, policy_ids=None): self.external_control_id = control_id self.external_control_name = f"Control {control_id}" - self.mcsb_control_id = "DP-3" - self.mcsb_control_name = "Encrypt data in transit" - self.mcsb_domain = "Data Protection" + self.policy_category = "Data Protection" self.confidence_score = 0.9 self.reasoning = "Relevant." self.azure_policy_ids = list(policy_ids or []) diff --git a/app/tests/test_coverage_mapping.py b/app/tests/test_coverage_mapping.py index 6bffa22..e78628a 100644 --- a/app/tests/test_coverage_mapping.py +++ b/app/tests/test_coverage_mapping.py @@ -59,9 +59,6 @@ def _mapping( return ControlMapping( external_control_id=control_id, external_control_name=name or f"Control {control_id}", - mcsb_control_id="GS-1", - mcsb_control_name="Governance and strategy", - mcsb_domain="Governance", confidence_score=confidence, reasoning=reasoning, azure_policy_ids=policy_ids, diff --git a/app/tests/test_defender_recommendations_not_invented.py b/app/tests/test_defender_recommendations_not_invented.py index 8f9fd30..46e1296 100644 --- a/app/tests/test_defender_recommendations_not_invented.py +++ b/app/tests/test_defender_recommendations_not_invented.py @@ -61,9 +61,6 @@ def test_map_control_discards_a_recommendation_the_model_invented(monkeypatch): """ svc = object.__new__(AIMappingService) svc.catalog = None - svc.mcsb_service = SimpleNamespace( - get_controls_for_external_control=lambda *a, **k: [] - ) svc.control_classification = SimpleNamespace( classify=lambda *a, **k: _async_return(unknown_classification()) ) @@ -81,9 +78,6 @@ def fake_sovereignty_context(*args, **kwargs): invented = ControlMapping( external_control_id="AC-1", external_control_name="Access control", - mcsb_control_id="IM-1", - mcsb_control_name="Protect identities", - mcsb_domain="Identity", confidence_score=0.9, reasoning="Relevant", mapping_type="exact", diff --git a/app/tests/test_defender_standard.py b/app/tests/test_defender_standard.py index 24ddcf4..514fa69 100644 --- a/app/tests/test_defender_standard.py +++ b/app/tests/test_defender_standard.py @@ -30,9 +30,6 @@ def _initiative(): ControlMapping( external_control_id="AC-1", external_control_name="Access control", - mcsb_control_id="IM-1", - mcsb_control_name="Protect identities", - mcsb_domain="Identity", confidence_score=0.9, reasoning="Relevant", azure_policy_ids=[G1], diff --git a/app/tests/test_frontend_design_guard.py b/app/tests/test_frontend_design_guard.py index d1426ba..d3cab38 100644 --- a/app/tests/test_frontend_design_guard.py +++ b/app/tests/test_frontend_design_guard.py @@ -185,8 +185,6 @@ def test_static_reference_getters_are_cached(): ``_self`` param so the cache key excludes the client instance.""" src = API_CLIENT.read_text(encoding="utf-8") for method in ( - "get_mcsb_controls", - "get_mcsb_domains", "get_sovereignty_summary", "get_sovereignty_objectives", "get_sovereignty_archetypes", diff --git a/app/tests/test_initiative_builder_drops.py b/app/tests/test_initiative_builder_drops.py index f6c93f8..1b8886a 100644 --- a/app/tests/test_initiative_builder_drops.py +++ b/app/tests/test_initiative_builder_drops.py @@ -49,8 +49,6 @@ def _mapping(control_id, policy_ids): control_id=control_id, control_title=f"Control {control_id}", domain="Data Protection", - mcsb_control_id="DP-3", - mcsb_control_name="Encrypt data in transit", confidence_score=0.9, mapping_rationale="Relevant", azure_policies=[ diff --git a/app/tests/test_initiative_grouping.py b/app/tests/test_initiative_grouping.py index 157d1bf..57bae5a 100644 --- a/app/tests/test_initiative_grouping.py +++ b/app/tests/test_initiative_grouping.py @@ -56,9 +56,6 @@ def _mapping(control_id, policy_ids, name=None): return ControlMapping( external_control_id=control_id, external_control_name=name or f"Control {control_id}", - mcsb_control_id="IM-1", - mcsb_control_name="Protect identities", - mcsb_domain="Identity", confidence_score=0.9, reasoning="Relevant control", azure_policy_ids=policy_ids, diff --git a/app/tests/test_mapping_concurrency.py b/app/tests/test_mapping_concurrency.py index 8996cfb..b3987f1 100644 --- a/app/tests/test_mapping_concurrency.py +++ b/app/tests/test_mapping_concurrency.py @@ -23,9 +23,6 @@ def _mapping(control: ExternalControl) -> ControlMapping: return ControlMapping( external_control_id=control.control_id, external_control_name=control.control_name, - mcsb_control_id="IM-1", - mcsb_control_name="Identity", - mcsb_domain="Identity", confidence_score=0.8, reasoning="Test mapping", mapping_type="partial", diff --git a/app/tests/test_parameterized_policy_filtering.py b/app/tests/test_parameterized_policy_filtering.py index d3f39d2..42dca4a 100644 --- a/app/tests/test_parameterized_policy_filtering.py +++ b/app/tests/test_parameterized_policy_filtering.py @@ -29,9 +29,6 @@ def _mapping(control_id: str, policy_ids: list[str]) -> ControlMapping: return ControlMapping( external_control_id=control_id, external_control_name=f"Control {control_id}", - mcsb_control_id="BR-1", - mcsb_control_name="Ensure regular backups", - mcsb_domain="Backup and Recovery", confidence_score=0.9, reasoning="Relevant control", azure_policy_ids=policy_ids, diff --git a/app/tests/test_pipeline_cancellation.py b/app/tests/test_pipeline_cancellation.py index 7d0e900..67fb474 100644 --- a/app/tests/test_pipeline_cancellation.py +++ b/app/tests/test_pipeline_cancellation.py @@ -77,8 +77,6 @@ def _map(extraction_arg, config, progress_callback=None): control_id=control.control_id, control_title=control.control_title, domain=control.domain, - mcsb_control_id="DP-3", - mcsb_control_name="Encrypt data in transit", confidence_score=0.9, mapping_rationale="Relevant", is_automatable=False, diff --git a/app/tests/test_pipeline_manual_register.py b/app/tests/test_pipeline_manual_register.py index bd31b18..c187a13 100644 --- a/app/tests/test_pipeline_manual_register.py +++ b/app/tests/test_pipeline_manual_register.py @@ -54,8 +54,6 @@ def _mapping(control_id, **kwargs): control_id=control_id, control_title="A control", domain="Data Protection", - mcsb_control_id="DP-3", - mcsb_control_name="Encrypt data", confidence_score=0.9, mapping_rationale="Relevant", is_automatable=False, diff --git a/app/tests/test_pipeline_policy_mapper.py b/app/tests/test_pipeline_policy_mapper.py index 90416e2..90163f5 100644 --- a/app/tests/test_pipeline_policy_mapper.py +++ b/app/tests/test_pipeline_policy_mapper.py @@ -74,9 +74,7 @@ class _FakeMapping: def __init__(self, control_id, **kwargs): self.external_control_id = control_id self.external_control_name = kwargs.get("name", f"Control {control_id}") - self.mcsb_control_id = kwargs.get("mcsb_id", "DP-3") - self.mcsb_control_name = kwargs.get("mcsb_name", "Encrypt data in transit") - self.mcsb_domain = kwargs.get("mcsb_domain", "Data Protection") + self.policy_category = kwargs.get("policy_category", "Data Protection") self.confidence_score = kwargs.get("confidence", 0.9) self.reasoning = kwargs.get("reasoning", "Relevant control") self.azure_policy_ids = kwargs.get("policy_ids", []) diff --git a/app/tests/test_policy_activity_recording.py b/app/tests/test_policy_activity_recording.py index 6a9bb66..56d00e1 100644 --- a/app/tests/test_policy_activity_recording.py +++ b/app/tests/test_policy_activity_recording.py @@ -26,9 +26,6 @@ def _valid_mapping(extra: dict | None = None) -> dict: m = { "external_control_id": "EXT-1", "external_control_name": "Encrypt data at rest", - "mcsb_control_id": "DP-4", - "mcsb_control_name": "Enable data at rest encryption by default", - "mcsb_domain": "Data Protection", "confidence_score": 0.9, "reasoning": "Direct match on encryption-at-rest intent.", "azure_policy_ids": ["4e6c27d5-a6ee-49cf-b2b4-d8fe90fa2b8b"], diff --git a/app/tests/test_policy_bundle_readme.py b/app/tests/test_policy_bundle_readme.py index 17bd846..4470321 100644 --- a/app/tests/test_policy_bundle_readme.py +++ b/app/tests/test_policy_bundle_readme.py @@ -19,9 +19,6 @@ def _request() -> PolicyGenerationRequest: ControlMapping( external_control_id="2.3.2.1", external_control_name="Control 2.3.2.1", - mcsb_control_id="IM-1", - mcsb_control_name="Protect identities", - mcsb_domain="Identity", confidence_score=0.9, reasoning="Relevant", azure_policy_ids=["18adea5e-f416-4d0f-8aa8-d24321e3e274"], diff --git a/app/tests/test_policy_exporter_refids.py b/app/tests/test_policy_exporter_refids.py index f3fb39f..b585c29 100644 --- a/app/tests/test_policy_exporter_refids.py +++ b/app/tests/test_policy_exporter_refids.py @@ -57,9 +57,6 @@ def _mapping(control_id, policy_ids, confidence=0.9): return ControlMapping( external_control_id=control_id, external_control_name=f"Control {control_id}", - mcsb_control_id="IM-1", - mcsb_control_name="Protect identities", - mcsb_domain="Identity", confidence_score=confidence, reasoning="Relevant control", azure_policy_ids=policy_ids, diff --git a/app/tests/test_policy_generation_versions.py b/app/tests/test_policy_generation_versions.py index 6f6c826..9894952 100644 --- a/app/tests/test_policy_generation_versions.py +++ b/app/tests/test_policy_generation_versions.py @@ -24,9 +24,6 @@ def _mapping(with_sovereignty: bool = False) -> ControlMapping: return ControlMapping( external_control_id="CTRL-1", external_control_name="Example control", - mcsb_control_id="IM-1", - mcsb_control_name="Protect identities", - mcsb_domain="Identity", confidence_score=0.9, reasoning="Relevant control", azure_policy_ids=["policy-id"], diff --git a/app/tests/test_policy_guid_existence.py b/app/tests/test_policy_guid_existence.py index 8d39786..4ce06d1 100644 --- a/app/tests/test_policy_guid_existence.py +++ b/app/tests/test_policy_guid_existence.py @@ -57,9 +57,6 @@ def _mapping(control_id: str, policy_ids: list[str], confidence: float = 0.9) -> return ControlMapping( external_control_id=control_id, external_control_name=f"Control {control_id}", - mcsb_control_id="IM-1", - mcsb_control_name="Protect identities", - mcsb_domain="Identity", confidence_score=confidence, reasoning="Relevant control", azure_policy_ids=policy_ids, @@ -154,8 +151,6 @@ def _pipeline_mapping(control_id: str, policy_ids: list[str]): control_id=control_id, control_title=f"Control {control_id}", domain="Identity", - mcsb_control_id="IM-1", - mcsb_control_name="Protect identities", confidence_score=0.9, mapping_rationale="Relevant", azure_policies=[ diff --git a/app/tests/test_policy_guid_stripping.py b/app/tests/test_policy_guid_stripping.py index c191139..58e5804 100644 --- a/app/tests/test_policy_guid_stripping.py +++ b/app/tests/test_policy_guid_stripping.py @@ -27,9 +27,6 @@ def _mapping(control_id: str, policy_ids: list[str], confidence: float = 0.9) -> return ControlMapping( external_control_id=control_id, external_control_name=f"Control {control_id}", - mcsb_control_id="IM-1", - mcsb_control_name="Protect identities", - mcsb_domain="Identity", confidence_score=confidence, reasoning="Relevant control", azure_policy_ids=policy_ids, @@ -147,8 +144,6 @@ def _pipeline_mapping(control_id: str, policy_ids: list[str]) -> ControlPolicyMa control_id=control_id, control_title=f"Control {control_id}", domain="Identity", - mcsb_control_id="IM-1", - mcsb_control_name="Protect identities", confidence_score=0.9, mapping_rationale="Relevant", azure_policies=[ diff --git a/app/tests/test_sovereignty_residency.py b/app/tests/test_sovereignty_residency.py index 6232f7b..f87d4e9 100644 --- a/app/tests/test_sovereignty_residency.py +++ b/app/tests/test_sovereignty_residency.py @@ -19,9 +19,6 @@ def _residency_mapping() -> ControlMapping: return ControlMapping( external_control_id="RES-1", external_control_name="Restrict data residency", - mcsb_control_id="NS-1", - mcsb_control_name="Network segmentation", - mcsb_domain="Network Security", confidence_score=0.95, reasoning="Require data residency enforcement.", azure_policy_ids=[], diff --git a/app/tests/test_system_policy_filtering.py b/app/tests/test_system_policy_filtering.py index 63490c0..3bd5614 100644 --- a/app/tests/test_system_policy_filtering.py +++ b/app/tests/test_system_policy_filtering.py @@ -31,9 +31,6 @@ def _mapping(control_id: str, policy_ids: list[str]) -> ControlMapping: return ControlMapping( external_control_id=control_id, external_control_name=f"Control {control_id}", - mcsb_control_id="IM-1", - mcsb_control_name="Protect identities", - mcsb_domain="Identity", confidence_score=0.9, reasoning="Relevant control", azure_policy_ids=policy_ids, diff --git a/compliance-pipeline/.env.template b/compliance-pipeline/.env.template deleted file mode 100644 index b3fc406..0000000 --- a/compliance-pipeline/.env.template +++ /dev/null @@ -1,23 +0,0 @@ -# Compliance Pipeline — .env template -# Copy to .env and fill in your values - -# Azure OpenAI (required) -AZURE_OPENAI_ENDPOINT=https://your-openai-resource.openai.azure.com/ -AZURE_OPENAI_DEPLOYMENT_NAME=gpt-4.1 -AZURE_OPENAI_API_VERSION=2024-12-01-preview - -# Authentication — choose ONE: -# Option 1: API Key -# AZURE_OPENAI_API_KEY=your-api-key-here - -# Option 2: DefaultAzureCredential (recommended) -# Just run: az login -# The pipeline will use your Azure CLI credentials automatically. - -# Pipeline settings (optional) -# AI_MAX_TOKENS=16000 -# AI_BATCH_SIZE=5 -# MIN_CONFIDENCE=0.5 -# INCLUDE_LOW_CONFIDENCE=true -# MAX_PDF_PAGES=200 -# OUTPUT_DIR=./output diff --git a/compliance-pipeline/README.md b/compliance-pipeline/README.md deleted file mode 100644 index 8d70f70..0000000 --- a/compliance-pipeline/README.md +++ /dev/null @@ -1,213 +0,0 @@ -# ComplianceIQ Compliance Pipeline - -**PDF → Controls → Azure Policy → Defender for Cloud — in one command.** - -This pipeline automates the entire journey from a compliance control PDF document to a deployable Azure Policy initiative for Microsoft Defender for Cloud regulatory compliance. - -## What It Does - -``` -┌─────────────┐ ┌──────────────────┐ ┌──────────────────┐ ┌────────────┐ ┌─────────────────┐ -│ Compliance │ │ AI Control │ │ Azure Policy │ │ Validate │ │ Initiative │ -│ PDF Input │────▶│ Extraction │────▶│ Mapping (LLM) │────▶│ Mappings │────▶│ Artifacts │ -│ │ │ (Azure OpenAI) │ │ │ │ │ │ (JSON + PS1) │ -└─────────────┘ └──────────────────┘ └──────────────────┘ └────────────┘ └─────────────────┘ - PDF Stage 1 Stage 2 Stage 3 Stage 4 -``` - -### Stage 1: PDF Text Extraction -- Extracts all text from the PDF using `pypdf` -- Handles multi-page documents (up to 200 pages) -- Chunks large documents for LLM processing - -### Stage 2: AI Control Extraction -- Sends PDF text to Azure OpenAI (GPT-4.1/GPT-4o) -- Uses **structured outputs** to extract every control with: - - Control ID, title, description, domain, type - - Sub-controls and requirements - - Framework metadata (name, version, authority, region) - -### Stage 3: Azure Policy Mapping -- Maps each control to: - - **MCSB controls** (Microsoft Cloud Security Benchmark) - - **Azure Policy definitions** (real built-in policy GUIDs) - - **Defender for Cloud recommendations** -- Identifies automatable vs manual controls -- Provides confidence scores and rationale - -### Stage 4: Validation -- Validates all policy GUIDs are proper format -- Checks MCSB control ID formats -- Flags low-confidence mappings for review -- Ensures all controls have mappings - -### Stage 5: Generate Initiative Artifacts -Produces the exact file format expected by Azure: - -| File | Purpose | -|------|---------| -| `_Initiative.json` | Main initiative definition (Policy Set Definition) | -| `policies.json` | Policy definition references with group assignments | -| `groups.json` | Policy definition groups (one per control) | -| `params.json` | Parameters (e.g., allowed locations) | -| `Deploy-Initiative.ps1` | PowerShell script for Azure deployment | -| `deploy-initiative.sh` | Azure CLI script for deployment | -| `_Mappings.csv` | Complete mapping report | -| `validation_report.json` | Validation results | - -## Quick Start - -### 1. Prerequisites - -```bash -# Python 3.10+ -pip install -r requirements.txt - -# Azure authentication (choose one) -az login # Azure CLI (recommended) -# OR set AZURE_OPENAI_API_KEY in .env -``` - -### 2. Configure - -```bash -cp .env.template .env -# Edit .env with your Azure OpenAI endpoint -``` - -### 3. Run - -```bash -# Basic — process a compliance PDF -python pipeline.py ./my-framework.pdf - -# With custom output directory -python pipeline.py ./my-framework.pdf --output ./frameworks/my-framework - -# With Azure region restrictions -python pipeline.py ./my-framework.pdf --locations uaenorth,uaecentral - -# Higher confidence threshold -python pipeline.py ./my-framework.pdf --min-confidence 0.7 - -# Verbose for debugging -python pipeline.py ./my-framework.pdf --verbose -``` - -### 4. Deploy to Azure - -```powershell -# PowerShell -cd output/my-framework -.\Deploy-Initiative.ps1 - -# With management group scope -.\Deploy-Initiative.ps1 -ManagementGroupId "mg-compliance" - -# Create and assign in one step -.\Deploy-Initiative.ps1 -AssignAfterCreation -``` - -```bash -# Azure CLI -cd output/my-framework -bash deploy-initiative.sh - -# With management group scope -bash deploy-initiative.sh --management-group mg-compliance -``` - -## Example Output - -Running against an Oman CDC document: - -``` -╔══════════════════════════════════════════════════════════════╗ -║ ComplianceIQ Compliance Pipeline ║ -║ PDF → Controls → Azure Policy → Defender for Cloud ║ -╚══════════════════════════════════════════════════════════════╝ - -────────────────────────────────────────────────────────────── - Stage 1: PDF Text Extraction -────────────────────────────────────────────────────────────── - - ✓ Extracted 45,230 characters from 28 pages - -────────────────────────────────────────────────────────────── - Stage 2: AI Control Extraction (Azure OpenAI) -────────────────────────────────────────────────────────────── - - ✓ Framework: Oman CDC Cloud Security Controls - ✓ Authority: Cyber Defense Centre (CDC) - ✓ Region: Oman - ✓ Controls: 28 extracted - - Domain Breakdown: - Network Security: 4 - Identity & Access Management: 3 - Data Protection & Encryption: 5 - ... - -────────────────────────────────────────────────────────────── - Stage 3: Azure Policy Mapping (Azure OpenAI) -────────────────────────────────────────────────────────────── - - ✓ Mapped: 28 controls - ✓ Automatable: 19 (via Azure Policy) - ✓ Manual: 9 (require attestation) - ✓ Policies: 37 unique Azure Policy definitions - ✓ Confidence: 0.82 average - -──────════════════════════════════════════════════════════════ - Pipeline Complete — 47.3s -══════════════════════════════════════════════════════════════ -``` - -## Architecture - -``` -compliance-pipeline/ -├── pipeline.py # CLI orchestrator (main entry point) -├── config.py # Configuration loader -├── models.py # Pydantic models (structured outputs) -├── pdf_extractor.py # PDF → raw text -├── control_extractor.py # Raw text → structured controls (LLM) -├── policy_mapper.py # Controls → Azure Policy mappings (LLM) -├── validator.py # Mapping validation -├── initiative_builder.py # Generate JSON + PS1 + CSV artifacts -├── requirements.txt # Python dependencies -├── .env.template # Environment variable template -└── README.md # This file -``` - -## How It Works with Defender for Cloud - -The generated initiative follows the exact format used by built-in regulatory standards in Defender for Cloud: - -1. **Policy Definition Groups** = your compliance controls -2. **Policy Definitions** = Azure Policy rules mapped to each control -3. Each policy is assigned to one or more groups (controls) -4. Non-automatable controls appear as "Manual attestation required" - -Once deployed and assigned: -- Defender for Cloud evaluates all resources against the initiative -- Compliance dashboard shows per-control status -- Non-compliant resources are flagged with remediation guidance -- Manual controls require evidence upload in the portal - -## Supported PDF Formats - -- ✅ Text-based PDFs (most modern compliance documents) -- ✅ Multi-language documents (Arabic, English, etc.) -- ✅ Documents up to 200 pages -- ❌ Scanned/image-only PDFs (OCR not yet supported) -- ❌ Password-protected PDFs - -## Integration with Existing ComplianceIQ - -This pipeline generates the same artifact format as the existing frameworks in `../framework/`: -- `framework/Oman CDC/` — CDC_Initiative.json, cdc_policies.json, cdc_groups.json -- `framework/SAMA/` — SAMA_Cybersecurity_Framework.json, policies.json -- `framework/SITA Cloud Compliance Framework/` — similar structure - -The generated `Deploy-Initiative.ps1` follows the same pattern as `CreateCDCInitiative.ps1`. diff --git a/compliance-pipeline/config.py b/compliance-pipeline/config.py deleted file mode 100644 index e58602f..0000000 --- a/compliance-pipeline/config.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -Configuration for the compliance pipeline. -Loads settings from environment variables or .env file. -""" - -import os -from pathlib import Path -from dataclasses import dataclass, field -from typing import Optional - - -@dataclass -class PipelineConfig: - """Pipeline configuration loaded from environment.""" - - # Azure OpenAI - azure_openai_endpoint: str = "" - azure_openai_deployment: str = "gpt-4.1" - azure_openai_api_version: str = "2024-12-01-preview" - azure_openai_api_key: Optional[str] = None # If not set, uses DefaultAzureCredential - - # Model settings - max_tokens: int = 16000 - batch_size: int = 5 # Controls per LLM call for mapping - - # Pipeline settings - min_confidence_threshold: float = 0.5 # Include mappings above this confidence - include_low_confidence: bool = True # Include low-confidence with warnings - max_pdf_pages: int = 200 # Safety limit for PDF page count - - # Output - output_dir: str = "./output" - - @classmethod - def from_env(cls, env_file: Optional[str] = None) -> "PipelineConfig": - """Load config from environment, optionally reading a .env file first.""" - if env_file: - _load_dotenv(env_file) - elif Path(".env").exists(): - _load_dotenv(".env") - # Also check parent app/.env - agent_env = Path(__file__).parent.parent / "app" / "backend" / ".env" - if agent_env.exists(): - _load_dotenv(str(agent_env)) - - return cls( - azure_openai_endpoint=os.getenv("AZURE_OPENAI_ENDPOINT", ""), - azure_openai_deployment=os.getenv("AZURE_OPENAI_DEPLOYMENT_NAME", "gpt-4.1"), - azure_openai_api_version=os.getenv("AZURE_OPENAI_API_VERSION", "2024-12-01-preview"), - azure_openai_api_key=os.getenv("AZURE_OPENAI_API_KEY"), - max_tokens=int(os.getenv("AI_MAX_TOKENS", "16000")), - batch_size=int(os.getenv("AI_BATCH_SIZE", "5")), - min_confidence_threshold=float(os.getenv("MIN_CONFIDENCE", "0.5")), - include_low_confidence=os.getenv("INCLUDE_LOW_CONFIDENCE", "true").lower() == "true", - max_pdf_pages=int(os.getenv("MAX_PDF_PAGES", "200")), - output_dir=os.getenv("OUTPUT_DIR", "./output"), - ) - - def validate(self) -> list[str]: - """Return list of config validation errors (empty = valid).""" - errors = [] - if not self.azure_openai_endpoint: - errors.append("AZURE_OPENAI_ENDPOINT is required") - if not self.azure_openai_deployment: - errors.append("AZURE_OPENAI_DEPLOYMENT_NAME is required") - return errors - - -def _load_dotenv(path: str): - """Minimal .env loader — no external dependency needed.""" - try: - with open(path) as f: - for line in f: - line = line.strip() - if not line or line.startswith("#") or "=" not in line: - continue - key, _, value = line.partition("=") - key = key.strip() - value = value.strip().strip("\"'") - if key and key not in os.environ: # Don't override existing env vars - os.environ[key] = value - except FileNotFoundError: - pass diff --git a/compliance-pipeline/control_extractor.py b/compliance-pipeline/control_extractor.py deleted file mode 100644 index 799e14e..0000000 --- a/compliance-pipeline/control_extractor.py +++ /dev/null @@ -1,247 +0,0 @@ -""" -LLM-based Control Extraction Engine. -Uses Azure OpenAI with structured outputs to extract compliance controls from raw PDF text. -""" - -import json -import logging -from typing import Optional - -from models import ControlExtractionResult, ExtractedControl -from config import PipelineConfig -from pdf_extractor import chunk_text - -logger = logging.getLogger(__name__) - -# ── System prompt for control extraction ────────────────────────────────────── - -EXTRACTION_SYSTEM_PROMPT = """You are an expert compliance analyst specializing in cybersecurity, data protection, and cloud governance frameworks from the Middle East, Africa, and global regulatory bodies. - -Your task is to analyze the raw text extracted from a compliance control document (PDF) and produce a structured extraction of ALL controls found in the document. - -## Extraction Rules - -1. **Identify every control** in the document. Controls may be numbered (e.g., TR-01, POL-03, Section 4.1) or listed as requirements, objectives, or mandates. - -2. **Assign a control ID** using the document's own numbering. If the document uses IDs like "TR-01", "POL-03", etc., preserve them exactly. If the document uses section numbers (e.g., 4.1, 4.2), use those. If no numbering exists, create sequential IDs like "CTRL-001", "CTRL-002", etc. - -3. **Classify each control's domain** into one of these categories: - - Network Security - - Identity & Access Management - - Data Protection & Encryption - - Logging & Monitoring - - Endpoint Security - - Vulnerability Management - - Backup & Recovery - - Incident Response - - Risk Management - - Governance & Policy - - Physical Security - - Cloud Security - - AI & Emerging Technology - - Privacy & Data Sovereignty - - Compliance & Audit - - Supply Chain / Third Party - - Business Continuity - -4. **Classify each control's type** as one of: - - Technical: Can be enforced or audited via technical means (Azure Policy, Defender) - - Policy: Requires organizational policy or procedure - - Contractual: Relates to contracts with cloud providers - - Management: Management oversight and governance - - Operational: Day-to-day operational procedures - - Governance: Overarching governance and frameworks - -5. **Capture the full description** of each control — not just the title but the complete requirement text. - -6. **Identify sub-controls** if a control has multiple sub-requirements (e.g., a, b, c). - -7. **Identify the framework metadata**: name, version, issuing authority, and country/region. - -8. **Be thorough** — it is critical to extract ALL controls, not just the first few. Scan the entire document. - -9. **Do NOT invent controls** that are not in the document. Only extract what is explicitly stated. - -10. **Provide a summary** of the framework's purpose and scope.""" - - -def get_openai_client(config: PipelineConfig): - """Create Azure OpenAI client using either API key or DefaultAzureCredential.""" - from openai import AzureOpenAI - - if config.azure_openai_api_key: - return AzureOpenAI( - azure_endpoint=config.azure_openai_endpoint, - api_key=config.azure_openai_api_key, - api_version=config.azure_openai_api_version, - ) - else: - from azure.identity import DefaultAzureCredential, get_bearer_token_provider - - credential = DefaultAzureCredential() - token_provider = get_bearer_token_provider( - credential, "https://cognitiveservices.azure.com/.default" - ) - return AzureOpenAI( - azure_endpoint=config.azure_openai_endpoint, - azure_ad_token_provider=token_provider, - api_version=config.azure_openai_api_version, - ) - - -def extract_controls_from_text( - pdf_text: str, - config: PipelineConfig, - pdf_metadata: Optional[dict] = None, -) -> ControlExtractionResult: - """ - Use Azure OpenAI to extract structured controls from raw PDF text. - - If the text is too large for a single call, it is chunked and results are merged. - - Args: - pdf_text: Raw text extracted from PDF. - config: Pipeline configuration. - pdf_metadata: Optional PDF metadata (title, author, etc.). - - Returns: - ControlExtractionResult with all extracted controls. - """ - client = get_openai_client(config) - - # Build context from metadata - metadata_context = "" - if pdf_metadata: - metadata_context = ( - f"\nPDF Metadata:\n" - f" Title: {pdf_metadata.get('title', 'Unknown')}\n" - f" Author: {pdf_metadata.get('author', 'Unknown')}\n" - f" Pages: {pdf_metadata.get('pages', 'Unknown')}\n" - ) - - # Chunk if necessary (gpt-4.1 has ~128k context but we stay conservative) - chunks = chunk_text(pdf_text, max_chars=100000) - - if len(chunks) == 1: - # Single call — send entire text - return _extract_single(client, config, chunks[0], metadata_context) - else: - # Multi-chunk: extract from each chunk then merge - logger.info(f"Document split into {len(chunks)} chunks for extraction") - return _extract_multi_chunk(client, config, chunks, metadata_context) - - -def _extract_single( - client, - config: PipelineConfig, - text: str, - metadata_context: str, -) -> ControlExtractionResult: - """Extract controls from a single text chunk.""" - - user_prompt = f"""{metadata_context} - -## Document Text - -{text} - ---- - -Extract ALL compliance controls from this document. Be thorough — capture every control, requirement, and sub-requirement. -Return the structured result with framework metadata and a complete list of controls.""" - - logger.info(f"Sending {len(user_prompt):,} chars to Azure OpenAI for control extraction...") - - completion = client.beta.chat.completions.parse( - model=config.azure_openai_deployment, - messages=[ - {"role": "system", "content": EXTRACTION_SYSTEM_PROMPT}, - {"role": "user", "content": user_prompt}, - ], - response_format=ControlExtractionResult, - max_completion_tokens=config.max_tokens, - ) - - result = completion.choices[0].message.parsed - - if not result: - raise ValueError("LLM returned empty extraction result") - - logger.info( - f"Extracted {len(result.controls)} controls from '{result.framework_name}'" - ) - return result - - -def _extract_multi_chunk( - client, - config: PipelineConfig, - chunks: list[str], - metadata_context: str, -) -> ControlExtractionResult: - """Extract controls from multiple chunks and merge results.""" - - all_controls: list[ExtractedControl] = [] - seen_ids: set[str] = set() - framework_name = "" - framework_version = None - issuing_authority = None - country_or_region = None - summary = "" - - for i, chunk in enumerate(chunks): - logger.info(f"Processing chunk {i + 1}/{len(chunks)} ({len(chunk):,} chars)") - - user_prompt = f"""{metadata_context} - -## Document Text (Part {i + 1} of {len(chunks)}) - -{chunk} - ---- - -Extract ALL compliance controls found in this portion of the document. -This is part {i + 1} of {len(chunks)} parts of the same document. -Be thorough — capture every control, requirement, and sub-requirement found in this section.""" - - completion = client.beta.chat.completions.parse( - model=config.azure_openai_deployment, - messages=[ - {"role": "system", "content": EXTRACTION_SYSTEM_PROMPT}, - {"role": "user", "content": user_prompt}, - ], - response_format=ControlExtractionResult, - max_completion_tokens=config.max_tokens, - ) - - result = completion.choices[0].message.parsed - if not result: - logger.warning(f"Chunk {i + 1} returned empty result") - continue - - # Take metadata from first chunk - if i == 0: - framework_name = result.framework_name - framework_version = result.framework_version - issuing_authority = result.issuing_authority - country_or_region = result.country_or_region - summary = result.summary - - # Deduplicate controls by ID - for ctrl in result.controls: - if ctrl.control_id not in seen_ids: - all_controls.append(ctrl) - seen_ids.add(ctrl.control_id) - else: - logger.debug(f"Skipping duplicate control: {ctrl.control_id}") - - logger.info(f"Chunk {i + 1}: found {len(result.controls)} controls ({len(all_controls)} total unique)") - - return ControlExtractionResult( - framework_name=framework_name, - framework_version=framework_version, - issuing_authority=issuing_authority, - country_or_region=country_or_region, - controls=all_controls, - summary=summary, - ) diff --git a/compliance-pipeline/initiative_builder.py b/compliance-pipeline/initiative_builder.py deleted file mode 100644 index 7139447..0000000 --- a/compliance-pipeline/initiative_builder.py +++ /dev/null @@ -1,642 +0,0 @@ -""" -Initiative Builder Module. -Generates all Defender for Cloud regulatory compliance initiative artifacts: - - initiative.json (main initiative definition with policyDefinitionGroups) - - policies.json (policy definition references with group assignments) - - groups.json (group definitions — one per control) - - params.json (parameters, if any) - - Deploy-Initiative.ps1 (PowerShell script to import into Azure) - - deploy-initiative.sh (Azure CLI script) - - mappings.csv (complete mapping report) - -Output format matches the Oman CDC / SAMA pattern used by Defender for Cloud. -""" - -import csv -import json -import logging -import re -from datetime import datetime -from pathlib import Path -from typing import Optional - -from models import ( - ControlExtractionResult, - ControlPolicyMapping, - ValidationReport, - InitiativeGroup, - PolicyDefinitionRef, -) - -logger = logging.getLogger(__name__) - - -def _sanitize_group_name(control_id: str) -> str: - """Convert a control ID to a valid Azure policy group name (alphanumeric + underscore).""" - return re.sub(r"[^a-zA-Z0-9_]", "_", control_id) - - -def build_initiative_artifacts( - extraction: ControlExtractionResult, - mappings: list[ControlPolicyMapping], - validation: ValidationReport, - output_dir: str, - allowed_locations: Optional[list[str]] = None, -) -> list[str]: - """ - Generate all initiative artifact files. - - Args: - extraction: The extracted framework controls. - mappings: The validated control-to-policy mappings. - validation: Validation report. - output_dir: Directory to write output files. - allowed_locations: Optional Azure regions for location policies. - - Returns: - List of file paths created. - """ - out = Path(output_dir) - out.mkdir(parents=True, exist_ok=True) - - files_created: list[str] = [] - - # Sanitized framework name for file naming - fw_safe = re.sub(r"[^a-zA-Z0-9]+", "_", extraction.framework_name).strip("_") - - # ── 1. groups.json ──────────────────────────────────────────────────── - groups = _build_groups(extraction, mappings) - groups_path = out / "groups.json" - _write_json(groups_path, groups) - files_created.append(str(groups_path)) - - # ── 2. policies.json ────────────────────────────────────────────────── - policies = _build_policies(mappings) - policies_path = out / "policies.json" - _write_json(policies_path, policies) - files_created.append(str(policies_path)) - - # ── 3. params.json ──────────────────────────────────────────────────── - params = _build_params(allowed_locations) - params_path = out / "params.json" - _write_json(params_path, params) - files_created.append(str(params_path)) - - # ── 4. initiative.json (main definition) ────────────────────────────── - initiative = _build_initiative(extraction, mappings, groups, policies, params) - initiative_path = out / f"{fw_safe}_Initiative.json" - _write_json(initiative_path, initiative) - files_created.append(str(initiative_path)) - - # ── 5. Deploy-Initiative.ps1 ────────────────────────────────────────── - ps_path = out / "Deploy-Initiative.ps1" - ps_content = _build_powershell_script(extraction, fw_safe) - ps_path.write_text(ps_content, encoding="utf-8") - files_created.append(str(ps_path)) - - # ── 6. deploy-initiative.sh ─────────────────────────────────────────── - sh_path = out / "deploy-initiative.sh" - sh_content = _build_cli_script(extraction, fw_safe) - sh_path.write_text(sh_content, encoding="utf-8") - files_created.append(str(sh_path)) - - # ── 7. mappings.csv (full mapping report) ───────────────────────────── - csv_path = out / f"{fw_safe}_Mappings.csv" - _write_mappings_csv(csv_path, extraction, mappings) - files_created.append(str(csv_path)) - - # ── 8. validation_report.json ───────────────────────────────────────── - report_path = out / "validation_report.json" - _write_json(report_path, validation.model_dump()) - files_created.append(str(report_path)) - - logger.info(f"Generated {len(files_created)} files in {out}/") - return files_created - - -# ── Builders ────────────────────────────────────────────────────────────────── - - -def _build_groups( - extraction: ControlExtractionResult, - mappings: list[ControlPolicyMapping], -) -> list[dict]: - """Build policyDefinitionGroups — one group per control.""" - groups = [] - mapping_lookup = {m.control_id: m for m in mappings} - - for ctrl in extraction.controls: - group_name = _sanitize_group_name(ctrl.control_id) - mapping = mapping_lookup.get(ctrl.control_id) - - display_name = f"{ctrl.control_id}: {ctrl.control_title}" - description = ctrl.control_description - - if mapping and not mapping.is_automatable: - description += " [MANUAL ATTESTATION REQUIRED]" - if mapping.manual_attestation_note: - description += f" — {mapping.manual_attestation_note}" - - groups.append({ - "name": group_name, - "displayName": display_name, - "description": description, - }) - - return groups - - -def _build_policies(mappings: list[ControlPolicyMapping]) -> list[dict]: - """Build policyDefinitions array with group assignments.""" - policy_refs: list[dict] = [] - seen_combos: set[str] = set() - - for mapping in mappings: - if not mapping.azure_policies: - continue - - group_name = _sanitize_group_name(mapping.control_id) - - for policy in mapping.azure_policies: - pid = policy.policy_definition_id - full_id = f"/providers/Microsoft.Authorization/policyDefinitions/{pid}" - - # Create a unique reference ID - ref_id = pid - combo_key = f"{pid}|{group_name}" - - if combo_key in seen_combos: - continue - seen_combos.add(combo_key) - - # Check if this policy already has an entry (might belong to multiple groups) - existing = next( - (p for p in policy_refs if p["PolicyDefinitionId"] == full_id), - None, - ) - - if existing: - # Add this group to existing policy entry - if group_name not in existing["GroupNames"]: - existing["GroupNames"].append(group_name) - else: - policy_refs.append({ - "PolicyDefinitionReferenceId": ref_id, - "PolicyDefinitionId": full_id, - "Parameters": {}, - "GroupNames": [group_name], - }) - - return policy_refs - - -def _build_params(allowed_locations: Optional[list[str]] = None) -> dict: - """Build parameters object.""" - params = {} - - if allowed_locations: - params["listOfAllowedLocations"] = { - "type": "Array", - "metadata": { - "displayName": "Allowed locations", - "description": "The list of locations that can be specified when deploying resources.", - }, - "defaultValue": allowed_locations, - } - - return params - - -def _build_initiative( - extraction: ControlExtractionResult, - mappings: list[ControlPolicyMapping], - groups: list[dict], - policies: list[dict], - params: dict, -) -> dict: - """Build the complete initiative JSON (Azure Policy Set Definition).""" - automatable = sum(1 for m in mappings if m.is_automatable) - manual = len(mappings) - automatable - - metadata = { - "category": "Regulatory Compliance", - "version": "1.0.0", - "source": "ComplianceIQ Compliance Pipeline (AI-Generated)", - "generatedDate": datetime.utcnow().isoformat() + "Z", - "frameworkName": extraction.framework_name, - } - - if extraction.framework_version: - metadata["frameworkVersion"] = extraction.framework_version - if extraction.issuing_authority: - metadata["authority"] = extraction.issuing_authority - if extraction.country_or_region: - metadata["country"] = extraction.country_or_region - - metadata["totalControls"] = len(extraction.controls) - metadata["automatableControls"] = automatable - metadata["manualControls"] = manual - - return { - "properties": { - "displayName": f"{extraction.framework_name} Compliance Controls", - "policyType": "Custom", - "description": extraction.summary, - "metadata": metadata, - "parameters": params, - "policyDefinitionGroups": groups, - "policyDefinitions": policies, - } - } - - -def _build_powershell_script( - extraction: ControlExtractionResult, - fw_safe: str, -) -> str: - """Generate PowerShell deployment script for Defender for Cloud.""" - initiative_file = f"{fw_safe}_Initiative.json" - name_slug = fw_safe.replace("_", "-") - - return f'''# ============================================================================ -# {extraction.framework_name} — Azure Policy Initiative Deployment -# Generated by ComplianceIQ Compliance Pipeline -# Date: {datetime.utcnow().strftime("%Y-%m-%d %H:%M UTC")} -# ============================================================================ -# -# This script creates a custom regulatory compliance initiative in Azure -# and optionally assigns it to a scope for Defender for Cloud monitoring. -# -# Prerequisites: -# - Az PowerShell module installed (Install-Module -Name Az) -# - Authenticated to Azure (Connect-AzAccount) -# - Contributor or Policy Contributor role at target scope -# - Files in current directory: groups.json, policies.json, params.json -# -# Usage: -# .\\Deploy-Initiative.ps1 # Current subscription -# .\\Deploy-Initiative.ps1 -ManagementGroupId "mg-compliance" # Management group -# .\\Deploy-Initiative.ps1 -AssignAfterCreation # Create + assign -# ============================================================================ - -param( - [Parameter(Mandatory=$false)] - [string]$Scope = "", - - [Parameter(Mandatory=$false)] - [string]$ManagementGroupId = "", - - [Parameter(Mandatory=$false)] - [switch]$AssignAfterCreation, - - [Parameter(Mandatory=$false)] - [string]$Location = "southafricanorth" -) - -$ErrorActionPreference = "Stop" - -Write-Host "" -Write-Host "========================================================" -ForegroundColor Cyan -Write-Host " {extraction.framework_name}" -ForegroundColor Cyan -Write-Host " Regulatory Compliance Initiative Deployment" -ForegroundColor Cyan -Write-Host "========================================================" -ForegroundColor Cyan -Write-Host "" - -# ── Determine target scope ──────────────────────────────────────────────── -if ($ManagementGroupId) {{ - $TargetScope = "/providers/Microsoft.Management/managementGroups/$ManagementGroupId" - Write-Host "[Scope] Management Group: $ManagementGroupId" -ForegroundColor Yellow -}} elseif ($Scope) {{ - $TargetScope = $Scope - Write-Host "[Scope] Custom: $Scope" -ForegroundColor Yellow -}} else {{ - $context = Get-AzContext - if (-not $context) {{ - Write-Error "Not authenticated to Azure. Run Connect-AzAccount first." - exit 1 - }} - $TargetScope = "/subscriptions/$($context.Subscription.Id)" - Write-Host "[Scope] Subscription: $($context.Subscription.Name) ($($context.Subscription.Id))" -ForegroundColor Yellow -}} -Write-Host "" - -# ── Verify required files ───────────────────────────────────────────────── -$requiredFiles = @("groups.json", "policies.json", "params.json") -$missingFiles = @() -foreach ($file in $requiredFiles) {{ - if (-not (Test-Path $file)) {{ - $missingFiles += $file - }} -}} - -if ($missingFiles.Count -gt 0) {{ - Write-Host "ERROR: Missing required files:" -ForegroundColor Red - $missingFiles | ForEach-Object {{ Write-Host " - $_" -ForegroundColor Red }} - Write-Host "" - Write-Host "Ensure you are running this script from the output directory" -ForegroundColor Yellow - Write-Host "containing groups.json, policies.json, and params.json." -ForegroundColor Yellow - exit 1 -}} - -Write-Host "[Files] All required files found" -ForegroundColor Green -Write-Host "" - -# ── Load definition files ───────────────────────────────────────────────── -Write-Host "Loading policy definition files..." -ForegroundColor Gray - -$groups = Get-Content -Raw groups.json -$policies = Get-Content -Raw policies.json -$params = Get-Content -Raw params.json - -# Validate JSON -try {{ - $null = $groups | ConvertFrom-Json - $null = $policies | ConvertFrom-Json - $null = $params | ConvertFrom-Json - Write-Host "[Validate] JSON files are valid" -ForegroundColor Green -}} catch {{ - Write-Error "Invalid JSON in definition files: $_" - exit 1 -}} - -Write-Host "" - -# ── Create the Policy Initiative ────────────────────────────────────────── -$initiativeName = "{name_slug}-compliance" -$displayName = "{extraction.framework_name} Compliance Controls" -$description = @" -{extraction.summary[:500] if extraction.summary else f"Regulatory compliance initiative for {extraction.framework_name}."} -"@ - -$metadata = @{{ - category = "Regulatory Compliance" - version = "1.0.0" - source = "ComplianceIQ Compliance Pipeline" - generatedDate = "{datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")}" -}} | ConvertTo-Json -Compress - -Write-Host "Creating initiative: $displayName" -ForegroundColor Green -Write-Host " Name: $initiativeName" -ForegroundColor Gray -Write-Host "" - -try {{ - $initParams = @{{ - Name = $initiativeName - DisplayName = $displayName - Description = $description - Metadata = $metadata - GroupDefinition = $groups - PolicyDefinition = $policies - Parameter = $params - }} - - # Add management group scope if specified - if ($ManagementGroupId) {{ - $initParams["ManagementGroupName"] = $ManagementGroupId - }} - - $initiative = New-AzPolicySetDefinition @initParams - - Write-Host "" - Write-Host "========================================================" -ForegroundColor Green - Write-Host " SUCCESS: Initiative Created" -ForegroundColor Green - Write-Host "========================================================" -ForegroundColor Green - Write-Host "" - Write-Host " Name: $($initiative.Name)" -ForegroundColor White - Write-Host " Display Name: $($initiative.Properties.DisplayName)" -ForegroundColor White - Write-Host " Resource ID: $($initiative.ResourceId)" -ForegroundColor White - Write-Host "" - -}} catch {{ - Write-Host "" - Write-Host "========================================================" -ForegroundColor Red - Write-Host " ERROR: Initiative Creation Failed" -ForegroundColor Red - Write-Host "========================================================" -ForegroundColor Red - Write-Host "" - Write-Error $_.Exception.Message - Write-Host "" - Write-Host "Troubleshooting:" -ForegroundColor Yellow - Write-Host " 1. Verify you have Policy Contributor role" -ForegroundColor Gray - Write-Host " 2. Check that policy definition GUIDs exist in Azure" -ForegroundColor Gray - Write-Host " 3. Run: Connect-AzAccount to re-authenticate" -ForegroundColor Gray - Write-Host " 4. Check Azure subscription quota for custom initiatives" -ForegroundColor Gray - exit 1 -}} - -# ── Optional: Assign the Initiative ─────────────────────────────────────── -if ($AssignAfterCreation) {{ - Write-Host "Assigning initiative to scope..." -ForegroundColor Green - Write-Host "" - - $assignmentName = "{name_slug}-$(Get-Date -Format 'yyyyMMdd')" - - $assignParams = @{{ - Name = $assignmentName - DisplayName = "$displayName - Assessment" - Scope = $TargetScope - PolicySetDefinition = $initiative - Description = "Regulatory compliance assessment for {extraction.framework_name}" - EnforcementMode = "Default" - IdentityType = "SystemAssigned" - Location = $Location - }} - - try {{ - $assignment = New-AzPolicyAssignment @assignParams - - Write-Host " Assignment: $($assignment.Name)" -ForegroundColor White - Write-Host " Scope: $TargetScope" -ForegroundColor White - Write-Host "" - Write-Host "[OK] Initiative assigned successfully" -ForegroundColor Green - - }} catch {{ - Write-Host "WARNING: Assignment failed — initiative was created but not assigned" -ForegroundColor Yellow - Write-Host " Error: $($_.Exception.Message)" -ForegroundColor Gray - Write-Host " You can assign it manually from Azure Portal > Policy > Assignments" -ForegroundColor Gray - }} -}} - -# ── Next Steps ──────────────────────────────────────────────────────────── -Write-Host "" -Write-Host "========================================================" -ForegroundColor Cyan -Write-Host " Next Steps" -ForegroundColor Cyan -Write-Host "========================================================" -ForegroundColor Cyan -Write-Host "" -Write-Host " 1. Azure Portal > Policy > Definitions — verify the initiative" -ForegroundColor White -Write-Host " 2. Azure Portal > Policy > Assignments — assign to desired scope" -ForegroundColor White -Write-Host " 3. Defender for Cloud > Regulatory Compliance — view compliance" -ForegroundColor White -Write-Host " 4. Allow ~24 hours for initial compliance evaluation" -ForegroundColor White -Write-Host " 5. Review non-compliant resources and remediate" -ForegroundColor White -Write-Host "" -Write-Host " Tip: Controls marked [MANUAL ATTESTATION] require manual" -ForegroundColor Gray -Write-Host " evidence in Defender for Cloud > Regulatory Compliance." -ForegroundColor Gray -Write-Host "" -''' - - -def _build_cli_script( - extraction: ControlExtractionResult, - fw_safe: str, -) -> str: - """Generate Azure CLI deployment script.""" - name_slug = fw_safe.replace("_", "-") - desc = (extraction.summary[:500] if extraction.summary else - f"Regulatory compliance initiative for {extraction.framework_name}.") - - return f'''#!/bin/bash -# ============================================================================ -# {extraction.framework_name} — Azure Policy Initiative Deployment (CLI) -# Generated by ComplianceIQ Compliance Pipeline -# Date: {datetime.utcnow().strftime("%Y-%m-%d %H:%M UTC")} -# ============================================================================ -# -# Usage: -# chmod +x deploy-initiative.sh -# ./deploy-initiative.sh # Current subscription -# ./deploy-initiative.sh --management-group mg-compliance # Management group -# -# Prerequisites: -# - Azure CLI installed and authenticated (az login) -# - Files in current directory: groups.json, policies.json, params.json -# ============================================================================ - -set -euo pipefail - -INITIATIVE_NAME="{name_slug}-compliance" -DISPLAY_NAME="{extraction.framework_name} Compliance Controls" -DESCRIPTION="{desc}" -MGMT_GROUP="" - -# Parse arguments -while [[ $# -gt 0 ]]; do - case $1 in - --management-group|-m) - MGMT_GROUP="$2" - shift 2 - ;; - *) - echo "Unknown argument: $1" - exit 1 - ;; - esac -done - -echo "" -echo "========================================================" -echo " {extraction.framework_name}" -echo " Regulatory Compliance Initiative Deployment" -echo "========================================================" -echo "" - -# Verify files -for f in groups.json policies.json params.json; do - if [[ ! -f "$f" ]]; then - echo "ERROR: Missing required file: $f" - exit 1 - fi -done - -echo "[Files] All required files found" -echo "" - -# Build scope args -SCOPE_ARGS="" -if [[ -n "$MGMT_GROUP" ]]; then - SCOPE_ARGS="--management-group $MGMT_GROUP" - echo "[Scope] Management Group: $MGMT_GROUP" -else - SUB=$(az account show --query id -o tsv 2>/dev/null) - echo "[Scope] Subscription: $SUB" -fi -echo "" - -# Create the initiative -echo "Creating initiative: $DISPLAY_NAME" -echo "" - -az policy set-definition create \\ - --name "$INITIATIVE_NAME" \\ - --display-name "$DISPLAY_NAME" \\ - --description "$DESCRIPTION" \\ - --definitions @policies.json \\ - --definition-groups @groups.json \\ - --params @params.json \\ - --metadata category="Regulatory Compliance" \\ - $SCOPE_ARGS - -echo "" -echo "========================================================" -echo " SUCCESS: Initiative created" -echo "========================================================" -echo "" -echo " Next steps:" -echo " 1. Azure Portal > Policy > Definitions — verify" -echo " 2. Azure Portal > Policy > Assignments — assign" -echo " 3. Defender for Cloud > Regulatory Compliance — review" -echo "" -''' - - -def _write_mappings_csv( - csv_path: Path, - extraction: ControlExtractionResult, - mappings: list[ControlPolicyMapping], -): - """Write a comprehensive mapping report as CSV.""" - mapping_lookup = {m.control_id: m for m in mappings} - - fieldnames = [ - "Control_ID", - "Control_Title", - "Domain", - "Control_Type", - "MCSB_Control_ID", - "MCSB_Control_Name", - "Confidence", - "Azure_Policy_IDs", - "Azure_Policy_Names", - "Is_Automatable", - "Manual_Note", - "Defender_Recommendations", - "Mapping_Rationale", - ] - - with open(csv_path, "w", newline="", encoding="utf-8") as f: - writer = csv.DictWriter(f, fieldnames=fieldnames) - writer.writeheader() - - for ctrl in extraction.controls: - m = mapping_lookup.get(ctrl.control_id) - if m: - policy_ids = "; ".join(p.policy_definition_id for p in m.azure_policies) - policy_names = "; ".join(p.policy_name for p in m.azure_policies) - defender_recs = "; ".join(m.defender_recommendations) - else: - policy_ids = "" - policy_names = "" - defender_recs = "" - - writer.writerow({ - "Control_ID": ctrl.control_id, - "Control_Title": ctrl.control_title, - "Domain": ctrl.domain, - "Control_Type": ctrl.control_type, - "MCSB_Control_ID": m.mcsb_control_id if m else "", - "MCSB_Control_Name": m.mcsb_control_name if m else "", - "Confidence": f"{m.confidence_score:.2f}" if m else "", - "Azure_Policy_IDs": policy_ids, - "Azure_Policy_Names": policy_names, - "Is_Automatable": str(m.is_automatable) if m else "", - "Manual_Note": m.manual_attestation_note or "" if m else "", - "Defender_Recommendations": defender_recs, - "Mapping_Rationale": m.mapping_rationale if m else "", - }) - - logger.info(f"Wrote mapping report: {csv_path}") - - -def _write_json(path: Path, data) -> None: - """Write JSON to file with consistent formatting.""" - with open(path, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False, default=str) - logger.info(f"Wrote: {path}") diff --git a/compliance-pipeline/models.py b/compliance-pipeline/models.py deleted file mode 100644 index 543180f..0000000 --- a/compliance-pipeline/models.py +++ /dev/null @@ -1,211 +0,0 @@ -""" -Pydantic models for the PDF-to-Initiative compliance automation pipeline. -These models define the structured outputs expected from Azure OpenAI. -""" - -from pydantic import BaseModel, Field -from typing import List, Optional, Literal -from datetime import datetime - - -# ── Stage 1: Extracted Controls from PDF ────────────────────────────────────── - -class ExtractedControl(BaseModel): - """A single control extracted from a compliance PDF by the LLM.""" - - control_id: str = Field( - ..., description="Control identifier from the document (e.g., 'TR-01', 'POL-03')" - ) - control_title: str = Field( - ..., description="Short title of the control" - ) - control_description: str = Field( - ..., description="Full description of the control requirement" - ) - domain: str = Field( - ..., description="Security/compliance domain (e.g., Network Security, Identity & Access, Data Protection)" - ) - control_type: Literal["Technical", "Policy", "Contractual", "Management", "Operational", "Governance"] = Field( - ..., description="Category of the control" - ) - sub_controls: List[str] = Field( - default_factory=list, - description="List of sub-requirements or sub-controls if any" - ) - - -class ControlExtractionResult(BaseModel): - """Complete result of LLM-based control extraction from a PDF.""" - - framework_name: str = Field( - ..., description="Name of the compliance framework identified in the document" - ) - framework_version: Optional[str] = Field( - None, description="Version of the framework if identified" - ) - issuing_authority: Optional[str] = Field( - None, description="Organization that issued this framework" - ) - country_or_region: Optional[str] = Field( - None, description="Country or region the framework applies to" - ) - controls: List[ExtractedControl] = Field( - ..., description="List of all controls extracted from the document" - ) - summary: str = Field( - ..., description="Brief summary of the framework's purpose and scope" - ) - - -# ── Stage 2: Azure Policy Mappings ──────────────────────────────────────────── - -class AzurePolicyMapping(BaseModel): - """A single Azure Policy mapping for a control.""" - - policy_definition_id: str = Field( - ..., description="Azure Policy Definition GUID (e.g., '4e6c27d5-a6ee-49cf-b2b4-d8fe90fa2b8b')" - ) - policy_name: str = Field( - ..., description="Display name of the Azure Policy" - ) - policy_description: str = Field( - ..., description="Brief description of what this policy enforces" - ) - relevance: Literal["high", "medium", "low"] = Field( - ..., description="How relevant this policy is to the control" - ) - - -class ControlPolicyMapping(BaseModel): - """Complete mapping from a single control to Azure Policies.""" - - control_id: str = Field( - ..., description="The control ID being mapped" - ) - control_title: str = Field( - ..., description="The control title" - ) - domain: str = Field( - ..., description="Security domain" - ) - mcsb_control_id: str = Field( - ..., description="Best matching MCSB control ID (e.g., 'IM-1', 'NS-1', 'DP-3')" - ) - mcsb_control_name: str = Field( - ..., description="Name of the matched MCSB control" - ) - confidence_score: float = Field( - ..., ge=0.0, le=1.0, - description="Confidence of this mapping (0.0-1.0)" - ) - mapping_rationale: str = Field( - ..., description="Explanation of why this mapping was chosen" - ) - azure_policies: List[AzurePolicyMapping] = Field( - default_factory=list, - description="Azure Policy definitions that implement this control" - ) - defender_recommendations: List[str] = Field( - default_factory=list, - description="Relevant Microsoft Defender for Cloud recommendations" - ) - is_automatable: bool = Field( - ..., description="Whether this control can be enforced/audited via Azure Policy" - ) - manual_attestation_note: Optional[str] = Field( - None, description="Note about manual steps required if not fully automatable" - ) - - -class BatchPolicyMappingResult(BaseModel): - """Result of mapping a batch of controls to Azure Policies.""" - - mappings: List[ControlPolicyMapping] = Field( - ..., description="List of all control-to-policy mappings" - ) - - -# ── Stage 3: Validated Initiative ───────────────────────────────────────────── - -class ValidationIssue(BaseModel): - """A validation issue found in the mapping.""" - - severity: Literal["error", "warning", "info"] = Field( - ..., description="Severity level" - ) - control_id: str = Field( - ..., description="Control ID this issue relates to" - ) - message: str = Field( - ..., description="Description of the issue" - ) - suggestion: Optional[str] = Field( - None, description="Suggested fix" - ) - - -class ValidationReport(BaseModel): - """Report from initiative validation.""" - - is_valid: bool = Field(..., description="Whether the initiative passes validation") - total_controls: int = Field(..., description="Total controls in the initiative") - automatable_controls: int = Field(..., description="Controls with Azure Policy mappings") - manual_controls: int = Field(..., description="Controls requiring manual attestation") - unique_policies: int = Field(..., description="Number of unique Azure Policy definitions") - avg_confidence: float = Field(..., description="Average mapping confidence score") - issues: List[ValidationIssue] = Field(default_factory=list, description="Validation issues") - - -# ── Stage 4: Initiative Output Artifacts ────────────────────────────────────── - -class InitiativeGroup(BaseModel): - """A policy definition group in the initiative (maps to a control).""" - - name: str = Field(..., description="Group name (sanitized control ID)") - displayName: str = Field(..., description="Display name with full control reference") - description: str = Field(..., description="Control description") - - -class PolicyDefinitionRef(BaseModel): - """A policy definition reference within the initiative.""" - - PolicyDefinitionReferenceId: str = Field( - ..., description="Unique reference ID for this policy in the initiative" - ) - PolicyDefinitionId: str = Field( - ..., description="Full Azure Policy definition resource ID" - ) - Parameters: dict = Field(default_factory=dict, description="Parameter overrides") - GroupNames: List[str] = Field( - ..., description="List of group names this policy belongs to" - ) - - -class InitiativeDefinition(BaseModel): - """Complete Azure Policy Initiative (Policy Set Definition) for Defender for Cloud.""" - - properties: dict = Field(..., description="The full initiative properties") - - -class PipelineResult(BaseModel): - """Final result from the complete pipeline execution.""" - - framework_name: str - framework_version: Optional[str] = None - issuing_authority: Optional[str] = None - country_or_region: Optional[str] = None - - total_controls_extracted: int - total_policies_mapped: int - automatable_controls: int - manual_controls: int - avg_confidence: float - - validation: ValidationReport - - output_directory: str - files_generated: List[str] - - started_at: datetime - completed_at: datetime - duration_seconds: float diff --git a/compliance-pipeline/pdf_extractor.py b/compliance-pipeline/pdf_extractor.py deleted file mode 100644 index a077065..0000000 --- a/compliance-pipeline/pdf_extractor.py +++ /dev/null @@ -1,129 +0,0 @@ -""" -PDF Text Extraction Module. -Extracts and preprocesses text from compliance control PDF documents. -""" - -import logging -from pathlib import Path -from typing import Optional - -logger = logging.getLogger(__name__) - - -def extract_text_from_pdf(pdf_path: str, max_pages: int = 200) -> str: - """ - Extract all text from a PDF file, page by page. - - Args: - pdf_path: Path to the PDF file. - max_pages: Maximum number of pages to process (safety limit). - - Returns: - Concatenated text from all pages with page markers. - - Raises: - FileNotFoundError: If the PDF does not exist. - ValueError: If the file is not a PDF or is empty. - """ - path = Path(pdf_path) - - if not path.exists(): - raise FileNotFoundError(f"PDF not found: {pdf_path}") - if path.suffix.lower() != ".pdf": - raise ValueError(f"Not a PDF file: {pdf_path}") - if path.stat().st_size == 0: - raise ValueError(f"PDF file is empty: {pdf_path}") - - try: - import pypdf - except ImportError: - raise ImportError( - "pypdf is required for PDF extraction. Install it: pip install pypdf" - ) - - reader = pypdf.PdfReader(str(path)) - total_pages = len(reader.pages) - - if total_pages == 0: - raise ValueError(f"PDF has no pages: {pdf_path}") - - pages_to_process = min(total_pages, max_pages) - logger.info(f"Extracting text from {path.name}: {total_pages} pages (processing {pages_to_process})") - - text_parts: list[str] = [] - - for page_num in range(pages_to_process): - try: - page = reader.pages[page_num] - page_text = page.extract_text() - if page_text and page_text.strip(): - text_parts.append(f"\n=== PAGE {page_num + 1} ===\n{page_text}") - except Exception as e: - logger.warning(f"Failed to extract page {page_num + 1}: {e}") - text_parts.append(f"\n=== PAGE {page_num + 1} === [EXTRACTION FAILED]\n") - - full_text = "\n".join(text_parts) - - if not full_text.strip(): - raise ValueError( - f"No readable text found in {path.name}. " - "The PDF may be scanned/image-based. OCR is not currently supported." - ) - - logger.info( - f"Extracted {len(full_text):,} characters from {pages_to_process} pages" - ) - return full_text - - -def chunk_text(text: str, max_chars: int = 80000, overlap: int = 2000) -> list[str]: - """ - Split large text into overlapping chunks for LLM processing. - - Args: - text: Full extracted text. - max_chars: Maximum characters per chunk. - overlap: Number of overlapping characters between chunks. - - Returns: - List of text chunks. - """ - if len(text) <= max_chars: - return [text] - - chunks = [] - start = 0 - while start < len(text): - end = start + max_chars - chunk = text[start:end] - chunks.append(chunk) - start = end - overlap - - logger.info(f"Split text into {len(chunks)} chunks ({max_chars} chars each, {overlap} overlap)") - return chunks - - -def get_pdf_metadata(pdf_path: str) -> dict: - """ - Extract metadata from the PDF (title, author, etc.). - - Args: - pdf_path: Path to the PDF file. - - Returns: - Dictionary of metadata fields. - """ - import pypdf - - reader = pypdf.PdfReader(pdf_path) - meta = reader.metadata or {} - - return { - "title": meta.get("/Title", ""), - "author": meta.get("/Author", ""), - "subject": meta.get("/Subject", ""), - "creator": meta.get("/Creator", ""), - "producer": meta.get("/Producer", ""), - "pages": len(reader.pages), - "file_size_bytes": Path(pdf_path).stat().st_size, - } diff --git a/compliance-pipeline/pipeline.py b/compliance-pipeline/pipeline.py deleted file mode 100644 index 8dad157..0000000 --- a/compliance-pipeline/pipeline.py +++ /dev/null @@ -1,545 +0,0 @@ -#!/usr/bin/env python3 -""" -ComplianceIQ Compliance Pipeline — PDF/CSV to Defender for Cloud Initiative - -End-to-end automation: - PDF input: 1. Extract text from a compliance PDF - 2. Use Azure OpenAI to extract structured controls - 3. Map each control to Azure Policy definitions via LLM - 4. Validate all mappings - 5. Generate deployable initiative artifacts (JSON + PowerShell) - - CSV input: Skips stages 1-2. Reads pre-structured controls directly from a CSV - file, then maps to Azure Policies (stages 3-5). - -Usage: - python pipeline.py - python pipeline.py - python pipeline.py --framework-name "My Framework" - python pipeline.py --output ./my-output --min-confidence 0.6 - python pipeline.py --locations uaenorth,uaecentral - -CSV Column Names (flexible, case-insensitive): - Control ID: Control_ID, ControlID, ID, No - Domain: Domain, Category, Section, Group - Title: Control_Title, Title, Requirement, Name - Description: Control_Description, Description, Requirement_Summary, Details - Type: Control_Type, ControlType, Type - -Environment: - AZURE_OPENAI_ENDPOINT (required) Azure OpenAI endpoint URL - AZURE_OPENAI_DEPLOYMENT_NAME (optional) Model deployment name (default: gpt-4.1) - AZURE_OPENAI_API_KEY (optional) API key — if omitted, uses DefaultAzureCredential - AZURE_OPENAI_API_VERSION (optional) API version (default: 2024-12-01-preview) -""" - -import argparse -import csv -import json -import logging -import sys -import time -from datetime import datetime -from pathlib import Path - -# Add this directory to path for local imports -sys.path.insert(0, str(Path(__file__).parent)) - -from config import PipelineConfig -from pdf_extractor import extract_text_from_pdf, get_pdf_metadata -from control_extractor import extract_controls_from_text -from policy_mapper import map_controls_to_azure_policies -from validator import validate_mappings -from initiative_builder import build_initiative_artifacts -from models import PipelineResult, ControlExtractionResult, ExtractedControl - - -def setup_logging(verbose: bool = False) -> None: - """Configure structured logging.""" - level = logging.DEBUG if verbose else logging.INFO - fmt = "%(asctime)s [%(levelname)-7s] %(name)s — %(message)s" - logging.basicConfig(level=level, format=fmt, datefmt="%H:%M:%S") - # Quiet noisy libraries - logging.getLogger("httpx").setLevel(logging.WARNING) - logging.getLogger("openai").setLevel(logging.WARNING) - logging.getLogger("azure").setLevel(logging.WARNING) - - -def print_banner(): - """Print the pipeline banner.""" - print() - print("╔══════════════════════════════════════════════════════════════╗") - print("║ ComplianceIQ Compliance Pipeline ║") - print("║ PDF → Controls → Azure Policy → Defender for Cloud ║") - print("╚══════════════════════════════════════════════════════════════╝") - print() - - -def print_stage(num: int, title: str): - """Print a stage header.""" - print(f"\n{'─' * 60}") - print(f" Stage {num}: {title}") - print(f"{'─' * 60}\n") - - -def parse_csv_to_extraction( - csv_path: str, - framework_name: str | None = None, -) -> ControlExtractionResult: - """ - Parse a compliance framework CSV into a ControlExtractionResult. - - Supports flexible column names — detects the following variants: - - Control ID: Control_ID, ControlID, ID, control_id, id - - Domain: Domain, Category, Section, domain, category - - Title: Control_Title, ControlTitle, Title, Requirement, Name - - Description: Control_Description, Description, Requirement_Summary, - Requirements, Requirement_Text, Details, Text - - Control Type: Control_Type, ControlType, Type - - The CSV may optionally include a header row with these column names. - Any columns not recognised are ignored. - """ - path = Path(csv_path) - if not path.exists(): - raise FileNotFoundError(f"CSV file not found: {csv_path}") - - fw_name = framework_name or path.stem.replace("_", " ").replace("-", " ") - - # Column name aliases (lower-cased for matching) - ID_COLS = {"control_id", "controlid", "id", "ctrl_id", "ctrl id", "no", "number"} - DOMAIN_COLS = {"domain", "category", "section", "group", "pillar"} - TITLE_COLS = {"control_title", "controltitle", "title", "requirement", "name", - "control_name", "controlname", "control name", "requirement_name"} - DESC_COLS = {"control_description", "description", "requirement_summary", - "requirements", "requirement_text", "details", "text", - "summary", "description_text", "objective", "intent"} - TYPE_COLS = {"control_type", "controltype", "type"} - - def _find_col(headers: list[str], candidates: set[str]) -> int | None: - for i, h in enumerate(headers): - if h.lower().strip() in candidates: - return i - return None - - controls: list[ExtractedControl] = [] - - with open(csv_path, newline="", encoding="utf-8-sig") as f: - reader = csv.reader(f) - raw_rows = list(reader) - - if not raw_rows: - raise ValueError(f"CSV file is empty: {csv_path}") - - # Detect header row - first_row = [c.strip() for c in raw_rows[0]] - id_col = _find_col(first_row, ID_COLS) - domain_col = _find_col(first_row, DOMAIN_COLS) - title_col = _find_col(first_row, TITLE_COLS) - desc_col = _find_col(first_row, DESC_COLS) - type_col = _find_col(first_row, TYPE_COLS) - - # If no header matched, assume positional: col0=ID, col1=Domain, col2=Title, col3=Desc - has_header = any(v is not None for v in [id_col, domain_col, title_col, desc_col]) - data_rows = raw_rows[1:] if has_header else raw_rows - if not has_header: - id_col, domain_col, title_col, desc_col = 0, 1, 2, 3 - - seen_ids: set[str] = set() - for row_num, row in enumerate(data_rows, start=2 if has_header else 1): - if not row or all(c.strip() == "" for c in row): - continue # Skip blank rows - - def _get(col: int | None, fallback: str = "") -> str: - if col is None or col >= len(row): - return fallback - return row[col].strip() - - control_id = _get(id_col) or f"CTRL-{row_num:03d}" - domain = _get(domain_col) or "Governance & Policy" - title = _get(title_col) or control_id - description = _get(desc_col) or title - control_type = _get(type_col) or "Technical" - - # Deduplicate - if control_id in seen_ids: - control_id = f"{control_id}_{row_num}" - seen_ids.add(control_id) - - controls.append(ExtractedControl( - control_id=control_id, - control_title=title, - control_description=description, - domain=domain, - control_type=control_type, - sub_controls=[], - )) - - if not controls: - raise ValueError(f"No controls found in CSV: {csv_path}") - - return ControlExtractionResult( - framework_name=fw_name, - framework_version=None, - issuing_authority=None, - country_or_region=None, - controls=controls, - summary=f"{fw_name} compliance framework with {len(controls)} controls (imported from CSV).", - ) - - -def run_pipeline( - pdf_path: str | None = None, - csv_path: str | None = None, - framework_name: str | None = None, - output_dir: str = "./output", - min_confidence: float = 0.5, - allowed_locations: list[str] | None = None, - env_file: str | None = None, - verbose: bool = False, -) -> PipelineResult: - """ - Execute the full pipeline from either a PDF or a CSV file. - - For PDF inputs: runs all 5 stages (extract text → LLM controls → map → validate → artifacts). - For CSV inputs: skips stages 1-2 and goes directly to Azure Policy mapping. - - Args: - pdf_path: Path to the compliance PDF (mutually exclusive with csv_path). - csv_path: Path to a CSV with pre-structured controls (mutually exclusive with pdf_path). - framework_name: Override the framework name (useful for CSV inputs). - output_dir: Directory for output artifacts. - min_confidence: Minimum confidence threshold. - allowed_locations: Optional Azure regions for location policies. - env_file: Optional path to .env file. - verbose: Enable debug logging. - - Returns: - PipelineResult with summary and file paths. - """ - if not pdf_path and not csv_path: - raise ValueError("Either pdf_path or csv_path must be provided") - if pdf_path and csv_path: - raise ValueError("Provide either pdf_path or csv_path, not both") - logger = logging.getLogger("pipeline") - started_at = datetime.utcnow() - t0 = time.time() - - # ── Load config ─────────────────────────────────────────────────────── - config = PipelineConfig.from_env(env_file) - config.output_dir = output_dir - config.min_confidence_threshold = min_confidence - - errors = config.validate() - if errors: - for err in errors: - logger.error(f"Config error: {err}") - print("\n❌ Configuration errors — set required environment variables.") - print(" Required: AZURE_OPENAI_ENDPOINT") - print(" Optional: AZURE_OPENAI_API_KEY (or use az login for DefaultAzureCredential)") - sys.exit(1) - - # ══════════════════════════════════════════════════════════════════════ - # STAGE 1 & 2: Input Processing (PDF extraction + LLM, or CSV parse) - # ══════════════════════════════════════════════════════════════════════ - if csv_path: - print_stage(1, "CSV Control Import (skipping PDF extraction + LLM)") - extraction = parse_csv_to_extraction(csv_path, framework_name=framework_name) - print(f" ✓ Framework: {extraction.framework_name}") - print(f" ✓ Controls: {len(extraction.controls)} imported from CSV") - print(f" ✓ Source: {csv_path}") - else: - # ── STAGE 1: PDF Text Extraction ────────────────────────────────── - print_stage(1, "PDF Text Extraction") - - pdf_metadata = get_pdf_metadata(pdf_path) - logger.info(f"PDF: {Path(pdf_path).name} ({pdf_metadata['pages']} pages, {pdf_metadata['file_size_bytes']:,} bytes)") - if pdf_metadata.get("title"): - logger.info(f"Title: {pdf_metadata['title']}") - - pdf_text = extract_text_from_pdf(pdf_path, max_pages=config.max_pdf_pages) - print(f" ✓ Extracted {len(pdf_text):,} characters from {pdf_metadata['pages']} pages") - - # ── STAGE 2: LLM Control Extraction ────────────────────────────── - print_stage(2, "AI Control Extraction (Azure OpenAI)") - - print(f" Sending to {config.azure_openai_deployment}...") - extraction = extract_controls_from_text(pdf_text, config, pdf_metadata) - - # Override framework name if provided - if framework_name: - extraction.framework_name = framework_name - - if csv_path: - # If coming from CSV, print domain breakdown here (stage 2 equivalent) - print_stage(2, "Controls Overview") - print(f" ✓ Framework: {extraction.framework_name}") - - if not csv_path: - print(f" ✓ Framework: {extraction.framework_name}") - if extraction.framework_version: - print(f" ✓ Version: {extraction.framework_version}") - if extraction.issuing_authority: - print(f" ✓ Authority: {extraction.issuing_authority}") - if extraction.country_or_region: - print(f" ✓ Region: {extraction.country_or_region}") - print(f" ✓ Controls: {len(extraction.controls)} extracted") - - # Show domain breakdown - domains: dict[str, int] = {} - for ctrl in extraction.controls: - domains[ctrl.domain] = domains.get(ctrl.domain, 0) + 1 - print(f"\n Domain Breakdown:") - for domain, count in sorted(domains.items(), key=lambda x: -x[1]): - print(f" {domain}: {count}") - - # ══════════════════════════════════════════════════════════════════════ - # STAGE 3: Azure Policy Mapping - # ══════════════════════════════════════════════════════════════════════ - print_stage(3, "Azure Policy Mapping (Azure OpenAI)") - - def progress(current, total): - pct = (current / total) * 100 - print(f" Mapping controls: {current}/{total} ({pct:.0f}%)", end="\r") - - mappings = map_controls_to_azure_policies(extraction, config, progress_callback=progress) - print() # Clear the progress line - - automatable = sum(1 for m in mappings if m.is_automatable) - manual = len(mappings) - automatable - total_policies = sum(len(m.azure_policies) for m in mappings) - unique_policies = len({p.policy_definition_id for m in mappings for p in m.azure_policies}) - avg_conf = sum(m.confidence_score for m in mappings) / len(mappings) if mappings else 0 - - print(f" ✓ Mapped: {len(mappings)} controls") - print(f" ✓ Automatable: {automatable} (via Azure Policy)") - print(f" ✓ Manual: {manual} (require attestation)") - print(f" ✓ Policies: {unique_policies} unique Azure Policy definitions") - print(f" ✓ Confidence: {avg_conf:.2f} average") - - # ══════════════════════════════════════════════════════════════════════ - # STAGE 4: Validation - # ══════════════════════════════════════════════════════════════════════ - print_stage(4, "Mapping Validation") - - validation = validate_mappings(extraction, mappings, min_confidence) - - errors_count = sum(1 for i in validation.issues if i.severity == "error") - warnings_count = sum(1 for i in validation.issues if i.severity == "warning") - infos_count = sum(1 for i in validation.issues if i.severity == "info") - - status = "✓ PASSED" if validation.is_valid else "✗ FAILED" - color = "Green" if validation.is_valid else "Red" - - print(f" {status}") - print(f" Errors: {errors_count}") - print(f" Warnings: {warnings_count}") - print(f" Info: {infos_count}") - - if errors_count > 0: - print(f"\n Errors:") - for issue in validation.issues: - if issue.severity == "error": - print(f" ❌ [{issue.control_id}] {issue.message}") - if issue.suggestion: - print(f" → {issue.suggestion}") - - if warnings_count > 0 and verbose: - print(f"\n Warnings:") - for issue in validation.issues: - if issue.severity == "warning": - print(f" ⚠️ [{issue.control_id}] {issue.message}") - - # ══════════════════════════════════════════════════════════════════════ - # STAGE 5: Generate Initiative Artifacts - # ══════════════════════════════════════════════════════════════════════ - print_stage(5, "Generate Defender for Cloud Initiative") - - files = build_initiative_artifacts( - extraction=extraction, - mappings=mappings, - validation=validation, - output_dir=output_dir, - allowed_locations=allowed_locations, - ) - - print(f" Output directory: {output_dir}/") - print(f" Files generated:") - for f in files: - fname = Path(f).name - size = Path(f).stat().st_size - print(f" 📄 {fname} ({size:,} bytes)") - - # ══════════════════════════════════════════════════════════════════════ - # Summary - # ══════════════════════════════════════════════════════════════════════ - elapsed = time.time() - t0 - completed_at = datetime.utcnow() - - print(f"\n{'═' * 60}") - print(f" Pipeline Complete — {elapsed:.1f}s") - print(f"{'═' * 60}") - print(f" Framework: {extraction.framework_name}") - print(f" Controls: {len(extraction.controls)}") - print(f" Automatable: {automatable}") - print(f" Manual: {manual}") - print(f" Unique Policies: {unique_policies}") - print(f" Avg Confidence: {avg_conf:.2f}") - print(f" Validation: {'PASSED' if validation.is_valid else 'FAILED'}") - print() - print(f" Deploy to Azure:") - print(f" PowerShell: cd {output_dir} && .\\Deploy-Initiative.ps1") - print(f" Azure CLI: cd {output_dir} && bash deploy-initiative.sh") - print() - - return PipelineResult( - framework_name=extraction.framework_name, - framework_version=extraction.framework_version, - issuing_authority=extraction.issuing_authority, - country_or_region=extraction.country_or_region, - total_controls_extracted=len(extraction.controls), - total_policies_mapped=unique_policies, - automatable_controls=automatable, - manual_controls=manual, - avg_confidence=avg_conf, - validation=validation, - output_directory=output_dir, - files_generated=files, - started_at=started_at, - completed_at=completed_at, - duration_seconds=elapsed, - ) - - -def main(): - """CLI entry point.""" - parser = argparse.ArgumentParser( - description="ComplianceIQ Compliance Pipeline — PDF/CSV to Defender for Cloud Initiative", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -Examples: - # From PDF (full AI extraction) - python pipeline.py ./my-framework.pdf - - # From CSV (skips PDF extraction; goes straight to policy mapping) - python pipeline.py ./my-framework.csv - - # From CSV with explicit framework name - python pipeline.py ./my-framework.csv --framework-name "SAMA Cybersecurity Framework" - - # Custom output directory - python pipeline.py ./my-framework.pdf --output ./my-framework-output - - # With Azure region restrictions - python pipeline.py ./my-framework.pdf --locations uaenorth,uaecentral - - # Higher confidence threshold (exclude weak mappings) - python pipeline.py ./my-framework.pdf --min-confidence 0.7 - - # Verbose logging for debugging - python pipeline.py ./my-framework.pdf --verbose - -CSV Column Names (flexible, case‑insensitive): - Control ID: Control_ID, ControlID, ID, No - Domain: Domain, Category, Section, Group - Title: Control_Title, Title, Requirement, Name - Description: Control_Description, Description, Requirement_Summary, Details - Type: Control_Type, ControlType, Type - -Environment Variables: - AZURE_OPENAI_ENDPOINT Azure OpenAI endpoint (required) - AZURE_OPENAI_DEPLOYMENT_NAME Model deployment name (default: gpt-4.1) - AZURE_OPENAI_API_KEY API key (optional — uses DefaultAzureCredential if omitted) - AZURE_OPENAI_API_VERSION API version (default: 2024-12-01-preview) -""", - ) - - parser.add_argument( - "input", - help="Path to the compliance document — PDF or CSV", - ) - parser.add_argument( - "--framework-name", "-n", - default=None, - help="Override the framework name (useful when importing a CSV without a clear name)", - ) - parser.add_argument( - "--output", "-o", - default=None, - help="Output directory for generated files (default: ./output/)", - ) - parser.add_argument( - "--min-confidence", - type=float, - default=0.5, - help="Minimum confidence threshold for including mappings (default: 0.5)", - ) - parser.add_argument( - "--locations", - default=None, - help="Comma-separated Azure region names for location policies (e.g., uaenorth,uaecentral)", - ) - parser.add_argument( - "--env-file", - default=None, - help="Path to .env file with Azure OpenAI credentials", - ) - parser.add_argument( - "--verbose", "-v", - action="store_true", - help="Enable debug logging", - ) - parser.add_argument( - "--json-output", - action="store_true", - help="Also write pipeline result summary as JSON", - ) - - args = parser.parse_args() - - setup_logging(args.verbose) - print_banner() - - # Auto-detect input type from extension - input_path = Path(args.input) - is_csv = input_path.suffix.lower() == ".csv" - - if not input_path.exists(): - print(f"❌ File not found: {args.input}") - sys.exit(1) - - # Parse locations - locations = None - if args.locations: - locations = [loc.strip() for loc in args.locations.split(",") if loc.strip()] - - # Determine output directory - output_dir = args.output - if not output_dir: - output_dir = f"./output/{input_path.stem}" - - # Run the pipeline - result = run_pipeline( - pdf_path=None if is_csv else args.input, - csv_path=args.input if is_csv else None, - framework_name=args.framework_name, - output_dir=output_dir, - min_confidence=args.min_confidence, - allowed_locations=locations, - env_file=args.env_file, - verbose=args.verbose, - ) - - # Optionally write result summary - if args.json_output: - result_path = Path(output_dir) / "pipeline_result.json" - with open(result_path, "w") as f: - json.dump(result.model_dump(), f, indent=2, default=str) - print(f" Pipeline result: {result_path}") - - # Exit code based on validation - sys.exit(0 if result.validation.is_valid else 1) - - -if __name__ == "__main__": - main() diff --git a/compliance-pipeline/policy_mapper.py b/compliance-pipeline/policy_mapper.py deleted file mode 100644 index 7f7545b..0000000 --- a/compliance-pipeline/policy_mapper.py +++ /dev/null @@ -1,274 +0,0 @@ -""" -Azure Policy Mapping Engine. -Uses Azure OpenAI to map extracted controls to Azure Policy definitions, -MCSB controls, and Defender for Cloud recommendations. -""" - -import json -import logging -from typing import Optional - -from models import ( - ExtractedControl, - ControlExtractionResult, - ControlPolicyMapping, - AzurePolicyMapping, - BatchPolicyMappingResult, -) -from config import PipelineConfig -from control_extractor import get_openai_client - -logger = logging.getLogger(__name__) - -# ── System prompt for Azure Policy mapping ──────────────────────────────────── - -MAPPING_SYSTEM_PROMPT = """You are an expert Azure cloud security architect specializing in Azure Policy, Microsoft Cloud Security Benchmark (MCSB), and Microsoft Defender for Cloud. - -Your task is to map compliance framework controls to: -1. **Azure Policy definitions** — specific built-in policy GUIDs that enforce or audit the control -2. **MCSB controls** — the closest matching Microsoft Cloud Security Benchmark control -3. **Defender for Cloud recommendations** — relevant recommendations - -## Azure Policy Mapping Guidelines - -For EACH control, identify **3 to 8** Azure Policy definitions that enforce or audit the control requirement. -Do NOT map only 1 policy per control — always identify multiple policies that together provide broad coverage. -Cover different resource types: VMs, storage, databases, networking, identity, and container services where relevant. - -### Policy ID Format -Use ONLY the GUIDs listed below. Do NOT invent or guess GUIDs. - -**Identity & Access Management:** -- `4e6c27d5-a6ee-49cf-b2b4-d8fe90fa2b8b` — MFA should be enabled on accounts with owner permissions -- `9297c21d-2ed6-4474-b48f-163f75654ce3` — MFA should be enabled on accounts with write permissions -- `e3576e28-8b17-4677-84c3-db2990658d64` — MFA should be enabled on accounts with read permissions -- `a451c1ef-c6ca-483d-87ed-f49761e3ffb5` — Role-Based Access Control (RBAC) should be used on Kubernetes Services -- `34c877ad-507e-4c82-993e-3452a6e0ad3c` — Kubernetes Services should have Role-Based Access Control enabled -- `0b15565f-aa9e-48ba-8619-45960f2c314d` — There should be more than one owner assigned to your subscription -- `a8eff44e-8db1-4c48-82a2-64d4e30b56bc` — Deprecated accounts with owner permissions should be removed -- `94e1c2ac-cbbe-4cac-a2b5-2cb8b36ce676` — Deprecated accounts should be removed from your subscription -- `9bc48460-f641-4a27-9f38-efe33a4a3e9e` — An Azure Active Directory administrator should be provisioned for SQL servers -- `abfb7388-5bf4-4ad7-ba99-2cd2f41cebb9` — An Azure Active Directory administrator should be provisioned for SQL Managed Instance -- `0015ea4d-51ff-4ce3-8d8c-f3f8f0be26b8` — Audit usage of custom RBAC roles -- `1f314764-cb73-4fc9-b863-8eca98ac36e9` — RBAC should be used on Kubernetes Services -- `b0f33259-77d7-4c9e-aac6-3aabcfae693c` — Management ports of virtual machines should be protected with just-in-time network access control - -**Network Security:** -- `e71308d3-144b-4262-b144-efdc3cc90517` — Subnets should be associated with a Network Security Group -- `2c89a2e5-7285-40fe-afe0-ae8654b92fb2` — All network ports should be restricted on network security groups associated to your VM -- `fc5e4038-4584-4632-8c85-c0448d374b2c` — All network ports should be restricted on NSGs associated to virtual machines -- `bb91dfba-c30d-4263-9add-9c2384e659a6` — Remote debugging should be turned off for Web Applications -- `cb510bfd-1cba-4d9f-a1ea-bed557ae0564` — Remote debugging should be turned off for Function Apps -- `f9d614c5-c173-4d56-95a7-b4437057d193` — Remote debugging should be turned off for API Apps -- `1e66c121-a66d-4b99-b523-e2cf4bf16934` — Azure Firewall should be enabled to protect your virtual network -- `83e0d761-c550-47de-b1b6-359f2a30b354` — Adaptive network hardening recommendations should be applied on internet facing virtual machines -- `9dfea752-cf9d-4745-b47b-81b8330bbb9f` — SQL Managed Instance should have public endpoint access disabled -- `ae89ebf1-3572-4ab1-b1bd-ec5f00bab3a6` — Private endpoint connections on Azure SQL Database should be enabled -- `1c06e275-d63d-4540-b761-71f364c2111d` — Private endpoint should be enabled for Key Vault - -**Encryption & Data Protection:** -- `7595c971-233d-4bcf-bd18-596129188c49` — Transparent data encryption on SQL databases should be enabled -- `404c3081-a854-4457-ae30-26a93ef643f9` — Secure transfer to storage accounts should be enabled -- `4733ea7b-a883-42fe-8cac-97454c2a9e4a` — Storage accounts should restrict network access -- `a4af4a39-4135-4a60-8bf6-d34b2a1d4f2c` — Storage accounts should use customer-managed key for encryption -- `1f905d99-2622-4c13-100b-f7c8e2078bc5` — Azure Cosmos DB accounts should use customer-managed keys to encrypt data at rest -- `702dd420-7fcc-42c5-afe8-4026edd20fe0` — OS and data disks should be encrypted with a customer-managed key -- `18adea5e-f416-4d0f-8aa8-d24321e3e274` — PostgreSQL servers should use customer-managed keys to encrypt data at rest -- `0a370ff3-6cab-4e85-8995-295fd854c5b8` — SQL managed instances should use customer-managed keys to encrypt data at rest -- `0961003e-5a0a-4549-abde-af6a37f2724d` — Virtual machines should encrypt temp disks, caches, and data flows between Compute and Storage resources -- `67121cc7-ff16-4bfd-986d-b15f4c767a1b` — Cognitive Services accounts should enable data encryption with a customer-managed key -- `0725b4dd-7e76-479c-a735-68e7ee23d5ca` — Cognitive Services accounts should restrict network access - -**Logging & Monitoring:** -- `818719e5-1338-4776-9a9d-3c31e4df5986` — Log Analytics agent should be installed on your virtual machine for Azure Security Center monitoring -- `428256e6-1fac-4f48-a757-df34c2b3336d` — Audit diagnostic setting for listed resource types -- `89099bee-89e0-4b26-a5f4-165451757743` — SQL servers should be configured with 90 days auditing retention or higher -- `b954148f-4c11-4c38-8221-be76711e194e` — Advanced data security should be enabled on SQL Managed Instance -- `b0d14bf4-f90c-4e66-9457-1346f80b5a44` — Email notification to subscription owner for high severity alerts should be enabled -- `6e2593d9-add6-4083-9c9b-4b7d2188c899` — Email notifications to admins should be enabled in Microsoft Defender for SQL - -**Key Management:** -- `8e826246-c976-48f6-b03e-619bb92b3d82` — Key Vault keys should have an expiration date -- `5f0bc445-3935-4915-9981-011aa2b46147` — Key Vault secrets should have an expiration date -- `f4b53539-8df9-40e4-86c6-6b607703bd4e` — Keys should be backed by a hardware security module (HSM) -- `6a523b34-47f5-4a80-a97e-d3e2d8bca2d6` — Azure Key Vault Managed HSM should have purge protection enabled -- `c39ba22d-4428-4149-b981-98acef4f7277` — Azure Key Vault should have firewall enabled - -**Endpoint Security & Vulnerability Management:** -- `e96a9a5f-07ca-471b-9bc5-6a0f33cbd68f` — Vulnerability assessment should be enabled on your virtual machines -- `44e1ad92-5f90-4a45-83bb-81cd4695e9f4` — Machines should have vulnerability findings resolved -- `1b7aa243-0538-4a73-b824-0b3fc489d80c` — Vulnerability assessment should be enabled on SQL Managed Instance -- `013e242c-8828-4970-87b3-ab247555486d` — Endpoint protection should be installed on your machines -- `ac076320-ddcf-4066-b451-6154267e8ad2` — Monitor missing Endpoint Protection in Azure Security Center -- `86b3d65f-7626-441e-b690-81a8b71cff60` — System updates should be installed on your machines -- `bd876905-5b84-4f73-ab2d-2e7a7c4568d9` — A vulnerability assessment solution should be enabled on your virtual machines -- `9c276cf7-d6e0-4a09-a4b4-be5f06902a79` — VMSS system updates should be installed - -**Backup & Recovery:** -- `09024ccc-0c5f-475e-9457-b7c0d9ed487b` — Azure Backup should be enabled for Virtual Machines -- `013e242c-8828-4970-87b3-ab247555486d` — Endpoint protection health issues should be resolved on your machines -- `22bee202-a82f-4305-9a2a-6d7f44d4dedb` — Geo-redundant backup should be enabled for Azure Database for MySQL -- `e96a9a5f-07ca-471b-9bc5-6a0f33cbd68f` — Vulnerability assessment should be enabled on your virtual machines - -**Data Classification & Privacy:** -- `ca610c1d-041c-4332-9d88-7ed3094967c7` — Private endpoint connections on Azure SQL Database should be enabled -- `0b60c0b2-2dc2-4e1c-b5c9-abbed971de53` — Sensitive data in your SQL databases should be classified - -**Location / Data Residency:** -- `e56962a6-4747-49cd-b67b-bf8b01975c4c` — Allowed locations -- `37bc2e11-1d3c-4e6e-a9fc-62ca7b58b6c2` — Allowed locations for resource groups - -**Incident Response:** -- `2c89a2e5-7285-40fe-afe0-ae8654b92fb2` — Subscriptions should have a contact email address for security issues - -### ⛔ NEVER USE THESE GUIDs (deployment failures / not available): -- `e1145ab1-eb4f-43d8-911b-36ddf771d13f` — DO NOT USE (Azure Update Manager — not available) -- `055f3b15-58a8-4d91-a4f6-8437a6c8f7e8` — DO NOT USE (DDoS Standard — not available) -- `b79fa14e-238a-4c2d-b376-442ce508fc84` — DO NOT USE (DINE activity log — causes parameter conflict) -- `55d1f543-d1b0-4811-9663-d6d0dbc6326d` — DO NOT USE (DINE Cognitive Services — causes parameter conflict) - -### MCSB Control IDs -Map to the closest MCSB control using these domain prefixes: -- **NS** (Network Security): NS-1 through NS-10 -- **IM** (Identity Management): IM-1 through IM-9 -- **PA** (Privileged Access): PA-1 through PA-8 -- **DP** (Data Protection): DP-1 through DP-8 -- **AM** (Asset Management): AM-1 through AM-5 -- **LT** (Logging and Threat Detection): LT-1 through LT-7 -- **IR** (Incident Response): IR-1 through IR-6 -- **PV** (Posture and Vulnerability Management): PV-1 through PV-6 -- **ES** (Endpoint Security): ES-1 through ES-3 -- **BR** (Backup and Recovery): BR-1 through BR-4 -- **DS** (DevOps Security): DS-1 through DS-7 -- **GS** (Governance and Strategy): GS-1 through GS-10 - -### Automatable vs Manual -- If a control can be enforced or audited via Azure Policy: `is_automatable: true` -- If a control requires human processes, contracts, or governance: `is_automatable: false` - - Add a `manual_attestation_note` explaining what manual steps are needed - -### Confidence Score -- 0.9-1.0: Exact match — Azure Policy directly enforces this control -- 0.7-0.8: Strong — Policy covers the core requirement with minor gaps -- 0.5-0.6: Partial — Policy partially addresses the control -- 0.3-0.4: Weak — Only tangentially related -- 0.0-0.2: No direct Azure Policy available - -### Critical Rules -- ONLY use GUIDs from the list above. Do NOT invent or guess GUIDs. -- Map **3 to 8 policies per control** for automatable controls — never just 1 or 2. -- A single control can map to MULTIPLE Azure Policies. -- A single Azure Policy can appear in MULTIPLE control groups. -- If no Azure Policy exists for a control, set `is_automatable: false` and explain. -- Be precise with confidence scores.""" - - -def map_controls_to_azure_policies( - extraction: ControlExtractionResult, - config: PipelineConfig, - progress_callback=None, -) -> list[ControlPolicyMapping]: - """ - Map all extracted controls to Azure Policy definitions using Azure OpenAI. - - Controls are processed in batches to stay within token limits. - - Args: - extraction: The extracted controls from the PDF. - config: Pipeline configuration. - progress_callback: Optional callable(current, total) for progress updates. - - Returns: - List of ControlPolicyMapping objects. - """ - client = get_openai_client(config) - controls = extraction.controls - batch_size = config.batch_size - - all_mappings: list[ControlPolicyMapping] = [] - - # Process in batches - total_batches = (len(controls) + batch_size - 1) // batch_size - - for batch_idx in range(total_batches): - start = batch_idx * batch_size - end = min(start + batch_size, len(controls)) - batch = controls[start:end] - - logger.info( - f"Mapping batch {batch_idx + 1}/{total_batches} " - f"(controls {start + 1}-{end} of {len(controls)})" - ) - - batch_mappings = _map_batch( - client=client, - config=config, - controls=batch, - framework_name=extraction.framework_name, - ) - - all_mappings.extend(batch_mappings) - - if progress_callback: - progress_callback(end, len(controls)) - - logger.info(f"Completed mapping {len(all_mappings)} controls to Azure Policies") - return all_mappings - - -def _map_batch( - client, - config: PipelineConfig, - controls: list[ExtractedControl], - framework_name: str, -) -> list[ControlPolicyMapping]: - """Map a batch of controls via a single LLM call.""" - - # Build the control descriptions for the prompt - controls_text = "" - for ctrl in controls: - sub_text = "" - if ctrl.sub_controls: - sub_text = "\n Sub-controls: " + "; ".join(ctrl.sub_controls) - controls_text += f""" - - ID: {ctrl.control_id} - Title: {ctrl.control_title} - Domain: {ctrl.domain} - Type: {ctrl.control_type} - Description: {ctrl.control_description}{sub_text} -""" - - user_prompt = f"""## Framework: {framework_name} - -## Controls to Map ({len(controls)} controls) -{controls_text} - ---- - -For EACH control above, provide: -1. The best-matching MCSB control ID and name -2. All relevant Azure Policy definition GUIDs that enforce or audit this control -3. Relevant Defender for Cloud recommendations -4. Whether the control is automatable via Azure Policy -5. A confidence score and rationale - -Map ALL {len(controls)} controls. Do not skip any.""" - - logger.info(f"Sending {len(user_prompt):,} chars for policy mapping...") - - completion = client.beta.chat.completions.parse( - model=config.azure_openai_deployment, - messages=[ - {"role": "system", "content": MAPPING_SYSTEM_PROMPT}, - {"role": "user", "content": user_prompt}, - ], - response_format=BatchPolicyMappingResult, - max_completion_tokens=config.max_tokens, - ) - - result = completion.choices[0].message.parsed - if not result: - raise ValueError("LLM returned empty mapping result") - - logger.info(f"Mapped {len(result.mappings)} controls in this batch") - return result.mappings diff --git a/compliance-pipeline/requirements.txt b/compliance-pipeline/requirements.txt deleted file mode 100644 index 56d3879..0000000 --- a/compliance-pipeline/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -pypdf>=4.0.0 -openai>=1.40.0 -azure-identity>=1.15.0 -pydantic>=2.0.0 diff --git a/compliance-pipeline/validator.py b/compliance-pipeline/validator.py deleted file mode 100644 index 7bdc418..0000000 --- a/compliance-pipeline/validator.py +++ /dev/null @@ -1,256 +0,0 @@ -""" -Mapping Validation Module. -Validates the control-to-Azure-Policy mappings before generating initiative artifacts. -""" - -import logging -import re -from typing import Optional - -from models import ( - ControlPolicyMapping, - ControlExtractionResult, - ValidationReport, - ValidationIssue, -) - -logger = logging.getLogger(__name__) - -# Verified Azure Policy definition GUIDs (tested and confirmed deployable in MngEnvMCAP). -# Used to validate whether a GUID is likely real before deployment. -# GUIDs NOT in this set will generate an "info" warning — not an error. -KNOWN_POLICY_GUIDS = { - # Identity & Access Management - "4e6c27d5-a6ee-49cf-b2b4-d8fe90fa2b8b", # MFA owners - "9297c21d-2ed6-4474-b48f-163f75654ce3", # MFA write - "e3576e28-8b17-4677-84c3-db2990658d64", # MFA read - "a451c1ef-c6ca-483d-87ed-f49761e3ffb5", # K8s RBAC (audit) - "34c877ad-507e-4c82-993e-3452a6e0ad3c", # K8s RBAC enabled - "0b15565f-aa9e-48ba-8619-45960f2c314d", # Multiple subscription owners - "a8eff44e-8db1-4c48-82a2-64d4e30b56bc", # Deprecated owner accounts removed - "94e1c2ac-cbbe-4cac-a2b5-2cb8b36ce676", # Deprecated accounts removed - "9bc48460-f641-4a27-9f38-efe33a4a3e9e", # AAD admin SQL server - "abfb7388-5bf4-4ad7-ba99-2cd2f41cebb9", # AAD admin SQL MI - "0015ea4d-51ff-4ce3-8d8c-f3f8f0be26b8", # Custom RBAC roles audit - "1f314764-cb73-4fc9-b863-8eca98ac36e9", # RBAC K8s services - "b0f33259-77d7-4c9e-aac6-3aabcfae693c", # JIT VM access - # Network Security - "e71308d3-144b-4262-b144-efdc3cc90517", # NSG on subnets - "2c89a2e5-7285-40fe-afe0-ae8654b92fb2", # NSG ports restricted (VM) - "fc5e4038-4584-4632-8c85-c0448d374b2c", # NSG ports restricted (NSG) - "bb91dfba-c30d-4263-9add-9c2384e659a6", # Remote debug Web App off - "cb510bfd-1cba-4d9f-a1ea-bed557ae0564", # Remote debug Functions off - "f9d614c5-c173-4d56-95a7-b4437057d193", # Remote debug API App off - "1e66c121-a66d-4b99-b523-e2cf4bf16934", # Azure Firewall enabled - "83e0d761-c550-47de-b1b6-359f2a30b354", # Adaptive network hardening - "9dfea752-cf9d-4745-b47b-81b8330bbb9f", # SQL MI public endpoint disabled - "ae89ebf1-3572-4ab1-b1bd-ec5f00bab3a6", # SQL private endpoint - "1c06e275-d63d-4540-b761-71f364c2111d", # KeyVault private endpoint - # Encryption & Data Protection - "7595c971-233d-4bcf-bd18-596129188c49", # SQL TDE - "404c3081-a854-4457-ae30-26a93ef643f9", # Secure transfer storage - "4733ea7b-a883-42fe-8cac-97454c2a9e4a", # Storage network restrict - "a4af4a39-4135-4a60-8bf6-d34b2a1d4f2c", # Storage CMK - "1f905d99-2622-4c13-100b-f7c8e2078bc5", # Cosmos DB CMK - "702dd420-7fcc-42c5-afe8-4026edd20fe0", # Disk CMK - "18adea5e-f416-4d0f-8aa8-d24321e3e274", # PostgreSQL CMK - "0a370ff3-6cab-4e85-8995-295fd854c5b8", # SQL MI CMK - "0961003e-5a0a-4549-abde-af6a37f2724d", # Temp disk encryption - "67121cc7-ff16-4bfd-986d-b15f4c767a1b", # Cognitive Services CMK - "0725b4dd-7e76-479c-a735-68e7ee23d5ca", # Cognitive Services public network - # Logging & Monitoring - "818719e5-1338-4776-9a9d-3c31e4df5986", # Log Analytics agent on VM - "428256e6-1fac-4f48-a757-df34c2b3336d", # Diagnostic settings (no param) - "89099bee-89e0-4b26-a5f4-165451757743", # SQL audit retention 90 days - "b954148f-4c11-4c38-8221-be76711e194e", # SQL MI advanced security - "b0d14bf4-f90c-4e66-9457-1346f80b5a44", # Security contact email - "6e2593d9-add6-4083-9c9b-4b7d2188c899", # Defender SQL email notifications - # Key Management - "8e826246-c976-48f6-b03e-619bb92b3d82", # KV key expiration - "5f0bc445-3935-4915-9981-011aa2b46147", # KV secret expiration - "f4b53539-8df9-40e4-86c6-6b607703bd4e", # Keys backed by HSM - "6a523b34-47f5-4a80-a97e-d3e2d8bca2d6", # KV Managed HSM purge protection - "c39ba22d-4428-4149-b981-98acef4f7277", # KV firewall enabled - # Endpoint Security & Vulnerability - "e96a9a5f-07ca-471b-9bc5-6a0f33cbd68f", # Vuln assessment VMs - "44e1ad92-5f90-4a45-83bb-81cd4695e9f4", # VM vulnerability findings resolved - "1b7aa243-0538-4a73-b824-0b3fc489d80c", # Vuln assessment SQL MI - "013e242c-8828-4970-87b3-ab247555486d", # Endpoint protection - "ac076320-ddcf-4066-b451-6154267e8ad2", # Anti-malware / Endpoint Protection - "86b3d65f-7626-441e-b690-81a8b71cff60", # System updates (classic) - "bd876905-5b84-4f73-ab2d-2e7a7c4568d9", # Vulnerability assessment solution - "9c276cf7-d6e0-4a09-a4b4-be5f06902a79", # VMSS system updates - # Backup & Recovery - "09024ccc-0c5f-475e-9457-b7c0d9ed487b", # Azure Backup VMs - "22bee202-a82f-4305-9a2a-6d7f44d4dedb", # Geo-redundant backup MySQL - # Data Classification & Privacy - "ca610c1d-041c-4332-9d88-7ed3094967c7", # SQL private endpoint connections - "0b60c0b2-2dc2-4e1c-b5c9-abbed971de53", # SQL data classification - # Location / Data Residency - "e56962a6-4747-49cd-b67b-bf8b01975c4c", # Allowed locations - "37bc2e11-1d3c-4e6e-a9fc-62ca7b58b6c2", # Allowed locations for RGs - # Incident Response / Alerts - "8e86a5b6-b9bd-49d1-8e21-4bb8a0862222", # Adaptive application controls -} - -GUID_PATTERN = re.compile( - r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", - re.IGNORECASE, -) - -VALID_MCSB_PREFIXES = { - "NS", "IM", "PA", "DP", "AM", "LT", "IR", "PV", "ES", "BR", "DS", "GS", -} - - -def validate_mappings( - extraction: ControlExtractionResult, - mappings: list[ControlPolicyMapping], - min_confidence: float = 0.5, -) -> ValidationReport: - """ - Validate the complete set of control-to-policy mappings. - - Checks: - - All extracted controls have a mapping - - Policy definition IDs are valid GUIDs - - MCSB control IDs follow expected format - - Confidence scores are reasonable - - No duplicate policy references within a group - - Manual controls have attestation notes - - Args: - extraction: The original extracted controls. - mappings: The policy mappings to validate. - min_confidence: Threshold below which a warning is raised. - - Returns: - ValidationReport with all issues found. - """ - issues: list[ValidationIssue] = [] - total_controls = len(extraction.controls) - extracted_ids = {c.control_id for c in extraction.controls} - mapped_ids = {m.control_id for m in mappings} - - # ── Check: all controls were mapped ─────────────────────────────────── - unmapped = extracted_ids - mapped_ids - for ctrl_id in unmapped: - issues.append(ValidationIssue( - severity="error", - control_id=ctrl_id, - message=f"Control was extracted but not mapped to any Azure Policy", - suggestion="Re-run the mapping pipeline or manually add mapping", - )) - - # ── Per-mapping checks ──────────────────────────────────────────────── - all_policy_ids: set[str] = set() - automatable_count = 0 - manual_count = 0 - confidence_sum = 0.0 - - for mapping in mappings: - control_id = mapping.control_id - - # Check MCSB control ID format - if mapping.mcsb_control_id: - parts = mapping.mcsb_control_id.split("-") - if len(parts) < 2 or parts[0] not in VALID_MCSB_PREFIXES: - issues.append(ValidationIssue( - severity="warning", - control_id=control_id, - message=f"MCSB control ID '{mapping.mcsb_control_id}' has unexpected format", - suggestion=f"Expected format like 'NS-1', 'IM-6', 'DP-3'. Valid prefixes: {', '.join(sorted(VALID_MCSB_PREFIXES))}", - )) - - # Check confidence score - confidence_sum += mapping.confidence_score - if mapping.confidence_score < min_confidence: - issues.append(ValidationIssue( - severity="warning", - control_id=control_id, - message=f"Low confidence mapping ({mapping.confidence_score:.2f} < {min_confidence})", - suggestion="Review this mapping manually before deployment", - )) - - # Check Azure Policy IDs - if mapping.is_automatable: - automatable_count += 1 - if not mapping.azure_policies: - issues.append(ValidationIssue( - severity="error", - control_id=control_id, - message="Control marked as automatable but has no Azure Policy mappings", - suggestion="Add Azure Policy definitions or mark as not automatable", - )) - - for policy in mapping.azure_policies: - pid = policy.policy_definition_id - - # Validate GUID format - if not GUID_PATTERN.match(pid): - issues.append(ValidationIssue( - severity="error", - control_id=control_id, - message=f"Invalid policy GUID format: '{pid}'", - suggestion="Azure Policy definition IDs must be valid UUIDs", - )) - else: - all_policy_ids.add(pid) - - # Check if it's a known GUID - if pid not in KNOWN_POLICY_GUIDS: - issues.append(ValidationIssue( - severity="info", - control_id=control_id, - message=f"Policy GUID '{pid}' is not in the known-good list", - suggestion="Verify this GUID exists in your Azure environment before deployment", - )) - else: - manual_count += 1 - if not mapping.manual_attestation_note: - issues.append(ValidationIssue( - severity="warning", - control_id=control_id, - message="Non-automatable control missing manual attestation note", - suggestion="Add guidance on what manual steps or evidence are needed", - )) - - avg_confidence = confidence_sum / len(mappings) if mappings else 0.0 - - # ── Summary checks ──────────────────────────────────────────────────── - error_count = sum(1 for i in issues if i.severity == "error") - is_valid = error_count == 0 - - if not mappings: - issues.append(ValidationIssue( - severity="error", - control_id="N/A", - message="No controls were mapped", - suggestion="Check that the PDF contains extractable controls", - )) - is_valid = False - - report = ValidationReport( - is_valid=is_valid, - total_controls=total_controls, - automatable_controls=automatable_count, - manual_controls=manual_count, - unique_policies=len(all_policy_ids), - avg_confidence=round(avg_confidence, 3), - issues=issues, - ) - - # Log summary - errors = sum(1 for i in issues if i.severity == "error") - warnings = sum(1 for i in issues if i.severity == "warning") - infos = sum(1 for i in issues if i.severity == "info") - - logger.info( - f"Validation {'PASSED' if is_valid else 'FAILED'}: " - f"{total_controls} controls, {automatable_count} automatable, " - f"{len(all_policy_ids)} unique policies, " - f"avg confidence {avg_confidence:.2f}, " - f"{errors} errors, {warnings} warnings, {infos} info" - ) - - return report diff --git a/docs/BACKLOG.md b/docs/BACKLOG.md index e6f18d7..4f57daa 100644 --- a/docs/BACKLOG.md +++ b/docs/BACKLOG.md @@ -144,16 +144,26 @@ refresh can no longer do either. --- -## B6 — `compliance-pipeline/` carries a second, larger hardcoded GUID menu +## B6 — `compliance-pipeline/` carries a second, larger hardcoded GUID menu — FIXED **Observed.** A fourth mapping stack with its own `policy_mapper.py` and a 64-GUID menu, 32 of them overlapping the backend's 34. The backend pipeline is a fork of it. **Why it matters.** Fixing B1 leaves this copy wrong, so the defect can be reintroduced by anyone -who follows the older code. - -**Done looks like.** Either it is deleted, or it delegates to the same engine. A decision, not an -oversight. +who follows the older code. Hardcoding a policy menu is exactly the failure mode the AI mapping +engine exists to avoid — a curated shortlist caps recall at whatever the list's author thought of, +against a catalogue of 2,467 shipped definitions. + +**Fix.** Deleted the entire `compliance-pipeline/` directory (`pipeline.py`, `policy_mapper.py`, +`control_extractor.py`, `pdf_extractor.py`, `initiative_builder.py`, `validator.py`, `models.py`, +`config.py`, its own `requirements.txt` and `README.md`) rather than reworking it to delegate to +`AIMappingService`. It was a fully standalone CLI tool — confirmed nothing in `app/` (backend, +frontend, or tests) imported or referenced it — so making it delegate would mean building and +maintaining a second integration surface into the shared mapping engine for a tool nothing else +depends on. Removed its two remaining references in `README.md` and `docs/FUNCTIONAL_SPEC.md`. + +**Status.** Fixed on `main`. There is exactly one mapping engine left in the repository +(`AIMappingService`), reached by both the backend services path and the pipeline path (B1). --- diff --git a/docs/FUNCTIONAL_SPEC.md b/docs/FUNCTIONAL_SPEC.md index 595bd47..8968781 100644 --- a/docs/FUNCTIONAL_SPEC.md +++ b/docs/FUNCTIONAL_SPEC.md @@ -200,5 +200,3 @@ labelled B. * ISO 27018, SOC 1 and SOC 3 have no Azure clause metadata, so citations against them can never reach `GROUNDED`. * `comparison.py` `_run_build_job` has no cancellation check. -* `compliance-pipeline/` retains its own 64-identifier fork of the mapper. Out of scope, - recorded as a decision rather than an oversight.