Map controls directly to Azure Policy instead of a 10-control MCSB stub

.github/skills/complianceiq/reference/api.md

@@ -32,7 +32,7 @@ The deployed OpenAPI document reports **94 operations across 89 paths**
 
 ## Granular mapping (`/mapping`) — for advanced flows
 `POST /mapping/analyze` (job) · `POST /mapping/map-batch` · `POST /mapping/map-single`
-· `GET /mapping/status/{job_id}` · `GET /mapping/mcsb/{controls,domains}`.
+· `GET /mapping/status/{job_id}`.
 
 ## Policy generation (`/policy`) — **authenticated**
 `POST /policy/generate` (+ `/generate/{json,bicep,scripts,slz}`) ·

README.md

@@ -64,7 +64,6 @@ compliance-iq/
 │   ├── Oman_Government_Azure_Mappings.csv
 │   └── CATALOG_SUMMARY.md
 │
-├── compliance-pipeline/       # Standalone CLI tool for batch processing
 ├── framework/                 # Azure Policy initiative JSON & deployment scripts
 ├── reference_documents/       # Source compliance framework PDFs
 ├── templates/                 # Simplified control templates for gap analysis

app/README.md

@@ -183,8 +183,7 @@ app/
 │   │   │   ├── policy.py      # Policy initiative models
 │   │   │   └── sovereignty.py # SLZ / sovereignty models (NEW)
 │   │   ├── services/          # Business logic
-│   │   │   ├── ai_mapping_service.py   # AI mapping (MCSB + SLZ)
-│   │   │   ├── mcsb_service.py         # MCSB control loader
+│   │   │   ├── ai_mapping_service.py   # AI mapping (direct to Azure Policy + SLZ)
 │   │   │   ├── microsoft_learn_client.py # MS Learn policy search
 │   │   │   ├── policy_service.py       # Policy gen (+ SLZ initiatives)
 │   │   │   └── sovereignty_service.py  # SLZ data service (NEW)
@@ -208,13 +207,12 @@ app/
 │   │   ├── 1_📁_Upload_Controls.py
 │   │   ├── 2_🤖_AI_Mapping.py    # Shows SLZ level badges
 │   │   ├── 3_✏️_Review_Edit.py    # Sovereignty filter + panels
-│   │   └── 4_📦_Export_Policy.py  # MCSB + SLZ export tabs
+│   │   └── 4_📦_Export_Policy.py  # Azure Policy + SLZ export tabs
 │   ├── utils/                 # API client (+ SLZ methods)
 │   ├── app.py                 # Main app (SLZ status in sidebar)
 │   └── requirements.txt
 │
 ├── data/                      # Reference data
-│   ├── mcsb/                  # MCSB controls
 │   └── examples/              # Sample files
 │
 ├── tests/                     # Test suite
@@ -277,7 +275,6 @@ SAMA-AC-01,Strong Authentication,Enforce MFA and disable legacy protocols
 ```json
 {
   "external_control_id": "SAMA-AC-01",
-  "mcsb_control_id": "IM-6",
   "confidence_score": 0.92,
   "reasoning": "Both controls focus on enforcing MFA...",
   "azure_policy_ids": ["4e6c27d5-a6ee-49cf-b2b4-d8fe90fa2b8b"]

app/backend/README.md

@@ -91,8 +91,6 @@ Once the server is running:
 - `POST /api/v1/mapping/map-single` - Map single control
 - `POST /api/v1/mapping/analyze` - Batch mapping (async)
 - `GET /api/v1/mapping/status/{job_id}` - Job status
-- `GET /api/v1/mapping/mcsb/controls` - Get MCSB controls
-- `GET /api/v1/mapping/mcsb/domains` - Get MCSB domains
 
 ### Policy Generation
 
@@ -114,9 +112,7 @@ Expected output:
 {
   "status": "healthy",
   "version": "1.0.0",
-  "azure_openai_connected": true,
-  "mcsb_controls_loaded": true,
-  "mcsb_control_count": 10
+  "azure_openai_connected": true
 }
 ```
 
@@ -135,12 +131,6 @@ curl -X POST http://localhost:8000/api/v1/mapping/map-single \
   }' | jq
 ```
 
-### Test MCSB Controls Retrieval
-
-```bash
-curl http://localhost:8000/api/v1/mapping/mcsb/controls | jq
-```
-
 ## 🚀 Azure Container Apps Deployment Notes
 
 - Container Apps pull images from ACR using the system-assigned managed identity (AcrPull granted in Bicep). No ACR admin credentials are needed at runtime.
@@ -162,7 +152,6 @@ backend/
 │   │   ├── mapping.py
 │   │   └── policy.py
 │   ├── services/            # Business logic
-│   │   ├── mcsb_service.py
 │   │   ├── ai_mapping_service.py
 │   │   └── policy_service.py
 │   ├── api/routes/          # API endpoints
@@ -232,14 +221,6 @@ from app.auth import test_azure_openai_connection
 test_azure_openai_connection()  # Should return True
 ```
 
-### MCSB Controls Not Loading
-
-The service includes 10 default MCSB controls for demonstration. For full MCSB catalog:
-
-1. Download from GitHub SecurityBenchmarks
-2. Place JSON at `../data/mcsb/mcsb_v1_controls.json`
-3. Restart server
-
 ## 📝 Development
 
 ### Run with Auto-reload

app/backend/app/api/routes/health.py

@@ -8,7 +8,7 @@
 
 from app import __version__
 from app.auth import test_azure_openai_connection
-from app.services import get_mcsb_service, get_sovereignty_service, get_policy_catalog_service
+from app.services import get_sovereignty_service, get_policy_catalog_service
 
 logger = logging.getLogger(__name__)
 router = APIRouter(tags=["health"])
@@ -19,9 +19,6 @@ class HealthResponse(BaseModel):
     status: str
     version: str
     azure_openai_connected: bool
-    mcsb_controls_loaded: bool
-    mcsb_control_count: int
-    mcsb_is_demonstration_data: bool = False
     slz_policies_loaded: bool = False
     slz_policy_count: int = 0
     policy_catalog_count: int = 0
@@ -46,19 +43,6 @@ async def health_check():
         logger.error(f"Azure OpenAI health check failed: {e}")
         azure_openai_connected = False
 
-    # Check MCSB service
-    try:
-        mcsb_service = get_mcsb_service()
-        controls = mcsb_service.get_all_controls()
-        mcsb_controls_loaded = True
-        mcsb_control_count = len(controls)
-        mcsb_is_demonstration_data = mcsb_service.is_demonstration_data
-    except Exception as e:
-        logger.error(f"MCSB service health check failed: {e}")
-        mcsb_controls_loaded = False
-        mcsb_control_count = 0
-        mcsb_is_demonstration_data = False
-
     # Check SLZ sovereignty service
     try:
         slz_service = get_sovereignty_service()
@@ -81,12 +65,9 @@ async def health_check():
         policy_catalog_source = "error"
 
     return HealthResponse(
-        status="healthy" if (azure_openai_connected and mcsb_controls_loaded) else "degraded",
+        status="healthy" if azure_openai_connected else "degraded",
         version=__version__,
         azure_openai_connected=azure_openai_connected,
-        mcsb_controls_loaded=mcsb_controls_loaded,
-        mcsb_control_count=mcsb_control_count,
-        mcsb_is_demonstration_data=mcsb_is_demonstration_data,
         slz_policies_loaded=slz_policies_loaded,
         slz_policy_count=slz_policy_count,
         policy_catalog_count=policy_catalog_count,

app/backend/app/api/routes/mapping.py

@@ -18,7 +18,7 @@
     MappingJob,
 )
 from app.models.mapping import record_mapping_activity
-from app.services import get_ai_mapping_service, get_mcsb_service
+from app.services import get_ai_mapping_service
 from app.db import cosmos_client
 from app.config import get_settings
 
@@ -268,58 +268,6 @@ async def get_mapping_status(job_id: str):
     return job
 
 
-@router.get("/mcsb/controls")
-async def get_mcsb_controls(domain: Optional[str] = None):
-    """
-    Get MCSB controls, optionally filtered by domain.
-
-    Args:
-        domain: Optional domain filter
-
-    Returns:
-        List of MCSB controls. ``is_demonstration_data`` is True whenever no
-        real MCSB catalog file is loaded, in which case the returned
-        ``defender_recommendations`` are illustrative examples, not verified
-        against a live Microsoft Defender for Cloud subscription (see
-        docs/BACKLOG.md B4).
-    """
-    try:
-        mcsb_service = get_mcsb_service()
-
-        if domain:
-            controls = mcsb_service.get_controls_by_domain(domain)
-        else:
-            controls = mcsb_service.get_all_controls()
-
-        return {
-            "controls": controls,
-            "count": len(controls),
-            "is_demonstration_data": mcsb_service.is_demonstration_data,
-        }
-
-    except Exception as e:
-        logger.error(f"Failed to get MCSB controls: {e}")
-        raise HTTPException(status_code=500, detail=str(e))
-
-
-@router.get("/mcsb/domains")
-async def get_mcsb_domains():
-    """Get all MCSB security domains."""
-    try:
-        mcsb_service = get_mcsb_service()
-        domains = mcsb_service.get_all_domains()
-
-        return {
-            "domains": domains,
-            "count": len(domains),
-            "is_demonstration_data": mcsb_service.is_demonstration_data,
-        }
-
-    except Exception as e:
-        logger.error(f"Failed to get MCSB domains: {e}")
-        raise HTTPException(status_code=500, detail=str(e))
-
-
 async def process_mapping_job(
     job_id: str,
     controls: List[ExternalControl],

app/backend/app/models/__init__.py

@@ -2,7 +2,7 @@
 Pydantic models for the AI Control Mapping Agent.
 """
 
-from app.models.control import ExternalControl, MCSBControl, FrameworkUpload
+from app.models.control import ExternalControl, FrameworkUpload
 from app.models.mapping import (
     ControlMapping,
     MappingBatch,
@@ -60,7 +60,6 @@
 __all__ = [
     # Control models
     "ExternalControl",
-    "MCSBControl",
     "FrameworkUpload",
     # Mapping models
     "ControlMapping",

app/backend/app/models/control.py

@@ -28,47 +28,6 @@ class ExternalControl(BaseModel):
     })
 
 
-class MCSBControl(BaseModel):
-    """Model for Microsoft Cloud Security Benchmark control."""
-
-    control_id: str = Field(..., description="MCSB control ID (e.g., IM-1)")
-    domain: str = Field(..., description="Security domain")
-    control_name: str = Field(..., description="Control title")
-    description: str = Field(..., description="Full control description")
-    azure_policy_ids: List[str] = Field(
-        default_factory=list,
-        description="Associated Azure Policy definition GUIDs"
-    )
-    defender_recommendations: List[str] = Field(
-        default_factory=list,
-        description=(
-            "Illustrative Microsoft Defender for Cloud recommendation names, "
-            "not verified against a live Defender for Cloud subscription. "
-            "Populated only when the deployment serves demonstration data "
-            "(see MCSBService.is_demonstration_data / docs/BACKLOG.md B4)."
-        ),
-    )
-    related_frameworks: Dict[str, List[str]] = Field(
-        default_factory=dict,
-        description="Mappings to other frameworks (CIS, NIST, etc.)"
-    )
-
-    model_config = ConfigDict(json_schema_extra={
-        "example": {
-            "control_id": "IM-1",
-            "domain": "Identity Management",
-            "control_name": "Use centralized identity and authentication system",
-            "description": "Use a centralized identity and authentication system...",
-            "azure_policy_ids": ["4e6c27d5-a6ee-49cf-b2b4-d8fe90fa2b8b"],
-            "defender_recommendations": ["Enable MFA for all users"],
-            "related_frameworks": {
-                "CIS": ["CIS-5.1"],
-                "NIST": ["IA-2"]
-            }
-        }
-    })
-
-
 class FrameworkUpload(BaseModel):
     """Model for uploaded framework control data."""
 

app/backend/app/models/mapping.py

@@ -83,15 +83,18 @@ class ControlMapping(BaseModel):
     external_control_id: str = Field(..., description="External framework control ID")
     external_control_name: str = Field(..., description="External control name")
 
-    mcsb_control_id: str = Field(..., description="Mapped MCSB control ID")
-    mcsb_control_name: str = Field(..., description="Mapped MCSB control name")
-    mcsb_domain: str = Field(..., description="MCSB security domain")
-
     confidence_score: float = Field(
         ...,
         ge=0.0,
         le=1.0,
-        description="Confidence score for this mapping (0.0 to 1.0)"
+        description=(
+            "Confidence that the selected azure_policy_ids (or, for non-"
+            "enforceable controls, the coverage classification) genuinely "
+            "match this control's literal text - scored against the actual "
+            "Azure Policy definitions retrieved for it, not against any "
+            "intermediate control taxonomy. See SYSTEM_PROMPT for the "
+            "worked calibration examples this rubric is grounded in."
+        ),
     )
 
     reasoning: str = Field(..., description="Explanation for why this mapping was chosen")
@@ -106,6 +109,19 @@ class ControlMapping(BaseModel):
         description="Type of mapping relationship"
     )
 
+    policy_category: Optional[str] = Field(
+        default=None,
+        description=(
+            "Grouping label derived from the catalog `category` of the "
+            "selected azure_policy_ids (e.g. 'Key Vault', 'Storage', "
+            "'Network'), or the external control's own domain when no policy "
+            "was selected. Always server-computed after the model responds - "
+            "never model-authored - because it is a resolvable fact about the "
+            "real catalog entries chosen, not a judgement call. Replaces the "
+            "old mcsb_domain fallback."
+        ),
+    )
+
     control_type: Optional[str] = Field(
         default=None,
         description=(
@@ -312,13 +328,11 @@ class ControlMapping(BaseModel):
         "example": {
             "external_control_id": "SAMA-AC-01",
             "external_control_name": "Strong Authentication",
-            "mcsb_control_id": "IM-6",
-            "mcsb_control_name": "Use strong authentication controls",
-            "mcsb_domain": "Identity Management",
             "confidence_score": 0.92,
             "reasoning": "Both controls focus on enforcing MFA and strong authentication mechanisms",
             "azure_policy_ids": ["4e6c27d5-a6ee-49cf-b2b4-d8fe90fa2b8b"],
             "mapping_type": "exact",
+            "policy_category": "Identity",
             "control_type": "Technical",
             "coverage_category": "A_AzurePolicy",
             "azure_enforceable": True,

app/backend/app/models/policy.py

@@ -291,7 +291,14 @@ class ManualControlEntry(BaseModel):
             "'Microsoft attested'"
         ),
     )
-    mcsb_control_id: str = Field("", description="Associated MCSB control ID, if any")
+    policy_category: str = Field(
+        "",
+        description=(
+            "Grouping label derived from the catalog category of any mapped "
+            "policies, or the control's own domain. Replaces the old "
+            "mcsb_control_id field."
+        ),
+    )
     responsibility: str = Field(
         "",
         description=(

app/backend/app/pipeline/initiative_builder.py

@@ -622,8 +622,7 @@ def _write_mappings_csv(
         "Coverage_Gap",
         "Outside_Step",
         "Enforcement_Plane",
-        "MCSB_Control_ID",
-        "MCSB_Control_Name",
+        "Policy_Category",
         "Confidence",
         "Azure_Policy_IDs",
         "Azure_Policy_Names",
@@ -683,8 +682,7 @@ def _joined(values) -> str:
                 "Coverage_Gap": str(getattr(m, "coverage_gap", False)) if m else "",
                 "Outside_Step": (getattr(m, "outside_step", None) or "") if m else "",
                 "Enforcement_Plane": (getattr(m, "enforcement_plane", None) or "") if m else "",
-                "MCSB_Control_ID": m.mcsb_control_id if m else "",
-                "MCSB_Control_Name": m.mcsb_control_name if m else "",
+                "Policy_Category": (getattr(m, "policy_category", None) or "") if m else "",
                 "Confidence": f"{m.confidence_score:.2f}" if m else "",
                 "Azure_Policy_IDs": policy_ids,
                 "Azure_Policy_Names": policy_names,
@@ -786,7 +784,7 @@ def _write_coverage_reports(
                 "control_type",
                 "coverage_category",
                 "coverage_display",
-                "mcsb_control_id",
+                "policy_category",
                 "responsibility",
                 "evidence_source",
                 "enforcement_plane",