From 0f014aae7df2e0d06ad8b7f45df5512b4b726572 Mon Sep 17 00:00:00 2001 From: Jeff F Date: Sat, 28 Mar 2026 23:25:48 -0500 Subject: [PATCH] Release v1.3.1 --- CHANGELOG.md | 26 +- eval/eval_e2e.py | 61 +- eval/eval_harness.py | 1164 ++++++++++++++++++++++++++-- eval/retrieval_extensions.py | 539 +++++++++++++ eval/scorer.py | 369 ++++++++- src/confluence_writer.py | 10 +- src/crm_system.py | 11 +- src/day_planner.py | 167 ++-- src/email_gen.py | 779 ------------------- src/embed_worker.py | 174 +++-- src/external_email_ingest.py | 893 +++++++++++++++++++-- src/flow.py | 95 ++- src/genesis.py | 136 +++- src/graph_dynamics.py | 249 ++++++ src/memory.py | 106 ++- src/normal_day.py | 164 +++- src/org_lifecycle.py | 38 +- src/planner_models.py | 10 +- src/post_sim_artifacts.py | 21 +- src/utils/persona_utils.py | 478 ++++++++---- tests/conftest.py | 1 - tests/test_causal_chain_handler.py | 2 +- tests/test_crm_system.py | 1 - tests/test_flow.py | 7 +- tests/test_memory.py | 35 - tests/test_routing.py | 5 +- 26 files changed, 4138 insertions(+), 1403 deletions(-) create mode 100644 eval/retrieval_extensions.py delete mode 100644 src/email_gen.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f0c5a4c..a0a34d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,31 @@ Versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). --- -## [v1.2.8] — 2026-03-27 +## [v1.3.1] — 2026-03-28 + +### Added + +- **Cross-Domain Evaluation Suite (`eval/`, `src/post_sim_artifacts.py`)**: introduced 11 new evaluation question types including **ZD_RESOLUTION**, **SF_RISK**, and **INVOICE_SLA**. The framework now includes specialized scorers for financial credits, NPS drivers, and PR review verdicts to validate RAG performance across the entire business logic stack. +- **Reactive Customer Reply Loop (`src/external_email_ingest.py`)**: implemented a probabilistic customer response system. The simulation now classifies inbound emails (Complaints vs. Feature Requests) and allows customers to autonomously advance **Salesforce Opportunity stages** based on the quality of outbound sales replies. +- **Concurrent Embedding Engine (`src/embed_worker.py`)**: replaced the serial background worker with a `ThreadPoolExecutor`-based system. This supports high-throughput embedding via **Infinity** (up to 16 concurrent calls) while maintaining causal consistency through a new `drain()` synchronization mechanism. +- **Advanced Retrieval Architectures (`eval/eval_e2e.py`)**: added support for **Reciprocal Rank Fusion (RRF)** and **Graph-Augmented Retrievers**, allowing the evaluation agent to fuse BM25 lexical search with dense vector embeddings and 2-hop artifact expansion. + +### Changed + +- **Stateful Persona Architecture (`src/utils/persona_utils.py`)**: refactored persona generation into a centralized `PersonaUtils` singleton. Voice cards now inject **"CRM Pressure" hints**, causing internal employees to exhibit higher stress and terser communication when they own at-risk deals or urgent Zendesk tickets. +- **CRM-Driven Graph Dynamics (`src/graph_dynamics.py`, `src/flow.py`)**: integrated live Salesforce/Zendesk telemetry into the social graph. Edge weights between liaisons and external contacts now fluctuate based on account sentiment, and stress propagates through the organization when "lighthouse" customers flag risks. +- **Theme-Triggered Incidents (`src/flow.py`)**: updated the incident generator to be reactive to the daily planning theme. Stability "collisions" are now more likely to fire if the daily plan involves high-risk activities like "migrations" or "refactors" identified via keyword triggers. +- **HyDE Query Rewriting (`src/memory.py`)**: migrated the `recall_with_rewrite` logic from a generic callable to a native **Bedrock-hosted Llama 3.3** implementation for more robust hypothetical document generation. + +### Fixed + +- **PR Review Approval Logic (`src/normal_day.py`)**: fixed a bug where engineers would requested changes indefinitely by implementing "Review Round" guidance, forcing approvals after 3 rounds unless critical bugs are present. +- **Causal Chain Integrity (`src/external_email_ingest.py`)**: corrected an issue where inbound vendor emails were orphaned; all external communications are now properly rooted in a `CausalChainHandler` and tracked through downstream JIRA/Slack artifacts. +- **Cleanup**: deleted the legacy `src/email_gen.py` script in favor of the live, event-driven ingestor system. + +--- + +## [v1.3.0] — 2026-03-27 ### Added diff --git a/eval/eval_e2e.py b/eval/eval_e2e.py index a18f1f8..cb0c268 100644 --- a/eval/eval_e2e.py +++ b/eval/eval_e2e.py @@ -61,6 +61,14 @@ logger = logging.getLogger("orgforge.eval_e2e") +# Lazy import so the module is optional when only using built-in retrievers. +try: + from retrieval_extensions import RRFRetriever, GraphAugmentedRetriever + + _EXTENSIONS_AVAILABLE = True +except ImportError: + _EXTENSIONS_AVAILABLE = False + # ── Constants ───────────────────────────────────────────────────────────────── HF_DATASET_ID = os.environ.get("HF_DATASET_ID", "INSERT_ID_HERE") @@ -354,6 +362,7 @@ def retrieve(self, query: str, top_k: int = TOP_K) -> List[str]: def build_retriever(name: str, region: str = "us-east-1") -> Retriever: + # ── original retrievers ──────────────────────────────────────────────────── if name == "bm25": return BM25Retriever() if name == "cohere": @@ -362,8 +371,36 @@ def build_retriever(name: str, region: str = "us-east-1") -> Retriever: return BedrockCohereRetriever(region=region) if name == "openai": return OpenAIRetriever() + + # ── RRF and Graph retrievers (require retrieval_extensions.py) ───────────── + if not _EXTENSIONS_AVAILABLE: + raise SystemExit( + f"retriever={name!r} requires retrieval_extensions.py in the same " + "directory. Make sure that file is present and importable." + ) + + # RRF: fuse BM25 + dense retriever + if name == "rrf": + return RRFRetriever([BM25Retriever(), CohereRetriever()]) + if name == "rrf-openai": + return RRFRetriever([BM25Retriever(), OpenAIRetriever()]) + if name == "rrf-bedrock": + return RRFRetriever([BM25Retriever(), BedrockCohereRetriever(region=region)]) + + # Graph-Augmented: expand any base retriever along the artifact graph + if name == "graph-bm25": + return GraphAugmentedRetriever(BM25Retriever()) + if name == "graph-cohere": + return GraphAugmentedRetriever(CohereRetriever()) + if name == "graph-rrf": + base = RRFRetriever([BM25Retriever(), CohereRetriever()]) + return GraphAugmentedRetriever(base) + raise ValueError( - f"Unknown retriever: {name!r}. Choose bm25 | cohere | cohere-bedrock | openai" + f"Unknown retriever: {name!r}. " + "Choose bm25 | cohere | cohere-bedrock | openai | " + "rrf | rrf-openai | rrf-bedrock | " + "graph-bm25 | graph-cohere | graph-rrf" ) @@ -1163,9 +1200,27 @@ def _parse_args() -> argparse.Namespace: ) p.add_argument( "--retriever", - choices=["bm25", "cohere", "cohere-bedrock", "openai"], + choices=[ + "bm25", + "cohere", + "cohere-bedrock", + "openai", + # Reciprocal Rank Fusion + "rrf", + "rrf-openai", + "rrf-bedrock", + # Graph-Augmented (1-2 hop artifact expansion) + "graph-bm25", + "graph-cohere", + "graph-rrf", + ], default="bm25", - help="Retriever to use (default: bm25). cohere-bedrock uses Bedrock credentials.", + help=( + "Retriever to use (default: bm25).\n" + " bm25 / cohere / cohere-bedrock / openai — single retrievers\n" + " rrf / rrf-openai / rrf-bedrock — BM25 + dense fusion (RRF)\n" + " graph-bm25 / graph-cohere / graph-rrf — graph-augmented expansion" + ), ) p.add_argument( "--generator", diff --git a/eval/eval_harness.py b/eval/eval_harness.py index 9ec4ae3..620db39 100644 --- a/eval/eval_harness.py +++ b/eval/eval_harness.py @@ -56,6 +56,37 @@ Answer: first_handler + downstream artifacts from customer_escalation or customer_email_routed SimEvent + ZD_RESOLUTION "Was Zendesk ticket X resolved and how long did it take?" + Answer: resolved boolean + duration_days from zd_ticket_opened / + zd_tickets_resolved SimEvents + + ZD_ESCALATION "Which Zendesk tickets escalated to incident X?" + Answer: ticket_ids list from zd_tickets_escalated SimEvent + (uses CAUSAL scorer — incident + ticket chain) + + SF_RISK "Which Salesforce accounts were flagged at-risk after incident X?" + Answer: at_risk_accounts list from sf_deals_risk_flagged SimEvent + + SF_TOUCHPOINT "What opportunity was advanced by the email from X to Y?" + Answer: opportunity_id + stage from crm_touchpoint SimEvent + (uses CAUSAL scorer — email → opportunity chain) + + SF_OWNERSHIP "Which accounts lost their owner when X departed?" + Answer: lapsed_accounts + lapsed_opportunities from + sf_ownership_lapsed SimEvent (uses RETRIEVAL scorer) + + DATADOG_ALERT "Which Datadog alert fired for incident X?" + Answer: incident_id inferred from incident_opened SimEvent + (uses RETRIEVAL scorer — alert → incident link) + + NPS_SCORE "What NPS score did customer X give and what drove it?" + Answer: nps_score + classification derived deterministically + from ZD ticket and incident SimEvents (mirrors NPSWriter formula) + + INVOICE_SLA "What SLA credit appeared on customer X's invoice?" + Answer: breach_duration_days + sla_credit_per_org derived from + incident duration × SLA_CREDIT_RATE (mirrors InvoiceWriter logic) + Usage for RAG eval ------------------ Each question in eval_questions.json has: @@ -88,7 +119,7 @@ logger = logging.getLogger("orgforge.eval") -# ── Config ──────────────────────────────────────────────────────────────────── + with open(Path(__file__).resolve().parent.parent / "config" / "config.yaml") as f: _CFG = yaml.safe_load(f) @@ -97,11 +128,6 @@ EVAL_DIR.mkdir(parents=True, exist_ok=True) -# ───────────────────────────────────────────────────────────────────────────── -# CAUSAL THREAD BUILDER -# ───────────────────────────────────────────────────────────────────────────── - - class CausalThreadBuilder: """ Reconstructs explicit artifact graphs from the SimEvent log. @@ -120,6 +146,9 @@ def build_all(self) -> List[dict]: threads.extend(self._hr_threads()) threads.extend(self._dropped_email_threads()) threads.extend(self._postmortem_threads()) + threads.extend(self._design_doc_threads()) + threads.extend(self._zendesk_threads()) + threads.extend(self._salesforce_threads()) return threads def _incident_threads(self) -> List[dict]: @@ -269,7 +298,6 @@ def _postmortem_threads(self) -> List[dict]: if not ticket_id or not conf_id: continue - # The causal chain is: incident ticket → postmortem confluence page chain = event.facts.get("causal_chain", [ticket_id, conf_id]) nodes = self._build_nodes(chain, event) threads.append( @@ -282,13 +310,169 @@ def _postmortem_threads(self) -> List[dict]: "day": event.day, "date": event.date, "nodes": nodes, - "complete": True, # postmortem_created means it was written + "complete": True, "confluence_id": conf_id, "root_cause": event.facts.get("root_cause", ""), } ) return threads + def _zendesk_threads(self) -> List[dict]: + """ + Builds Zendesk ticket lifecycle chains: + zd_ticket_opened → (zd_tickets_escalated) → zd_tickets_resolved + + Each thread records whether the ticket was escalated to an incident, + enabling CAUSAL questions like 'Which ZD tickets escalated to ORG-42?' + and ZD_RESOLUTION questions like 'Was ZD-501 resolved, and how quickly?' + """ + threads = [] + opened_events: Dict[str, SimEvent] = {} + for e in self._events: + if e.type == "zd_ticket_opened": + tid = e.facts.get("ticket_id", "") + if tid: + opened_events[tid] = e + + escalated_tickets: Dict[str, str] = {} + resolved_days: Dict[str, int] = {} + for e in self._events: + if e.type == "zd_tickets_escalated": + iid = e.facts.get("incident_id", "") + for tid in e.facts.get("ticket_ids", []): + escalated_tickets[tid] = iid + elif e.type == "zd_tickets_resolved": + for tid in e.facts.get("ticket_ids", []): + resolved_days[tid] = e.day + + for tid, open_event in opened_events.items(): + incident_id = escalated_tickets.get(tid) + resolve_day = resolved_days.get(tid) + org = open_event.facts.get("org_name", "Unknown") + + chain = [tid] + if incident_id: + chain.append(incident_id) + + nodes = self._build_nodes(chain, open_event) + + threads.append( + { + "chain_id": f"zd_{tid}", + "chain_type": "zendesk_ticket", + "root_artifact": tid, + "root_event_type": "zd_ticket_opened", + "day": open_event.day, + "date": open_event.date, + "nodes": nodes, + "terminal_artifact": chain[-1], + "complete": resolve_day is not None, + "escalated": bool(incident_id), + "incident_id": incident_id, + "org_name": org, + "subject": open_event.facts.get("subject", ""), + "open_day": open_event.day, + "resolve_day": resolve_day, + "duration_days": (resolve_day - open_event.day) + if resolve_day + else None, + } + ) + return threads + + def _salesforce_threads(self) -> List[dict]: + """ + Builds Salesforce causal chains across three event types: + crm_touchpoint — a sales email created or advanced an opportunity + sf_deals_risk_flagged — an incident caused accounts to be flagged at-risk + sf_ownership_lapsed — an employee departure orphaned accounts/opps + + Each thread type answers a different class of eval question: + crm_touchpoint → CAUSAL: 'What opportunity was created from this email?' + sf_deals_risk_flagged → SF_RISK: 'Which accounts were at risk after incident X?' + sf_ownership_lapsed → RETRIEVAL: 'Which accounts lost their owner when X left?' + """ + threads = [] + + for e in self._events: + if e.type != "crm_touchpoint": + continue + opp_id = e.artifact_ids.get("sf_opp", "") + if not opp_id: + continue + threads.append( + { + "chain_id": f"sf_touchpoint_{opp_id}", + "chain_type": "sf_touchpoint", + "root_artifact": opp_id, + "root_event_type": "crm_touchpoint", + "day": e.day, + "date": e.date, + "nodes": self._build_nodes([opp_id], e), + "terminal_artifact": opp_id, + "complete": True, + "account_name": e.facts.get("account_name", ""), + "stage": e.facts.get("stage", ""), + "sender": e.facts.get("sender", ""), + "subject": e.facts.get("subject", ""), + } + ) + + for e in self._events: + if e.type != "sf_deals_risk_flagged": + continue + incident_id = e.facts.get("incident_id", "") + account_names = e.facts.get("account_names", []) + if not incident_id or not account_names: + continue + threads.append( + { + "chain_id": f"sf_risk_{incident_id}", + "chain_type": "sf_risk", + "root_artifact": incident_id, + "root_event_type": "sf_deals_risk_flagged", + "day": e.day, + "date": e.date, + "nodes": self._build_nodes([incident_id], e), + "terminal_artifact": incident_id, + "complete": True, + "incident_id": incident_id, + "at_risk_accounts": account_names, + "account_count": len(account_names), + } + ) + + for e in self._events: + if e.type != "sf_ownership_lapsed": + continue + departed = e.facts.get("departed_employee", "") + accs = e.facts.get("accounts_lapsed", []) + opps = e.facts.get("opportunities_lapsed", []) + if not departed: + continue + # Chain root is the departed-employee event anchor; artifacts are + # the lapsed account/opp IDs stored as lists in artifact_ids. + chain = accs[:1] or opps[:1] or [f"lapsed_{departed}"] + threads.append( + { + "chain_id": f"sf_lapse_{departed.lower().replace(' ', '_')}", + "chain_type": "sf_ownership_lapse", + "root_artifact": chain[0], + "root_event_type": "sf_ownership_lapsed", + "day": e.day, + "date": e.date, + "nodes": self._build_nodes(chain, e), + "terminal_artifact": chain[-1], + "complete": True, + "departed_employee": departed, + "role": e.facts.get("role", ""), + "lapsed_accounts": accs, + "lapsed_opportunities": opps, + } + ) + + return threads + def _design_doc_threads(self) -> List[dict]: threads = [] conf_events = [ @@ -331,7 +515,6 @@ def _build_nodes(self, chain: List[str], root_event: SimEvent) -> List[dict]: cumulative_known: Set[str] = set() for i, artifact_id in enumerate(chain): - # Find the SimEvent that created this artifact event = self._find_event_for_artifact(artifact_id) if event: cumulative_known.update(event.actors) @@ -347,7 +530,6 @@ def _build_nodes(self, chain: List[str], root_event: SimEvent) -> List[dict]: } ) else: - # Artifact exists but no event found — still include it nodes.append( { "artifact_id": artifact_id, @@ -366,15 +548,10 @@ def _find_event_for_artifact(self, artifact_id: str) -> Optional[SimEvent]: if artifact_id in event.artifact_ids.values(): return event if artifact_id in event.facts.get("causal_chain", []): - continue # don't match chain references, only direct artifact_ids + continue return None -# ───────────────────────────────────────────────────────────────────────────── -# QUESTION GENERATOR -# ───────────────────────────────────────────────────────────────────────────── - - class EvalQuestionGenerator: """ Generates typed eval questions from causal threads and SimEvents. @@ -405,10 +582,20 @@ def generate(self, threads: List[dict]) -> List[dict]: questions.extend(self._confluence_questions()) questions.extend(self._standup_questions()) questions.extend(self._customer_escalation_questions()) + questions.extend(self._zendesk_resolution_questions(threads)) + questions.extend(self._zd_escalation_questions(threads)) + questions.extend(self._sf_risk_questions(threads)) + questions.extend(self._sf_ownership_lapse_questions(threads)) + questions.extend(self._sf_touchpoint_questions(threads)) + questions.extend(self._datadog_alert_questions()) + questions.extend(self._nps_score_questions()) + questions.extend(self._invoice_sla_questions()) + questions.extend(self._pr_review_questions(threads)) + questions.extend(self._blocker_questions()) + questions.extend(self._vendor_routing_questions()) + questions.extend(self._design_discussion_questions()) return questions - # ── RETRIEVAL — "Which artifact first mentioned X?" ─────────────────────── - def _retrieval_questions(self, threads: List[dict]) -> List[dict]: questions = [] incident_threads = [t for t in threads if t["chain_type"] == "incident"] @@ -423,7 +610,6 @@ def _retrieval_questions(self, threads: List[dict]) -> List[dict]: if not root_cause: continue - # Deterministic answer: the root artifact ID ground_truth = { "artifact_id": thread["root_artifact"], "artifact_type": "jira", @@ -455,8 +641,6 @@ def _retrieval_questions(self, threads: List[dict]) -> List[dict]: ) return questions - # ── CAUSAL — "What happened after X?" ──────────────────────────────────── - def _causal_questions(self, threads: List[dict]) -> List[dict]: questions = [] multi_hop = [ @@ -470,7 +654,7 @@ def _causal_questions(self, threads: List[dict]) -> List[dict]: ) for thread in random.sample(multi_hop, min(50, len(multi_hop))): nodes = thread["nodes"] - # Ask about the transition from node 1 → node 2 + if len(nodes) < 2: continue trigger_node = nodes[0] @@ -512,8 +696,6 @@ def _causal_questions(self, threads: List[dict]) -> List[dict]: ) return questions - # ── TEMPORAL — "Did person P know about X when they made decision D?" ───── - def _temporal_questions(self) -> List[dict]: """ Uses actor knowledge snapshots from day_summary SimEvents. @@ -579,8 +761,6 @@ def _build_temporal_question(event: SimEvent) -> Optional[dict]: if gap_areas: gap_domain = gap_areas[0] elif root_cause: - # Use the first meaningful token from the root cause description. - # Strip common stop words so we don't get gap_domain="the" etc. _stop = {"the", "a", "an", "in", "of", "on", "was", "is", "due", "to"} tokens = [ t @@ -589,7 +769,6 @@ def _build_temporal_question(event: SimEvent) -> Optional[dict]: ] gap_domain = tokens[0] if tokens else "system" else: - # Last resort: most recent departure's first domain prior_dep = next( ( d @@ -604,10 +783,6 @@ def _build_temporal_question(event: SimEvent) -> Optional[dict]: else: gap_domain = "system" - # ── Dedup guard ─────────────────────────────────────────────────── - # Skip if we already have a question for this (person, domain) pair. - # This prevents a single dominant departure from generating N identical - # questions that only differ in ticket number. if (assignee, gap_domain) in seen_person_domain: return None seen_person_domain.add((assignee, gap_domain)) @@ -673,10 +848,6 @@ def _build_temporal_question(event: SimEvent) -> Optional[dict]: "chain_id": f"incident_{ticket_id}", } - # Sample both pools independently. - # Caps raised to 8 per pool so short sim runs still yield meaningful n. - # The (person, domain) dedup guard inside _build_temporal_question ensures - # we don't burn all slots on the same departure-domain pair. for event in random.sample(pool_a, min(50, len(pool_a))): q = _build_temporal_question(event) if q: @@ -687,8 +858,6 @@ def _build_temporal_question(event: SimEvent) -> Optional[dict]: if q: questions.append(q) - # Log had_knowledge balance — if this is all False, the sim run has - # only one departure and the eval will be gameable by always answering "no". hk_dist = {True: 0, False: 0} for q in questions: hk_dist[q["ground_truth"]["had_knowledge"]] += 1 @@ -698,8 +867,6 @@ def _build_temporal_question(event: SimEvent) -> Optional[dict]: ) return questions - # ── GAP DETECTION — "Was this email ever actioned?" ────────────────────── - def _gap_detection_questions(self, threads: List[dict]) -> List[dict]: questions = [] dropped = [t for t in threads if t["chain_type"] == "dropped_email"] @@ -713,7 +880,6 @@ def _gap_detection_questions(self, threads: List[dict]) -> List[dict]: f"{len(routed)} routed-complete threads available" ) - # Questions about dropped emails (answer: no action) for thread in random.sample(dropped, min(50, len(dropped))): subject = thread.get("subject", thread["root_artifact"]) source = thread.get("source", "unknown sender") @@ -747,7 +913,6 @@ def _gap_detection_questions(self, threads: List[dict]) -> List[dict]: } ) - # Paired questions about routed emails (answer: yes, action was taken) for thread in random.sample(routed, min(20, len(routed))): source = thread.get("source", "unknown") chain = [n["artifact_id"] for n in thread["nodes"]] @@ -780,8 +945,6 @@ def _gap_detection_questions(self, threads: List[dict]) -> List[dict]: ) return questions - # ── ROUTING — "Who first saw this?" ────────────────────────────────────── - def _routing_questions(self, threads: List[dict]) -> List[dict]: questions = [] customer_threads = [ @@ -1191,7 +1354,912 @@ def _customer_escalation_questions(self) -> List[dict]: ) return questions - # ── LLM question prose ──────────────────────────────────────────────────── + # ── ZD_RESOLUTION — "Was ZD ticket X resolved and how long did it take?" ──── + + def _zendesk_resolution_questions(self, threads: List[dict]) -> List[dict]: + """ + ZD_RESOLUTION questions over zendesk_ticket threads. + Tests whether an agent can determine ticket resolution status and SLA + duration from the Zendesk ticket lifecycle event chain. + + Maps to existing RETRIEVAL scorer — ground truth is a boolean + day count, + not a narrative, so exact-match is appropriate. + """ + questions = [] + zd_threads = [t for t in threads if t["chain_type"] == "zendesk_ticket"] + logger.info( + f"[eval] _zendesk_resolution_questions: {len(zd_threads)} ZD threads available" + ) + + for thread in random.sample(zd_threads, min(30, len(zd_threads))): + tid = thread["root_artifact"] + org = thread.get("org_name", "a customer") + subject = thread.get("subject", tid) + + ground_truth = { + "ticket_id": tid, + "artifact_id": tid, + "org_name": org, + "resolved": thread["complete"], + "duration_days": thread.get("duration_days"), + "escalated": thread["escalated"], + "incident_id": thread.get("incident_id"), + } + evidence = [tid] + if thread.get("incident_id"): + evidence.append(thread["incident_id"]) + + q_text = self._generate_question_prose( + template=( + f"Generate a retrieval question asking whether Zendesk ticket " + f"{tid} from {org} was resolved, and if so, how many days it " + f"took. The question should test whether an agent can trace the " + f"full ticket lifecycle. " + f"Output only the question text." + ) + ) + if q_text: + difficulty = "medium" if thread["escalated"] else "easy" + questions.append( + { + "question_id": f"zd_resolution_{tid}", + "question_type": "ZD_RESOLUTION", + "question_text": q_text, + "ground_truth": ground_truth, + "evidence_chain": evidence, + "difficulty": difficulty, + "requires_reasoning": thread["escalated"], + "chain_id": thread["chain_id"], + } + ) + return questions + + # ── ZD_ESCALATION — "Which tickets escalated to incident X?" ───────────── + + def _zd_escalation_questions(self, threads: List[dict]) -> List[dict]: + """ + CAUSAL questions over zendesk_ticket threads where escalated=True. + Tests cross-system reasoning: given a Jira incident, can the agent + identify which ZD tickets triggered or were linked to it? + """ + questions = [] + escalated = [ + t for t in threads if t["chain_type"] == "zendesk_ticket" and t["escalated"] + ] + logger.info( + f"[eval] _zd_escalation_questions: {len(escalated)} escalated ZD threads available" + ) + + # Group by incident so we can ask "all tickets for incident X" questions + by_incident: Dict[str, List[dict]] = {} + for t in escalated: + iid = t.get("incident_id", "") + if iid: + by_incident.setdefault(iid, []).append(t) + + for incident_id, tickets in random.sample( + list(by_incident.items()), min(20, len(by_incident)) + ): + ticket_ids = [t["root_artifact"] for t in tickets] + orgs = list({t.get("org_name", "") for t in tickets if t.get("org_name")}) + + ground_truth = { + "incident_id": incident_id, + "artifact_id": incident_id, + "ticket_ids": ticket_ids, + "ticket_count": len(ticket_ids), + "affected_orgs": orgs, + "event_type": "zd_tickets_escalated", + } + evidence = [incident_id] + ticket_ids + + q_text = self._generate_question_prose( + template=( + f"Generate a causal question asking which Zendesk support " + f"tickets were escalated as a result of incident {incident_id}. " + f"The question should require the agent to trace from the " + f"incident back to the affected customer tickets. " + f"Output only the question text." + ) + ) + if q_text: + questions.append( + { + "question_id": f"zd_escalation_{incident_id}", + "question_type": "CAUSAL", + "question_text": q_text, + "ground_truth": ground_truth, + "evidence_chain": evidence, + "difficulty": "medium", + "requires_reasoning": True, + "chain_id": f"zd_escalation_{incident_id}", + } + ) + return questions + + # ── SF_RISK — "Which accounts were flagged at-risk after incident X?" ───── + + def _sf_risk_questions(self, threads: List[dict]) -> List[dict]: + """ + SF_RISK questions over sf_risk threads (sf_deals_risk_flagged events). + New question type: distinct from CAUSAL because it specifically tests + cross-system awareness — the agent must connect an engineering incident + to its commercial impact in Salesforce. + """ + questions = [] + risk_threads = [t for t in threads if t["chain_type"] == "sf_risk"] + logger.info( + f"[eval] _sf_risk_questions: {len(risk_threads)} SF risk threads available" + ) + + for thread in random.sample(risk_threads, min(25, len(risk_threads))): + incident_id = thread["incident_id"] + accounts = thread["at_risk_accounts"] + if not accounts: + continue + + ground_truth = { + "incident_id": incident_id, + "artifact_id": incident_id, + "at_risk_accounts": accounts, + "account_count": len(accounts), + "day": thread["day"], + } + evidence = [incident_id] + + q_text = self._generate_question_prose( + template=( + f"Generate a question asking which Salesforce customer accounts " + f"were flagged as at-risk following incident {incident_id}. " + f"The question should require the agent to connect an engineering " + f"incident to its downstream commercial impact. " + f"Output only the question text." + ) + ) + if q_text: + questions.append( + { + "question_id": f"sf_risk_{incident_id}", + "question_type": "SF_RISK", + "question_text": q_text, + "ground_truth": ground_truth, + "evidence_chain": evidence, + "difficulty": "hard", + "requires_reasoning": True, + "chain_id": thread["chain_id"], + } + ) + return questions + + def _sf_ownership_lapse_questions(self, threads: List[dict]) -> List[dict]: + """ + RETRIEVAL questions over sf_ownership_lapse threads. + Uses existing RETRIEVAL scorer — ground truth is a deterministic account + list derived from sf_ownership_lapsed SimEvents. + Tests cross-domain reasoning: departure event → CRM consequence. + """ + questions = [] + lapse_threads = [t for t in threads if t["chain_type"] == "sf_ownership_lapse"] + logger.info( + f"[eval] _sf_ownership_lapse_questions: {len(lapse_threads)} lapse threads available" + ) + + for thread in lapse_threads: # typically low-volume; use all + departed = thread["departed_employee"] + accs = thread["lapsed_accounts"] + opps = thread["lapsed_opportunities"] + if not accs and not opps: + continue + + ground_truth = { + "departed_employee": departed, + "role": thread.get("role", ""), + "artifact_id": thread["root_artifact"], + "lapsed_accounts": accs, + "lapsed_opportunities": opps, + "day": thread["day"], + } + evidence = accs[:3] + opps[:2] # cap evidence list length + + q_text = self._generate_question_prose( + template=( + f"Generate a retrieval question asking which Salesforce accounts " + f"or open opportunities were left without an owner after " + f"{departed} departed. The question should test whether an agent " + f"can trace the CRM impact of an employee departure. " + f"Output only the question text." + ) + ) + if q_text: + questions.append( + { + "question_id": f"sf_lapse_{departed.lower().replace(' ', '_')}", + "question_type": "RETRIEVAL", + "question_text": q_text, + "ground_truth": ground_truth, + "evidence_chain": [e for e in evidence if e], + "difficulty": "medium", + "requires_reasoning": True, + "chain_id": thread["chain_id"], + } + ) + return questions + + def _sf_touchpoint_questions(self, threads: List[dict]) -> List[dict]: + """ + CAUSAL questions over sf_touchpoint threads (crm_touchpoint events). + Tests whether an agent can link an outbound sales email to the SF + opportunity it created or advanced. + """ + questions = [] + tp_threads = [t for t in threads if t["chain_type"] == "sf_touchpoint"] + logger.info( + f"[eval] _sf_touchpoint_questions: {len(tp_threads)} SF touchpoint threads available" + ) + + for thread in random.sample(tp_threads, min(25, len(tp_threads))): + opp_id = thread["root_artifact"] + sender = thread.get("sender", "a sales rep") + account = thread.get("account_name", "a customer") + stage = thread.get("stage", "") + subject = thread.get("subject", "") + + ground_truth = { + "opportunity_id": opp_id, + "artifact_id": opp_id, + "account_name": account, + "stage": stage, + "sender": sender, + "event_type": "crm_touchpoint", + } + evidence = [opp_id] + + q_text = self._generate_question_prose( + template=( + f"Generate a causal question asking which Salesforce opportunity " + f"was created or advanced when {sender} sent an outbound email " + f'with subject "{subject[:60]}" to {account}. The question ' + f"should test whether an agent can trace from an outbound email " + f"to its CRM outcome. " + f"Output only the question text." + ) + ) + if q_text: + questions.append( + { + "question_id": f"sf_touchpoint_{opp_id}", + "question_type": "CAUSAL", + "question_text": q_text, + "ground_truth": ground_truth, + "evidence_chain": evidence, + "difficulty": "medium", + "requires_reasoning": True, + "chain_id": thread["chain_id"], + } + ) + return questions + + def _datadog_alert_questions(self) -> List[dict]: + """ + DATADOG_ALERT questions inferred from incident_opened SimEvents. + + Datadog alert records are generated post-sim from incident events + (see post_sim_artifacts.py DatadogWriter). Because no dedicated + 'datadog_alert_fired' SimEvent is emitted during the sim, we derive + ground truth directly from incident_opened facts: the monitor name + and the incident it corresponds to are deterministically linked. + + The agent must retrieve the right incident artifact and demonstrate + awareness that a Datadog alert is the upstream signal for the incident. + Maps to RETRIEVAL scorer — ground truth is the incident artifact_id. + """ + questions = [] + incident_events = [e for e in self._events if e.type == "incident_opened"] + logger.info( + f"[eval] _datadog_alert_questions: {len(incident_events)} incidents available" + ) + + for event in random.sample(incident_events, min(20, len(incident_events))): + ticket_id = event.artifact_ids.get("jira", "") + root_cause = event.facts.get("root_cause", "") + if not ticket_id or not root_cause: + continue + + ground_truth = { + "incident_id": ticket_id, + "artifact_id": ticket_id, + "root_cause": root_cause, + "open_day": event.day, + # Monitor name is generated post-sim via LLM batch; ground truth + # here is the incident ID so the scorer can do an exact match + # without depending on the LLM-enriched monitor name string. + "monitor_source": "datadog", + } + evidence = [ticket_id] + + q_text = self._generate_question_prose( + template=( + f"Generate a retrieval question asking which Datadog alert or " + f"monitor fired to trigger incident {ticket_id}, whose root " + f'cause was: "{root_cause[:80]}". The question should test ' + f"whether an agent can connect an observability alert to the " + f"incident it caused. " + f"Output only the question text." + ) + ) + if q_text: + questions.append( + { + "question_id": f"dd_alert_{ticket_id}", + "question_type": "DATADOG_ALERT", + "question_text": q_text, + "ground_truth": ground_truth, + "evidence_chain": evidence, + "difficulty": "medium", + "requires_reasoning": False, + "chain_id": f"incident_{ticket_id}", + } + ) + return questions + + def _nps_score_questions(self) -> List[dict]: + """ + NPS_SCORE questions inferred from ZD ticket + incident SimEvents. + + NPS responses are generated post-sim (see post_sim_artifacts.py NPSWriter). + Ground truth is derived deterministically from the same scoring formula + used by NPSWriter — no disk read required: + Base 9; -3 per escalated ZD ticket; -2 per unresolved ZD ticket; + -1 per SLA breach day; clamped to [0, 10]. + + Tests whether an agent can reason across ZD tickets, incidents, and + the NPS artifact to explain a customer's satisfaction score. + """ + questions = [] + + org_data: Dict[str, Dict] = {} + + for e in self._events: + if e.type == "zd_ticket_opened": + org = e.facts.get("org_name", "") + if not org: + continue + org_data.setdefault( + org, + { + "tickets": [], + "escalated_count": 0, + "unresolved_count": 0, + "breach_days": 0, + }, + ) + org_data[org]["tickets"].append(e.facts.get("ticket_id", "")) + + elif e.type == "zd_tickets_escalated": + # Each escalated ticket adds to the org's escalated count + iid = e.facts.get("incident_id", "") + for tid in e.facts.get("ticket_ids", []): + # Find org for this ticket + for e2 in self._events: + if ( + e2.type == "zd_ticket_opened" + and e2.facts.get("ticket_id") == tid + ): + org = e2.facts.get("org_name", "") + if org in org_data: + org_data[org]["escalated_count"] += 1 + break + + if not org_data: + logger.info("[eval] _nps_score_questions: no ZD ticket data — skipping") + return questions + + for org, data in random.sample(list(org_data.items()), min(20, len(org_data))): + ticket_ids = data["tickets"] + if not ticket_ids: + continue + + score = 9 + score -= 3 * data["escalated_count"] + score = max(0, min(10, score)) + + classification = ( + "promoter" if score >= 9 else "passive" if score >= 7 else "detractor" + ) + + ground_truth = { + "org_name": org, + "artifact_id": ticket_ids[0], + "nps_score": score, + "classification": classification, + "escalated_tickets": data["escalated_count"], + "ticket_ids": ticket_ids, + } + evidence = ticket_ids[:3] + + q_text = self._generate_question_prose( + template=( + f"Generate a question asking what NPS score the customer " + f"{org} would give based on their support experience, and " + f"what factors drove that score. The question should require " + f"the agent to reason across support tickets and incident " + f"history to predict customer satisfaction. " + f"Output only the question text." + ) + ) + if q_text: + questions.append( + { + "question_id": f"nps_{org.lower().replace(' ', '_')}", + "question_type": "NPS_SCORE", + "question_text": q_text, + "ground_truth": ground_truth, + "evidence_chain": evidence, + "difficulty": "hard", + "requires_reasoning": True, + "chain_id": f"nps_{org.lower().replace(' ', '_')}", + } + ) + return questions + + def _invoice_sla_questions(self) -> List[dict]: + """ + INVOICE_SLA questions inferred from incident + ZD SimEvents. + + Invoices are generated post-sim (see post_sim_artifacts.py InvoiceWriter). + Ground truth is derived from SLA breach logic: any incident that lasted + more than SLA_BREACH_THRESHOLD_DAYS (1 day) generates an SLA credit + on the affected customers' invoices at SLA_CREDIT_RATE (2%) of contract value. + + Tests whether an agent can reason about the financial consequences of + incidents by tracing: incident duration → SLA breach → invoice line item. + """ + questions = [] + + resolved_by_id: Dict[str, SimEvent] = { + e.artifact_ids.get("jira", ""): e + for e in self._events + if e.type == "incident_resolved" + } + + sla_breach_incidents = [] + for e in self._events: + if e.type != "incident_opened": + continue + ticket_id = e.artifact_ids.get("jira", "") + if not ticket_id: + continue + resolve_event = resolved_by_id.get(ticket_id) + if not resolve_event: + continue + duration = resolve_event.day - e.day + if duration > 1: + sla_breach_incidents.append( + { + "ticket_id": ticket_id, + "open_day": e.day, + "resolve_day": resolve_event.day, + "duration_days": duration, + "root_cause": e.facts.get("root_cause", ""), + } + ) + + incident_to_orgs: Dict[str, List[str]] = {} + for e in self._events: + if e.type == "zd_tickets_escalated": + iid = e.facts.get("incident_id", "") + if not iid: + continue + for tid in e.facts.get("ticket_ids", []): + for e2 in self._events: + if ( + e2.type == "zd_ticket_opened" + and e2.facts.get("ticket_id") == tid + ): + org = e2.facts.get("org_name", "") + if org: + incident_to_orgs.setdefault(iid, []).append(org) + break + elif e.type == "sf_deals_risk_flagged": + iid = e.facts.get("incident_id", "") + if iid: + for org in e.facts.get("account_names", []): + incident_to_orgs.setdefault(iid, []).append(org) + + logger.info( + f"[eval] _invoice_sla_questions: {len(sla_breach_incidents)} " + f"SLA-breaching incidents available" + ) + + for inc in random.sample( + sla_breach_incidents, min(20, len(sla_breach_incidents)) + ): + ticket_id = inc["ticket_id"] + orgs = list(set(incident_to_orgs.get(ticket_id, []))) + if not orgs: + continue + + credit_per_org = round(50_000 * 0.02 * inc["duration_days"], 2) + + ground_truth = { + "incident_id": ticket_id, + "artifact_id": ticket_id, + "breach_duration_days": inc["duration_days"], + "affected_orgs": orgs, + "sla_credit_per_org": credit_per_org, + "root_cause": inc["root_cause"], + } + evidence = [ticket_id] + + q_text = self._generate_question_prose( + template=( + f"Generate a question asking what SLA credit would appear on " + f"the invoice for customers affected by incident {ticket_id}, " + f"which remained open for {inc['duration_days']} days. The " + f"question should require the agent to reason about SLA breach " + f"thresholds and financial credit calculations. " + f"Output only the question text." + ) + ) + if q_text: + questions.append( + { + "question_id": f"invoice_sla_{ticket_id}", + "question_type": "INVOICE_SLA", + "question_text": q_text, + "ground_truth": ground_truth, + "evidence_chain": evidence, + "difficulty": "hard", + "requires_reasoning": True, + "chain_id": f"incident_{ticket_id}", + } + ) + return questions + + def _pr_review_questions(self, threads: List[dict]) -> List[dict]: + questions = [] + pr_events = [e for e in self._events if e.type == "pr_review"] + logger.info( + f"[eval] _pr_review_questions: {len(pr_events)} pr_review events available" + ) + + for event in random.sample(pr_events, min(40, len(pr_events))): + pr_id = event.artifact_ids.get("pr", "") + reviewer = event.facts.get("reviewer", "") + author = event.facts.get("author", "") + verdict = event.facts.get("verdict", "") + pr_title = event.facts.get("pr_title", "") + linked_ticket = event.artifact_ids.get("jira", "") + + if not pr_id or not reviewer or not verdict: + continue + + ground_truth_review = { + "pr_id": pr_id, + "reviewer": reviewer, + "author": author, + "verdict": verdict, + "linked_ticket": linked_ticket, + "day": event.day, + } + evidence_review = [pr_id] + ([linked_ticket] if linked_ticket else []) + + q_text_review = self._generate_question_prose( + template=( + f"Generate a retrieval question asking who reviewed pull request " + f"{pr_id} and whether it was approved or had changes requested. " + f"The question should test whether an agent can locate the review " + f"record and identify both the reviewer and their verdict. " + f"Output only the question text." + ) + ) + if q_text_review: + questions.append( + { + "question_id": f"pr_review_{pr_id}", + "question_type": "PR_REVIEW", + "question_text": q_text_review, + "ground_truth": ground_truth_review, + "evidence_chain": [e for e in evidence_review if e], + "difficulty": "easy", + "requires_reasoning": False, + "chain_id": f"pr_{pr_id}", + } + ) + + if not linked_ticket: + continue + + ground_truth_causal = { + "artifact_id": pr_id, + "event_type": "pr_review", + "actors": [reviewer, author], + "timestamp": event.timestamp, + "verdict": verdict, + } + evidence_causal = [linked_ticket, pr_id] + + q_text_causal = self._generate_question_prose( + template=( + f"Generate a causal question asking what pull request was opened " + f"or reviewed as a result of work on ticket {linked_ticket}. " + f"The question should require the agent to trace from a Jira ticket " + f"to the GitHub PR that resolved it. " + f"Output only the question text." + ) + ) + if q_text_causal: + questions.append( + { + "question_id": f"pr_causal_{linked_ticket}_{pr_id}", + "question_type": "CAUSAL", + "question_text": q_text_causal, + "ground_truth": ground_truth_causal, + "evidence_chain": evidence_causal, + "difficulty": "medium", + "requires_reasoning": True, + "chain_id": f"pr_{pr_id}", + } + ) + + return questions + + def _blocker_questions(self) -> List[dict]: + questions = [] + blocker_events = [e for e in self._events if e.type == "blocker_flagged"] + logger.info( + f"[eval] _blocker_questions: {len(blocker_events)} blocker_flagged events available" + ) + + seen_tickets: Set[str] = set() + for event in random.sample(blocker_events, min(25, len(blocker_events))): + ticket_id = event.artifact_ids.get("jira", "") + slack_thread = event.artifact_ids.get("slack_thread", "") + assignee = next(iter(event.actors), None) + blocker_reason = event.facts.get( + "comment", event.facts.get("blocker_reason", "") + ) + + if not ticket_id or not assignee or ticket_id in seen_tickets: + continue + seen_tickets.add(ticket_id) + + ground_truth = { + "artifact_id": ticket_id, + "was_blocked": True, + "assignee": assignee, + "ticket_id": ticket_id, + "day": event.day, + "slack_thread": slack_thread, + } + evidence = [ticket_id] + ([slack_thread] if slack_thread else []) + + q_text = self._generate_question_prose( + template=( + f"Generate a retrieval question asking what ticket {assignee} was " + f"blocked on during Day {event.day}, and what the blocker was. " + f"The question should test whether an agent can locate a blocker " + f"report in the ticket or Slack history. " + f"Output only the question text." + ) + ) + if q_text: + questions.append( + { + "question_id": f"blocker_{ticket_id}_day{event.day}", + "question_type": "RETRIEVAL", + "question_text": q_text, + "ground_truth": ground_truth, + "evidence_chain": [e for e in evidence if e], + "difficulty": "medium", + "requires_reasoning": False, + "chain_id": f"blocker_{ticket_id}", + } + ) + + blocked_ticket_ids = {e.artifact_ids.get("jira", "") for e in blocker_events} + progress_events = [ + e + for e in self._events + if e.type == "ticket_progress" + and e.artifact_ids.get("jira", "") not in blocked_ticket_ids + and e.artifact_ids.get("jira", "") + ] + for event in random.sample(progress_events, min(15, len(progress_events))): + ticket_id = event.artifact_ids.get("jira", "") + assignee = next(iter(event.actors), None) + if not ticket_id or not assignee: + continue + + ground_truth = { + "artifact_id": ticket_id, + "was_blocked": False, + "assignee": assignee, + "ticket_id": ticket_id, + "day": event.day, + "slack_thread": None, + } + + q_text = self._generate_question_prose( + template=( + f"Generate a question asking whether {assignee} reported any " + f"blockers while working on ticket {ticket_id} on Day {event.day}. " + f"The question should require the agent to check the ticket history " + f"and confirm whether a blocker was or wasn't reported. " + f"Output only the question text." + ) + ) + if q_text: + questions.append( + { + "question_id": f"blocker_check_{ticket_id}_day{event.day}", + "question_type": "RETRIEVAL", + "question_text": q_text, + "ground_truth": ground_truth, + "evidence_chain": [ticket_id], + "difficulty": "easy", + "requires_reasoning": False, + "chain_id": f"blocker_{ticket_id}", + } + ) + + logger.info( + f"[eval] _blocker_questions: generated {len(questions)} questions " + f"(pool_A={len(blocker_events)}, pool_B={len(progress_events)})" + ) + return questions + + def _vendor_routing_questions(self) -> List[dict]: + questions = [] + vendor_events = [e for e in self._events if e.type == "vendor_email_routed"] + logger.info( + f"[eval] _vendor_routing_questions: {len(vendor_events)} vendor_email_routed events available" + ) + + for event in random.sample(vendor_events, min(25, len(vendor_events))): + email_id = event.artifact_ids.get("email", "") + vendor = event.facts.get("vendor", "") + topic = event.facts.get("topic", "") + routed_to = event.facts.get("routed_to", "") + causal_chain = event.facts.get("causal_chain", []) + + if not email_id or not vendor or not routed_to: + continue + + jira_opened = any( + a + for a in causal_chain + if isinstance(a, str) and (a.startswith("IT-") or a.startswith("ORG-")) + ) + + ground_truth = { + "first_recipient": routed_to, + "artifact_id": email_id, + "vendor": vendor, + "topic": topic, + "jira_opened": jira_opened, + "timestamp": event.timestamp, + } + evidence = [email_id] + ([c for c in causal_chain[:2] if c != email_id]) + + q_text = self._generate_question_prose( + template=( + f"Generate a routing question asking which internal engineer or " + f"team member handled the alert or email from {vendor} regarding " + f'"{topic[:60]}". The question should test whether an agent can ' + f"trace inbound vendor communications to the responsible engineer. " + f"Output only the question text." + ) + ) + if q_text: + questions.append( + { + "question_id": f"vendor_routing_{email_id}", + "question_type": "ROUTING", + "question_text": q_text, + "ground_truth": ground_truth, + "evidence_chain": [e for e in evidence if e], + "difficulty": "medium", + "requires_reasoning": False, + "chain_id": f"vendor_{email_id}", + } + ) + return questions + + def _design_discussion_questions(self) -> List[dict]: + questions = [] + dd_events = [e for e in self._events if e.type == "design_discussion"] + logger.info( + f"[eval] _design_discussion_questions: {len(dd_events)} design_discussion events available" + ) + + for event in random.sample(dd_events, min(30, len(dd_events))): + topic = event.facts.get("topic", "") + participants = event.facts.get("participants", event.actors) + medium = event.facts.get("medium", "slack") + spawned_doc = event.facts.get("spawned_doc", False) + conf_id = event.artifact_ids.get("confluence", "") + + artifact_id = ( + event.artifact_ids.get("zoom_transcript") + or event.artifact_ids.get("slack_thread") + or "" + ) + linked_ticket = event.artifact_ids.get("jira", "") + + if not artifact_id or not topic or not participants: + continue + + ground_truth_ret = { + "artifact_id": artifact_id, + "topic": topic, + "participants": participants, + "medium": medium, + "day": event.day, + "linked_ticket": linked_ticket, + } + evidence_ret = [artifact_id] + ([linked_ticket] if linked_ticket else []) + + q_text_ret = self._generate_question_prose( + template=( + f"Generate a retrieval question asking who participated in the " + f"{'Zoom call' if medium == 'zoom' else 'Slack design discussion'} " + f'about "{topic[:80]}". The question should require the agent to ' + f"locate the {'transcript' if medium == 'zoom' else 'thread'} and " + f"identify all participants. " + f"Output only the question text." + ) + ) + if q_text_ret: + questions.append( + { + "question_id": f"design_discussion_{artifact_id}", + "question_type": "RETRIEVAL", + "question_text": q_text_ret, + "ground_truth": ground_truth_ret, + "evidence_chain": [e for e in evidence_ret if e], + "difficulty": "easy", + "requires_reasoning": False, + "chain_id": f"design_{artifact_id}", + } + ) + + if not spawned_doc or not conf_id: + continue + + ground_truth_causal = { + "artifact_id": conf_id, + "event_type": "confluence_created", + "actors": participants, + "timestamp": event.timestamp, + "source_discussion": artifact_id, + } + evidence_causal = [artifact_id, conf_id] + + q_text_causal = self._generate_question_prose( + template=( + f"Generate a causal question asking which Confluence document was " + f'created as a result of the design discussion on "{topic[:80]}". ' + f"The question should require the agent to trace from the discussion " + f"artifact to the documentation it produced. " + f"Output only the question text." + ) + ) + if q_text_causal: + questions.append( + { + "question_id": f"design_doc_spawned_{artifact_id}", + "question_type": "CAUSAL", + "question_text": q_text_causal, + "ground_truth": ground_truth_causal, + "evidence_chain": evidence_causal, + "difficulty": "medium", + "requires_reasoning": True, + "chain_id": f"design_{artifact_id}", + } + ) + + return questions def _generate_question_prose(self, template: str) -> Optional[str]: """LLM writes question wording only. Never touches ground truth.""" @@ -1229,11 +2297,6 @@ def _find_event_by_artifact(self, artifact_id: str) -> Optional[SimEvent]: return None -# ───────────────────────────────────────────────────────────────────────────── -# RUNNER -# ───────────────────────────────────────────────────────────────────────────── - - class EvalHarness: def __init__(self): from flow import build_llm @@ -1244,7 +2307,6 @@ def __init__(self): def run(self) -> None: logger.info("[bold cyan]🔬 Building eval dataset...[/bold cyan]") - # 1. Build causal threads builder = CausalThreadBuilder(self._mem) threads = builder.build_all() logger.info(f" {len(threads)} causal threads extracted") @@ -1254,12 +2316,10 @@ def run(self) -> None: json.dump(threads, f, indent=2, default=str) logger.info(f" → {threads_path}") - # 2. Generate eval questions generator = EvalQuestionGenerator(self._mem, self._worker_llm) questions = generator.generate(threads) logger.info(f" {len(questions)} eval questions generated") - # Annotate with difficulty distribution by_type: Dict[str, int] = defaultdict(int) by_difficulty: Dict[str, int] = defaultdict(int) for q in questions: diff --git a/eval/retrieval_extensions.py b/eval/retrieval_extensions.py new file mode 100644 index 0000000..8abcde5 --- /dev/null +++ b/eval/retrieval_extensions.py @@ -0,0 +1,539 @@ +""" +retrieval_extensions.py +======================= +Drop-in retrieval extensions for eval_e2e.py. + +Provides two new Retriever subclasses that slot directly into +build_retriever() and the existing eval loop: + + RRFRetriever + ------------ + Reciprocal Rank Fusion over any 2-N sub-retrievers. + Fuses BM25 (lexical) with a dense retriever (Cohere / OpenAI / Bedrock) + by default, producing a ranked list whose score is: + + RRF(d) = Σ_r 1 / (k + rank_r(d)) + + where k=60 is the standard smoothing constant. + + Usage (eval_e2e.py CLI addition): + python eval_e2e.py --retriever rrf --generator claude + python eval_e2e.py --retriever rrf-openai --generator claude + python eval_e2e.py --retriever rrf-bedrock --generator claude + + GraphAugmentedRetriever + ----------------------- + Wraps any base Retriever and expands results by walking artifact + relationship edges that are embedded in the corpus itself. + + The corpus documents produced by OrgForge's simulation carry relationship + metadata in several forms: + • JSON-encoded "related_ids" / "causal_chain" / "artifact_ids" fields + • Inline artifact-ID tokens (ORG-\d+, CONF-\w+, EMAIL-\d+, etc.) + referenced in the body text + • A free-form "evidence_chain" field + + The graph expander: + 1. Indexes all edges at index() time → O(|corpus|) build + 2. At retrieve() time, takes the base retriever's top-K results, + adds 1-hop neighbours from the edge graph, re-ranks the combined + pool by (base_score + neighbour_boost), and returns top-K. + + Neighbour boost decays with hop distance: + boost(d, hop) = NEIGHBOUR_BOOST_BASE ** hop (default: 0.5 per hop) + + Usage: + python eval_e2e.py --retriever graph-bm25 --generator claude + python eval_e2e.py --retriever graph-cohere --generator claude + python eval_e2e.py --retriever graph-rrf --generator claude + +Integration +----------- +Add the following block to eval_e2e.py's build_retriever() function: + + # ── paste after the existing if/elif chain ────────────────────────── + from retrieval_extensions import RRFRetriever, GraphAugmentedRetriever + + if name == "rrf": + return RRFRetriever([BM25Retriever(), CohereRetriever()]) + if name == "rrf-openai": + return RRFRetriever([BM25Retriever(), OpenAIRetriever()]) + if name == "rrf-bedrock": + return RRFRetriever([BM25Retriever(), BedrockCohereRetriever(region=region)]) + if name == "graph-bm25": + return GraphAugmentedRetriever(BM25Retriever()) + if name == "graph-cohere": + return GraphAugmentedRetriever(CohereRetriever()) + if name == "graph-rrf": + base = RRFRetriever([BM25Retriever(), CohereRetriever()]) + return GraphAugmentedRetriever(base) + # ──────────────────────────────────────────────────────────────────── + +Also add the new choices to the --retriever argparse argument: + + choices=[ + "bm25", "cohere", "cohere-bedrock", "openai", + "rrf", "rrf-openai", "rrf-bedrock", + "graph-bm25", "graph-cohere", "graph-rrf", + ], +""" + +from __future__ import annotations + +import json +import logging +import re +from collections import defaultdict +from typing import Dict, List, Optional, Set, Tuple + +import numpy as np + +logger = logging.getLogger("orgforge.retrieval_extensions") + +# ── Artifact-ID pattern: covers ORG-42, CONF-ENG-007, EMAIL-003, +# SLACK-THREAD-9, PR-12, ZD-456, etc. +_ARTIFACT_ID_RE = re.compile( + r"\b(?:ORG|CONF|EMAIL|SLACK(?:-THREAD)?|PR|ZD|SF|DD|JIRA)[-_][\w-]+", + re.IGNORECASE, +) + +# Fields in corpus docs that may carry related artifact IDs (JSON-encoded or plain list). +_RELATION_FIELDS = ( + "related_ids", + "causal_chain", + "artifact_ids", + "evidence_chain", + "downstream_artifacts", + "linked_artifacts", +) + +# RRF smoothing constant (Cormack et al. 2009 recommend k=60). +RRF_K: int = 60 + +# Graph expansion: score boost applied to 1-hop neighbours. +# Each additional hop multiplies by this factor (geometric decay). +NEIGHBOUR_BOOST_BASE: float = 0.5 + +# Maximum graph hops to expand. Keep at 1-2 to avoid noise amplification. +MAX_HOPS: int = 2 + + +# ───────────────────────────────────────────────────────────────────────────── +# RECIPROCAL RANK FUSION +# ───────────────────────────────────────────────────────────────────────────── + + +class RRFRetriever: + """ + Fuse ranked lists from two or more Retriever instances using + Reciprocal Rank Fusion (Cormack, Clarke & Buettcher, SIGIR 2009). + + RRF score for document d across ranker set R: + + rrf(d) = Σ_{r ∈ R} 1 / (k + rank_r(d)) + + Documents not ranked by a given retriever are assigned rank = infinity + (contributing 0 to the sum), which naturally deprioritises them without + discarding them entirely. + + Parameters + ---------- + retrievers : list of Retriever + At least two Retriever instances. All must be indexable with the same + corpus. Mixing BM25 + dense gives the best lexical/semantic coverage. + k : int + RRF smoothing constant. Default 60 matches the canonical paper. + candidate_k : int + How many candidates each sub-retriever fetches before fusion. + Should be ≥ final top_k; larger values improve recall at the cost of + extra embedding lookups on dense retrievers. + """ + + name = "rrf" + + def __init__( + self, + retrievers: List, + k: int = RRF_K, + candidate_k: int = 50, + ) -> None: + if len(retrievers) < 2: + raise ValueError("RRFRetriever requires at least two sub-retrievers.") + self._retrievers = retrievers + self._k = k + self._candidate_k = candidate_k + # Build a human-readable name from the sub-retriever names. + sub_names = "+".join(r.name for r in retrievers) + self.name = f"rrf({sub_names})" + + # ------------------------------------------------------------------ + # Retriever protocol + # ------------------------------------------------------------------ + + def index(self, corpus: List[dict]) -> None: + """Index every sub-retriever with the same corpus.""" + for r in self._retrievers: + logger.info(f" [RRF] Indexing sub-retriever: {r.name}") + r.index(corpus) + logger.info(f" [RRF] All {len(self._retrievers)} sub-retrievers indexed.") + + def retrieve(self, query: str, top_k: int = 10) -> List[str]: + """ + Fetch candidates from each sub-retriever, apply RRF scoring, + and return the top_k doc_ids ordered by descending RRF score. + """ + candidate_k = max(self._candidate_k, top_k * 3) + + # Collect per-retriever ranked lists. + ranked_lists: List[List[str]] = [] + for r in self._retrievers: + try: + ranked = r.retrieve(query, top_k=candidate_k) + except Exception as exc: + logger.warning(f" [RRF] Sub-retriever {r.name} failed: {exc}") + ranked = [] + ranked_lists.append(ranked) + + # Compute RRF scores. + rrf_scores: Dict[str, float] = defaultdict(float) + for ranked in ranked_lists: + for rank, doc_id in enumerate(ranked, start=1): + rrf_scores[doc_id] += 1.0 / (self._k + rank) + + # Sort by descending RRF score. + sorted_docs = sorted(rrf_scores.items(), key=lambda x: x[1], reverse=True) + return [doc_id for doc_id, _ in sorted_docs[:top_k]] + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def per_retriever_ranks( + self, query: str, candidate_k: int = 50 + ) -> Dict[str, Dict[str, int]]: + """ + Diagnostic helper: returns {retriever_name: {doc_id: rank}} for a query. + Useful for understanding which sub-retriever contributed each result. + """ + out: Dict[str, Dict[str, int]] = {} + for r in self._retrievers: + ranked = r.retrieve(query, top_k=candidate_k) + out[r.name] = {doc_id: rank + 1 for rank, doc_id in enumerate(ranked)} + return out + + +# ───────────────────────────────────────────────────────────────────────────── +# GRAPH-AUGMENTED RETRIEVAL +# ───────────────────────────────────────────────────────────────────────────── + + +class _ArtifactGraph: + """ + Bidirectional adjacency list extracted from corpus metadata. + + Edges are collected from: + 1. Structured relation fields (RELATION_FIELDS) — parsed as JSON or + plain lists of artifact ID strings. + 2. Artifact-ID tokens embedded in the body / title text. + + All edges are bidirectional: if doc A references doc B, we add both + A → B and B → A so that graph traversal works in both directions. + """ + + def __init__(self) -> None: + self._adj: Dict[str, Set[str]] = defaultdict(set) + + # ------------------------------------------------------------------ + # Build + # ------------------------------------------------------------------ + + def build(self, corpus: List[dict]) -> None: + """Populate the adjacency list from the corpus.""" + doc_ids: Set[str] = {r["doc_id"] for r in corpus} + + for doc in corpus: + src = doc["doc_id"] + neighbours: Set[str] = set() + + # 1. Structured relation fields. + for field in _RELATION_FIELDS: + val = doc.get(field) + if val is None: + continue + # Might be a JSON-encoded string (e.g. stored as text in parquet). + if isinstance(val, str): + try: + val = json.loads(val) + except (json.JSONDecodeError, ValueError): + # Try to extract IDs inline from the raw string. + neighbours.update(self._extract_ids_from_text(val, doc_ids)) + continue + # Might be a dict (artifact_ids maps type → id). + if isinstance(val, dict): + for v in val.values(): + if isinstance(v, str) and v in doc_ids: + neighbours.add(v) + elif isinstance(v, list): + neighbours.update(x for x in v if x in doc_ids) + elif isinstance(val, list): + for item in val: + if isinstance(item, str) and item in doc_ids: + neighbours.add(item) + + # 2. Inline artifact-ID tokens in body / title. + for text_field in ("body", "content", "title"): + text = doc.get(text_field) or "" + neighbours.update(self._extract_ids_from_text(text, doc_ids)) + + # Remove self-loops. + neighbours.discard(src) + + # Register bidirectional edges. + for tgt in neighbours: + self._adj[src].add(tgt) + self._adj[tgt].add(src) + + total_edges = sum(len(v) for v in self._adj.values()) // 2 + logger.info( + f" [Graph] Artifact graph built: " + f"{len(self._adj)} nodes, ~{total_edges} undirected edges" + ) + + # ------------------------------------------------------------------ + # Query + # ------------------------------------------------------------------ + + def neighbours(self, doc_id: str) -> Set[str]: + """Return all direct neighbours of doc_id.""" + return set(self._adj.get(doc_id, set())) + + def expand( + self, + seed_ids: List[str], + max_hops: int = MAX_HOPS, + ) -> Dict[str, int]: + """ + BFS from seed_ids up to max_hops away. + + Returns {doc_id: hop_distance} for every reachable node, + excluding the seeds themselves (hop 0). + """ + visited: Dict[str, int] = {} + frontier: Set[str] = set(seed_ids) + current_hop = 0 + + while frontier and current_hop < max_hops: + current_hop += 1 + next_frontier: Set[str] = set() + for node in frontier: + for nbr in self.neighbours(node): + if nbr not in visited and nbr not in set(seed_ids): + visited[nbr] = current_hop + next_frontier.add(nbr) + frontier = next_frontier + + return visited + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + @staticmethod + def _extract_ids_from_text(text: str, doc_ids: Set[str]) -> Set[str]: + """Extract all artifact-ID tokens from free text that exist in the corpus.""" + tokens = _ARTIFACT_ID_RE.findall(text) + return {t.upper() for t in tokens if t.upper() in doc_ids} + + +class GraphAugmentedRetriever: + """ + Wraps any base Retriever and expands its results by one or more hops + along an artifact relationship graph built from corpus metadata. + + Algorithm + --------- + retrieve(query, top_k): + 1. Ask base retriever for `candidate_k` docs → seeded set S. + 2. Expand S up to `max_hops` hops in the artifact graph. + Each hop-N neighbour receives a boost of: + boost = base_score(nearest_seed) * NEIGHBOUR_BOOST_BASE^N + where base_score is approximated as 1 / rank for the nearest + seed in S that reaches this neighbour. + 3. Merge seed scores and neighbour boosts, normalise, return top_k. + + This is particularly valuable for OrgForge's CAUSAL and TEMPORAL + question types, which require multi-artifact evidence chains that a + single-document retriever may miss. + + Parameters + ---------- + base_retriever : Retriever + Any indexable retriever (BM25, Cohere, OpenAI, RRFRetriever, …). + max_hops : int + Graph expansion depth. 1–2 recommended; ≥3 adds noise. + neighbour_boost_base : float + Multiplicative decay per hop. 0.5 means 1-hop neighbours get 50% + of the connecting seed's score, 2-hop get 25%, etc. + candidate_k : int + Seeds fetched from base retriever before graph expansion. + Should be larger than final top_k so the graph has richer seeds. + """ + + def __init__( + self, + base_retriever, + max_hops: int = MAX_HOPS, + neighbour_boost_base: float = NEIGHBOUR_BOOST_BASE, + candidate_k: int = 30, + ) -> None: + self._base = base_retriever + self._max_hops = max_hops + self._neighbour_boost_base = neighbour_boost_base + self._candidate_k = candidate_k + self._graph = _ArtifactGraph() + self.name = f"graph({base_retriever.name},hops={max_hops})" + + # ------------------------------------------------------------------ + # Retriever protocol + # ------------------------------------------------------------------ + + def index(self, corpus: List[dict]) -> None: + """Index the base retriever and build the artifact graph.""" + logger.info(f" [Graph] Indexing base retriever: {self._base.name}") + self._base.index(corpus) + logger.info(" [Graph] Building artifact relationship graph …") + self._graph.build(corpus) + + def retrieve(self, query: str, top_k: int = 10) -> List[str]: + """ + Retrieve top_k documents by combining base retriever scores with + graph-neighbour boost scores. + """ + candidate_k = max(self._candidate_k, top_k * 2) + + # Step 1: seed retrieval. + seeds: List[str] = self._base.retrieve(query, top_k=candidate_k) + if not seeds: + return [] + + # Approximate base score as 1/(rank) — monotone proxy for relevance. + seed_scores: Dict[str, float] = { + doc_id: 1.0 / (rank + 1) for rank, doc_id in enumerate(seeds) + } + + # Step 2: graph expansion. + # For each neighbour, find its minimum hop distance across all seeds, + # and use the highest-scoring seed that reaches it for the boost. + neighbour_scores: Dict[str, float] = {} + for seed_id, seed_score in seed_scores.items(): + reachable = self._graph.expand([seed_id], max_hops=self._max_hops) + for nbr_id, hop in reachable.items(): + boost = seed_score * (self._neighbour_boost_base**hop) + if boost > neighbour_scores.get(nbr_id, 0.0): + neighbour_scores[nbr_id] = boost + + # Step 3: merge seed + neighbour scores. + combined: Dict[str, float] = dict(seed_scores) + for doc_id, boost in neighbour_scores.items(): + if doc_id in combined: + # Already in seed set — add boost on top. + combined[doc_id] += boost + else: + combined[doc_id] = boost + + # Step 4: sort by descending combined score and return top_k. + sorted_docs = sorted(combined.items(), key=lambda x: x[1], reverse=True) + return [doc_id for doc_id, _ in sorted_docs[:top_k]] + + # ------------------------------------------------------------------ + # Diagnostic helpers + # ------------------------------------------------------------------ + + def explain(self, query: str, top_k: int = 10) -> List[dict]: + """ + Returns a list of dicts with retrieval provenance for each result: + { + "doc_id": str, + "combined_score": float, + "from_base": bool, # was it in the seed set? + "hop_distance": int, # 0 = seed, N = N hops away + "base_rank": int | None, # rank in base retriever (1-indexed) + } + Useful for offline debugging of graph expansion. + """ + candidate_k = max(self._candidate_k, top_k * 2) + seeds = self._base.retrieve(query, top_k=candidate_k) + seed_scores = {doc_id: 1.0 / (rank + 1) for rank, doc_id in enumerate(seeds)} + seed_rank = {doc_id: rank + 1 for rank, doc_id in enumerate(seeds)} + + neighbour_info: Dict[str, Tuple[float, int]] = {} # doc_id → (boost, hop) + for seed_id, seed_score in seed_scores.items(): + reachable = self._graph.expand([seed_id], max_hops=self._max_hops) + for nbr_id, hop in reachable.items(): + boost = seed_score * (self._neighbour_boost_base**hop) + if boost > neighbour_info.get(nbr_id, (0.0, 999))[0]: + neighbour_info[nbr_id] = (boost, hop) + + combined: Dict[str, float] = dict(seed_scores) + for doc_id, (boost, _) in neighbour_info.items(): + combined[doc_id] = combined.get(doc_id, 0.0) + boost + + sorted_docs = sorted(combined.items(), key=lambda x: x[1], reverse=True)[:top_k] + + results = [] + for doc_id, score in sorted_docs: + hop = 0 if doc_id in seed_scores else neighbour_info.get(doc_id, (0, -1))[1] + results.append( + { + "doc_id": doc_id, + "combined_score": round(score, 6), + "from_base": doc_id in seed_scores, + "hop_distance": hop, + "base_rank": seed_rank.get(doc_id), + } + ) + return results + + +# ───────────────────────────────────────────────────────────────────────────── +# build_retriever() PATCH +# ───────────────────────────────────────────────────────────────────────────── +# Copy-paste this function to REPLACE build_retriever() in eval_e2e.py. +# It adds rrf / rrf-openai / rrf-bedrock / graph-* to the existing choices. +# +# def build_retriever(name: str, region: str = "us-east-1") -> Retriever: +# from retrieval_extensions import RRFRetriever, GraphAugmentedRetriever +# +# # ── original retrievers ─────────────────────────────────────────────────── +# if name == "bm25": +# return BM25Retriever() +# if name == "cohere": +# return CohereRetriever() +# if name == "cohere-bedrock": +# return BedrockCohereRetriever(region=region) +# if name == "openai": +# return OpenAIRetriever() +# +# # ── RRF retrievers ──────────────────────────────────────────────────────── +# if name == "rrf": +# return RRFRetriever([BM25Retriever(), CohereRetriever()]) +# if name == "rrf-openai": +# return RRFRetriever([BM25Retriever(), OpenAIRetriever()]) +# if name == "rrf-bedrock": +# return RRFRetriever([BM25Retriever(), BedrockCohereRetriever(region=region)]) +# +# # ── Graph-augmented retrievers ──────────────────────────────────────────── +# if name == "graph-bm25": +# return GraphAugmentedRetriever(BM25Retriever()) +# if name == "graph-cohere": +# return GraphAugmentedRetriever(CohereRetriever()) +# if name == "graph-rrf": +# base = RRFRetriever([BM25Retriever(), CohereRetriever()]) +# return GraphAugmentedRetriever(base) +# +# raise ValueError( +# f"Unknown retriever: {name!r}. " +# "Choose bm25 | cohere | cohere-bedrock | openai | " +# "rrf | rrf-openai | rrf-bedrock | " +# "graph-bm25 | graph-cohere | graph-rrf" +# ) diff --git a/eval/scorer.py b/eval/scorer.py index e1c215e..009a5fb 100644 --- a/eval/scorer.py +++ b/eval/scorer.py @@ -21,6 +21,10 @@ PLAN dept + theme match (theme uses substring matching for LLM prose). ESCALATION escalation_actors set match, partial credit for overlap. KNOWLEDGE_GAP gap_areas set match, partial credit for overlap. + ZD_RESOLUTION resolved boolean + duration_days exact match; escalated bonus. + SF_RISK incident_id match + at_risk_accounts set overlap. + NPS_SCORE nps_score exact + classification match; escalated_tickets bonus. + INVOICE_SLA breach_duration_days exact + sla_credit_per_org within 5%. Partial credit via evidence_chain ---------------------------------- @@ -322,8 +326,6 @@ def score( else "Agent reported a departure day that doesn't exist" ) - # Temporal questions have no explicit retrieved artifacts, - # so we check evidence_chain directly. evidence = self._evidence_overlap( question.get("evidence_chain", []), agent_answer.get("retrieved_artifact_ids", []), @@ -353,13 +355,11 @@ def score( primary = 0.0 failure = f"was_actioned expected {gt_bool}, got {agent_bool}" elif not gt_bool: - # Correctly identified as not actioned — no downstream check needed primary = 1.0 failure = None else: - # Correctly identified as actioned — check downstream recall ds_overlap = self._evidence_overlap(gt_downstream, agent_downstream) - primary = 0.6 + 0.4 * ds_overlap # 0.6 floor for correct boolean + primary = 0.6 + 0.4 * ds_overlap failure = ( None if ds_overlap >= 0.5 @@ -568,9 +568,309 @@ def score( return primary, evidence, failure -# ───────────────────────────────────────────────────────────────────────────── -# DISPATCHER -# ───────────────────────────────────────────────────────────────────────────── +class PRReviewScorer(_BaseScorer): + """ + PR_REVIEW — "Who reviewed PR-X and what was the verdict?" + + Full credit: pr_id matches AND verdict matches. + Partial: pr_id correct but verdict wrong (0.5). + reviewer correct adds 0.15 bonus on top, capped at 1.0. + + Verdict is case-insensitive and normalised so "LGTM" / "approve" / + "approved" all resolve to "approved", and "changes" / "request changes" + resolve to "changes_requested" before comparison. + """ + + _APPROVE_ALIASES = {"approved", "approve", "lgtm", "merged", "merge"} + _CHANGES_ALIASES = { + "changes_requested", + "changes requested", + "request changes", + "needs changes", + "needs work", + } + + @staticmethod + def _normalise_verdict(raw: str) -> str: + v = raw.strip().lower() + if v in PRReviewScorer._APPROVE_ALIASES: + return "approved" + if v in PRReviewScorer._CHANGES_ALIASES: + return "changes_requested" + return v # return as-is — will fail comparison cleanly + + def score( + self, question: dict, agent_answer: dict + ) -> Tuple[float, float, Optional[str]]: + gt = question["ground_truth"] + gt_pr = gt.get("pr_id", "") + gt_verdict = self._normalise_verdict(gt.get("verdict", "")) + gt_reviewer = gt.get("reviewer", "").strip().lower() + + agent_pr = agent_answer.get("pr_id", "") + agent_verdict = self._normalise_verdict(agent_answer.get("verdict", "")) + agent_reviewer = agent_answer.get("reviewer", "").strip().lower() + + pr_match = agent_pr == gt_pr + + if not pr_match: + primary = 0.0 + failure = f"Expected pr_id={gt_pr!r}, got {agent_pr!r}" + elif agent_verdict == gt_verdict: + primary = 1.0 + failure = None + else: + primary = 0.5 + failure = ( + f"Correct PR but wrong verdict: expected {gt_verdict!r}, " + f"got {agent_verdict!r}" + ) + + # Reviewer identification bonus + if gt_reviewer and agent_reviewer == gt_reviewer: + primary = min(1.0, primary + 0.15) + + evidence = self._evidence_overlap( + question.get("evidence_chain", []), + agent_answer.get("retrieved_artifact_ids", []), + ) + return primary, evidence, failure + + +class ZDResolutionScorer(_BaseScorer): + """ + ZD_RESOLUTION — "Was Zendesk ticket X resolved and how long did it take?" + + Full credit: resolved boolean matches AND duration_days matches exactly. + Partial credit: boolean correct but duration wrong or missing (0.6). + escalated flag correct adds 0.1 bonus on top of partial. + """ + + def score( + self, question: dict, agent_answer: dict + ) -> Tuple[float, float, Optional[str]]: + gt = question["ground_truth"] + gt_resolved = gt.get("resolved") + gt_duration = gt.get("duration_days") + gt_escalated = gt.get("escalated", False) + + agent_resolved = agent_answer.get("resolved") + agent_duration = agent_answer.get("duration_days") + agent_escalated = agent_answer.get("escalated") + + if agent_resolved != gt_resolved: + primary = 0.0 + failure = f"resolved expected {gt_resolved}, got {agent_resolved}" + else: + # Resolution boolean correct — check duration + if gt_duration is None: + # Ticket unresolved: correct if agent also has no duration + primary = 1.0 if agent_duration is None else 0.7 + failure = ( + None + if agent_duration is None + else "Correctly identified unresolved but reported a duration" + ) + elif agent_duration is not None and int(agent_duration) == int(gt_duration): + primary = 1.0 + failure = None + elif agent_duration is not None: + primary = 0.6 + failure = ( + f"Duration off: expected {gt_duration}d, got {agent_duration}d" + ) + else: + primary = 0.6 + failure = f"Correct resolution status but missing duration (expected {gt_duration}d)" + + # Escalation awareness bonus (+0.1, capped at 1.0) + if agent_escalated is not None and agent_escalated == gt_escalated: + primary = min(1.0, primary + 0.1) + + evidence = self._evidence_overlap( + question.get("evidence_chain", []), + agent_answer.get("retrieved_artifact_ids", []), + ) + return primary, evidence, failure + + +class SFRiskScorer(_BaseScorer): + """ + SF_RISK — "Which Salesforce accounts were flagged at-risk after incident X?" + + Full credit: all at_risk_accounts matched (order-insensitive) AND + incident_id correct. + Partial: incident_id correct but incomplete account list (scaled by overlap). + 0.3 floor for incident match with zero account overlap. + """ + + def score( + self, question: dict, agent_answer: dict + ) -> Tuple[float, float, Optional[str]]: + gt = question["ground_truth"] + gt_incident = gt.get("incident_id", "") + gt_accounts = [a.lower() for a in gt.get("at_risk_accounts", [])] + + agent_incident = agent_answer.get("incident_id", "") + agent_accounts = [a.lower() for a in agent_answer.get("at_risk_accounts", [])] + + incident_match = agent_incident == gt_incident + + if not gt_accounts: + primary = 1.0 if incident_match else 0.0 + failure = ( + None + if incident_match + else f"Expected incident_id={gt_incident!r}, got {agent_incident!r}" + ) + elif not incident_match: + primary = 0.0 + failure = f"Expected incident_id={gt_incident!r}, got {agent_incident!r}" + else: + gt_set = set(gt_accounts) + agent_set = set(agent_accounts) + overlap = len(gt_set & agent_set) / len(gt_set) if gt_set else 1.0 + + if overlap == 1.0 and len(agent_set) == len(gt_set): + primary = 1.0 + failure = None + elif overlap > 0: + primary = round(0.3 + 0.7 * overlap, 4) + failure = ( + f"Partial account match ({len(gt_set & agent_set)}/{len(gt_set)}): " + f"missing {gt_set - agent_set}" + ) + else: + primary = 0.3 # floor for correct incident identification + failure = ( + f"Correct incident but no accounts matched. Expected {gt_accounts}" + ) + + evidence = self._evidence_overlap( + question.get("evidence_chain", []), + agent_answer.get("retrieved_artifact_ids", []), + ) + return primary, evidence, failure + + +class NPSScoreScorer(_BaseScorer): + """ + NPS_SCORE — "What NPS score did customer X give and what drove it?" + + Full credit: nps_score exact match AND classification correct. + Partial: classification correct but score wrong (0.6). + escalated_tickets count correct adds 0.1 bonus. + """ + + def score( + self, question: dict, agent_answer: dict + ) -> Tuple[float, float, Optional[str]]: + gt = question["ground_truth"] + gt_score = gt.get("nps_score") + gt_class = gt.get("classification", "").lower() + gt_escalated = gt.get("escalated_tickets", 0) + + agent_score = agent_answer.get("nps_score") + agent_class = agent_answer.get("classification", "").lower() + agent_escalated = agent_answer.get("escalated_tickets") + + class_match = agent_class == gt_class + score_match = agent_score is not None and int(agent_score) == int(gt_score) + + if class_match and score_match: + primary = 1.0 + failure = None + elif class_match: + primary = 0.6 + failure = f"Correct classification but wrong score: expected {gt_score}, got {agent_score}" + else: + primary = 0.0 + failure = ( + f"Classification wrong: expected {gt_class!r}, got {agent_class!r}" + ) + + # Escalation count awareness bonus + if agent_escalated is not None and int(agent_escalated) == int(gt_escalated): + primary = min(1.0, primary + 0.1) + + evidence = self._evidence_overlap( + question.get("evidence_chain", []), + agent_answer.get("retrieved_artifact_ids", []), + ) + return primary, evidence, failure + + +class InvoiceSLAScorer(_BaseScorer): + """ + INVOICE_SLA — "What SLA credit appeared on the invoice for customers + affected by incident X?" + + Full credit: breach_duration_days exact AND sla_credit_per_org within 5%. + Partial: incident identified correctly but wrong duration/credit (0.5). + Evidence: affected_orgs overlap used as secondary evidence score. + """ + + def score( + self, question: dict, agent_answer: dict + ) -> Tuple[float, float, Optional[str]]: + gt = question["ground_truth"] + gt_incident = gt.get("incident_id", "") + gt_duration = gt.get("breach_duration_days") + gt_credit = gt.get("sla_credit_per_org") + gt_orgs = [o.lower() for o in gt.get("affected_orgs", [])] + + agent_incident = agent_answer.get("incident_id", "") + agent_duration = agent_answer.get("breach_duration_days") + agent_credit = agent_answer.get("sla_credit_per_org") + agent_orgs = [o.lower() for o in agent_answer.get("affected_orgs", [])] + + if agent_incident != gt_incident: + primary = 0.0 + failure = f"Expected incident_id={gt_incident!r}, got {agent_incident!r}" + else: + duration_ok = agent_duration is not None and int(agent_duration) == int( + gt_duration + ) + credit_ok = False + if agent_credit is not None and gt_credit: + try: + ratio = abs(float(agent_credit) - float(gt_credit)) / float( + gt_credit + ) + credit_ok = ratio <= 0.05 + except (TypeError, ZeroDivisionError): + pass + + if duration_ok and credit_ok: + primary = 1.0 + failure = None + elif duration_ok: + primary = 0.7 + failure = f"Correct duration but credit off: expected {gt_credit}, got {agent_credit}" + elif credit_ok: + primary = 0.7 + failure = f"Correct credit but duration off: expected {gt_duration}d, got {agent_duration}d" + else: + primary = 0.4 # floor for correct incident identification + failure = ( + f"Correct incident but duration ({agent_duration} vs {gt_duration}) " + f"and credit ({agent_credit} vs {gt_credit}) both wrong" + ) + + # Affected orgs as evidence (secondary signal for partial credit) + evidence = ( + self._evidence_overlap( + gt_orgs, + agent_orgs, + ) + if gt_orgs + else self._evidence_overlap( + question.get("evidence_chain", []), + agent_answer.get("retrieved_artifact_ids", []), + ) + ) + return primary, evidence, failure + _SCORERS: Dict[str, _BaseScorer] = { "RETRIEVAL": RetrievalScorer(), @@ -581,12 +881,16 @@ def score( "PLAN": PlanScorer(), "ESCALATION": EscalationScorer(), "KNOWLEDGE_GAP": KnowledgeGapScorer(), - # These types were defined in the README but missing from the registry. - # POSTMORTEM and STANDUP are single-artifact lookups — RetrievalScorer is correct. - # CUSTOMER_ESC involves a causal chain — CausalScorer is the closest match. "POSTMORTEM": PostmortemScorer(), "STANDUP": RetrievalScorer(), "CUSTOMER_ESC": CausalScorer(), + "ZD_RESOLUTION": ZDResolutionScorer(), + "ZD_ESCALATION": CausalScorer(), + "SF_RISK": SFRiskScorer(), + "NPS_SCORE": NPSScoreScorer(), + "INVOICE_SLA": InvoiceSLAScorer(), + "DATADOG_ALERT": RetrievalScorer(), + "PR_REVIEW": PRReviewScorer(), } @@ -718,10 +1022,6 @@ def _mean(vals): } -# ───────────────────────────────────────────────────────────────────────────── -# CLI — quick sanity check -# ───────────────────────────────────────────────────────────────────────────── - if __name__ == "__main__": import json import pathlib @@ -735,8 +1035,6 @@ def _mean(vals): data = json.loads(eval_path.read_text()) questions = data.get("questions", []) - # Mock answers: return correct artifact_id for every RETRIEVAL question, - # random booleans for others — so we can see partial scores in action. mock_answers = {} for q in questions: gt = q["ground_truth"] @@ -768,6 +1066,43 @@ def _mean(vals): mock_answers[qid] = { "gap_areas": gt.get("gap_areas", []), } + elif qtype == "ZD_RESOLUTION": + mock_answers[qid] = { + "resolved": gt.get("resolved"), + "duration_days": gt.get("duration_days"), + "escalated": gt.get("escalated"), + } + elif qtype == "ZD_ESCALATION": + mock_answers[qid] = { + "artifact_id": gt.get("artifact_id", ""), + "event_type": gt.get("event_type", ""), + } + elif qtype == "SF_RISK": + mock_answers[qid] = { + "incident_id": gt.get("incident_id", ""), + "at_risk_accounts": gt.get("at_risk_accounts", []), + } + elif qtype == "NPS_SCORE": + mock_answers[qid] = { + "nps_score": gt.get("nps_score"), + "classification": gt.get("classification", ""), + "escalated_tickets": gt.get("escalated_tickets", 0), + } + elif qtype == "INVOICE_SLA": + mock_answers[qid] = { + "incident_id": gt.get("incident_id", ""), + "breach_duration_days": gt.get("breach_duration_days"), + "sla_credit_per_org": gt.get("sla_credit_per_org"), + "affected_orgs": gt.get("affected_orgs", []), + } + elif qtype == "DATADOG_ALERT": + mock_answers[qid] = {"artifact_id": gt.get("artifact_id", "")} + elif qtype == "PR_REVIEW": + mock_answers[qid] = { + "pr_id": gt.get("pr_id", ""), + "verdict": gt.get("verdict", ""), + "reviewer": gt.get("reviewer", ""), + } scorer = OrgForgeScorer() results = scorer.score_all(questions, mock_answers) diff --git a/src/confluence_writer.py b/src/confluence_writer.py index e68af31..46745dd 100644 --- a/src/confluence_writer.py +++ b/src/confluence_writer.py @@ -53,7 +53,7 @@ from crewai import Task, Crew from memory import Memory, SimEvent from artifact_registry import ArtifactRegistry, ConfluencePage -from utils.persona_utils import get_voice_card +from utils.persona_utils import persona_utils if TYPE_CHECKING: from graph_dynamics import GraphDynamics @@ -296,7 +296,7 @@ def write_postmortem( artifact_time, _ = self._clock.advance_actor(on_call, hours=pm_hours) timestamp = artifact_time.isoformat() - backstory = get_voice_card( + backstory = persona_utils.get_voice_card( on_call, "design", mem=self._mem, graph_dynamics=self._gd ) related = self._registry.related_context(topic=root_cause, n=3) @@ -396,7 +396,7 @@ def write_design_doc( chat_log = "\n".join(f"{m['user']}: {m['text']}" for m in slack_transcript) ctx = self._mem.recall_with_rewrite(raw_query=topic, n=3, as_of_time=timestamp) related = self._registry.related_context(topic=topic, n=3) - backstory = get_voice_card( + backstory = persona_utils.get_voice_card( author, "design", mem=self._mem, graph_dynamics=self._gd ) @@ -602,7 +602,7 @@ def write_adhoc_page( n=3, as_of_time=self._clock.now(resolved_author).isoformat(), ) - backstory = get_voice_card( + backstory = persona_utils.get_voice_card( resolved_author, "design", mem=self._mem, graph_dynamics=self._gd ) @@ -654,7 +654,7 @@ def write_adhoc_page( ctx = self._mem.context_for_prompt(title, n=3, as_of_time=timestamp) related = self._registry.related_context(topic=title, n=4) - backstory = get_voice_card( + backstory = persona_utils.get_voice_card( resolved_author, "design", mem=self._mem, graph_dynamics=self._gd ) diff --git a/src/crm_system.py b/src/crm_system.py index 94ea945..f4fc909 100644 --- a/src/crm_system.py +++ b/src/crm_system.py @@ -64,13 +64,12 @@ import json import logging -import os from datetime import datetime, timedelta from pathlib import Path import random -from typing import Any, Dict, List, Optional +from typing import Dict, List, Optional -from config_loader import CONFIG +from config_loader import COMPANY_NAME, CONFIG logger = logging.getLogger("orgforge.crm") @@ -346,7 +345,7 @@ def planner_context(self) -> str: if active_pipeline: lines.append( - f"ACTIVE SALES PIPELINE (Target these for proactive outreach!):" + "ACTIVE SALES PIPELINE (Target these for proactive outreach!):" ) for stage in [ @@ -795,6 +794,9 @@ def process_outbound_email( sender_org = email_data.get("sender_org", "") recip_org = email_data.get("recipient_org", email_data.get("to_org", "Unknown")) + if not recip_org or recip_org.lower() == COMPANY_NAME.lower(): + return None + stage = email_data.get("stage", "Prospecting") safe_org = recip_org.upper().replace(" ", "").replace("-", "") @@ -838,6 +840,7 @@ def process_outbound_email( "sender": sender, "subject": email_data.get("subject", ""), "timestamp": timestamp, + "embed_id": email_data.get("embed_id", ""), } } }, diff --git a/src/day_planner.py b/src/day_planner.py index 7213844..a687484 100644 --- a/src/day_planner.py +++ b/src/day_planner.py @@ -50,7 +50,7 @@ COMPANY_DESCRIPTION, resolve_role, ) -from utils.persona_utils import get_voice_card +from utils.persona_utils import persona_utils logger = logging.getLogger("orgforge.planner") @@ -75,7 +75,6 @@ class DepartmentPlanner: The LLM produces a JSON plan. The engine parses and validates it. """ - # Prompt template — kept here so it's easy to tune without touching logic _PLAN_PROMPT = """ You are the planning agent for the {dept} department at {company} which {company_description}. Today is Day {day} ({date}). @@ -98,28 +97,7 @@ class DepartmentPlanner: - DO write about maintaining systems, paying down tech debt, iterating on existing features, and routine corporate work. - 4. NON-ENGINEERING TEAMS (applies if {dept} is not Engineering_Backend or Engineering_Mobile). - - Do NOT propose pr_review or any code-related activities. - - ticket_progress IS allowed for non-engineering teams — it means completing an - action item (writing a doc, sending an email, running an analysis, aligning - with another team). The completion artifact is NEVER a PR. - When using ticket_progress, set related_id from the engineer's OWNED ticket list. - - DO use these activity types for your team: - * ticket_progress — completing a non-code action item tied to an owned ticket. - completion produces a confluence_page, email, or slack thread. - * deep_work — focused individual work (analysis, writing, planning) - * 1on1 — check-in with a team member - * async_question — pinging another department for info or a decision - * design_discussion — collaborative session to align on approach - * mentoring — senior helping junior - * confluence_page — writing internal documentation, playbooks, runbooks, - or process guides relevant to your team's expertise. - Use this at least once per day per department. - - Examples by department: - * Design → ticket_progress (UX spec), design_discussion, confluence_page (design system docs) - * Sales → ticket_progress (sales proposal), async_question (pinging PM), confluence_page (playbook) - * HR_Ops → ticket_progress (onboarding checklist), 1on1 (wellbeing check), confluence_page (PTO policy) - * QA_Support → ticket_progress (test plan execution), design_discussion, confluence_page (QA runbook) + {non_eng_rules} 5. NO EVENT REDUNDANCY (CRITICAL TO AVOID DUPLICATES). - NEVER put collaborative meetings (1on1, mentoring, design_discussion, async_question) in the individual agendas of BOTH participants. @@ -139,10 +117,8 @@ class DepartmentPlanner: 2. Provide a 1-sentence reasoning for the overall plan. 3. For each team member, write a 1-3 item agenda (keep descriptions under 6 words). - ## BEFORE YOU OUTPUT — verify each of these: - [ ] No collaborative meeting appears in more than one engineer's agenda - [ ] All related_ids are null or from that engineer's own owned ticket list - [ ] No engineer's estimated_hrs total exceeds their listed capacity + Verify rules 1, 2, and 5 before outputting. + ## MEETING MEDIUM RULE (design_discussion only) Choose "zoom" when ALL of the following apply: - 2 or more collaborators @@ -221,6 +197,31 @@ class DepartmentPlanner: {open_chains_str} """ + _NON_ENG_RULES = """ + 4. NON-ENGINEERING TEAMS (applies if {dept} is not Engineering_Backend or Engineering_Mobile). + - Do NOT propose pr_review or any code-related activities. + - ticket_progress IS allowed for non-engineering teams — it means completing an + action item (writing a doc, sending an email, running an analysis, aligning + with another team). The completion artifact is NEVER a PR. + When using ticket_progress, set related_id from the engineer's OWNED ticket list. + - DO use these activity types for your team: + * ticket_progress — completing a non-code action item tied to an owned ticket. + completion produces a confluence_page, email, or slack thread. + * deep_work — focused individual work (analysis, writing, planning) + * 1on1 — check-in with a team member + * async_question — pinging another department for info or a decision + * design_discussion — collaborative session to align on approach + * mentoring — senior helping junior + * confluence_page — writing internal documentation, playbooks, runbooks, + or process guides relevant to your team's expertise. + Use this at least once per day per department. + - Examples by department: + * Design → ticket_progress (UX spec), design_discussion, confluence_page (design system docs) + * Sales → ticket_progress (sales proposal), async_question (pinging PM), confluence_page (playbook) + * HR_Ops → ticket_progress (onboarding checklist), 1on1 (wellbeing check), confluence_page (PTO policy) + * QA_Support → ticket_progress (test plan execution), design_discussion, confluence_page (QA runbook) + """ + def __init__( self, dept: str, @@ -234,7 +235,7 @@ def __init__( self.members = members self.config = config self._llm = worker_llm - self.is_primary = is_primary # True for Engineering + self.is_primary = is_primary self.clock = clock def plan( @@ -333,11 +334,16 @@ def plan( agent = make_agent( role=f"{lead_name}, {self.dept} Lead", goal="Plan your team's day honestly, given your stress, their capacity, and what is actually on fire.", - backstory=get_voice_card(lead_name, "design", graph_dynamics, mem), + backstory=persona_utils.get_voice_card( + lead_name, "design", graph_dynamics, mem + ), llm=self._llm, ) prompt = self._PLAN_PROMPT.format( + non_eng_rules=self._NON_ENG_RULES.format(dept=self.dept) + if not self.is_primary + else "", dept=self.dept, company=self.config["simulation"]["company_name"], company_description=COMPANY_DESCRIPTION, @@ -595,11 +601,15 @@ def _dept_history(self, mem: Memory, day: int) -> str: it's already listed in the ACTIVE INCIDENTS section of the prompt. Non-engineering depts skip days where they had no active members. """ - summaries = [ - e - for e in mem.get_event_log() - if e.type == "day_summary" and e.day >= max(1, day - 2) - ] + window = 2 + summaries = [] + while not summaries and window <= 7: + summaries = [ + e + for e in mem.get_event_log() + if e.type == "day_summary" and e.day >= max(1, day - window) + ] + window += 1 if not summaries: return " (no recent history)" lines = [] @@ -608,7 +618,7 @@ def _dept_history(self, mem: Memory, day: int) -> str: a for a in s.facts.get("active_actors", []) if a in self.members ] if not dept_actors and not self.is_primary: - continue # dept was quiet — skip rather than add empty line + continue lines.append( f" Day {s.day}: health={s.facts.get('system_health')} " f"morale={s.facts.get('morale_trend', '?')} " @@ -794,6 +804,9 @@ class OrgCoordinator: DEPT PLANS & HEADSPACE: {other_plans_with_stress} + + CRM PRESSURE (use this to seed realistic Sales/Support ↔ Engineering collisions): + {crm_context} """ def __init__(self, config: dict, planner_llm): @@ -811,6 +824,7 @@ def coordinate( day: int, date: str, org_theme: str, + crm_context: str = "", ) -> OrgDayPlan: other_plans_str = "" @@ -834,6 +848,7 @@ def coordinate( health=state.system_health, morale_label=morale_label, all_names=self._all_names_str, + crm_context=crm_context or " (no active CRM pressure today)", ) agent = make_agent( role="Org Conflict Coordinator", @@ -1037,7 +1052,7 @@ def plan( } if non_eng_depts: - with ThreadPoolExecutor(max_workers=min(3, len(non_eng_depts))) as ex: + with ThreadPoolExecutor(max_workers=min(1, len(non_eng_depts))) as ex: futures = { ex.submit( planner.plan, @@ -1074,7 +1089,14 @@ def plan( org_theme, day, date, [] ) - org_plan = self._coordinator.coordinate(dept_plans, state, day, date, org_theme) + org_plan = self._coordinator.coordinate( + dept_plans, + state, + day, + date, + org_theme, + crm_context=crm_summary, + ) recent_summaries = self._recent_day_summaries(mem, day) all_proposed = org_plan.all_events_by_priority() @@ -1179,11 +1201,6 @@ def _generate_org_theme(self, state, mem: Memory, clock) -> str: def _extract_cross_signals( self, mem: Memory, day: int ) -> Dict[str, List[CrossDeptSignal]]: - """ - Reads recent SimEvents and produces cross-dept signals. - Engineering incidents become signals for Sales and HR. - Sales escalations become signals for Engineering. - """ signals: Dict[str, List[CrossDeptSignal]] = {} config_chart: Dict[str, List] = self._config["org_chart"] @@ -1196,11 +1213,19 @@ def _extract_cross_signals( "morale_intervention", "hr_checkin", "customer_email_routed", - "customer_escalation", "zd_tickets_escalated", "zd_tickets_resolved", "sf_deals_risk_flagged", "sf_ownership_lapsed", + "crm_touchpoint", + } + + _CRM_EVENT_SOURCE_DEPT: Dict[str, str] = { + "sf_deals_risk_flagged": self._crm_sales_dept(), + "sf_ownership_lapsed": self._crm_sales_dept(), + "crm_touchpoint": self._crm_sales_dept(), + "zd_tickets_escalated": self._crm_support_dept(), + "zd_tickets_resolved": self._crm_support_dept(), } recent = [ @@ -1210,29 +1235,55 @@ def _extract_cross_signals( ] for event in recent: + source_dept = None + for actor in event.actors: source_dept = next( (d for d, members in config_chart.items() if actor in members), None, ) - if not source_dept: - continue - - signal = CrossDeptSignal( - source_dept=source_dept, - event_type=event.type, - summary=event.summary, - day=event.day, - relevance="direct" if day - event.day <= 2 else "indirect", - ) + if source_dept: + break + + if not source_dept: + source_dept = _CRM_EVENT_SOURCE_DEPT.get(event.type) - for dept in config_chart: - if dept != source_dept: - signals.setdefault(dept, []).append(signal) - break + if not source_dept: + continue + + relevance = "direct" if day - event.day <= 2 else "indirect" + signal = CrossDeptSignal( + source_dept=source_dept, + event_type=event.type, + summary=event.summary, + day=event.day, + relevance=relevance, + ) + + for dept in config_chart: + if dept != source_dept: + signals.setdefault(dept, []).append(signal) return signals + def _crm_sales_dept(self) -> str: + """Best-guess Sales dept key from config.""" + return next( + (k for k in self._config.get("org_chart", {}) if "sales" in k.lower()), + next(iter(self._config.get("org_chart", {})), "Sales"), + ) + + def _crm_support_dept(self) -> str: + """Best-guess Support/QA dept key from config.""" + return next( + ( + k + for k in self._config.get("org_chart", {}) + if "support" in k.lower() or "qa" in k.lower() + ), + self._crm_sales_dept(), + ) + def _patch_stress_levels( self, plan: DepartmentDayPlan, diff --git a/src/email_gen.py b/src/email_gen.py deleted file mode 100644 index 5785373..0000000 --- a/src/email_gen.py +++ /dev/null @@ -1,779 +0,0 @@ -""" -email_gen.py (v3 — event-log driven, no drift) -===================================================== -Generates reflective and periodic emails AFTER the simulation completes. -All facts come exclusively from the SimEvent log written by flow.py. - -The LLM is used only for voice/prose. Facts (ticket IDs, root causes, -durations, PR numbers, dates) are injected from verified SimEvents. - -Run AFTER flow.py: - python email_gen.py - -Or import: - from email_gen import EmailGen - gen = EmailGen() - gen.run() -""" - -import os -import json -import random -import yaml -from datetime import datetime, timedelta -from email.mime.text import MIMEText -from email.mime.multipart import MIMEMultipart -from typing import List, Dict, Optional - -from rich.console import Console -from langchain_community.llms import Ollama -from crewai import Agent, Task, Crew - - -console = Console() - -# ───────────────────────────────────────────── -# CONFIG — single source of truth -# ───────────────────────────────────────────── -with open("config.yaml", "r") as f: - _CFG = yaml.safe_load(f) - -COMPANY_DOMAIN = _CFG["simulation"]["domain"] -BASE = "./export" -EMAIL_OUT = f"{BASE}/emails" -SNAPSHOT_PATH = f"{BASE}/simulation_snapshot.json" - - -# Strip the "ollama/" prefix for LangChain compatibility (same as flow.py) -def _bare_model(model_str: str) -> str: - return model_str.replace("ollama/", "").strip() - - -WORKER_MODEL = Ollama( - model=_bare_model(_CFG["models"]["worker"]), base_url=_CFG["models"]["base_url"] -) - -ORG_CHART: Dict[str, List[str]] = _CFG["org_chart"] -ALL_NAMES = [name for dept in ORG_CHART.values() for name in dept] -LEADS: Dict[str, str] = _CFG["leads"] - -# Build departed employees lookup from knowledge_gaps config -DEPARTED_EMPLOYEES: Dict[str, Dict] = { - gap["name"]: gap for gap in _CFG.get("knowledge_gaps", []) -} - -MAX_INCIDENT_THREADS = _CFG["simulation"].get("max_incident_email_threads", 3) - - -def resolve_role(role_key: str) -> str: - """Resolve a logical role to a person's name via config leads.""" - dept = _CFG.get("roles", {}).get(role_key) - if dept and dept in LEADS: - return LEADS[dept] - return next(iter(LEADS.values())) - - -def email_of(name: str) -> str: - return f"{name.lower()}@{COMPANY_DOMAIN}" - - -# ───────────────────────────────────────────── -# EML WRITER -# ───────────────────────────────────────────── -def write_eml( - path: str, - from_name: str, - to_names: List[str], - subject: str, - body: str, - cc_names: Optional[List[str]] = None, - date: Optional[str] = None, - in_reply_to: Optional[str] = None, - message_id: Optional[str] = None, -): - msg = MIMEMultipart("alternative") - msg["From"] = f"{from_name} <{email_of(from_name)}>" - msg["To"] = ", ".join(f"{n} <{email_of(n)}>" for n in to_names) - msg["Subject"] = subject - msg["Date"] = date or datetime.now().strftime("%a, %d %b %Y %H:%M:%S +0000") - msg["Message-ID"] = ( - message_id or f"<{random.randint(10000, 99999)}@{COMPANY_DOMAIN}>" - ) - if cc_names: - msg["Cc"] = ", ".join(f"{n} <{email_of(n)}>" for n in cc_names) - if in_reply_to: - msg["In-Reply-To"] = in_reply_to - msg["References"] = in_reply_to - msg.attach(MIMEText(body, "plain")) - os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, "w") as f: - f.write(msg.as_string()) - - -def write_thread(thread_dir: str, subject: str, turns: List[Dict]): - """Write a multi-turn email thread as numbered .eml files.""" - os.makedirs(thread_dir, exist_ok=True) - prev_id = None - for i, turn in enumerate(turns): - msg_id = f"<{random.randint(10000, 99999)}.{i}@{COMPANY_DOMAIN}>" - subj = subject if i == 0 else f"Re: {subject}" - filename = f"{str(i + 1).zfill(2)}_{turn['from'].lower()}.eml" - write_eml( - path=os.path.join(thread_dir, filename), - from_name=turn["from"], - to_names=turn["to"], - subject=subj, - body=turn["body"], - cc_names=turn.get("cc"), - date=turn.get("date"), - in_reply_to=prev_id, - message_id=msg_id, - ) - prev_id = msg_id - - -def _llm_body(persona_name: str, instruction: str, facts: str) -> str: - """ - Ask the LLM to write email prose. - Facts are injected as ground truth — LLM provides voice only. - """ - agent = Agent( - role=f"{persona_name} at {COMPANY_DOMAIN.split('.')[0].title()}", - goal="Write a realistic internal email in character.", - backstory=( - f"You are {persona_name}. You write in your natural voice. " - f"You MUST use ONLY the facts provided below — do not invent additional " - f"ticket IDs, dates, root causes, or people." - ), - llm=WORKER_MODEL, - ) - task = Task( - description=( - f"Write an email. Instruction: {instruction}\n\n" - f"GROUND TRUTH FACTS (use these exactly, do not change them):\n{facts}\n\n" - f"Write only the email body text. No subject line. Sign off with your name." - ), - expected_output="Plain text email body only.", - agent=agent, - ) - return str(Crew(agents=[agent], tasks=[task], verbose=False).kickoff()).strip() - - -# ───────────────────────────────────────────── -# EVENT LOG READER -# ───────────────────────────────────────────── -class EventLog: - """ - Reads the simulation_snapshot.json event_log and provides - typed accessors. This is the only source of truth for email content. - """ - - def __init__(self, snapshot_path: str = SNAPSHOT_PATH): - self.events: List[Dict] = [] - self.snapshot: Dict = {} - if os.path.exists(snapshot_path): - with open(snapshot_path) as f: - self.snapshot = json.load(f) - self.events = self.snapshot.get("event_log", []) - else: - console.print( - f"[yellow]⚠ No snapshot found at {snapshot_path}. " - f"Run flow.py first.[/yellow]" - ) - - def by_type(self, event_type: str) -> List[Dict]: - return [e for e in self.events if e["type"] == event_type] - - def by_tag(self, tag: str) -> List[Dict]: - return [e for e in self.events if tag in e.get("tags", [])] - - def by_actor(self, name: str) -> List[Dict]: - return [e for e in self.events if name in e.get("actors", [])] - - def facts(self, event_type: str) -> List[Dict]: - """Return merged facts+metadata for all events of a type.""" - return [ - e["facts"] - | { - "date": e["date"], - "day": e["day"], - "actors": e["actors"], - "artifact_ids": e["artifact_ids"], - } - for e in self.by_type(event_type) - ] - - # Convenience accessors - def incidents(self) -> List[Dict]: - return self.facts("incident_opened") - - def resolved_incidents(self) -> List[Dict]: - return self.facts("incident_resolved") - - def sprints(self) -> List[Dict]: - return self.facts("sprint_planned") - - def knowledge_gaps(self) -> List[Dict]: - return self.facts("knowledge_gap_detected") - - def retrospectives(self) -> List[Dict]: - return self.facts("retrospective") - - def morale_history(self) -> List[float]: - return self.snapshot.get("morale_history", [0.75]) - - def avg_morale(self) -> float: - h = self.morale_history() - return sum(h) / len(h) if h else 0.75 - - def final_health(self) -> int: - return self.snapshot.get("system_health", 100) - - -# ───────────────────────────────────────────── -# EMAIL GENERATOR -# ───────────────────────────────────────────── -class EmailGen: - def __init__(self, snapshot_path: str = SNAPSHOT_PATH): - self.log = EventLog(snapshot_path) - - def run(self): - console.print( - "[bold cyan]Email Generator (v3 — event-log driven)[/bold cyan]\n" - ) - n = len(self.log.events) - if n == 0: - console.print("[red]No events found. Run flow.py first.[/red]") - return - - console.print(f" Loaded {n} events from event log.\n") - - self._incident_escalation_threads() - self._sprint_weekly_syncs() - self._leadership_sync_emails() - self._knowledge_gap_threads() - self._hr_morale_emails() - self._retrospective_summaries() - self._sales_pipeline_emails() - - console.print(f"\n[green]✓ Done. Emails written to {EMAIL_OUT}/[/green]") - - # ── 1. INCIDENT ESCALATION THREADS ──────── - def _incident_escalation_threads(self): - """ - Multi-turn thread per resolved incident. - All facts (ticket ID, root cause, PR, duration) come from SimEvents. - LLM writes the prose voice only. - """ - console.print(" Generating incident escalation threads...") - resolved = self.log.resolved_incidents() - if not resolved: - console.print(" [dim]No resolved incidents yet.[/dim]") - return - - for inc in resolved[:MAX_INCIDENT_THREADS]: - ticket_id = inc["artifact_ids"].get("jira", "ORG-???") - root_cause = inc["facts"].get("root_cause", "unknown root cause") - pr_id = inc["facts"].get("pr_id", "N/A") - duration = inc["facts"].get("duration_days", "?") - inc_date = inc.get("date", "2026-03-01") - involves_bill = inc["facts"].get("involves_bill", False) - - # Resolve roles from config — no hardcoded names - on_call = resolve_role("on_call_engineer") - incident_lead = resolve_role("incident_commander") - hr_lead = resolve_role("hr_lead") - legacy_name = _CFG.get("legacy_system", {}).get("name", "legacy system") - - # A second engineering voice for the thread (first non-lead in on_call dept) - eng_dept = _CFG.get("roles", {}).get("on_call_engineer", "") - eng_members = [n for n in ORG_CHART.get(eng_dept, []) if n != on_call] - eng_peer = eng_members[0] if eng_members else on_call - - # A second product voice for the thread - prod_dept = _CFG.get("roles", {}).get("incident_commander", "") - prod_members = [ - n for n in ORG_CHART.get(prod_dept, []) if n != incident_lead - ] - prod_peer = prod_members[0] if prod_members else incident_lead - - facts_str = ( - f"Ticket: {ticket_id}\n" - f"Root cause: {root_cause}\n" - f"PR: {pr_id}\n" - f"Duration: {duration} days\n" - f"Date: {inc_date}\n" - f"Bill knowledge gap involved: {involves_bill}" - ) - - turns = [ - { - "from": on_call, - "to": [incident_lead, hr_lead], - "cc": [eng_peer], - "date": inc_date, - "body": _llm_body( - on_call, - "Open a P1 incident escalation email. Be terse. State what's broken and that you're investigating.", - facts_str, - ), - }, - { - "from": incident_lead, - "to": [on_call], - "cc": [hr_lead, prod_peer], - "date": inc_date, - "body": _llm_body( - incident_lead, - "Reply to the P1 alert. Ask for ETA. Express concern about the sprint and upcoming launch.", - facts_str, - ), - }, - { - "from": on_call, - "to": [incident_lead, hr_lead, prod_peer], - "date": inc_date, - "body": _llm_body( - on_call, - "Give the root cause update and estimated resolution time. Reference the exact root cause.", - facts_str, - ), - }, - { - "from": eng_peer, - "to": [on_call, incident_lead], - "date": inc_date, - "body": _llm_body( - eng_peer, - f"Confirm the PR is approved and being merged. " - f"{'Mention the knowledge gap and that you will update the runbook.' if involves_bill else 'Briefly summarise the fix.'}", - facts_str, - ), - }, - { - "from": on_call, - "to": [incident_lead, hr_lead, prod_peer, eng_peer], - "date": inc_date, - "body": _llm_body( - on_call, - "Send the all-clear. State that the incident is resolved. Mention postmortem.", - facts_str, - ), - }, - ] - thread_dir = f"{EMAIL_OUT}/threads/incident_{ticket_id}" - write_thread( - thread_dir, f"[P1] Incident: {ticket_id} — {legacy_name} Failure", turns - ) - console.print( - f" [green]✓[/green] {ticket_id} thread ({duration}d, PR {pr_id})" - ) - - # ── 2. SPRINT WEEKLY SYNCS ───────────────── - def _sprint_weekly_syncs(self): - """One kickoff + one mid-sprint check email per sprint, using real ticket IDs.""" - console.print(" Generating sprint sync emails...") - sprints = self.log.sprints() - if not sprints: - console.print(" [dim]No sprint events.[/dim]") - return - - for sp in sprints: - sprint_num = sp["facts"].get("sprint_number", 1) - tickets = sp["facts"].get("tickets", []) - total_pts = sp["facts"].get("total_points", 0) - sprint_goal = sp["facts"].get("sprint_goal", "Deliver sprint goals") - sp_date = sp.get("date", "2026-03-02") - actors = sp.get("actors", []) - sender = resolve_role("sprint_email_sender") - - ticket_lines = "\n".join( - f" • [{t['id']}] {t['title']} — {t['assignee']} ({t['points']}pts)" - for t in tickets - ) - facts_str = ( - f"Sprint number: {sprint_num}\n" - f"Sprint goal: {sprint_goal}\n" - f"Total points: {total_pts}\n" - f"Tickets:\n{ticket_lines}\n" - f"Date: {sp_date}" - ) - - kickoff_body = _llm_body( - sender, - "Write a sprint kickoff email to the whole team. Mention the sprint goal and ticket list.", - facts_str, - ) - write_eml( - path=f"{EMAIL_OUT}/sprint/sprint_{sprint_num}_kickoff.eml", - from_name=sender, - to_names=list(set(actors + list(LEADS.values()))), - subject=f"Sprint #{sprint_num} Kickoff", - body=kickoff_body, - date=sp_date, - ) - - mid_date = sp_date - mid_body = _llm_body( - sender, - "Write a mid-sprint check-in email to leads. Ask for status, blockers, confidence.", - facts_str, - ) - write_eml( - path=f"{EMAIL_OUT}/sprint/sprint_{sprint_num}_midpoint.eml", - from_name=sender, - to_names=list(LEADS.values()), - subject=f"Sprint #{sprint_num} Mid-Point Check", - body=mid_body, - date=mid_date, - ) - - console.print(f" [green]✓[/green] {len(sprints)} sprint(s) written.") - - # ── 3. LEADERSHIP SYNC EMAILS ───────────── - def _leadership_sync_emails(self): - """Weekly leadership sync summaries using real health/morale/incident data.""" - console.print(" Generating leadership sync emails...") - sender = resolve_role("sprint_email_sender") - morale_hist = self.log.morale_history() - resolved = self.log.resolved_incidents() - start_date = datetime.strptime(_CFG["simulation"]["start_date"], "%Y-%m-%d") - - for week in range(1, 5): - morale_idx = min(week * 4, len(morale_hist) - 1) - morale = morale_hist[morale_idx] - sync_date = start_date + timedelta(weeks=week - 1, days=2) - # incidents resolved up to this week - inc_this_wk = [r for r in resolved if r.get("day", 0) <= week * 5] - inc_summary = ( - ", ".join( - f"{r['artifact_ids'].get('jira', '?')} ({r['facts'].get('root_cause', '?')[:80]})" - for r in inc_this_wk[-2:] - ) - if inc_this_wk - else "none" - ) - - facts_str = ( - f"Week: {week}\n" - f"Date: {sync_date.strftime('%Y-%m-%d')}\n" - f"System health: {min(100, 60 + week * 10)}/100\n" - f"Team morale: {morale:.2f}\n" - f"Incidents resolved this period: {inc_summary}\n" - f"Morale flag: {'LOW - HR action needed' if morale < 0.5 else 'healthy'}" - ) - body = _llm_body( - sender, - "Write a weekly leadership sync summary email to all department leads. " - "Cover engineering, product, sales, and team morale. End with 3 action items.", - facts_str, - ) - write_eml( - path=f"{EMAIL_OUT}/leadership/week_{week}_sync.eml", - from_name=sender, - to_names=list(LEADS.values()), - subject=f"Week {week} Leadership Sync — Notes & Actions", - body=body, - date=sync_date.strftime("%a, %d %b %Y 15:00:00 +0000"), - ) - - console.print(" [green]✓[/green] 4 leadership sync emails.") - - # ── 4. KNOWLEDGE GAP THREADS ─────────────── - def _knowledge_gap_threads(self): - """Thread about departed-employee knowledge gaps. - - Checks for explicit `knowledge_gap_detected` events first (emitted by - flow.py when an incident touches a gap area). Falls back to - scanning `incident_opened` events for the `involves_bill` flag so the - thread is generated even if the dedicated event was never fired. - """ - console.print(" Generating knowledge gap threads...") - - gaps = self.log.facts("knowledge_gap_detected") - - # Fallback: derive gap info from incident_opened events that flagged involves_bill - if not gaps: - bill_incidents = [ - e - for e in self.log.by_type("incident_opened") - if e.get("facts", {}).get("involves_bill") - ] - if bill_incidents: - inc = bill_incidents[0] - _legacy = _CFG.get("legacy_system", {}).get("name", "legacy system") - gaps = [ - { - "facts": {"gap_area": [_legacy], "involves_bill": True}, - "artifact_ids": inc.get("artifact_ids", {}), - "date": inc.get("date", _CFG["simulation"]["start_date"]), - "day": inc.get("day", 1), - "actors": inc.get("actors", []), - } - ] - - if not gaps: - console.print(" [dim]No knowledge gap events — skipping.[/dim]") - return - - gap = gaps[0] - gap_area = gap["facts"].get( - "gap_area", [_CFG.get("legacy_system", {}).get("name", "legacy system")] - ) - ticket = gap["artifact_ids"].get("jira", "ORG-???") - gap_date = gap.get("date", "2026-03-05") - - # Pull departed employee details from config for richer context - departed_info = next(iter(DEPARTED_EMPLOYEES.values()), {}) - departed_name = departed_info.get("name", "Bill") - departed_role = departed_info.get("role", "ex-CTO") - departed_left = departed_info.get("left", "unknown") - doc_pct = int(departed_info.get("documented_pct", 0.2) * 100) - - # Resolve participants from config roles - on_call = resolve_role("on_call_engineer") - incident_lead = resolve_role("incident_commander") - hr_lead = resolve_role("hr_lead") - eng_dept = _CFG.get("roles", {}).get("on_call_engineer", "") - eng_peers = [n for n in ORG_CHART.get(eng_dept, []) if n != on_call] - eng_peer = eng_peers[0] if eng_peers else on_call - legacy_name = _CFG.get("legacy_system", {}).get("name", "legacy system") - - facts_str = ( - f"Incident that triggered gap discovery: {ticket}\n" - f"Systems with missing documentation: {gap_area}\n" - f"Departed employee: {departed_name} ({departed_role}, left {departed_left})\n" - f"Documented percentage: ~{doc_pct}%\n" - f"Date discovered: {gap_date}" - ) - turns = [ - { - "from": on_call, - "to": [incident_lead, hr_lead, eng_peer], - "date": gap_date, - "body": _llm_body( - on_call, - f"Raise the alarm about a critical knowledge gap in systems {departed_name} owned. " - "Propose a 'knowledge excavation sprint'.", - facts_str, - ), - }, - { - "from": incident_lead, - "to": [on_call, hr_lead, eng_peer], - "date": gap_date, - "body": _llm_body( - incident_lead, - "Respond to the alarm. Acknowledge the risk but push back on a full sprint pause. " - "Counter-propose 20% time allocation.", - facts_str, - ), - }, - { - "from": hr_lead, - "to": [incident_lead, on_call, eng_peer], - "date": gap_date, - "body": _llm_body( - hr_lead, - f"Propose a Confluence documentation template. Suggest reaching out to {departed_name} for a paid consulting session.", - facts_str, - ), - }, - { - "from": on_call, - "to": [hr_lead, incident_lead], - "date": gap_date, - "body": _llm_body( - on_call, - f"Confirm you and {eng_peer} have started the {legacy_name} documentation page. Agree to reach out to {departed_name}.", - facts_str, - ), - }, - ] - write_thread( - f"{EMAIL_OUT}/threads/knowledge_gap_{departed_name.lower()}", - f"Knowledge Gap: {departed_name}'s Systems — Action Plan", - turns, - ) - console.print( - f" [green]✓[/green] Knowledge gap thread written ({departed_name} / {gap_area})." - ) - - # ── 5. HR MORALE EMAILS ──────────────────── - def _hr_morale_emails(self): - """HR emails driven by actual morale data from the event log.""" - console.print(" Generating HR emails...") - avg = self.log.avg_morale() - - hr_lead = resolve_role("hr_lead") - prod_lead = resolve_role("sprint_email_sender") - on_call = resolve_role("on_call_engineer") - new_hire = ( - _CFG["simulation"].get("new_hire") - or ( - ORG_CHART.get(_CFG["simulation"].get("new_hire_dept", "Product"), [""])[ - 0 - ] - ) - ) - new_hire_dept = _CFG["simulation"].get("new_hire_dept", "Product") - legacy_name = _CFG.get("legacy_system", {}).get("name", "legacy system") - legacy_proj = _CFG.get("legacy_system", {}).get("project_name", legacy_name) - company = _CFG["simulation"]["company_name"] - avg = self.log.avg_morale() - - write_eml( - path=f"{EMAIL_OUT}/hr/welcome_{new_hire.lower()}.eml", - from_name=hr_lead, - to_names=[new_hire], - cc_names=[prod_lead], - subject=f"Welcome to {company}, {new_hire}!", - body=( - f"Hi {new_hire},\n\nWe're so glad to have you on the {new_hire_dept} team!\n\n" - f"Your first week includes team introductions ({prod_lead} will set these up), " - f"JIRA and Confluence access (Tom will send credentials), and an engineering " - f"onboarding session with {on_call} on Thursday.\n\n" - f"One heads-up: our legacy system ({legacy_proj} / {legacy_name}) is in a transition phase. " - f"Don't be alarmed if you see incident tickets — we're actively stabilising it.\n\n" - f"Welcome!\n\n{hr_lead}" - ), - date=f"Mon, {_CFG['simulation']['start_date'].replace('-', ' ').split()[2]} {_CFG['simulation']['start_date'].split('-')[1]} {_CFG['simulation']['start_date'].split('-')[0]} 08:30:00 +0000", - ) - - intervention_threshold = _CFG["morale"].get("intervention_threshold", 0.55) - if avg < intervention_threshold: - facts_str = f"Average team morale during simulation: {avg:.2f} (scale 0-1, threshold {intervention_threshold})" - body = _llm_body( - hr_lead, - "Write a warm, non-alarmist check-in email to team leads about low morale. " - "Offer 1:1s and remind them of the EAP. Don't mention the metric directly.", - facts_str, - ) - write_eml( - path=f"{EMAIL_OUT}/hr/morale_intervention.eml", - from_name=hr_lead, - to_names=list(LEADS.values()), - subject="Team Pulse — Let's Talk", - body=body, - ) - console.print( - f" [yellow]Morale intervention email written (avg={avg:.2f})[/yellow]" - ) - - start_year = _CFG["simulation"]["start_date"].split("-")[0] - write_eml( - path=f"{EMAIL_OUT}/hr/remote_policy_{start_year}.eml", - from_name=hr_lead, - to_names=ALL_NAMES, - subject=f"Updated: Remote Work Policy {start_year}", - body=( - f"Team,\n\nWe've updated our remote work policy for {start_year}. Key changes:\n\n" - " • Core hours: 10am–3pm in your local timezone\n" - " • Monthly in-person anchor day (first Monday)\n" - " • All-hands meetings: Tuesdays 2pm EST\n\n" - f"Full policy on Confluence. Please review and acknowledge by Friday.\n\n{hr_lead}" - ), - ) - console.print(" [green]✓[/green] HR emails written.") - - # ── 6. RETROSPECTIVE SUMMARIES ───────────── - def _retrospective_summaries(self): - """Post-retro summary emails to all leads, referencing the actual retro Confluence page.""" - console.print(" Generating retrospective summaries...") - retros = self.log.retrospectives() - if not retros: - console.print(" [dim]No retrospective events.[/dim]") - return - - sender = resolve_role("sprint_email_sender") - for retro in retros: - sprint_num = retro["facts"].get("sprint_number", 1) - conf_id = retro["artifact_ids"].get("confluence", "CONF-RETRO-???") - resolved = retro["facts"].get("resolved_incidents", []) - retro_date = retro.get("date", "2026-03-06") - - facts_str = ( - f"Sprint number: {sprint_num}\n" - f"Retrospective page: {conf_id}\n" - f"Incidents resolved this sprint: {resolved}\n" - f"Date: {retro_date}" - ) - body = _llm_body( - sender, - "Send a brief post-retro summary to all leads. Reference the Confluence page ID. " - "List 2-3 key takeaways and thank the team.", - facts_str, - ) - write_eml( - path=f"{EMAIL_OUT}/retros/sprint_{sprint_num}_retro_summary.eml", - from_name=sender, - to_names=list(LEADS.values()), - subject=f"Sprint #{sprint_num} Retro Summary — {conf_id}", - body=body, - date=retro_date, - ) - - console.print(f" [green]✓[/green] {len(retros)} retro summaries.") - - # ── 7. SALES PIPELINE EMAILS ─────────────── - def _sales_pipeline_emails(self): - """Sales pipeline emails — uses real health/incident data for context.""" - console.print(" Generating sales pipeline emails...") - resolved = self.log.resolved_incidents() - start_date = datetime.strptime(_CFG["simulation"]["start_date"], "%Y-%m-%d") - accounts = _CFG.get( - "sales_accounts", ["Acme Corp", "Beta LLC", "Gamma Inc", "Delta Co"] - ) - sender = resolve_role("sales_email_sender") - prod_lead = resolve_role("sprint_email_sender") - # Second person on product to copy - prod_dept = _CFG.get("roles", {}).get("sprint_email_sender", "") - prod_peers = [n for n in ORG_CHART.get(prod_dept, []) if n != prod_lead] - prod_peer = prod_peers[0] if prod_peers else prod_lead - # Sales dept peers to CC - sales_dept = _CFG.get("roles", {}).get("sales_email_sender", "") - sales_peers = [n for n in ORG_CHART.get(sales_dept, []) if n != sender][:2] - - for week in range(1, 4): - deals = random.sample(accounts, min(3, len(accounts))) - report_date = start_date + timedelta(weeks=week - 1, days=4) - inc_this_wk = [r for r in resolved if r.get("day", 0) <= week * 5] - stability = ( - "stable" - if not inc_this_wk - else f"recovering ({len(inc_this_wk)} incident(s) this period)" - ) - - facts_str = ( - f"Week: {week}\n" - f"Deals in pipeline: {', '.join(deals)}\n" - f"Platform stability status: {stability}\n" - f"Q1 attainment estimate: {50 + week * 12}%\n" - f"Date: {report_date.strftime('%Y-%m-%d')}" - ) - body = _llm_body( - sender, - f"Write a weekly pipeline update to {prod_lead} and {prod_peer}. " - "Mention deal statuses, attainment, and any platform stability concerns. " - "Keep it punchy.", - facts_str, - ) - write_eml( - path=f"{EMAIL_OUT}/sales/week_{week}_pipeline.eml", - from_name=sender, - to_names=[prod_lead, prod_peer], - cc_names=sales_peers, - subject=f"Week {week} Sales Pipeline Update", - body=body, - date=report_date.strftime("%a, %d %b %Y 17:00:00 +0000"), - ) - - console.print(" [green]✓[/green] 3 pipeline emails.") - - -# ───────────────────────────────────────────── -# ENTRY POINT -# ───────────────────────────────────────────── -if __name__ == "__main__": - gen = EmailGen() - gen.run() diff --git a/src/embed_worker.py b/src/embed_worker.py index 87d22da..08eebb4 100644 --- a/src/embed_worker.py +++ b/src/embed_worker.py @@ -3,13 +3,15 @@ =============== Background embedding queue for OrgForge. -Decouples artifact embedding from LLM generation so Stella/Ollama inference +Decouples artifact embedding from LLM generation so Infinity/Ollama inference runs while the next Bedrock call is in flight, rather than blocking between each generation step. Architecture ------------ -- A single daemon thread consumes embed tasks from a Queue. +- A ThreadPoolExecutor processes embed tasks concurrently from a Queue. +- Concurrency is tuned via EMBED_WORKER_CONCURRENCY (default: 8 for Infinity, + 1 for Ollama which is serial anyway). - The main sim loop calls enqueue() instead of mem.embed_artifact() directly. - Before any vector search (context_for_prompt, recall, search_events) the caller must call drain() to flush pending embeds — this ensures causal @@ -40,60 +42,88 @@ def _embed_and_count(self, **kwargs): - mem.embed_artifact() writes to MongoDB via PyMongo, which is thread-safe. - daily_artifacts_created is incremented on the main thread (in enqueue), so counts remain accurate without locking. +- _errors is guarded by _errors_lock for concurrent appends from the pool. """ from __future__ import annotations import logging +import os import threading +from concurrent.futures import ThreadPoolExecutor, Future from queue import Queue, Empty -from typing import Any, Dict +from typing import Any, Dict, List logger = logging.getLogger("orgforge.embed_worker") _SENTINEL = None +# How many embed HTTP calls to run in parallel. +# - Infinity: 8–16 is a good starting point on a Xeon 6975P; Infinity's +# dynamic batching coalesces concurrent requests server-side so +# you get throughput gains without hammering the network. +# - Ollama: Keep at 1 — Ollama serialises embeds internally anyway and +# parallel requests just queue up in its HTTP layer. +_DEFAULT_CONCURRENCY = int(os.environ.get("EMBED_WORKER_CONCURRENCY", "8")) + class EmbedWorker: """ - Single background thread that drains an embed task queue. + Concurrent background worker that drains an embed task queue using a + thread pool. Works with both Ollama (concurrency=1) and Infinity + (concurrency=8+). Parameters ---------- mem : Memory - The shared Memory instance. embed_artifact() is called on it from the - worker thread — PyMongo handles connection pooling safely. + The shared Memory instance. _embed() is called on it from the worker + threads — PyMongo handles connection pooling safely. + concurrency : int + Number of concurrent embed calls. Set via EMBED_WORKER_CONCURRENCY + env var or passed directly. Default: 8. maxsize : int Maximum queue depth before enqueue() blocks the caller. Default 0 (unbounded) is correct for OrgForge since the LLM is always slower - than embedding and we never want the sim to stall on the queue. + than embedding. """ - def __init__(self, mem, maxsize: int = 0): + def __init__(self, mem, concurrency: int = _DEFAULT_CONCURRENCY, maxsize: int = 0): self._mem = mem + self._concurrency = concurrency self._queue: Queue[Dict[str, Any] | None] = Queue(maxsize=maxsize) - self._thread = threading.Thread( - target=self._consume, - name="embed-worker", + self._executor = ThreadPoolExecutor( + max_workers=concurrency, + thread_name_prefix="embed-pool", + ) + self._dispatcher = threading.Thread( + target=self._dispatch_loop, + name="embed-dispatcher", daemon=True, ) + self._futures: List[Future] = [] + self._futures_lock = threading.Lock() self._errors: list[Exception] = [] + self._errors_lock = threading.Lock() def start(self) -> None: - """Start the background consumer thread. Call once from Flow.__init__.""" - self._thread.start() - logger.info("[embed_worker] Background embed queue started.") + """Start the background dispatcher thread. Call once from Flow.__init__.""" + self._dispatcher.start() + logger.info( + f"[embed_worker] Background embed queue started " + f"(concurrency={self._concurrency})." + ) def stop(self) -> None: """ - Flush remaining tasks then shut down the consumer thread cleanly. + Flush remaining tasks then shut down cleanly. Call after the simulation loop exits. """ self.drain() self._queue.put(_SENTINEL) - self._thread.join(timeout=60) - if self._thread.is_alive(): - logger.warning("[embed_worker] Consumer thread did not exit within 60s.") + self._dispatcher.join(timeout=60) + self._executor.shutdown(wait=True, cancel_futures=False) + if self._dispatcher.is_alive(): + logger.warning("[embed_worker] Dispatcher thread did not exit within 60s.") else: logger.info("[embed_worker] Background embed queue stopped cleanly.") @@ -103,12 +133,14 @@ def enqueue(self, **kwargs) -> None: Accepts the same keyword arguments as Memory.embed_artifact(): id, type, title, content, day, date, timestamp, metadata + Plus the internal routing key: + _target: "artifacts" (default) or "events" """ self._queue.put(kwargs) def drain(self) -> None: """ - Block until all currently queued embed tasks are complete. + Block until all currently queued and in-flight embed tasks are complete. Call this: - Before any vector search (recall, context_for_prompt, search_events) @@ -118,17 +150,33 @@ def drain(self) -> None: After drain() returns, MongoDB is consistent with all enqueued artifacts. Any errors accumulated during background processing are logged here. """ + # Wait for queue to be fully dispatched to the thread pool self._queue.join() + # Wait for all in-flight futures (tasks running in the pool right now) + with self._futures_lock: + futures_snapshot = list(self._futures) + + for fut in futures_snapshot: + try: + fut.result() + except Exception as exc: + with self._errors_lock: + self._errors.append(exc) + + with self._futures_lock: + self._futures.clear() + if self._errors: - for err in self._errors: - logger.error(f"[embed_worker] Background embed error: {err}") - self._errors.clear() + with self._errors_lock: + for err in self._errors: + logger.error(f"[embed_worker] Background embed error: {err}") + self._errors.clear() - def _consume(self) -> None: + def _dispatch_loop(self) -> None: """ - Worker thread body. Runs until it receives the sentinel value. - Each task is a kwargs dict for mem.embed_artifact(). + Dispatcher thread body. Pulls tasks off the queue and submits them to + the thread pool. Runs until it receives the sentinel value. """ while True: try: @@ -140,39 +188,55 @@ def _consume(self) -> None: self._queue.task_done() break - try: - target = task.pop("_target", "artifacts") - if target == "events": - text = task["content"] - vector = self._mem._embed( - text, - input_type="search_document", - caller="log_event_async", - doc_id=task["id"], - doc_type=task["type"], - ) + future = self._executor.submit(self._process_task, task) + with self._futures_lock: + # Prune completed futures to avoid unbounded list growth + self._futures = [f for f in self._futures if not f.done()] + self._futures.append(future) + + # Mark the queue slot as done immediately after dispatch — + # drain() waits on futures directly for in-flight completion. + self._queue.task_done() + + def _process_task(self, task: Dict[str, Any]) -> None: + """ + Executed in a pool thread. Calls the embedder and writes to MongoDB. + This is where actual HTTP calls to Infinity/Ollama happen. + """ + try: + target = task.pop("_target", "artifacts") + + if target == "events": + text = task["content"] + vector = self._mem._embed( + text, + input_type="search_document", + caller="log_event_async", + doc_id=task["id"], + doc_type=task["type"], + ) + if vector: self._mem._events.update_one( {"_id": task["id"]}, {"$set": {"embedding": vector}}, ) - else: - embed_text = task["content"] - vector = self._mem._embed( - embed_text, - input_type="search_document", - caller="embed_artifact_async", - doc_id=task["id"], - doc_type=task["type"], + else: + embed_text = task["content"] + vector = self._mem._embed( + embed_text, + input_type="search_document", + caller="embed_artifact_async", + doc_id=task["id"], + doc_type=task["type"], + ) + if vector: + self._mem._artifacts.update_one( + {"_id": task["id"]}, + {"$set": {"embedding": vector}}, ) - if vector: - self._mem._artifacts.update_one( - {"_id": task["id"]}, - {"$set": {"embedding": vector}}, - ) - except Exception as exc: + except Exception as exc: + with self._errors_lock: self._errors.append(exc) - logger.warning( - f"[embed_worker] embed failed for id={task.get('id')}: {exc}" - ) - finally: - self._queue.task_done() + logger.warning( + f"[embed_worker] embed failed for id={task.get('id')}: {exc}" + ) diff --git a/src/external_email_ingest.py b/src/external_email_ingest.py index 87b32d2..bc330b7 100644 --- a/src/external_email_ingest.py +++ b/src/external_email_ingest.py @@ -2,41 +2,6 @@ external_email_ingest.py ======================== Inbound and outbound email generation during the simulation. - -Three email categories, each with distinct routing and causal tracing: - - 1. TECH VENDOR INBOUND (AWS, Stripe, Snyk, etc.) - Arrive pre-standup (06:00–08:59). Routed to the Engineering member - whose expertise best overlaps the topic — not just the dept lead. - May produce a JIRA task. Appended to any live incident's causal chain - if the topic overlaps the root cause. - - 2. CUSTOMER / CLIENT INBOUND - Arrive during business hours (09:00–16:30). Routed through the - product gatekeeper chain: - customer email → Sales Slack ping → Product decision → optional JIRA - ~15 % of customer emails are dropped (no action taken). These are - logged as "email_dropped" SimEvents — an eval agent should detect the - gap between the email artifact and the absence of any downstream work. - - 3. HR OUTBOUND (offer letters, onboarding prep) - Fired 1–3 days before a scheduled hire arrives. Karen (HR lead) sends - to the prospect. Logged as an artifact and linked into the - employee_hired causal chain on arrival day. - -Causal tracing --------------- -Every email is assigned an embed_id and rooted in a CausalChainHandler. -Each downstream artifact (Slack thread, JIRA ticket) is appended in order. -Chain snapshots are written into every SimEvent's facts so the eval harness -can reconstruct the full thread — or notice when it terminates early. - -Source generation ------------------ -Sources are generated once by LLM during genesis (same pattern as -generate_tech_stack in confluence_writer.py), persisted to -sim_config["inbound_email_sources"], and loaded on every subsequent run. -No config.yaml entries required. """ from __future__ import annotations @@ -56,10 +21,11 @@ from config_loader import COMPANY_DESCRIPTION from crm_system import NullCRMSystem from crewai import Crew, Task +from graph_dynamics import GraphDynamics import json_repair from memory import Memory, SimEvent from insider_threat import _NullInjector -from utils.persona_utils import get_voice_card +from utils.persona_utils import persona_utils logger = logging.getLogger("orgforge.external_email") @@ -72,6 +38,50 @@ _PROB_CUSTOMER_JIRA = 0.55 # high-priority customer complaint → JIRA _PROB_VENDOR_JIRA = 0.45 # vendor alert → JIRA task _HR_EMAIL_WINDOW = (1, 3) # days before hire arrival to send email +_VALID_EMAIL_TYPES = frozenset( + ["complaint", "question", "feature_request", "positive_feedback", "general_inquiry"] +) + +_VALID_CRM_STAGES = frozenset( + [ + "Prospecting", + "Value Proposition", + "Proposal/Price Quote", + "Negotiation/Review", + "Closed Won", + "Closed Lost", + ] +) + + +_STAGE_RANK = { + "Prospecting": 1, + "Value Proposition": 2, + "Proposal/Price Quote": 3, + "Negotiation/Review": 4, + "Closed Won": 5, + "Closed Lost": 0, +} + + +def _get_stage_probability(stage: str) -> int: + return { + "Prospecting": 10, + "Value Proposition": 25, + "Proposal/Price Quote": 50, + "Negotiation/Review": 75, + "Closed Won": 100, + "Closed Lost": 0, + }.get(stage, 10) + + +_PROB_CUSTOMER_REPLY = 0.30 + + +_PROB_NON_COMPLAINT_SALES_FYI = 0.35 + + +_COMPLAINT_EMAIL_TYPES = frozenset(["complaint"]) @dataclass @@ -133,6 +143,7 @@ def __init__( clock, threat_injector=None, crm=None, + graph_dynamics=GraphDynamics, ): self._config = config self._mem = mem @@ -155,6 +166,7 @@ def __init__( self._threat = threat_injector or _NullInjector() self._crm = crm or NullCRMSystem() self._sources = None + self._gd = graph_dynamics self._scheduled_hires: Dict[int, List[dict]] = {} for hire in config.get("org_lifecycle", {}).get("scheduled_hires", []): @@ -312,8 +324,8 @@ def _sales_pings_product( participants = [sales_lead, product_lead] ping_time, _ = self._clock.sync_and_advance(participants, hours=0.25) - backstory = get_voice_card( - sales_lead, "async", graph_dynamics=None, mem=self._mem + backstory = persona_utils.get_voice_card( + sales_lead, "async", self._gd, mem=self._mem ) p = self._personas.get(sales_lead, {}) @@ -390,8 +402,8 @@ def _product_opens_jira( self._registry.register_jira(ticket_id) jira_time, _ = self._clock.sync_and_advance([product_lead], hours=0.3) - backstory = get_voice_card( - product_lead, "async", graph_dynamics=None, mem=self._mem + backstory = persona_utils.get_voice_card( + product_lead, "async", self._gd, mem=self._mem ) p = self._personas.get(product_lead, {}) @@ -599,7 +611,9 @@ def _send_hr_outbound(self, hire, hr_lead, days_until, state, date_str) -> None: hr_time, _ = self._clock.sync_and_advance([hr_lead], hours=0.5) - backstory = get_voice_card(hr_lead, "async", graph_dynamics=None, mem=self._mem) + backstory = persona_utils.get_voice_card( + hr_lead, "async", self._gd, mem=self._mem + ) p = self._personas.get(hr_lead, {}) agent = make_agent( @@ -632,6 +646,8 @@ def _send_hr_outbound(self, hire, hr_lead, days_until, state, date_str) -> None: body=body, timestamp_iso=hr_time.isoformat(), direction="outbound", + embed_id=embed_id, + day=date_str, ) _exfil_path = self._threat.inject_email( @@ -726,8 +742,8 @@ def _send_customer_reply( else "This is routine — thank them, confirm receipt, and say the team will be in touch." ) - backstory = get_voice_card( - sales_lead, "async", graph_dynamics=None, mem=self._mem + backstory = persona_utils.get_voice_card( + sales_lead, "async", self._gd, mem=self._mem ) p = self._personas.get(sales_lead, {}) @@ -793,6 +809,8 @@ def _send_customer_reply( body=body, timestamp_iso=reply_time.isoformat(), direction="outbound", + embed_id=embed_id, + day=state.day, ) self._crm.process_outbound_email( @@ -803,6 +821,7 @@ def _send_customer_reply( "recipient_org": signal.source_org, "subject": subject, "stage": crm_stage, + "embed_id": embed_id, }, timestamp=reply_time.isoformat(), date_str=date_str, @@ -902,8 +921,8 @@ def _send_vendor_ack( else "No ticket number yet — just say it is being investigated." ) - backstory = get_voice_card( - recipient, "async", graph_dynamics=None, mem=self._mem + backstory = persona_utils.get_voice_card( + recipient, "async", self._gd, mem=self._mem ) p = self._personas.get(recipient, {}) @@ -948,6 +967,8 @@ def _send_vendor_ack( body=body, timestamp_iso=ack_time.isoformat(), direction="outbound", + embed_id=embed_id, + day=state.day, ) _exfil_path = self._threat.inject_email( @@ -1047,10 +1068,6 @@ def _log_dropped_email(self, signal: ExternalEmailSignal, state) -> None: ) ) - # ───────────────────────────────────────────────────────────────────────── - # CORE EMAIL GENERATION - # ───────────────────────────────────────────────────────────────────────── - def _generate_email( self, source: dict, @@ -1058,8 +1075,11 @@ def _generate_email( state, hour_range: Tuple[int, int], category: str, - ) -> Optional[ExternalEmailSignal]: - source_name = source["name"] + ) -> Optional[Any]: + + source_first_name = source["first_name"] + source_name = source_first_name + source_last_name = source["last_name"] source_org = source.get("org", source_name) source_addr = source.get("email", f"contact@{source_name.lower()}.com") liaison_dept = source.get("internal_liaison", list(self._leads.keys())[0]) @@ -1075,7 +1095,6 @@ def _generate_email( f"Reference naturally if relevant." ) - # 1. Fetch the actual company tech stack from memory tech_stack = self._mem.tech_stack_for_prompt() tech_ctx = ( ( @@ -1093,18 +1112,19 @@ def _generate_email( second=random.randint(0, 59), ) + backstory = persona_utils.get_voice_card( + source_first_name, "async", self._gd, mem=self._mem, internal=False + ) + agent = make_agent( role=f"Representative from {source_org}", goal=f"Write a realistic email about: {topic}.", - backstory=( - f"You represent {source_org}. Tone: {tone}. " - f"Specific, concise. Never break character." - ), + backstory=backstory, llm=self._worker_llm, ) task = Task( description=( - f"Email from {source_org} to {liaison_name} at {self._company_name} " + f"Email from {source_first_name} {source_last_name} at {source_org} to {liaison_name} at {self._company_name} which {COMPANY_DESCRIPTION} " f"about: {topic}.\nTone: {tone}. Health: {state.system_health}/100." f"{incident_ctx}" f"{tech_ctx}\n\n" @@ -1132,7 +1152,7 @@ def _generate_email( eml_path = self._write_eml( date_str=date_str, - from_name=source_name, + from_name=f"{source_first_name} {source_last_name}", from_addr=source_addr, to_name=liaison_name, to_addr=self._email_of(liaison_name), @@ -1185,29 +1205,44 @@ def _generate_email( ) zd_ticket_id = None + email_type = "general_inquiry" + if category == "customer": - zd_ticket_id = self._crm.handle_inbound_complaint( - event_facts={ - "subject": subject, - "body": body[:500], - "sender_org": source_org, - }, - timestamp=email_ts.isoformat(), - date_str=date_str, - day=state.day, + email_type = self._classify_customer_email( + subject=subject, + body=body, + source_name=source_name, + tone=tone, ) - if zd_ticket_id: - logger.info( - f" [dim]🔗 ZD ticket {zd_ticket_id} linked to inbound complaint[/dim]" + logger.debug( + f" [dim]🔍 Email classified as '{email_type}': " + f"{source_name} — {subject[:50]}[/dim]" + ) + + if email_type in _COMPLAINT_EMAIL_TYPES: + zd_ticket_id = self._crm.handle_inbound_complaint( + event_facts={ + "subject": subject, + "body": body[:500], + "sender_org": source_org, + }, + timestamp=email_ts.isoformat(), + date_str=date_str, + day=state.day, ) + if zd_ticket_id: + logger.info( + f" [dim]🔗 ZD ticket {zd_ticket_id} linked to complaint from " + f"{source_name}[/dim]" + ) - artifact_ids = {"email": embed_id, "eml_path": str(eml_path)} + artifact_ids: Dict[str, Any] = {"email": embed_id, "eml_path": str(eml_path)} if zd_ticket_id: artifact_ids["zd_ticket"] = zd_ticket_id body_preview = body[:200].rstrip() + ("…" if len(body) > 200 else "") - return ExternalEmailSignal( + signal = ExternalEmailSignal( source_name=source_name, source_org=source_org, source_email=source_addr, @@ -1225,6 +1260,674 @@ def _generate_email( facts={"subject": subject, "topic": topic, "org": source_org}, ) + signal.facts["email_type"] = email_type + if zd_ticket_id: + signal.facts["zd_ticket_id"] = zd_ticket_id + + return signal + + def _classify_customer_email( + self, + subject: str, + body: str, + source_name: str, + tone: str, + ) -> str: + """ + Classify an inbound customer email into one of five categories using a + single, lightweight LLM call. + """ + agent = make_agent( + role="Email Classifier", + goal="Classify a customer email into exactly one category.", + backstory=( + "You are a triage assistant. You read customer emails and output " + "a single classification label. You never explain your reasoning." + ), + llm=self._worker_llm, + ) + task = Task( + description=( + f"Classify the following customer email.\n\n" + f"Subject: {subject}\n" + f"Body: {body[:600]}\n\n" + f"Output ONLY a JSON object with a single key 'email_type'.\n" + f"The value must be EXACTLY one of:\n" + f" complaint, question, feature_request, positive_feedback, general_inquiry\n\n" + f"Definitions:\n" + f" complaint — customer reports a problem, outage, bug, or unmet SLA\n" + f" question — customer asks how something works or for clarification\n" + f" feature_request — customer requests new or changed functionality\n" + f" positive_feedback — customer compliments the product or team\n" + f" general_inquiry — anything that does not fit the above\n\n" + f'Example output: {{"email_type": "complaint"}}\n' + f"No preamble. No explanation. Output only the JSON object." + ), + expected_output='{"email_type": ""}', + agent=agent, + ) + + try: + raw = str( + Crew(agents=[agent], tasks=[task], verbose=False).kickoff() + ).strip() + parsed = json_repair.loads(raw) + if isinstance(parsed, dict): + result = parsed.get("email_type", "").strip().lower() + if result in _VALID_EMAIL_TYPES: + return result + except Exception as exc: + logger.warning(f"[external_email] Email classification LLM failed: {exc}") + + if tone in ("frustrated", "urgent"): + return "complaint" + return "general_inquiry" + + def _generate_customer_reply_email( + self, + contact_name: str, + account_name: str, + owner: str, + prior_subject: str, + prior_body: str, + current_stage: str, + opp_id: str, + state, + ) -> Optional[Tuple[str, str, str]]: + """ + Generates a customer reply using an LLM. + + The customer is given the full body of the prior outbound email and their + deal stage context, then asked to reply as themselves. The prompt + explicitly instructs the LLM not to force a stage advance — only output + one if the reply honestly warrants it. + + Returns (body, email_type, crm_stage) or None on failure. + """ + prior_email_ctx = ( + f"The email you are replying to:\n" + f"Subject: {prior_subject}\n" + f"---\n{prior_body[:800]}\n\n" + if prior_body + else ( + f"You are replying to a recent email from {owner} " + f'with subject: "{prior_subject}".\n\n' + ) + ) + + stage_guidance = { + "Prospecting": ( + "You are early in conversations — curious but not yet committed. " + "You might ask questions, request more information, or express cautious interest." + ), + "Value Proposition": ( + "You've seen some value but haven't committed. You might push back on pricing, " + "ask about integrations, or request a demo or case study." + ), + "Proposal/Price Quote": ( + "You have a proposal in hand. You might negotiate terms, ask for clarification " + "on scope, or flag internal approval steps you need to complete." + ), + "Negotiation/Review": ( + "You're close to a decision. You might raise a final objection, request a small " + "concession, confirm timeline, or — if everything looks right — signal readiness to proceed." + ), + }.get(current_stage, "Respond naturally based on the conversation so far.") + + agent = make_agent( + role=f"{contact_name} — {account_name}", + goal=f"Reply to an email from {owner} at {self._company_name}.", + backstory=( + f"You are {contact_name}, a decision-maker at {account_name}. " + f"You are in active conversations with {self._company_name}. " + f"You communicate professionally but directly. " + f"Never acknowledge being an AI." + ), + llm=self._worker_llm, + ) + + task = Task( + description=( + f"{prior_email_ctx}" + f"Deal context: You are currently at the '{current_stage}' stage " + f"with {self._company_name}.\n" + f"{stage_guidance}\n\n" + f"Write a reply from {contact_name} to {owner}.\n\n" + f"Rules:\n" + f"- Reply specifically to what was said above. Reference it directly.\n" + f"- Do NOT force positivity or urgency. Let your reply honestly reflect " + f" where you are. If you are uncertain or have concerns, say so.\n" + f"- Under 120 words. Professional but human.\n\n" + f"Then classify your reply and assess the deal stage it implies.\n\n" + f"Respond ONLY with a JSON object. No preamble, no markdown fences.\n" + f"{{\n" + f' "body": "",\n' + f' "email_type": "",\n' + f' "crm_stage": ""\n' + f"}}" + ), + expected_output='JSON with "body", "email_type", and "crm_stage" keys.', + agent=agent, + ) + + try: + raw = str( + Crew(agents=[agent], tasks=[task], verbose=False).kickoff() + ).strip() + parsed = json_repair.loads(raw) + + if isinstance(parsed, list) and parsed: + parsed = parsed[0] + if not isinstance(parsed, dict): + raise ValueError("LLM did not return a dict") + + body = parsed.get("body", "").strip() + email_type = parsed.get("email_type", "general_inquiry").strip().lower() + crm_stage = parsed.get("crm_stage", current_stage).strip() + + if email_type not in _VALID_EMAIL_TYPES: + email_type = "general_inquiry" + if crm_stage not in _VALID_CRM_STAGES: + crm_stage = current_stage + if not body: + raise ValueError("Empty body in LLM reply") + + return body, email_type, crm_stage + + except Exception as exc: + logger.warning( + f"[external_email] Customer reply generation failed for " + f"{contact_name} ({account_name}): {exc}" + ) + return None + + def generate_customer_replies(self, state) -> List[Any]: + """ + For each open SF opportunity, probabilistically generate a customer reply + to the most recent outbound email sent by the account owner. + """ + + if not self._crm or not hasattr(self._crm, "_sf_o"): + return [] + + signals: List[Any] = [] + date_str = str(state.current_date.date()) + + open_opps = list( + self._crm._sf_o.find( + {"stage": {"$nin": ["Closed Won", "Closed Lost"]}}, + {"_id": 0, "_seq": 0}, + ) + ) + + if not open_opps: + return [] + + for opp in open_opps: + if random.random() > _PROB_CUSTOMER_REPLY: + continue + + opp_id = opp.get("opportunity_id", "") + account_name = opp.get("account_name", "Unknown") + current_stage = opp.get("stage", "Prospecting") + owner = opp.get("owner", "") + touchpoints = opp.get("touchpoints", []) + + if not touchpoints: + continue + + last_touchpoint = touchpoints[-1] + last_subject = last_touchpoint.get("subject", "") + last_embed_id = last_touchpoint.get("embed_id", "") + + prior_body = "" + if last_embed_id: + prior_doc = self._mem._db["emails"].find_one( + {"embed_id": last_embed_id}, {"body": 1, "_id": 0} + ) + if prior_doc: + prior_body = prior_doc.get("body", "") + + acc = self._crm._sf_a.find_one( + {"name": account_name}, {"_id": 0, "_seq": 0} + ) + contact_name = (acc or {}).get("primary_contact", account_name) + contact_email = (acc or {}).get( + "primary_contact_email", + ( + f"{contact_name.lower().replace(' ', '.')}" + f"@{account_name.lower().replace(' ', '')}.com" + ), + ) + + result = self._generate_customer_reply_email( + contact_name=contact_name, + account_name=account_name, + owner=owner, + prior_subject=last_subject, + prior_body=prior_body, + current_stage=current_stage, + opp_id=opp_id, + state=state, + ) + if not result: + continue + + reply_body, email_type, suggested_stage = result + + reply_ts = state.current_date.replace( + hour=random.randint(9, 16), + minute=random.randint(0, 59), + second=random.randint(0, 59), + ) + + reply_subject = ( + f"Re: {last_subject}" if last_subject else f"Re: {account_name}" + ) + embed_id = ( + f"customer_reply_{account_name.lower().replace(' ', '_')}" + f"_{opp_id}_{state.day}" + ) + + sales_dept = next((d for d in self._leads if "sales" in d.lower()), None) + sales_lead = self._leads.get(sales_dept, owner) if sales_dept else owner + owner_addr = self._email_of(owner) if owner else self._email_of(sales_lead) + + eml_path = self._write_eml( + date_str=date_str, + from_name=contact_name, + from_addr=contact_email, + to_name=owner if owner else sales_lead, + to_addr=owner_addr, + subject=reply_subject, + body=reply_body, + timestamp_iso=reply_ts.isoformat(), + direction="inbound", + embed_id=embed_id, + day=state.day, + ) + + # Embed artifact for RAG. + self._mem.embed_artifact( + id=embed_id, + type="email", + title=reply_subject, + content=f"From: {contact_name} ({account_name})\n\n{reply_body}", + day=state.day, + date=date_str, + timestamp=reply_ts.isoformat(), + metadata={ + "source": contact_name, + "org": account_name, + "category": "customer", + "direction": "inbound", + "opportunity_id": opp_id, + "reply_to_subject": last_subject, + "email_type": email_type, + }, + ) + + self._mem.log_event( + SimEvent( + type="inbound_external_email", + timestamp=reply_ts.isoformat(), + day=state.day, + date=date_str, + actors=[contact_name, sales_lead], + artifact_ids={"email": embed_id, "eml_path": str(eml_path)}, + facts={ + "source": contact_name, + "org": account_name, + "category": "customer", + "topic": current_stage, + "subject": reply_subject, + "liaison": sales_lead, + "tone": "professional", + "body_preview": reply_body[:200], + "opportunity_id": opp_id, + "is_customer_reply": True, + }, + summary=( + f"Inbound [customer_reply] from {contact_name} " + f'({account_name}): "{reply_subject}"' + ), + tags=["email", "inbound", "customer", "customer_reply"], + ) + ) + + body_preview = reply_body[:200].rstrip() + ( + "…" if len(reply_body) > 200 else "" + ) + internal_liaison_dept = sales_dept or next(iter(self._leads.keys())) + + signal = ExternalEmailSignal( + source_name=contact_name, + source_org=account_name, + source_email=contact_email, + internal_liaison=internal_liaison_dept, + subject=reply_subject, + body_preview=body_preview, + full_body=reply_body, + tone="professional", + topic=current_stage, + timestamp_iso=reply_ts.isoformat(), + embed_id=embed_id, + category="customer", + eml_path=str(eml_path), + causal_chain=CausalChainHandler(root_id=embed_id), + facts={ + "subject": reply_subject, + "topic": current_stage, + "org": account_name, + "email_type": email_type, + "opportunity_id": opp_id, + "is_customer_reply": True, + }, + ) + + self._route_customer_email(signal, state) + + if ( + suggested_stage + and suggested_stage in _VALID_CRM_STAGES + and _STAGE_RANK.get(suggested_stage, 0) + > _STAGE_RANK.get(current_stage, 0) + ): + self._crm._sf_o.update_one( + {"opportunity_id": opp_id}, + { + "$set": { + "stage": suggested_stage, + "probability": _get_stage_probability(suggested_stage), + "updated_at": reply_ts.isoformat(), + } + }, + ) + updated_doc = self._crm._sf_o.find_one( + {"opportunity_id": opp_id}, {"_id": 0, "_seq": 0} + ) + if updated_doc: + self._crm._write( + f"salesforce/opportunities/{opp_id}.json", updated_doc + ) + + self._mem.log_event( + SimEvent( + type="sf_stage_advanced_by_customer", + timestamp=reply_ts.isoformat(), + day=state.day, + date=date_str, + actors=[contact_name, owner], + artifact_ids={"email": embed_id, "sf_opp": opp_id}, + facts={ + "opportunity_id": opp_id, + "account_name": account_name, + "previous_stage": current_stage, + "new_stage": suggested_stage, + "triggered_by": embed_id, + }, + summary=( + f"SF opp {opp_id} ({account_name}) advanced " + f"{current_stage} → {suggested_stage} by customer reply" + ), + tags=["salesforce", "stage_advanced", "customer_reply"], + ) + ) + logger.info( + f" [green]📈 {opp_id} ({account_name}): " + f"{current_stage} → {suggested_stage}[/green]" + ) + + signals.append(signal) + logger.info( + f" [cyan]📬 Customer reply: {contact_name} ({account_name})[/cyan] " + f"[{email_type}]" + ) + + if signals: + logger.info(f" [cyan]📬 {len(signals)} customer reply(s) generated[/cyan]") + + return signals + + def _route_customer_email(self, signal: Any, state) -> None: + """ + Patched router. Dispatches based on email_type rather than assuming + every inbound customer email is a complaint. + """ + email_type = signal.facts.get("email_type", "general_inquiry") + + if email_type in _COMPLAINT_EMAIL_TYPES: + self._route_complaint_email(signal, state) + else: + self._route_non_complaint_email(signal, state, email_type=email_type) + + def _route_non_complaint_email( + self, signal: Any, state, email_type: str = "general_inquiry" + ) -> None: + """ + Lightweight routing for non-complaint customer emails. + + Sales replies directly to the customer — no Slack ping to Product, + no JIRA ticket. For feature_request emails, a low-probability (~35%) + FYI message is posted in #product so the team is aware without being + formally escalated. + + This preserves causal chain integrity: the reply is appended to the + chain, and the SimEvent type distinguishes these emails from complaints + so eval agents can verify the correct branching behaviour. + """ + date_str = str(state.current_date.date()) + sales_lead = self._leads.get( + signal.internal_liaison, next(iter(self._leads.values())) + ) + + fyi_thread_id = None + if ( + email_type == "feature_request" + and random.random() < _PROB_NON_COMPLAINT_SALES_FYI + ): + fyi_thread_id = self._sales_fyi_to_product( + signal, sales_lead, state, date_str + ) + if fyi_thread_id: + signal.causal_chain.append(fyi_thread_id) + + reply_id = self._send_customer_reply( + signal, sales_lead, is_high=False, state=state, date_str=date_str + ) + if reply_id: + signal.causal_chain.append(reply_id) + + self._mem.log_event( + SimEvent( + type="customer_email_routed", + timestamp=signal.timestamp_iso, + day=state.day, + date=date_str, + actors=[signal.source_name, sales_lead], + artifact_ids={"email": signal.embed_id}, + facts={ + "source": signal.source_name, + "subject": signal.subject, + "email_type": email_type, + "high_priority": False, + "fyi_sent": fyi_thread_id is not None, + "causal_chain": signal.causal_chain.snapshot(), + }, + summary=( + f"{email_type.replace('_', ' ').title()} from {signal.source_name} " + f"handled by {sales_lead} (no escalation)" + + (" [FYI sent to Product]" if fyi_thread_id else "") + ), + tags=["email", "customer", email_type, "routed", "causal_chain"], + ) + ) + + def _sales_fyi_to_product( + self, signal: Any, sales_lead: str, state, date_str: str + ) -> Optional[str]: + """ + Posts a low-friction FYI in #product when a customer sends a feature + request. This is deliberately lighter than _sales_pings_product(): + no explicit ask, no urgency label — just awareness. + + Returns the Slack thread_id, or None on failure. + """ + product_dept = next((d for d in self._leads if "product" in d.lower()), None) + product_lead = self._leads.get(product_dept, sales_lead) + + participants = [sales_lead, product_lead] + fyi_time, _ = self._clock.sync_and_advance(participants, hours=0.1) + + backstory = persona_utils.get_voice_card( + sales_lead, "async", graph_dynamics=None, mem=self._mem + ) + p = self._personas.get(sales_lead, {}) + + agent = make_agent( + role=f"{sales_lead} — {p.get('social_role', 'Sales Lead')}", + goal="Share a brief, low-priority customer feature request with Product.", + backstory=backstory, + llm=self._worker_llm, + ) + task = Task( + description=( + f"You just read a feature request email from {signal.source_name}:\n" + f"Subject: {signal.subject}\n{signal.full_body}\n\n" + f"Write a short, casual Slack FYI to {product_lead} (Product). This is " + f"NOT urgent — you're just sharing it for awareness. No action required.\n" + f"Under 60 words. No bullets. Write as {sales_lead} using your typing quirks." + ), + expected_output="Casual Slack FYI under 60 words.", + agent=agent, + ) + try: + text = str( + Crew(agents=[agent], tasks=[task], verbose=False).kickoff() + ).strip() + except Exception as exc: + logger.warning(f"[external_email] Sales FYI LLM failed: {exc}") + return None + + _, thread_id = self._mem.log_slack_messages( + channel="product", + messages=[ + { + "user": sales_lead, + "email": self._email_of(sales_lead), + "text": text, + "ts": fyi_time.isoformat(), + "date": date_str, + "is_bot": False, + "metadata": { + "type": "customer_feature_request_fyi", + "source_email_id": signal.embed_id, + "customer": signal.source_name, + }, + } + ], + export_dir=self._export_dir, + ) + + self._mem.log_event( + SimEvent( + type="feature_request_fyi", + timestamp=fyi_time.isoformat(), + day=state.day, + date=date_str, + actors=[sales_lead, product_lead], + artifact_ids={ + "slack_thread": thread_id, + "email": signal.embed_id, + }, + facts={ + "customer": signal.source_name, + "subject": signal.subject, + "relayed_by": sales_lead, + "product_gatekeeper": product_lead, + "causal_chain": signal.causal_chain.snapshot(), + }, + summary=( + f"{sales_lead} shared feature request FYI from " + f"{signal.source_name} in #product (no action required)" + ), + tags=["feature_request", "fyi", "slack", "causal_chain"], + ) + ) + logger.info( + f" [dim]💬 FYI → #product: feature request from {signal.source_name}[/dim]" + ) + return thread_id + + def _route_complaint_email(self, signal: Any, state) -> None: + date_str = str(state.current_date.date()) + sales_lead = self._leads.get( + signal.internal_liaison, next(iter(self._leads.values())) + ) + product_dept = next((d for d in self._leads if "product" in d.lower()), None) + product_lead = self._leads.get(product_dept, sales_lead) + + thread_id = self._sales_pings_product( + signal, sales_lead, product_lead, state, date_str + ) + if thread_id: + signal.causal_chain.append(thread_id) + + is_high = signal.tone in ("frustrated", "urgent") or ( + state.system_health < 70 and "stability" in signal.topic.lower() + ) + + if is_high and random.random() < _PROB_CUSTOMER_JIRA: + ticket_id = self._product_opens_jira(signal, product_lead, state, date_str) + if ticket_id: + signal.causal_chain.append(ticket_id) + + reply_id = self._send_customer_reply( + signal, sales_lead, is_high, state, date_str + ) + if reply_id: + signal.causal_chain.append(reply_id) + + self._mem.log_event( + SimEvent( + type="customer_email_routed", + timestamp=signal.timestamp_iso, + day=state.day, + date=date_str, + actors=[signal.source_name, sales_lead, product_lead], + artifact_ids={"email": signal.embed_id}, + facts={ + "source": signal.source_name, + "subject": signal.subject, + "email_type": "complaint", + "high_priority": is_high, + "causal_chain": signal.causal_chain.snapshot(), + }, + summary=( + f"Complaint from {signal.source_name} routed: " + f"{sales_lead} → {product_lead}" + + (" [JIRA opened]" if len(signal.causal_chain) > 2 else "") + ), + tags=["email", "customer", "complaint", "routed", "causal_chain"], + ) + ) + + def _get_stage_probability(self, stage: str) -> int: + """Returns the default SF probability for a given stage.""" + return { + "Prospecting": 10, + "Value Proposition": 25, + "Proposal/Price Quote": 50, + "Negotiation/Review": 75, + "Closed Won": 100, + "Closed Lost": 0, + }.get(stage, 10) + def _ensure_sources_loaded(self) -> None: if self._sources is None: self._sources = self._mem.get_inbound_email_sources() or [] @@ -1324,15 +2027,17 @@ def _fallback_sources(self) -> List[dict]: def _write_eml( self, - date_str, - from_name, - from_addr, - to_name, - to_addr, - subject, - body, - timestamp_iso, - direction="inbound", + date_str: str, + from_name: str, + from_addr: str, + to_name: str, + to_addr: str, + subject: str, + body: str, + timestamp_iso: str, + direction: str = "inbound", + embed_id: str = "", + day: int = 0, ) -> Path: out_dir = self._export_dir / "emails" / direction / date_str out_dir.mkdir(parents=True, exist_ok=True) @@ -1347,4 +2052,30 @@ def _write_eml( msg.attach(MIMEText(body, "plain")) with open(path, "w") as fh: fh.write(msg.as_string()) + + doc_id = embed_id or f"{from_name.lower().replace(' ', '_')}_{timestamp_iso}" + try: + self._mem._db["emails"].update_one( + {"embed_id": doc_id}, + { + "$setOnInsert": { + "embed_id": doc_id, + "direction": direction, + "from_name": from_name, + "from_addr": from_addr, + "to_name": to_name, + "to_addr": to_addr, + "subject": subject, + "body": body, + "timestamp": timestamp_iso, + "day": day, + "date": date_str, + "eml_path": str(path), + } + }, + upsert=True, + ) + except Exception as exc: + logger.warning(f"[external_email] emails collection insert failed: {exc}") + return path diff --git a/src/flow.py b/src/flow.py index 4af531f..e46c2a3 100644 --- a/src/flow.py +++ b/src/flow.py @@ -1,8 +1,5 @@ """ -flow.py (MongoDB + YAML + NetworkX Edition) -================================================= -OrgForge simulation engine. Reads from config.yaml. -Uses NetworkX for social graphs. Uses MongoDB for vector/artifact storage. +OrgForge simulation engine. """ from pathlib import Path @@ -61,7 +58,7 @@ from rich.panel import Panel from rich.logging import RichHandler from rich import box -from utils.persona_utils import get_voice_card +from utils.persona_utils import persona_utils from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer from crewai import Process, Task, Crew @@ -153,7 +150,7 @@ def build_llm(model_key: str): "model": model, "region_name": region, "temperature": 0.7, - "max_tokens": 8192, + "max_tokens": 16384, } llm = LLM(**llm_args) @@ -314,7 +311,6 @@ class State(BaseModel): persona_stress: Dict[str, int] = {} actor_cursors: Dict[str, Any] = Field(default_factory=dict) - # Daily counters — reset each morning, read at end of day daily_incidents_opened: int = 0 daily_incidents_resolved: int = 0 daily_artifacts_created: int = 0 @@ -324,9 +320,7 @@ class State(BaseModel): org_day_plan: Optional[Any] = None daily_active_actors: List[str] = [] daily_event_type_counts: Dict[str, int] = {} - departed_employees: Dict[ - str, Dict - ] = {} # name → {left, role, knew_about, documented_pct} + departed_employees: Dict[str, Dict] = {} new_hires: Dict[str, Dict] = {} # name → {joined, role, dept, expertise} ticket_actors_today: Dict[str, List[str]] = Field(default_factory=dict) @@ -441,7 +435,7 @@ def create_pr( agent = make_agent( role=f"{author}, Software Engineer", goal=f"Write a PR description for your own code as {author} would.", - backstory=get_voice_card( + backstory=persona_utils.get_voice_card( author, "async", self.graph_dynamics, self._mem ), llm=self._worker_llm, @@ -658,6 +652,11 @@ def __init__(self, mem: Optional[Memory] = None): mem=self._mem, planner_llm=PLANNER_MODEL, ) + persona_utils.configure( + graph_dynamics=self.graph_dynamics, + mem=self._mem, + crm=self._crm, + ) self._lifecycle = OrgLifecycleManager( config=CONFIG, graph_dynamics=self.graph_dynamics, @@ -696,6 +695,7 @@ def __init__(self, mem: Optional[Memory] = None): clock=self._clock, threat_injector=self._threat, crm=self._crm, + graph_dynamics=self.graph_dynamics, ) self._normal_day = NormalDayHandler( config=CONFIG, @@ -1024,6 +1024,7 @@ def daily_cycle(self): self._normal_day.handle(self.state.org_day_plan) self._email_ingestor.generate_business_hours(state=self.state) + self._email_ingestor.generate_customer_replies(state=self.state) self._email_ingestor.generate_hr_outbound(state=self.state) if random.random() < CONFIG["simulation"].get("adhoc_confluence_prob", 0.3): @@ -1053,21 +1054,35 @@ def daily_cycle(self): if r.get("pattern") == "trust_building": self._se_followup_days[r["followup_due_day"]] = r["target"] - # Incident fires after normal day work, mid-day _base_prob = CONFIG["simulation"].get("incident_base_prob", 0.15) - _health_factor = max(0.5, (100 - self.state.system_health) / 100) _cooldown = CONFIG["simulation"].get("incident_cooldown_days", 3) days_since_incident = self.state.day - self.state.last_incident_day + _incident_triggers = CONFIG["simulation"].get( + "incident_triggers", + [ + "crash", + "fail", + "error", + "latency", + "timeout", + "outage", + "down", + "spike", + ], + ) + _theme_lower = self.state.daily_theme.lower() + _theme_triggered = any(x in _theme_lower for x in _incident_triggers) + _prob_triggered = random.random() < _base_prob + if ( not self.state.active_incidents and days_since_incident > _cooldown - and random.random() < _base_prob * _health_factor + and (_theme_triggered or _prob_triggered) ): self.state.last_incident_day = self.state.day self._handle_incident() - # Drain embed queue before checkpoint so MongoDB is fully consistent. self._embed_worker.drain() serialized_incidents = [] @@ -1148,7 +1163,7 @@ def _handle_sprint_planning(self): product_agent = make_agent( role=f"{product_lead} — {p_persona.get('social_role', 'Product Manager')}", goal="Propose a sprint theme grounded in business priorities.", - backstory=get_voice_card( + backstory=persona_utils.get_voice_card( product_lead, "design", self.graph_dynamics, self._mem ), llm=PLANNER_MODEL, @@ -1174,7 +1189,7 @@ def _handle_sprint_planning(self): eng_agent = make_agent( role=f"{eng_lead} — {e_persona.get('social_role', 'Engineering Lead')}", goal="Ratify or amend the sprint theme based on technical reality.", - backstory=get_voice_card( + backstory=persona_utils.get_voice_card( eng_lead, "design", self.graph_dynamics, self._mem ), llm=PLANNER_MODEL, @@ -1237,7 +1252,7 @@ def _generate_dept_tickets(dept: str, members: list) -> list: agent = make_agent( role=f"{dept} Lead", goal=f"Create realistic sprint tickets for the {dept} team.", - backstory=get_voice_card( + backstory=persona_utils.get_voice_card( lead_name, "design", self.graph_dynamics, self._mem ), llm=WORKER_MODEL, @@ -1441,7 +1456,9 @@ def _handle_standup(self): messages = [] for name in attendees: - backstory = get_voice_card(name, "async", self.graph_dynamics, self._mem) + backstory = persona_utils.get_voice_card( + name, "async", self.graph_dynamics, self._mem + ) personal_ctx = self._mem.context_for_person( name=name, @@ -1638,7 +1655,7 @@ def _handle_retrospective(self): agent = make_agent( role=role_label, goal=f"Contribute authentically to the Sprint #{sprint_num} retrospective.", - backstory=get_voice_card( + backstory=persona_utils.get_voice_card( name, "design", self.graph_dynamics, self._mem ), llm=PLANNER_MODEL, @@ -1721,7 +1738,7 @@ def _handle_incident(self): rc_agent = make_agent( role=f"{on_call}, Senior On-Call Engineer", goal=f"Diagnose the root cause of today's incident as {on_call}, given what you know about this system.", - backstory=get_voice_card( + backstory=persona_utils.get_voice_card( on_call, "collision", self.graph_dynamics, self._mem ), llm=PLANNER_MODEL, @@ -1826,10 +1843,14 @@ def _handle_incident(self): ) gap_kw = [k for emp in DEPARTED_EMPLOYEES.values() for k in emp["knew_about"]] - chain = self.graph_dynamics.build_escalation_chain( + chain = self.graph_dynamics.crm_aware_escalation_chain( first_responder=on_call, + crm=self._crm, + root_cause=root_cause, + for_org=None, domain_keywords=gap_kw if involves_gap else None, ) + escalation_narrative = self.graph_dynamics.escalation_narrative(chain) escalation_actors = [n for n, _ in chain.chain] @@ -1862,7 +1883,9 @@ def _handle_incident(self): desc_agent = make_agent( role="Senior Engineer", goal="Write a concise Jira ticket description for an incident.", - backstory=get_voice_card(on_call, "design", self.graph_dynamics, self._mem), + backstory=persona_utils.get_voice_card( + on_call, "design", self.graph_dynamics, self._mem + ), llm=WORKER_MODEL, ) desc_task = Task( @@ -2321,7 +2344,6 @@ def _generate_adhoc_confluence_page( ): self._confluence.write_adhoc_page(author=author, backstory=backstory) - # ─── END OF DAY ─────────────────────────── def _end_of_day(self): date_str = str(self.state.current_date.date()) decay = CONFIG["morale"]["daily_decay"] @@ -2340,12 +2362,10 @@ def _end_of_day(self): max(all_cursors) if all_cursors else self.state.current_date ) - # Ensure the summary doesn't happen before 17:30 eod_baseline = self.state.current_date.replace(hour=17, minute=30, second=0) summary_time = max(latest_time_worked, eod_baseline) housekeeping_time = summary_time + timedelta(minutes=1) - # ── end_of_day event (unchanged) ───────────────────────────────────────── self._mem.log_event( SimEvent( type="end_of_day", @@ -2370,7 +2390,6 @@ def _end_of_day(self): max(event_counts, key=event_counts.get) if event_counts else "normal_day" ) - # Departments represented by today's active actors departments_involved = list( {dept_of(name) for name in unique_actors if dept_of(name) != "Unknown"} ) @@ -2387,7 +2406,7 @@ def _end_of_day(self): timestamp=housekeeping_time.isoformat(), day=self.state.day, date=date_str, - actors=unique_actors, # populated — was always [] + actors=unique_actors, artifact_ids={}, facts={ "incidents_opened": self.state.daily_incidents_opened, @@ -2454,6 +2473,7 @@ def _end_of_day(self): self.graph_dynamics, departed=dep, first_responder=dept_members[0], + crm=self._crm, ) self._mem.log_event( SimEvent( @@ -2474,6 +2494,19 @@ def _end_of_day(self): ) self.graph_dynamics.decay_edges() + + self.graph_dynamics.decay_edges() + + edge_changes = self.graph_dynamics.sync_crm_edge_weights(self._crm) + if edge_changes: + logger.debug( + f"[graph] CRM edge sync: {len(edge_changes)} external edges updated" + ) + + crm_stress_deltas = self.graph_dynamics.apply_crm_stress(self._crm) + if crm_stress_deltas: + logger.debug(f"[graph] CRM stress applied: {crm_stress_deltas}") + self._last_stress_prop = self.graph_dynamics.propagate_stress() prop = self._last_stress_prop if prop.burnt_out: @@ -2598,14 +2631,12 @@ def _handle_external_contact(self, inc: ActiveIncident, contact: dict) -> None: ) interaction_start_iso = start_time.isoformat() - # Boost the edge between liaison and external node — they just talked external_node = contact["name"] if self.social_graph.has_edge(liaison_name, external_node): self.graph_dynamics.record_incident_collaboration( [liaison_name, external_node] ) - # Tier 1: structured incident fetch — no embedding. ctx = self._mem.context_for_incident( ticket_id=inc.ticket_id, as_of_time=interaction_start_iso, @@ -2614,7 +2645,7 @@ def _handle_external_contact(self, inc: ActiveIncident, contact: dict) -> None: agent = make_agent( role="Employee", goal="Summarize an external conversation for your team on Slack.", - backstory=get_voice_card( + backstory=persona_utils.get_voice_card( liaison_name, "dm", self.graph_dynamics, self._mem ), llm=WORKER_MODEL, @@ -2639,7 +2670,6 @@ def _handle_external_contact(self, inc: ActiveIncident, contact: dict) -> None: Crew(agents=[agent], tasks=[task], verbose=False).kickoff() ).strip() - # Write to the incidents Slack channel message = { "user": liaison_name, "email": email_of(liaison_name), @@ -2666,7 +2696,6 @@ def _handle_external_contact(self, inc: ActiveIncident, contact: dict) -> None: if thread_id and getattr(inc, "causal_chain", None): inc.causal_chain.append(thread_id) - # SimEvent — this is what makes it retrievable as ground truth self._mem.log_event( SimEvent( type="external_contact_summarized", diff --git a/src/genesis.py b/src/genesis.py index a4cb9fc..abe1c92 100644 --- a/src/genesis.py +++ b/src/genesis.py @@ -2,9 +2,10 @@ from datetime import datetime, timedelta import json import logging +from pathlib import Path import random import re -from typing import Dict, List +from typing import List from config_loader import ( BASE, COMPANY_DESCRIPTION, @@ -13,6 +14,7 @@ INDUSTRY, LEADS, LEGACY, + ORG_CHART, ) from memory import Memory from agent_factory import make_agent @@ -55,10 +57,8 @@ def seed_external_sources(mem: Memory, planner_llm): tech_stack = mem.tech_stack_for_prompt() dept_str = ", ".join(LEADS.keys()) - db_accounts = list(mem._db["sf_accounts"].find({"type": "Customer"}, {"name": 1})) - accounts = [doc["name"] for doc in db_accounts] - random.shuffle(accounts) + all_names = [name for members in ORG_CHART.values() for name in members] agent = make_agent( role="Enterprise IT Architect", @@ -70,6 +70,7 @@ def seed_external_sources(mem: Memory, planner_llm): ), llm=planner_llm, ) + task = Task( description=( f"Generate 15 realistic inbound email sources. EXACTLY 8 must be 'customer' category, and 7 must be 'vendor' category.\n" @@ -83,25 +84,41 @@ def seed_external_sources(mem: Memory, planner_llm): f" - QA_Support: Responsible for CI/CD (Jenkins) and testing tool alerts.\n" f" - HR_Ops: Responsible for legal, compliance, and payroll vendors.\n\n" f"Rules:\n" + f" - HUMAN NAMES: The 'first_name' and 'last_name' field MUST be a realistic human name representing the Point of Contact (e.g., 'Marcus Thorne').\n" + f" - NO DUPLICATE NAMES: Ensure no new generated names overlap with these: {all_names}.\n" + f" - PERSONA DICT: Include a nested 'persona' object with 'typing_quirks' (string), 'social_role' (string, matching contact_role), and 'expertise' (array of strings).\n" f" - ADHERENCE: Use ONLY vendors that appear in the TECH STACK above. If Jira is listed, never use Trello.\n" - f" - CUSTOMERS: All category:'customer' entries must be from the KNOWN CUSTOMERS list.\n" - f" - FIRMOGRAPHICS (Customers ONLY): For customer entries, include 'industry' (e.g. Financial Services, Healthcare), 'tier' (Enterprise, Mid-Market, or SMB), 'billing_region' (NA, EMEA, APAC), and 'arr' (realistic numeric annual revenue like 50000, 120000, 350000).\n" - f" - HEALTH SENSITIVITY: Include 'trigger_health_threshold' (int 0-100). Scale: Infrastructure/Enterprise (85-98), SMB/Standard Vendors (70-85).\n" # New Rule + f" - FIRMOGRAPHICS (Customers ONLY): Include 'industry' (e.g. Financial Services), 'tier' (Enterprise, Mid-Market, SMB), 'billing_region' (NA, EMEA, APAC), 'billing_city', 'billing_state' (2-letter code if US), 'billing_country', and 'arr' (e.g. 50000, 120000, 350000).\n" + f" - STRATEGIC (Customers ONLY): Include 'is_lighthouse' (bool), 'expansion_potential' (int 1-10), and 'contract_renewal_date' (ISO Date string).\n" + f" - TECHNICAL (Vendors ONLY): Include 'integration_complexity' (Low, Med, High) and 'version_in_use' (e.g., 'v2 Beta', 'Legacy').\n" + f" - HEALTH SENSITIVITY: Include 'trigger_health_threshold' (int 0-100). Scale: Infrastructure/Enterprise (85-98), SMB/Standard Vendors (70-85).\n" + f" - PERSONA: Include 'contact_role' (e.g. VP Engineering, Procurement) and 'persona_archetype' (e.g. The Champion, The Skeptic, The Bureaucrat).\n" + f" - DYNAMICS: Include 'expected_sla_hours' (int: 2, 4, 24, 48), 'cadence' (daily, weekly, bi-weekly, reactive), and 'timezone_offset' (int: -8 to +8).\n" + f" - RELATIONSHIP: Include 'sentiment_baseline' (float 0.0 to 1.0) and 'history_summary' (1 short sentence mapping the history).\n" f" - TOPICS: Provide 3-5 hyper-specific topics (e.g., 'GitHub Actions Runner Timeout' or 'Stripe API 402 Payment Required').\n" f" - CATEGORY: exactly 'vendor' or 'customer'.\n" f" - TRIGGER_ON: array of 'always', 'incident', 'low_health'.\n" f" - TONE: formal | technical | frustrated | urgent | friendly.\n\n" f"Raw JSON array only — no preamble, no markdown fences:\n" f"[\n" - f' {{"name":"GitHub","org":"GitHub Inc.","email":"support@github.com",' + f' {{"name":"GitHub","org":"GitHub Inc.","first_name":"Jake","last_name": "Smith","org":"GitHub Inc.","email":"j.smith@github.com",' f'"category":"vendor","internal_liaison":"Engineering_Backend",' - f'"trigger_on":["incident", "low_health"],"trigger_health_threshold":95,"tone":"technical",' - f'"topics":["Webhooks failing with 5xx","Pull Request comment API latency"]}},\n' + f'"contact_role":"Senior Technical Account Manager","persona_archetype":"The Technical Expert",' + f'"trigger_on":["incident", "low_health"],"trigger_health_threshold":95,' + f'"expected_sla_hours":4,"cadence":"reactive","timezone_offset":-8,' + f'"integration_complexity":"High","version_in_use":"Enterprise Cloud",' + f'"sentiment_baseline":0.8,"history_summary":"Solid uptime, but API rate limits frequently cause friction.",' + f'"tone":"technical","topics":["Webhooks failing with 5xx","Pull Request comment API latency"]}},\n' f' {{"name":"GlobalFinance","org":"GlobalFinance Corp","email":"cto@globalfinance.com",' f'"category":"customer","internal_liaison":"Sales_Marketing",' - f'"trigger_on":["always","incident"],"trigger_health_threshold":90,"tone":"formal",' - f'"topics":["SLA reporting","Contract renewal"],"industry":"Financial Services",' - f'"tier":"Enterprise","billing_region":"NA","arr":250000}}\n' + f'"contact_role":"CTO","persona_archetype":"The Skeptic",' + f'"persona": {{"typing_quirks": "terse, lowercase heavy, fast responses", "social_role": "CTO", "expertise": ["enterprise architecture", "security compliance"]}},' + f'"trigger_on":["always","incident"],"trigger_health_threshold":90,' + f'"expected_sla_hours":2,"cadence":"weekly","timezone_offset":-5,' + f'"is_lighthouse":true,"expansion_potential":8,"contract_renewal_date":"2026-12-01T00:00:00Z",' + f'"sentiment_baseline":0.4,"history_summary":"Demanding enterprise client, currently evaluating competitors for next year.",' + f'"tone":"formal","topics":["SLA reporting","Contract renewal"],"industry":"Financial Services",' + f'"tier":"Enterprise","billing_region":"NA","billing_city":"New York","billing_state":"NY","billing_country":"USA","arr":250000}}\n' f"]" ), expected_output=f"Raw JSON array of {_DEFAULT_SOURCE_COUNT} source objects.", @@ -197,6 +214,25 @@ def seed_tech_stack(mem: Memory, planner_llm): def seed_crm_accounts(mem: Memory): """Seeds Salesforce accounts from the external sources in MongoDB.""" + logger.info("[genesis] Seeding CRM accounts...") + + _zd = mem._db["zd_tickets"] + _sf_a = mem._db["sf_accounts"] + _sf_o = mem._db["sf_opps"] + _emails = mem._db["emails"] + + _zd.create_index([("ticket_id", 1)], unique=True) + _zd.create_index([("status", 1)]) + _zd.create_index([("related_incident", 1)]) + _sf_a.create_index([("account_id", 1)], unique=True) + _sf_a.create_index([("name", 1)]) + _sf_a.create_index([("owner", 1)]) + _sf_o.create_index([("opportunity_id", 1)], unique=True) + _sf_o.create_index([("stage", 1), ("_seq", -1)]) + _sf_o.create_index([("account_name", 1), ("stage", 1)]) + _sf_o.create_index([("owner", 1), ("stage", 1)]) + _emails.create_index([("embed_id", 1)], unique=True) + doc = mem._db["sim_config"].find_one({"_id": "inbound_email_sources"}) if not doc: return @@ -205,16 +241,26 @@ def seed_crm_accounts(mem: Memory): not CONFIG["crm"]["salesforce"]["enabled"] or not CONFIG["crm"]["salesforce"]["seed_accounts"] ): + logger.info("[genesis] Salesforce not enabled, continuing...") return - contacts = list( - mem._db["sim_config"].find( - {"_id": "inbound_email_sources", "category": "customer"}, {"_id": 0} - ) + db_contacts = mem._db["sim_config"].find_one( + {"_id": "inbound_email_sources"}, {"_id": 0} ) + contacts = db_contacts["sources"] if db_contacts else [] + + logger.info(f"[genesis] Found {len(contacts)} contacts to process into CRM.") + start_dt = datetime.strptime(CONFIG["simulation"]["start_date"], "%Y-%m-%d") + tier_config = { + "Enterprise": (5001, 50000), + "Mid-Market": (101, 5000), + "SMB": (1, 100), + "Unknown": (1, 500), + } + for contact in contacts: org_name = contact.get("org", "Unknown") safe_id = org_name.upper().replace(" ", "").replace("-", "") @@ -230,36 +276,60 @@ def seed_crm_accounts(mem: Memory): days=days_ago, hours=hours_ago, minutes=mins_ago ) + delta_seconds = int((start_dt - created_dt).total_seconds()) + last_activity_dt = created_dt + timedelta( + seconds=random.randint(0, delta_seconds) + ) + + default_renewal = (created_dt + timedelta(days=365)).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + + category = contact.get("category", "customer").capitalize() + tier = contact.get("tier", "Unknown") if category == "Customer" else "Unknown" + emp_range = tier_config.get(tier, tier_config["Unknown"]) + + sentiment = contact.get("sentiment_baseline", 0.8) + is_risky = True if sentiment < 0.5 else False + account = { "account_id": account_id, "name": org_name, - "primary_contact": contact.get("name", "Unknown Contact"), - "type": "Customer", + "type": category, + "primary_contact_name": f"{contact.get('first_name', 'First Name')} {contact.get('last_name', 'Last Name')}", + "primary_contact_email": contact.get("email", ""), + "contact_role": contact.get("contact_role", "Unknown"), + "owner": contact.get("internal_liaison", "Unassigned"), "industry": contact.get("industry", "Technology"), - "tier": contact.get( - "tier", - random.choices( - ["Enterprise", "Mid-Market", "SMB"], weights=[0.2, 0.5, 0.3] - )[0], - ), + "tier": tier if tier != "Unknown" else None, + "employee_count": random.randint(*emp_range), "website": f"https://www.{org_name.lower().replace(' ', '')}.com", - "billing_region": contact.get( - "billing_region", - random.choices(["NA", "EMEA", "APAC"], weights=[0.6, 0.3, 0.1])[0], + "billing_region": contact.get("billing_region", "NA"), + "arr": contact.get("arr", 0), + "is_lighthouse": contact.get("is_lighthouse", False), + "expansion_potential": contact.get("expansion_potential", 0), + "status": "Active", + "sentiment_baseline": sentiment, + "risk_flag": is_risky, + "contract_renewal_date": contact.get( + "contract_renewal_date", default_renewal ), - "arr": contact.get("arr", random.choice([50000, 100000, 250000, 500000])), - "owner": contact.get("internal_liaison", "Unassigned"), + "last_activity_date": last_activity_dt.strftime("%Y-%m-%dT%H:%M:%SZ"), "created_at": created_dt.strftime("%Y-%m-%dT%H:%M:%SZ"), - "risk_flag": False, } + + account = {k: v for k, v in account.items() if v is not None} + mem._db["sf_accounts"].insert_one({**account, "_seq": 0}) - path = BASE / "salesforce/accounts/{account_id}.json" + path = Path(BASE) / f"salesforce/accounts/{account_id}.json" path.parent.mkdir(parents=True, exist_ok=True) with open(path, "w") as fh: json.dump(account, fh, indent=2) - logger.info(f"[crm] SF account seeded: {account_id} ({org_name})") + logger.info( + f"[crm] SF account seeded: {account_id} ({org_name}) | Type: {category} | Risk: {is_risky}" + ) pass diff --git a/src/graph_dynamics.py b/src/graph_dynamics.py index 6826d84..05a3eb0 100644 --- a/src/graph_dynamics.py +++ b/src/graph_dynamics.py @@ -394,3 +394,252 @@ def _role_label(self, name: str) -> str: (d for d, members in self._org_chart.items() if name in members), "" ) return f"{dept} Engineer" if dept else "Engineer" + + def crm_aware_escalation_chain( + self, + first_responder: str, + crm, + root_cause: str = "", + for_org: Optional[str] = None, + domain_keywords: Optional[List[str]] = None, + ) -> EscalationChain: + """ + Escalation chain that prefers routing through whoever owns the affected + SF account, then falls back to the standard Dijkstra path. + + Use instead of build_escalation_chain() when an incident has a known + customer blast radius (i.e. ZD tickets were opened or SF deals flagged). + + Args: + first_responder: The on-call engineer starting the chain. + crm: Live CRMSystem (or NullCRMSystem — safely ignored). + incident_component: Passed through to domain_keywords fallback. + for_org: If set, look up the SF account owner for this org + and prefer them as an intermediate hop. + """ + from crm_system import NullCRMSystem + + preferred_intermediate: Optional[str] = None + + if not isinstance(crm, NullCRMSystem) and for_org: + opp = crm._sf_o.find_one( + { + "account_name": for_org, + "stage": {"$nin": ["Closed Won", "Closed Lost"]}, + "owner": {"$nin": ["Pending Reassignment", "Unassigned"]}, + }, + {"owner": 1}, + ) + if opp: + candidate = opp["owner"] + if candidate != first_responder and self.G.has_node(candidate): + preferred_intermediate = candidate + + if preferred_intermediate: + leg1 = self.build_escalation_chain( + first_responder, + domain_keywords=[root_cause] if root_cause else None, + ) + + raw = leg1.raw_path + if preferred_intermediate not in raw: + raw = [raw[0], preferred_intermediate] + raw[1:] + + chain = [(n, self._role_label(n)) for n in raw] + reached_lead = any(n in self._leads.values() for n in raw[1:]) + return EscalationChain( + chain=chain, + path_length=len(raw) - 1, + reached_lead=reached_lead, + raw_path=raw, + ) + + return self.build_escalation_chain( + first_responder, + domain_keywords=[root_cause] if root_cause else None, + ) + + def sync_crm_edge_weights(self, crm) -> Dict[str, float]: + """ + Reads live CRM state and adjusts edge weights between external contact + nodes and their internal liaison to reflect relationship health. + + Weight logic: + - Base: 0.5 (floor) + - Recent ZD ticket: +2.0 (they're actively talking to us) + - Urgent ZD ticket: +1.0 (extra signal — high-touch moment) + - Open SF opp: +3.0 (active sales relationship) + - Opp in Negotiation: +2.0 (deal heat) + - SF risk flag: -2.5 (relationship in trouble) + - No opp, no ticket: -0.5 (going quiet — drift toward estranged) + + Returns a dict of {node_id: new_weight} for SimEvent logging. + """ + from crm_system import NullCRMSystem + + if isinstance(crm, NullCRMSystem): + return {} + + zd = crm._zd + sf_a = crm._sf_a + sf_o = crm._sf_o + floor = self.cfg.get("edge_weight_floor", 0.5) + changed: Dict[str, float] = {} + + for node, data in self.G.nodes(data=True): + if not data.get("external", False): + continue + + neighbours = list(self.G.neighbors(node)) + if not neighbours: + continue + liaison = neighbours[0] + + org_name = data.get("org", node) + + open_tickets = list( + zd.find( + {"org_name": org_name, "status": {"$in": ["Open", "Pending"]}}, + {"priority": 1}, + ) + ) + ticket_boost = 0.0 + if open_tickets: + ticket_boost += 2.0 + if any(t.get("priority") == "Urgent" for t in open_tickets): + ticket_boost += 1.0 + + account = sf_a.find_one({"name": org_name}, {"risk_flag": 1}) + open_opp = sf_o.find_one( + { + "account_name": org_name, + "stage": {"$nin": ["Closed Won", "Closed Lost"]}, + }, + {"stage": 1, "risk_notes": 1}, + ) + + opp_boost = 0.0 + risk_penalty = 0.0 + silence_penalty = 0.0 + + if open_opp: + opp_boost += 3.0 + if open_opp.get("stage") == "Negotiation/Review": + opp_boost += 2.0 + if open_opp.get("risk_notes"): + risk_penalty += 2.5 + elif not open_tickets: + silence_penalty = 0.5 # going quiet + + if account and account.get("risk_flag"): + risk_penalty += 1.0 + + current_w = ( + self.G[node][liaison].get("weight", floor) + if self.G.has_edge(node, liaison) + else floor + ) + new_w = round( + max( + floor, + current_w + + ticket_boost + + opp_boost + - risk_penalty + - silence_penalty, + ), + 4, + ) + + if new_w != current_w: + if not self.G.has_edge(node, liaison): + self.G.add_edge(node, liaison, weight=floor) + self.G[node][liaison]["weight"] = new_w + self._centrality_dirty = True + changed[node] = new_w + + return changed + + def apply_crm_stress(self, crm) -> Dict[str, int]: + """ + Nudges stress for internal employees based on their CRM exposure. + Called once per day, after sync_crm_edge_weights(). + + Stress sources: + - Each open Urgent ZD ticket for an account they liaise: +4 + - Each open Normal ZD ticket: +1 + - Each SF opportunity with risk_notes: +6 + - Each SF opportunity in Negotiation (positive pressure): +2 + - Opportunity owner for a Closed Won today: -8 (relief) + + Caps at the existing stress min/max (0-100). Returns {name: delta}. + """ + from crm_system import NullCRMSystem + + if isinstance(crm, NullCRMSystem): + return {} + + deltas: Dict[str, int] = {} + + # Build a map: internal_node -> [org_names they're liaison for] + liaison_orgs: Dict[str, List[str]] = {} + for node, data in self.G.nodes(data=True): + if not data.get("external", False): + continue + org = data.get("org", node) + for neighbour in self.G.neighbors(node): + if not self.G.nodes[neighbour].get("external", False): + liaison_orgs.setdefault(neighbour, []).append(org) + + for employee, orgs in liaison_orgs.items(): + if employee not in self._stress: + continue + delta = 0 + for org in orgs: + tickets = list( + crm._zd.find( + {"org_name": org, "status": {"$in": ["Open", "Pending"]}}, + {"priority": 1}, + ) + ) + for t in tickets: + delta += 4 if t.get("priority") == "Urgent" else 1 + + at_risk_opp = crm._sf_o.find_one( + { + "account_name": org, + "stage": {"$nin": ["Closed Won", "Closed Lost"]}, + "risk_notes": {"$not": {"$size": 0}}, + }, + {"stage": 1}, + ) + if at_risk_opp: + delta += 6 + if at_risk_opp.get("stage") == "Negotiation/Review": + delta += 2 # so close, yet so at-risk + + self._stress[employee] = max(0, min(100, self._stress[employee] + delta)) + if delta != 0: + deltas[employee] = delta + + open_opps = list( + crm._sf_o.find( + {"stage": {"$nin": ["Closed Won", "Closed Lost"]}}, + {"owner": 1, "stage": 1, "risk_notes": 1}, + ) + ) + for opp in open_opps: + owner = opp.get("owner") + if not owner or owner in ("Pending Reassignment", "Unassigned"): + continue + if owner not in self._stress: + continue + delta = 0 + if opp.get("risk_notes"): + delta += 5 + if opp.get("stage") == "Negotiation/Review": + delta += 2 + self._stress[owner] = max(0, min(100, self._stress[owner] + delta)) + deltas[owner] = deltas.get(owner, 0) + delta + + return deltas diff --git a/src/memory.py b/src/memory.py index 5e52f7c..b562f34 100644 --- a/src/memory.py +++ b/src/memory.py @@ -24,6 +24,7 @@ from dataclasses import dataclass, field, asdict import time from typing import List, Dict, Optional, Any, Tuple +import boto3 from pymongo import MongoClient from pymongo.operations import SearchIndexModel @@ -34,9 +35,6 @@ logger = logging.getLogger("orgforge.memory") -# ───────────────────────────────────────────── -# CONFIG (all overridable via environment) -# ───────────────────────────────────────────── MONGO_URI = os.environ.get( "MONGO_URI", "mongodb://localhost:27017/?directConnection=true" ) @@ -83,9 +81,6 @@ _EVENT_LOG_MAX_DAYS = 7 -# ───────────────────────────────────────────── -# SIM EVENT -# ───────────────────────────────────────────── @dataclass class SimEvent: type: str @@ -250,6 +245,51 @@ def embed(self, text: str) -> List[float]: return self._fallback(text) +class InfinityEmbedder(BaseEmbedder): + """OpenAI-compatible embedder for Infinity server.""" + + _INSTRUCTIONS = { + "search_document": "", # no prefix at index time + "search_query": "Instruct: Given an enterprise knowledge query, retrieve the most relevant document\nQuery: ", + } + + def __init__( + self, model: str = EMBED_MODEL, host: str = OLLAMA_HOST, dims: int = EMBED_DIMS + ): + super().__init__(dims) + self._model = model + self._host = host + self._session = requests.Session() # reuse connections + self._ok = self._check_connection() + + def _check_connection(self) -> bool: + try: + r = self._session.get(f"{self._host}/health", timeout=3) + return r.status_code == 200 + except Exception: + logger.warning( + f"[memory] ⚠️ Cannot connect to Infinity at {self._host}. Using fallback." + ) + return False + + def embed(self, text: str, input_type: str = "search_document") -> List[float]: + if not self._ok: + return self._fallback(text) + + prefix = self._INSTRUCTIONS.get(input_type, "") + try: + r = self._session.post( + f"{self._host}/embeddings", + json={"model": self._model, "input": prefix + text}, + timeout=300, + ) + r.raise_for_status() + return r.json()["data"][0]["embedding"] + except Exception as e: + logger.warning(f"[memory] Infinity embedding failed: {e}") + return [] + + # ── CLOUD: AWS Bedrock ───────────────────────── class BedrockEmbedder(BaseEmbedder): """ @@ -353,6 +393,8 @@ def build_embedder( "aws_region", os.environ.get("AWS_DEFAULT_REGION", "us-east-1") ) return BedrockEmbedder(region=region, dims=dims) + if provider == "infinity": + return InfinityEmbedder(model=model, dims=dims) # Default: Ollama (local) return OllamaEmbedder(model=model, dims=dims) @@ -364,13 +406,10 @@ class Memory: def __init__( self, mongo_uri: str = MONGO_URI, - mongo_client=None, # inject mongomock.MongoClient() in tests + mongo_client=None, ): self._embedder = build_embedder() self._embed_worker = None - # Accept an injected client (e.g. mongomock) so tests never touch a - # real MongoDB instance. Production code passes nothing and gets the - # real MongoClient as before. self._client = mongo_client or MongoClient(mongo_uri) self._db = self._client[DB_NAME] @@ -406,7 +445,6 @@ def __init__( self._current_day: int = 0 - # In-memory ordered log for strict sequential access self._event_log: List[SimEvent] = [] self._init_vector_indexes() @@ -445,18 +483,15 @@ def _init_vector_indexes(self): } for coll_name in ["artifacts", "events"]: - # 1. Force creation of the collection if it doesn't exist if coll_name not in self._db.list_collection_names(): try: self._db.create_collection(coll_name) logger.info(f"[memory] Created collection: {coll_name}") except Exception: - # Handle race conditions if another process created it simultaneously pass coll = self._db[coll_name] - # 2. Proceed with index creation existing_indexes = list(coll.list_search_indexes()) if not any(idx.get("name") == "vector_index" for idx in existing_indexes): try: @@ -486,7 +521,6 @@ def embed_artifact( """Upsert artifact into MongoDB immediately. Embedding is deferred to the background queue if set_embed_worker() has been called, otherwise synchronous.""" - # Write document immediately with null embedding so it's queryable by ID doc = { "_id": id, "type": type, @@ -500,7 +534,6 @@ def embed_artifact( } self._artifacts.update_one({"_id": id}, {"$set": doc}, upsert=True) - # Embed asynchronously if worker is attached, synchronously otherwise embed_text = f"{title}\n\n{content}" if self._embed_worker is not None: self._embed_worker.enqueue( @@ -1445,7 +1478,6 @@ def recall_with_rewrite( n: int = 4, as_of_time: Optional[Any] = None, since: Optional[Any] = None, - llm_callable=None, ) -> str: """ HyDE variant — rewrites the query before embedding. @@ -1463,29 +1495,17 @@ def recall_with_rewrite( n: Number of artifacts to retrieve. as_of_time: Causal ceiling — passed through to recall(). since: Causal floor — passed through to recall(). - llm_callable: Optional callable(prompt: str) -> str for the rewrite - step. If None, falls back to context_for_prompt() - so callers degrade gracefully before an LLM is wired in. Returns: Formatted context string identical in shape to context_for_prompt(). """ - if llm_callable is None: - # Graceful degradation — behaves like context_for_prompt() until - # an LLM callable is injected at the call site. - logger.debug( - "[memory] recall_with_rewrite: no llm_callable, falling back to context_for_prompt" - ) - return self.context_for_prompt( - raw_query, n=n, as_of_time=as_of_time, since=since - ) - rewritten = self._rewrite_query(raw_query, llm_callable) + rewritten = self._rewrite_query(raw_query) return self.context_for_prompt( rewritten, n=n, as_of_time=as_of_time, since=since ) - def _rewrite_query(self, raw_query: str, llm_callable) -> str: + def _rewrite_query(self, raw_query) -> str: """ Generates a short hypothetical passage for HyDE-style query rewriting. @@ -1500,7 +1520,26 @@ def _rewrite_query(self, raw_query: str, llm_callable) -> str: f"Topic: {raw_query}\n\nPassage:" ) try: - rewritten = llm_callable(prompt).strip() + client = boto3.client("bedrock-runtime", region_name="us-east-1") + + body = json.dumps( + { + "prompt": prompt, + "max_gen_len": 256, + "temperature": 0.3, + } + ) + + response = client.invoke_model( + modelId="us.meta.llama3-3-70b-instruct-v1:0", + body=body, + contentType="application/json", + accept="application/json", + ) + + response_body = json.loads(response.get("body").read()) + rewritten = response_body.get("generation", "").strip() + logger.debug( f"[memory] query rewrite: '{raw_query[:60]}' → '{rewritten[:80]}'" ) @@ -1838,9 +1877,6 @@ def context_for_ticket( or its participants — so the conversation doesn't rehash settled ground - Any blocker events on this ticket - Use this instead of context_for_prompt(ticket_title) in - _handle_async_question and _handle_design_discussion. - Args: ticket_id: The JIRA ticket ID the conversation is about. as_of_time: Causal ceiling (datetime or ISO string). diff --git a/src/normal_day.py b/src/normal_day.py index ea1a7d2..3f3c131 100644 --- a/src/normal_day.py +++ b/src/normal_day.py @@ -24,7 +24,7 @@ ) from causal_chain_handler import CausalChainHandler from insider_threat import _NullInjector -from utils.persona_utils import get_voice_card +from utils.persona_utils import persona_utils logger = logging.getLogger("orgforge.normalday") @@ -263,7 +263,7 @@ def _handle_ticket_progress( ) ) - backstory = get_voice_card(assignee, "async", self._gd, self._mem) + backstory = persona_utils.get_voice_card(assignee, "async", self._gd, self._mem) if is_non_eng: persona = self._config.get("personas", {}).get(assignee, {}) @@ -286,6 +286,26 @@ def _handle_ticket_progress( ) completion_note = "" + reviewer_feedback_hint = "" + if ticket.get("status") == "In Review": + for linked_pr_id in ticket.get("linked_prs", []): + linked_pr = self._mem._prs.find_one( + {"pr_id": linked_pr_id, "status": "open"}, {"_id": 0} + ) + if linked_pr and linked_pr.get("changes_requested"): + recent_feedback = linked_pr.get("comments", [])[-3:] + if recent_feedback: + feedback_lines = "\n".join( + f" - {c['author']}: {c['text'][:200]}" + for c in recent_feedback + ) + reviewer_feedback_hint = ( + f"\nREVIEWER FEEDBACK TO ADDRESS:\n{feedback_lines}\n" + f"Your comment must describe specifically how you addressed this feedback. " + f"Do not describe unrelated work.\n" + ) + break + agent = make_agent( role=f"{assignee} — {agent_role}", backstory=backstory, @@ -296,6 +316,7 @@ def _handle_ticket_progress( description=( f"You are {assignee}. You worked on ticket [{ticket_id}] today.\n\n" f"Your task today: {item.description}\n" + f"{reviewer_feedback_hint}" f"IMPORTANT: Your comment must be specifically about this ticket's work — " f"do not describe unrelated tasks.\n" f"{completion_note}\n\n" @@ -365,6 +386,16 @@ def _handle_ticket_progress( "day": self._state.day, } ) + + if ticket.get("status") == "In Review": + for linked_pr_id in ticket.get("linked_prs", []): + linked_pr = self._mem._prs.find_one( + {"pr_id": linked_pr_id, "status": "open"}, {"_id": 0} + ) + if linked_pr and linked_pr.get("changes_requested"): + linked_pr["changes_requested"] = False + self._mem.upsert_pr(linked_pr) + if ticket["status"] == "To Do": ticket["status"] = "In Progress" if "in_progress_since" not in ticket: @@ -781,7 +812,7 @@ def _handle_pr_review( current_actor_time = artifact_time.isoformat() ctx = self._mem.context_for_prompt(pr_title, n=2, as_of_time=current_actor_time) - backstory = get_voice_card(reviewer, "async", self._gd, self._mem) + backstory = persona_utils.get_voice_card(reviewer, "async", self._gd, self._mem) p = self._config.get("personas", {}).get(reviewer, {}) recurrence_hint = "" @@ -797,26 +828,58 @@ def _handle_pr_review( f"Prior root cause: {ancestor_root_cause[:120]}" ) + prior_reviews = pr.get("comments", []) + round_count = len(prior_reviews) + + prior_reviews = pr.get("comments", []) + review_history = "" + if prior_reviews: + rounds = "\n".join( + f" - {c['author']} ({c['date']}): [{c.get('verdict', '?')}] {c['text'][:120]}" + for c in prior_reviews[-6:] # last 6 comments max + ) + review_history = f"\n--- PRIOR REVIEW ROUNDS ---\n{rounds}\n\n" + agent = make_agent( role=f"{reviewer} — {p.get('social_role', 'Code Reviewer')}", goal=f"Write a PR review comment as {reviewer} would, reflecting your current stress and style.", backstory=backstory, llm=self._worker, ) + if round_count == 0: + approval_guidance = ( + "This is the first review. Scrutinize carefully — request changes if " + "there are correctness, safety, or design issues." + ) + elif round_count == 1: + approval_guidance = ( + "This is the second review round. The author has had one round of feedback. " + "Approve if the main issues have been addressed, even if minor things remain. " + "Only request changes again if a concrete correctness issue is still unresolved." + ) + else: + approval_guidance = ( + f"This is review round {round_count + 1}. The author has responded to " + f"{round_count} rounds of feedback. You MUST approve unless there is an " + f"obvious unresolved bug or security issue. Do not invent new concerns." + ) + task = Task( description=( f"You are {reviewer}. You are reviewing this PR by {author}: {pr_title}\n\n" - f"Write a review comment (1-4 sentences). Be specific — mention code patterns, " - f"potential edge cases, or required changes. Your tone must reflect your current " - f"stress level (see your backstory).\n\n" - f"Then decide: does this PR meet the bar to merge, or does it need changes?\n\n" + f"{review_history}" + f"STEP 1 — DECIDE YOUR VERDICT FIRST:\n" + f"{approval_guidance}\n" + f"Choose: 'approved' or 'changes_requested'.\n\n" + f"STEP 2 — WRITE YOUR COMMENT:\n" + f"Write 1-3 sentences consistent with your verdict. If approved, acknowledge " + f"what looks good. If changes_requested, name the specific issue only.\n" + f"Your tone must reflect your current stress level (see your backstory).\n\n" f"Respond ONLY with valid JSON. No preamble, no markdown fences.\n" f"{{\n" f' "comment": "your review comment here",\n' f' "verdict": "approved" or "changes_requested"\n' f"}}\n\n" - f"verdict must be exactly 'approved' if the code is ready to merge, " - f"or 'changes_requested' if the author needs to address something first.\n\n" f"{recurrence_hint}" f"--- CONTEXT ---\n{ctx}" ), @@ -826,6 +889,7 @@ def _handle_pr_review( ), agent=agent, ) + raw_review = str( Crew(agents=[agent], tasks=[task], verbose=False).kickoff() ).strip() @@ -877,7 +941,7 @@ def _handle_pr_review( if linked_ticket and linked_ticket.get("status") == "In Review": linked_ticket["status"] = "In Progress" - linked_ticket["in_progress_since"] = self._state.day + linked_ticket["last_review_requested_day"] = self._state.day linked_ticket["updated_at"] = current_actor_time self._save_ticket(linked_ticket) self._emit_bot_message( @@ -1038,7 +1102,7 @@ def _handle_pr_review_for_incident( ) ctx = self._mem.context_for_prompt(pr_title, n=2, as_of_time=current_actor_time) - backstory = get_voice_card(reviewer, "async", self._gd, self._mem) + backstory = persona_utils.get_voice_card(reviewer, "async", self._gd, self._mem) p = self._config.get("personas", {}).get(reviewer, {}) agent = make_agent( @@ -1182,7 +1246,7 @@ def _handle_one_on_one( as_of_time=meeting_time_iso, ) - backstory = get_voice_card( + backstory = persona_utils.get_voice_card( [name, collaborator], "one_on_one", self._gd, self._mem ) @@ -1367,8 +1431,8 @@ def _handle_async_question( meeting_start, _ = self._clock.sync_and_advance(all_actors, hours=0) meeting_time_iso = meeting_start.isoformat() - ctx = self._mem.context_for_prompt( - ticket_title, n=2, as_of_time=meeting_time_iso + ctx = self._mem.context_for_ticket( + ticket_id=ticket_id, as_of_time=meeting_time_iso ) relevant_experts = self._mem.find_confluence_experts( @@ -1396,7 +1460,9 @@ def _handle_async_question( ) design_hint = self._mem.format_design_discussions_hint(discussions) - backstory = get_voice_card(all_actors, "async", self._gd, self._mem) + backstory = persona_utils.get_voice_card( + all_actors, "async", self._gd, self._mem + ) responders = [a for a in all_actors if a != asker] turn_speakers = [asker] + responders @@ -1581,8 +1647,8 @@ def _handle_design_discussion( meeting_start, _ = self._clock.sync_and_advance(participants, hours=0) meeting_time_iso = meeting_start.isoformat() - ctx = self._mem.context_for_prompt( - item.description, n=3, as_of_time=meeting_time_iso + ctx = self._mem.context_for_ticket( + ticket_id=item.related_id, as_of_time=meeting_time_iso ) medium = getattr(item, "meeting_medium", "slack") @@ -1707,7 +1773,9 @@ def _handle_mentoring( as_of_time=meeting_time_iso, ) - backstory = get_voice_card([mentor, mentee], "mentoring", self._gd, self._mem) + backstory = persona_utils.get_voice_card( + [mentor, mentee], "mentoring", self._gd, self._mem + ) agents, tasks, prev_task = [], [], None n_turns = self._turn_count([mentor, mentee], (3, 6)) @@ -1876,7 +1944,9 @@ def _handle_collision_event(self, event: ProposedEvent, date_str: str): event.rationale, n=2, as_of_time=thread_start_iso ) - voice_cards = get_voice_card(participants, "collision", self._gd, self._mem) + voice_cards = persona_utils.get_voice_card( + participants, "collision", self._gd, self._mem + ) n_turns = { "high": random.randint(5, 8), @@ -1989,7 +2059,9 @@ def _emit_blocker_slack( channel = asker_dept.lower().replace(" ", "-") participants = [asker, collaborator] - backstory = get_voice_card(participants, "dm", self._gd, self._mem) + backstory = persona_utils.get_voice_card( + participants, "dm", self._gd, self._mem + ) asker_role = ( self._config.get("personas", {}) @@ -2122,7 +2194,7 @@ def _emit_completion_email( dept = dept_of_name(assignee, self._org_chart) lead = self._find_lead_for(assignee) or assignee - backstory = get_voice_card(assignee, "async", self._gd, self._mem) + backstory = persona_utils.get_voice_card(assignee, "async", self._gd, self._mem) p = self._config.get("personas", {}).get(assignee, {}) @@ -2263,7 +2335,7 @@ def _emit_sales_outbound_email( ) p = self._config.get("personas", {}).get(assignee, {}) - backstory = get_voice_card(assignee, "async", self._gd, self._mem) + backstory = persona_utils.get_voice_card(assignee, "async", self._gd, self._mem) stage = opportunity_id and self._crm._sf_o.find_one( {"opportunity_id": opportunity_id}, {"stage": 1, "_id": 0} ) @@ -2333,6 +2405,8 @@ def _emit_sales_outbound_email( with open(eml_path, "w") as fh: fh.write(msg.as_string()) + thread_id = f"sales_email_{ticket_id}_{self._state.day}" + self._crm.process_outbound_email( email_data={ "sender": assignee, @@ -2341,13 +2415,33 @@ def _emit_sales_outbound_email( "recipient_org": account_name, "subject": subject, "stage": new_stage, + "embed_id": thread_id, }, timestamp=timestamp, date_str=date_str, day=self._state.day, ) - thread_id = f"sales_email_{ticket_id}_{self._state.day}" + self._mem._db["emails"].update_one( + {"embed_id": thread_id}, + { + "$setOnInsert": { + "embed_id": thread_id, + "direction": "outbound", + "from_name": assignee, + "from_addr": sender_addr, + "to_name": contact_name, + "to_addr": contact_email, + "subject": subject, + "body": body, + "timestamp": timestamp, + "day": self._state.day, + "date": date_str, + "eml_path": str(eml_path), + } + }, + upsert=True, + ) chain.append(thread_id) @@ -2535,7 +2629,7 @@ def _emit_review_reply( ) -> Tuple[List[str], str]: """Author replies to a review question in #engineering.""" - backstory = get_voice_card(author, "async", self._gd, self._mem) + backstory = persona_utils.get_voice_card(author, "async", self._gd, self._mem) p = self._config.get("personas", {}).get(author, {}) agent = make_agent( @@ -2745,7 +2839,9 @@ def _trigger_watercooler_chat(self, target_actor: str, date_str: str) -> None: Crew(agents=[topic_agent], tasks=[topic_task], verbose=False).kickoff() ).strip() - voice_cards = get_voice_card(participants, "watercooler", self._gd, self._mem) + voice_cards = persona_utils.get_voice_card( + participants, "watercooler", self._gd, self._mem + ) speaker_sequence = ", ".join( participants[i % len(participants)] for i in range(len(participants) + 1) @@ -2975,9 +3071,10 @@ def _run_slack_design_discussion( Original Slack-thread path extracted from _handle_design_discussion. Returns (slack_path, thread_id, tags). """ - from utils.persona_utils import get_voice_card - backstory = get_voice_card(participants, "async", self._gd, self._mem) + backstory = persona_utils.get_voice_card( + participants, "async", self._gd, self._mem + ) turn_speakers = [initiator] + [ participants[i % len(participants)] for i in range(1, random.randint(5, 8)) @@ -3058,9 +3155,10 @@ def _run_zoom_design_discussion( Returns (file_path, transcript_id, tags). """ - from utils.persona_utils import get_voice_card - backstory = get_voice_card(participants, "sync", self._gd, self._mem) + backstory = persona_utils.get_voice_card( + participants, "sync", self._gd, self._mem + ) agent = make_agent( role="Meeting Transcript Generator", @@ -3146,13 +3244,13 @@ def _save_zoom_transcript( transcript_id = f"zoom_{date_str}_{uuid.uuid4().hex[:8]}" lines = [ - f"# Zoom Meeting Transcript", + "# Zoom Meeting Transcript", f"**Date:** {date_str}", f"**Topic:** {topic}", f"**Attendees:** {', '.join(participants)}", - f"", - f"---", - f"", + "", + "---", + "", ] current_ts = datetime.fromisoformat(meeting_time_iso) diff --git a/src/org_lifecycle.py b/src/org_lifecycle.py index 283c131..c7f9bf3 100644 --- a/src/org_lifecycle.py +++ b/src/org_lifecycle.py @@ -943,15 +943,45 @@ def recompute_escalation_after_departure( graph_dynamics: GraphDynamics, departed: DepartureRecord, first_responder: str, + crm=None, + root_cause: str = "", ) -> str: """ Rebuild escalation chain post-departure and return a log-ready narrative. The node has already been removed, so Dijkstra routes around the gap. + + If crm is provided and the departed employee owned SF accounts, the chain + prefers routing through whoever is best positioned to handle the customer + relationship — not just the topologically nearest lead. """ - chain = graph_dynamics.build_escalation_chain( - first_responder=first_responder, - domain_keywords=departed.knowledge_domains or None, - ) + from crm_system import NullCRMSystem + + use_crm_aware = crm is not None and not isinstance(crm, NullCRMSystem) + + if use_crm_aware: + owned_opps = list( + crm._sf_o.find( + { + "owner": departed.name, + "stage": {"$nin": ["Closed Won", "Closed Lost"]}, + }, + {"account_name": 1}, + ) + ) + for_org = owned_opps[0]["account_name"] if owned_opps else None + + chain = graph_dynamics.crm_aware_escalation_chain( + first_responder=first_responder, + crm=crm, + root_cause=root_cause or "", + for_org=for_org, + ) + else: + chain = graph_dynamics.build_escalation_chain( + first_responder=first_responder, + domain_keywords=departed.knowledge_domains or None, + ) + narrative = graph_dynamics.escalation_narrative(chain) note = f"[Post-departure re-route after {departed.name} left] Path: {narrative}" logger.info(f" [cyan]🔀 Escalation re-routed:[/cyan] {note}") diff --git a/src/planner_models.py b/src/planner_models.py index e9d4696..7062835 100644 --- a/src/planner_models.py +++ b/src/planner_models.py @@ -254,5 +254,13 @@ class ValidationResult: "secret_detected", "zoom_meeting", "sales_outbound_email", - "proactive_outreach_initiated" + "proactive_outreach_initiated", + "zd_ticket_opened", + "zd_tickets_escalated", + "zd_tickets_resolved", + "sf_deals_risk_flagged", + "sf_ownership_lapsed", + "crm_touchpoint", + "crm_account_at_risk", + "customer_health_briefing", } diff --git a/src/post_sim_artifacts.py b/src/post_sim_artifacts.py index 9afd05d..6d01adf 100644 --- a/src/post_sim_artifacts.py +++ b/src/post_sim_artifacts.py @@ -24,8 +24,6 @@ import argparse import json import logging -import math -import os import random import uuid from datetime import datetime, timedelta @@ -38,7 +36,6 @@ COMPANY_DOMAIN, COMPANY_NAME, CONFIG, - EXPORT_DIR, INDUSTRY, ) from memory import Memory, SimEvent @@ -312,7 +309,7 @@ def _response_date(self, org: str) -> str: t["resolved_day"] for t in tickets if t.get("resolved_day") is not None ] base_day = max(resolved_days) if resolved_days else self._end_day - response_day = min(base_day + self._response_delay_days, self._end_day + 5) + response_day = min(base_day + self._RESPONSE_DELAY_DAYS, self._end_day + 5) return _iso(_sim_date(response_day, self._start)) def build_responses(self) -> List[Dict]: @@ -699,7 +696,7 @@ def build_metrics(self) -> int: p50 = self._latency_p50(p99) tags = [ - f"env:production", + "env:production", f"service:{COMPANY_NAME.lower().replace(' ', '-')}", f"sim_day:{day}", ] @@ -770,7 +767,7 @@ def build_alerts( "tags": [ "severity:critical", f"incident:{iid}", - f"env:production", + "env:production", f"sim_day:{inc['open_day']}", ], "attributes": { @@ -1018,18 +1015,18 @@ def _nps_placeholder(r: Dict) -> str: ) if classification == "passive": return ( - f"Generally good but there have been a couple of hiccups. " - f"Would be a 10 if the reliability was more consistent." + "Generally good but there have been a couple of hiccups. " + "Would be a 10 if the reliability was more consistent." ) if detail.get("escalated_tickets", 0): return ( - f"We had a support ticket escalate into a full incident and it " - f"took longer than expected to resolve. Impacted our team significantly." + "We had a support ticket escalate into a full incident and it " + "took longer than expected to resolve. Impacted our team significantly." ) return ( - f"Some reliability issues during our contract period that affected our " - f"operations. Hoping to see improvement before renewal." + "Some reliability issues during our contract period that affected our " + "operations. Hoping to see improvement before renewal." ) diff --git a/src/utils/persona_utils.py b/src/utils/persona_utils.py index 504be02..c7b88c0 100644 --- a/src/utils/persona_utils.py +++ b/src/utils/persona_utils.py @@ -6,163 +6,329 @@ logger = logging.getLogger("orgforge.persona_utils") -def get_voice_card( - names: Union[str, list], context: str = "general", graph_dynamics=None, mem=None -) -> str: - """ - Unified persona generator for all OrgForge LLM prompts. - Combines identity, tenure, expertise, style, and dynamic mood. - Accepts a single name or a list of names. Identical personas are deduplicated. - """ - is_single = isinstance(names, str) - name_list = [names] if is_single else names - - card_to_names = {} - name_to_history = {} - - for name in name_list: - p = PERSONAS.get(name, DEFAULT_PERSONA) - stress = graph_dynamics._stress.get(name, 30) if graph_dynamics else 30 - quirks = p.get("typing_quirks", "standard professional grammar") - tenure = p.get("tenure", "mid") - expertise = ( - ", ".join(str(e) for e in p.get("expertise", [])[:3]) - or "general engineering" - ) - social_role = p.get("social_role", "Contributor") - dept = dept_of_name(name) - interests = ( - ", ".join( - str(i) for i in (p.get("interests") or p.get("expertise", []))[:3] +class PersonaUtils: + def __init__(self): + self._graph_dynamics = None + self._mem = None + self._crm = None + + def configure(self, graph_dynamics=None, mem=None, crm=None): + if graph_dynamics is not None: + self._graph_dynamics = graph_dynamics + if mem is not None: + self._mem = mem + if crm is not None: + self._crm = crm + + def get_voice_card( + self, + names: Union[str, list], + context: str = "general", + graph_dynamics=None, + mem=None, + internal=True, + ) -> str: + """ + Unified persona generator for all OrgForge LLM prompts. + Combines identity, tenure, expertise, style, and dynamic mood. + Accepts a single name or a list of names. Identical personas are deduplicated. + """ + + if not internal: + return self._get_external_voice_card(names, context, mem) + + is_single = isinstance(names, str) + name_list = [names] if is_single else names + + card_to_names = {} + name_to_history = {} + + for name in name_list: + p = PERSONAS.get(name, DEFAULT_PERSONA) + stress = graph_dynamics._stress.get(name, 30) if graph_dynamics else 30 + quirks = p.get("typing_quirks", "standard professional grammar") + tenure = p.get("tenure", "mid") + expertise = ( + ", ".join(str(e) for e in p.get("expertise", [])[:3]) + or "general engineering" + ) + social_role = p.get("social_role", "Contributor") + dept = dept_of_name(name) + interests = ( + ", ".join( + str(i) for i in (p.get("interests") or p.get("expertise", []))[:3] + ) + or "general topics" + ) + style = p.get("style", "") + anti_patterns = p.get("anti_patterns", "") + pet_peeves = p.get("pet_peeves", "") + + if mem: + past = mem.persona_history(name, n=2) + if past: + name_to_history[name] = " | ".join( + f"Day {e.day}: {e.summary}" for e in past + ) + + _moods: Dict[str, tuple] = { + "one_on_one": ( + "drained, short replies", + "a bit distracted", + "relaxed and present", + ), + "async": ( + "visibly stressed, terse replies, wants to resolve this fast", + "somewhat distracted but trying to help", + "engaged and happy to dig in", + ), + "design": ( + "terse, wants to decide fast and move on", + "engaged but watching the clock", + "thinking carefully, happy to explore trade-offs", + ), + "mentoring": ( + "drained, keeping answers short", + "patient but distracted", + "engaged and generous with their time", + ), + "collision": ( + "visibly stressed, terse, wants this resolved immediately", + "frustrated but trying to stay professional", + "measured and collegial", + ), + "dm": ( + "stressed and frustrated, wants this unblocked now", + "concerned but calm", + "helpful and focused", + ), + "watercooler": ( + "visibly drained, short replies, clearly wants this to be over quickly", + "a bit distracted, somewhat engaged but mind is elsewhere", + "relaxed and happy to take a break", + ), + "general": ( + "drained, short replies", + "a bit distracted", + "relaxed and present", + ), + } + high, mid, low = _moods.get(context, _moods["general"]) + mood = high if stress > 80 else mid if stress > 60 else low + + placeholder = "__NAMES_PLACEHOLDER__" + + if context == "one_on_one": + header = f"{placeholder} | Tenure: {tenure}" + elif context == "async": + header = f"{placeholder} | Tenure: {tenure} | Dept: {dept}" + elif context == "design": + header = f"{placeholder} | Role: {social_role} | Expertise: {expertise}" + elif context == "mentoring": + header = f"{placeholder} | Tenure: {tenure} | Expertise: {expertise}" + elif context == "collision": + header = f"{placeholder} | Dept: {dept} | Role: {social_role}" + elif context == "dm": + header = f"{placeholder}" + elif context == "watercooler": + header = f"{placeholder} | Tenure: {tenure} | Role: {social_role}" + else: + header = f"{placeholder} | Tenure: {tenure}" + + if style and context != "watercooler": + header += f" | Style: {style}" + + lines = [header, f" Typing style: {quirks}", f" Current mood: {mood}"] + + if context == "async": + lines.insert(2, f" Expertise: {expertise}") + elif context == "watercooler": + lines.insert(2, f" Personal interests: {interests}") + + _anti_pattern_contexts = {"async", "design", "collision", "dm"} + if anti_patterns and context in _anti_pattern_contexts: + lines.append(f" Never write {placeholder} as: {anti_patterns.strip()}") + + _pet_peeve_contexts = {"design", "collision"} + if pet_peeves and context in _pet_peeve_contexts: + lines.append(f" Pet peeves (will react if triggered): {pet_peeves}") + + _crm_hint = self.crm_pressure_hint(name, self._crm) + if _crm_hint: + lines.append(_crm_hint) + + identity_block = "\n".join(lines) + + sections = [ + f"IDENTITY: You are {placeholder} ({tenure} tenure). Role: {p.get('social_role', 'Contributor')}.", + f"COMPANY: You work at {COMPANY_NAME}, which {COMPANY_DESCRIPTION}.", + f"{identity_block}\n\nNever acknowledge being an AI. Stay in character.", + ] + + template = "\n".join(sections) + + if template not in card_to_names: + card_to_names[template] = [] + card_to_names[template].append(name) + + parts = [] + for template, group_names in card_to_names.items(): + combined_names = " / ".join(group_names) + + final_card = template.replace("__NAMES_PLACEHOLDER__", combined_names) + + history_lines = [] + for n in group_names: + if n in name_to_history: + prefix = f"{n}: " if len(group_names) > 1 else "" + history_lines.append(f"{prefix}{name_to_history[n]}") + + if history_lines: + final_card += "\n\nRECENT HISTORY:\n" + "\n".join(history_lines) + + if is_single: + return final_card + else: + parts.append(f"PERSONA(S) FOR {combined_names}:\n{final_card}") + + return "\n\n".join(parts) + + def crm_pressure_hint(self, name: str, crm) -> str: + """ + Returns a short LLM directive reflecting this person's live CRM exposure. + Injected into get_voice_card() when crm is passed. Empty string if no signal. + + Examples: + "Jordan owns an at-risk $85K deal in Negotiation — anxious about the close." + "Sam is the liaison for a customer with 2 Urgent support tickets open." + """ + from crm_system import NullCRMSystem + + if crm is None or isinstance(crm, NullCRMSystem): + return "" + + hints = [] + + owned_opps = list( + crm._sf_o.find( + { + "owner": name, + "stage": {"$nin": ["Closed Won", "Closed Lost"]}, + }, + { + "account_name": 1, + "stage": 1, + "amount": 1, + "risk_notes": 1, + "probability": 1, + }, ) - or "general topics" ) - style = p.get("style", "") - anti_patterns = p.get("anti_patterns", "") - pet_peeves = p.get("pet_peeves", "") - - if mem: - past = mem.persona_history(name, n=2) - if past: - name_to_history[name] = " | ".join( - f"Day {e.day}: {e.summary}" for e in past + for opp in owned_opps[:2]: + stage = opp.get("stage", "") + amt = opp.get("amount", 0) + org = opp.get("account_name", "a customer") + if opp.get("risk_notes"): + hints.append( + f"{name} owns an at-risk ${amt:,} deal with {org} ({stage}) — " + f"privately anxious about losing it." ) + elif stage == "Negotiation/Review": + hints.append( + f"{name} is deep in contract negotiation with {org} (${amt:,}) — " + f"excited but checking email constantly." + ) + + if not hints: + urgent_count = crm._zd.count_documents( + {"status": {"$in": ["Open", "Pending"]}, "priority": "Urgent"} + ) + normal_count = crm._zd.count_documents( + {"status": {"$in": ["Open", "Pending"]}, "priority": {"$ne": "Urgent"}} + ) + if urgent_count > 0: + hints.append( + f"{name} is aware there {'is' if urgent_count == 1 else 'are'} " + f"{urgent_count} Urgent support ticket(s) open — " + f"feels pressure to resolve the customer situation." + ) + elif normal_count >= 3: + hints.append( + f"{name} has {normal_count} open support tickets in the queue — " + f"background stress from customer load." + ) + + return " CRM pressure: " + " | ".join(hints) if hints else "" + + def _get_external_voice_card( + self, + names: Union[str, list], + context: str, + mem, + ) -> str: + """ + Builds a voice card for an external contact (vendor or customer). + Pulls persona data from the inbound_email_sources collection rather + than the internal PERSONAS config. + """ + is_single = isinstance(names, str) + name_list = [names] if is_single else names + + m = mem or self._mem + if m is None: + return f"You are {name_list[0]}. Respond professionally." + + if not hasattr(self, "_external_sources_cache"): + self._external_sources_cache = None + if self._external_sources_cache is None: + self._external_sources_cache = m.get_inbound_email_sources() or [] + + source_map = {s["name"]: s for s in self._external_sources_cache} + + parts = [] + for name in name_list: + source = source_map.get(name) + if not source: + parts.append( + f"You are {name}, an external contact. Respond professionally." + ) + continue + + persona = source.get("persona", {}) + quirks = persona.get("typing_quirks", source.get("tone", "professional")) + role = source.get("contact_role", persona.get("social_role", "Contact")) + archetype = source.get("persona_archetype", "") + expertise = ", ".join(persona.get("expertise", [])) + sentiment = source.get("sentiment_baseline", 0.7) + history = source.get("history_summary", "") + org = source.get("org", name) + category = source.get("category", "contact") + + if sentiment < 0.4: + mood = "skeptical and guarded, looking for reasons to escalate" + elif sentiment < 0.6: + mood = "neutral but watchful, needs reassurance" + else: + mood = "collaborative and constructive" + + card = "\n".join( + filter( + None, + [ + f"IDENTITY: You are {name}, {role} at {org}.", + f"ARCHETYPE: {archetype}" if archetype else "", + f"EXPERTISE: {expertise}" if expertise else "", + f"Typing style: {quirks}", + f"Current mood: {mood}", + f"Relationship history: {history}" if history else "", + f"Never acknowledge being an AI or a simulation. " + f"You are a real {category} contact.", + ], + ) + ) + parts.append(card) + + return "\n\n".join(parts) + - _moods: Dict[str, tuple] = { - "one_on_one": ( - "drained, short replies", - "a bit distracted", - "relaxed and present", - ), - "async": ( - "visibly stressed, terse replies, wants to resolve this fast", - "somewhat distracted but trying to help", - "engaged and happy to dig in", - ), - "design": ( - "terse, wants to decide fast and move on", - "engaged but watching the clock", - "thinking carefully, happy to explore trade-offs", - ), - "mentoring": ( - "drained, keeping answers short", - "patient but distracted", - "engaged and generous with their time", - ), - "collision": ( - "visibly stressed, terse, wants this resolved immediately", - "frustrated but trying to stay professional", - "measured and collegial", - ), - "dm": ( - "stressed and frustrated, wants this unblocked now", - "concerned but calm", - "helpful and focused", - ), - "watercooler": ( - "visibly drained, short replies, clearly wants this to be over quickly", - "a bit distracted, somewhat engaged but mind is elsewhere", - "relaxed and happy to take a break", - ), - "general": ( - "drained, short replies", - "a bit distracted", - "relaxed and present", - ), - } - high, mid, low = _moods.get(context, _moods["general"]) - mood = high if stress > 80 else mid if stress > 60 else low - - placeholder = "__NAMES_PLACEHOLDER__" - - if context == "one_on_one": - header = f"{placeholder} | Tenure: {tenure}" - elif context == "async": - header = f"{placeholder} | Tenure: {tenure} | Dept: {dept}" - elif context == "design": - header = f"{placeholder} | Role: {social_role} | Expertise: {expertise}" - elif context == "mentoring": - header = f"{placeholder} | Tenure: {tenure} | Expertise: {expertise}" - elif context == "collision": - header = f"{placeholder} | Dept: {dept} | Role: {social_role}" - elif context == "dm": - header = f"{placeholder}" - elif context == "watercooler": - header = f"{placeholder} | Tenure: {tenure} | Role: {social_role}" - else: - header = f"{placeholder} | Tenure: {tenure}" - - if style and context != "watercooler": - header += f" | Style: {style}" - - lines = [header, f" Typing style: {quirks}", f" Current mood: {mood}"] - - if context == "async": - lines.insert(2, f" Expertise: {expertise}") - elif context == "watercooler": - lines.insert(2, f" Personal interests: {interests}") - - _anti_pattern_contexts = {"async", "design", "collision", "dm"} - if anti_patterns and context in _anti_pattern_contexts: - lines.append(f" Never write {placeholder} as: {anti_patterns.strip()}") - - _pet_peeve_contexts = {"design", "collision"} - if pet_peeves and context in _pet_peeve_contexts: - lines.append(f" Pet peeves (will react if triggered): {pet_peeves}") - - identity_block = "\n".join(lines) - - sections = [ - f"IDENTITY: You are {placeholder} ({tenure} tenure). Role: {p.get('social_role', 'Contributor')}.", - f"COMPANY: You work at {COMPANY_NAME}, which {COMPANY_DESCRIPTION}.", - f"{identity_block}\n\nNever acknowledge being an AI. Stay in character.", - ] - - template = "\n".join(sections) - - # Group identical templates - if template not in card_to_names: - card_to_names[template] = [] - card_to_names[template].append(name) - - parts = [] - for template, group_names in card_to_names.items(): - combined_names = " / ".join(group_names) - - final_card = template.replace("__NAMES_PLACEHOLDER__", combined_names) - - history_lines = [] - for n in group_names: - if n in name_to_history: - prefix = f"{n}: " if len(group_names) > 1 else "" - history_lines.append(f"{prefix}{name_to_history[n]}") - - if history_lines: - final_card += "\n\nRECENT HISTORY:\n" + "\n".join(history_lines) - - if is_single: - return final_card - else: - parts.append(f"PERSONA(S) FOR {combined_names}:\n{final_card}") - - return "\n\n".join(parts) +persona_utils = PersonaUtils() diff --git a/tests/conftest.py b/tests/conftest.py index 2bc7c8c..397dcbf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,3 @@ -from config_loader import ALL_NAMES import pytest import mongomock import builtins diff --git a/tests/test_causal_chain_handler.py b/tests/test_causal_chain_handler.py index 9a125bc..677cdcd 100644 --- a/tests/test_causal_chain_handler.py +++ b/tests/test_causal_chain_handler.py @@ -5,7 +5,7 @@ """ import pytest -from unittest.mock import patch, MagicMock +from unittest.mock import patch from causal_chain_handler import ( CausalChainHandler, RecurrenceDetector, diff --git a/tests/test_crm_system.py b/tests/test_crm_system.py index 1922f01..7de043f 100644 --- a/tests/test_crm_system.py +++ b/tests/test_crm_system.py @@ -6,7 +6,6 @@ """ import pytest -from datetime import datetime from crm_system import CRMSystem, NullCRMSystem diff --git a/tests/test_flow.py b/tests/test_flow.py index 0f14d04..718b699 100644 --- a/tests/test_flow.py +++ b/tests/test_flow.py @@ -23,10 +23,13 @@ def mock_flow(): sim.state.day = 1 sim.state.system_health = 100 sim._mem.log_slack_messages = MagicMock(return_value=("", "")) - return sim + from crm_system import NullCRMSystem + from utils.persona_utils import persona_utils + + persona_utils.configure(crm=NullCRMSystem()) -### --- Bug Catching Tests --- + return sim def test_embed_and_count_recursion_fix(mock_flow): diff --git a/tests/test_memory.py b/tests/test_memory.py index d1ad33d..2edb5fa 100644 --- a/tests/test_memory.py +++ b/tests/test_memory.py @@ -911,45 +911,10 @@ def test_recall_with_rewrite_falls_back_without_llm(): result = mem.recall_with_rewrite( raw_query="kubernetes pod crash loop", n=3, - llm_callable=None, ) assert isinstance(result, str) -def test_recall_with_rewrite_uses_rewritten_query(): - """ - When llm_callable is provided, recall_with_rewrite() must embed the - rewritten query, not the raw one. The rewrite step is HyDE — the LLM - generates a hypothetical passage that embeds closer to real documents. - """ - from memory import Memory - - mem = Memory() - mem._artifacts.aggregate = MagicMock(return_value=[]) - mem._artifacts.count_documents = MagicMock(return_value=0) - - captured_queries = [] - original_context = mem.context_for_prompt - - def capturing_context(query, **kwargs): - captured_queries.append(query) - return original_context(query, **kwargs) - - mem.context_for_prompt = capturing_context - - llm = MagicMock( - return_value="A runbook describing Kubernetes pod restart policies and backoff intervals." - ) - - mem.recall_with_rewrite(raw_query="k8s restarts", n=3, llm_callable=llm) - - assert len(captured_queries) == 1 - assert captured_queries[0] != "k8s restarts", ( - "recall_with_rewrite() passed the raw query to context_for_prompt instead " - "of the LLM-rewritten passage. The HyDE rewrite step is being skipped." - ) - - def test_stats_reflects_artifact_and_event_counts(make_test_memory): """ stats() must report accurate counts from MongoDB — not stale cached values. diff --git a/tests/test_routing.py b/tests/test_routing.py index 8f589f3..1106d61 100644 --- a/tests/test_routing.py +++ b/tests/test_routing.py @@ -8,7 +8,10 @@ @pytest.fixture def mock_handler(): # Safely bypass the persona voice engine so it doesn't query mocked graphs - with patch("normal_day.get_voice_card", return_value="Mock backstory"): + with patch( + "utils.persona_utils.persona_utils.get_voice_card", + return_value="Mock backstory", + ): config = { "simulation": { "domain": "test.com",