"""Builds the script-vs-live end-to-end comparison PDF.

Reads /tmp/agent2_e2e_results.json (live agent output, from tests_e2e_compare.py) and compares
against the original DEMO_SCRIPT (the hardcoded reference script this rebuild replaces).
"""
import json

from reportlab.lib import colors
from reportlab.lib.pagesizes import letter
from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
from reportlab.lib.units import inch
from reportlab.platypus import (
    PageBreak, Paragraph, SimpleDocTemplate, Spacer, Table, TableStyle,
)

OUT_PATH = "/Users/sarah350252/SA AI agent 2/e2e_comparison_report.pdf"

REFERENCE = [
    {
        "script_turn": 1,
        "headline": "Customer is frustrated after being told the DD would be reapplied. "
                    "Acknowledge first, then investigate.",
        "next_moves": ["Acknowledge his frustration and apologise for the situation",
                       "Ask to place him on hold while you investigate",
                       "Check account notes to confirm what the previous advisor told him",
                       "Check billing history to confirm how many Direct Debits have been missed"],
        "suggested_wording": None,
    },
    {
        "script_turn": 2,
        "headline": "Issue identified: Previous advisor gave incorrect information.",
        "next_moves": ["Check for vulnerability — ask if anything is affecting the "
                       "customer's ability to keep up with payments."],
        "suggested_wording": "I can see there have been multiple missed payments. Is "
                              "anything affecting your ability to keep up with payments at "
                              "the moment? We ask so we can make sure we give you the right "
                              "support.",
    },
    {
        "script_turn": 3,
        "headline": "Vulnerability disclosed: reduced income due to short time at work.",
        "next_moves": ["Thank the customer for sharing this",
                       "Ask permission to record the vulnerability so the right support "
                       "can be provided"],
        "suggested_wording": "Thank you for letting me know — I'm sorry to hear that. With "
                              "your permission, I'd like to record this on your account so "
                              "we can make sure you get the right support. Are you happy "
                              "for me to do that?",
    },
    {
        "script_turn": 4,
        "headline": "Permission granted — record the vulnerability now.",
        "next_moves": ["Reinstate the DD mandate (customer is vulnerable)",
                       "Take manual payment for the outstanding balance",
                       "Explain that future missed payments could again put the policy at "
                       "risk of cancellation"],
        "suggested_wording": None,
    },
    {
        "script_turn": 5,
        "headline": "Status: Resolved",
        "next_moves": ["Log the complaint in AWS.",
                       "Select Complaint Cause: Agent gave incorrect information.",
                       "Select Complaint Sub-cause: Incorrect Direct Debit reapplication "
                       "advice."],
        "suggested_wording": None,
    },
    {
        "script_turn": 6,
        "headline": "Status: Resolved (case note)",
        "next_moves": ["Copy these over into customer account notes",
                       "Close the complaint as resolved"],
        "suggested_wording": None,
    },
]

# Maps each live-agent turn (from tests_e2e_compare.py) to the reference script turn it
# corresponds to in intent — not always 1:1, since the live agent runs the real two-procedure
# design (CMP-DD-REJECT-001 + PROC-VULNERABILITY-FLOW), which has finer-grained real steps
# than the old single flattened script. See the Structural Notes section of the PDF.
LIVE_TO_SCRIPT = {1: 1, 2: 2, 3: 3, 4: 4, 5: 4, 6: 5, 7: 6}

STRUCTURAL_NOTES = [
    "Turns 1–7 were run against the live, deployed agent (AWS App Runner, account "
    "039324591237, service esure-complaint-guidance-groq) using the real, esure-sourced "
    "procedures (CMP-DD-REJECT-001, PROC-VULNERABILITY-FLOW) — not the old hardcoded demo "
    "script. The reference script column below is the original scripted answer for "
    "comparison only; it is not fed to the live agent.",
    "The model is Groq-hosted openai/gpt-oss-120b, called directly (OpenAI SDK pointed at "
    "Groq's own endpoint) rather than through the EXL AI Gateway or AWS Bedrock. Both of "
    "those were tried first and ruled out for this account specifically: the gateway's own "
    "WAF (Incapsula) rejects this App Runner service's egress IP (confirmed by direct "
    "testing), and Bedrock's Anthropic models aren't enabled in this account without a "
    "separate use-case form. Groq was confirmed reachable from this exact service before "
    "switching, not assumed.",
    "Turn 2 of the live run supplies both the billing-history fact and the advisor-confirmation "
    "fact in one message, which the real procedure's S2+S3 steps both consume in a single "
    "pass — this naturally collapses back to the script's 6-turn pacing rather than needing "
    "two separate turns for what the script treats as one.",
    "The real procedure separates \"vulnerability confirmed\" (its own step, V3) from "
    "\"resolution offered\" (CMP-DD-REJECT-001's S5) — two genuinely distinct steps in the "
    "real source material. The old script's turn 4 compressed both into one reply. The live "
    "run therefore takes 2 turns (4 and 5 below) to cover what the script did in 1 — this is "
    "a structural consequence of using the real procedures, not a content deviation.",
    "The real CMP-DD-REJECT-001 procedure (v4, from esure's own Billing SOP.docx / skills "
    "doc) originally ended at S6 (log the complaint, give the reference number) with no "
    "separate case-note-posting step. At the user's explicit request, an S7 step was added "
    "on 2026-10-17 to match the script's separate \"ready\" turn — this step is clearly "
    "marked in knowledge_base/esure-procedures.json and knowledge_base/README.md as "
    "demo-authored content added for parity with the script, not itself from an esure "
    "source document, so it isn't mistaken for SME-sourced guidance later.",
]

KNOWN_FINDINGS = [
    ("Forced tool_choice unsupported", "The gateway's Bedrock Claude models reject a forced "
     "tool_choice (named function). Fixed by switching to tool_choice=\"auto\" with explicit "
     "\"you MUST call this tool\" prompting, per the gateway's own error message."),
    ("Vulnerability interrupt never exited", "The harness's step-position logic re-entered "
     "the vulnerability interrupt forever once triggered, because it never checked whether "
     "vulnerability_confirmed had already been set. Fixed by checking that flag before "
     "re-entering V2/V3."),
    ("Grounding judge missing current-turn context", "The grounding/relevance judge only saw "
     "prior conversation history, not the very message being responded to — so it couldn't "
     "see a fact (e.g. consent) confirmed in the current turn, and wrongly flagged the "
     "response for stating it. Fixed by including the current turn's note in the judge's "
     "context."),
    ("Grounding judge missing the step's own instruction", "The grounding judge only checked "
     "claims against explicitly cited documents, not the procedure step's own instruction "
     "text — even though that instruction IS real, verbatim source content from the ported "
     "procedure. This caused a false hold on turn 5 (the model correctly relayed the real "
     "S5 instruction to reinstate the mandate and take manual payment, but was held because "
     "that text wasn't separately supplied as \"source material\"). Fixed by including the "
     "step instruction in the grounding context."),
    ("Model over-elaborating a terse instruction", "On the S6 (closing/logging) step, the "
     "model initially added plausible-sounding but unsupported procedural detail (a "
     "\"dropdown list\", escalation routes, a read-the-number-back requirement) around a "
     "genuinely terse real instruction. The grounding check correctly held this. Fixed with "
     "an explicit prompt instruction: keep the answer as terse as the instruction when the "
     "instruction itself is terse, rather than filling in plausible procedure around it."),
    ("Relevance judge missing conversation context", "Judged alone, a bare confirmation like "
     "\"ready\" has no visible connection to a response that correctly proceeds with the next "
     "step — the relevance check initially called turn 7's correct response \"unrelated\" "
     "because it didn't re-explain context the representative already had. Fixed by giving "
     "the relevance judge the full conversation, with an explicit instruction that a short "
     "confirmation is responsive if the response correctly acts on it."),
    ("Switched off the EXL AI Gateway and AWS Bedrock entirely for this deployment", "Both "
     "were tested directly from this App Runner service and ruled out: the gateway's own WAF "
     "(Incapsula) rejects this service's egress IP outright (confirmed with a live request, "
     "not assumed), and this AWS account's Bedrock doesn't have Anthropic models enabled "
     "without a separate use-case form an account admin would need to submit. Groq's public "
     "API, called directly via the OpenAI SDK pointed at Groq's own endpoint, was confirmed "
     "reachable from this exact service and is what's live now."),
    ("litellm produced an intermittent, unreproducible model_not_found error", "Using litellm "
     "as a multi-provider translation layer in front of Groq intermittently failed with "
     "\"model `llama-3.3-70b-versatile` does not exist\" — a model this code never references "
     "anywhere. Root cause not fully isolated, but calling Groq's native OpenAI-compatible "
     "endpoint directly (dropping litellm as a dependency for this client) removed the "
     "failure mode entirely."),
    ("Groq's free-tier rate limit (8000 TPM) surfaced as what looked like a reasoning bug", "A "
     "429 from Groq, if unhandled, left a fact silently unextracted on that turn — "
     "indistinguishable from the model simply failing to understand an unambiguous \"yes\" "
     "unless you knew to look for it. Fixed by parsing Groq's own reported wait time from the "
     "429 body and retrying with it, rather than guessing a backoff."),
    ("Even at temperature=0, Groq occasionally missed an unambiguous yes/no confirmation", "An "
     "isolated call with the exact same input reliably extracted the right fact, but the same "
     "call inside the full multi-turn conversation occasionally didn't — residual "
     "non-determinism, a known characteristic of Groq's fast-inference hardware even at "
     "temperature=0. Added a narrow, harness-level deterministic check (only for the two "
     "binary yes/no confirmations the procedure explicitly waits on, only when unambiguous, "
     "never overriding a \"no\") as a safety net — not a replacement for the model's own "
     "extraction, which still runs and is still what any nuanced case depends on."),
    ("App Runner repeatedly rolled back to the old gateway config on update", "Updating the "
     "existing esure-complaint-guidance-v2 service's image/env config three times in a row "
     "resulted in long (20-40 min) stalls that ultimately rolled back to the original, "
     "already-working gateway configuration, even after confirming the correct code was "
     "actually built and pushed to ECR. Rather than keep fighting that service's apparent "
     "stuck revision history, a fresh service (esure-complaint-guidance-groq) was created "
     "from a clean slate with the image pinned by exact digest rather than the mutable "
     ":latest tag — it deployed correctly on the first attempt."),
]


def _p(text, style):
    return Paragraph(text.replace("\n", "<br/>"), style)


def build():
    live = json.load(open("/tmp/agent2_e2e_results.json"))
    styles = getSampleStyleSheet()
    body = ParagraphStyle("body", parent=styles["BodyText"], fontSize=9, leading=12)
    small = ParagraphStyle("small", parent=styles["BodyText"], fontSize=8, leading=10,
                            textColor=colors.HexColor("#555555"))
    h1 = styles["Title"]
    h2 = styles["Heading2"]
    h3 = styles["Heading3"]

    doc = SimpleDocTemplate(OUT_PATH, pagesize=letter,
                             topMargin=0.6 * inch, bottomMargin=0.6 * inch,
                             leftMargin=0.55 * inch, rightMargin=0.55 * inch)
    story = []

    story.append(Paragraph("Complaint Guidance Agent — End-to-End Comparison", h1))
    story.append(Paragraph(
        "Live agent, deployed on AWS App Runner (Groq-hosted model, real procedures, "
        "structured output) vs. the original hardcoded demo script, same missed-Direct-Debit "
        "complaint scenario. Run directly against the live production URL, not a local test.",
        styles["Normal"]))
    story.append(Spacer(1, 14))

    story.append(Paragraph("Result summary", h2))
    total = len(live)
    passed = sum(1 for t in live if t["ok"])
    story.append(Paragraph(
        f"{passed}/{total} live turns passed full verification (citation existence + "
        f"grounding + relevance) on the final run. All content is structured output "
        f"(Pydantic-validated <code>TurnResponse</code>: headline / next_moves / "
        f"suggested_wording / citations), not free-form prose — the same shape the "
        f"reference script uses, so it can be compared field-by-field below.", body))
    story.append(Spacer(1, 10))

    story.append(Paragraph("Structural notes — read before the turn-by-turn table", h2))
    for note in STRUCTURAL_NOTES:
        story.append(Paragraph("• " + note, body))
        story.append(Spacer(1, 4))
    story.append(PageBreak())

    story.append(Paragraph("Turn-by-turn comparison", h2))
    for turn in live:
        script_turn_num = LIVE_TO_SCRIPT.get(turn["turn"])
        ref = REFERENCE[script_turn_num - 1] if script_turn_num else None

        story.append(Paragraph(
            f"Live turn {turn['turn']} (stage {turn['stage']})"
            + (f" — corresponds to script turn {script_turn_num}" if ref else
               " — no corresponding script turn (see structural notes)"), h3))
        story.append(Paragraph(f"<b>Representative said:</b> {turn['input']}", small))
        story.append(Spacer(1, 4))

        rows = [[
            Paragraph("<b>Field</b>", body),
            Paragraph("<b>Reference script</b>", body),
            Paragraph("<b>Live agent (actual)</b>", body),
        ]]
        if ref:
            rows.append([_p("Headline", body), _p(ref["headline"], body),
                         _p(turn["headline"] or "(held — see verification)", body)])
            rows.append([_p("Next moves", body),
                         _p("\n".join("• " + m for m in ref["next_moves"]), body),
                         _p("\n".join("• " + m for m in turn["next_moves"]) or "(none)", body)])
            rows.append([_p("Suggested wording", body),
                         _p(ref["suggested_wording"] or "(none)", body),
                         _p(turn["suggested_wording"] or "(none)", body)])
        else:
            rows.append([_p("Headline", body), _p("(n/a)", body),
                         _p(turn["headline"] or "(held)", body)])
            rows.append([_p("Next moves", body), _p("(n/a)", body),
                         _p("\n".join("• " + m for m in turn["next_moves"]) or "(none)", body)])

        table = Table(rows, colWidths=[1.0 * inch, 2.8 * inch, 2.8 * inch])
        table.setStyle(TableStyle([
            ("GRID", (0, 0), (-1, -1), 0.5, colors.HexColor("#cccccc")),
            ("BACKGROUND", (0, 0), (-1, 0), colors.HexColor("#eef2f1")),
            ("VALIGN", (0, 0), (-1, -1), "TOP"),
            ("FONTSIZE", (0, 0), (-1, -1), 8.5),
        ]))
        story.append(table)
        story.append(Spacer(1, 4))

        verdict_color = "#1e6f5c" if turn["ok"] else "#b4530a"
        verdict_text = "PASSED verification" if turn["ok"] else "HELD — " + "; ".join(turn["violations"])
        story.append(Paragraph(
            f'<font color="{verdict_color}"><b>{verdict_text}</b></font>', body))
        story.append(Spacer(1, 14))

    story.append(PageBreak())
    story.append(Paragraph("Real issues found and fixed during this testing", h2))
    story.append(Paragraph(
        "Each of these was caught by actually running the scenario end-to-end against the "
        "live gateway, not by inspection — listed here so the fixes are traceable to a real "
        "failure, not assumed.", body))
    story.append(Spacer(1, 8))
    for title, desc in KNOWN_FINDINGS:
        story.append(Paragraph(f"<b>{title}</b>", body))
        story.append(Paragraph(desc, body))
        story.append(Spacer(1, 8))

    doc.build(story)
    print("Wrote", OUT_PATH)


if __name__ == "__main__":
    build()