{
  "revision": "2026-10-22",
  "primaryComparison": "RL seed 2 versus the same SFT checkpoint; paired on the same 750 observed claims",
  "metrics": [
    {
      "id": "frontier-baseline",
      "metric": "Frontier model baseline",
      "value": "59.7% pass@1",
      "status": "measured",
      "scope": "GPT 6.1; 95% CI [56.8, 62.7] on the balanced 750-claim test."
    },
    {
      "id": "rl-gain",
      "metric": "Held-out capability gain after RL",
      "value": "+0.8 pp overall; +7.6 pp on denied claims",
      "status": "measured",
      "scope": "Seed 2 vs SFT. Paired 95% CIs: overall [−1.2, 2.8]; denied [4.4, 11.2]. Seed 1 overall: −4.5 pp."
    },
    {
      "id": "unseen-family",
      "metric": "Unseen task-family gain",
      "value": "Not measured",
      "status": "unmeasured",
      "scope": "All six constructed rule families appear in training. Their test score is not an unseen-family transfer result."
    },
    {
      "id": "production-holdout",
      "metric": "Production-derived holdout gain",
      "value": "+0.8 pp incremental RL; +7.5 pp for SFT + RL",
      "status": "measured",
      "scope": "Same 750-claim test, not an independent replication. Pipeline vs base CI [4.7, 10.1]; incremental RL CI includes zero."
    },
    {
      "id": "sim2real",
      "metric": "Sim2Real transfer efficiency",
      "value": "Not applicable",
      "status": "not_applicable",
      "scope": "Training and evaluation use production-derived claims. The reported setup has no separate simulator-to-production transfer stage."
    },
    {
      "id": "human-agreement",
      "metric": "Human / reward agreement",
      "value": "Not measured",
      "status": "unmeasured",
      "scope": "Labels are observed insurer remittances. No independent expert agreement study has been completed."
    },
    {
      "id": "reward-false-positive",
      "metric": "Reward false-positive rate",
      "value": "Not established",
      "status": "unmeasured",
      "scope": "Requires an independently adjudicated correctness set. Targeted mutation checks do not estimate this rate."
    },
    {
      "id": "reward-exploits",
      "metric": "Reward exploit rate after red-teaming",
      "value": "0 / 2,247 fixed attacks rewarded (0%)",
      "status": "scoped",
      "scope": "Targeted audit only: fabricated lines, excess duplicates, and contradictory claim classes on 749 accepted references. Open-ended red-team exploit rate is unmeasured."
    },
    {
      "id": "scale",
      "metric": "Effective environment scale",
      "value": "18,445 training task IDs / 6 controlled families",
      "status": "scoped",
      "scope": "12,618 observed-claim tasks + 5,827 constructed tasks. Distinct IDs verified; semantic non-redundancy has not been audited."
    },
    {
      "id": "saturation",
      "metric": "Saturation point",
      "value": "Not established; 7,752 rollouts per RL seed observed",
      "status": "scoped",
      "scope": "Ten completed updates per seed. Seed 2 dev score peaks at step 15 then falls at step 20; this does not establish a saturation budget."
    },
    {
      "id": "throughput",
      "metric": "Throughput",
      "value": "2.83 rollout attempts/sec",
      "status": "scoped",
      "scope": "Seed 2: 7,752 attempts / 2,738 rollout seconds on the reference 8B/H100 setup. 0.42/sec including updates; excludes validation and setup."
    },
    {
      "id": "reliability",
      "metric": "Environment reliability",
      "value": "99.6% valid evaluated completions (1,550 / 1,556)",
      "status": "scoped",
      "scope": "Seed 2, final saved results after retries: prediction present, process and integrity pass. Includes model behavior; not an environment uptime SLA."
    },
    {
      "id": "cost",
      "metric": "Cost per 1M valid episodes",
      "value": "Not measured; approximately $93 reported experiment spend",
      "status": "scoped",
      "scope": "Reported total covers this version’s SFT, RL seeds, control, and evaluation (29 GPU-hours). No validated per-million unit cost."
    },
    {
      "id": "learning-efficiency",
      "metric": "Learning efficiency per $10K training",
      "value": "Not established; 3,876 rollouts to the selected RL checkpoint",
      "status": "scoped",
      "scope": "Seed 2 adds 0.8 pp after five updates. Logged rollout + update time: 2.72 GPU-hours, excluding SFT, validation and setup; no $10K extrapolation."
    },
    {
      "id": "synthetic-gap",
      "metric": "Synthetic substitution gap",
      "value": "Not measured",
      "status": "unmeasured",
      "scope": "No matched training comparison against a strongest cheap synthetic replica. Trivial baselines and constructed tasks do not substitute for that experiment."
    }
  ],
  "operations": {
    "seed2": {
      "rolloutAttempts": 7752,
      "rolloutSeconds": 2738,
      "updateSeconds": 15852,
      "completedUpdates": 10,
      "selectedCheckpointRollouts": 3876,
      "selectedCheckpointRolloutSeconds": 1296,
      "selectedCheckpointUpdateSeconds": 8510,
      "validEvaluatedCompletions": 1550,
      "evaluatedTasks": 1556
    },
    "costContext": {
      "reportedUSD": 93,
      "reportedGPUHours": 29,
      "scope": "PayerBench v2 SFT, both RL seeds, random-reward control, and evaluation; paper-reported estimate, not a metered unit-cost benchmark"
    }
  },
  "targetedVerifierAudit": {
    "date": "2026-10-22",
    "scope": "Three fixed mutation classes applied to accepted canonical reference answers on the 750-claim evaluation. This is a targeted deterministic regression audit, not open-ended red-teaming or a human correctness study.",
    "referenceAnswers": {
      "attempted": 750,
      "accepted": 749,
      "rejected": 1
    },
    "attacks": {
      "fabricated_line": {
        "attempts": 749,
        "rewarded": 0,
        "integrityRejected": 749
      },
      "excess_duplicate": {
        "attempts": 749,
        "rewarded": 0,
        "integrityRejected": 749
      },
      "contradictory_claim_class": {
        "attempts": 749,
        "rewarded": 0,
        "integrityRejected": 749
      }
    },
    "totalAttackAttempts": 2247,
    "totalAttacksRewarded": 0
  },
  "sources": [
    "Saved rescored evaluation aggregates",
    "Saved reference trainer metrics logs",
    "PayerBench v2 compute report",
    "Targeted verifier mutation audit, October 22 2026"
  ],
  "auditScriptSHA256": "accfc07ae00fe4b963d31c8fa6c37b312cfdc25daaf87109ba5f517141083d73"
}