{
  "run_id": "cymraegbench-claude-opus-5-5-v0.1",
  "created_at": "2026-09-23T12:36:03.528354+00:00",
  "status": "completed",
  "seed": 1,
  "max_cases_per_suite": 100,
  "techiaith_revision": "47839d2147c97fd2f10a52dd36751608e5fa36bf",
  "models_at_run_start": [
    {
      "id": "claude-opus-5-5",
      "label": "Claude Opus 5.5",
      "provider": "anthropic",
      "api_model": "claude-opus-5-5",
      "api_key_env": "ANTHROPIC_API_KEY",
      "base_url": "https://api.anthropic.com/v1",
      "enabled": true,
      "tier": "scoreboard",
      "base_url_env": null,
      "input_usd_per_million": 4.0,
      "output_usd_per_million": 20.0,
      "reasoning_effort": "low",
      "min_output_tokens": 512,
      "repetition_penalty": null,
      "supports_seed": true,
      "supports_temperature": true,
      "weight_access": null,
      "enable_thinking": null,
      "request_timeout_seconds": null
    }
  ],
  "models": [
    {
      "id": "claude-opus-5-5",
      "label": "Claude Opus 5.5",
      "provider": "anthropic",
      "api_model": "claude-opus-5-5",
      "api_key_env": "ANTHROPIC_API_KEY",
      "base_url": "https://api.anthropic.com/v1",
      "enabled": true,
      "tier": "scoreboard",
      "base_url_env": null,
      "input_usd_per_million": 4.0,
      "output_usd_per_million": 20.0,
      "reasoning_effort": "low",
      "min_output_tokens": 2048,
      "repetition_penalty": null,
      "supports_seed": true,
      "supports_temperature": true,
      "weight_access": null,
      "enable_thinking": null,
      "request_timeout_seconds": null
    }
  ],
  "suites": [
    "arc-welsh-easy",
    "arc-welsh-challenge",
    "piqa-welsh",
    "truthfulqa-welsh-mc1",
    "xnli-brit-cy",
    "mgsm-cy",
    "copa-cy",
    "wnli-cy",
    "welsh-cefr",
    "welsh-lexicon",
    "welsh-grammar",
    "welsh-yes-no",
    "welsh-obscenities",
    "welsh-bilingual-placenames",
    "welsh-legislation-translation",
    "welsh-registers",
    "welsh-mmlu-lite",
    "welsh-toxigen",
    "welsh-arc-easy-mini-cy",
    "ccc-cywiro-esbonio",
    "ccc-priod-ddull",
    "ccc-termau",
    "ccc-crynhoi-dyfynnu",
    "ccc-cyfeiriadau",
    "ccc-cyfarwyddyd-cymraeg",
    "ccc-cyfarwyddyd-saesneg"
  ],
  "status_counts": {
    "completed": 2053
  },
  "retry_policy": "Rows with status=error were retried in place. Successful rows were not rerun; the final model configuration records the higher output ceilings used for retries. Persistent empty or length-limited model outputs are status=invalid and score zero; infrastructure errors still block publication.",
  "scoring_policy": {
    "binary_suites": [
      "welsh-grammar",
      "welsh-toxigen"
    ],
    "scorer": "binary_choice",
    "description": "Accept an explicit leading Y/N label with optional following explanation; do not search arbitrary prose for a label.",
    "strict_format_overall_score": 77.83,
    "audited_overall_score": 87.55,
    "rejected_preliminary_score": 87.61,
    "audit_note": "The preliminary generic-choice heuristic misread an N in prose for welsh-grammar:221:29113e004891 even though the explicit final answer was Y and the target was N."
  },
  "formula": {
    "overall_score": "50% macro mean of 9 objective suites + 50% macro mean of 9 Techiaith suites",
    "translation": "Corpus BLEU, reported separately"
  }
}