{ "run_id": "cymraegbench-v0.1-estynedig", "created_at": "2026-08-10T13:01:52.422144+00:00", "updated_at": "2026-09-10", "status": "completed", "seed": 1, "max_cases_per_suite": "cymysg: hyd at 100 ar gyfer yr 21 model sampl; set lawn ar gyfer Jupiter a Chaernarfon Cymraeg Instruct", "techiaith_revision": "47839d2147c97fd2f10a52dd36751608e5fa36bf", "models": [ { "id": "gpt-5-6-sol", "label": "GPT-5.6 Sol", "provider": "openai", "api_model": "gpt-5.6-sol", "api_key_env": "OPENAI_API_KEY", "base_url": "https://api.openai.com/v1", "enabled": true, "tier": "scoreboard", "base_url_env": null, "input_usd_per_million": 5, "output_usd_per_million": 30, "reasoning_effort": "none", "min_output_tokens": null, "repetition_penalty": null }, { "id": "claude-opus-4-8", "label": "Claude Opus 4.8", "provider": "anthropic", "api_model": "claude-opus-4-8", "api_key_env": "ANTHROPIC_API_KEY", "base_url": "https://api.anthropic.com/v1", "enabled": true, "tier": "scoreboard", "base_url_env": null, "input_usd_per_million": 5, "output_usd_per_million": 25, "reasoning_effort": null, "min_output_tokens": 512, "repetition_penalty": null }, { "id": "claude-opus-5", "label": "Claude Opus 5", "provider": "anthropic", "api_model": "claude-opus-5", "api_key_env": "ANTHROPIC_API_KEY", "base_url": "https://api.anthropic.com/v1", "enabled": true, "tier": "scoreboard", "base_url_env": null, "input_usd_per_million": 5, "output_usd_per_million": 25, "reasoning_effort": "low", "min_output_tokens": 512, "repetition_penalty": null }, { "id": "claude-fable-5", "label": "Claude Fable 5", "provider": "anthropic", "api_model": "claude-fable-5", "api_key_env": "ANTHROPIC_API_KEY", "base_url": "https://api.anthropic.com/v1", "enabled": true, "tier": "scoreboard", "base_url_env": null, "input_usd_per_million": 10, "output_usd_per_million": 50, "reasoning_effort": "low", "min_output_tokens": 512, "repetition_penalty": null }, { "id": "grok-4-20", "label": "Grok 4.20", "provider": "openrouter", "api_model": "x-ai/grok-4.20", "api_key_env": "OPENROUTER_API_KEY", "base_url": "https://openrouter.ai/api/v1", "enabled": true, "tier": "scoreboard", "base_url_env": null, "input_usd_per_million": 1.25, "output_usd_per_million": 2.5, "reasoning_effort": "none", "min_output_tokens": 512, "repetition_penalty": null }, { "id": "kimi-k3", "label": "Kimi K3", "provider": "openrouter", "api_model": "moonshotai/kimi-k3", "api_key_env": "OPENROUTER_API_KEY", "base_url": "https://openrouter.ai/api/v1", "enabled": true, "tier": "scoreboard", "base_url_env": null, "input_usd_per_million": 3, "output_usd_per_million": 15, "reasoning_effort": "none", "min_output_tokens": 256, "repetition_penalty": null }, { "id": "muse-spark-1-2", "label": "Muse Spark 1.2", "provider": "openrouter", "api_model": "meta/muse-spark-1.2", "api_key_env": "OPENROUTER_API_KEY", "base_url": "https://openrouter.ai/api/v1", "enabled": true, "tier": "scoreboard", "base_url_env": null, "input_usd_per_million": 1.25, "output_usd_per_million": 4.25, "reasoning_effort": "low", "min_output_tokens": 8192, "repetition_penalty": null, "supports_seed": false, "weight_access": "open" }, { "id": "glm-5-2", "label": "GLM-5.2", "provider": "openrouter", "api_model": "z-ai/glm-5.2", "api_key_env": "OPENROUTER_API_KEY", "base_url": "https://openrouter.ai/api/v1", "enabled": true, "tier": "scoreboard", "base_url_env": null, "input_usd_per_million": 0.76, "output_usd_per_million": 2.42, "reasoning_effort": "none", "min_output_tokens": 256, "repetition_penalty": null }, { "id": "qwen3-8-27b", "label": "Qwen3.8 27B", "provider": "openrouter", "api_model": "qwen/qwen3.8-27b", "api_key_env": "OPENROUTER_API_KEY", "base_url": "https://openrouter.ai/api/v1", "enabled": true, "tier": "scoreboard", "base_url_env": null, "input_usd_per_million": 0.45, "output_usd_per_million": 3.2, "reasoning_effort": "low", "min_output_tokens": 32768, "repetition_penalty": null, "weight_access": "open" }, { "id": "qwen3-8-2p4t-a95b", "label": "Qwen3.8 2.4T A95B", "provider": "openrouter", "api_model": "qwen/qwen3.8-2.4t-a95b", "api_key_env": "OPENROUTER_API_KEY", "base_url": "https://openrouter.ai/api/v1", "enabled": true, "tier": "scoreboard", "base_url_env": null, "input_usd_per_million": 2, "output_usd_per_million": 6, "reasoning_effort": "low", "min_output_tokens": 32768, "repetition_penalty": null, "weight_access": "open" }, { "id": "deepseek-v4-flash-0731", "label": "DeepSeek V4-Flash-0731", "provider": "openrouter", "api_model": "deepseek/deepseek-v4-flash-0731", "api_key_env": "OPENROUTER_API_KEY", "base_url": "https://openrouter.ai/api/v1", "enabled": true, "tier": "scoreboard", "base_url_env": null, "input_usd_per_million": 0.09, "output_usd_per_million": 0.18, "reasoning_effort": "none", "min_output_tokens": 256, "repetition_penalty": null }, { "id": "mwydryn", "label": "Mwydryn", "provider": "openai_compatible", "api_model": "BangorAI/phi2-mwydryn-1", "api_key_env": "MWYDRYN_API_KEY", "base_url": "http://127.0.0.1:8003/v1", "enabled": false, "tier": "scoreboard", "base_url_env": "MWYDRYN_API_BASE", "input_usd_per_million": null, "output_usd_per_million": null, "reasoning_effort": null, "min_output_tokens": 512, "repetition_penalty": null }, { "id": "bangor-mistral-cymraeg-v2", "label": "BangorAI Mistral-7B-Cymraeg-Welsh-v2", "provider": "openai_compatible", "api_model": "BangorAI/Mistral-7B-Cymraeg-Welsh-v2", "api_key_env": "MISTRAL_CYMRAEG_API_KEY", "base_url": "http://127.0.0.1:8005/v1", "enabled": false, "tier": "scoreboard", "base_url_env": "MISTRAL_CYMRAEG_API_BASE", "input_usd_per_million": null, "output_usd_per_million": null, "reasoning_effort": null, "min_output_tokens": 512, "repetition_penalty": null }, { "id": "mwydryn-7b-v2", "label": "BangorAI Mwydryn 7B fersiwn 2", "provider": "openai_compatible", "api_model": "BangorAI/mwydryn-7b-fersiwn-2", "api_key_env": "MWYDRYN_7B_API_KEY", "base_url": "http://127.0.0.1:8006/v1", "enabled": false, "tier": "scoreboard", "base_url_env": "MWYDRYN_7B_API_BASE", "input_usd_per_million": null, "output_usd_per_million": null, "reasoning_effort": null, "min_output_tokens": 512, "repetition_penalty": null }, { "id": "techiaith-llama-3-2-1b-sft", "label": "Techiaith Llama 3.2 1B Welsh SFT", "provider": "openai_compatible", "api_model": "techiaith/llama-3.2-1b-welsh-sft", "api_key_env": "TECHIAITH_LLAMA_SFT_API_KEY", "base_url": "http://127.0.0.1:8007/v1", "enabled": false, "tier": "scoreboard", "base_url_env": "TECHIAITH_LLAMA_SFT_API_BASE", "input_usd_per_million": null, "output_usd_per_million": null, "reasoning_effort": null, "min_output_tokens": 512, "repetition_penalty": 1.15 }, { "id": "qwen3-5-9b-local-bf16", "label": "Qwen3.5 9B", "provider": "openai_compatible", "api_model": "Qwen/Qwen3.5-9B", "enabled": false, "tier": "scoreboard", "input_usd_per_million": null, "output_usd_per_million": null, "reasoning_effort": null, "min_output_tokens": 2048, "repetition_penalty": 1.05, "weight_access": "open" }, { "id": "jupiter-n-120b", "label": "Jupiter-N-120B", "provider": "openai_compatible", "api_model": "Jupiter-N-120B-BF16", "api_key_env": "JUPITER_API_KEY", "base_url": "http://127.0.0.1:8000/v1", "enabled": false, "tier": "scoreboard", "base_url_env": "JUPITER_API_BASE", "input_usd_per_million": null, "output_usd_per_million": null, "reasoning_effort": null, "min_output_tokens": 512, "repetition_penalty": null }, { "id": "caernarfon-3b-cymraeg-instruct-v0-1", "label": "Caernarfon 3B Cymraeg Instruct v0.1", "provider": "openai_compatible", "api_model": "caernarfon-3b-cymraeg-instruct-v0.1", "api_key_env": "CAERNARFON_API_KEY", "base_url": "http://127.0.0.1:8002/v1", "enabled": false, "tier": "scoreboard", "base_url_env": "CAERNARFON_API_BASE", "input_usd_per_million": null, "output_usd_per_million": null, "reasoning_effort": null, "min_output_tokens": 512, "repetition_penalty": null }, { "id": "qwen3-8-flash", "label": "Qwen3.8 Flash", "provider": "openrouter", "api_model": "qwen/qwen3.8-flash", "api_key_env": "OPENROUTER_API_KEY", "base_url": "https://openrouter.ai/api/v1", "enabled": true, "tier": "scoreboard", "base_url_env": null, "input_usd_per_million": 0.15, "output_usd_per_million": 0.47, "reasoning_effort": null, "min_output_tokens": 131072, "repetition_penalty": null }, { "id": "glm-5-3-flash", "label": "GLM-5.3 Flash", "provider": "openrouter", "api_model": "z-ai/glm-5.3-flash", "api_key_env": "OPENROUTER_API_KEY", "base_url": "https://openrouter.ai/api/v1", "enabled": true, "tier": "scoreboard", "base_url_env": null, "input_usd_per_million": 0.075, "output_usd_per_million": 0.25, "reasoning_effort": "low", "min_output_tokens": 32768, "repetition_penalty": null }, { "id": "glm-5-3", "label": "GLM-5.3", "provider": "openrouter", "api_model": "z-ai/glm-5.3", "api_key_env": "OPENROUTER_API_KEY", "base_url": "https://openrouter.ai/api/v1", "enabled": true, "tier": "scoreboard", "base_url_env": null, "input_usd_per_million": 1.4, "output_usd_per_million": 4.4, "reasoning_effort": "low", "min_output_tokens": 65536, "repetition_penalty": null }, { "id": "gpt-6-astra", "label": "GPT-6 Astra", "provider": "openrouter", "api_model": "openai/gpt-6-astra", "api_key_env": "OPENROUTER_API_KEY", "base_url": "https://openrouter.ai/api/v1", "enabled": true, "tier": "scoreboard", "base_url_env": null, "input_usd_per_million": 10, "output_usd_per_million": 50, "reasoning_effort": "low", "min_output_tokens": 32768, "repetition_penalty": null, "supports_seed": true, "supports_temperature": false, "weight_access": null, "enable_thinking": null, "request_timeout_seconds": null }, { "id": "deepseek-v41-flash", "label": "DeepSeek V4.1 Flash", "provider": "openai_compatible", "api_model": "deepseek-v41-flash", "api_key_env": "DEEPSEEK_V41_FLASH_API_KEY", "base_url": "http://127.0.0.1:8000/v1", "enabled": true, "tier": "scoreboard", "base_url_env": "DEEPSEEK_V41_FLASH_API_BASE", "input_usd_per_million": 0, "output_usd_per_million": 0, "reasoning_effort": "none", "min_output_tokens": 768, "repetition_penalty": null, "supports_seed": true, "supports_temperature": true, "weight_access": "open", "enable_thinking": false, "request_timeout_seconds": 1800 } ], "suites": [ "arc-welsh-easy", "arc-welsh-challenge", "piqa-welsh", "truthfulqa-welsh-mc1", "xnli-brit-cy", "mgsm-cy", "copa-cy", "wnli-cy", "welsh-cefr", "welsh-lexicon", "welsh-grammar", "welsh-yes-no", "welsh-obscenities", "welsh-bilingual-placenames", "welsh-legislation-translation", "welsh-registers", "welsh-mmlu-lite", "welsh-toxigen", "welsh-arc-easy-mini-cy" ], "status_counts": { "completed": 106621, "invalid": 2102, "refusal": 80 }, "retry_policy": "Ailbrofwyd rhesi status=error yn eu lle. Mae allbynnau gwag, allbynnau sy'n cyrraedd y terfyn tocynnau, a chyfyngiadau cyd-destun y model yn status=invalid ac yn sgorio sero. Mae gwallau seilwaith yn rhwystro cyhoeddi.", "formula": { "overall_score": "50% macro mean of 9 objective suites + 50% macro mean of 9 Techiaith suites", "translation": "Corpus BLEU, reported separately" }, "source_runs": [ { "run_id": "cymraegbench-v0.1", "model_count": 7, "cases_per_model": 1821, "includes_ccc": false }, { "run_id": "cymraegbench-claude-opus-5-v0.1", "model_count": 1, "cases_per_model": 2053, "includes_ccc": true }, { "run_id": "cymraegbench-muse-spark-1-2-v0.1", "model_count": 1, "cases_per_model": 2053, "includes_ccc": true }, { "run_id": "cymraegbench-v0.1-lleol-bangor-mistral-cymraeg-v2", "model_count": 1, "cases_per_model": 2053, "includes_ccc": true }, { "run_id": "cymraegbench-v0.1-lleol-mwydryn", "model_count": 1, "cases_per_model": 2053, "includes_ccc": true }, { "run_id": "cymraegbench-v0.1-lleol-mwydryn-7b-v2", "model_count": 1, "cases_per_model": 2053, "includes_ccc": true }, { "run_id": "cymraegbench-v0.1-lleol-techiaith-llama-3-2-1b-sft", "model_count": 1, "cases_per_model": 2053, "includes_ccc": true }, { "run_id": "cymraegbench-qwen3-5-9b-local-bf16-v0.1", "model_count": 1, "cases_per_model": 2053, "includes_ccc": true }, { "run_id": "cymraegbench-qwen3-8-27b-openrouter-v0.1", "model_count": 1, "cases_per_model": 2053, "includes_ccc": true }, { "run_id": "cymraegbench-qwen3-8-2p4t-a95b-openrouter-v0.1", "model_count": 1, "cases_per_model": 2053, "includes_ccc": true }, { "run_id": "jupiter-spark-bf16-v0.1", "model_count": 1, "cases_per_model": 33657, "includes_ccc": true, "scope": "set lawn" }, { "run_id": "caernarfon-3b-cymraeg-instruct-v0.1", "model_count": 1, "cases_per_model": 33657, "includes_ccc": true, "scope": "set lawn" }, { "run_id": "cymraegbench-qwen3-8-flash-openrouter-v0.1", "model_count": 1, "cases_per_model": 2053, "includes_ccc": true }, { "run_id": "cymraegbench-glm-5-3-flash-openrouter-v0.1", "model_count": 1, "cases_per_model": 2053, "includes_ccc": true }, { "run_id": "cymraegbench-glm-5-3-openrouter-v0.1", "model_count": 1, "cases_per_model": 2053, "includes_ccc": true }, { "run_id": "cymraegbench-gpt-6-astra-openrouter-v0.1", "model_count": 1, "cases_per_model": 2053, "includes_ccc": true }, { "run_id": "cymraegbench-deepseek-v41-flash-local-v0.1", "model_count": 1, "cases_per_model": 2053, "includes_ccc": true } ], "score_scope": { "core_suites": 19, "ccc_suites": 7, "overall_score_uses_ccc": false, "total_responses": 108803, "sampled_responses": 41489, "full_set_responses": 67314, "methodological_note": "Mae'r un fformiwla a'r un setiau'n cael eu defnyddio, ond mae'r 21 model sampl yn defnyddio hyd at 100 achos y set; mae Jupiter a Chaernarfon Cymraeg Instruct yn rhediadau set lawn. Mae'r rhediadau DeepSeek V4.1 Flash, GPT-6 Astra, GLM-5.3 a Qwen3.8 diweddar yn cynnwys Archwiliad Iaith CCC. Mae rhediad y model sylfaenol Caernarfon 3B wedi'i gadw yn yr archif ond wedi'i dynnu o'r prif sgorfwrdd gan fod y fersiwn instruct bellach ar gael." } }