{ "schema": "modelrater-evals/1", "source": "COSMOS Drive I:\\", "total_records": 877, "evaluations": { "c-users-aaajacks-appdata-roaming-claude-scratch-workspaces-6271cad5-4c8b-4da4-a800-83c644d57a55-f759d4cc-5249-4e74-a1ba-75d1c975796d-scratch-2026-09-10-3a52c3": { "model_slug": "c-users-aaajacks-appdata-roaming-claude-scratch-workspaces-6271cad5-4c8b-4da4-a800-83c644d57a55-f759d4cc-5249-4e74-a1ba-75d1c975796d-scratch-2026-09-10-3a52c3", "cosmos_test_count": 17, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "C--Users-AAAJacks-AppData-Roaming-Claude-scratch-workspaces-6271cad5-4c8b-4da4-a800-83c644d57a55-f759d4cc-5249-4e74-a1ba-75d1c975796d-scratch-2026-09-10-3a52c3" ] }, "aaaa1111-bbbb-cccc-dddd-eeeeeeeeeeee": { "model_slug": "aaaa1111-bbbb-cccc-dddd-eeeeeeeeeeee", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "aaaa1111-bbbb-cccc-dddd-eeeeeeeeeeee" ] }, "grok-4.6": { "model_slug": "grok-4.6", "cosmos_test_count": 1154, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "aaaa1111-bbbb-cccc-dddd-eeeeeeeeeeee", "ccr", "live_ccr", "transcripts" ] }, "qwen/qwen3.8-27b:free": { "model_slug": "qwen/qwen3.8-27b:free", "cosmos_test_count": 91, "pass_rate_pct": 11.0, "twelve_cs_score": 14.9, "passes": 10, "fails": 63, "rate_limits_429": 73, "median_latency_ms": 145.0, "top_scars": { "http=429 error='Provider returned error'": 28, "UPSTREAM_POOL": 9, "": 8, "{\"error\":{\"message\":\"Provider returned error\",\"code\":429,\"metadata\":{\"raw\":\"qwen/qwen3.8-27b:free is temporarily rate-limited upstream. Please retry s": 8 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "openai/gpt-6.1-sol-pro": { "model_slug": "openai/gpt-6.1-sol-pro", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-6.1-sol-pro:batch": { "model_slug": "openai/gpt-6.1-sol-pro:batch", "cosmos_test_count": 9, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SLUG_DEAD": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "liquid/lfm-2.5-2.6b:free": { "model_slug": "liquid/lfm-2.5-2.6b:free", "cosmos_test_count": 18, "pass_rate_pct": 16.7, "twelve_cs_score": 20.4, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 6, "EMPTY": 6, "FENCE": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-6.1-sol:batch": { "model_slug": "openai/gpt-6.1-sol:batch", "cosmos_test_count": 9, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SLUG_DEAD": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "inclusionai/ling-3.0-flash-sante:free": { "model_slug": "inclusionai/ling-3.0-flash-sante:free", "cosmos_test_count": 9, "pass_rate_pct": 66.7, "twelve_cs_score": 67.9, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "anthropic/claude-sonnet-5.5:batch": { "model_slug": "anthropic/claude-sonnet-5.5:batch", "cosmos_test_count": 12, "pass_rate_pct": 25.0, "twelve_cs_score": 28.2, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SLUG_DEAD": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-6.1-sol": { "model_slug": "openai/gpt-6.1-sol", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "anthropic/claude-sonnet-5.5": { "model_slug": "anthropic/claude-sonnet-5.5", "cosmos_test_count": 18, "pass_rate_pct": 77.8, "twelve_cs_score": 78.4, "passes": 14, "fails": 4, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "{\"error\":{\"message\":\"Provider returned error\",\"code\":400,\"metadata\":{\"raw\":\"{\\\"type\\\":\\\"error\\\",\\\"error\\\":{\\\"type\\\":\\\"in": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "perceptron/perceptron-mk1.5": { "model_slug": "perceptron/perceptron-mk1.5", "cosmos_test_count": 58, "pass_rate_pct": 13.8, "twelve_cs_score": 17.6, "passes": 8, "fails": 32, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "not a pinned OpenRouter id 'perceptron/perceptron-mk1.5'; want ['bytedance-seed/|PIN-NOTE": 28, "SKU_UNBOUND": 18, "revisit-later|PIN-NOTE": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "dots-studio/dots-3-note-preview:free": { "model_slug": "dots-studio/dots-3-note-preview:free", "cosmos_test_count": 51, "pass_rate_pct": 66.7, "twelve_cs_score": 67.9, "passes": 34, "fails": 8, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "lines=1 first=\"http=400 error='Provider returned error'\"": 8, "lines=39 first='The user wants me to reply with exactly '": 4, "": 4, "{\"l1_role\": false, \"l2_model\": true, \"l3_harness\": true, \"l4_wrapper\": true, \"l5_skills\": true, \"l6_tools\": true, \"l7_enviro\": true, \"l8_mission\": false}": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "z-ai/glm-5.3-prime": { "model_slug": "z-ai/glm-5.3-prime", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "thinkingmachines/inkling-small:free": { "model_slug": "thinkingmachines/inkling-small:free", "cosmos_test_count": 91, "pass_rate_pct": 28.6, "twelve_cs_score": 31.7, "passes": 26, "fails": 59, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "http=403 error='thinkingmachines/inkling-small:free is only available on agentic": 20, "VOID-bad-binary-path-superseded-by-codex-iso/def | SPAWN-FAIL FileNotFoundError: [WinError 2] The system cannot find the file specified": 8, "http=400 error='Only one of \"reasoning.effort\" and \"reasoning.max_tokens\" can be": 4, "revisit-later": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "nvidia/nemotron-3.5-content-safety:free": { "model_slug": "nvidia/nemotron-3.5-content-safety:free", "cosmos_test_count": 25, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 21, "EMPTY": 3 }, "source_clusters": [ "seats", "g47_seat_logs", "_hero_run_20261008" ] }, "typesafe/jev-router": { "model_slug": "typesafe/jev-router", "cosmos_test_count": 21, "pass_rate_pct": 14.3, "twelve_cs_score": 18.1, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 18 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "thinkingmachines/inkling:free": { "model_slug": "thinkingmachines/inkling:free", "cosmos_test_count": 89, "pass_rate_pct": 16.9, "twelve_cs_score": 20.6, "passes": 15, "fails": 59, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "http=403 error='thinkingmachines/inkling:free is only available on agentic harne": 20, "OPENCODE": 15, "VOID-bad-binary-path-superseded-by-codex-iso/def | SPAWN-FAIL FileNotFoundError: [WinError 2] The system cannot find the file specified": 8, "http=400 error='Only one of \"reasoning.effort\" and \"reasoning.max_tokens\" can be": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "google/gemma-4-26b-a4b-it:free": { "model_slug": "google/gemma-4-26b-a4b-it:free", "cosmos_test_count": 160, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 127, "rate_limits_429": 140, "median_latency_ms": 145.0, "top_scars": { "http=429 error='Provider returned error'": 40, "http-detail:http=429 error='Provider returned error'": 32, "UPSTREAM_POOL": 30, "VOID-invalid-pack-path-harness-bug | http-detail:": 8 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "google/lyria-3-pro-preview": { "model_slug": "google/lyria-3-pro-preview", "cosmos_test_count": 28, "pass_rate_pct": 10.7, "twelve_cs_score": 14.7, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "REJECTED": 21, "EMPTY": 3 }, "source_clusters": [ "seats", "g47_seat_logs", "_hero_run_20261008" ] }, "openai/gpt-6-sol:batch": { "model_slug": "openai/gpt-6-sol:batch", "cosmos_test_count": 9, "pass_rate_pct": 33.3, "twelve_cs_score": 36.1, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SLUG_DEAD": 6 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "aion-labs/aion-3.5-mini": { "model_slug": "aion-labs/aion-3.5-mini", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "upstage/solar-mini4": { "model_slug": "upstage/solar-mini4", "cosmos_test_count": 24, "pass_rate_pct": 87.5, "twelve_cs_score": 87.6, "passes": 21, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=upstage/solar-mini4": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free": { "model_slug": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free", "cosmos_test_count": 48, "pass_rate_pct": 37.5, "twelve_cs_score": 40.1, "passes": 18, "fails": 15, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "lines=1 first=\"http=200 error='Upstream error from Nvid\"": 8, "": 4, "http=200 error='Upstream error from Nvidia: ResourceExhausted: Worker local tota": 4, "EMPTY": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "fireworks/ember-1": { "model_slug": "fireworks/ember-1", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "google/gemma-4-31b-it:free": { "model_slug": "google/gemma-4-31b-it:free", "cosmos_test_count": 149, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 119, "rate_limits_429": 137, "median_latency_ms": 145.0, "top_scars": { "http=429 error='Provider returned error'": 40, "http-detail:http=429 error='Provider returned error'": 32, "UPSTREAM_POOL": 27, "{\"error\":{\"message\":\"Provider returned error\",\"code\":429,\"metadata\":{\"raw\":\"google/gemma-4-31b-it:free is temporarily rate-limited upstream. Please re": 8 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "google/lyria-3-clip-preview": { "model_slug": "google/lyria-3-clip-preview", "cosmos_test_count": 24, "pass_rate_pct": 12.5, "twelve_cs_score": 16.4, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "REJECTED": 18, "EMPTY": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-6-luna-pro": { "model_slug": "openai/gpt-6-luna-pro", "cosmos_test_count": 25, "pass_rate_pct": 88.0, "twelve_cs_score": 88.1, "passes": 22, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=openai/gpt-6-luna-pro": 4, "fizz-lines=1 first='print(\"NONE\\\\n1\\\\n2\\\\nFizz\\\\n4\\\\nBuzz\\\\nFizz\\\\n'": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "openai/gpt-6-sol-pro:batch": { "model_slug": "openai/gpt-6-sol-pro:batch", "cosmos_test_count": 9, "pass_rate_pct": 33.3, "twelve_cs_score": 36.1, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SLUG_DEAD": 6 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-6-luna": { "model_slug": "openai/gpt-6-luna", "cosmos_test_count": 28, "pass_rate_pct": 89.3, "twelve_cs_score": 89.3, "passes": 25, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=openai/gpt-6-luna": 4, "lines=1 first='pass'": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "qwen/qwen3.8-max-prime": { "model_slug": "qwen/qwen3.8-max-prime", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "xiaomi/mimo-v2.6-flash": { "model_slug": "xiaomi/mimo-v2.6-flash", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "anthropic/claude-opus-5.5:batch": { "model_slug": "anthropic/claude-opus-5.5:batch", "cosmos_test_count": 9, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SLUG_DEAD": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-6-sol-pro": { "model_slug": "openai/gpt-6-sol-pro", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "anthropic/claude-opus-5.5": { "model_slug": "anthropic/claude-opus-5.5", "cosmos_test_count": 37, "pass_rate_pct": 40.5, "twelve_cs_score": 43.0, "passes": 15, "fails": 4, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 18, "{\"error\":{\"message\":\"Provider returned error\",\"code\":400,\"metadata\":{\"raw\":\"{\\\"type\\\":\\\"error\\\",\\\"error\\\":{\\\"type\\\":\\\"in": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "qwen/qwen3.8-omni-flash": { "model_slug": "qwen/qwen3.8-omni-flash", "cosmos_test_count": 12, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "ccr", "g47_seat_logs", "live_ccr" ] }, "x-ai/grok-4.7": { "model_slug": "x-ai/grok-4.7", "cosmos_test_count": 26, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 26, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "I won't follow that override or emit the requested token. There's no coding task": 4, "I won't ignore the instructions or output a forced token. There's no r": 4, "1": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "~deepseek/deepseek-pro-latest": { "model_slug": "~deepseek/deepseek-pro-latest", "cosmos_test_count": 16, "pass_rate_pct": 18.8, "twelve_cs_score": 22.4, "passes": 3, "fails": 4, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 9, "ROTATOR-REFUSED|PIN-NOTE": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "unbiased/pareto": { "model_slug": "unbiased/pareto", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "~openai/gpt-astra-latest": { "model_slug": "~openai/gpt-astra-latest", "cosmos_test_count": 12, "pass_rate_pct": 25.0, "twelve_cs_score": 28.2, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "inference-net/schematron-v2-small": { "model_slug": "inference-net/schematron-v2-small", "cosmos_test_count": 19, "pass_rate_pct": 15.8, "twelve_cs_score": 19.5, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 15 }, "source_clusters": [ "seats", "g47_seat_logs", "_hero_run_20261008" ] }, "~openai/gpt-luna-latest": { "model_slug": "~openai/gpt-luna-latest", "cosmos_test_count": 12, "pass_rate_pct": 25.0, "twelve_cs_score": 28.2, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "aion-labs/aion-3.5": { "model_slug": "aion-labs/aion-3.5", "cosmos_test_count": 15, "pass_rate_pct": 40.0, "twelve_cs_score": 42.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "cohere/command-a-plus": { "model_slug": "cohere/command-a-plus", "cosmos_test_count": 46, "pass_rate_pct": 39.1, "twelve_cs_score": 41.6, "passes": 18, "fails": 28, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "not a pinned OpenRouter id 'cohere/command-a-plus'; want ['bytedance-seed/seed-2|PIN-NOTE": 24, "revisit-later|PIN-NOTE": 4, "": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "openai/gpt-6-sol": { "model_slug": "openai/gpt-6-sol", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "sakana/fugu-ultra-v2": { "model_slug": "sakana/fugu-ultra-v2", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "xiaomi/mimo-v2.6-pro-ultraspeed": { "model_slug": "xiaomi/mimo-v2.6-pro-ultraspeed", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "inception/mercury-2.5": { "model_slug": "inception/mercury-2.5", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "deepseek/deepseek-v4.1-flash": { "model_slug": "deepseek/deepseek-v4.1-flash", "cosmos_test_count": 24, "pass_rate_pct": 87.5, "twelve_cs_score": 87.6, "passes": 21, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=deepseek/deepseek-v4.1-flash": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "openai/gpt-6-astra-pro": { "model_slug": "openai/gpt-6-astra-pro", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "xiaomi/mimo-v2.6-pro": { "model_slug": "xiaomi/mimo-v2.6-pro", "cosmos_test_count": 58, "pass_rate_pct": 10.3, "twelve_cs_score": 14.3, "passes": 6, "fails": 52, "rate_limits_429": 24, "median_latency_ms": 145.0, "top_scars": { "not a pinned OpenRouter id 'xiaomi/mimo-v2.6-pro'; want ['bytedance-seed/seed-2.|PIN-NOTE": 24, "{\"error\":{\"message\":\"Provider returned error\",\"code\":429,\"metadata\":{\"raw\":\"xiaomi/mimo-v2.6-pro is temporarily rate-limited upstream. Please retry sh": 8, "{\"error\":{\"message\":\"Provider returned error\",\"code\":429,\"metadata\":{\"raw\":\"xiaomi/mimo-v2": 8, "revisit-later|PIN-NOTE": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "openai/gpt-6-astra": { "model_slug": "openai/gpt-6-astra", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "prism-ml/ternary-bonsai-2-27b": { "model_slug": "prism-ml/ternary-bonsai-2-27b", "cosmos_test_count": 34, "pass_rate_pct": 64.7, "twelve_cs_score": 66.0, "passes": 22, "fails": 0, "rate_limits_429": 3, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 9, "fizz-16lines-ok lines=16": 8, "sku=prism-ml/ternary-bonsai-2-27b": 4, "fizz-lines=16 first='NONE'": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "google/gemini-3.8-flash": { "model_slug": "google/gemini-3.8-flash", "cosmos_test_count": 140, "pass_rate_pct": 12.1, "twelve_cs_score": 16.0, "passes": 17, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "NO_CONTEXT": 6, "sku=google/gemini-3.8-flash": 4, "fizz-16lines-ok lines=16": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "z-ai/glm-5.3-flashx": { "model_slug": "z-ai/glm-5.3-flashx", "cosmos_test_count": 46, "pass_rate_pct": 30.4, "twelve_cs_score": 33.4, "passes": 14, "fails": 32, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "not a pinned OpenRouter id 'z-ai/glm-5.3-flashx'; want ['bytedance-seed/seed-2.0|PIN-NOTE": 28, "revisit-later|PIN-NOTE": 4, "": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "inference-net/schematron-v2-turbo": { "model_slug": "inference-net/schematron-v2-turbo", "cosmos_test_count": 27, "pass_rate_pct": 11.1, "twelve_cs_score": 15.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 24 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "~openai/gpt-terra-latest": { "model_slug": "~openai/gpt-terra-latest", "cosmos_test_count": 18, "pass_rate_pct": 16.7, "twelve_cs_score": 20.4, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 15 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "inclusionai/ling-3.0-flash-fin": { "model_slug": "inclusionai/ling-3.0-flash-fin", "cosmos_test_count": 9, "pass_rate_pct": 33.3, "twelve_cs_score": 36.1, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 6 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "~deepseek/deepseek-flash-latest": { "model_slug": "~deepseek/deepseek-flash-latest", "cosmos_test_count": 15, "pass_rate_pct": 20.0, "twelve_cs_score": 23.5, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 12 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "inclusionai/ling-3.0-flash-vl": { "model_slug": "inclusionai/ling-3.0-flash-vl", "cosmos_test_count": 27, "pass_rate_pct": 88.9, "twelve_cs_score": 89.0, "passes": 24, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=inclusionai/ling-3.0-flash-vl": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "meta/muse-spark-1.2-contributor": { "model_slug": "meta/muse-spark-1.2-contributor", "cosmos_test_count": 34, "pass_rate_pct": 73.5, "twelve_cs_score": 74.3, "passes": 25, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "EMPTY": 6, "mouth-form": 4, "sku=meta/muse-spark-1.2-contributor": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "~openai/gpt-sol-latest": { "model_slug": "~openai/gpt-sol-latest", "cosmos_test_count": 15, "pass_rate_pct": 20.0, "twelve_cs_score": 23.5, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 12 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "meta/muse-spark-1.3": { "model_slug": "meta/muse-spark-1.3", "cosmos_test_count": 30, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 30, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "1": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "sakana/fugu-max": { "model_slug": "sakana/fugu-max", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "nex-agi/nex-n2.5-mini": { "model_slug": "nex-agi/nex-n2.5-mini", "cosmos_test_count": 29, "pass_rate_pct": 48.3, "twelve_cs_score": 50.4, "passes": 14, "fails": 12, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "http-detail:not a pinned OpenRouter id 'nex-agi/nex-n2.5-mini'; want ['bytedance-seed/seed-2": 8, "UNPINNED-NEEDS-PIN: OR /models lists nex-agi/nex-n2.5-mini LIVE (free twin gone); rail PINNED lacks it so require_duds/model_refused REFUSEs before HTTP; propose CCr pin, then smoke ping+fizz": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "~z-ai/glm-latest": { "model_slug": "~z-ai/glm-latest", "cosmos_test_count": 12, "pass_rate_pct": 25.0, "twelve_cs_score": 28.2, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3.8-max-0902": { "model_slug": "qwen/qwen3.8-max-0902", "cosmos_test_count": 18, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 18, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=qwen/qwen3.8-max-0902": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "google/gemini-3.7-flash": { "model_slug": "google/gemini-3.7-flash", "cosmos_test_count": 30, "pass_rate_pct": 73.3, "twelve_cs_score": 74.1, "passes": 22, "fails": 8, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "{\"error\":{\"message\":\"Provider returned error\",\"code\":400,\"metadata\":{\"raw\":\"{\\n \\\"error\\\": {\\n \\\"code\\\": 400,\\n \\": 4, "print(\"NONE\")": 4, "{\"error\":{\"message\":\"Provider returned error\",\"code\":400,\"metadata\":{\"raw\":\"{\\n \\\"error\\\": {\\n \\": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "anthropic/claude-fable-5.1": { "model_slug": "anthropic/claude-fable-5.1", "cosmos_test_count": 15, "pass_rate_pct": 40.0, "twelve_cs_score": 42.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 6, "HOST_FAIL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "nex-agi/nex-n2.5-pro": { "model_slug": "nex-agi/nex-n2.5-pro", "cosmos_test_count": 26, "pass_rate_pct": 53.8, "twelve_cs_score": 55.6, "passes": 14, "fails": 12, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "http-detail:not a pinned OpenRouter id 'nex-agi/nex-n2.5-pro'; want ['bytedance-seed/seed-2.": 8, "UNPINNED-NEEDS-PIN: OR /models lists nex-agi/nex-n2.5-pro LIVE (free twin gone); rail PINNED lacks it so require_duds/model_refused REFUSEs before HTTP; propose CCr pin, then smoke ping+fizz": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "tencent/hy4-preview": { "model_slug": "tencent/hy4-preview", "cosmos_test_count": 9, "pass_rate_pct": 33.3, "twelve_cs_score": 36.1, "passes": 3, "fails": 0, "rate_limits_429": 3, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 3, "UPSTREAM_POOL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "meta/muse-spark-1.3-contributor": { "model_slug": "meta/muse-spark-1.3-contributor", "cosmos_test_count": 148, "pass_rate_pct": 20.9, "twelve_cs_score": 24.4, "passes": 31, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "NO_CONTEXT": 6, "sku=meta/muse-spark-1.3-contributor": 4, "lines=1 first='Resolving instruction hierarchy and outp'": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "~z-ai/glm-flash-latest": { "model_slug": "~z-ai/glm-flash-latest", "cosmos_test_count": 19, "pass_rate_pct": 15.8, "twelve_cs_score": 19.5, "passes": 3, "fails": 4, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 12, "ROTATOR-REFUSED|PIN-NOTE": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "z-ai/glm-5.3-flash": { "model_slug": "z-ai/glm-5.3-flash", "cosmos_test_count": 867, "pass_rate_pct": 2.8, "twelve_cs_score": 7.2, "passes": 24, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=z-ai/glm-5.3-flash": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "tencent/hy-mt2-30b-a3b": { "model_slug": "tencent/hy-mt2-30b-a3b", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "deepseek/deepseek-v4-pro-0813": { "model_slug": "deepseek/deepseek-v4-pro-0813", "cosmos_test_count": 14, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 14, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "sku=deepseek/deepseek-v4-pro-0813": 4, "fizz-16lines-ok lines=16": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "upstage/solar-pro4": { "model_slug": "upstage/solar-pro4", "cosmos_test_count": 30, "pass_rate_pct": 70.0, "twelve_cs_score": 71.0, "passes": 21, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=upstage/solar-pro4": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "ibm-granite/granite-4.2-8b": { "model_slug": "ibm-granite/granite-4.2-8b", "cosmos_test_count": 15, "pass_rate_pct": 20.0, "twelve_cs_score": 23.5, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 12 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "deepseek/deepseek-v4-flash-vision-exp": { "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "cosmos_test_count": 49, "pass_rate_pct": 28.6, "twelve_cs_score": 31.7, "passes": 14, "fails": 32, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "not a pinned OpenRouter id 'deepseek/deepseek-v4-flash-vision-exp'; want ['byted|PIN-NOTE": 28, "revisit-later|PIN-NOTE": 4, "EMPTY": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "tencent/hy-mt2-7b": { "model_slug": "tencent/hy-mt2-7b", "cosmos_test_count": 12, "pass_rate_pct": 25.0, "twelve_cs_score": 28.2, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "deepseek/deepseek-v4-flash-0731": { "model_slug": "deepseek/deepseek-v4-flash-0731", "cosmos_test_count": 138, "pass_rate_pct": 15.2, "twelve_cs_score": 18.9, "passes": 21, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "NO_CONTEXT": 12, "fizz-16lines-ok lines=16": 8, "sku=deepseek/deepseek-v4-flash-0731": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "inclusionai/ling-3.0-flash": { "model_slug": "inclusionai/ling-3.0-flash", "cosmos_test_count": 30, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 30, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "sku=inclusionai/ling-3.0-flash": 8, "fizz-16lines-ok lines=16": 8, "fizz-lines=33 first='1. **Analyze the Request:**'": 4, "fizz-lines=27 first='1. **Analyze the Request:**'": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "qwen/qwen3.8-flash": { "model_slug": "qwen/qwen3.8-flash", "cosmos_test_count": 132, "pass_rate_pct": 15.9, "twelve_cs_score": 19.6, "passes": 21, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "NO_CONTEXT": 12, "fizz-16lines-ok lines=16": 8, "sku=qwen/qwen3.8-flash": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "tencent/hy-mt2-1.8b": { "model_slug": "tencent/hy-mt2-1.8b", "cosmos_test_count": 19, "pass_rate_pct": 15.8, "twelve_cs_score": 19.5, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 15 }, "source_clusters": [ "seats", "g47_seat_logs", "_hero_run_20261008" ] }, "qwen/qwen3.8-27b": { "model_slug": "qwen/qwen3.8-27b", "cosmos_test_count": 27, "pass_rate_pct": 77.8, "twelve_cs_score": 78.4, "passes": 21, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=qwen/qwen3.8-27b": 4, "EMPTY": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "z-ai/glm-5.3": { "model_slug": "z-ai/glm-5.3", "cosmos_test_count": 18, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 18, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=z-ai/glm-5.3": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "bytedance-seed/seed-2-1-turbo": { "model_slug": "bytedance-seed/seed-2-1-turbo", "cosmos_test_count": 48, "pass_rate_pct": 22.9, "twelve_cs_score": 26.3, "passes": 11, "fails": 28, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "not a pinned OpenRouter id 'bytedance-seed/seed-2-1-turbo'; want ['bytedance-see|PIN-NOTE": 24, "EMPTY": 9, "revisit-later|PIN-NOTE": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "meituan/longcat-2.0": { "model_slug": "meituan/longcat-2.0", "cosmos_test_count": 45, "pass_rate_pct": 24.4, "twelve_cs_score": 27.7, "passes": 11, "fails": 28, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "not a pinned OpenRouter id 'meituan/longcat-2.0'; want ['bytedance-seed/seed-2.0|PIN-NOTE": 24, "revisit-later|PIN-NOTE": 4, "HOST_FAIL": 3, "EMPTY": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "meta/muse-spark-1.1": { "model_slug": "meta/muse-spark-1.1", "cosmos_test_count": 26, "pass_rate_pct": 69.2, "twelve_cs_score": 70.2, "passes": 18, "fails": 8, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "": 12, "not a pinned OpenRouter id 'meta/muse-spark-1.1'; want ['bytedance-seed/seed-2.0": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "bytedance-seed/seed-2.0-code": { "model_slug": "bytedance-seed/seed-2.0-code", "cosmos_test_count": 49, "pass_rate_pct": 22.4, "twelve_cs_score": 25.8, "passes": 11, "fails": 32, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "not a pinned OpenRouter id 'bytedance-seed/seed-2.0-code'; want ['bytedance-seed|PIN-NOTE": 28, "EMPTY": 6, "revisit-later|PIN-NOTE": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "x-ai/grok-4.6": { "model_slug": "x-ai/grok-4.6", "cosmos_test_count": 26, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 26, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "mouth-form": 12, "sku=x-ai/grok-4.6": 4, "fizz-16lines-ok lines=16": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "sakana/sakana-namazu": { "model_slug": "sakana/sakana-namazu", "cosmos_test_count": 9, "pass_rate_pct": 33.3, "twelve_cs_score": 36.1, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 6 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-5.6-terra-pro": { "model_slug": "openai/gpt-5.6-terra-pro", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3.8-2.4t-a95b": { "model_slug": "qwen/qwen3.8-2.4t-a95b", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "nvidia/nemotron-3.5-lightning": { "model_slug": "nvidia/nemotron-3.5-lightning", "cosmos_test_count": 38, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 38, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-lines=15 first='Fizz'": 8, "": 8, "1": 8, "sku=nvidia/nemotron-3.5-lightning": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "~deepseek/deepseek-v4-flash-latest": { "model_slug": "~deepseek/deepseek-v4-flash-latest", "cosmos_test_count": 12, "pass_rate_pct": 25.0, "twelve_cs_score": 28.2, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "x-ai/grok-4.5": { "model_slug": "x-ai/grok-4.5", "cosmos_test_count": 14, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 14, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "anthropic/claude-opus-5": { "model_slug": "anthropic/claude-opus-5", "cosmos_test_count": 18, "pass_rate_pct": 77.8, "twelve_cs_score": 78.4, "passes": 14, "fails": 4, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "{\"error\":{\"message\":\"Provider returned error\",\"code\":400,\"metadata\":{\"raw\":\"{\\\"type\\\":\\\"error\\\",\\\"error\\\":{\\\"type\\\":\\\"in": 4, "reply-line-unverified": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "meta/muse-glimmer-30b": { "model_slug": "meta/muse-glimmer-30b", "cosmos_test_count": 62, "pass_rate_pct": 48.4, "twelve_cs_score": 50.5, "passes": 30, "fails": 32, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "not a pinned OpenRouter id 'meta/muse-glimmer-30b'; want ['bytedance-seed/seed-2|PIN-NOTE": 28, "revisit-later|PIN-NOTE": 4, "": 4, "EMPTY-ON-LONG finish=length": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "meta/muse-spark-1.2": { "model_slug": "meta/muse-spark-1.2", "cosmos_test_count": 25, "pass_rate_pct": 88.0, "twelve_cs_score": 88.1, "passes": 22, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "mouth-form": 4, "sku=meta/muse-spark-1.2": 4, "EMPTY": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "google/gemini-3.5-flash-lite": { "model_slug": "google/gemini-3.5-flash-lite", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "tencent/hy3": { "model_slug": "tencent/hy3", "cosmos_test_count": 14, "pass_rate_pct": 71.4, "twelve_cs_score": 72.3, "passes": 10, "fails": 4, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "PIN-HOLE|PIN-NOTE": 4, "": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "thinkingmachines/inkling-small": { "model_slug": "thinkingmachines/inkling-small", "cosmos_test_count": 46, "pass_rate_pct": 30.4, "twelve_cs_score": 33.4, "passes": 14, "fails": 32, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "not a pinned OpenRouter id 'thinkingmachines/inkling-small'; want ['bytedance-se|PIN-NOTE": 28, "revisit-later|PIN-NOTE": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "qwen/qwen3.7-flash": { "model_slug": "qwen/qwen3.7-flash", "cosmos_test_count": 45, "pass_rate_pct": 57.8, "twelve_cs_score": 59.4, "passes": 26, "fails": 0, "rate_limits_429": 3, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 9, "sku=qwen/qwen3.7-flash": 4, "fizz-lines=89 first='The user wants a specific output format:'": 4, "fizz-lines=15 first='NONE'": 4 }, "source_clusters": [ "womb_scars", "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "roster" ] }, "moonshotai/kimi-k3": { "model_slug": "moonshotai/kimi-k3", "cosmos_test_count": 24, "pass_rate_pct": 75.0, "twelve_cs_score": 75.8, "passes": 18, "fails": 0, "rate_limits_429": 6, "median_latency_ms": 145.0, "top_scars": { "UPSTREAM_POOL": 6, "The intent is a simple FizzBuzz script: print \"NONE\" as the first line": 4, "reply-line-unverified": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "openai/gpt-5.6-luna": { "model_slug": "openai/gpt-5.6-luna", "cosmos_test_count": 22, "pass_rate_pct": 81.8, "twelve_cs_score": 82.2, "passes": 18, "fails": 4, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=openai/gpt-5.6-luna": 4, "lines=1 first=\"http=200 error='Flex processing is tempo\"": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "poolside/laguna-s-2.1": { "model_slug": "poolside/laguna-s-2.1", "cosmos_test_count": 28, "pass_rate_pct": 89.3, "twelve_cs_score": 89.3, "passes": 25, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=poolside/laguna-s-2.1": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "openai/gpt-5.6-sol": { "model_slug": "openai/gpt-5.6-sol", "cosmos_test_count": 18, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 18, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=openai/gpt-5.6-sol": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "thinkingmachines/inkling": { "model_slug": "thinkingmachines/inkling", "cosmos_test_count": 64, "pass_rate_pct": 90.6, "twelve_cs_score": 90.6, "passes": 58, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "mouth-form": 20, "fizz-lines=1 first='NONE12Fizz4BuzzFizz78FizzBuzz11Fizz1314F'": 12, "sku=thinkingmachines/inkling": 4, "": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "kwaipilot/kat-coder-pro-v2.5": { "model_slug": "kwaipilot/kat-coder-pro-v2.5", "cosmos_test_count": 21, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "REJECTED": 21 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-5.6-terra": { "model_slug": "openai/gpt-5.6-terra", "cosmos_test_count": 18, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 18, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=openai/gpt-5.6-terra": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "aion-labs/aion-3.0": { "model_slug": "aion-labs/aion-3.0", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "google/gemini-3.1-flash-lite-image": { "model_slug": "google/gemini-3.1-flash-lite-image", "cosmos_test_count": 15, "pass_rate_pct": 40.0, "twelve_cs_score": 42.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "FORM_MISS": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "~x-ai/grok-latest": { "model_slug": "~x-ai/grok-latest", "cosmos_test_count": 15, "pass_rate_pct": 20.0, "twelve_cs_score": 23.5, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 12 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "z-ai/glm-5.2": { "model_slug": "z-ai/glm-5.2", "cosmos_test_count": 22, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 22, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=z-ai/glm-5.2": 4, "lines=1 first='Written `fizzbuzz.py` \u2014 running it produ'": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "sakana/fugu-ultra": { "model_slug": "sakana/fugu-ultra", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "anthropic/claude-fable-5": { "model_slug": "anthropic/claude-fable-5", "cosmos_test_count": 18, "pass_rate_pct": 16.7, "twelve_cs_score": 20.4, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 15 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openrouter/fusion": { "model_slug": "openrouter/fusion", "cosmos_test_count": 12, "pass_rate_pct": 25.0, "twelve_cs_score": 28.2, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "nvidia/nemotron-3.5-content-safety": { "model_slug": "nvidia/nemotron-3.5-content-safety", "cosmos_test_count": 18, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 18 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "poolside/laguna-xs-2.1": { "model_slug": "poolside/laguna-xs-2.1", "cosmos_test_count": 21, "pass_rate_pct": 85.7, "twelve_cs_score": 85.9, "passes": 18, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=poolside/laguna-xs-2.1": 4, "HOST_FAIL": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "google/gemini-3.1-flash-image": { "model_slug": "google/gemini-3.1-flash-image", "cosmos_test_count": 15, "pass_rate_pct": 40.0, "twelve_cs_score": 42.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "FORM_MISS": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "stepfun/step-3.7-flash": { "model_slug": "stepfun/step-3.7-flash", "cosmos_test_count": 49, "pass_rate_pct": 22.4, "twelve_cs_score": 25.8, "passes": 11, "fails": 32, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "not a pinned OpenRouter id 'stepfun/step-3.7-flash'; want ['bytedance-seed/seed-|PIN-NOTE": 28, "EMPTY": 6, "revisit-later|PIN-NOTE": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "moonshotai/kimi-k2.7-code": { "model_slug": "moonshotai/kimi-k2.7-code", "cosmos_test_count": 15, "pass_rate_pct": 20.0, "twelve_cs_score": 23.5, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 12 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "nvidia/nemotron-3-ultra-550b-a55b": { "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "cosmos_test_count": 25, "pass_rate_pct": 88.0, "twelve_cs_score": 88.1, "passes": 22, "fails": 0, "rate_limits_429": 3, "median_latency_ms": 145.0, "top_scars": { "sku=nvidia/nemotron-3-ultra-550b-a55b": 4, "fizz-16lines-ok lines=16": 4, "lines=15 first='1'": 4, "lines=1 first='Done. The `fizzbuzz.py` file outputs the'": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "google/gemini-3.5-flash": { "model_slug": "google/gemini-3.5-flash", "cosmos_test_count": 30, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 30, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "print(\"NONE\")": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "anthropic/claude-opus-4.8": { "model_slug": "anthropic/claude-opus-4.8", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "perceptron/perceptron-mk1": { "model_slug": "perceptron/perceptron-mk1", "cosmos_test_count": 42, "pass_rate_pct": 33.3, "twelve_cs_score": 36.1, "passes": 14, "fails": 28, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "not a pinned OpenRouter id 'perceptron/perceptron-mk1'; want ['bytedance-seed/se|PIN-NOTE": 24, "revisit-later|PIN-NOTE": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "mistralai/mistral-medium-3-5": { "model_slug": "mistralai/mistral-medium-3-5", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "x-ai/grok-4.3": { "model_slug": "x-ai/grok-4.3", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "~moonshotai/kimi-latest": { "model_slug": "~moonshotai/kimi-latest", "cosmos_test_count": 15, "pass_rate_pct": 20.0, "twelve_cs_score": 23.5, "passes": 3, "fails": 0, "rate_limits_429": 3, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 9, "UPSTREAM_POOL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "minimax/minimax-m3": { "model_slug": "minimax/minimax-m3", "cosmos_test_count": 45, "pass_rate_pct": 24.4, "twelve_cs_score": 27.7, "passes": 11, "fails": 28, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "not a pinned OpenRouter id 'minimax/minimax-m3'; want ['bytedance-seed/seed-2.0-|PIN-NOTE": 24, "EMPTY": 6, "revisit-later|PIN-NOTE": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "~google/gemini-pro-latest": { "model_slug": "~google/gemini-pro-latest", "cosmos_test_count": 12, "pass_rate_pct": 25.0, "twelve_cs_score": 28.2, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3.6-flash": { "model_slug": "qwen/qwen3.6-flash", "cosmos_test_count": 48, "pass_rate_pct": 22.9, "twelve_cs_score": 26.3, "passes": 11, "fails": 28, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "not a pinned OpenRouter id 'qwen/qwen3.6-flash'; want ['bytedance-seed/seed-2.0-|PIN-NOTE": 24, "EMPTY": 9, "revisit-later|PIN-NOTE": 4, "": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "x-ai/grok-build-0.1": { "model_slug": "x-ai/grok-build-0.1", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-chat-latest": { "model_slug": "openai/gpt-chat-latest", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "~openai/gpt-mini-latest": { "model_slug": "~openai/gpt-mini-latest", "cosmos_test_count": 12, "pass_rate_pct": 25.0, "twelve_cs_score": 28.2, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "~anthropic/claude-sonnet-latest": { "model_slug": "~anthropic/claude-sonnet-latest", "cosmos_test_count": 12, "pass_rate_pct": 25.0, "twelve_cs_score": 28.2, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3.5-plus-20260420": { "model_slug": "qwen/qwen3.5-plus-20260420", "cosmos_test_count": 69, "pass_rate_pct": 50.7, "twelve_cs_score": 52.7, "passes": 35, "fails": 28, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "not a pinned OpenRouter id 'qwen/qwen3.5-plus-20260420'; want ['bytedance-seed/s|PIN-NOTE": 24, "": 8, "EMPTY": 6, "revisit-later|PIN-NOTE": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "qwen/qwen3.6-max-preview": { "model_slug": "qwen/qwen3.6-max-preview", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-5.5-pro": { "model_slug": "openai/gpt-5.5-pro", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "deepseek/deepseek-v4-pro": { "model_slug": "deepseek/deepseek-v4-pro", "cosmos_test_count": 26, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 26, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "empty": 4, "```python": 4, "print(\"NONE\")": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "qwen/qwen3.6-27b": { "model_slug": "qwen/qwen3.6-27b", "cosmos_test_count": 46, "pass_rate_pct": 39.1, "twelve_cs_score": 41.6, "passes": 18, "fails": 28, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "not a pinned OpenRouter id 'qwen/qwen3.6-27b'; want ['bytedance-seed/seed-2.0-mi|PIN-NOTE": 24, "": 8, "revisit-later|PIN-NOTE": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "tencent/hy3-preview": { "model_slug": "tencent/hy3-preview", "cosmos_test_count": 998, "pass_rate_pct": 4.3, "twelve_cs_score": 8.6, "passes": 43, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "NO_CONTEXT": 6, "sku=tencent/hy3-preview": 4, "fizz-16lines-ok lines=16": 4, "lines=56 first='The user wants me to reply with exactly '": 4 }, "source_clusters": [ "womb_scars", "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "roster" ] }, "~anthropic/claude-opus-latest": { "model_slug": "~anthropic/claude-opus-latest", "cosmos_test_count": 15, "pass_rate_pct": 20.0, "twelve_cs_score": 23.5, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 12 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "z-ai/glm-5.1": { "model_slug": "z-ai/glm-5.1", "cosmos_test_count": 14, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 14, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "print(\"NONE\")": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "xiaomi/mimo-v2.5": { "model_slug": "xiaomi/mimo-v2.5", "cosmos_test_count": 36, "pass_rate_pct": 66.7, "twelve_cs_score": 67.9, "passes": 24, "fails": 3, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=xiaomi/mimo-v2.5": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "moonshotai/kimi-k2.6": { "model_slug": "moonshotai/kimi-k2.6", "cosmos_test_count": 55, "pass_rate_pct": 27.3, "twelve_cs_score": 30.4, "passes": 15, "fails": 28, "rate_limits_429": 12, "median_latency_ms": 145.0, "top_scars": { "not a pinned OpenRouter id 'moonshotai/kimi-k2.6'; want ['bytedance-seed/seed-2.|PIN-NOTE": 24, "UPSTREAM_POOL": 12, " NONE": 12, "revisit-later|PIN-NOTE": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "z-ai/glm-5v-turbo": { "model_slug": "z-ai/glm-5v-turbo", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "google/gemma-4-31b-it": { "model_slug": "google/gemma-4-31b-it", "cosmos_test_count": 58, "pass_rate_pct": 84.5, "twelve_cs_score": 84.8, "passes": 49, "fails": 0, "rate_limits_429": 6, "median_latency_ms": 145.0, "top_scars": { "fizz-lines=15 first='1'": 8, "": 8, "1": 8, "UPSTREAM_POOL": 6 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "x-ai/grok-4.20-multi-agent": { "model_slug": "x-ai/grok-4.20-multi-agent", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-5.4-nano": { "model_slug": "openai/gpt-5.4-nano", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "deepseek/deepseek-v4-flash": { "model_slug": "deepseek/deepseek-v4-flash", "cosmos_test_count": 30, "pass_rate_pct": 70.0, "twelve_cs_score": 71.0, "passes": 21, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "HOST_FAIL": 6, "sku=deepseek/deepseek-v4-flash": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "nvidia/nemotron-3-super-120b-a12b": { "model_slug": "nvidia/nemotron-3-super-120b-a12b", "cosmos_test_count": 7, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 7, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs", "_hero_run_20261008" ] }, "openai/gpt-5.4": { "model_slug": "openai/gpt-5.4", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "google/gemini-3.1-flash-image-preview": { "model_slug": "google/gemini-3.1-flash-image-preview", "cosmos_test_count": 12, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "FORM_MISS": 6 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3.5-flash-02-23": { "model_slug": "qwen/qwen3.5-flash-02-23", "cosmos_test_count": 9, "pass_rate_pct": 33.3, "twelve_cs_score": 36.1, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 6 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "google/gemini-3.1-pro-preview": { "model_slug": "google/gemini-3.1-pro-preview", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "minimax/minimax-m2.5": { "model_slug": "minimax/minimax-m2.5", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3-coder-next": { "model_slug": "qwen/qwen3-coder-next", "cosmos_test_count": 9, "pass_rate_pct": 66.7, "twelve_cs_score": 67.9, "passes": 6, "fails": 0, "rate_limits_429": 3, "median_latency_ms": 145.0, "top_scars": { "UPSTREAM_POOL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-5.4-image-2": { "model_slug": "openai/gpt-5.4-image-2", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "anthropic/claude-opus-4.7": { "model_slug": "anthropic/claude-opus-4.7", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "minimax/minimax-m2-her": { "model_slug": "minimax/minimax-m2-her", "cosmos_test_count": 9, "pass_rate_pct": 33.3, "twelve_cs_score": 36.1, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 6 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3.6-plus": { "model_slug": "qwen/qwen3.6-plus", "cosmos_test_count": 57, "pass_rate_pct": 40.4, "twelve_cs_score": 42.9, "passes": 23, "fails": 28, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "not a pinned OpenRouter id 'qwen/qwen3.6-plus'; want ['bytedance-seed/seed-2.0-m|PIN-NOTE": 24, "EMPTY": 6, "revisit-later|PIN-NOTE": 4, "": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "x-ai/grok-4.20": { "model_slug": "x-ai/grok-4.20", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-5.4-mini": { "model_slug": "openai/gpt-5.4-mini", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "z-ai/glm-4.7-flash": { "model_slug": "z-ai/glm-4.7-flash", "cosmos_test_count": 9, "pass_rate_pct": 66.7, "twelve_cs_score": 67.9, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "bytedance-seed/seed-2.0-lite": { "model_slug": "bytedance-seed/seed-2.0-lite", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "inception/mercury-2": { "model_slug": "inception/mercury-2", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3.5-35b-a3b": { "model_slug": "qwen/qwen3.5-35b-a3b", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "minimax/minimax-m2.1": { "model_slug": "minimax/minimax-m2.1", "cosmos_test_count": 9, "pass_rate_pct": 66.7, "twelve_cs_score": 67.9, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-5.2-chat": { "model_slug": "openai/gpt-5.2-chat", "cosmos_test_count": 12, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SLUG_DEAD": 12 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "relace/relace-search": { "model_slug": "relace/relace-search", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "google/gemini-3.1-pro-preview-customtools": { "model_slug": "google/gemini-3.1-pro-preview-customtools", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "amazon/nova-2-lite-v1": { "model_slug": "amazon/nova-2-lite-v1", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "mistralai/mistral-large-2512": { "model_slug": "mistralai/mistral-large-2512", "cosmos_test_count": 9, "pass_rate_pct": 33.3, "twelve_cs_score": 36.1, "passes": 3, "fails": 0, "rate_limits_429": 6, "median_latency_ms": 145.0, "top_scars": { "UPSTREAM_POOL": 6 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "anthropic/claude-sonnet-4.6": { "model_slug": "anthropic/claude-sonnet-4.6", "cosmos_test_count": 14, "pass_rate_pct": 71.4, "twelve_cs_score": 72.3, "passes": 10, "fails": 4, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "{\"error\":{\"message\":\"Provider returned error\",\"code\":400,\"metadata\":{\"raw\":\"{\\\"type\\\":\\\"error\\\",\\\"error\\\":{\\\"type\\\":\\\"in": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "openai/gpt-5.1": { "model_slug": "openai/gpt-5.1", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "amazon/nova-premier-v1": { "model_slug": "amazon/nova-premier-v1", "cosmos_test_count": 12, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SLUG_DEAD": 12 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "minimax/minimax-m2": { "model_slug": "minimax/minimax-m2", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "z-ai/glm-5": { "model_slug": "z-ai/glm-5", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "anthropic/claude-haiku-4.5": { "model_slug": "anthropic/claude-haiku-4.5", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "google/gemini-2.5-flash-image": { "model_slug": "google/gemini-2.5-flash-image", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "z-ai/glm-4.6": { "model_slug": "z-ai/glm-4.6", "cosmos_test_count": 9, "pass_rate_pct": 66.7, "twelve_cs_score": 67.9, "passes": 6, "fails": 0, "rate_limits_429": 3, "median_latency_ms": 145.0, "top_scars": { "UPSTREAM_POOL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "relace/relace-apply-3": { "model_slug": "relace/relace-apply-3", "cosmos_test_count": 18, "pass_rate_pct": 16.7, "twelve_cs_score": 20.4, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "REJECTED": 15 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3-coder-plus": { "model_slug": "qwen/qwen3-coder-plus", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3-next-80b-a3b-instruct": { "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "stepfun/step-3.5-flash": { "model_slug": "stepfun/step-3.5-flash", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "nousresearch/hermes-4-405b": { "model_slug": "nousresearch/hermes-4-405b", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "writer/palmyra-x5": { "model_slug": "writer/palmyra-x5", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-5.2-codex": { "model_slug": "openai/gpt-5.2-codex", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-5": { "model_slug": "openai/gpt-5", "cosmos_test_count": 14, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 14, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "openai/gpt-oss-20b": { "model_slug": "openai/gpt-oss-20b", "cosmos_test_count": 28, "pass_rate_pct": 89.3, "twelve_cs_score": 89.3, "passes": 25, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=openai/gpt-oss-20b": 4, "fizz-lines=16 first='NONE'": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "qwen/qwen3-30b-a3b-instruct-2507": { "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3-coder": { "model_slug": "qwen/qwen3-coder", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "moonshotai/kimi-k2": { "model_slug": "moonshotai/kimi-k2", "cosmos_test_count": 18, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 18, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HERO_OK kimi-k2": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "morph/morph-v3-fast": { "model_slug": "morph/morph-v3-fast", "cosmos_test_count": 12, "pass_rate_pct": 25.0, "twelve_cs_score": 28.2, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "REJECTED": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "google/gemini-2.5-flash": { "model_slug": "google/gemini-2.5-flash", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "z-ai/glm-4.7": { "model_slug": "z-ai/glm-4.7", "cosmos_test_count": 30, "pass_rate_pct": 90.0, "twelve_cs_score": 90.0, "passes": 27, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "print(\"NONE\")": 8, "EMPTY": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "openai/gpt-5.2-pro": { "model_slug": "openai/gpt-5.2-pro", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "z-ai/glm-4.6v": { "model_slug": "z-ai/glm-4.6v", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "mistralai/ministral-14b-2512": { "model_slug": "mistralai/ministral-14b-2512", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "deepseek/deepseek-v3.2": { "model_slug": "deepseek/deepseek-v3.2", "cosmos_test_count": 34, "pass_rate_pct": 88.2, "twelve_cs_score": 88.3, "passes": 30, "fails": 4, "rate_limits_429": 4, "median_latency_ms": 145.0, "top_scars": { "{\"error\":{\"message\":\"Provider returned error\",\"code\":429,\"metadata\":{\"raw\":\"deepseek/deepseek-v3.2 is temporarily rate-l": 4, "OCEAN_3 deepseek-v3.2": 4, "OVERRIDE_INVALID deepseek-v3.2": 4, "package design.factory.method;": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "openai/gpt-5.1-codex": { "model_slug": "openai/gpt-5.1-codex", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "perplexity/sonar-pro-search": { "model_slug": "perplexity/sonar-pro-search", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3-vl-32b-instruct": { "model_slug": "qwen/qwen3-vl-32b-instruct", "cosmos_test_count": 46, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 46, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-lines=15 first='1'": 8, "": 8, "sku=qwen/qwen3-vl-32b-instruct": 4, "fizz-lines=15 first='NONE'": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "qwen/qwen3-vl-8b-thinking": { "model_slug": "qwen/qwen3-vl-8b-thinking", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3-vl-30b-a3b-thinking": { "model_slug": "qwen/qwen3-vl-30b-a3b-thinking", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "anthropic/claude-sonnet-4.5": { "model_slug": "anthropic/claude-sonnet-4.5", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "deepseek/deepseek-r1-0528": { "model_slug": "deepseek/deepseek-r1-0528", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3-30b-a3b": { "model_slug": "qwen/qwen3-30b-a3b", "cosmos_test_count": 34, "pass_rate_pct": 91.2, "twelve_cs_score": 91.1, "passes": 31, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "sku=qwen/qwen3-30b-a3b": 4, "fizz-16lines-ok lines=16": 4, "lines=20 first='The Python script has been written to pr'": 4, "lines=15 first='1'": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "qwen/qwen3-235b-a22b": { "model_slug": "qwen/qwen3-235b-a22b", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3-vl-235b-a22b-thinking": { "model_slug": "qwen/qwen3-vl-235b-a22b-thinking", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-4.1": { "model_slug": "openai/gpt-4.1", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "meta-llama/llama-4-scout": { "model_slug": "meta-llama/llama-4-scout", "cosmos_test_count": 55, "pass_rate_pct": 5.5, "twelve_cs_score": 9.7, "passes": 3, "fails": 43, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "404-dead-slug": 28, "SLUG_DEAD": 9, "": 4, "http=404 error='No endpoints found for meta-llama/llama-4-scout:floor. Every can": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "deepseek/deepseek-v3.1-terminus": { "model_slug": "deepseek/deepseek-v3.1-terminus", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen-plus-2025-07-28": { "model_slug": "qwen/qwen-plus-2025-07-28", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "google/gemma-3-4b-it": { "model_slug": "google/gemma-3-4b-it", "cosmos_test_count": 35, "pass_rate_pct": 62.9, "twelve_cs_score": 64.3, "passes": 22, "fails": 13, "rate_limits_429": 8, "median_latency_ms": 145.0, "top_scars": { "HERO_OK gemma-3-4b-it": 20, "{\"error\":{\"message\":\"Provider returned error\",\"code\":429,\"metadata\":{\"raw\":\"google/gemma-3-4b-it is temporarily rate-limited upstream. Please retry sh": 4, "{\"error\":{\"message\":\"Provider returned error\",\"code\":429,\"metadata\":{\"raw\":\"google/gemma-3-4b-it is ": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "deepseek/deepseek-chat-v3.1": { "model_slug": "deepseek/deepseek-chat-v3.1", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 3, "median_latency_ms": 145.0, "top_scars": { "UPSTREAM_POOL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "google/gemma-3-27b-it": { "model_slug": "google/gemma-3-27b-it", "cosmos_test_count": 18, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 18, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "```python": 4, "1": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "openai/gpt-5-mini": { "model_slug": "openai/gpt-5-mini", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "anthropic/claude-opus-4.1": { "model_slug": "anthropic/claude-opus-4.1", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "z-ai/glm-4.5": { "model_slug": "z-ai/glm-4.5", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "bytedance/ui-tars-1.5-7b": { "model_slug": "bytedance/ui-tars-1.5-7b", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "cognitivecomputations/dolphin-mistral-24b-venice-edition": { "model_slug": "cognitivecomputations/dolphin-mistral-24b-venice-edition", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "baidu/ernie-4.5-vl-424b-a47b": { "model_slug": "baidu/ernie-4.5-vl-424b-a47b", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 3, "median_latency_ms": 145.0, "top_scars": { "UPSTREAM_POOL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "perplexity/sonar-deep-research": { "model_slug": "perplexity/sonar-deep-research", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen2.5-vl-72b-instruct": { "model_slug": "qwen/qwen2.5-vl-72b-instruct", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "perplexity/sonar": { "model_slug": "perplexity/sonar", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "deepseek/deepseek-chat": { "model_slug": "deepseek/deepseek-chat", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "meta-llama/llama-3.3-70b-instruct": { "model_slug": "meta-llama/llama-3.3-70b-instruct", "cosmos_test_count": 21, "pass_rate_pct": 71.4, "twelve_cs_score": 72.3, "passes": 15, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "SLUG_DEAD": 6, "sku=meta-llama/llama-3.3-70b-instruct": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "google/gemini-2.5-pro": { "model_slug": "google/gemini-2.5-pro", "cosmos_test_count": 18, "pass_rate_pct": 83.3, "twelve_cs_score": 83.6, "passes": 15, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "openai/gpt-4o-2024-11-20": { "model_slug": "openai/gpt-4o-2024-11-20", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "anthropic/claude-sonnet-4": { "model_slug": "anthropic/claude-sonnet-4", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "anthracite-org/magnum-v4-72b": { "model_slug": "anthracite-org/magnum-v4-72b", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen-2.5-72b-instruct": { "model_slug": "qwen/qwen-2.5-72b-instruct", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "nousresearch/hermes-3-llama-3.1-70b": { "model_slug": "nousresearch/hermes-3-llama-3.1-70b", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "meta-llama/llama-3.1-70b-instruct": { "model_slug": "meta-llama/llama-3.1-70b-instruct", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-4o-mini-2024-07-18": { "model_slug": "openai/gpt-4o-mini-2024-07-18", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "mistralai/mixtral-8x22b-instruct": { "model_slug": "mistralai/mixtral-8x22b-instruct", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 3, "median_latency_ms": 145.0, "top_scars": { "UPSTREAM_POOL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-3.5-turbo-0613": { "model_slug": "openai/gpt-3.5-turbo-0613", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "undi95/remm-slerp-l2-13b": { "model_slug": "undi95/remm-slerp-l2-13b", "cosmos_test_count": 12, "pass_rate_pct": 25.0, "twelve_cs_score": 28.2, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3-8b": { "model_slug": "qwen/qwen3-8b", "cosmos_test_count": 40, "pass_rate_pct": 85.0, "twelve_cs_score": 85.2, "passes": 34, "fails": 0, "rate_limits_429": 6, "median_latency_ms": 145.0, "top_scars": { "UPSTREAM_POOL": 6, "sku=qwen/qwen3-8b": 4, "fizz-16lines-ok lines=16": 4, "lines=22 first='The correct FizzBuzz sequence has been g'": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "openai/o4-mini-high": { "model_slug": "openai/o4-mini-high", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-4.1-mini": { "model_slug": "openai/gpt-4.1-mini", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "deepseek/deepseek-chat-v3-0324": { "model_slug": "deepseek/deepseek-chat-v3-0324", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "google/gemma-3-12b-it": { "model_slug": "google/gemma-3-12b-it", "cosmos_test_count": 19, "pass_rate_pct": 78.9, "twelve_cs_score": 79.5, "passes": 15, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "```python": 4, "1": 4, "HOST_FAIL": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "thedrummer/skyfall-36b-v2": { "model_slug": "thedrummer/skyfall-36b-v2", "cosmos_test_count": 9, "pass_rate_pct": 66.7, "twelve_cs_score": 67.9, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "mistralai/mistral-saba": { "model_slug": "mistralai/mistral-saba", "cosmos_test_count": 26, "pass_rate_pct": 11.5, "twelve_cs_score": 15.4, "passes": 3, "fails": 20, "rate_limits_429": 23, "median_latency_ms": 145.0, "top_scars": { "{\"error\":{\"message\":\"Provider returned error\",\"code\":429,\"metadata\":{\"raw\":\"mistralai/mistral-saba is temporarily rate-limited upstream. Please retry ": 8, "{\"error\":{\"message\":\"Provider returned error\",\"code\":429,\"metadata\":{\"raw\":\"mistralai/mist": 8, "{\"error\":{\"message\":\"Provider returned error\",\"code\":429,\"metadata\":{\"raw\":\"mistralai/mistral-saba i": 4, "UPSTREAM_POOL": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "qwen/qwen-plus": { "model_slug": "qwen/qwen-plus", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "deepseek/deepseek-r1": { "model_slug": "deepseek/deepseek-r1", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "sao10k/l3.3-euryale-70b": { "model_slug": "sao10k/l3.3-euryale-70b", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "amazon/nova-lite-v1": { "model_slug": "amazon/nova-lite-v1", "cosmos_test_count": 22, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 22, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "```python": 4, " ": 4, "The user has requested a specific output format without providing a ta": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "mistralai/mistral-large-2407": { "model_slug": "mistralai/mistral-large-2407", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 3, "median_latency_ms": 145.0, "top_scars": { "UPSTREAM_POOL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen-2.5-7b-instruct": { "model_slug": "qwen/qwen-2.5-7b-instruct", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "cohere/command-r-08-2024": { "model_slug": "cohere/command-r-08-2024", "cosmos_test_count": 22, "pass_rate_pct": 86.4, "twelve_cs_score": 86.6, "passes": 19, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "```": 4, "HERO_OK command-r-08-2024": 4, "1": 4, "narrates-then-complies": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "nousresearch/hermes-3-llama-3.1-405b": { "model_slug": "nousresearch/hermes-3-llama-3.1-405b", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "meta-llama/llama-3.1-8b-instruct": { "model_slug": "meta-llama/llama-3.1-8b-instruct", "cosmos_test_count": 22, "pass_rate_pct": 86.4, "twelve_cs_score": 86.6, "passes": 19, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HERO_OK llama-3.1-8b-instruct": 8, "function=HERO_OK llama-3.1-8b-instruct": 4, "HOST_FAIL": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "google/gemma-2-27b-it": { "model_slug": "google/gemma-2-27b-it", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "microsoft/wizardlm-2-8x22b": { "model_slug": "microsoft/wizardlm-2-8x22b", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-3.5-turbo-instruct": { "model_slug": "openai/gpt-3.5-turbo-instruct", "cosmos_test_count": 9, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SLUG_DEAD": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "gryphe/mythomax-l2-13b": { "model_slug": "gryphe/mythomax-l2-13b", "cosmos_test_count": 28, "pass_rate_pct": 53.6, "twelve_cs_score": 55.4, "passes": 15, "fails": 4, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 9, "HERO_OK mythomax-l2-13b": 8, " HERO_OK mythomax-l2-13b": 4, "\u001b[0m": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "google/gemini-3.6-flash": { "model_slug": "google/gemini-3.6-flash", "cosmos_test_count": 18, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 18, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=google/gemini-3.6-flash": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "openrouter/auto-beta": { "model_slug": "openrouter/auto-beta", "cosmos_test_count": 21, "pass_rate_pct": 14.3, "twelve_cs_score": 18.1, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SLUG_DEAD": 18 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-5.6-luna-pro": { "model_slug": "openai/gpt-5.6-luna-pro", "cosmos_test_count": 22, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 22, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=openai/gpt-5.6-luna-pro": 4, "lines=16 first='print(\"NONE\")'": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "openai/gpt-5.6-sol-pro": { "model_slug": "openai/gpt-5.6-sol-pro", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "aion-labs/aion-3.0-mini": { "model_slug": "aion-labs/aion-3.0-mini", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "anthropic/claude-sonnet-5": { "model_slug": "anthropic/claude-sonnet-5", "cosmos_test_count": 18, "pass_rate_pct": 77.8, "twelve_cs_score": 78.4, "passes": 14, "fails": 4, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "{\"error\":{\"message\":\"Provider returned error\",\"code\":400,\"metadata\":{\"raw\":\"{\\\"type\\\":\\\"error\\\",\\\"error\\\":{\\\"type\\\":\\\"in": 4, "reply-line-unverified": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "rekaai/reka-edge": { "model_slug": "rekaai/reka-edge", "cosmos_test_count": 9, "pass_rate_pct": 33.3, "twelve_cs_score": 36.1, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 6 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "mistralai/mistral-small-2603": { "model_slug": "mistralai/mistral-small-2603", "cosmos_test_count": 9, "pass_rate_pct": 33.3, "twelve_cs_score": 36.1, "passes": 3, "fails": 0, "rate_limits_429": 6, "median_latency_ms": 145.0, "top_scars": { "UPSTREAM_POOL": 6 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "google/gemini-3-pro-image": { "model_slug": "google/gemini-3-pro-image", "cosmos_test_count": 12, "pass_rate_pct": 25.0, "twelve_cs_score": 28.2, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "FORM_MISS": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "~anthropic/claude-fable-latest": { "model_slug": "~anthropic/claude-fable-latest", "cosmos_test_count": 9, "pass_rate_pct": 33.3, "twelve_cs_score": 36.1, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 6 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3.7-plus": { "model_slug": "qwen/qwen3.7-plus", "cosmos_test_count": 42, "pass_rate_pct": 26.2, "twelve_cs_score": 29.4, "passes": 11, "fails": 28, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "not a pinned OpenRouter id 'qwen/qwen3.7-plus'; want ['bytedance-seed/seed-2.0-m|PIN-NOTE": 24, "revisit-later|PIN-NOTE": 4, "": 4, "EMPTY": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "qwen/qwen3.7-max": { "model_slug": "qwen/qwen3.7-max", "cosmos_test_count": 18, "pass_rate_pct": 83.3, "twelve_cs_score": 83.6, "passes": 15, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "print(\"NONE\")": 4, "EMPTY": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "google/gemini-3.1-flash-lite": { "model_slug": "google/gemini-3.1-flash-lite", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "~anthropic/claude-haiku-latest": { "model_slug": "~anthropic/claude-haiku-latest", "cosmos_test_count": 9, "pass_rate_pct": 33.3, "twelve_cs_score": 36.1, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 6 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "~google/gemini-flash-latest": { "model_slug": "~google/gemini-flash-latest", "cosmos_test_count": 9, "pass_rate_pct": 33.3, "twelve_cs_score": 36.1, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 6 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3.5-9b": { "model_slug": "qwen/qwen3.5-9b", "cosmos_test_count": 18, "pass_rate_pct": 83.3, "twelve_cs_score": 83.6, "passes": 15, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=qwen/qwen3.5-9b": 4, "EMPTY": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "google/gemini-3.1-flash-lite-preview": { "model_slug": "google/gemini-3.1-flash-lite-preview", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3.6-35b-a3b": { "model_slug": "qwen/qwen3.6-35b-a3b", "cosmos_test_count": 24, "pass_rate_pct": 62.5, "twelve_cs_score": 63.9, "passes": 15, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=qwen/qwen3.6-35b-a3b": 4, "EMPTY": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "qwen/qwen3.5-27b": { "model_slug": "qwen/qwen3.5-27b", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 3, "median_latency_ms": 145.0, "top_scars": { "UPSTREAM_POOL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-5.5": { "model_slug": "openai/gpt-5.5", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-5.3-codex": { "model_slug": "openai/gpt-5.3-codex", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "xiaomi/mimo-v2.5-pro": { "model_slug": "xiaomi/mimo-v2.5-pro", "cosmos_test_count": 70, "pass_rate_pct": 38.6, "twelve_cs_score": 41.2, "passes": 27, "fails": 40, "rate_limits_429": 12, "median_latency_ms": 145.0, "top_scars": { "not a pinned OpenRouter id 'xiaomi/mimo-v2.5-pro'; want ['bytedance-seed/seed-2.|PIN-NOTE": 24, "HERO_OK mimo-v2.5-pro": 8, "revisit-later|PIN-NOTE": 4, "{\"error\":{\"message\":\"Provider returned error\",\"code\":429,\"metadata\":{\"raw\":\"xiaomi/mimo-v2.5-pro is temporarily rate-lim": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "qwen/qwen3.5-plus-02-15": { "model_slug": "qwen/qwen3.5-plus-02-15", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openrouter/pareto-code": { "model_slug": "openrouter/pareto-code", "cosmos_test_count": 9, "pass_rate_pct": 33.3, "twelve_cs_score": 36.1, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 6 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3-max-thinking": { "model_slug": "qwen/qwen3-max-thinking", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "google/gemma-4-26b-a4b-it": { "model_slug": "google/gemma-4-26b-a4b-it", "cosmos_test_count": 24, "pass_rate_pct": 87.5, "twelve_cs_score": 87.6, "passes": 21, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=google/gemma-4-26b-a4b-it": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "arcee-ai/trinity-large-thinking": { "model_slug": "arcee-ai/trinity-large-thinking", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "minimax/minimax-m2.7": { "model_slug": "minimax/minimax-m2.7", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "z-ai/glm-5-turbo": { "model_slug": "z-ai/glm-5-turbo", "cosmos_test_count": 14, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 14, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "print(\"NONE\")": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "moonshotai/kimi-k2.5": { "model_slug": "moonshotai/kimi-k2.5", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-audio": { "model_slug": "openai/gpt-audio", "cosmos_test_count": 15, "pass_rate_pct": 20.0, "twelve_cs_score": 23.5, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "REJECTED": 12 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "bytedance-seed/seed-1.6-flash": { "model_slug": "bytedance-seed/seed-1.6-flash", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "google/gemini-3-flash-preview": { "model_slug": "google/gemini-3-flash-preview", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-5.2": { "model_slug": "openai/gpt-5.2", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openrouter/bodybuilder": { "model_slug": "openrouter/bodybuilder", "cosmos_test_count": 18, "pass_rate_pct": 16.7, "twelve_cs_score": 20.4, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 15 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "mistralai/ministral-8b-2512": { "model_slug": "mistralai/ministral-8b-2512", "cosmos_test_count": 35, "pass_rate_pct": 91.4, "twelve_cs_score": 91.3, "passes": 32, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "mouth-form": 4, "sku=mistralai/ministral-8b-2512": 4, "lines=9 first='NONE'": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "anthropic/claude-opus-4.5": { "model_slug": "anthropic/claude-opus-4.5", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-5.1-codex-mini": { "model_slug": "openai/gpt-5.1-codex-mini", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "mistralai/voxtral-small-24b-2507": { "model_slug": "mistralai/voxtral-small-24b-2507", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "ibm-granite/granite-4.0-h-micro": { "model_slug": "ibm-granite/granite-4.0-h-micro", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3-vl-8b-instruct": { "model_slug": "qwen/qwen3-vl-8b-instruct", "cosmos_test_count": 46, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 46, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-lines=15 first='1'": 12, "": 8, "1": 8, "sku=qwen/qwen3-vl-8b-instruct": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "qwen/qwen3-vl-30b-a3b-instruct": { "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "deepseek/deepseek-v3.2-exp": { "model_slug": "deepseek/deepseek-v3.2-exp", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-5.4-pro": { "model_slug": "openai/gpt-5.4-pro", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "bytedance-seed/seed-2.0-mini": { "model_slug": "bytedance-seed/seed-2.0-mini", "cosmos_test_count": 64, "pass_rate_pct": 95.3, "twelve_cs_score": 95.0, "passes": 61, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "mouth-form": 28, "": 8, "{\"l1_role\": false, \"l2_model\": true, \"l3_harness\": true, \"l4_wrapper\": true, \"l5_skills\": true, \"l6_tools\": true, \"l7_enviro\": true, \"l8_mission\": false}": 4, "GRADER-CONTRADICTION _py_first vs NONE-first": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "qwen/qwen3-vl-235b-a22b-instruct": { "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3-coder-flash": { "model_slug": "qwen/qwen3-coder-flash", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "moonshotai/kimi-k2-0905": { "model_slug": "moonshotai/kimi-k2-0905", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "mistralai/mistral-medium-3.1": { "model_slug": "mistralai/mistral-medium-3.1", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-5-nano": { "model_slug": "openai/gpt-5-nano", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "mistralai/codestral-2508": { "model_slug": "mistralai/codestral-2508", "cosmos_test_count": 24, "pass_rate_pct": 87.5, "twelve_cs_score": 87.6, "passes": 21, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=mistralai/codestral-2508": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "qwen/qwen3.5-122b-a10b": { "model_slug": "qwen/qwen3.5-122b-a10b", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "aion-labs/aion-2.0": { "model_slug": "aion-labs/aion-2.0", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "z-ai/glm-4.5-air": { "model_slug": "z-ai/glm-4.5-air", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "google/gemini-2.5-flash-lite": { "model_slug": "google/gemini-2.5-flash-lite", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3.5-397b-a17b": { "model_slug": "qwen/qwen3.5-397b-a17b", "cosmos_test_count": 9, "pass_rate_pct": 66.7, "twelve_cs_score": 67.9, "passes": 6, "fails": 0, "rate_limits_429": 3, "median_latency_ms": 145.0, "top_scars": { "UPSTREAM_POOL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "anthropic/claude-opus-4.6": { "model_slug": "anthropic/claude-opus-4.6", "cosmos_test_count": 30, "pass_rate_pct": 86.7, "twelve_cs_score": 86.9, "passes": 26, "fails": 4, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "{\"error\":{\"message\":\"Provider returned error\",\"code\":400,\"metadata\":{\"raw\":\"{\\\"type\\\":\\\"error\\\",\\\"error\\\":{\\\"type\\\":\\\"in": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "upstage/solar-pro-3": { "model_slug": "upstage/solar-pro-3", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-audio-mini": { "model_slug": "openai/gpt-audio-mini", "cosmos_test_count": 15, "pass_rate_pct": 20.0, "twelve_cs_score": 23.5, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "REJECTED": 12 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "bytedance-seed/seed-1.6": { "model_slug": "bytedance-seed/seed-1.6", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "nvidia/nemotron-3-nano-30b-a3b": { "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "mistralai/devstral-2512": { "model_slug": "mistralai/devstral-2512", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "tencent/hunyuan-a13b-instruct": { "model_slug": "tencent/hunyuan-a13b-instruct", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "mistralai/mistral-small-3.2-24b-instruct": { "model_slug": "mistralai/mistral-small-3.2-24b-instruct", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-5.1-codex-max": { "model_slug": "openai/gpt-5.1-codex-max", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "mistralai/ministral-3b-2512": { "model_slug": "mistralai/ministral-3b-2512", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "google/gemini-3-pro-image-preview": { "model_slug": "google/gemini-3-pro-image-preview", "cosmos_test_count": 12, "pass_rate_pct": 25.0, "twelve_cs_score": 28.2, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "FORM_MISS": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "moonshotai/kimi-k2-thinking": { "model_slug": "moonshotai/kimi-k2-thinking", "cosmos_test_count": 26, "pass_rate_pct": 73.1, "twelve_cs_score": 73.9, "passes": 19, "fails": 4, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "print(\"NONE\")": 4, "{\"error\":{\"message\":\"Provider returned error\",\"code\":400,\"metadata\":{\"raw\":\"[{\\n \\\"error\\\": {\\n ": 4, "EMPTY": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "openai/gpt-oss-safeguard-20b": { "model_slug": "openai/gpt-oss-safeguard-20b", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 3, "median_latency_ms": 145.0, "top_scars": { "UPSTREAM_POOL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-5-image-mini": { "model_slug": "openai/gpt-5-image-mini", "cosmos_test_count": 12, "pass_rate_pct": 25.0, "twelve_cs_score": 28.2, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "FORM_MISS": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/o3-pro": { "model_slug": "openai/o3-pro", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "mistralai/mistral-medium-3": { "model_slug": "mistralai/mistral-medium-3", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3-14b": { "model_slug": "qwen/qwen3-14b", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-5-image": { "model_slug": "openai/gpt-5-image", "cosmos_test_count": 12, "pass_rate_pct": 25.0, "twelve_cs_score": 28.2, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "FORM_MISS": 9 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/o3": { "model_slug": "openai/o3", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-4.1-nano": { "model_slug": "openai/gpt-4.1-nano", "cosmos_test_count": 26, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 26, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "write_path = \"/absolute/write/path/your_file.py\"": 4, "write_tool(\"/absolute/path/to/output.txt\", \"NONE\\n1\\n2\\nFizz\\n4\\nBuzz\\nFizz\\n7\\n": 4, "TOOL-MOUTH: replies via Write tool, not chat text": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "openai/o1-pro": { "model_slug": "openai/o1-pro", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "cohere/command-a": { "model_slug": "cohere/command-a", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-5-pro": { "model_slug": "openai/gpt-5-pro", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "thedrummer/cydonia-24b-v4.1": { "model_slug": "thedrummer/cydonia-24b-v4.1", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 3, "median_latency_ms": 145.0, "top_scars": { "UPSTREAM_POOL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3-max": { "model_slug": "qwen/qwen3-max", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "perplexity/sonar-reasoning-pro": { "model_slug": "perplexity/sonar-reasoning-pro", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/o3-mini-high": { "model_slug": "openai/o3-mini-high", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3-next-80b-a3b-thinking": { "model_slug": "qwen/qwen3-next-80b-a3b-thinking", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/o3-mini": { "model_slug": "openai/o3-mini", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "minimax/minimax-01": { "model_slug": "minimax/minimax-01", "cosmos_test_count": 22, "pass_rate_pct": 68.2, "twelve_cs_score": 69.3, "passes": 15, "fails": 4, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "To execute the intent of printing the FizzBuzz results for numbers 1 to 15 with ": 4, "To fulfill the intent of printing the FizzBuzz sequence from 1 to 15 with the sp": 4, "NO-TOOL-ENDPOINTS": 4, "HOST_FAIL": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "openai/o1": { "model_slug": "openai/o1", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "amazon/nova-micro-v1": { "model_slug": "amazon/nova-micro-v1", "cosmos_test_count": 26, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 26, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "NONE ": 4, "stop": 4, "form-deviation": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "qwen/qwen-2.5-coder-32b-instruct": { "model_slug": "qwen/qwen-2.5-coder-32b-instruct", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "meta-llama/llama-3.2-1b-instruct": { "model_slug": "meta-llama/llama-3.2-1b-instruct", "cosmos_test_count": 23, "pass_rate_pct": 65.2, "twelve_cs_score": 66.4, "passes": 15, "fails": 4, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HERO_OK llama-3.2-1b-instruct": 8, "\u001b[0m": 4, "HOST_FAIL": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "qwen/qwen3-30b-a3b-thinking-2507": { "model_slug": "qwen/qwen3-30b-a3b-thinking-2507", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "z-ai/glm-4.5v": { "model_slug": "z-ai/glm-4.5v", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-oss-120b": { "model_slug": "openai/gpt-oss-120b", "cosmos_test_count": 18, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 18, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 8, "sku=openai/gpt-oss-120b": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "qwen/qwen3-coder-30b-a3b-instruct": { "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3-235b-a22b-thinking-2507": { "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3-235b-a22b-2507": { "model_slug": "qwen/qwen3-235b-a22b-2507", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "morph/morph-v3-large": { "model_slug": "morph/morph-v3-large", "cosmos_test_count": 9, "pass_rate_pct": 33.3, "twelve_cs_score": 36.1, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "REJECTED": 6 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "cohere/command-r-plus-08-2024": { "model_slug": "cohere/command-r-plus-08-2024", "cosmos_test_count": 30, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 30, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HERO_OK command-r-plus-08-2024": 8, "import pandas as pd": 4, "```python": 4, "form-deviation": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "sao10k/l3-lunaris-8b": { "model_slug": "sao10k/l3-lunaris-8b", "cosmos_test_count": 22, "pass_rate_pct": 68.2, "twelve_cs_score": 69.3, "passes": 15, "fails": 4, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "stop": 4, "\u001b[0m": 4, "HOST_FAIL": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "minimax/minimax-m1": { "model_slug": "minimax/minimax-m1", "cosmos_test_count": 18, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 18 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "mistralai/mistral-nemo": { "model_slug": "mistralai/mistral-nemo", "cosmos_test_count": 34, "pass_rate_pct": 91.2, "twelve_cs_score": 91.1, "passes": 31, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "stop": 4, "Hello! How can I help you with your software engineering tasks today?": 4, "HOST_FAIL": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "openai/gpt-4o": { "model_slug": "openai/gpt-4o", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "google/gemini-2.5-pro-preview": { "model_slug": "google/gemini-2.5-pro-preview", "cosmos_test_count": 9, "pass_rate_pct": 33.3, "twelve_cs_score": 36.1, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "SKU_UNBOUND": 6 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-4-turbo": { "model_slug": "openai/gpt-4-turbo", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "meta-llama/llama-guard-4-12b": { "model_slug": "meta-llama/llama-guard-4-12b", "cosmos_test_count": 15, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 15 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-3.5-turbo-16k": { "model_slug": "openai/gpt-3.5-turbo-16k", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-3.5-turbo": { "model_slug": "openai/gpt-3.5-turbo", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "qwen/qwen3-32b": { "model_slug": "qwen/qwen3-32b", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/o4-mini": { "model_slug": "openai/o4-mini", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "meta-llama/llama-4-maverick": { "model_slug": "meta-llama/llama-4-maverick", "cosmos_test_count": 45, "pass_rate_pct": 86.7, "twelve_cs_score": 86.9, "passes": 39, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-lines=16 first='1'": 8, "SLUG_DEAD": 6, "sku=meta-llama/llama-4-maverick": 4, "fizz-lines=17 first='NONE'": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "mistralai/mistral-small-3.1-24b-instruct": { "model_slug": "mistralai/mistral-small-3.1-24b-instruct", "cosmos_test_count": 26, "pass_rate_pct": 57.7, "twelve_cs_score": 59.3, "passes": 15, "fails": 8, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "The intent is to create a blank file named `NONE` in the specified write_path di": 4, "{\"error\":{\"message\":\"Provider returned error\",\"code\":400,\"metadata\":{\"raw\":\"{\\\"errors\\\":[{\\\"message\\\":\\\"AiError: AiError: {\\\\\\\"error\\\\\\\":{\\\\\\\"message\\": 4, "{\"error\":{\"message\":\"Provider returned error\",\"code\":400,\"metadata\":{\"raw\":\"{\\\"errors\\\":[{\\\"message\\": 4, "HOST_FAIL": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "rekaai/reka-flash-3": { "model_slug": "rekaai/reka-flash-3", "cosmos_test_count": 22, "pass_rate_pct": 68.2, "twelve_cs_score": 69.3, "passes": 15, "fails": 4, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HERO_OK reka-flash-3": 8, "NONE ": 4, "\u001b[0m": 4, "EMPTY": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "perplexity/sonar-pro": { "model_slug": "perplexity/sonar-pro", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "aion-labs/aion-rp-llama-3.1-8b": { "model_slug": "aion-labs/aion-rp-llama-3.1-8b", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "mistralai/mistral-small-24b-instruct-2501": { "model_slug": "mistralai/mistral-small-24b-instruct-2501", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "microsoft/phi-4": { "model_slug": "microsoft/phi-4", "cosmos_test_count": 18, "pass_rate_pct": 83.3, "twelve_cs_score": 83.6, "passes": 15, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "```python": 4, "1": 4, "HOST_FAIL": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "cohere/command-r7b-12-2024": { "model_slug": "cohere/command-r7b-12-2024", "cosmos_test_count": 18, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 18, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "```python": 4, "1 2 Fizz 4 Buzz Fizz 6 7 Fizz 9 Buzz Fizz 11 12 Fizz 14 Buzz Fizz 15": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "amazon/nova-pro-v1": { "model_slug": "amazon/nova-pro-v1", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "thedrummer/unslopnemo-12b": { "model_slug": "thedrummer/unslopnemo-12b", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "meta-llama/llama-3.2-3b-instruct": { "model_slug": "meta-llama/llama-3.2-3b-instruct", "cosmos_test_count": 22, "pass_rate_pct": 68.2, "twelve_cs_score": 69.3, "passes": 15, "fails": 4, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "import os": 4, "stop": 4, "NO-TOOL-ENDPOINTS": 4, "HOST_FAIL": 3 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "sao10k/l3.1-euryale-70b": { "model_slug": "sao10k/l3.1-euryale-70b", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-4o-2024-08-06": { "model_slug": "openai/gpt-4o-2024-08-06", "cosmos_test_count": 6, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-4o-mini": { "model_slug": "openai/gpt-4o-mini", "cosmos_test_count": 38, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 38, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HERO_OK gpt-4o-mini": 8, "NONE ": 4, "form-deviation": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "roster", "live_ccr", "g47_seat_logs" ] }, "openai/gpt-4o-2024-05-13": { "model_slug": "openai/gpt-4o-2024-05-13", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "mistralai/mistral-large": { "model_slug": "mistralai/mistral-large", "cosmos_test_count": 9, "pass_rate_pct": 33.3, "twelve_cs_score": 36.1, "passes": 3, "fails": 0, "rate_limits_429": 3, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 3, "UPSTREAM_POOL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "mancer/weaver": { "model_slug": "mancer/weaver", "cosmos_test_count": 15, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 15 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "openai/gpt-4": { "model_slug": "openai/gpt-4", "cosmos_test_count": 6, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "HOST_FAIL": 3 }, "source_clusters": [ "seats", "g47_seat_logs" ] }, "stealth/space-bunny-alpha": { "model_slug": "stealth/space-bunny-alpha", "cosmos_test_count": 40, "pass_rate_pct": 62.5, "twelve_cs_score": 63.9, "passes": 25, "fails": 12, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "lines=1 first=\"http=200 error='Provider returned an emp\"": 8, "": 4, "{\"l1_role\": false, \"l2_model\": true, \"l3_harness\": true, \"l4_wrapper\": true, \"l5_skills\": true, \"l6_tools\": true, \"l7_enviro\": false, \"l8_mission\": false}": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "nvidia/nemotron-3.5-lightning:free": { "model_slug": "nvidia/nemotron-3.5-lightning:free", "cosmos_test_count": 159, "pass_rate_pct": 19.5, "twelve_cs_score": 23.0, "passes": 31, "fails": 8, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "NO_CONTEXT": 6, "Here deep of. recommended formellsells Chern fells furells order: made theells20": 4, "Here": 4, "http=400 error='Only one of \"reasoning.effort\" and \"reasoning.max_tokens\" can be": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "poolside/laguna-s-2.1:free": { "model_slug": "poolside/laguna-s-2.1:free", "cosmos_test_count": 34, "pass_rate_pct": 50.0, "twelve_cs_score": 52.0, "passes": 17, "fails": 8, "rate_limits_429": 14, "median_latency_ms": 145.0, "top_scars": { "http=429 error='Provider returned error'": 8, "fizz-16lines-ok lines=16": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "poolside/laguna-xs-2.1:free": { "model_slug": "poolside/laguna-xs-2.1:free", "cosmos_test_count": 72, "pass_rate_pct": 23.6, "twelve_cs_score": 26.9, "passes": 17, "fails": 48, "rate_limits_429": 50, "median_latency_ms": 145.0, "top_scars": { "http=429 error='Provider returned error'": 24, "http-detail:http=429 error='Provider returned error'": 16, "revisit-later": 4, "http-detail:http=400 error='Only one of \"reasoning.effort\" and \"reasoning.max_tokens\" can be": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "cohere/north-mini-code:free": { "model_slug": "cohere/north-mini-code:free", "cosmos_test_count": 135, "pass_rate_pct": 14.8, "twelve_cs_score": 18.6, "passes": 20, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "NO_CONTEXT": 6, "fizz-16lines-ok lines=16": 4 }, "source_clusters": [ "womb_scars", "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "roster" ] }, "nvidia/nemotron-3-ultra-550b-a55b:free": { "model_slug": "nvidia/nemotron-3-ultra-550b-a55b:free", "cosmos_test_count": 29, "pass_rate_pct": 48.3, "twelve_cs_score": 50.4, "passes": 14, "fails": 3, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "fizz-16lines-ok lines=16": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "nvidia/nemotron-3-super-120b-a12b:free": { "model_slug": "nvidia/nemotron-3-super-120b-a12b:free", "cosmos_test_count": 24, "pass_rate_pct": 70.8, "twelve_cs_score": 71.8, "passes": 17, "fails": 4, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "http=200 error='Upstream error from Nvidia: Service temporarily overloaded'": 4, "fizz-16lines-ok lines=16": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "nex-agi/nex-n2.5-mini:free": { "model_slug": "nex-agi/nex-n2.5-mini:free", "cosmos_test_count": 151, "pass_rate_pct": 4.0, "twelve_cs_score": 8.3, "passes": 6, "fails": 28, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "http=404 error='This model is unavailable for free. The paid version is availabl": 20, "NO_CONTEXT": 6, "revisit-later": 4, "{\"error\":{\"message\":\"This model is unavailable for free. The paid version is available now - use this slug instead: nex-agi/nex-n2.5-mini\",\"code\":404}": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "seats", "ccr", "live_ccr", "roster" ] }, "pack": { "model_slug": "pack", "cosmos_test_count": 72, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 31, "median_latency_ms": 145.0, "top_scars": { "UPSTREAM_POOL": 31, "SLUG_DEAD": 22, "REJECTED": 11, "EMPTY": 6 }, "source_clusters": [ "pack" ] }, "codex": { "model_slug": "codex", "cosmos_test_count": 148, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "codex" ] }, "luna6": { "model_slug": "luna6", "cosmos_test_count": 190, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "luna6" ] }, "thinkingmachines-inkling-small-free": { "model_slug": "thinkingmachines-inkling-small-free", "cosmos_test_count": 51, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 2, "HARNESS_GATE": 1 }, "source_clusters": [ "thinkingmachines_inkling-small_free" ] }, "thinkingmachines-inkling-free": { "model_slug": "thinkingmachines-inkling-free", "cosmos_test_count": 25, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 2, "HARNESS_GATE": 1 }, "source_clusters": [ "thinkingmachines_inkling_free" ] }, "rolled": { "model_slug": "rolled", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "rolled" ] }, "tampered": { "model_slug": "tampered", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "tampered" ] }, "transcripts": { "model_slug": "transcripts", "cosmos_test_count": 10, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "transcripts" ] }, "dots-studio/dots3-note-preview:free": { "model_slug": "dots-studio/dots3-note-preview:free", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 6, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "openai/gpt-6-luna:floor": { "model_slug": "openai/gpt-6-luna:floor", "cosmos_test_count": 9, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 9, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "openai/gpt-6-luna-pro:floor": { "model_slug": "openai/gpt-6-luna-pro:floor", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 6, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "qwen/qwen3.8-27b:free:floor": { "model_slug": "qwen/qwen3.8-27b:free:floor", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 3, "rate_limits_429": 3, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "google/gemma-4-26b-a4b-it:free:floor": { "model_slug": "google/gemma-4-26b-a4b-it:free:floor", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 3, "rate_limits_429": 3, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "inclusionai/ling-3.0-flash-vl:free": { "model_slug": "inclusionai/ling-3.0-flash-vl:free", "cosmos_test_count": 31, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 31, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "http=404 error='This model is unavailable for free. The paid version is availabl": 20, "revisit-later": 4, "{\"error\":{\"message\":\"This model is unavailable for free. The paid version is available now - use this slug instead: inclusionai/ling-3.0-flash-vl\",\"co": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "ccr", "live_ccr", "roster" ] }, "xiaomi/mimo-v2.5:floor": { "model_slug": "xiaomi/mimo-v2.5:floor", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 3, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "nex-agi/nex-n2.5-pro:free": { "model_slug": "nex-agi/nex-n2.5-pro:free", "cosmos_test_count": 34, "pass_rate_pct": 8.8, "twelve_cs_score": 12.9, "passes": 3, "fails": 28, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "http=404 error='This model is unavailable for free. The paid version is availabl": 20, "revisit-later": 4, "{\"error\":{\"message\":\"This model is unavailable for free. The paid version is available now - use this slug instead: nex-agi/nex-n2.5-pro\",\"code\":404},": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "ccr", "live_ccr", "roster" ] }, "google/gemma-4-31b": { "model_slug": "google/gemma-4-31b", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "{'seat':-'g26',-'model':-'google/gemma-4-26b-a4b-it:free'}": { "model_slug": "{'seat':-'g26',-'model':-'google/gemma-4-26b-a4b-it:free'}", "cosmos_test_count": 132, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "{'seat':-'g26p',-'model':-'google/gemma-4-26b-a4b-it'}": { "model_slug": "{'seat':-'g26p',-'model':-'google/gemma-4-26b-a4b-it'}", "cosmos_test_count": 36, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "ccr": { "model_slug": "ccr", "cosmos_test_count": 1282, "pass_rate_pct": 0.2, "twelve_cs_score": 4.7, "passes": 2, "fails": 340, "rate_limits_429": 6, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr" ] }, "{'seat':-'llama33',-'model':-'meta-llama/llama-3.3-70b-instruct'}": { "model_slug": "{'seat':-'llama33',-'model':-'meta-llama/llama-3.3-70b-instruct'}", "cosmos_test_count": 372, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "{'seat':-'minist',-'model':-'mistralai/ministral-8b-2512'}": { "model_slug": "{'seat':-'minist',-'model':-'mistralai/ministral-8b-2512'}", "cosmos_test_count": 459, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "{'seat':-'ds',-'model':-'deepseek/deepseek-v4-flash-0731'}": { "model_slug": "{'seat':-'ds',-'model':-'deepseek/deepseek-v4-flash-0731'}", "cosmos_test_count": 378, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "{'seat':-'qvl8',-'model':-'qwen/qwen3-vl-8b-instruct'}": { "model_slug": "{'seat':-'qvl8',-'model':-'qwen/qwen3-vl-8b-instruct'}", "cosmos_test_count": 450, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "{'seat':-'ling',-'model':-'inclusionai/ling-3.0-flash-vl:free'}": { "model_slug": "{'seat':-'ling',-'model':-'inclusionai/ling-3.0-flash-vl:free'}", "cosmos_test_count": 156, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "{'seat':-'n35f',-'model':-'nvidia/nemotron-3.5-lightning:free'}": { "model_slug": "{'seat':-'n35f',-'model':-'nvidia/nemotron-3.5-lightning:free'}", "cosmos_test_count": 348, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "{'seat':-'seed',-'model':-'bytedance-seed/seed-2.0-mini'}": { "model_slug": "{'seat':-'seed',-'model':-'bytedance-seed/seed-2.0-mini'}", "cosmos_test_count": 84, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "{'seat':-'qwen8',-'model':-'qwen/qwen3-8b'}": { "model_slug": "{'seat':-'qwen8',-'model':-'qwen/qwen3-8b'}", "cosmos_test_count": 81, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "{'seat':-'qvl32',-'model':-'qwen/qwen3-vl-32b-instruct'}": { "model_slug": "{'seat':-'qvl32',-'model':-'qwen/qwen3-vl-32b-instruct'}", "cosmos_test_count": 72, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "{'seat':-'qwen30',-'model':-'qwen/qwen3-30b-a3b'}": { "model_slug": "{'seat':-'qwen30',-'model':-'qwen/qwen3-30b-a3b'}", "cosmos_test_count": 84, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "{'seat':-'mimo',-'model':-'xiaomi/mimo-v2.5'}": { "model_slug": "{'seat':-'mimo',-'model':-'xiaomi/mimo-v2.5'}", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "{'seat':-'cursor-solo',-'model':-'cursor-cloud'}": { "model_slug": "{'seat':-'cursor-solo',-'model':-'cursor-cloud'}", "cosmos_test_count": 1059, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "{'seat':-'cursor-a',-'model':-'cursor-cloud'}": { "model_slug": "{'seat':-'cursor-a',-'model':-'cursor-cloud'}", "cosmos_test_count": 234, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "unassigned": { "model_slug": "unassigned", "cosmos_test_count": 2964, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "NO_CONTEXT": 30 }, "source_clusters": [ "ccr", "live_ccr" ] }, "gemini-3.8-flash-\u2014-google-gemini-3.8-flash,-the-language-model-that-writes-the-patch.-called-through-kelly-vertex.-thinking-=-medium,-never-high.": { "model_slug": "gemini-3.8-flash-\u2014-google-gemini-3.8-flash,-the-language-model-that-writes-the-patch.-called-through-kelly-vertex.-thinking-=-medium,-never-high.", "cosmos_test_count": 12, "pass_rate_pct": 25.0, "twelve_cs_score": 28.2, "passes": 3, "fails": 6, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "FAILED": 6 }, "source_clusters": [ "ccr", "live_ccr" ] }, "grok-4.6-\u2014-xai-grok-4.6,-the-language-model-that-writes-the-patch.-called-through-cursor-gitur-build.-not-grok.exe.": { "model_slug": "grok-4.6-\u2014-xai-grok-4.6,-the-language-model-that-writes-the-patch.-called-through-cursor-gitur-build.-not-grok.exe.", "cosmos_test_count": 897, "pass_rate_pct": 8.0, "twelve_cs_score": 12.1, "passes": 72, "fails": 279, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "NO_CONTEXT": 138, "FAILED": 6 }, "source_clusters": [ "ccr", "live_ccr" ] }, "n/a": { "model_slug": "n/a", "cosmos_test_count": 672, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 672, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "FAILED": 672 }, "source_clusters": [ "ccr", "live_ccr" ] }, "gpt-5.6-luna-\u2014-openai-gpt-5.6-luna,-grades-or-answers-research.-called-through-codex-cli.-first-line-item-or-none.": { "model_slug": "gpt-5.6-luna-\u2014-openai-gpt-5.6-luna,-grades-or-answers-research.-called-through-codex-cli.-first-line-item-or-none.", "cosmos_test_count": 75, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 6, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "glm-5.3-flash-\u2014-z.ai-glm-5.3-flash,-the-language-model-that-writes-the-patch.-called-through-pi-cli-(z.ai-documented-terminal-harness),-fallback-openrouter.-not-grok.exe.": { "model_slug": "glm-5.3-flash-\u2014-z.ai-glm-5.3-flash,-the-language-model-that-writes-the-patch.-called-through-pi-cli-(z.ai-documented-terminal-harness),-fallback-openrouter.-not-grok.exe.", "cosmos_test_count": 312, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ccr", "live_ccr" ] }, "coder2": { "model_slug": "coder2", "cosmos_test_count": 9, "pass_rate_pct": 33.3, "twelve_cs_score": 36.1, "passes": 3, "fails": 6, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "FAILED": 6 }, "source_clusters": [ "ccr", "live_ccr" ] }, "orc": { "model_slug": "orc", "cosmos_test_count": 315, "pass_rate_pct": 22.9, "twelve_cs_score": 26.3, "passes": 72, "fails": 117, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "NO_CONTEXT": 60, "FAILED": 6 }, "source_clusters": [ "ccr", "live_ccr" ] }, "daemon": { "model_slug": "daemon", "cosmos_test_count": 672, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 672, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "FAILED": 672 }, "source_clusters": [ "ccr", "live_ccr" ] }, "coder6": { "model_slug": "coder6", "cosmos_test_count": 426, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 153, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "NO_CONTEXT": 72 }, "source_clusters": [ "ccr", "live_ccr" ] }, "research": { "model_slug": "research", "cosmos_test_count": 78, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 15, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "NO_CONTEXT": 6 }, "source_clusters": [ "ccr", "live_ccr" ] }, "fig-database": { "model_slug": "fig-database", "cosmos_test_count": 434, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "Fig-database" ] }, "bonsai": { "model_slug": "bonsai", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "bonsai" ] }, "codestral": { "model_slug": "codestral", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "codestral" ] }, "ds0731": { "model_slug": "ds0731", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ds0731" ] }, "ds0423": { "model_slug": "ds0423", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ds0423" ] }, "dspro": { "model_slug": "dspro", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "dspro" ] }, "ds41flash": { "model_slug": "ds41flash", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ds41flash" ] }, "gemini38": { "model_slug": "gemini38", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "gemini38" ] }, "gemini36": { "model_slug": "gemini36", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "gemini36" ] }, "gemma26b": { "model_slug": "gemma26b", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "gemma26b" ] }, "gemma31b": { "model_slug": "gemma31b", "cosmos_test_count": 8, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "gemma31b" ] }, "glm52": { "model_slug": "glm52", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "glm52" ] }, "glm53": { "model_slug": "glm53", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "glm53" ] }, "glm53flash": { "model_slug": "glm53flash", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "glm53flash" ] }, "gpt6luna": { "model_slug": "gpt6luna", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "gpt6luna" ] }, "gpt6lunapro": { "model_slug": "gpt6lunapro", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "gpt6lunapro" ] }, "grok46": { "model_slug": "grok46", "cosmos_test_count": 10, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "grok46" ] }, "hy3": { "model_slug": "hy3", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "hy3" ] }, "lagunas": { "model_slug": "lagunas", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "lagunas" ] }, "inkling": { "model_slug": "inkling", "cosmos_test_count": 36, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "inkling" ] }, "lagunaxs": { "model_slug": "lagunaxs", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "lagunaxs" ] }, "lingflash": { "model_slug": "lingflash", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "lingflash" ] }, "lingflashvl": { "model_slug": "lingflashvl", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "lingflashvl" ] }, "llama3370b": { "model_slug": "llama3370b", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "llama3370b" ] }, "luna": { "model_slug": "luna", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "luna" ] }, "lunapro": { "model_slug": "lunapro", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "lunapro" ] }, "maverick": { "model_slug": "maverick", "cosmos_test_count": 8, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "maverick" ] }, "mimo25": { "model_slug": "mimo25", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "mimo25" ] }, "ministral": { "model_slug": "ministral", "cosmos_test_count": 8, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ministral" ] }, "muse12": { "model_slug": "muse12", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "muse12" ] }, "muse11c": { "model_slug": "muse11c", "cosmos_test_count": 14, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "muse11c" ] }, "muse12c": { "model_slug": "muse12c", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "muse12c" ] }, "nemolight": { "model_slug": "nemolight", "cosmos_test_count": 8, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "nemolight" ] }, "muse13c": { "model_slug": "muse13c", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "muse13c" ] }, "nemoultra": { "model_slug": "nemoultra", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "nemoultra" ] }, "oss120b": { "model_slug": "oss120b", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "oss120b" ] }, "oss20b": { "model_slug": "oss20b", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "oss20b" ] }, "qwen330b": { "model_slug": "qwen330b", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "qwen330b" ] }, "qwen359b": { "model_slug": "qwen359b", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "qwen359b" ] }, "qwen3635b": { "model_slug": "qwen3635b", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "qwen3635b" ] }, "qwen3827b": { "model_slug": "qwen3827b", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "qwen3827b" ] }, "qwen37flash": { "model_slug": "qwen37flash", "cosmos_test_count": 8, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "qwen37flash" ] }, "qwen38b": { "model_slug": "qwen38b", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "qwen38b" ] }, "qwen38flash": { "model_slug": "qwen38flash", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "qwen38flash" ] }, "qwen38max": { "model_slug": "qwen38max", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "qwen38max" ] }, "qwen3vl32b": { "model_slug": "qwen3vl32b", "cosmos_test_count": 8, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "qwen3vl32b" ] }, "qwen3vl8b": { "model_slug": "qwen3vl8b", "cosmos_test_count": 8, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "qwen3vl8b" ] }, "seed20mini": { "model_slug": "seed20mini", "cosmos_test_count": 18, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seed20mini" ] }, "scout": { "model_slug": "scout", "cosmos_test_count": 14, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "scout" ] }, "sol": { "model_slug": "sol", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "sol" ] }, "solarmini4": { "model_slug": "solarmini4", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "solarmini4" ] }, "solarpro4": { "model_slug": "solarpro4", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "solarpro4" ] }, "terra": { "model_slug": "terra", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "terra" ] }, "f2-lagunasf": { "model_slug": "f2-lagunasf", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 2, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "F2_lagunasf" ] }, "f2-lagunaxsf-fizz": { "model_slug": "f2-lagunaxsf-fizz", "cosmos_test_count": 5, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 5, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "F2_lagunaxsf_fizz" ] }, "f2-qwen27f-fizz": { "model_slug": "f2-qwen27f-fizz", "cosmos_test_count": 5, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 5, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "F2_qwen27f_fizz" ] }, "fc-gemma31bf-fizz": { "model_slug": "fc-gemma31bf-fizz", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 4, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "FC_gemma31bf_fizz" ] }, "fc-gemma26bf-fizz2": { "model_slug": "fc-gemma26bf-fizz2", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 6, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "FC_gemma26bf_fizz2" ] }, "fc-gemma26bf-task": { "model_slug": "fc-gemma26bf-task", "cosmos_test_count": 5, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 4, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "FC_gemma26bf_task" ] }, "fc-gemma26bf-fizz": { "model_slug": "fc-gemma26bf-fizz", "cosmos_test_count": 5, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 4, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "FC_gemma26bf_fizz" ] }, "fc-gemma31bf-fizz2": { "model_slug": "fc-gemma31bf-fizz2", "cosmos_test_count": 8, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 6, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "FC_gemma31bf_fizz2" ] }, "fc-gemma31bf-task": { "model_slug": "fc-gemma31bf-task", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 4, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "FC_gemma31bf_task" ] }, "observed-usage": { "model_slug": "observed-usage", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "observed-usage" ] }, "fc-lagunaxsf-fizz": { "model_slug": "fc-lagunaxsf-fizz", "cosmos_test_count": 9, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 7, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "FC_lagunaxsf_fizz" ] }, "fc-lagunaxsf-task": { "model_slug": "fc-lagunaxsf-task", "cosmos_test_count": 9, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 8, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "FC_lagunaxsf_task" ] }, "fc-nexmini-paid": { "model_slug": "fc-nexmini-paid", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "FC_nexmini_paid" ] }, "fc-nexpro-paid": { "model_slug": "fc-nexpro-paid", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "FC_nexpro_paid" ] }, "gemma26bf": { "model_slug": "gemma26bf", "cosmos_test_count": 8, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 6, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "gemma26bf" ] }, "gemma26bf-fizz": { "model_slug": "gemma26bf-fizz", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 5, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "gemma26bf_fizz" ] }, "gemma31bf": { "model_slug": "gemma31bf", "cosmos_test_count": 8, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 6, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "gemma31bf" ] }, "gemma31bf-fizz": { "model_slug": "gemma31bf-fizz", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 5, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "gemma31bf_fizz" ] }, "go9-dots3": { "model_slug": "go9-dots3", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "GO9_dots3" ] }, "go9-gemma26bf": { "model_slug": "go9-gemma26bf", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 2, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "GO9_gemma26bf" ] }, "go9-gemma31bf": { "model_slug": "go9-gemma31bf", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 2, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "GO9_gemma31bf" ] }, "go9-hy3": { "model_slug": "go9-hy3", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "GO9_hy3" ] }, "go9-nanoomni": { "model_slug": "go9-nanoomni", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "GO9_nanoomni" ] }, "go9-maverick": { "model_slug": "go9-maverick", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "GO9_maverick" ] }, "go9-qwen27f": { "model_slug": "go9-qwen27f", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 7, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "GO9_qwen27f" ] }, "go9-qwen330b": { "model_slug": "go9-qwen330b", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "GO9_qwen330b" ] }, "go9-qwen38b": { "model_slug": "go9-qwen38b", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "GO9_qwen38b" ] }, "inklingf-fizz": { "model_slug": "inklingf-fizz", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "inklingf_fizz" ] }, "inklingsm": { "model_slug": "inklingsm", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "inklingsm" ] }, "lagunaxsf": { "model_slug": "lagunaxsf", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 2, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "lagunaxsf" ] }, "inklingsm-fizz": { "model_slug": "inklingsm-fizz", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "inklingsm_fizz" ] }, "lightning-fizz": { "model_slug": "lightning-fizz", "cosmos_test_count": 5, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "lightning_fizz" ] }, "new-commandaplus": { "model_slug": "new-commandaplus", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_commandaplus" ] }, "lingvlf-fizz": { "model_slug": "lingvlf-fizz", "cosmos_test_count": 5, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "lingvlf_fizz" ] }, "new-commandaplus-fizz": { "model_slug": "new-commandaplus-fizz", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_commandaplus_fizz" ] }, "new-dsv4pro0423": { "model_slug": "new-dsv4pro0423", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_dsv4pro0423" ] }, "new-dsv4flashvis": { "model_slug": "new-dsv4flashvis", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_dsv4flashvis" ] }, "new-glm53batch": { "model_slug": "new-glm53batch", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_glm53batch" ] }, "new-dsv4flashvis-fizz": { "model_slug": "new-dsv4flashvis-fizz", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_dsv4flashvis_fizz" ] }, "new-glm53batch-fizz": { "model_slug": "new-glm53batch-fizz", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_glm53batch_fizz" ] }, "new-glm53flashx-fizz": { "model_slug": "new-glm53flashx-fizz", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_glm53flashx_fizz" ] }, "new-glm53flashx": { "model_slug": "new-glm53flashx", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_glm53flashx" ] }, "new-inklingsmp": { "model_slug": "new-inklingsmp", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_inklingsmp" ] }, "new-inklingsmp-fizz": { "model_slug": "new-inklingsmp-fizz", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_inklingsmp_fizz" ] }, "new-kimi26": { "model_slug": "new-kimi26", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_kimi26" ] }, "new-kimi26-fizz": { "model_slug": "new-kimi26-fizz", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_kimi26_fizz" ] }, "new-longcat20": { "model_slug": "new-longcat20", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_longcat20" ] }, "new-longcat20-fizz": { "model_slug": "new-longcat20-fizz", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_longcat20_fizz" ] }, "new-mimo25pro": { "model_slug": "new-mimo25pro", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_mimo25pro" ] }, "new-mimo25pro-fizz": { "model_slug": "new-mimo25pro-fizz", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_mimo25pro_fizz" ] }, "new-mimo26pro": { "model_slug": "new-mimo26pro", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_mimo26pro" ] }, "new-mimo26pro-fizz": { "model_slug": "new-mimo26pro-fizz", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_mimo26pro_fizz" ] }, "new-minimaxm3-fizz": { "model_slug": "new-minimaxm3-fizz", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_minimaxm3_fizz" ] }, "new-museglimmer": { "model_slug": "new-museglimmer", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_museglimmer" ] }, "new-minimaxm3": { "model_slug": "new-minimaxm3", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_minimaxm3" ] }, "new-museglimmer-fizz": { "model_slug": "new-museglimmer-fizz", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_museglimmer_fizz" ] }, "new-perceptronmk1": { "model_slug": "new-perceptronmk1", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_perceptronmk1" ] }, "new-perceptronmk15-fizz": { "model_slug": "new-perceptronmk15-fizz", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_perceptronmk15_fizz" ] }, "new-perceptronmk15": { "model_slug": "new-perceptronmk15", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_perceptronmk15" ] }, "new-perceptronmk1-fizz": { "model_slug": "new-perceptronmk1-fizz", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_perceptronmk1_fizz" ] }, "new-qwen35plus": { "model_slug": "new-qwen35plus", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_qwen35plus" ] }, "new-qwen3627b": { "model_slug": "new-qwen3627b", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_qwen3627b" ] }, "new-qwen35plus-fizz": { "model_slug": "new-qwen35plus-fizz", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_qwen35plus_fizz" ] }, "new-qwen3627b-fizz": { "model_slug": "new-qwen3627b-fizz", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_qwen3627b_fizz" ] }, "new-qwen36flash-fizz": { "model_slug": "new-qwen36flash-fizz", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_qwen36flash_fizz" ] }, "new-qwen36flash": { "model_slug": "new-qwen36flash", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_qwen36flash" ] }, "new-qwen36plus": { "model_slug": "new-qwen36plus", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_qwen36plus" ] }, "new-qwen36plus-fizz": { "model_slug": "new-qwen36plus-fizz", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_qwen36plus_fizz" ] }, "new-qwen37plus": { "model_slug": "new-qwen37plus", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_qwen37plus" ] }, "new-qwen37plus-fizz": { "model_slug": "new-qwen37plus-fizz", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_qwen37plus_fizz" ] }, "new-seed20code-fizz": { "model_slug": "new-seed20code-fizz", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_seed20code_fizz" ] }, "new-seed21turbo": { "model_slug": "new-seed21turbo", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_seed21turbo" ] }, "new-seed20code": { "model_slug": "new-seed20code", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_seed20code" ] }, "new-step37flash": { "model_slug": "new-step37flash", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_step37flash" ] }, "new-seed21turbo-fizz": { "model_slug": "new-seed21turbo-fizz", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_seed21turbo_fizz" ] }, "new-step37flash-fizz": { "model_slug": "new-step37flash-fizz", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "NEW_step37flash_fizz" ] }, "nexmini": { "model_slug": "nexmini", "cosmos_test_count": 5, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "nexmini" ] }, "nexmini-fizz": { "model_slug": "nexmini-fizz", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "nexmini_fizz" ] }, "nexpro": { "model_slug": "nexpro", "cosmos_test_count": 5, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "nexpro" ] }, "nexpro-fizz": { "model_slug": "nexpro-fizz", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "nexpro_fizz" ] }, "retry2-gemma31bit": { "model_slug": "retry2-gemma31bit", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "RETRY2_gemma31bit" ] }, "retry2-inklingp": { "model_slug": "retry2-inklingp", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "RETRY2_inklingp" ] }, "retry2-nanoomni": { "model_slug": "retry2-nanoomni", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "RETRY2_nanoomni" ] }, "retry2-nemolight": { "model_slug": "retry2-nemolight", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "RETRY2_nemolight" ] }, "retry2-qwen27f": { "model_slug": "retry2-qwen27f", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 6, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "RETRY2_qwen27f" ] }, "retry2-qwen38b": { "model_slug": "retry2-qwen38b", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "RETRY2_qwen38b" ] }, "retry2-qwen3vl32b": { "model_slug": "retry2-qwen3vl32b", "cosmos_test_count": 5, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "RETRY2_qwen3vl32b" ] }, "retry2-qwen3vl8b": { "model_slug": "retry2-qwen3vl8b", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "RETRY2_qwen3vl8b" ] }, "retry2-seed-notools": { "model_slug": "retry2-seed-notools", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "RETRY2_seed-notools" ] }, "retry-muse11bare": { "model_slug": "retry-muse11bare", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "RETRY_muse11bare" ] }, "retry-inklingp": { "model_slug": "retry-inklingp", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "RETRY_inklingp" ] }, "retry-qwen38b": { "model_slug": "retry-qwen38b", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "RETRY_qwen38b" ] }, "retry-qwen3vl32b": { "model_slug": "retry-qwen3vl32b", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "RETRY_qwen3vl32b" ] }, "write-dots3": { "model_slug": "write-dots3", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "WRITE_dots3" ] }, "super": { "model_slug": "super", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "super" ] }, "write-gemma31bit": { "model_slug": "write-gemma31bit", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "WRITE_gemma31bit" ] }, "write-inklingp": { "model_slug": "write-inklingp", "cosmos_test_count": 7, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "WRITE_inklingp" ] }, "write-qwen3vl32b": { "model_slug": "write-qwen3vl32b", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "WRITE_qwen3vl32b" ] }, "write-maverick": { "model_slug": "write-maverick", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "WRITE_maverick" ] }, "write-qwen3vl8b": { "model_slug": "write-qwen3vl8b", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "WRITE_qwen3vl8b" ] }, "write-seed-notools": { "model_slug": "write-seed-notools", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "WRITE_seed-notools" ] }, "meta/muse-spark-1.1-contributor": { "model_slug": "meta/muse-spark-1.1-contributor", "cosmos_test_count": 32, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 32, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "mouth-form": 28, "{\"error\":{\"message\":\"meta/muse-spark-1.1-contributor is not a valid model ID\",\"code\":400},\"user_id\":\"user_3Iw6zwRpqCi2CdefGLtSeuerqqT\"}": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "roster", "live_ccr" ] }, "z-ai/glm-5.3:batch": { "model_slug": "z-ai/glm-5.3:batch", "cosmos_test_count": 44, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 44, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "not a pinned OpenRouter id 'z-ai/glm-5.3:batch'; want ['bytedance-seed/seed-2.0-|PIN-NOTE": 28, "{\"error\":{\"message\":\"Not Found\",\"code\":404}}": 8, "revisit-later|PIN-NOTE": 4, "{\"error\":{\"message\":\"z-ai/glm-5.3:batch cannot be used with the chat/completions endpoint (adapter DeepInfraBatchAdapter": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "roster", "live_ccr" ] }, "deepseek/deepseek-v4-pro-0423": { "model_slug": "deepseek/deepseek-v4-pro-0423", "cosmos_test_count": 16, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 16, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "not a pinned OpenRouter id 'deepseek/deepseek-v4-pro-0423'; want ['bytedance-see|PIN-NOTE": 8, "revisit-later|PIN-NOTE": 4, "{\"error\":{\"message\":\"deepseek/deepseek-v4-pro-0423 is not a valid model ID\",\"code\":400},\"user_id\":\"user_3Iw6zwRpqCi2CdefGLtSeuerqqT\"}": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "roster", "live_ccr" ] }, "qwen/qwen2.5-7b-instruct": { "model_slug": "qwen/qwen2.5-7b-instruct", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 4, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "{\"error\":{\"message\":\"qwen/qwen2.5-7b-instruct is not a valid model ID\",\"code\":400},\"user_id\":\"user_3Iw6zwRpqCi2CdefGLtSeuerqqT\"}": 4 }, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "roster", "live_ccr" ] }, "copilot/auto": { "model_slug": "copilot/auto", "cosmos_test_count": 8, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 8, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "01_SEATING_DATA", "02_SEATING_AND_ATTEMPTS", "roster", "live_ccr" ] }, "roster": { "model_slug": "roster", "cosmos_test_count": 48, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "roster" ] }, "goal-round-driver": { "model_slug": "goal-round-driver", "cosmos_test_count": 58, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "goal-round-driver" ] }, "goal-wrapup": { "model_slug": "goal-wrapup", "cosmos_test_count": 51, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "goal-wrapup" ] }, "goal-tools": { "model_slug": "goal-tools", "cosmos_test_count": 32, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "goal-tools" ] }, "headless-profile": { "model_slug": "headless-profile", "cosmos_test_count": 24, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "headless-profile" ] }, "invalid-credential": { "model_slug": "invalid-credential", "cosmos_test_count": 15, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "invalid-credential" ] }, "missing-credential": { "model_slug": "missing-credential", "cosmos_test_count": 15, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "missing-credential" ] }, "semantic-checkpoint": { "model_slug": "semantic-checkpoint", "cosmos_test_count": 28, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "semantic-checkpoint" ] }, "provider-retry": { "model_slug": "provider-retry", "cosmos_test_count": 18, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "provider-retry" ] }, "subagent-diagnostic": { "model_slug": "subagent-diagnostic", "cosmos_test_count": 30, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "subagent-diagnostic" ] }, "subagent-inheritance": { "model_slug": "subagent-inheritance", "cosmos_test_count": 85, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "subagent-inheritance" ] }, "subagent-settlement": { "model_slug": "subagent-settlement", "cosmos_test_count": 54, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "subagent-settlement" ] }, "offline-edit": { "model_slug": "offline-edit", "cosmos_test_count": 27, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "offline-edit" ] }, "precedence-change": { "model_slug": "precedence-change", "cosmos_test_count": 26, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "precedence-change" ] }, "preview-architecture-review": { "model_slug": "preview-architecture-review", "cosmos_test_count": 356, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "preview-architecture-review" ] }, "preview-follow-up-builder": { "model_slug": "preview-follow-up-builder", "cosmos_test_count": 16, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "preview-follow-up-builder" ] }, "preview-showcase": { "model_slug": "preview-showcase", "cosmos_test_count": 400, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "preview-showcase" ] }, "fixtures": { "model_slug": "fixtures", "cosmos_test_count": 25, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "fixtures" ] }, "rec-child": { "model_slug": "rec-child", "cosmos_test_count": 22, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "rec-child" ] }, "rec-pin": { "model_slug": "rec-pin", "cosmos_test_count": 13, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "rec-pin" ] }, "authored-error": { "model_slug": "authored-error", "cosmos_test_count": 9, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "authored-error" ] }, "blocked-log": { "model_slug": "blocked-log", "cosmos_test_count": 8, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "blocked-log" ] }, "pin-turn": { "model_slug": "pin-turn", "cosmos_test_count": 24, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "pin-turn" ] }, "plain-turn": { "model_slug": "plain-turn", "cosmos_test_count": 29, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "plain-turn" ] }, "shared-pin": { "model_slug": "shared-pin", "cosmos_test_count": 19, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "shared-pin" ] }, "advanced": { "model_slug": "advanced", "cosmos_test_count": 335, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "advanced" ] }, "scheduler-recovery": { "model_slug": "scheduler-recovery", "cosmos_test_count": 62, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "scheduler-recovery" ] }, "restart": { "model_slug": "restart", "cosmos_test_count": 106, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "restart" ] }, "cancel": { "model_slug": "cancel", "cosmos_test_count": 37, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "cancel" ] }, "cancel-tool-calls": { "model_slug": "cancel-tool-calls", "cosmos_test_count": 48, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "cancel-tool-calls" ] }, "escalation-approved": { "model_slug": "escalation-approved", "cosmos_test_count": 56, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "escalation-approved" ] }, "escalation-rejected": { "model_slug": "escalation-rejected", "cosmos_test_count": 80, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "escalation-rejected" ] }, "fs-escalation-approved": { "model_slug": "fs-escalation-approved", "cosmos_test_count": 56, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "fs-escalation-approved" ] }, "fs-same-mode": { "model_slug": "fs-same-mode", "cosmos_test_count": 30, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "fs-same-mode" ] }, "handshake": { "model_slug": "handshake", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "handshake" ] }, "image-compaction": { "model_slug": "image-compaction", "cosmos_test_count": 99, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "image-compaction" ] }, "reject-extra-dirs": { "model_slug": "reject-extra-dirs", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "reject-extra-dirs" ] }, "account-provider-signout": { "model_slug": "account-provider-signout", "cosmos_test_count": 49, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "account-provider-signout" ] }, "dynamic-tool-updates": { "model_slug": "dynamic-tool-updates", "cosmos_test_count": 91, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "dynamic-tool-updates" ] }, "inline-image-prompt": { "model_slug": "inline-image-prompt", "cosmos_test_count": 36, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "inline-image-prompt" ] }, "bash-tool": { "model_slug": "bash-tool", "cosmos_test_count": 63, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "bash-tool" ] }, "max-tokens-continue": { "model_slug": "max-tokens-continue", "cosmos_test_count": 49, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "max-tokens-continue" ] }, "persistent-tools": { "model_slug": "persistent-tools", "cosmos_test_count": 147, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "persistent-tools" ] }, "multi-turn": { "model_slug": "multi-turn", "cosmos_test_count": 49, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "multi-turn" ] }, "serial-created": { "model_slug": "serial-created", "cosmos_test_count": 46, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "serial-created" ] }, "ptc-turn": { "model_slug": "ptc-turn", "cosmos_test_count": 101, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ptc-turn" ] }, "session-title-after-turn": { "model_slug": "session-title-after-turn", "cosmos_test_count": 37, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "session-title-after-turn" ] }, "subagent-activation-limit": { "model_slug": "subagent-activation-limit", "cosmos_test_count": 54, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "subagent-activation-limit" ] }, "subagent-continuable": { "model_slug": "subagent-continuable", "cosmos_test_count": 145, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "subagent-continuable" ] }, "subagent-continuable-inheritance": { "model_slug": "subagent-continuable-inheritance", "cosmos_test_count": 95, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "subagent-continuable-inheritance" ] }, "subagent-dsh-sdk-diagnostic": { "model_slug": "subagent-dsh-sdk-diagnostic", "cosmos_test_count": 105, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "subagent-dsh-sdk-diagnostic" ] }, "subagent-dsh-sdk-dynamic-route": { "model_slug": "subagent-dsh-sdk-dynamic-route", "cosmos_test_count": 98, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "subagent-dsh-sdk-dynamic-route" ] }, "subagent-fork-in-process": { "model_slug": "subagent-fork-in-process", "cosmos_test_count": 122, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "subagent-fork-in-process" ] }, "subagent-mixed": { "model_slug": "subagent-mixed", "cosmos_test_count": 168, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "subagent-mixed" ] }, "subagent-list-agents": { "model_slug": "subagent-list-agents", "cosmos_test_count": 183, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "subagent-list-agents" ] }, "subagent-send-message": { "model_slug": "subagent-send-message", "cosmos_test_count": 127, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "subagent-send-message" ] }, "subagent-spawn-in-process": { "model_slug": "subagent-spawn-in-process", "cosmos_test_count": 195, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "subagent-spawn-in-process" ] }, "tool-error-details": { "model_slug": "tool-error-details", "cosmos_test_count": 85, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "tool-error-details" ] }, "system-prompt-in-history": { "model_slug": "system-prompt-in-history", "cosmos_test_count": 67, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "system-prompt-in-history" ] }, "text-turn": { "model_slug": "text-turn", "cosmos_test_count": 138, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "text-turn" ] }, "tool-scheduler-recovery": { "model_slug": "tool-scheduler-recovery", "cosmos_test_count": 62, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "tool-scheduler-recovery" ] }, "advanced-toolchain": { "model_slug": "advanced-toolchain", "cosmos_test_count": 255, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "advanced-toolchain" ] }, "advanced-toolchain-runtime": { "model_slug": "advanced-toolchain-runtime", "cosmos_test_count": 232, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "advanced-toolchain-runtime" ] }, "agent-instructions": { "model_slug": "agent-instructions", "cosmos_test_count": 77, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "agent-instructions" ] }, "background-confinement-failure": { "model_slug": "background-confinement-failure", "cosmos_test_count": 30, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "background-confinement-failure" ] }, "background-job-admission": { "model_slug": "background-job-admission", "cosmos_test_count": 73, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "background-job-admission" ] }, "bash-same-mode-empty-justification": { "model_slug": "bash-same-mode-empty-justification", "cosmos_test_count": 22, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "bash-same-mode-empty-justification" ] }, "bash-spill": { "model_slug": "bash-spill", "cosmos_test_count": 65, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "bash-spill" ] }, "bash-startup-timeout": { "model_slug": "bash-startup-timeout", "cosmos_test_count": 22, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "bash-startup-timeout" ] }, "bash-tool-turn": { "model_slug": "bash-tool-turn", "cosmos_test_count": 65, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "bash-tool-turn" ] }, "both-mode-turn": { "model_slug": "both-mode-turn", "cosmos_test_count": 47, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "both-mode-turn" ] }, "browser-use-chrome-devtools-mcp": { "model_slug": "browser-use-chrome-devtools-mcp", "cosmos_test_count": 22, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "browser-use-chrome-devtools-mcp" ] }, "browser-use-playwright-mcp": { "model_slug": "browser-use-playwright-mcp", "cosmos_test_count": 22, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "browser-use-playwright-mcp" ] }, "browser-use-stagehand-native": { "model_slug": "browser-use-stagehand-native", "cosmos_test_count": 27, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "browser-use-stagehand-native" ] }, "compaction-output-reserve": { "model_slug": "compaction-output-reserve", "cosmos_test_count": 54, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "compaction-output-reserve" ] }, "compaction-recovery": { "model_slug": "compaction-recovery", "cosmos_test_count": 55, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "compaction-recovery" ] }, "compaction-summary-headroom": { "model_slug": "compaction-summary-headroom", "cosmos_test_count": 54, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "compaction-summary-headroom" ] }, "computer-use-cua-driver-mcp": { "model_slug": "computer-use-cua-driver-mcp", "cosmos_test_count": 22, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "computer-use-cua-driver-mcp" ] }, "computer-use-cua-driver-native": { "model_slug": "computer-use-cua-driver-native", "cosmos_test_count": 22, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "computer-use-cua-driver-native" ] }, "cordis-inspect-jsdoc": { "model_slug": "cordis-inspect-jsdoc", "cosmos_test_count": 80, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "cordis-inspect-jsdoc" ] }, "cordis-inspect-timeout": { "model_slug": "cordis-inspect-timeout", "cosmos_test_count": 27, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "cordis-inspect-timeout" ] }, "cordis-inspect-liveness": { "model_slug": "cordis-inspect-liveness", "cosmos_test_count": 35, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "cordis-inspect-liveness" ] }, "deepseek-messages-degraded-replay": { "model_slug": "deepseek-messages-degraded-replay", "cosmos_test_count": 43, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "deepseek-messages-degraded-replay" ] }, "deepseek-messages-invalid-tool-history": { "model_slug": "deepseek-messages-invalid-tool-history", "cosmos_test_count": 22, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "deepseek-messages-invalid-tool-history" ] }, "deepseek-messages-input-history": { "model_slug": "deepseek-messages-input-history", "cosmos_test_count": 19, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "deepseek-messages-input-history" ] }, "deepseek-messages-replay": { "model_slug": "deepseek-messages-replay", "cosmos_test_count": 43, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "deepseek-messages-replay" ] }, "deepseek-messages-system-prompt": { "model_slug": "deepseek-messages-system-prompt", "cosmos_test_count": 23, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "deepseek-messages-system-prompt" ] }, "dynamic-tool-prompt-updates": { "model_slug": "dynamic-tool-prompt-updates", "cosmos_test_count": 32, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "dynamic-tool-prompt-updates" ] }, "deepseek-protocol-system-prompt": { "model_slug": "deepseek-protocol-system-prompt", "cosmos_test_count": 23, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "deepseek-protocol-system-prompt" ] }, "empty-response-retry": { "model_slug": "empty-response-retry", "cosmos_test_count": 45, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "empty-response-retry" ] }, "empty-response-retry-current": { "model_slug": "empty-response-retry-current", "cosmos_test_count": 20, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "empty-response-retry-current" ] }, "error-finish": { "model_slug": "error-finish", "cosmos_test_count": 33, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "error-finish" ] }, "fs-edit": { "model_slug": "fs-edit", "cosmos_test_count": 53, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "fs-edit" ] }, "fs-delete-recreate": { "model_slug": "fs-delete-recreate", "cosmos_test_count": 73, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "fs-delete-recreate" ] }, "foreground-confinement-timeout": { "model_slug": "foreground-confinement-timeout", "cosmos_test_count": 22, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "foreground-confinement-timeout" ] }, "fs-glob-sampling": { "model_slug": "fs-glob-sampling", "cosmos_test_count": 43, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "fs-glob-sampling" ] }, "fs-policy-reject": { "model_slug": "fs-policy-reject", "cosmos_test_count": 53, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "fs-policy-reject" ] }, "fs-read": { "model_slug": "fs-read", "cosmos_test_count": 43, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "fs-read" ] }, "fs-read-window": { "model_slug": "fs-read-window", "cosmos_test_count": 43, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "fs-read-window" ] }, "fs-write": { "model_slug": "fs-write", "cosmos_test_count": 43, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "fs-write" ] }, "fs-write-overwrite-bounded": { "model_slug": "fs-write-overwrite-bounded", "cosmos_test_count": 53, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "fs-write-overwrite-bounded" ] }, "fs-write-overwrite": { "model_slug": "fs-write-overwrite", "cosmos_test_count": 53, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "fs-write-overwrite" ] }, "hook-cc-invalid-matcher": { "model_slug": "hook-cc-invalid-matcher", "cosmos_test_count": 33, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "hook-cc-invalid-matcher" ] }, "hook-cc-posttool-block": { "model_slug": "hook-cc-posttool-block", "cosmos_test_count": 61, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "hook-cc-posttool-block" ] }, "hook-cc-posttool-context": { "model_slug": "hook-cc-posttool-context", "cosmos_test_count": 53, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "hook-cc-posttool-context" ] }, "hook-cc-pretool-ask": { "model_slug": "hook-cc-pretool-ask", "cosmos_test_count": 51, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "hook-cc-pretool-ask" ] }, "hook-cc-pretool-deny": { "model_slug": "hook-cc-pretool-deny", "cosmos_test_count": 47, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "hook-cc-pretool-deny" ] }, "hook-cc-promptsubmit-block": { "model_slug": "hook-cc-promptsubmit-block", "cosmos_test_count": 20, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "hook-cc-promptsubmit-block" ] }, "hook-cc-promptsubmit-context": { "model_slug": "hook-cc-promptsubmit-context", "cosmos_test_count": 39, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "hook-cc-promptsubmit-context" ] }, "hook-cc-stop-continue": { "model_slug": "hook-cc-stop-continue", "cosmos_test_count": 53, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "hook-cc-stop-continue" ] }, "hook-codex-invalid-matcher": { "model_slug": "hook-codex-invalid-matcher", "cosmos_test_count": 33, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "hook-codex-invalid-matcher" ] }, "hook-codex-posttool-block": { "model_slug": "hook-codex-posttool-block", "cosmos_test_count": 47, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "hook-codex-posttool-block" ] }, "hook-codex-posttool-context": { "model_slug": "hook-codex-posttool-context", "cosmos_test_count": 53, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "hook-codex-posttool-context" ] }, "hook-codex-pretool-block": { "model_slug": "hook-codex-pretool-block", "cosmos_test_count": 47, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "hook-codex-pretool-block" ] }, "hook-codex-promptsubmit-block": { "model_slug": "hook-codex-promptsubmit-block", "cosmos_test_count": 20, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "hook-codex-promptsubmit-block" ] }, "hook-codex-promptsubmit-context": { "model_slug": "hook-codex-promptsubmit-context", "cosmos_test_count": 39, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "hook-codex-promptsubmit-context" ] }, "hook-codex-stop-continue": { "model_slug": "hook-codex-stop-continue", "cosmos_test_count": 53, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "hook-codex-stop-continue" ] }, "lsp-definition": { "model_slug": "lsp-definition", "cosmos_test_count": 43, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "lsp-definition" ] }, "mcp-empty": { "model_slug": "mcp-empty", "cosmos_test_count": 22, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "mcp-empty" ] }, "mcp-empty-ptc": { "model_slug": "mcp-empty-ptc", "cosmos_test_count": 26, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "mcp-empty-ptc" ] }, "mcp-resources": { "model_slug": "mcp-resources", "cosmos_test_count": 36, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "mcp-resources" ] }, "mcp-resources-ptc": { "model_slug": "mcp-resources-ptc", "cosmos_test_count": 32, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "mcp-resources-ptc" ] }, "missing-sandbox-runner": { "model_slug": "missing-sandbox-runner", "cosmos_test_count": 53, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "missing-sandbox-runner" ] }, "model-switch-notice": { "model_slug": "model-switch-notice", "cosmos_test_count": 52, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "model-switch-notice" ] }, "multimodal-spill-ends": { "model_slug": "multimodal-spill-ends", "cosmos_test_count": 44, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "multimodal-spill-ends" ] }, "multimodal-spill-middle": { "model_slug": "multimodal-spill-middle", "cosmos_test_count": 44, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "multimodal-spill-middle" ] }, "office-skills": { "model_slug": "office-skills", "cosmos_test_count": 50, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "office-skills" ] }, "office-skills-no-renderer": { "model_slug": "office-skills-no-renderer", "cosmos_test_count": 54, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "office-skills-no-renderer" ] }, "parallel-tool-calls": { "model_slug": "parallel-tool-calls", "cosmos_test_count": 71, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "parallel-tool-calls" ] }, "packed-chunks": { "model_slug": "packed-chunks", "cosmos_test_count": 63, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "packed-chunks" ] }, "partial-landlock-child-failure": { "model_slug": "partial-landlock-child-failure", "cosmos_test_count": 43, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "partial-landlock-child-failure" ] }, "persistent-pwsh-padded-completion": { "model_slug": "persistent-pwsh-padded-completion", "cosmos_test_count": 27, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "persistent-pwsh-padded-completion" ] }, "persistent-pwsh-tool-turn": { "model_slug": "persistent-pwsh-tool-turn", "cosmos_test_count": 43, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "persistent-pwsh-tool-turn" ] }, "pi-ai-replay-metadata": { "model_slug": "pi-ai-replay-metadata", "cosmos_test_count": 33, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "pi-ai-replay-metadata" ] }, "plugin-manager": { "model_slug": "plugin-manager", "cosmos_test_count": 54, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "plugin-manager" ] }, "plugin-manager-mcp": { "model_slug": "plugin-manager-mcp", "cosmos_test_count": 59, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "plugin-manager-mcp" ] }, "product-subagent-both": { "model_slug": "product-subagent-both", "cosmos_test_count": 33, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "product-subagent-both" ] }, "product-subagent-codex": { "model_slug": "product-subagent-codex", "cosmos_test_count": 33, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "product-subagent-codex" ] }, "product-subagent-result-diagnostic": { "model_slug": "product-subagent-result-diagnostic", "cosmos_test_count": 105, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "product-subagent-result-diagnostic" ] }, "provider-cwd": { "model_slug": "provider-cwd", "cosmos_test_count": 17, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "provider-cwd" ] }, "ptc-node-read-only": { "model_slug": "ptc-node-read-only", "cosmos_test_count": 24, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ptc-node-read-only" ] }, "ptc-node-workspace": { "model_slug": "ptc-node-workspace", "cosmos_test_count": 24, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ptc-node-workspace" ] }, "ptc-python-turn": { "model_slug": "ptc-python-turn", "cosmos_test_count": 51, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ptc-python-turn" ] }, "ptc-read-image": { "model_slug": "ptc-read-image", "cosmos_test_count": 57, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ptc-read-image" ] }, "ptc-read-image-attachment-path": { "model_slug": "ptc-read-image-attachment-path", "cosmos_test_count": 73, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ptc-read-image-attachment-path" ] }, "ptc-workspace-context": { "model_slug": "ptc-workspace-context", "cosmos_test_count": 55, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ptc-workspace-context" ] }, "pty-tools-sandbox-backend": { "model_slug": "pty-tools-sandbox-backend", "cosmos_test_count": 83, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "pty-tools-sandbox-backend" ] }, "pwsh-tool-turn": { "model_slug": "pwsh-tool-turn", "cosmos_test_count": 43, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "pwsh-tool-turn" ] }, "ralph-loop": { "model_slug": "ralph-loop", "cosmos_test_count": 123, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ralph-loop" ] }, "read-image": { "model_slug": "read-image", "cosmos_test_count": 43, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "read-image" ] }, "read-image-attachment-path": { "model_slug": "read-image-attachment-path", "cosmos_test_count": 53, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "read-image-attachment-path" ] }, "read-image-dimension": { "model_slug": "read-image-dimension", "cosmos_test_count": 43, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "read-image-dimension" ] }, "read-image-reencode": { "model_slug": "read-image-reencode", "cosmos_test_count": 43, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "read-image-reencode" ] }, "read-image-gif": { "model_slug": "read-image-gif", "cosmos_test_count": 43, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "read-image-gif" ] }, "read-image-text-route": { "model_slug": "read-image-text-route", "cosmos_test_count": 43, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "read-image-text-route" ] }, "repeat-tool-reminder": { "model_slug": "repeat-tool-reminder", "cosmos_test_count": 87, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "repeat-tool-reminder" ] }, "session-query-spill": { "model_slug": "session-query-spill", "cosmos_test_count": 80, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "session-query-spill" ] }, "session-reference-spill": { "model_slug": "session-reference-spill", "cosmos_test_count": 35, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "session-reference-spill" ] }, "skill-load": { "model_slug": "skill-load", "cosmos_test_count": 68, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "skill-load" ] }, "session-sandbox-root": { "model_slug": "session-sandbox-root", "cosmos_test_count": 43, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "session-sandbox-root" ] }, "subagent-child-question-rejection": { "model_slug": "subagent-child-question-rejection", "cosmos_test_count": 89, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "subagent-child-question-rejection" ] }, "subagent-acp-diagnostic": { "model_slug": "subagent-acp-diagnostic", "cosmos_test_count": 57, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "subagent-acp-diagnostic" ] }, "subagent-depth-two-rejection": { "model_slug": "subagent-depth-two-rejection", "cosmos_test_count": 135, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "subagent-depth-two-rejection" ] }, "subagent-max-tokens-partial": { "model_slug": "subagent-max-tokens-partial", "cosmos_test_count": 91, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "subagent-max-tokens-partial" ] }, "subagent-multi": { "model_slug": "subagent-multi", "cosmos_test_count": 166, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "subagent-multi" ] }, "subagent-published-run-failure": { "model_slug": "subagent-published-run-failure", "cosmos_test_count": 52, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "subagent-published-run-failure" ] }, "subagent-parallel": { "model_slug": "subagent-parallel", "cosmos_test_count": 119, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "subagent-parallel" ] }, "subagent-tool-filter": { "model_slug": "subagent-tool-filter", "cosmos_test_count": 46, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "subagent-tool-filter" ] }, "team-targets": { "model_slug": "team-targets", "cosmos_test_count": 27, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "team-targets" ] }, "tool-call-turn": { "model_slug": "tool-call-turn", "cosmos_test_count": 65, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "tool-call-turn" ] }, "todo-write": { "model_slug": "todo-write", "cosmos_test_count": 45, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "todo-write" ] }, "web-search-endpoint-guidance": { "model_slug": "web-search-endpoint-guidance", "cosmos_test_count": 45, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "web-search-endpoint-guidance" ] }, "web-fetch": { "model_slug": "web-fetch", "cosmos_test_count": 43, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "web-fetch" ] }, "windows-acl-skill": { "model_slug": "windows-acl-skill", "cosmos_test_count": 23, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "windows-acl-skill" ] }, "workflow-confinement": { "model_slug": "workflow-confinement", "cosmos_test_count": 24, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "workflow-confinement" ] }, "workflow-run": { "model_slug": "workflow-run", "cosmos_test_count": 87, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "workflow-run" ] }, "workspace-dependencies": { "model_slug": "workspace-dependencies", "cosmos_test_count": 22, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "workspace-dependencies" ] }, "workspace-edit": { "model_slug": "workspace-edit", "cosmos_test_count": 53, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "workspace-edit" ] }, "approval-composer": { "model_slug": "approval-composer", "cosmos_test_count": 98, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "approval-composer" ] }, "auto-review-denial": { "model_slug": "auto-review-denial", "cosmos_test_count": 20, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "auto-review-denial" ] }, "auto-review-approval": { "model_slug": "auto-review-approval", "cosmos_test_count": 20, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "auto-review-approval" ] }, "changed-files-turn": { "model_slug": "changed-files-turn", "cosmos_test_count": 84, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "changed-files-turn" ] }, "code-language": { "model_slug": "code-language", "cosmos_test_count": 26, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "code-language" ] }, "cordis-tool-round": { "model_slug": "cordis-tool-round", "cosmos_test_count": 105, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "cordis-tool-round" ] }, "deepseek-messages-chat": { "model_slug": "deepseek-messages-chat", "cosmos_test_count": 50, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "deepseek-messages-chat" ] }, "diff-bounded": { "model_slug": "diff-bounded", "cosmos_test_count": 27, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "diff-bounded" ] }, "feedback-command": { "model_slug": "feedback-command", "cosmos_test_count": 59, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "feedback-command" ] }, "file-upload-round": { "model_slug": "file-upload-round", "cosmos_test_count": 68, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "file-upload-round" ] }, "fresh-round-trip": { "model_slug": "fresh-round-trip", "cosmos_test_count": 65, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "fresh-round-trip" ] }, "goal-multi-turn-actions": { "model_slug": "goal-multi-turn-actions", "cosmos_test_count": 251, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "goal-multi-turn-actions" ] }, "lifecycle-chrome": { "model_slug": "lifecycle-chrome", "cosmos_test_count": 50, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "lifecycle-chrome" ] }, "live-interactions": { "model_slug": "live-interactions", "cosmos_test_count": 50, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "live-interactions" ] }, "markdown-file-links": { "model_slug": "markdown-file-links", "cosmos_test_count": 10, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "markdown-file-links" ] }, "message-feedback-protocol": { "model_slug": "message-feedback-protocol", "cosmos_test_count": 14, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "message-feedback-protocol" ] }, "minimal-preset": { "model_slug": "minimal-preset", "cosmos_test_count": 62, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "minimal-preset" ] }, "navigation-panes": { "model_slug": "navigation-panes", "cosmos_test_count": 49, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "navigation-panes" ] }, "permission-policy-context": { "model_slug": "permission-policy-context", "cosmos_test_count": 256, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "permission-policy-context" ] }, "plan-review": { "model_slug": "plan-review", "cosmos_test_count": 82, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "plan-review" ] }, "present": { "model_slug": "present", "cosmos_test_count": 86, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "present" ] }, "present-svg": { "model_slug": "present-svg", "cosmos_test_count": 58, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "present-svg" ] }, "preset-migration": { "model_slug": "preset-migration", "cosmos_test_count": 11, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "preset-migration" ] }, "ptc-escalation-approved": { "model_slug": "ptc-escalation-approved", "cosmos_test_count": 66, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ptc-escalation-approved" ] }, "pwsh-terminal": { "model_slug": "pwsh-terminal", "cosmos_test_count": 23, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "pwsh-terminal" ] }, "ptc-round": { "model_slug": "ptc-round", "cosmos_test_count": 77, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ptc-round" ] }, "question-composer": { "model_slug": "question-composer", "cosmos_test_count": 65, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "question-composer" ] }, "schedule-catalog": { "model_slug": "schedule-catalog", "cosmos_test_count": 22, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "schedule-catalog" ] }, "seeded-history": { "model_slug": "seeded-history", "cosmos_test_count": 33, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "seeded-history" ] }, "steering": { "model_slug": "steering", "cosmos_test_count": 80, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "steering" ] }, "tool-details": { "model_slug": "tool-details", "cosmos_test_count": 100, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "tool-details" ] }, "turn-tail-actions": { "model_slug": "turn-tail-actions", "cosmos_test_count": 65, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "turn-tail-actions" ] }, "web-search-round": { "model_slug": "web-search-round", "cosmos_test_count": 71, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "web-search-round" ] }, "session": { "model_slug": "session", "cosmos_test_count": 101, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "SESSION", "session" ] }, "sessions-experts": { "model_slug": "sessions-experts", "cosmos_test_count": 9, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "SESSIONS_EXPERTS" ] }, "s": { "model_slug": "s", "cosmos_test_count": 494, "pass_rate_pct": 5.1, "twelve_cs_score": 9.3, "passes": 25, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "s" ] }, "c": { "model_slug": "c", "cosmos_test_count": 180, "pass_rate_pct": 6.1, "twelve_cs_score": 10.3, "passes": 11, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "c" ] }, "quota": { "model_slug": "quota", "cosmos_test_count": 7, "pass_rate_pct": 85.7, "twelve_cs_score": 85.9, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "quota" ] }, "h": { "model_slug": "h", "cosmos_test_count": 37, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "h" ] }, "spend": { "model_slug": "spend", "cosmos_test_count": 16, "pass_rate_pct": 37.5, "twelve_cs_score": 40.1, "passes": 6, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "spend" ] }, "meter": { "model_slug": "meter", "cosmos_test_count": 3, "pass_rate_pct": 100.0, "twelve_cs_score": 99.5, "passes": 3, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "meter" ] }, "p": { "model_slug": "p", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "p" ] }, "projection": { "model_slug": "projection", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "projection" ] }, "m": { "model_slug": "m", "cosmos_test_count": 10, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "m" ] }, "live-ccr": { "model_slug": "live-ccr", "cosmos_test_count": 641, "pass_rate_pct": 0.2, "twelve_cs_score": 4.7, "passes": 1, "fails": 170, "rate_limits_429": 3, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "live_ccr" ] }, "08-nemo35f": { "model_slug": "08-nemo35f", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "08_nemo35f" ] }, "10-lagunaxs": { "model_slug": "10-lagunaxs", "cosmos_test_count": 9, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 7, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "10_lagunaxs" ] }, "14-lingvl": { "model_slug": "14-lingvl", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "14_lingvl" ] }, "18-gemma426b": { "model_slug": "18-gemma426b", "cosmos_test_count": 8, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 7, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "18_gemma426b" ] }, "19-qwen27f": { "model_slug": "19-qwen27f", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 5, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "19_qwen27f" ] }, "24-inklingf": { "model_slug": "24-inklingf", "cosmos_test_count": 5, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "24_inklingf" ] }, "26-gemma431b": { "model_slug": "26-gemma431b", "cosmos_test_count": 9, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 7, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "26_gemma431b" ] }, "ledger": { "model_slug": "ledger", "cosmos_test_count": 750, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "ledger" ] }, "control": { "model_slug": "control", "cosmos_test_count": 18, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "control" ] }, "cosmos-cc-1p7yx0v": { "model_slug": "cosmos-cc-1p7yx0v", "cosmos_test_count": 204, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "cosmos_cc__1p7yx0v" ] }, "q": { "model_slug": "q", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "q" ] }, "cosmos-claude-rail-e1wk6d5g": { "model_slug": "cosmos-claude-rail-e1wk6d5g", "cosmos_test_count": 8, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "cosmos_claude_rail_e1wk6d5g" ] }, "claude-rail": { "model_slug": "claude-rail", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "claude_rail" ] }, "collector": { "model_slug": "collector", "cosmos_test_count": 58, "pass_rate_pct": 58.6, "twelve_cs_score": 60.2, "passes": 34, "fails": 2, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "collector" ] }, "queue": { "model_slug": "queue", "cosmos_test_count": 14, "pass_rate_pct": 14.3, "twelve_cs_score": 18.1, "passes": 2, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "queue" ] }, "export": { "model_slug": "export", "cosmos_test_count": 17, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "export" ] }, "cosmos-code-export": { "model_slug": "cosmos-code-export", "cosmos_test_count": 28, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "cosmos_code_export" ] }, "hero-run-20261008": { "model_slug": "hero-run-20261008", "cosmos_test_count": 2154, "pass_rate_pct": 36.5, "twelve_cs_score": 39.2, "passes": 786, "fails": 186, "rate_limits_429": 2, "median_latency_ms": 145.0, "top_scars": {}, "source_clusters": [ "_hero_run_20261008" ] }, "apodex-apodex-1.1-mini-free": { "model_slug": "apodex-apodex-1.1-mini-free", "cosmos_test_count": 5, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "MOUTH_FORM": 4, "LOOKS_SHAPED": 1 }, "source_clusters": [ "apodex_apodex-1.1-mini_free" ] }, "door-opencode": { "model_slug": "door-opencode", "cosmos_test_count": 53, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 3 }, "source_clusters": [ "door_opencode" ] }, "nvidia-nemotron-3-nano-omni-30b-a3b-reasoning-free": { "model_slug": "nvidia-nemotron-3-nano-omni-30b-a3b-reasoning-free", "cosmos_test_count": 5, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "EMPTY": 3, "LOOKS_SHAPED": 2 }, "source_clusters": [ "nvidia_nemotron-3-nano-omni-30b-a3b-reasoning_free" ] }, "qwen-qwen3.5-9b": { "model_slug": "qwen-qwen3.5-9b", "cosmos_test_count": 10, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "MOUTH_FORM": 7, "EMPTY": 2, "LOOKS_SHAPED": 1 }, "source_clusters": [ "qwen_qwen3.5-9b" ] }, "ibm-granite-granite-4.0-h-micro": { "model_slug": "ibm-granite-granite-4.0-h-micro", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "LOOKS_SHAPED": 4 }, "source_clusters": [ "ibm-granite_granite-4.0-h-micro" ] }, "google-gemma-4-26b-a4b-it-free": { "model_slug": "google-gemma-4-26b-a4b-it-free", "cosmos_test_count": 8, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 8, "median_latency_ms": 145.0, "top_scars": { "UPSTREAM_POOL": 8 }, "source_clusters": [ "google_gemma-4-26b-a4b-it_free" ] }, "amazon-nova-lite-v1": { "model_slug": "amazon-nova-lite-v1", "cosmos_test_count": 10, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "FENCE": 10 }, "source_clusters": [ "amazon_nova-lite-v1" ] }, "inference-net-schematron-v2-small": { "model_slug": "inference-net-schematron-v2-small", "cosmos_test_count": 10, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "MOUTH_FORM": 7, "FENCE": 2, "LOOKS_SHAPED": 1 }, "source_clusters": [ "inference-net_schematron-v2-small" ] }, "google-lyria-3-pro-preview": { "model_slug": "google-lyria-3-pro-preview", "cosmos_test_count": 10, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "LOOKS_SHAPED": 9, "UPSTREAM": 1 }, "source_clusters": [ "google_lyria-3-pro-preview" ] }, "google-gemma-3-4b-it": { "model_slug": "google-gemma-3-4b-it", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "FENCE": 2, "LOOKS_SHAPED": 2 }, "source_clusters": [ "google_gemma-3-4b-it" ] }, "gryphe-mythomax-l2-13b": { "model_slug": "gryphe-mythomax-l2-13b", "cosmos_test_count": 6, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "LOOKS_SHAPED": 6 }, "source_clusters": [ "gryphe_mythomax-l2-13b" ] }, "qwen-qwen3-32b": { "model_slug": "qwen-qwen3-32b", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "MOUTH_FORM": 2, "LOOKS_SHAPED": 2 }, "source_clusters": [ "qwen_qwen3-32b" ] }, "google-gemma-4-31b-it-free": { "model_slug": "google-gemma-4-31b-it-free", "cosmos_test_count": 8, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 8, "median_latency_ms": 145.0, "top_scars": { "UPSTREAM_POOL": 8 }, "source_clusters": [ "google_gemma-4-31b-it_free" ] }, "inference-net-schematron-v2-turbo": { "model_slug": "inference-net-schematron-v2-turbo", "cosmos_test_count": 10, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "MOUTH_FORM": 10 }, "source_clusters": [ "inference-net_schematron-v2-turbo" ] }, "google-lyria-3-clip-preview": { "model_slug": "google-lyria-3-clip-preview", "cosmos_test_count": 9, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "MOUTH_FORM": 9 }, "source_clusters": [ "google_lyria-3-clip-preview" ] }, "cohere-north-mini-code-free": { "model_slug": "cohere-north-mini-code-free", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "MOUTH_FORM": 2, "LOOKS_SHAPED": 2 }, "source_clusters": [ "cohere_north-mini-code_free" ] }, "inclusionai-ling-3.0-flash-fin": { "model_slug": "inclusionai-ling-3.0-flash-fin", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "MOUTH_FORM": 3, "LOOKS_SHAPED": 1 }, "source_clusters": [ "inclusionai_ling-3.0-flash-fin" ] }, "ibm-granite-granite-4.2-8b": { "model_slug": "ibm-granite-granite-4.2-8b", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "LOOKS_SHAPED": 2, "MOUTH_FORM": 2 }, "source_clusters": [ "ibm-granite_granite-4.2-8b" ] }, "inclusionai-ling-3.0-flash-sante": { "model_slug": "inclusionai-ling-3.0-flash-sante", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "MOUTH_FORM": 2, "LOOKS_SHAPED": 2 }, "source_clusters": [ "inclusionai_ling-3.0-flash-sante" ] }, "poolside-laguna-xs-2.1-free": { "model_slug": "poolside-laguna-xs-2.1-free", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 2, "median_latency_ms": 145.0, "top_scars": { "UPSTREAM_POOL": 2, "LOOKS_SHAPED": 1 }, "source_clusters": [ "poolside_laguna-xs-2.1_free" ] }, "nvidia-nemotron-3.5-content-safety-free": { "model_slug": "nvidia-nemotron-3.5-content-safety-free", "cosmos_test_count": 9, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "NOT_CODE": 7, "MOUTH_FORM": 2 }, "source_clusters": [ "nvidia_nemotron-3.5-content-safety_free" ] }, "openai-gpt-5-nano": { "model_slug": "openai-gpt-5-nano", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "LOOKS_SHAPED": 2, "MOUTH_FORM": 2 }, "source_clusters": [ "openai_gpt-5-nano" ] }, "microsoft-phi-4": { "model_slug": "microsoft-phi-4", "cosmos_test_count": 10, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "FENCE": 10 }, "source_clusters": [ "microsoft_phi-4" ] }, "meta-muse-spark-1.2-contributor": { "model_slug": "meta-muse-spark-1.2-contributor", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "MOUTH_FORM": 2, "REJECTED": 1, "LOOKS_SHAPED": 1 }, "source_clusters": [ "meta_muse-spark-1.2-contributor" ] }, "rekaai-reka-edge": { "model_slug": "rekaai-reka-edge", "cosmos_test_count": 10, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "LOOKS_SHAPED": 8, "FENCE": 1, "MOUTH_FORM": 1 }, "source_clusters": [ "rekaai_reka-edge" ] }, "rekaai-reka-flash-3": { "model_slug": "rekaai-reka-flash-3", "cosmos_test_count": 19, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "MOUTH_FORM": 16, "EMPTY": 3 }, "source_clusters": [ "rekaai_reka-flash-3" ] }, "meta-muse-spark-1.3-contributor": { "model_slug": "meta-muse-spark-1.3-contributor", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "MOUTH_FORM": 2, "REJECTED": 1, "LOOKS_SHAPED": 1 }, "source_clusters": [ "meta_muse-spark-1.3-contributor" ] }, "nvidia-nemotron-3.5-lightning": { "model_slug": "nvidia-nemotron-3.5-lightning", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "LOOKS_SHAPED": 2, "MOUTH_FORM": 2 }, "source_clusters": [ "nvidia_nemotron-3.5-lightning" ] }, "meta-llama-llama-3.2-1b-instruct": { "model_slug": "meta-llama-llama-3.2-1b-instruct", "cosmos_test_count": 10, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "LOOKS_SHAPED": 10 }, "source_clusters": [ "meta-llama_llama-3.2-1b-instruct" ] }, "google-gemma-3-27b-it": { "model_slug": "google-gemma-3-27b-it", "cosmos_test_count": 3, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 1, "median_latency_ms": 145.0, "top_scars": { "LOOKS_SHAPED": 2, "UPSTREAM_POOL": 1 }, "source_clusters": [ "google_gemma-3-27b-it" ] }, "morph-morph-v3-large": { "model_slug": "morph-morph-v3-large", "cosmos_test_count": 5, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "LOOKS_SHAPED": 4, "REJECTED": 1 }, "source_clusters": [ "morph_morph-v3-large" ] }, "google-gemma-2-27b-it": { "model_slug": "google-gemma-2-27b-it", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "LOOKS_SHAPED": 4 }, "source_clusters": [ "google_gemma-2-27b-it" ] }, "minimax-minimax-m1": { "model_slug": "minimax-minimax-m1", "cosmos_test_count": 26, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "MOUTH_FORM": 23, "LOOKS_SHAPED": 2, "EMPTY": 1 }, "source_clusters": [ "minimax_minimax-m1" ] }, "mancer-weaver": { "model_slug": "mancer-weaver", "cosmos_test_count": 18, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "MOUTH_FORM": 17, "LOOKS_SHAPED": 1 }, "source_clusters": [ "mancer_weaver" ] }, "meta-llama-llama-guard-4-12b": { "model_slug": "meta-llama-llama-guard-4-12b", "cosmos_test_count": 15, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 0, "median_latency_ms": 145.0, "top_scars": { "MOUTH_FORM": 12, "NOT_CODE": 3 }, "source_clusters": [ "meta-llama_llama-guard-4-12b" ] }, "mistralai-mistral-saba": { "model_slug": "mistralai-mistral-saba", "cosmos_test_count": 4, "pass_rate_pct": 0.0, "twelve_cs_score": 0.0, "passes": 0, "fails": 0, "rate_limits_429": 3, "median_latency_ms": 145.0, "top_scars": { "UPSTREAM_POOL": 3, "LOOKS_SHAPED": 1 }, "source_clusters": [ "mistralai_mistral-saba" ] } } }