{
  "schema": "modelrater-benchmarks/1",
  "benchmarks": {
    "anthropic/claude-3.5-sonnet": {
      "elo": 1374,
      "coding_elo": 1378,
      "swe_bench": 49.0,
      "humaneval": 93.7
    },
    "openai/gpt-4o": {
      "elo": 1332,
      "coding_elo": 1330,
      "swe_bench": 38.8,
      "humaneval": 90.2
    },
    "openai/gpt-4o-mini": {
      "elo": 1275,
      "coding_elo": 1260,
      "swe_bench": 28.5,
      "humaneval": 84.1
    },
    "qwen/qwen3-coder-30b-fp8": {
      "elo": 1295,
      "coding_elo": 1310,
      "swe_bench": 41.2,
      "humaneval": 93.1
    },
    "qwen/qwen-2.5-coder-32b-instruct": {
      "elo": 1284,
      "coding_elo": 1302,
      "swe_bench": 35.8,
      "humaneval": 92.7
    },
    "google/gemini-flash-1.5": {
      "elo": 1268,
      "coding_elo": 1255,
      "swe_bench": 26.0,
      "humaneval": 82.5
    },
    "google/gemini-2.5-flash": {
      "elo": 1320,
      "coding_elo": 1325,
      "swe_bench": 36.5,
      "humaneval": 88.0
    },
    "vertex/gemini-3.8-flash": {
      "elo": 1328,
      "coding_elo": 1335,
      "swe_bench": 37.8,
      "humaneval": 89.5
    },
    "deepseek/deepseek-chat": {
      "elo": 1315,
      "coding_elo": 1320,
      "swe_bench": 36.2,
      "humaneval": 89.4
    },
    "meta-llama/llama-3.3-70b-instruct": {
      "elo": 1290,
      "coding_elo": 1285,
      "swe_bench": 32.4,
      "humaneval": 87.5
    },
    "meta-llama/llama-3.1-8b-instruct": {
      "elo": 1180,
      "coding_elo": 1165,
      "swe_bench": 18.0,
      "humaneval": 72.0
    },
    "mistralai/mistral-large-2407": {
      "elo": 1265,
      "coding_elo": 1260,
      "swe_bench": 27.5,
      "humaneval": 84.0
    },
    "x-ai/grok-2": {
      "elo": 1280,
      "coding_elo": 1275,
      "swe_bench": 31.0,
      "humaneval": 86.5
    },
    "z-ai/glm-4": {
      "elo": 1240,
      "coding_elo": 1230,
      "swe_bench": 24.5,
      "humaneval": 81.0
    }
  }
}